def stopscan(self, id, cli=None):
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
scanstatus = scaninfo[5]
print(scanstatus)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if scanstatus == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not scanstatus == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
scanstatus + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if not cli:
raise cherrypy.HTTPRedirect("/")
else:
return json.dumps(["SUCCESS", ""])
def stopscan(self, id, cli=None):
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
scanstatus = scaninfo[5]
print(scanstatus)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if scanstatus == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not scanstatus == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
scanstatus + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if not cli:
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
docroot=self.docroot,
errors=list())
else:
return json.dumps(["SUCCESS", ""])
def stopscanmulti(self, ids):
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID specified.")
scanname = str(scaninfo[0])
scanstatus = scaninfo[5]
if scanstatus == "FINISHED":
error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and scanstatus == "ABORTED":
error.append("Scan '" + scanname + "' is already aborted.")
errState = True
if not errState:
#set the scanstatus in the db to "ABORT-REQUESTED"
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
raise cherrypy.HTTPRedirect("/")
def stopscanmulti(self, ids):
#Removing All globalScanStatus Refs
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID specified.")
scanname = str(scaninfo[0])
scanstatus = scaninfo[5]
if scanstatus == "FINISHED":
error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and scanstatus == "ABORTED":
error.append("Scan '" + scanname + "' is already aborted.")
errState = True
if not errState:
#set the scanstatus in the db to "ABORT-REQUESTED"
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
errors=error,
docroot=self.docroot)
def stopscanmulti(self, ids):
"""Stop a scan
Args:
ids (str): comma separated list of scan IDs
Note:
Unnecessary for now given that only one simultaneous scan is permitted
"""
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
scaninfo = dbh.scanInstanceGet(id)
if not scaninfo:
return self.error("Invalid scan ID: %s" % id)
scanname = str(scaninfo[0])
scanstatus = scaninfo[5]
if scanstatus == "FINISHED":
error.append(
"Scan '%s' is in a finished state. <a href='/scandelete?id=%s&confirm=1'>Maybe you want to delete it instead?</a>"
% (scanname, id))
continue
if scanstatus == "ABORTED":
error.append("Scan '" + scanname + "' is already aborted.")
continue
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
raise cherrypy.HTTPRedirect("/")
def test_scan_instance_set(self):
"""
Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None)
"""
sfdb = SpiderFootDb(self.default_options, False)
scan_instance = 'example scan instance'
sfdb.scanInstanceSet(scan_instance, None, None, None)
self.assertEqual('TBD', 'TBD')
def test_scan_instance_get_should_return_a_list(self):
"""
Test scanInstanceGet(self, instanceId)
"""
sfdb = SpiderFootDb(self.default_options, False)
scan_instance = 'example scan instance'
sfdb.scanInstanceSet(scan_instance, None, None, None)
scan_instance_get = sfdb.scanInstanceGet(scan_instance)
self.assertIsInstance(scan_instance_get, list)
def stopscan(self, id, cli=None):
"""Stop a scan.
Args:
id (str): scan ID
cli: TBD
"""
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if not scaninfo:
if cli:
return json.dumps(["ERROR", "Invalid scan ID."])
return self.error("Invalid scan ID.")
scanstatus = scaninfo[5]
if scanstatus == "ABORTED":
if cli:
return json.dumps(["ERROR", "Scan already aborted."])
return self.error("The scan is already aborted.")
if not scanstatus == "RUNNING":
if cli:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
return self.error(
"The running scan is currently in the state '%s', please try again later or restart SpiderFoot."
% scanstatus)
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if cli:
return json.dumps(["SUCCESS", ""])
raise cherrypy.HTTPRedirect("/")
def test_scanInstanceSet_argument_instanceId_of_invalid_type_should_raise_TypeError(self):
"""
Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None)
"""
sfdb = SpiderFootDb(self.default_options, False)
started = None
ended = None
status = None
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_instance_set = sfdb.scanInstanceSet(invalid_type, started, ended, status)
def startScan(self):
self.moduleInstances = dict()
dbh = SpiderFootDb(self.config)
self.sf.setDbh(dbh)
aborted = False
# Create a unique ID for this scan and create it in the back-end DB.
self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target)
self.sf.setScanId(self.config['__guid__'])
self.myId = self.config['__guid__']
dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target)
dbh.scanInstanceSet(self.config['__guid__'],
time.time() * 1000, None, 'STARTING')
self.status = "STARTING"
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
dbh.scanConfigSet(self.config['__guid__'],
self.sf.configSerialize(self.config))
self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.")
# moduleList = list of modules the user wants to run
try:
for modName in self.moduleList:
if modName == '':
continue
module = __import__('modules.' + modName, globals(), locals(),
[modName])
mod = getattr(module, modName)()
mod.__name__ = modName
# A bit hacky: we pass the database object as part of the config. This
# object should only be used by the internal SpiderFoot modules writing
# to the database, which at present is only sfp_stor_db.
# Individual modules cannot create their own SpiderFootDb instance or
# we'll get database locking issues, so it all goes through this.
self.config['__sfdb__'] = dbh
# Set up the module
# Configuration is a combined global config with module-specific options
#modConfig = deepcopy(self.config)
modConfig = self.config['__modules__'][modName]['opts']
for opt in self.config.keys():
modConfig[opt] = self.config[opt]
mod.clearListeners(
) # clear any listener relationships from the past
mod.setup(self.sf, self.target, modConfig)
self.moduleInstances[modName] = mod
self.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
for module in self.moduleInstances.values():
for listenerModule in self.moduleInstances.values():
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() != None:
module.registerListener(listenerModule)
dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING')
self.status = "RUNNING"
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target,
"SpiderFoot UI")
dbh.scanEventStore(self.config['__guid__'], rootEvent)
# Start the modules sequentially.
for module in self.moduleInstances.values():
# Check in case the user requested to stop the scan between modules initializing
if module.checkForStop():
dbh.scanInstanceSet(self.config['__guid__'],
status='ABORTING')
self.status = "ABORTING"
aborted = True
break
# Many modules' start() method will return None, as most will rely on
# notifications during the scan from other modules.
module.start()
# Check if any of the modules ended due to being stopped
for module in self.moduleInstances.values():
if module.checkForStop():
aborted = True
if aborted:
self.sf.status("Scan [" + self.config['__guid__'] +
"] aborted.")
dbh.scanInstanceSet(self.config['__guid__'], None,
time.time() * 1000, 'ABORTED')
self.status = "ABORTED"
else:
self.sf.status("Scan [" + self.config['__guid__'] +
"] completed.")
dbh.scanInstanceSet(self.config['__guid__'], None,
time.time() * 1000, 'FINISHED')
self.status = "FINISHED"
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.sf.error("Unhandled exception encountered during scan. " + \
"Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " +
str(e))
dbh.scanInstanceSet(self.config['__guid__'], None,
time.time() * 1000, 'ERROR-FAILED')
self.status = "ERROR-FAILED"
self.moduleInstances = None
dbh.close()
self.sf.setDbh(None)
self.sf.setScanId(None)
class SpiderFootScanner():
def __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True):
"""Initialize SpiderFootScanner object.
Args:
scanName (str): name of the scan
scanId (str): unique ID of the scan
scanTarget (str): scan target
targetType (str): scan target type
moduleList (list): list of modules to run
globalOpts (dict): scan options
start (bool): start the scan immediately
Returns:
None
Todo:
Eventually change this to be able to control multiple scan instances
"""
if not isinstance(scanName, str):
raise TypeError("scanName is %s; expected str()" % type(scanName))
if not isinstance(scanId, str):
raise TypeError("scanId is %s; expected str()" % type(scanId))
if not isinstance(scanTarget, str):
raise TypeError("scanTarget is %s; expected str()" % type(scanTarget))
if not isinstance(targetType, str):
raise TypeError("targetType is %s; expected str()" % type(targetType))
if not isinstance(moduleList, list):
raise TypeError("moduleList is %s; expected list()" % type(moduleList))
if not isinstance(globalOpts, dict):
raise TypeError("globalOpts is %s; expected dict()" % type(globalOpts))
if not globalOpts:
raise ValueError("globalOpts is empty")
self.moduleInstances = dict()
self.config = deepcopy(globalOpts)
self.sf = SpiderFoot(self.config)
self.dbh = SpiderFootDb(self.config)
self.targetValue = scanTarget
self.targetType = targetType
self.moduleList = moduleList
self.modconfig = dict()
self.scanName = scanName
self.sf.setDbh(self.dbh)
# Create a unique ID for this scan and create it in the back-end DB.
self.scanId = scanId
self.sf.setGUID(self.scanId)
self.dbh.scanInstanceCreate(self.scanId, self.scanName, self.targetValue)
# Create our target
try:
self.target = SpiderFootTarget(self.targetValue, self.targetType)
except ValueError as e:
self.sf.status("Scan [%s] failed: %s" % (self.scanId, e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError("Invalid target: %s" % e)
except TypeError as e:
self.sf.status("Scan [%s] failed: %s" % (self.scanId, e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
raise TypeError("Invalid target: %s" % e)
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
self.dbh.scanConfigSet(self.scanId, self.sf.configSerialize(deepcopy(self.config)))
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
if self.config['_socks1type']:
socksDns = self.config['_socks6dns']
socksAddr = self.config['_socks2addr']
socksPort = int(self.config['_socks3port'])
socksUsername = self.config['_socks4user'] or ''
socksPassword = self.config['_socks5pwd'] or ''
proxy = "%s:%s" % (socksAddr, socksPort)
if socksUsername or socksPassword:
proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy)
if self.config['_socks1type'] == '4':
proxy = 'socks4://' + proxy
elif self.config['_socks1type'] == '5':
proxy = 'socks5://' + proxy
elif self.config['_socks1type'] == 'HTTP':
proxy = 'http://' + proxy
elif self.config['_socks1type'] == 'TOR':
proxy = 'socks5h://' + proxy
else:
raise ValueError("Invalid SOCKS proxy type: %s" % self.config["_socks1ttype"])
self.sf.debug("SOCKS: %s:%s (%s:%s)" % (socksAddr, socksPort, socksUsername, socksPassword))
self.sf.updateSocket(proxy)
else:
self.sf.revertSocket()
# Override the default DNS server
if self.config['_dnsserver']:
res = dns.resolver.Resolver()
res.nameservers = [self.config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.config['_useragent'] = self.sf.optValueToData(self.config['_useragent'])
# Get internet TLDs
tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata is None:
self.config['_internettlds'] = self.sf.optValueToData(self.config['_internettlds'])
self.sf.cachePut("internet_tlds", self.config['_internettlds'])
else:
self.config["_internettlds"] = tlddata.splitlines()
self.setStatus("INITIALIZING", time.time() * 1000, None)
if start:
self.startScan()
def getId(self):
"""Retrieve the unique scan identifier.
Returns:
str: scan id
"""
if hasattr(self, 'scanId'):
return self.scanId
return None
def setStatus(self, status, started=None, ended=None):
"""Set the status of the currently running scan (if any).
Args:
status (str): scan status
started (str): TBD
ended (str): TBD
Returns:
None
"""
if status not in ["INITIALIZING", "STARTING", "STARTED", "RUNNING", "ABORT-REQUESTED", "ABORTED", "ABORTING", "FINISHED", "ERROR-FAILED"]:
raise ValueError("Invalid scan status '%s'" % status)
self.status = status
self.dbh.scanInstanceSet(self.scanId, started, ended, status)
return None
def startScan(self):
"""Start running a scan."""
aborted = False
self.setStatus("STARTING", time.time() * 1000, None)
self.sf.status("Scan [" + self.scanId + "] initiated.")
try:
# moduleList = list of modules the user wants to run
for modName in self.moduleList:
if modName == '':
continue
try:
module = __import__('modules.' + modName, globals(), locals(),
[modName])
except ImportError:
self.sf.error("Failed to load module: " + modName, False)
continue
mod = getattr(module, modName)()
mod.__name__ = modName
# Module may have been renamed or removed
if modName not in self.config['__modules__']:
continue
# Set up the module
# Configuration is a combined global config with module-specific options
self.modconfig[modName] = deepcopy(self.config['__modules__'][modName]['opts'])
for opt in list(self.config.keys()):
self.modconfig[modName][opt] = deepcopy(self.config[opt])
mod.clearListeners() # clear any listener relationships from the past
mod.setup(self.sf, self.modconfig[modName])
mod.setDbh(self.dbh)
mod.setScanId(self.scanId)
# Give modules a chance to 'enrich' the original target with
# aliases of that target.
newTarget = mod.enrichTarget(self.target)
if newTarget is not None:
self.target = newTarget
self.moduleInstances[modName] = mod
# Override the module's local socket module
# to be the SOCKS one.
if self.config['_socks1type'] != '':
mod._updateSocket(socket)
# Set up event output filters if requested
if self.config['__outputfilter']:
mod.setOutputFilter(self.config['__outputfilter'])
self.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
for module in list(self.moduleInstances.values()):
# Register the target with the module
module.setTarget(self.target)
for listenerModule in list(self.moduleInstances.values()):
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() is not None:
module.registerListener(listenerModule)
# Now we are ready to roll..
self.setStatus("RUNNING")
# Create a pseudo module for the root event to originate from
psMod = SpiderFootPlugin()
psMod.__name__ = "SpiderFoot UI"
psMod.setTarget(self.target)
psMod.setDbh(self.dbh)
psMod.clearListeners()
for mod in list(self.moduleInstances.values()):
if mod.watchedEvents() is not None:
psMod.registerListener(mod)
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("ROOT", self.targetValue, "", None)
psMod.notifyListeners(rootEvent)
firstEvent = SpiderFootEvent(self.targetType, self.targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# Special case.. check if an INTERNET_NAME is also a domain
if self.targetType == 'INTERNET_NAME':
if self.sf.isDomain(self.targetValue, self.config['_internettlds']):
firstEvent = SpiderFootEvent('DOMAIN_NAME', self.targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# If in interactive mode, loop through this shared global variable
# waiting for inputs, and process them until my status is set to
# FINISHED.
# Check in case the user requested to stop the scan between modules
# initializing
for module in list(self.moduleInstances.values()):
if module.checkForStop():
self.setStatus('ABORTING')
aborted = True
break
if aborted:
self.sf.status("Scan [" + self.scanId + "] aborted.")
self.setStatus("ABORTED", None, time.time() * 1000)
else:
self.sf.status("Scan [" + self.scanId + "] completed.")
self.setStatus("FINISHED", None, time.time() * 1000)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
"encountered during scan. Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.sf.status("Scan [" + self.scanId + "] failed: " + str(e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
self.dbh.close()
def startScan(self):
self.moduleInstances = dict()
dbh = SpiderFootDb(self.config)
self.sf.setDbh(dbh)
aborted = False
# Create a unique ID for this scan and create it in the back-end DB.
self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target)
self.sf.setScanId(self.config['__guid__'])
self.myId = self.config['__guid__']
dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target)
dbh.scanInstanceSet(self.config['__guid__'], time.time() * 1000, None, 'STARTING')
self.status = "STARTING"
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
dbh.scanConfigSet(self.config['__guid__'], self.sf.configSerialize(self.config))
self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.")
# moduleList = list of modules the user wants to run
try:
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
if self.config['_socks1type'] != '':
socksType = socks.PROXY_TYPE_SOCKS4
socksDns = self.config['_socks6dns']
socksAddr = self.config['_socks2addr']
socksPort = int(self.config['_socks3port'])
socksUsername = ''
socksPassword = ''
if self.config['_socks1type'] == '4':
socksType = socks.PROXY_TYPE_SOCKS4
if self.config['_socks1type'] == '5':
socksType = socks.PROXY_TYPE_SOCKS5
socksUsername = self.config['_socks4user']
socksPassword = self.config['_socks5pwd']
if self.config['_socks1type'] == 'HTTP':
socksType = socks.PROXY_TYPE_HTTP
self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \
"(" + socksUsername + ":" + socksPassword + ")")
socks.setdefaultproxy(socksType, socksAddr, socksPort,
socksDns, socksUsername, socksPassword)
# Override the default socket and getaddrinfo calls with the
# SOCKS ones
socket.socket = socks.socksocket
socket.create_connection = socks.create_connection
socket.getaddrinfo = socks.getaddrinfo
self.sf.updateSocket(socket)
# Override the default DNS server
if self.config['_dnsserver'] != "":
res = dns.resolver.Resolver()
res.nameservers = [ self.config['_dnsserver'] ]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.config['_useragent'] = self.sf.optValueToData(self.config['_useragent'])
# Get internet TLDs
tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata == None:
self.config['_internettlds'] = self.sf.optValueToData(self.config['_internettlds'])
self.sf.cachePut("internet_tlds", self.config['_internettlds'])
else:
self.config["_internettlds"] = tlddata.splitlines()
for modName in self.moduleList:
if modName == '':
continue
module = __import__('modules.' + modName, globals(), locals(), [modName])
mod = getattr(module, modName)()
mod.__name__ = modName
# A bit hacky: we pass the database object as part of the config. This
# object should only be used by the internal SpiderFoot modules writing
# to the database, which at present is only sfp__stor_db.
# Individual modules cannot create their own SpiderFootDb instance or
# we'll get database locking issues, so it all goes through this.
self.config['__sfdb__'] = dbh
# Set up the module
# Configuration is a combined global config with module-specific options
#modConfig = deepcopy(self.config)
modConfig = self.config['__modules__'][modName]['opts']
for opt in self.config.keys():
modConfig[opt] = self.config[opt]
mod.clearListeners() # clear any listener relationships from the past
mod.setup(self.sf, self.target, modConfig)
self.moduleInstances[modName] = mod
# Override the module's local socket module
# to be the SOCKS one.
if self.config['_socks1type'] != '':
mod._updateSocket(socket)
self.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
for module in self.moduleInstances.values():
for listenerModule in self.moduleInstances.values():
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() != None:
module.registerListener(listenerModule)
dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING')
self.status = "RUNNING"
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target, "SpiderFoot UI")
dbh.scanEventStore(self.config['__guid__'], rootEvent)
# Start the modules sequentially.
for module in self.moduleInstances.values():
# Check in case the user requested to stop the scan between modules initializing
if module.checkForStop():
dbh.scanInstanceSet(self.config['__guid__'], status='ABORTING')
self.status = "ABORTING"
aborted = True
break
# Many modules' start() method will return None, as most will rely on
# notifications during the scan from other modules.
module.start()
# Check if any of the modules ended due to being stopped
for module in self.moduleInstances.values():
if module.checkForStop():
aborted = True
if aborted:
self.sf.status("Scan [" + self.config['__guid__'] + "] aborted.")
dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ABORTED')
self.status = "ABORTED"
else:
self.sf.status("Scan [" + self.config['__guid__'] + "] completed.")
dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'FINISHED')
self.status = "FINISHED"
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
"encountered during scan. Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " + str(e))
dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ERROR-FAILED')
self.status = "ERROR-FAILED"
self.moduleInstances = None
dbh.close()
self.sf.setDbh(None)
self.sf.setScanId(None)
def startScan(self):
self.moduleInstances = dict()
dbh = SpiderFootDb(self.config)
self.sf.setDbh(dbh)
aborted = False
# Create a unique ID for this scan and create it in the back-end DB.
self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target)
self.sf.setScanId(self.config['__guid__'])
self.myId = self.config['__guid__']
dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target)
dbh.scanInstanceSet(self.config['__guid__'], time.time() * 1000, None, 'STARTING')
self.status = "STARTING"
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
dbh.scanConfigSet(self.config['__guid__'], self.sf.configSerialize(self.config))
self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.")
# moduleList = list of modules the user wants to run
try:
for modName in self.moduleList:
if modName == '':
continue
module = __import__('modules.' + modName, globals(), locals(), [modName])
mod = getattr(module, modName)()
mod.__name__ = modName
# A bit hacky: we pass the database object as part of the config. This
# object should only be used by the internal SpiderFoot modules writing
# to the database, which at present is only sfp__stor_db.
# Individual modules cannot create their own SpiderFootDb instance or
# we'll get database locking issues, so it all goes through this.
self.config['__sfdb__'] = dbh
# Set up the module
# Configuration is a combined global config with module-specific options
#modConfig = deepcopy(self.config)
modConfig = self.config['__modules__'][modName]['opts']
for opt in self.config.keys():
modConfig[opt] = self.config[opt]
mod.clearListeners() # clear any listener relationships from the past
mod.setup(self.sf, self.target, modConfig)
self.moduleInstances[modName] = mod
self.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
for module in self.moduleInstances.values():
for listenerModule in self.moduleInstances.values():
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() != None:
module.registerListener(listenerModule)
dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING')
self.status = "RUNNING"
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target, "SpiderFoot UI")
dbh.scanEventStore(self.config['__guid__'], rootEvent)
# Start the modules sequentially.
for module in self.moduleInstances.values():
# Check in case the user requested to stop the scan between modules initializing
if module.checkForStop():
dbh.scanInstanceSet(self.config['__guid__'], status='ABORTING')
self.status = "ABORTING"
aborted = True
break
# Many modules' start() method will return None, as most will rely on
# notifications during the scan from other modules.
module.start()
# Check if any of the modules ended due to being stopped
for module in self.moduleInstances.values():
if module.checkForStop():
aborted = True
if aborted:
self.sf.status("Scan [" + self.config['__guid__'] + "] aborted.")
dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ABORTED')
self.status = "ABORTED"
else:
self.sf.status("Scan [" + self.config['__guid__'] + "] completed.")
dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'FINISHED')
self.status = "FINISHED"
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
"encountered during scan. Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " + str(e))
dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ERROR-FAILED')
self.status = "ERROR-FAILED"
self.moduleInstances = None
dbh.close()
self.sf.setDbh(None)
self.sf.setScanId(None)
def startScan(self):
self.moduleInstances = dict()
dbh = SpiderFootDb(self.config)
self.sf.setDbh(dbh)
aborted = False
# Create a unique ID for this scan and create it in the back-end DB.
self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target)
self.sf.setScanId(self.config['__guid__'])
self.myId = self.config['__guid__']
dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target)
dbh.scanInstanceSet(self.config['__guid__'],
time.time() * 1000, None, 'STARTING')
self.status = "STARTING"
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
dbh.scanConfigSet(self.config['__guid__'],
self.sf.configSerialize(self.config))
self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.")
# moduleList = list of modules the user wants to run
try:
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
if self.config['_socks1type'] != '':
socksType = socks.PROXY_TYPE_SOCKS4
socksDns = self.config['_socks6dns']
socksAddr = self.config['_socks2addr']
socksPort = int(self.config['_socks3port'])
socksUsername = ''
socksPassword = ''
if self.config['_socks1type'] == '4':
socksType = socks.PROXY_TYPE_SOCKS4
if self.config['_socks1type'] == '5':
socksType = socks.PROXY_TYPE_SOCKS5
socksUsername = self.config['_socks4user']
socksPassword = self.config['_socks5pwd']
if self.config['_socks1type'] == 'HTTP':
socksType = socks.PROXY_TYPE_HTTP
self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \
"(" + socksUsername + ":" + socksPassword + ")")
socks.setdefaultproxy(socksType, socksAddr, socksPort,
socksDns, socksUsername, socksPassword)
# Override the default socket and getaddrinfo calls with the
# SOCKS ones
socket.socket = socks.socksocket
socket.create_connection = socks.create_connection
socket.getaddrinfo = socks.getaddrinfo
self.sf.updateSocket(socket)
# Override the default DNS server
if self.config['_dnsserver'] != "":
res = dns.resolver.Resolver()
res.nameservers = [self.config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.config['_useragent'] = self.sf.optValueToData(
self.config['_useragent'])
# Get internet TLDs
tlddata = self.sf.cacheGet("internet_tlds",
self.config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata == None:
self.config['_internettlds'] = self.sf.optValueToData(
self.config['_internettlds'])
self.sf.cachePut("internet_tlds", self.config['_internettlds'])
else:
self.config["_internettlds"] = tlddata.splitlines()
for modName in self.moduleList:
if modName == '':
continue
module = __import__('modules.' + modName, globals(), locals(),
[modName])
mod = getattr(module, modName)()
mod.__name__ = modName
# A bit hacky: we pass the database object as part of the config. This
# object should only be used by the internal SpiderFoot modules writing
# to the database, which at present is only sfp__stor_db.
# Individual modules cannot create their own SpiderFootDb instance or
# we'll get database locking issues, so it all goes through this.
self.config['__sfdb__'] = dbh
# Set up the module
# Configuration is a combined global config with module-specific options
#modConfig = deepcopy(self.config)
modConfig = self.config['__modules__'][modName]['opts']
for opt in self.config.keys():
modConfig[opt] = self.config[opt]
mod.clearListeners(
) # clear any listener relationships from the past
mod.setup(self.sf, self.target, modConfig)
self.moduleInstances[modName] = mod
# Override the module's local socket module
# to be the SOCKS one.
if self.config['_socks1type'] != '':
mod._updateSocket(socket)
self.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
for module in self.moduleInstances.values():
for listenerModule in self.moduleInstances.values():
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() != None:
module.registerListener(listenerModule)
dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING')
self.status = "RUNNING"
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target,
"SpiderFoot UI")
dbh.scanEventStore(self.config['__guid__'], rootEvent)
# Start the modules sequentially.
for module in self.moduleInstances.values():
# Check in case the user requested to stop the scan between modules initializing
if module.checkForStop():
dbh.scanInstanceSet(self.config['__guid__'],
status='ABORTING')
self.status = "ABORTING"
aborted = True
break
# Many modules' start() method will return None, as most will rely on
# notifications during the scan from other modules.
module.start()
# Check if any of the modules ended due to being stopped
for module in self.moduleInstances.values():
if module.checkForStop():
aborted = True
if aborted:
self.sf.status("Scan [" + self.config['__guid__'] +
"] aborted.")
dbh.scanInstanceSet(self.config['__guid__'], None,
time.time() * 1000, 'ABORTED')
self.status = "ABORTED"
else:
self.sf.status("Scan [" + self.config['__guid__'] +
"] completed.")
dbh.scanInstanceSet(self.config['__guid__'], None,
time.time() * 1000, 'FINISHED')
self.status = "FINISHED"
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
"encountered during scan. Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " +
str(e))
dbh.scanInstanceSet(self.config['__guid__'], None,
time.time() * 1000, 'ERROR-FAILED')
self.status = "ERROR-FAILED"
self.moduleInstances = None
dbh.close()
self.sf.setDbh(None)
self.sf.setScanId(None)