def stopscan(self, id, cli=None): dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) scanstatus = scaninfo[5] print(scanstatus) if scaninfo is None: if not cli: return self.error("Invalid scan ID.") else: return json.dumps(["ERROR", "Invalid scan ID."]) if scanstatus == "ABORTED": if not cli: return self.error("The scan is already aborted.") else: return json.dumps(["ERROR", "Scan already aborted."]) if not scanstatus == "RUNNING": if not cli: return self.error("The running scan is currently in the state '" + \ scanstatus + "', please try again later or restart " + \ " SpiderFoot.") else: return json.dumps( ["ERROR", "Scan in an invalid state for stopping."]) dbh.scanInstanceSet(id, status="ABORT-REQUESTED") if not cli: raise cherrypy.HTTPRedirect("/") else: return json.dumps(["SUCCESS", ""])
def stopscan(self, id, cli=None): dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) scanstatus = scaninfo[5] print(scanstatus) if scaninfo is None: if not cli: return self.error("Invalid scan ID.") else: return json.dumps(["ERROR", "Invalid scan ID."]) if scanstatus == "ABORTED": if not cli: return self.error("The scan is already aborted.") else: return json.dumps(["ERROR", "Scan already aborted."]) if not scanstatus == "RUNNING": if not cli: return self.error("The running scan is currently in the state '" + \ scanstatus + "', please try again later or restart " + \ " SpiderFoot.") else: return json.dumps( ["ERROR", "Scan in an invalid state for stopping."]) dbh.scanInstanceSet(id, status="ABORT-REQUESTED") if not cli: templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list()) else: return json.dumps(["SUCCESS", ""])
def stopscanmulti(self, ids): dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): errState = False scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: return self.error("Invalid scan ID specified.") scanname = str(scaninfo[0]) scanstatus = scaninfo[5] if scanstatus == "FINISHED": error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \ id + "&confirm=1'>Maybe you want to delete it instead?</a>") errState = True if not errState and scanstatus == "ABORTED": error.append("Scan '" + scanname + "' is already aborted.") errState = True if not errState: #set the scanstatus in the db to "ABORT-REQUESTED" dbh.scanInstanceSet(id, status="ABORT-REQUESTED") raise cherrypy.HTTPRedirect("/")
def stopscanmulti(self, ids): #Removing All globalScanStatus Refs dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): errState = False scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: return self.error("Invalid scan ID specified.") scanname = str(scaninfo[0]) scanstatus = scaninfo[5] if scanstatus == "FINISHED": error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \ id + "&confirm=1'>Maybe you want to delete it instead?</a>") errState = True if not errState and scanstatus == "ABORTED": error.append("Scan '" + scanname + "' is already aborted.") errState = True if not errState: #set the scanstatus in the db to "ABORT-REQUESTED" dbh.scanInstanceSet(id, status="ABORT-REQUESTED") templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, errors=error, docroot=self.docroot)
def stopscanmulti(self, ids): """Stop a scan Args: ids (str): comma separated list of scan IDs Note: Unnecessary for now given that only one simultaneous scan is permitted """ dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): scaninfo = dbh.scanInstanceGet(id) if not scaninfo: return self.error("Invalid scan ID: %s" % id) scanname = str(scaninfo[0]) scanstatus = scaninfo[5] if scanstatus == "FINISHED": error.append( "Scan '%s' is in a finished state. <a href='/scandelete?id=%s&confirm=1'>Maybe you want to delete it instead?</a>" % (scanname, id)) continue if scanstatus == "ABORTED": error.append("Scan '" + scanname + "' is already aborted.") continue dbh.scanInstanceSet(id, status="ABORT-REQUESTED") raise cherrypy.HTTPRedirect("/")
def test_scan_instance_set(self): """ Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None) """ sfdb = SpiderFootDb(self.default_options, False) scan_instance = 'example scan instance' sfdb.scanInstanceSet(scan_instance, None, None, None) self.assertEqual('TBD', 'TBD')
def test_scan_instance_get_should_return_a_list(self): """ Test scanInstanceGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) scan_instance = 'example scan instance' sfdb.scanInstanceSet(scan_instance, None, None, None) scan_instance_get = sfdb.scanInstanceGet(scan_instance) self.assertIsInstance(scan_instance_get, list)
def stopscan(self, id, cli=None): """Stop a scan. Args: id (str): scan ID cli: TBD """ dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) if not scaninfo: if cli: return json.dumps(["ERROR", "Invalid scan ID."]) return self.error("Invalid scan ID.") scanstatus = scaninfo[5] if scanstatus == "ABORTED": if cli: return json.dumps(["ERROR", "Scan already aborted."]) return self.error("The scan is already aborted.") if not scanstatus == "RUNNING": if cli: return json.dumps( ["ERROR", "Scan in an invalid state for stopping."]) return self.error( "The running scan is currently in the state '%s', please try again later or restart SpiderFoot." % scanstatus) dbh.scanInstanceSet(id, status="ABORT-REQUESTED") if cli: return json.dumps(["SUCCESS", ""]) raise cherrypy.HTTPRedirect("/")
def test_scanInstanceSet_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None) """ sfdb = SpiderFootDb(self.default_options, False) started = None ended = None status = None invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError) as cm: scan_instance_set = sfdb.scanInstanceSet(invalid_type, started, ended, status)
def startScan(self): self.moduleInstances = dict() dbh = SpiderFootDb(self.config) self.sf.setDbh(dbh) aborted = False # Create a unique ID for this scan and create it in the back-end DB. self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target) self.sf.setScanId(self.config['__guid__']) self.myId = self.config['__guid__'] dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target) dbh.scanInstanceSet(self.config['__guid__'], time.time() * 1000, None, 'STARTING') self.status = "STARTING" # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList dbh.scanConfigSet(self.config['__guid__'], self.sf.configSerialize(self.config)) self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.") # moduleList = list of modules the user wants to run try: for modName in self.moduleList: if modName == '': continue module = __import__('modules.' + modName, globals(), locals(), [modName]) mod = getattr(module, modName)() mod.__name__ = modName # A bit hacky: we pass the database object as part of the config. This # object should only be used by the internal SpiderFoot modules writing # to the database, which at present is only sfp_stor_db. # Individual modules cannot create their own SpiderFootDb instance or # we'll get database locking issues, so it all goes through this. self.config['__sfdb__'] = dbh # Set up the module # Configuration is a combined global config with module-specific options #modConfig = deepcopy(self.config) modConfig = self.config['__modules__'][modName]['opts'] for opt in self.config.keys(): modConfig[opt] = self.config[opt] mod.clearListeners( ) # clear any listener relationships from the past mod.setup(self.sf, self.target, modConfig) self.moduleInstances[modName] = mod self.sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in self.moduleInstances.values(): for listenerModule in self.moduleInstances.values(): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() != None: module.registerListener(listenerModule) dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING') self.status = "RUNNING" # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target, "SpiderFoot UI") dbh.scanEventStore(self.config['__guid__'], rootEvent) # Start the modules sequentially. for module in self.moduleInstances.values(): # Check in case the user requested to stop the scan between modules initializing if module.checkForStop(): dbh.scanInstanceSet(self.config['__guid__'], status='ABORTING') self.status = "ABORTING" aborted = True break # Many modules' start() method will return None, as most will rely on # notifications during the scan from other modules. module.start() # Check if any of the modules ended due to being stopped for module in self.moduleInstances.values(): if module.checkForStop(): aborted = True if aborted: self.sf.status("Scan [" + self.config['__guid__'] + "] aborted.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ABORTED') self.status = "ABORTED" else: self.sf.status("Scan [" + self.config['__guid__'] + "] completed.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'FINISHED') self.status = "FINISHED" except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.sf.error("Unhandled exception encountered during scan. " + \ "Please report this as a bug: " + \ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False) self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " + str(e)) dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ERROR-FAILED') self.status = "ERROR-FAILED" self.moduleInstances = None dbh.close() self.sf.setDbh(None) self.sf.setScanId(None)
class SpiderFootScanner(): def __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan scanTarget (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Returns: None Todo: Eventually change this to be able to control multiple scan instances """ if not isinstance(scanName, str): raise TypeError("scanName is %s; expected str()" % type(scanName)) if not isinstance(scanId, str): raise TypeError("scanId is %s; expected str()" % type(scanId)) if not isinstance(scanTarget, str): raise TypeError("scanTarget is %s; expected str()" % type(scanTarget)) if not isinstance(targetType, str): raise TypeError("targetType is %s; expected str()" % type(targetType)) if not isinstance(moduleList, list): raise TypeError("moduleList is %s; expected list()" % type(moduleList)) if not isinstance(globalOpts, dict): raise TypeError("globalOpts is %s; expected dict()" % type(globalOpts)) if not globalOpts: raise ValueError("globalOpts is empty") self.moduleInstances = dict() self.config = deepcopy(globalOpts) self.sf = SpiderFoot(self.config) self.dbh = SpiderFootDb(self.config) self.targetValue = scanTarget self.targetType = targetType self.moduleList = moduleList self.modconfig = dict() self.scanName = scanName self.sf.setDbh(self.dbh) # Create a unique ID for this scan and create it in the back-end DB. self.scanId = scanId self.sf.setGUID(self.scanId) self.dbh.scanInstanceCreate(self.scanId, self.scanName, self.targetValue) # Create our target try: self.target = SpiderFootTarget(self.targetValue, self.targetType) except ValueError as e: self.sf.status("Scan [%s] failed: %s" % (self.scanId, e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError("Invalid target: %s" % e) except TypeError as e: self.sf.status("Scan [%s] failed: %s" % (self.scanId, e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) raise TypeError("Invalid target: %s" % e) # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList self.dbh.scanConfigSet(self.scanId, self.sf.configSerialize(deepcopy(self.config))) # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.config['_socks1type']: socksDns = self.config['_socks6dns'] socksAddr = self.config['_socks2addr'] socksPort = int(self.config['_socks3port']) socksUsername = self.config['_socks4user'] or '' socksPassword = self.config['_socks5pwd'] or '' proxy = "%s:%s" % (socksAddr, socksPort) if socksUsername or socksPassword: proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy) if self.config['_socks1type'] == '4': proxy = 'socks4://' + proxy elif self.config['_socks1type'] == '5': proxy = 'socks5://' + proxy elif self.config['_socks1type'] == 'HTTP': proxy = 'http://' + proxy elif self.config['_socks1type'] == 'TOR': proxy = 'socks5h://' + proxy else: raise ValueError("Invalid SOCKS proxy type: %s" % self.config["_socks1ttype"]) self.sf.debug("SOCKS: %s:%s (%s:%s)" % (socksAddr, socksPort, socksUsername, socksPassword)) self.sf.updateSocket(proxy) else: self.sf.revertSocket() # Override the default DNS server if self.config['_dnsserver']: res = dns.resolver.Resolver() res.nameservers = [self.config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.config['_useragent'] = self.sf.optValueToData(self.config['_useragent']) # Get internet TLDs tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.config['_internettlds'] = self.sf.optValueToData(self.config['_internettlds']) self.sf.cachePut("internet_tlds", self.config['_internettlds']) else: self.config["_internettlds"] = tlddata.splitlines() self.setStatus("INITIALIZING", time.time() * 1000, None) if start: self.startScan() def getId(self): """Retrieve the unique scan identifier. Returns: str: scan id """ if hasattr(self, 'scanId'): return self.scanId return None def setStatus(self, status, started=None, ended=None): """Set the status of the currently running scan (if any). Args: status (str): scan status started (str): TBD ended (str): TBD Returns: None """ if status not in ["INITIALIZING", "STARTING", "STARTED", "RUNNING", "ABORT-REQUESTED", "ABORTED", "ABORTING", "FINISHED", "ERROR-FAILED"]: raise ValueError("Invalid scan status '%s'" % status) self.status = status self.dbh.scanInstanceSet(self.scanId, started, ended, status) return None def startScan(self): """Start running a scan.""" aborted = False self.setStatus("STARTING", time.time() * 1000, None) self.sf.status("Scan [" + self.scanId + "] initiated.") try: # moduleList = list of modules the user wants to run for modName in self.moduleList: if modName == '': continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.sf.error("Failed to load module: " + modName, False) continue mod = getattr(module, modName)() mod.__name__ = modName # Module may have been renamed or removed if modName not in self.config['__modules__']: continue # Set up the module # Configuration is a combined global config with module-specific options self.modconfig[modName] = deepcopy(self.config['__modules__'][modName]['opts']) for opt in list(self.config.keys()): self.modconfig[modName][opt] = deepcopy(self.config[opt]) mod.clearListeners() # clear any listener relationships from the past mod.setup(self.sf, self.modconfig[modName]) mod.setDbh(self.dbh) mod.setScanId(self.scanId) # Give modules a chance to 'enrich' the original target with # aliases of that target. newTarget = mod.enrichTarget(self.target) if newTarget is not None: self.target = newTarget self.moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.config['_socks1type'] != '': mod._updateSocket(socket) # Set up event output filters if requested if self.config['__outputfilter']: mod.setOutputFilter(self.config['__outputfilter']) self.sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in list(self.moduleInstances.values()): # Register the target with the module module.setTarget(self.target) for listenerModule in list(self.moduleInstances.values()): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() is not None: module.registerListener(listenerModule) # Now we are ready to roll.. self.setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(self.target) psMod.setDbh(self.dbh) psMod.clearListeners() for mod in list(self.moduleInstances.values()): if mod.watchedEvents() is not None: psMod.registerListener(mod) # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.targetType, self.targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.targetType == 'INTERNET_NAME': if self.sf.isDomain(self.targetValue, self.config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing for module in list(self.moduleInstances.values()): if module.checkForStop(): self.setStatus('ABORTING') aborted = True break if aborted: self.sf.status("Scan [" + self.scanId + "] aborted.") self.setStatus("ABORTED", None, time.time() * 1000) else: self.sf.status("Scan [" + self.scanId + "] completed.") self.setStatus("FINISHED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \ "encountered during scan. Please report this as a bug: " + \ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False) self.sf.status("Scan [" + self.scanId + "] failed: " + str(e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) self.dbh.close()
def startScan(self): self.moduleInstances = dict() dbh = SpiderFootDb(self.config) self.sf.setDbh(dbh) aborted = False # Create a unique ID for this scan and create it in the back-end DB. self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target) self.sf.setScanId(self.config['__guid__']) self.myId = self.config['__guid__'] dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target) dbh.scanInstanceSet(self.config['__guid__'], time.time() * 1000, None, 'STARTING') self.status = "STARTING" # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList dbh.scanConfigSet(self.config['__guid__'], self.sf.configSerialize(self.config)) self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.") # moduleList = list of modules the user wants to run try: # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.config['_socks1type'] != '': socksType = socks.PROXY_TYPE_SOCKS4 socksDns = self.config['_socks6dns'] socksAddr = self.config['_socks2addr'] socksPort = int(self.config['_socks3port']) socksUsername = '' socksPassword = '' if self.config['_socks1type'] == '4': socksType = socks.PROXY_TYPE_SOCKS4 if self.config['_socks1type'] == '5': socksType = socks.PROXY_TYPE_SOCKS5 socksUsername = self.config['_socks4user'] socksPassword = self.config['_socks5pwd'] if self.config['_socks1type'] == 'HTTP': socksType = socks.PROXY_TYPE_HTTP self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \ "(" + socksUsername + ":" + socksPassword + ")") socks.setdefaultproxy(socksType, socksAddr, socksPort, socksDns, socksUsername, socksPassword) # Override the default socket and getaddrinfo calls with the # SOCKS ones socket.socket = socks.socksocket socket.create_connection = socks.create_connection socket.getaddrinfo = socks.getaddrinfo self.sf.updateSocket(socket) # Override the default DNS server if self.config['_dnsserver'] != "": res = dns.resolver.Resolver() res.nameservers = [ self.config['_dnsserver'] ] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.config['_useragent'] = self.sf.optValueToData(self.config['_useragent']) # Get internet TLDs tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata == None: self.config['_internettlds'] = self.sf.optValueToData(self.config['_internettlds']) self.sf.cachePut("internet_tlds", self.config['_internettlds']) else: self.config["_internettlds"] = tlddata.splitlines() for modName in self.moduleList: if modName == '': continue module = __import__('modules.' + modName, globals(), locals(), [modName]) mod = getattr(module, modName)() mod.__name__ = modName # A bit hacky: we pass the database object as part of the config. This # object should only be used by the internal SpiderFoot modules writing # to the database, which at present is only sfp__stor_db. # Individual modules cannot create their own SpiderFootDb instance or # we'll get database locking issues, so it all goes through this. self.config['__sfdb__'] = dbh # Set up the module # Configuration is a combined global config with module-specific options #modConfig = deepcopy(self.config) modConfig = self.config['__modules__'][modName]['opts'] for opt in self.config.keys(): modConfig[opt] = self.config[opt] mod.clearListeners() # clear any listener relationships from the past mod.setup(self.sf, self.target, modConfig) self.moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.config['_socks1type'] != '': mod._updateSocket(socket) self.sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in self.moduleInstances.values(): for listenerModule in self.moduleInstances.values(): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() != None: module.registerListener(listenerModule) dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING') self.status = "RUNNING" # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target, "SpiderFoot UI") dbh.scanEventStore(self.config['__guid__'], rootEvent) # Start the modules sequentially. for module in self.moduleInstances.values(): # Check in case the user requested to stop the scan between modules initializing if module.checkForStop(): dbh.scanInstanceSet(self.config['__guid__'], status='ABORTING') self.status = "ABORTING" aborted = True break # Many modules' start() method will return None, as most will rely on # notifications during the scan from other modules. module.start() # Check if any of the modules ended due to being stopped for module in self.moduleInstances.values(): if module.checkForStop(): aborted = True if aborted: self.sf.status("Scan [" + self.config['__guid__'] + "] aborted.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ABORTED') self.status = "ABORTED" else: self.sf.status("Scan [" + self.config['__guid__'] + "] completed.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'FINISHED') self.status = "FINISHED" except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \ "encountered during scan. Please report this as a bug: " + \ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False) self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " + str(e)) dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ERROR-FAILED') self.status = "ERROR-FAILED" self.moduleInstances = None dbh.close() self.sf.setDbh(None) self.sf.setScanId(None)
def startScan(self): self.moduleInstances = dict() dbh = SpiderFootDb(self.config) self.sf.setDbh(dbh) aborted = False # Create a unique ID for this scan and create it in the back-end DB. self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target) self.sf.setScanId(self.config['__guid__']) self.myId = self.config['__guid__'] dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target) dbh.scanInstanceSet(self.config['__guid__'], time.time() * 1000, None, 'STARTING') self.status = "STARTING" # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList dbh.scanConfigSet(self.config['__guid__'], self.sf.configSerialize(self.config)) self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.") # moduleList = list of modules the user wants to run try: for modName in self.moduleList: if modName == '': continue module = __import__('modules.' + modName, globals(), locals(), [modName]) mod = getattr(module, modName)() mod.__name__ = modName # A bit hacky: we pass the database object as part of the config. This # object should only be used by the internal SpiderFoot modules writing # to the database, which at present is only sfp__stor_db. # Individual modules cannot create their own SpiderFootDb instance or # we'll get database locking issues, so it all goes through this. self.config['__sfdb__'] = dbh # Set up the module # Configuration is a combined global config with module-specific options #modConfig = deepcopy(self.config) modConfig = self.config['__modules__'][modName]['opts'] for opt in self.config.keys(): modConfig[opt] = self.config[opt] mod.clearListeners() # clear any listener relationships from the past mod.setup(self.sf, self.target, modConfig) self.moduleInstances[modName] = mod self.sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in self.moduleInstances.values(): for listenerModule in self.moduleInstances.values(): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() != None: module.registerListener(listenerModule) dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING') self.status = "RUNNING" # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target, "SpiderFoot UI") dbh.scanEventStore(self.config['__guid__'], rootEvent) # Start the modules sequentially. for module in self.moduleInstances.values(): # Check in case the user requested to stop the scan between modules initializing if module.checkForStop(): dbh.scanInstanceSet(self.config['__guid__'], status='ABORTING') self.status = "ABORTING" aborted = True break # Many modules' start() method will return None, as most will rely on # notifications during the scan from other modules. module.start() # Check if any of the modules ended due to being stopped for module in self.moduleInstances.values(): if module.checkForStop(): aborted = True if aborted: self.sf.status("Scan [" + self.config['__guid__'] + "] aborted.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ABORTED') self.status = "ABORTED" else: self.sf.status("Scan [" + self.config['__guid__'] + "] completed.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'FINISHED') self.status = "FINISHED" except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \ "encountered during scan. Please report this as a bug: " + \ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False) self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " + str(e)) dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ERROR-FAILED') self.status = "ERROR-FAILED" self.moduleInstances = None dbh.close() self.sf.setDbh(None) self.sf.setScanId(None)
def startScan(self): self.moduleInstances = dict() dbh = SpiderFootDb(self.config) self.sf.setDbh(dbh) aborted = False # Create a unique ID for this scan and create it in the back-end DB. self.config['__guid__'] = dbh.scanInstanceGenGUID(self.target) self.sf.setScanId(self.config['__guid__']) self.myId = self.config['__guid__'] dbh.scanInstanceCreate(self.config['__guid__'], self.name, self.target) dbh.scanInstanceSet(self.config['__guid__'], time.time() * 1000, None, 'STARTING') self.status = "STARTING" # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList dbh.scanConfigSet(self.config['__guid__'], self.sf.configSerialize(self.config)) self.sf.status("Scan [" + self.config['__guid__'] + "] initiated.") # moduleList = list of modules the user wants to run try: # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.config['_socks1type'] != '': socksType = socks.PROXY_TYPE_SOCKS4 socksDns = self.config['_socks6dns'] socksAddr = self.config['_socks2addr'] socksPort = int(self.config['_socks3port']) socksUsername = '' socksPassword = '' if self.config['_socks1type'] == '4': socksType = socks.PROXY_TYPE_SOCKS4 if self.config['_socks1type'] == '5': socksType = socks.PROXY_TYPE_SOCKS5 socksUsername = self.config['_socks4user'] socksPassword = self.config['_socks5pwd'] if self.config['_socks1type'] == 'HTTP': socksType = socks.PROXY_TYPE_HTTP self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \ "(" + socksUsername + ":" + socksPassword + ")") socks.setdefaultproxy(socksType, socksAddr, socksPort, socksDns, socksUsername, socksPassword) # Override the default socket and getaddrinfo calls with the # SOCKS ones socket.socket = socks.socksocket socket.create_connection = socks.create_connection socket.getaddrinfo = socks.getaddrinfo self.sf.updateSocket(socket) # Override the default DNS server if self.config['_dnsserver'] != "": res = dns.resolver.Resolver() res.nameservers = [self.config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.config['_useragent'] = self.sf.optValueToData( self.config['_useragent']) # Get internet TLDs tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata == None: self.config['_internettlds'] = self.sf.optValueToData( self.config['_internettlds']) self.sf.cachePut("internet_tlds", self.config['_internettlds']) else: self.config["_internettlds"] = tlddata.splitlines() for modName in self.moduleList: if modName == '': continue module = __import__('modules.' + modName, globals(), locals(), [modName]) mod = getattr(module, modName)() mod.__name__ = modName # A bit hacky: we pass the database object as part of the config. This # object should only be used by the internal SpiderFoot modules writing # to the database, which at present is only sfp__stor_db. # Individual modules cannot create their own SpiderFootDb instance or # we'll get database locking issues, so it all goes through this. self.config['__sfdb__'] = dbh # Set up the module # Configuration is a combined global config with module-specific options #modConfig = deepcopy(self.config) modConfig = self.config['__modules__'][modName]['opts'] for opt in self.config.keys(): modConfig[opt] = self.config[opt] mod.clearListeners( ) # clear any listener relationships from the past mod.setup(self.sf, self.target, modConfig) self.moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.config['_socks1type'] != '': mod._updateSocket(socket) self.sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in self.moduleInstances.values(): for listenerModule in self.moduleInstances.values(): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() != None: module.registerListener(listenerModule) dbh.scanInstanceSet(self.config['__guid__'], status='RUNNING') self.status = "RUNNING" # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("INITIAL_TARGET", self.target, "SpiderFoot UI") dbh.scanEventStore(self.config['__guid__'], rootEvent) # Start the modules sequentially. for module in self.moduleInstances.values(): # Check in case the user requested to stop the scan between modules initializing if module.checkForStop(): dbh.scanInstanceSet(self.config['__guid__'], status='ABORTING') self.status = "ABORTING" aborted = True break # Many modules' start() method will return None, as most will rely on # notifications during the scan from other modules. module.start() # Check if any of the modules ended due to being stopped for module in self.moduleInstances.values(): if module.checkForStop(): aborted = True if aborted: self.sf.status("Scan [" + self.config['__guid__'] + "] aborted.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ABORTED') self.status = "ABORTED" else: self.sf.status("Scan [" + self.config['__guid__'] + "] completed.") dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'FINISHED') self.status = "FINISHED" except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \ "encountered during scan. Please report this as a bug: " + \ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False) self.sf.status("Scan [" + self.config['__guid__'] + "] failed: " + str(e)) dbh.scanInstanceSet(self.config['__guid__'], None, time.time() * 1000, 'ERROR-FAILED') self.status = "ERROR-FAILED" self.moduleInstances = None dbh.close() self.sf.setDbh(None) self.sf.setScanId(None)