def __startScan(self) -> None: """Start running a scan. Raises: AssertionError: Never actually raised. """ failed = True try: self.__setStatus("STARTING", time.time() * 1000, None) self.__sf.status( f"Scan [{self.__scanId}] for '{self.__target.targetValue}' initiated." ) self.eventQueue = queue.Queue() self.__sharedThreadPool.start() # moduleList = list of modules the user wants to run self.__sf.debug(f"Loading {len(self.__moduleList)} modules ...") for modName in self.__moduleList: if not modName: continue # Module may have been renamed or removed if modName not in self.__config['__modules__']: self.__sf.error(f"Failed to load module: {modName}") continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.__sf.error(f"Failed to load module: {modName}") continue try: mod = getattr(module, modName)() mod.__name__ = modName except Exception: self.__sf.error( f"Module {modName} initialization failed: {traceback.format_exc()}" ) continue # Set up the module options, scan ID, database handle and listeners try: # Configuration is a combined global config with module-specific options self.__modconfig[modName] = deepcopy( self.__config['__modules__'][modName]['opts']) for opt in list(self.__config.keys()): self.__modconfig[modName][opt] = deepcopy( self.__config[opt]) # clear any listener relationships from the past mod.clearListeners() mod.setScanId(self.__scanId) mod.setSharedThreadPool(self.__sharedThreadPool) mod.setDbh(self.__dbh) mod.setup(self.__sf, self.__modconfig[modName]) except Exception: self.__sf.error( f"Module {modName} initialization failed: {traceback.format_exc()}" ) mod.errorState = True continue # Override the module's local socket module to be the SOCKS one. if self.__config['_socks1type'] != '': try: mod._updateSocket(socket) except Exception as e: self.__sf.error( f"Module {modName} socket setup failed: {e}") continue # Set up event output filters if requested if self.__config['__outputfilter']: try: mod.setOutputFilter(self.__config['__outputfilter']) except Exception as e: self.__sf.error( f"Module {modName} output filter setup failed: {e}" ) continue # Give modules a chance to 'enrich' the original target with aliases of that target. try: newTarget = mod.enrichTarget(self.__target) if newTarget is not None: self.__target = newTarget except Exception as e: self.__sf.error( f"Module {modName} target enrichment failed: {e}") continue # Register the target with the module try: mod.setTarget(self.__target) except Exception as e: self.__sf.error( f"Module {modName} failed to set target '{self.__target}': {e}" ) continue # Set up the outgoing event queue try: mod.outgoingEventQueue = self.eventQueue mod.incomingEventQueue = queue.Queue() except Exception as e: self.__sf.error( f"Module {modName} event queue setup failed: {e}") continue self.__moduleInstances[modName] = mod self.__sf.status(f"{modName} module loaded.") self.__sf.debug( f"Scan [{self.__scanId}] loaded {len(self.__moduleInstances)} modules." ) if not self.__moduleInstances: self.__setStatus("ERROR-FAILED", None, time.time() * 1000) self.__dbh.close() return # sort modules by priority self.__moduleInstances = OrderedDict( sorted(self.__moduleInstances.items(), key=lambda m: m[-1]._priority)) # Now we are ready to roll.. self.__setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(self.__target) psMod.setDbh(self.__dbh) psMod.clearListeners() psMod.outgoingEventQueue = self.eventQueue psMod.incomingEventQueue = queue.Queue() # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.__targetType == 'INTERNET_NAME': if self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing scanstatus = self.__dbh.scanInstanceGet(self.__scanId) if scanstatus and scanstatus[5] == "ABORT-REQUESTED": raise AssertionError("ABORT-REQUESTED") # start threads self.waitForThreads() failed = False except (KeyboardInterrupt, AssertionError): self.__sf.status(f"Scan [{self.__scanId}] aborted.") self.__setStatus("ABORTED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.__sf.error( f"Unhandled exception ({e.__class__.__name__}) encountered during scan." + "Please report this as a bug: " + +repr( traceback.format_exception(exc_type, exc_value, exc_traceback))) self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) finally: if not failed: self.__setStatus("FINISHED", None, time.time() * 1000) self.runCorrelations() self.__sf.status(f"Scan [{self.__scanId}] completed.") self.__dbh.close()
def __startScan(self, threaded=True): """Start running a scan. Args: threaded (bool): whether to thread modules """ aborted = False self.__setStatus("STARTING", time.time() * 1000, None) self.__sf.status(f"Scan [{self.__scanId}] initiated.") if threaded: self.eventQueue = queue.Queue() try: # moduleList = list of modules the user wants to run for modName in self.__moduleList: if modName == '': continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.__sf.error(f"Failed to load module: {modName}") continue mod = getattr(module, modName)() mod.__name__ = modName # Module may have been renamed or removed if modName not in self.__config['__modules__']: continue # Set up the module # Configuration is a combined global config with module-specific options self.__modconfig[modName] = deepcopy( self.__config['__modules__'][modName]['opts']) for opt in list(self.__config.keys()): self.__modconfig[modName][opt] = deepcopy( self.__config[opt]) mod.clearListeners( ) # clear any listener relationships from the past mod.setup(self.__sf, self.__modconfig[modName]) mod.setDbh(self.__dbh) mod.setScanId(self.__scanId) # Give modules a chance to 'enrich' the original target with # aliases of that target. newTarget = mod.enrichTarget(self.__target) if newTarget is not None: self.__target = newTarget self.__moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.__config['_socks1type'] != '': mod._updateSocket(socket) # Set up event output filters if requested if self.__config['__outputfilter']: mod.setOutputFilter(self.__config['__outputfilter']) # Register the target with the module mod.setTarget(self.__target) if threaded: # Set up the outgoing event queue mod.outgoingEventQueue = self.eventQueue mod.incomingEventQueue = queue.Queue() self.__sf.status(modName + " module loaded.") # sort modules by priority self.__moduleInstances = OrderedDict( sorted(self.__moduleInstances.items(), key=lambda m: m[-1]._priority)) if not threaded: # Register listener modules and then start all modules sequentially for module in list(self.__moduleInstances.values()): for listenerModule in list( self.__moduleInstances.values()): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() is not None: module.registerListener(listenerModule) # Now we are ready to roll.. self.__setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(self.__target) psMod.setDbh(self.__dbh) psMod.clearListeners() if threaded: psMod.outgoingEventQueue = self.eventQueue psMod.incomingEventQueue = queue.Queue() else: for mod in list(self.__moduleInstances.values()): if mod.watchedEvents() is not None: psMod.registerListener(mod) # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.__targetType == 'INTERNET_NAME': if self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing for mod in list(self.__moduleInstances.values()): if mod.checkForStop(): self.__setStatus('ABORTING') aborted = True break # start threads if threaded and not aborted: self.waitForThreads() if aborted: self.__sf.status(f"Scan [{self.__scanId}] aborted.") self.__setStatus("ABORTED", None, time.time() * 1000) else: self.__sf.status(f"Scan [{self.__scanId}] completed.") self.__setStatus("FINISHED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.__sf.error( f"Unhandled exception ({e.__class__.__name__}) encountered during scan." + "Please report this as a bug: " + repr( traceback.format_exception(exc_type, exc_value, exc_traceback))) self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) self.__dbh.close()