def test_start(self): """ Test start(self) """ sfp = SpiderFootPlugin() sfp.start()
def test_register_listener(self): """ Test registerListener(self, listener) """ sfp = SpiderFootPlugin() sfp.registerListener(None) self.assertEqual('TBD', 'TBD')
def test_getTarget_unitialised_target_should_raise(self): """ Test getTarget(self) """ sfp = SpiderFootPlugin() with self.assertRaises(TypeError): sfp.getTarget()
def test_getScanId_unitialised_scanid_should_raise_TypeError(self): """ Test getScanId(self) """ sfp = SpiderFootPlugin() with self.assertRaises(TypeError): sfp.getScanId()
def test_clearListeners(self): """ Test clearListeners(self) """ sfp = SpiderFootPlugin() sfp.clearListeners() self.assertEqual('TBD', 'TBD')
def test_watchedEvents_should_return_a_list(self): """ Test watchedEvents(self) """ sfp = SpiderFootPlugin() watched_events = sfp.watchedEvents() self.assertIsInstance(watched_events, list)
def test_producedEvents_should_return_a_list(self): """ Test producedEvents(self) """ sfp = SpiderFootPlugin() produced_events = sfp.producedEvents() self.assertIsInstance(produced_events, list)
def test_enrichTargetargument_target_should_enrih_target(self): """ Test enrichTarget(self, target) """ sfp = SpiderFootPlugin() sfp.enrichTarget(None) self.assertEqual('TBD', 'TBD')
def test_updateSocket(self): """ Test _updateSocket(self, sock) """ sfp = SpiderFootPlugin() sfp._updateSocket(None) self.assertEqual('TBD', 'TBD')
def test_tempStorage_should_return_a_dict(self): """ Test tempStorage(self) """ sfp = SpiderFootPlugin() temp_storage = sfp.tempStorage() self.assertIsInstance(temp_storage, dict)
def test_set_dbhargument_dbh_should_set_database_handle(self): """ Test setDbh(self, dbh) """ sfdb = SpiderFootDb(self.default_options, False) sfp = SpiderFootPlugin() sfp.setDbh(sfdb) self.assertIsInstance(sfp.__sfdb__, SpiderFootDb)
def test_setOutputFilter_should_set_output_filter(self): """ Test setOutputFilter(self, types) """ sfp = SpiderFootPlugin() output_filter = "test filter" sfp.setOutputFilter("test filter") self.assertEqual(output_filter, sfp.__outputFilter__)
def test_start(self): """ Test start(self) """ sf = SpiderFoot(self.default_options) sfp = SpiderFootPlugin() sfp.sf = sf sfp.start()
def test_notifyListeners_output_filter_unmatched_should_not_notify_listener_modules( self): """ Test notifyListeners(self, sfEvent) """ sfp = SpiderFootPlugin() sfdb = SpiderFootDb(self.default_options, False) sfp.setDbh(sfdb) target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME") sfp.setTarget(target) event_type = 'ROOT' event_data = 'test data' module = 'test module' source_event = None evt = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'test event type' event_data = 'test data' module = 'test module' source_event = evt evt = SpiderFootEvent(event_type, event_data, module, source_event) sfp.__outputFilter__ = "example unmatched event type" sfp.notifyListeners(evt) self.assertEqual('TBD', 'TBD')
def test_notifyListeners_invalid_event_should_raise_TypeError(self): """ Test notifyListeners(self, sfEvent) """ sfp = SpiderFootPlugin() invalid_types = [None, "", list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfp.notifyListeners(None)
def test_setScanId_argument_id_invalid_type_should_raise_TypeError(self): """ Test setScanId(self, id) """ sfp = SpiderFootPlugin() invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfp.setScanId(invalid_type)
def test_setTarget_should_set_a_target(self): """ Test setTarget(self, target) """ sfp = SpiderFootPlugin() target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME") sfp.setTarget(target) get_target = sfp.getTarget().targetValue self.assertIsInstance(get_target, str) self.assertEqual("spiderfoot.net", get_target)
def test_handleEvent(self): """ Test handleEvent(self, sfEvent) """ event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) sfp = SpiderFootPlugin() sfp.handleEvent(evt)
def test_getScanId_should_return_a_string(self): """ Test getScanId(self) """ sfp = SpiderFootPlugin() scan_id = 'example scan id' sfp.setScanId(scan_id) get_scan_id = sfp.getScanId() self.assertIsInstance(get_scan_id, str) self.assertEqual(scan_id, get_scan_id)
def test_setScanId_argument_id_should_set_a_scan_id(self): """ Test setScanId(self, id) """ sfp = SpiderFootPlugin() scan_id = '1234' sfp.setScanId(scan_id) get_scan_id = sfp.getScanId() self.assertIsInstance(get_scan_id, str) self.assertEqual(scan_id, get_scan_id)
def test_checkForStop(self): """ Test checkForStop(self) """ sfp = SpiderFootPlugin() class DatabaseStub: def scanInstanceGet(self, scanId): return [None, None, None, None, None, status] sfp.__sfdb__ = DatabaseStub() sfp.__scanId__ = 'example scan id' # pseudo-parameterized test scan_statuses = [(None, False), ("anything", False), ("RUNNING", False), ("ABORT-REQUESTED", True)] for status, expectedReturnValue in scan_statuses: returnValue = sfp.checkForStop() self.assertEqual(returnValue, expectedReturnValue, status)
def test_checkForStop(self): """ Test checkForStop(self) Todo: include other statuses for completeness """ sfp = SpiderFootPlugin() class DatabaseStub: def scanInstanceGet(self, scanId): return [None, None, None, None, None, status] sfp.__sfdb__ = DatabaseStub() sfp.__scanId__ = 'example scan id' # pseudo-parameterized test for status, expectedReturnValue in [("RUNNING", False), ("ABORT-REQUESTED", True)]: returnValue = sfp.checkForStop() self.assertEqual(returnValue, expectedReturnValue, status)
def test_notifyListeners_event_type_and_data_same_as_source_event_source_event_should_story_only( self): """ Test notifyListeners(self, sfEvent) """ sfp = SpiderFootPlugin() sfdb = SpiderFootDb(self.default_options, False) sfp.setDbh(sfdb) event_type = 'ROOT' event_data = 'test data' module = 'test module' source_event = None evt = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'test event type' event_data = 'test data' module = 'test module' source_event = evt evt = SpiderFootEvent(event_type, event_data, module, source_event) source_event = evt evt = SpiderFootEvent(event_type, event_data, module, source_event) source_event = evt evt = SpiderFootEvent(event_type, event_data, module, source_event) sfp.notifyListeners(evt) self.assertEqual('TBD', 'TBD')
def test_notifyListeners(self): """ Test notifyListeners(self, sfEvent) Todo: Test with source event """ sfp = SpiderFootPlugin() class DatabaseStub: def scanInstanceGet(self, scanId): return [None, None, None, None, None, None] sfp.__sfdb__ = DatabaseStub() event_type = 'ROOT' event_data = 'test data' module = 'test module' source_event = None evt = SpiderFootEvent(event_type, event_data, module, source_event) sfp.notifyListeners(evt) self.assertEqual('TBD', 'TBD')
def test_setup(self): """ Test setup(self, sf, userOpts=dict()) """ sfp = SpiderFootPlugin() sfp.setup(None) sfp.setup(None, None) self.assertEqual('TBD', 'TBD')
def test_notifyListeners_should_notify_listener_modules(self): """ Test notifyListeners(self, sfEvent) Todo: Test with source event """ sfp = SpiderFootPlugin() sfdb = SpiderFootDb(self.default_options, False) sfp.setDbh(sfdb) event_type = 'ROOT' event_data = 'test data' module = 'test module' source_event = None evt = SpiderFootEvent(event_type, event_data, module, source_event) sfp.notifyListeners(evt) self.assertEqual('TBD', 'TBD')
def __startScan(self, threaded=True): """Start running a scan. Args: threaded (bool): whether to thread modules """ aborted = False self.__setStatus("STARTING", time.time() * 1000, None) self.__sf.status(f"Scan [{self.__scanId}] initiated.") if threaded: self.eventQueue = queue.Queue() try: # moduleList = list of modules the user wants to run for modName in self.__moduleList: if modName == '': continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.__sf.error(f"Failed to load module: {modName}") continue mod = getattr(module, modName)() mod.__name__ = modName # Module may have been renamed or removed if modName not in self.__config['__modules__']: continue # Set up the module # Configuration is a combined global config with module-specific options self.__modconfig[modName] = deepcopy( self.__config['__modules__'][modName]['opts']) for opt in list(self.__config.keys()): self.__modconfig[modName][opt] = deepcopy( self.__config[opt]) mod.clearListeners( ) # clear any listener relationships from the past mod.setup(self.__sf, self.__modconfig[modName]) mod.setDbh(self.__dbh) mod.setScanId(self.__scanId) # Give modules a chance to 'enrich' the original target with # aliases of that target. newTarget = mod.enrichTarget(self.__target) if newTarget is not None: self.__target = newTarget self.__moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.__config['_socks1type'] != '': mod._updateSocket(socket) # Set up event output filters if requested if self.__config['__outputfilter']: mod.setOutputFilter(self.__config['__outputfilter']) # Register the target with the module mod.setTarget(self.__target) if threaded: # Set up the outgoing event queue mod.outgoingEventQueue = self.eventQueue mod.incomingEventQueue = queue.Queue() self.__sf.status(modName + " module loaded.") # sort modules by priority self.__moduleInstances = OrderedDict( sorted(self.__moduleInstances.items(), key=lambda m: m[-1]._priority)) if not threaded: # Register listener modules and then start all modules sequentially for module in list(self.__moduleInstances.values()): for listenerModule in list( self.__moduleInstances.values()): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() is not None: module.registerListener(listenerModule) # Now we are ready to roll.. self.__setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(self.__target) psMod.setDbh(self.__dbh) psMod.clearListeners() if threaded: psMod.outgoingEventQueue = self.eventQueue psMod.incomingEventQueue = queue.Queue() else: for mod in list(self.__moduleInstances.values()): if mod.watchedEvents() is not None: psMod.registerListener(mod) # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.__targetType == 'INTERNET_NAME': if self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing for mod in list(self.__moduleInstances.values()): if mod.checkForStop(): self.__setStatus('ABORTING') aborted = True break # start threads if threaded and not aborted: self.waitForThreads() if aborted: self.__sf.status(f"Scan [{self.__scanId}] aborted.") self.__setStatus("ABORTED", None, time.time() * 1000) else: self.__sf.status(f"Scan [{self.__scanId}] completed.") self.__setStatus("FINISHED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.__sf.error( f"Unhandled exception ({e.__class__.__name__}) encountered during scan." + "Please report this as a bug: " + repr( traceback.format_exception(exc_type, exc_value, exc_traceback))) self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) self.__dbh.close()
def __startScan(self) -> None: """Start running a scan. Raises: AssertionError: Never actually raised. """ failed = True try: self.__setStatus("STARTING", time.time() * 1000, None) self.__sf.status( f"Scan [{self.__scanId}] for '{self.__target.targetValue}' initiated." ) self.eventQueue = queue.Queue() self.__sharedThreadPool.start() # moduleList = list of modules the user wants to run self.__sf.debug(f"Loading {len(self.__moduleList)} modules ...") for modName in self.__moduleList: if not modName: continue # Module may have been renamed or removed if modName not in self.__config['__modules__']: self.__sf.error(f"Failed to load module: {modName}") continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.__sf.error(f"Failed to load module: {modName}") continue try: mod = getattr(module, modName)() mod.__name__ = modName except Exception: self.__sf.error( f"Module {modName} initialization failed: {traceback.format_exc()}" ) continue # Set up the module options, scan ID, database handle and listeners try: # Configuration is a combined global config with module-specific options self.__modconfig[modName] = deepcopy( self.__config['__modules__'][modName]['opts']) for opt in list(self.__config.keys()): self.__modconfig[modName][opt] = deepcopy( self.__config[opt]) # clear any listener relationships from the past mod.clearListeners() mod.setScanId(self.__scanId) mod.setSharedThreadPool(self.__sharedThreadPool) mod.setDbh(self.__dbh) mod.setup(self.__sf, self.__modconfig[modName]) except Exception: self.__sf.error( f"Module {modName} initialization failed: {traceback.format_exc()}" ) mod.errorState = True continue # Override the module's local socket module to be the SOCKS one. if self.__config['_socks1type'] != '': try: mod._updateSocket(socket) except Exception as e: self.__sf.error( f"Module {modName} socket setup failed: {e}") continue # Set up event output filters if requested if self.__config['__outputfilter']: try: mod.setOutputFilter(self.__config['__outputfilter']) except Exception as e: self.__sf.error( f"Module {modName} output filter setup failed: {e}" ) continue # Give modules a chance to 'enrich' the original target with aliases of that target. try: newTarget = mod.enrichTarget(self.__target) if newTarget is not None: self.__target = newTarget except Exception as e: self.__sf.error( f"Module {modName} target enrichment failed: {e}") continue # Register the target with the module try: mod.setTarget(self.__target) except Exception as e: self.__sf.error( f"Module {modName} failed to set target '{self.__target}': {e}" ) continue # Set up the outgoing event queue try: mod.outgoingEventQueue = self.eventQueue mod.incomingEventQueue = queue.Queue() except Exception as e: self.__sf.error( f"Module {modName} event queue setup failed: {e}") continue self.__moduleInstances[modName] = mod self.__sf.status(f"{modName} module loaded.") self.__sf.debug( f"Scan [{self.__scanId}] loaded {len(self.__moduleInstances)} modules." ) if not self.__moduleInstances: self.__setStatus("ERROR-FAILED", None, time.time() * 1000) self.__dbh.close() return # sort modules by priority self.__moduleInstances = OrderedDict( sorted(self.__moduleInstances.items(), key=lambda m: m[-1]._priority)) # Now we are ready to roll.. self.__setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(self.__target) psMod.setDbh(self.__dbh) psMod.clearListeners() psMod.outgoingEventQueue = self.eventQueue psMod.incomingEventQueue = queue.Queue() # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.__targetType == 'INTERNET_NAME': if self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing scanstatus = self.__dbh.scanInstanceGet(self.__scanId) if scanstatus and scanstatus[5] == "ABORT-REQUESTED": raise AssertionError("ABORT-REQUESTED") # start threads self.waitForThreads() failed = False except (KeyboardInterrupt, AssertionError): self.__sf.status(f"Scan [{self.__scanId}] aborted.") self.__setStatus("ABORTED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.__sf.error( f"Unhandled exception ({e.__class__.__name__}) encountered during scan." + "Please report this as a bug: " + +repr( traceback.format_exception(exc_type, exc_value, exc_traceback))) self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) finally: if not failed: self.__setStatus("FINISHED", None, time.time() * 1000) self.runCorrelations() self.__sf.status(f"Scan [{self.__scanId}] completed.") self.__dbh.close()
def test_init(self): """ Test __init__(self) """ sfp = SpiderFootPlugin() self.assertIsInstance(sfp, SpiderFootPlugin)