def test_start(self):
"""
Test start(self)
"""
sfp = SpiderFootPlugin()
sfp.start()
def test_register_listener(self):
"""
Test registerListener(self, listener)
"""
sfp = SpiderFootPlugin()
sfp.registerListener(None)
self.assertEqual('TBD', 'TBD')
def test_getTarget_unitialised_target_should_raise(self):
"""
Test getTarget(self)
"""
sfp = SpiderFootPlugin()
with self.assertRaises(TypeError):
sfp.getTarget()
def test_getScanId_unitialised_scanid_should_raise_TypeError(self):
"""
Test getScanId(self)
"""
sfp = SpiderFootPlugin()
with self.assertRaises(TypeError):
sfp.getScanId()
def test_clearListeners(self):
"""
Test clearListeners(self)
"""
sfp = SpiderFootPlugin()
sfp.clearListeners()
self.assertEqual('TBD', 'TBD')
def test_watchedEvents_should_return_a_list(self):
"""
Test watchedEvents(self)
"""
sfp = SpiderFootPlugin()
watched_events = sfp.watchedEvents()
self.assertIsInstance(watched_events, list)
def test_producedEvents_should_return_a_list(self):
"""
Test producedEvents(self)
"""
sfp = SpiderFootPlugin()
produced_events = sfp.producedEvents()
self.assertIsInstance(produced_events, list)
def test_enrichTargetargument_target_should_enrih_target(self):
"""
Test enrichTarget(self, target)
"""
sfp = SpiderFootPlugin()
sfp.enrichTarget(None)
self.assertEqual('TBD', 'TBD')
def test_updateSocket(self):
"""
Test _updateSocket(self, sock)
"""
sfp = SpiderFootPlugin()
sfp._updateSocket(None)
self.assertEqual('TBD', 'TBD')
def test_tempStorage_should_return_a_dict(self):
"""
Test tempStorage(self)
"""
sfp = SpiderFootPlugin()
temp_storage = sfp.tempStorage()
self.assertIsInstance(temp_storage, dict)
def test_set_dbhargument_dbh_should_set_database_handle(self):
"""
Test setDbh(self, dbh)
"""
sfdb = SpiderFootDb(self.default_options, False)
sfp = SpiderFootPlugin()
sfp.setDbh(sfdb)
self.assertIsInstance(sfp.__sfdb__, SpiderFootDb)
def test_setOutputFilter_should_set_output_filter(self):
"""
Test setOutputFilter(self, types)
"""
sfp = SpiderFootPlugin()
output_filter = "test filter"
sfp.setOutputFilter("test filter")
self.assertEqual(output_filter, sfp.__outputFilter__)
def test_start(self):
"""
Test start(self)
"""
sf = SpiderFoot(self.default_options)
sfp = SpiderFootPlugin()
sfp.sf = sf
sfp.start()
def test_notifyListeners_output_filter_unmatched_should_not_notify_listener_modules(
self):
"""
Test notifyListeners(self, sfEvent)
"""
sfp = SpiderFootPlugin()
sfdb = SpiderFootDb(self.default_options, False)
sfp.setDbh(sfdb)
target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME")
sfp.setTarget(target)
event_type = 'ROOT'
event_data = 'test data'
module = 'test module'
source_event = None
evt = SpiderFootEvent(event_type, event_data, module, source_event)
event_type = 'test event type'
event_data = 'test data'
module = 'test module'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, module, source_event)
sfp.__outputFilter__ = "example unmatched event type"
sfp.notifyListeners(evt)
self.assertEqual('TBD', 'TBD')
def test_notifyListeners_invalid_event_should_raise_TypeError(self):
"""
Test notifyListeners(self, sfEvent)
"""
sfp = SpiderFootPlugin()
invalid_types = [None, "", list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
sfp.notifyListeners(None)
def test_setScanId_argument_id_invalid_type_should_raise_TypeError(self):
"""
Test setScanId(self, id)
"""
sfp = SpiderFootPlugin()
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
sfp.setScanId(invalid_type)
def test_setTarget_should_set_a_target(self):
"""
Test setTarget(self, target)
"""
sfp = SpiderFootPlugin()
target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME")
sfp.setTarget(target)
get_target = sfp.getTarget().targetValue
self.assertIsInstance(get_target, str)
self.assertEqual("spiderfoot.net", get_target)
def test_handleEvent(self):
"""
Test handleEvent(self, sfEvent)
"""
event_type = 'ROOT'
event_data = 'example event data'
module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, module, source_event)
sfp = SpiderFootPlugin()
sfp.handleEvent(evt)
def test_getScanId_should_return_a_string(self):
"""
Test getScanId(self)
"""
sfp = SpiderFootPlugin()
scan_id = 'example scan id'
sfp.setScanId(scan_id)
get_scan_id = sfp.getScanId()
self.assertIsInstance(get_scan_id, str)
self.assertEqual(scan_id, get_scan_id)
def test_setScanId_argument_id_should_set_a_scan_id(self):
"""
Test setScanId(self, id)
"""
sfp = SpiderFootPlugin()
scan_id = '1234'
sfp.setScanId(scan_id)
get_scan_id = sfp.getScanId()
self.assertIsInstance(get_scan_id, str)
self.assertEqual(scan_id, get_scan_id)
def test_checkForStop(self):
"""
Test checkForStop(self)
"""
sfp = SpiderFootPlugin()
class DatabaseStub:
def scanInstanceGet(self, scanId):
return [None, None, None, None, None, status]
sfp.__sfdb__ = DatabaseStub()
sfp.__scanId__ = 'example scan id'
# pseudo-parameterized test
scan_statuses = [(None, False), ("anything", False),
("RUNNING", False), ("ABORT-REQUESTED", True)]
for status, expectedReturnValue in scan_statuses:
returnValue = sfp.checkForStop()
self.assertEqual(returnValue, expectedReturnValue, status)
def test_checkForStop(self):
"""
Test checkForStop(self)
Todo:
include other statuses for completeness
"""
sfp = SpiderFootPlugin()
class DatabaseStub:
def scanInstanceGet(self, scanId):
return [None, None, None, None, None, status]
sfp.__sfdb__ = DatabaseStub()
sfp.__scanId__ = 'example scan id'
# pseudo-parameterized test
for status, expectedReturnValue in [("RUNNING", False),
("ABORT-REQUESTED", True)]:
returnValue = sfp.checkForStop()
self.assertEqual(returnValue, expectedReturnValue, status)
def test_notifyListeners_event_type_and_data_same_as_source_event_source_event_should_story_only(
self):
"""
Test notifyListeners(self, sfEvent)
"""
sfp = SpiderFootPlugin()
sfdb = SpiderFootDb(self.default_options, False)
sfp.setDbh(sfdb)
event_type = 'ROOT'
event_data = 'test data'
module = 'test module'
source_event = None
evt = SpiderFootEvent(event_type, event_data, module, source_event)
event_type = 'test event type'
event_data = 'test data'
module = 'test module'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, module, source_event)
source_event = evt
evt = SpiderFootEvent(event_type, event_data, module, source_event)
source_event = evt
evt = SpiderFootEvent(event_type, event_data, module, source_event)
sfp.notifyListeners(evt)
self.assertEqual('TBD', 'TBD')
def test_notifyListeners(self):
"""
Test notifyListeners(self, sfEvent)
Todo:
Test with source event
"""
sfp = SpiderFootPlugin()
class DatabaseStub:
def scanInstanceGet(self, scanId):
return [None, None, None, None, None, None]
sfp.__sfdb__ = DatabaseStub()
event_type = 'ROOT'
event_data = 'test data'
module = 'test module'
source_event = None
evt = SpiderFootEvent(event_type, event_data, module, source_event)
sfp.notifyListeners(evt)
self.assertEqual('TBD', 'TBD')
def test_setup(self):
"""
Test setup(self, sf, userOpts=dict())
"""
sfp = SpiderFootPlugin()
sfp.setup(None)
sfp.setup(None, None)
self.assertEqual('TBD', 'TBD')
def test_notifyListeners_should_notify_listener_modules(self):
"""
Test notifyListeners(self, sfEvent)
Todo:
Test with source event
"""
sfp = SpiderFootPlugin()
sfdb = SpiderFootDb(self.default_options, False)
sfp.setDbh(sfdb)
event_type = 'ROOT'
event_data = 'test data'
module = 'test module'
source_event = None
evt = SpiderFootEvent(event_type, event_data, module, source_event)
sfp.notifyListeners(evt)
self.assertEqual('TBD', 'TBD')
def __startScan(self, threaded=True):
"""Start running a scan.
Args:
threaded (bool): whether to thread modules
"""
aborted = False
self.__setStatus("STARTING", time.time() * 1000, None)
self.__sf.status(f"Scan [{self.__scanId}] initiated.")
if threaded:
self.eventQueue = queue.Queue()
try:
# moduleList = list of modules the user wants to run
for modName in self.__moduleList:
if modName == '':
continue
try:
module = __import__('modules.' + modName, globals(),
locals(), [modName])
except ImportError:
self.__sf.error(f"Failed to load module: {modName}")
continue
mod = getattr(module, modName)()
mod.__name__ = modName
# Module may have been renamed or removed
if modName not in self.__config['__modules__']:
continue
# Set up the module
# Configuration is a combined global config with module-specific options
self.__modconfig[modName] = deepcopy(
self.__config['__modules__'][modName]['opts'])
for opt in list(self.__config.keys()):
self.__modconfig[modName][opt] = deepcopy(
self.__config[opt])
mod.clearListeners(
) # clear any listener relationships from the past
mod.setup(self.__sf, self.__modconfig[modName])
mod.setDbh(self.__dbh)
mod.setScanId(self.__scanId)
# Give modules a chance to 'enrich' the original target with
# aliases of that target.
newTarget = mod.enrichTarget(self.__target)
if newTarget is not None:
self.__target = newTarget
self.__moduleInstances[modName] = mod
# Override the module's local socket module
# to be the SOCKS one.
if self.__config['_socks1type'] != '':
mod._updateSocket(socket)
# Set up event output filters if requested
if self.__config['__outputfilter']:
mod.setOutputFilter(self.__config['__outputfilter'])
# Register the target with the module
mod.setTarget(self.__target)
if threaded:
# Set up the outgoing event queue
mod.outgoingEventQueue = self.eventQueue
mod.incomingEventQueue = queue.Queue()
self.__sf.status(modName + " module loaded.")
# sort modules by priority
self.__moduleInstances = OrderedDict(
sorted(self.__moduleInstances.items(),
key=lambda m: m[-1]._priority))
if not threaded:
# Register listener modules and then start all modules sequentially
for module in list(self.__moduleInstances.values()):
for listenerModule in list(
self.__moduleInstances.values()):
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() is not None:
module.registerListener(listenerModule)
# Now we are ready to roll..
self.__setStatus("RUNNING")
# Create a pseudo module for the root event to originate from
psMod = SpiderFootPlugin()
psMod.__name__ = "SpiderFoot UI"
psMod.setTarget(self.__target)
psMod.setDbh(self.__dbh)
psMod.clearListeners()
if threaded:
psMod.outgoingEventQueue = self.eventQueue
psMod.incomingEventQueue = queue.Queue()
else:
for mod in list(self.__moduleInstances.values()):
if mod.watchedEvents() is not None:
psMod.registerListener(mod)
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None)
psMod.notifyListeners(rootEvent)
firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# Special case.. check if an INTERNET_NAME is also a domain
if self.__targetType == 'INTERNET_NAME':
if self.__sf.isDomain(self.__targetValue,
self.__config['_internettlds']):
firstEvent = SpiderFootEvent('DOMAIN_NAME',
self.__targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# If in interactive mode, loop through this shared global variable
# waiting for inputs, and process them until my status is set to
# FINISHED.
# Check in case the user requested to stop the scan between modules
# initializing
for mod in list(self.__moduleInstances.values()):
if mod.checkForStop():
self.__setStatus('ABORTING')
aborted = True
break
# start threads
if threaded and not aborted:
self.waitForThreads()
if aborted:
self.__sf.status(f"Scan [{self.__scanId}] aborted.")
self.__setStatus("ABORTED", None, time.time() * 1000)
else:
self.__sf.status(f"Scan [{self.__scanId}] completed.")
self.__setStatus("FINISHED", None, time.time() * 1000)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.__sf.error(
f"Unhandled exception ({e.__class__.__name__}) encountered during scan."
+ "Please report this as a bug: " + repr(
traceback.format_exception(exc_type, exc_value,
exc_traceback)))
self.__sf.status(f"Scan [{self.__scanId}] failed: {e}")
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
self.__dbh.close()
def __startScan(self) -> None:
"""Start running a scan.
Raises:
AssertionError: Never actually raised.
"""
failed = True
try:
self.__setStatus("STARTING", time.time() * 1000, None)
self.__sf.status(
f"Scan [{self.__scanId}] for '{self.__target.targetValue}' initiated."
)
self.eventQueue = queue.Queue()
self.__sharedThreadPool.start()
# moduleList = list of modules the user wants to run
self.__sf.debug(f"Loading {len(self.__moduleList)} modules ...")
for modName in self.__moduleList:
if not modName:
continue
# Module may have been renamed or removed
if modName not in self.__config['__modules__']:
self.__sf.error(f"Failed to load module: {modName}")
continue
try:
module = __import__('modules.' + modName, globals(),
locals(), [modName])
except ImportError:
self.__sf.error(f"Failed to load module: {modName}")
continue
try:
mod = getattr(module, modName)()
mod.__name__ = modName
except Exception:
self.__sf.error(
f"Module {modName} initialization failed: {traceback.format_exc()}"
)
continue
# Set up the module options, scan ID, database handle and listeners
try:
# Configuration is a combined global config with module-specific options
self.__modconfig[modName] = deepcopy(
self.__config['__modules__'][modName]['opts'])
for opt in list(self.__config.keys()):
self.__modconfig[modName][opt] = deepcopy(
self.__config[opt])
# clear any listener relationships from the past
mod.clearListeners()
mod.setScanId(self.__scanId)
mod.setSharedThreadPool(self.__sharedThreadPool)
mod.setDbh(self.__dbh)
mod.setup(self.__sf, self.__modconfig[modName])
except Exception:
self.__sf.error(
f"Module {modName} initialization failed: {traceback.format_exc()}"
)
mod.errorState = True
continue
# Override the module's local socket module to be the SOCKS one.
if self.__config['_socks1type'] != '':
try:
mod._updateSocket(socket)
except Exception as e:
self.__sf.error(
f"Module {modName} socket setup failed: {e}")
continue
# Set up event output filters if requested
if self.__config['__outputfilter']:
try:
mod.setOutputFilter(self.__config['__outputfilter'])
except Exception as e:
self.__sf.error(
f"Module {modName} output filter setup failed: {e}"
)
continue
# Give modules a chance to 'enrich' the original target with aliases of that target.
try:
newTarget = mod.enrichTarget(self.__target)
if newTarget is not None:
self.__target = newTarget
except Exception as e:
self.__sf.error(
f"Module {modName} target enrichment failed: {e}")
continue
# Register the target with the module
try:
mod.setTarget(self.__target)
except Exception as e:
self.__sf.error(
f"Module {modName} failed to set target '{self.__target}': {e}"
)
continue
# Set up the outgoing event queue
try:
mod.outgoingEventQueue = self.eventQueue
mod.incomingEventQueue = queue.Queue()
except Exception as e:
self.__sf.error(
f"Module {modName} event queue setup failed: {e}")
continue
self.__moduleInstances[modName] = mod
self.__sf.status(f"{modName} module loaded.")
self.__sf.debug(
f"Scan [{self.__scanId}] loaded {len(self.__moduleInstances)} modules."
)
if not self.__moduleInstances:
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
self.__dbh.close()
return
# sort modules by priority
self.__moduleInstances = OrderedDict(
sorted(self.__moduleInstances.items(),
key=lambda m: m[-1]._priority))
# Now we are ready to roll..
self.__setStatus("RUNNING")
# Create a pseudo module for the root event to originate from
psMod = SpiderFootPlugin()
psMod.__name__ = "SpiderFoot UI"
psMod.setTarget(self.__target)
psMod.setDbh(self.__dbh)
psMod.clearListeners()
psMod.outgoingEventQueue = self.eventQueue
psMod.incomingEventQueue = queue.Queue()
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None)
psMod.notifyListeners(rootEvent)
firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# Special case.. check if an INTERNET_NAME is also a domain
if self.__targetType == 'INTERNET_NAME':
if self.__sf.isDomain(self.__targetValue,
self.__config['_internettlds']):
firstEvent = SpiderFootEvent('DOMAIN_NAME',
self.__targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# If in interactive mode, loop through this shared global variable
# waiting for inputs, and process them until my status is set to
# FINISHED.
# Check in case the user requested to stop the scan between modules
# initializing
scanstatus = self.__dbh.scanInstanceGet(self.__scanId)
if scanstatus and scanstatus[5] == "ABORT-REQUESTED":
raise AssertionError("ABORT-REQUESTED")
# start threads
self.waitForThreads()
failed = False
except (KeyboardInterrupt, AssertionError):
self.__sf.status(f"Scan [{self.__scanId}] aborted.")
self.__setStatus("ABORTED", None, time.time() * 1000)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.__sf.error(
f"Unhandled exception ({e.__class__.__name__}) encountered during scan."
+ "Please report this as a bug: " + +repr(
traceback.format_exception(exc_type, exc_value,
exc_traceback)))
self.__sf.status(f"Scan [{self.__scanId}] failed: {e}")
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
finally:
if not failed:
self.__setStatus("FINISHED", None, time.time() * 1000)
self.runCorrelations()
self.__sf.status(f"Scan [{self.__scanId}] completed.")
self.__dbh.close()
def test_init(self):
"""
Test __init__(self)
"""
sfp = SpiderFootPlugin()
self.assertIsInstance(sfp, SpiderFootPlugin)
