def test_get_equivalents_should_return_a_list(self):
"""
Test _getEquivalents(self, typeName)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
equivalents = target._getEquivalents(target_type)
self.assertEqual(list, type(equivalents))
def test_matches_should_return_a_boolean(self):
"""
Test matches(self, value, includeParents=False, includeChildren=True)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
matches = target.matches(None)
self.assertEqual(bool, type(matches))
def test_set_alias(self):
"""
Test setAlias(self, value, typeName)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
set_alias = target.setAlias(None, None)
self.assertEqual('TBD', 'TBD')
def test_get_addresses(self):
"""
Test getAddresses(self)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
target.getAddresses()
self.assertEqual('TBD', 'TBD')
def test_get_names_should_return_a_list(self):
"""
Test getNames(self)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
names = target.getNames()
self.assertEqual(list, type(names))
def test_matches(self):
"""
Test matches(self, value, includeParents=False, includeChildren=True)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
target.matches(None)
self.assertEqual('TBD', 'TBD')
def test_init_supported_target_types(self):
"""
Test __init__(self, targetValue, typeName)
"""
target_value = 'example target value'
for target_type in self.valid_target_types:
with self.subTest(target_type=target_type):
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(SpiderFootTarget, type(target))
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
def test_processHost_should_return_SpiderFootEvent(self):
"""
Test processHost(self, host, parentEvent, affiliate=None)
"""
sf = SpiderFoot(self.default_options)
module = sfp_dnsresolve()
module.setup(sf, dict())
target_value = '127.0.0.1'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = 'example module'
source_event = ''
parent_event = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.processHost("127.0.0.1", parent_event, None)
self.assertIsInstance(result, SpiderFootEvent)
self.assertIsNone(result)
def test_processDomain_should_return_None(self):
"""
Test processDomain(self, domainName, parentEvent, affil=False, host=None)
Todo:
review: why should this return None?
"""
sf = SpiderFoot(self.default_options)
module = sfp_dnsresolve()
module.setup(sf, dict())
target_value = 'example.local'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = 'example module'
source_event = ''
parent_event = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.processDomain('www.example.local', parent_event, None, None)
self.assertIsNone(result)
def test_init_unsupported_target_type_should_raise(self):
"""
Test __init__(self, targetValue, typeName)
"""
with self.assertRaises(ValueError) as cm:
target = SpiderFootTarget('example target value',
'example target type')
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_iban()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = 'sfp_test_module'
source_event = ''
source_event = SpiderFootEvent(event_type, event_data, event_module,
source_event)
event_type = 'TARGET_WEB_CONTENT'
event_data = 'example data'
event_module = 'sfp_test_module'
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
self.assertEqual('TBD', 'TBD')
self.assertIsNone(result)
def test_target_aliases_attribute_should_return_a_list(self):
"""
Test targetAliases attribute
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
self.assertIsInstance(target.targetAliases, list)
def test_resolve_targets_should_return_list(self):
"""
Test resolveTargets(self, target, validateReverse)
"""
sf = SpiderFoot(self.default_options)
target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME")
resolve_targets = sf.resolveTargets(target, False)
self.assertIsInstance(resolve_targets, list)
self.assertIn('spiderfoot.net', resolve_targets)
def test_target_value_attribute_should_return_a_string(self):
"""
Test targetValue attribute
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
self.assertIsInstance(target.targetValue, str)
self.assertEqual(target_value, target.targetValue)
def test_init_unsupported_target_type_should_exit(self):
"""
Test __init__(self, targetValue, typeName)
"""
target_value = 'example target value'
with self.assertRaises(SystemExit) as cm:
target_type = 'example target type'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(cm.exception.code, -1)
def test_set_target_should_set_a_target(self):
"""
Test setTarget(self, target)
"""
sfp = SpiderFootPlugin()
target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME")
sfp.setTarget(target)
get_target = sfp.getTarget().targetValue
self.assertIsInstance(get_target, str)
self.assertEqual("spiderfoot.net", get_target)
def test_resolve_targets_should_return_list(self):
"""
Test resolveTargets(self, target, validateReverse)
"""
sf = SpiderFoot(self.default_options)
invalid_types = [None, "", list()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
resolve_targets = sf.resolveTargets(invalid_type, False)
self.assertIsInstance(resolve_targets, list)
target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME")
resolve_targets = sf.resolveTargets(target, False)
self.assertIsInstance(resolve_targets, list)
self.assertIn('spiderfoot.net', resolve_targets)
target = SpiderFootTarget("1.1.1.1", "IP_ADDRESS")
resolve_targets = sf.resolveTargets(target, False)
self.assertIsInstance(resolve_targets, list)
self.assertIn('1.1.1.1', resolve_targets)
def test_enrichTarget(self):
"""
Test enrichTarget(self, target)
"""
sf = SpiderFoot(self.default_options)
module = sfp_dnsresolve()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
result = module.enrichTarget(target)
self.assertIsInstance(result, SpiderFootTarget)
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_company()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_hostio()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'DOMAIN_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_init_invalid_target_value_should_raise(self):
"""
Test __init__(self, targetValue, typeName)
"""
with self.assertRaises(TypeError) as cm:
target = SpiderFootTarget(None, 'IP_ADDRESS')
def __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True):
"""Initialize SpiderFootScanner object.
Args:
scanName (str): name of the scan
scanId (str): unique ID of the scan
scanTarget (str): scan target
targetType (str): scan target type
moduleList (list): list of modules to run
globalOpts (dict): scan options
start (bool): start the scan immediately
Returns:
None
"""
if not isinstance(scanName, str):
raise TypeError("scanName is %s; expected str()" % type(scanName))
if not isinstance(scanId, str):
raise TypeError("scanId is %s; expected str()" % type(scanId))
if not isinstance(scanTarget, str):
raise TypeError("scanTarget is %s; expected str()" % type(scanTarget))
if not isinstance(targetType, str):
raise TypeError("targetType is %s; expected str()" % type(targetType))
if not isinstance(moduleList, list):
raise TypeError("moduleList is %s; expected list()" % type(moduleList))
if not isinstance(globalOpts, dict):
raise TypeError("globalOpts is %s; expected dict()" % type(globalOpts))
if not globalOpts:
raise ValueError("globalOpts is empty")
self.moduleInstances = dict()
self.config = deepcopy(globalOpts)
self.sf = SpiderFoot(self.config)
self.dbh = SpiderFootDb(self.config)
self.targetValue = scanTarget
self.targetType = targetType
self.moduleList = moduleList
self.modconfig = dict()
self.scanName = scanName
self.sf.setDbh(self.dbh)
# Create a unique ID for this scan and create it in the back-end DB.
self.scanId = scanId
self.sf.setGUID(self.scanId)
self.dbh.scanInstanceCreate(self.scanId, self.scanName, self.targetValue)
# Create our target
try:
self.target = SpiderFootTarget(self.targetValue, self.targetType)
except ValueError as e:
self.sf.status("Scan [%s] failed: %s" % (self.scanId, e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError("Invalid target: %s" % e)
except TypeError as e:
self.sf.status("Scan [%s] failed: %s" % (self.scanId, e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
raise TypeError("Invalid target: %s" % e)
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
self.dbh.scanConfigSet(self.scanId, self.sf.configSerialize(deepcopy(self.config)))
self.setStatus("INITIALIZING", time.time() * 1000, None)
if start:
self.startScan()
def __init__(self,
scanName,
scanId,
targetValue,
targetType,
moduleList,
globalOpts,
start=True):
"""Initialize SpiderFootScanner object.
Args:
scanName (str): name of the scan
scanId (str): unique ID of the scan
targetValue (str): scan target
targetType (str): scan target type
moduleList (list): list of modules to run
globalOpts (dict): scan options
start (bool): start the scan immediately
Raises:
TypeError: arg type was invalid
ValueError: arg value was invalid
Todo:
Eventually change this to be able to control multiple scan instances
"""
if not isinstance(globalOpts, dict):
raise TypeError(
f"globalOpts is {type(globalOpts)}; expected dict()")
if not globalOpts:
raise ValueError("globalOpts is empty")
self.__config = deepcopy(globalOpts)
self.__dbh = SpiderFootDb(self.__config)
if not isinstance(scanName, str):
raise TypeError(f"scanName is {type(scanName)}; expected str()")
if not scanName:
raise ValueError("scanName value is blank")
self.__scanName = scanName
if not isinstance(scanId, str):
raise TypeError(f"scanId is {type(scanId)}; expected str()")
if not scanId:
raise ValueError("scanId value is blank")
if not isinstance(targetValue, str):
raise TypeError(
f"targetValue is {type(targetValue)}; expected str()")
if not targetValue:
raise ValueError("targetValue value is blank")
self.__targetValue = targetValue
if not isinstance(targetType, str):
raise TypeError(
f"targetType is {type(targetType)}; expected str()")
if not targetType:
raise ValueError("targetType value is blank")
self.__targetType = targetType
if not isinstance(moduleList, list):
raise TypeError(
f"moduleList is {type(moduleList)}; expected list()")
if not moduleList:
raise ValueError("moduleList is empty")
self.__moduleList = moduleList
self.__sf = SpiderFoot(self.__config)
self.__sf.dbh = self.__dbh
# Create a unique ID for this scan in the back-end DB.
if not isinstance(scanId, str):
raise TypeError(f"scanId is {type(scanId)}; expected str()")
if scanId:
self.__scanId = scanId
else:
self.__scanId = self.__sf.genScanInstanceId()
self.__sf.scanId = self.__scanId
self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName,
self.__targetValue)
# Create our target
try:
self.__target = SpiderFootTarget(self.__targetValue,
self.__targetType)
except (TypeError, ValueError) as e:
self.__sf.status(f"Scan [{self.__scanId}] failed: {e}")
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError(f"Invalid target: {e}")
# Save the config current set for this scan
self.__config['_modulesenabled'] = self.__moduleList
self.__dbh.scanConfigSet(
self.__scanId, self.__sf.configSerialize(deepcopy(self.__config)))
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
if self.__config['_socks1type']:
socksDns = self.__config['_socks6dns']
socksAddr = self.__config['_socks2addr']
socksPort = int(self.__config['_socks3port'])
socksUsername = self.__config['_socks4user'] or ''
socksPassword = self.__config['_socks5pwd'] or ''
proxy = f"{socksAddr}:{socksPort}"
if socksUsername or socksPassword:
proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy)
if self.__config['_socks1type'] == '4':
proxy = 'socks4://' + proxy
elif self.__config['_socks1type'] == '5':
proxy = 'socks5://' + proxy
elif self.__config['_socks1type'] == 'HTTP':
proxy = 'http://' + proxy
elif self.__config['_socks1type'] == 'TOR':
proxy = 'socks5h://' + proxy
else:
raise ValueError(
f"Invalid SOCKS proxy type: {self.__config['_socks1ttype']}"
)
self.__sf.debug(
f"SOCKS: {socksAddr}:{socksPort} ({socksUsername}:{socksPassword})"
)
self.__sf.socksProxy = proxy
else:
self.__sf.socksProxy = None
# Override the default DNS server
if self.__config['_dnsserver']:
res = dns.resolver.Resolver()
res.nameservers = [self.__config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.__config['_useragent'] = self.__sf.optValueToData(
self.__config['_useragent'])
# Get internet TLDs
tlddata = self.__sf.cacheGet("internet_tlds",
self.__config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata is None:
self.__config['_internettlds'] = self.__sf.optValueToData(
self.__config['_internettlds'])
self.__sf.cachePut("internet_tlds", self.__config['_internettlds'])
else:
self.__config["_internettlds"] = tlddata.splitlines()
self.__setStatus("INITIALIZING", time.time() * 1000, None)
if start:
self.__startScan()
def startScan(self):
global globalScanStatus
self.ts = threading.local()
self.ts.moduleInstances = dict()
self.ts.sf = SpiderFoot(self.temp['config'])
self.ts.config = deepcopy(self.temp['config'])
self.ts.dbh = SpiderFootDb(self.temp['config'])
self.ts.targetValue = self.temp['targetValue']
self.ts.targetType = self.temp['targetType']
self.ts.moduleList = self.temp['moduleList']
self.ts.modconfig = dict()
self.ts.scanName = self.temp['scanName']
self.ts.scanId = self.temp['scanId']
aborted = False
self.ts.sf.setDbh(self.ts.dbh)
# Create a unique ID for this scan and create it in the back-end DB.
self.ts.sf.setGUID(self.ts.scanId)
self.ts.dbh.scanInstanceCreate(self.ts.scanId, self.ts.scanName,
self.ts.targetValue)
self.setStatus("STARTING", time.time() * 1000, None)
# Create our target
target = SpiderFootTarget(self.ts.targetValue, self.ts.targetType)
# Save the config current set for this scan
self.ts.config['_modulesenabled'] = self.ts.moduleList
self.ts.dbh.scanConfigSet(
self.ts.scanId,
self.ts.sf.configSerialize(deepcopy(self.ts.config)))
self.ts.sf.status("Scan [" + self.ts.scanId + "] initiated.")
# moduleList = list of modules the user wants to run
try:
# Process global options that point to other places for data
# Save default socket methods that will be overridden
if not hasattr(socket, 'savedsocket'):
socket.savedsocket = socket.socket
socket.savedcreate_connection = socket.create_connection
#socket.savedgetaddrinfo = socket.getaddrinfo
# If a SOCKS server was specified, set it up
if self.ts.config['_socks1type'] != '':
socksType = socks.PROXY_TYPE_SOCKS4
socksDns = self.ts.config['_socks6dns']
socksAddr = self.ts.config['_socks2addr']
socksPort = int(self.ts.config['_socks3port'])
socksUsername = ''
socksPassword = ''
if self.ts.config['_socks1type'] == '4':
socksType = socks.PROXY_TYPE_SOCKS4
if self.ts.config['_socks1type'] == '5':
socksType = socks.PROXY_TYPE_SOCKS5
socksUsername = self.ts.config['_socks4user']
socksPassword = self.ts.config['_socks5pwd']
if self.ts.config['_socks1type'] == 'HTTP':
socksType = socks.PROXY_TYPE_HTTP
if self.ts.config['_socks1type'] == 'TOR':
socksType = socks.PROXY_TYPE_SOCKS5
self.ts.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \
"(" + socksUsername + ":" + socksPassword + ")")
socks.setdefaultproxy(socksType, socksAddr, socksPort,
socksDns, socksUsername, socksPassword)
# Override the default socket and getaddrinfo calls with the
# SOCKS ones. Just ensure we don't also try and SOCKS-proxy
# connectivity to the TOR control port.
def _create_connection(address,
timeout=None,
source_address=None):
if socksAddr not in address:
sock = socks.socksocket()
sock.setproxy(socks.PROXY_TYPE_SOCKS5, socksAddr,
socksPort)
sock.settimeout(self.ts.config['_fetchtimeout'])
sock.connect(address)
return sock
else:
sock = socket.socket
sock.settimeout(self.ts.config['_fetchtimeout'])
sock.connect(address)
return sock
socket.socket = socks.socksocket
socket.setdefaulttimeout(self.ts.config['_fetchtimeout'])
socket.create_connection = _create_connection
#socket.getaddrinfo = socks.getaddrinfo
self.ts.sf.updateSocket(socket)
else:
# BUG: If the user had a SOCKS proxy set
# and then decided to unset it, the original socket class
# is not reverted to its default state - we still have
# the SOCKS version of socket.
socket.socket = socket.savedsocket
socket.setdefaulttimeout(self.ts.config['_fetchtimeout'])
socket.create_connection = socket.savedcreate_connection
#socket.getaddrinfo = socket.savedgetaddrinfo
self.ts.sf.revertSocket()
# Override the default DNS server
if self.ts.config['_dnsserver'] != "":
res = dns.resolver.Resolver()
res.nameservers = [self.ts.config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.ts.config['_useragent'] = self.ts.sf.optValueToData(
self.ts.config['_useragent'])
# Get internet TLDs
tlddata = self.ts.sf.cacheGet(
"internet_tlds", self.ts.config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata is None:
self.ts.config['_internettlds'] = self.ts.sf.optValueToData(
self.ts.config['_internettlds'])
self.ts.sf.cachePut("internet_tlds",
self.ts.config['_internettlds'])
else:
self.ts.config["_internettlds"] = tlddata.splitlines()
for modName in self.ts.moduleList:
if modName == '':
continue
module = __import__('modules.' + modName, globals(), locals(),
[modName])
mod = getattr(module, modName)()
mod.__name__ = modName
# Module may have been renamed or removed
if modName not in self.ts.config['__modules__']:
continue
# Set up the module
# Configuration is a combined global config with module-specific options
self.ts.modconfig[modName] = deepcopy(
self.ts.config['__modules__'][modName]['opts'])
for opt in self.ts.config.keys():
self.ts.modconfig[modName][opt] = deepcopy(
self.ts.config[opt])
mod.clearListeners(
) # clear any listener relationships from the past
mod.setup(self.ts.sf, self.ts.modconfig[modName])
mod.setDbh(self.ts.dbh)
mod.setScanId(self.ts.scanId)
# Give modules a chance to 'enrich' the original target with
# aliases of that target.
newTarget = mod.enrichTarget(target)
if newTarget is not None:
target = newTarget
self.ts.moduleInstances[modName] = mod
# Override the module's local socket module
# to be the SOCKS one.
if self.ts.config['_socks1type'] != '':
mod._updateSocket(socket)
# Set up event output filters if requested
if self.ts.config['__outputfilter']:
mod.setOutputFilter(self.ts.config['__outputfilter'])
self.ts.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
for module in self.ts.moduleInstances.values():
# Register the target with the module
module.setTarget(target)
for listenerModule in self.ts.moduleInstances.values():
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() is not None:
module.registerListener(listenerModule)
# Now we are ready to roll..
self.setStatus("RUNNING")
# Create a pseudo module for the root event to originate from
psMod = SpiderFootPlugin()
psMod.__name__ = "SpiderFoot UI"
psMod.setTarget(target)
psMod.clearListeners()
for mod in self.ts.moduleInstances.values():
if mod.watchedEvents() is not None:
psMod.registerListener(mod)
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("ROOT", self.ts.targetValue, "", None)
psMod.notifyListeners(rootEvent)
firstEvent = SpiderFootEvent(self.ts.targetType,
self.ts.targetValue, "SpiderFoot UI",
rootEvent)
psMod.notifyListeners(firstEvent)
# If in interactive mode, loop through this shared global variable
# waiting for inputs, and process them until my status is set to
# FINISHED.
# Check in case the user requested to stop the scan between modules
# initializing
for module in self.ts.moduleInstances.values():
if module.checkForStop():
self.setStatus('ABORTING')
aborted = True
break
if aborted:
self.ts.sf.status("Scan [" + self.ts.scanId + "] aborted.")
self.setStatus("ABORTED", None, time.time() * 1000)
else:
self.ts.sf.status("Scan [" + self.ts.scanId + "] completed.")
self.setStatus("FINISHED", None, time.time() * 1000)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.ts.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
"encountered during scan. Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.ts.sf.status("Scan [" + self.ts.scanId + "] failed: " +
str(e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
self.ts.dbh.close()
del self.ts
del self.temp
def __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True):
"""Initialize SpiderFootScanner object.
Args:
scanName (str): name of the scan
scanId (str): unique ID of the scan
scanTarget (str): scan target
targetType (str): scan target type
moduleList (list): list of modules to run
globalOpts (dict): scan options
start (bool): start the scan immediately
Returns:
None
Todo:
Eventually change this to be able to control multiple scan instances
"""
if not isinstance(scanName, str):
raise TypeError("scanName is %s; expected str()" % type(scanName))
if not isinstance(scanId, str):
raise TypeError("scanId is %s; expected str()" % type(scanId))
if not isinstance(scanTarget, str):
raise TypeError("scanTarget is %s; expected str()" % type(scanTarget))
if not isinstance(targetType, str):
raise TypeError("targetType is %s; expected str()" % type(targetType))
if not isinstance(moduleList, list):
raise TypeError("moduleList is %s; expected list()" % type(moduleList))
if not isinstance(globalOpts, dict):
raise TypeError("globalOpts is %s; expected dict()" % type(globalOpts))
if not globalOpts:
raise ValueError("globalOpts is empty")
self.moduleInstances = dict()
self.config = deepcopy(globalOpts)
self.sf = SpiderFoot(self.config)
self.dbh = SpiderFootDb(self.config)
self.targetValue = scanTarget
self.targetType = targetType
self.moduleList = moduleList
self.modconfig = dict()
self.scanName = scanName
self.sf.setDbh(self.dbh)
# Create a unique ID for this scan and create it in the back-end DB.
self.scanId = scanId
self.sf.setGUID(self.scanId)
self.dbh.scanInstanceCreate(self.scanId, self.scanName, self.targetValue)
# Create our target
try:
self.target = SpiderFootTarget(self.targetValue, self.targetType)
except ValueError as e:
self.sf.status("Scan [%s] failed: %s" % (self.scanId, e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError("Invalid target: %s" % e)
except TypeError as e:
self.sf.status("Scan [%s] failed: %s" % (self.scanId, e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
raise TypeError("Invalid target: %s" % e)
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
self.dbh.scanConfigSet(self.scanId, self.sf.configSerialize(deepcopy(self.config)))
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
if self.config['_socks1type']:
socksDns = self.config['_socks6dns']
socksAddr = self.config['_socks2addr']
socksPort = int(self.config['_socks3port'])
socksUsername = self.config['_socks4user'] or ''
socksPassword = self.config['_socks5pwd'] or ''
proxy = "%s:%s" % (socksAddr, socksPort)
if socksUsername or socksPassword:
proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy)
if self.config['_socks1type'] == '4':
proxy = 'socks4://' + proxy
elif self.config['_socks1type'] == '5':
proxy = 'socks5://' + proxy
elif self.config['_socks1type'] == 'HTTP':
proxy = 'http://' + proxy
elif self.config['_socks1type'] == 'TOR':
proxy = 'socks5h://' + proxy
else:
raise ValueError("Invalid SOCKS proxy type: %s" % self.config["_socks1ttype"])
self.sf.debug("SOCKS: %s:%s (%s:%s)" % (socksAddr, socksPort, socksUsername, socksPassword))
self.sf.updateSocket(proxy)
else:
self.sf.revertSocket()
# Override the default DNS server
if self.config['_dnsserver']:
res = dns.resolver.Resolver()
res.nameservers = [self.config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.config['_useragent'] = self.sf.optValueToData(self.config['_useragent'])
# Get internet TLDs
tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata is None:
self.config['_internettlds'] = self.sf.optValueToData(self.config['_internettlds'])
self.sf.cachePut("internet_tlds", self.config['_internettlds'])
else:
self.config["_internettlds"] = tlddata.splitlines()
self.setStatus("INITIALIZING", time.time() * 1000, None)
if start:
self.startScan()
def startScan(self):
"""Start running a scan."""
self.moduleInstances = dict()
self.sf = SpiderFoot(self.temp['config'])
self.config = deepcopy(self.temp['config'])
self.dbh = SpiderFootDb(self.temp['config'])
self.targetValue = self.temp['targetValue']
self.targetType = self.temp['targetType']
self.moduleList = self.temp['moduleList']
self.modconfig = dict()
self.scanName = self.temp['scanName']
self.scanId = self.temp['scanId']
aborted = False
self.sf.setDbh(self.dbh)
# Create a unique ID for this scan and create it in the back-end DB.
self.sf.setGUID(self.scanId)
self.dbh.scanInstanceCreate(self.scanId,
self.scanName, self.targetValue)
self.setStatus("STARTING", time.time() * 1000, None)
# Create our target
try:
target = SpiderFootTarget(self.targetValue, self.targetType)
except BaseException as e:
self.sf.status("Scan [%s] failed: %s" % (self.scanId, e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
return None
# Save the config current set for this scan
self.config['_modulesenabled'] = self.moduleList
self.dbh.scanConfigSet(self.scanId,
self.sf.configSerialize(deepcopy(self.config)))
self.sf.status("Scan [" + self.scanId + "] initiated.")
# moduleList = list of modules the user wants to run
try:
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
if self.config['_socks1type'] != '':
socksDns = self.config['_socks6dns']
socksAddr = self.config['_socks2addr']
socksPort = int(self.config['_socks3port'])
socksUsername = self.config['_socks4user'] or ''
socksPassword = self.config['_socks5pwd'] or ''
creds = ""
if socksUsername and socksPassword:
creds = socksUsername + ":" + socksPassword + "@"
proxy = creds + socksAddr + ":" + str(socksPort)
if self.config['_socks1type'] == '4':
proxy = 'socks4://' + proxy
elif self.config['_socks1type'] == '5':
proxy = 'socks5://' + proxy
elif self.config['_socks1type'] == 'HTTP':
proxy = 'http://' + proxy
elif self.config['_socks1type'] == 'TOR':
proxy = 'socks5h://' + proxy
self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \
"(" + socksUsername + ":" + socksPassword + ")")
self.sf.updateSocket(proxy)
else:
self.sf.revertSocket()
# Override the default DNS server
if self.config['_dnsserver'] != "":
res = dns.resolver.Resolver()
res.nameservers = [self.config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.config['_useragent'] = self.sf.optValueToData(
self.config['_useragent'])
# Get internet TLDs
tlddata = self.sf.cacheGet("internet_tlds",
self.config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata is None:
self.config['_internettlds'] = self.sf.optValueToData(
self.config['_internettlds'])
self.sf.cachePut("internet_tlds", self.config['_internettlds'])
else:
self.config["_internettlds"] = tlddata.splitlines()
for modName in self.moduleList:
if modName == '':
continue
try:
module = __import__('modules.' + modName, globals(), locals(),
[modName])
except ImportError:
self.sf.error("Failed to load module: " + modName, False)
continue
mod = getattr(module, modName)()
mod.__name__ = modName
# Module may have been renamed or removed
if modName not in self.config['__modules__']:
continue
# Set up the module
# Configuration is a combined global config with module-specific options
self.modconfig[modName] = deepcopy(self.config['__modules__'][modName]['opts'])
for opt in list(self.config.keys()):
self.modconfig[modName][opt] = deepcopy(self.config[opt])
mod.clearListeners() # clear any listener relationships from the past
mod.setup(self.sf, self.modconfig[modName])
mod.setDbh(self.dbh)
mod.setScanId(self.scanId)
# Give modules a chance to 'enrich' the original target with
# aliases of that target.
newTarget = mod.enrichTarget(target)
if newTarget is not None:
target = newTarget
self.moduleInstances[modName] = mod
# Override the module's local socket module
# to be the SOCKS one.
if self.config['_socks1type'] != '':
mod._updateSocket(socket)
# Set up event output filters if requested
if self.config['__outputfilter']:
mod.setOutputFilter(self.config['__outputfilter'])
self.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
for module in list(self.moduleInstances.values()):
# Register the target with the module
module.setTarget(target)
for listenerModule in list(self.moduleInstances.values()):
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
# Note the absence of a check for whether a module can register
# to itself. That is intentional because some modules will
# act on their own notifications (e.g. sfp_dns)!
if listenerModule.watchedEvents() is not None:
module.registerListener(listenerModule)
# Now we are ready to roll..
self.setStatus("RUNNING")
# Create a pseudo module for the root event to originate from
psMod = SpiderFootPlugin()
psMod.__name__ = "SpiderFoot UI"
psMod.setTarget(target)
psMod.setDbh(self.dbh)
psMod.clearListeners()
for mod in list(self.moduleInstances.values()):
if mod.watchedEvents() is not None:
psMod.registerListener(mod)
# Create the "ROOT" event which un-triggered modules will link events to
rootEvent = SpiderFootEvent("ROOT", self.targetValue, "", None)
psMod.notifyListeners(rootEvent)
firstEvent = SpiderFootEvent(self.targetType, self.targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# Special case.. check if an INTERNET_NAME is also a domain
if self.targetType == 'INTERNET_NAME':
if self.sf.isDomain(self.targetValue, self.config['_internettlds']):
firstEvent = SpiderFootEvent('DOMAIN_NAME', self.targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# If in interactive mode, loop through this shared global variable
# waiting for inputs, and process them until my status is set to
# FINISHED.
# Check in case the user requested to stop the scan between modules
# initializing
for module in list(self.moduleInstances.values()):
if module.checkForStop():
self.setStatus('ABORTING')
aborted = True
break
if aborted:
self.sf.status("Scan [" + self.scanId + "] aborted.")
self.setStatus("ABORTED", None, time.time() * 1000)
else:
self.sf.status("Scan [" + self.scanId + "] completed.")
self.setStatus("FINISHED", None, time.time() * 1000)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
"encountered during scan. Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.sf.status("Scan [" + self.scanId + "] failed: " + str(e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
self.dbh.close()
def test_init(self):
"""
Test __init__
"""
#target = SpiderFootTarget('junk', 'junk')
#self.assertEqual(target, None)
target_value = '127.0.0.1'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
target_value = '::1'
target_type = 'IPV6_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
target_value = '0.0.0.0/0'
target_type = 'NETBLOCK_OWNER'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
target_value = 'localhost.local'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
target_value = '*****@*****.**'
target_type = 'EMAILADDR'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
target_value = 'hello'
target_type = 'HUMAN_NAME'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
target_value = '1234'
target_type = 'BGP_AS_OWNER'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)
target_value = '+12345678901'
target_type = 'PHONE_NUMBER'
target = SpiderFootTarget(target_value, target_type)
self.assertEqual(target.getType(), target_type)
self.assertEqual(target.getValue(), target_value)