def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_webanalytics()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_cleanbrowsing()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'cleanbrowsing.org'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent_no_api_key_should_set_errorState(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_ipregistry()
module.setup(sf, dict())
target_value = "example target value"
target_type = "IP_ADDRESS"
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = "ROOT"
event_data = "example data"
event_module = ""
source_event = ""
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
self.assertTrue(module.errorState)
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_searchcode()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'DOMAIN_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_dnsdb()
module.setup(sf, dict())
target_value = "example target value"
target_type = "IP_ADDRESS"
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = "ROOT"
event_data = "example data"
event_module = ""
source_event = ""
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent_no_api_key_should_set_errorState(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_grayhatwarfare()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
self.assertTrue(module.errorState)
def test_handleEvent_no_tool_path_configured_should_set_errorState(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_tool_cmseek()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
self.assertTrue(module.errorState)
def test_handleEvent_event_data_web_content_not_containing_webframework_string_should_not_create_event(
self):
sf = SpiderFoot(self.default_options)
module = sfp_webframework()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(
module, sfp_webframework)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
event_type = 'TARGET_WEB_CONTENT'
event_data = 'example data'
event_module = 'example module'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
evt.actualSource = "https://spiderfoot.net/"
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent_event_data_affiliate_internet_name_not_matching_ad_server_should_not_return_event(self):
sf = SpiderFoot(self.default_options)
module = sfp_stevenblack_hosts()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
module.opts['_fetchtimeout'] = 15
module.optdescs['_fetchtimeout'] = ''
module.opts['_useragent'] = ''
module.optdescs['_useragent'] = ''
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(module, sfp_stevenblack_hosts)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
event_type = 'AFFILIATE_INTERNET_NAME'
event_data = 'no.ads.safe.local'
event_module = 'example module'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent_event_data_not_containing_cookie_should_not_return_event(
self):
sf = SpiderFoot(self.default_options)
module = sfp_cookie()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(
module, sfp_cookie)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
event_type = 'WEBSERVER_HTTPHEADERS'
event_data = '{"not cookie": "example cookie"}'
event_module = 'sfp_spider'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
evt.actualSource = "https://spiderfoot.net/example"
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_setAlias_should_set_alias(self):
"""
Test setAlias(self, value, typeName)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
target.setAlias("example value", "example type")
expected_aliases = [{'type': 'example type', 'value': 'example value'}]
target_aliases = target.targetAliases
self.assertEqual(expected_aliases, target_aliases)
# check duplicated aliases aren't created
target.setAlias("example value", "example type")
target_aliases = target.targetAliases
self.assertEqual(expected_aliases, target_aliases)
def __init__(self,
scanName: str,
scanId: str,
targetValue: str,
targetType: str,
moduleList: list,
globalOpts: dict,
start: bool = True) -> None:
"""Initialize SpiderFootScanner object.
Args:
scanName (str): name of the scan
scanId (str): unique ID of the scan
targetValue (str): scan target
targetType (str): scan target type
moduleList (list): list of modules to run
globalOpts (dict): scan options
start (bool): start the scan immediately
Raises:
TypeError: arg type was invalid
ValueError: arg value was invalid
Todo:
Eventually change this to be able to control multiple scan instances
"""
if not isinstance(globalOpts, dict):
raise TypeError(
f"globalOpts is {type(globalOpts)}; expected dict()")
if not globalOpts:
raise ValueError("globalOpts is empty")
self.__config = deepcopy(globalOpts)
self.__dbh = SpiderFootDb(self.__config)
if not isinstance(scanName, str):
raise TypeError(f"scanName is {type(scanName)}; expected str()")
if not scanName:
raise ValueError("scanName value is blank")
self.__scanName = scanName
if not isinstance(scanId, str):
raise TypeError(f"scanId is {type(scanId)}; expected str()")
if not scanId:
raise ValueError("scanId value is blank")
if not isinstance(targetValue, str):
raise TypeError(
f"targetValue is {type(targetValue)}; expected str()")
if not targetValue:
raise ValueError("targetValue value is blank")
self.__targetValue = targetValue
if not isinstance(targetType, str):
raise TypeError(
f"targetType is {type(targetType)}; expected str()")
if not targetType:
raise ValueError("targetType value is blank")
self.__targetType = targetType
if not isinstance(moduleList, list):
raise TypeError(
f"moduleList is {type(moduleList)}; expected list()")
if not moduleList:
raise ValueError("moduleList is empty")
self.__moduleList = moduleList
self.__sf = SpiderFoot(self.__config)
self.__sf.dbh = self.__dbh
# Create a unique ID for this scan in the back-end DB.
if scanId:
self.__scanId = scanId
else:
self.__scanId = SpiderFootHelpers.genScanInstanceId()
self.__sf.scanId = self.__scanId
self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName,
self.__targetValue)
# Create our target
try:
self.__target = SpiderFootTarget(self.__targetValue,
self.__targetType)
except (TypeError, ValueError) as e:
self.__sf.status(f"Scan [{self.__scanId}] failed: {e}")
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError(f"Invalid target: {e}") from None
# Save the config current set for this scan
self.__config['_modulesenabled'] = self.__moduleList
self.__dbh.scanConfigSet(
self.__scanId, self.__sf.configSerialize(deepcopy(self.__config)))
# Process global options that point to other places for data
# If a proxy server was specified, set it up
proxy_type = self.__config.get('_socks1type')
if proxy_type:
# TODO: allow DNS lookup to be configurable when using a proxy
# - proxy DNS lookup: socks5h:// and socks4a://
# - local DNS lookup: socks5:// and socks4://
if proxy_type == '4':
proxy_proto = 'socks4://'
elif proxy_type == '5':
proxy_proto = 'socks5://'
elif proxy_type == 'HTTP':
proxy_proto = 'http://'
elif proxy_type == 'TOR':
proxy_proto = 'socks5h://'
else:
self.__sf.status(
f"Scan [{self.__scanId}] failed: Invalid proxy type: {proxy_type}"
)
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError(f"Invalid proxy type: {proxy_type}")
proxy_host = self.__config.get('_socks2addr', '')
if not proxy_host:
self.__sf.status(
f"Scan [{self.__scanId}] failed: Proxy type is set ({proxy_type}) but proxy address value is blank"
)
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError(
f"Proxy type is set ({proxy_type}) but proxy address value is blank"
)
proxy_port = int(self.__config.get('_socks3port') or 0)
if not proxy_port:
if proxy_type in ['4', '5']:
proxy_port = 1080
elif proxy_type.upper() == 'HTTP':
proxy_port = 8080
elif proxy_type.upper() == 'TOR':
proxy_port = 9050
proxy_username = self.__config.get('_socks4user', '')
proxy_password = self.__config.get('_socks5pwd', '')
if proxy_username or proxy_password:
proxy_auth = f"{proxy_username}:{proxy_password}"
proxy = f"{proxy_proto}{proxy_auth}@{proxy_host}:{proxy_port}"
else:
proxy = f"{proxy_proto}{proxy_host}:{proxy_port}"
self.__sf.debug(f"Using proxy: {proxy}")
self.__sf.socksProxy = proxy
else:
self.__sf.socksProxy = None
# Override the default DNS server
if self.__config['_dnsserver']:
res = dns.resolver.Resolver()
res.nameservers = [self.__config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.__config['_useragent'] = self.__sf.optValueToData(
self.__config['_useragent'])
# Set up the Internet TLD list.
# If the cached does not exist or has expired, reload it from scratch.
tld_data = self.__sf.cacheGet("internet_tlds",
self.__config['_internettlds_cache'])
if tld_data is None:
tld_data = self.__sf.optValueToData(self.__config['_internettlds'])
if tld_data is None:
self.__sf.status(
f"Scan [{self.__scanId}] failed: Could not update TLD list"
)
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError("Could not update TLD list")
self.__sf.cachePut("internet_tlds", tld_data)
self.__config['_internettlds'] = tld_data.splitlines()
self.__setStatus("INITIALIZING", time.time() * 1000, None)
self.__sharedThreadPool = SpiderFootThreadPool(
threads=self.__config.get("_maxthreads", 3),
name='sharedThreadPool')
# Used when module threading is enabled
self.eventQueue = None
if start:
self.__startScan()
def __init__(self,
scanName,
scanId,
targetValue,
targetType,
moduleList,
globalOpts,
start=True):
"""Initialize SpiderFootScanner object.
Args:
scanName (str): name of the scan
scanId (str): unique ID of the scan
targetValue (str): scan target
targetType (str): scan target type
moduleList (list): list of modules to run
globalOpts (dict): scan options
start (bool): start the scan immediately
Raises:
TypeError: arg type was invalid
ValueError: arg value was invalid
Todo:
Eventually change this to be able to control multiple scan instances
"""
if not isinstance(globalOpts, dict):
raise TypeError(
f"globalOpts is {type(globalOpts)}; expected dict()")
if not globalOpts:
raise ValueError("globalOpts is empty")
self.__config = deepcopy(globalOpts)
self.__dbh = SpiderFootDb(self.__config)
if not isinstance(scanName, str):
raise TypeError(f"scanName is {type(scanName)}; expected str()")
if not scanName:
raise ValueError("scanName value is blank")
self.__scanName = scanName
if not isinstance(scanId, str):
raise TypeError(f"scanId is {type(scanId)}; expected str()")
if not scanId:
raise ValueError("scanId value is blank")
if not isinstance(targetValue, str):
raise TypeError(
f"targetValue is {type(targetValue)}; expected str()")
if not targetValue:
raise ValueError("targetValue value is blank")
self.__targetValue = targetValue
if not isinstance(targetType, str):
raise TypeError(
f"targetType is {type(targetType)}; expected str()")
if not targetType:
raise ValueError("targetType value is blank")
self.__targetType = targetType
if not isinstance(moduleList, list):
raise TypeError(
f"moduleList is {type(moduleList)}; expected list()")
if not moduleList:
raise ValueError("moduleList is empty")
self.__moduleList = moduleList
self.__sf = SpiderFoot(self.__config)
self.__sf.dbh = self.__dbh
# Create a unique ID for this scan in the back-end DB.
if not isinstance(scanId, str):
raise TypeError(f"scanId is {type(scanId)}; expected str()")
if scanId:
self.__scanId = scanId
else:
self.__scanId = self.__sf.genScanInstanceId()
self.__sf.scanId = self.__scanId
self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName,
self.__targetValue)
# Create our target
try:
self.__target = SpiderFootTarget(self.__targetValue,
self.__targetType)
except (TypeError, ValueError) as e:
self.__sf.status(f"Scan [{self.__scanId}] failed: {e}")
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)
raise ValueError(f"Invalid target: {e}")
# Save the config current set for this scan
self.__config['_modulesenabled'] = self.__moduleList
self.__dbh.scanConfigSet(
self.__scanId, self.__sf.configSerialize(deepcopy(self.__config)))
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
if self.__config['_socks1type']:
socksAddr = self.__config['_socks2addr']
socksPort = int(self.__config['_socks3port'])
socksUsername = self.__config['_socks4user'] or ''
socksPassword = self.__config['_socks5pwd'] or ''
proxy = f"{socksAddr}:{socksPort}"
if socksUsername or socksPassword:
proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy)
if self.__config['_socks1type'] == '4':
proxy = 'socks4://' + proxy
elif self.__config['_socks1type'] == '5':
proxy = 'socks5://' + proxy
elif self.__config['_socks1type'] == 'HTTP':
proxy = 'http://' + proxy
elif self.__config['_socks1type'] == 'TOR':
proxy = 'socks5h://' + proxy
else:
raise ValueError(
f"Invalid SOCKS proxy type: {self.__config['_socks1ttype']}"
)
self.__sf.debug(
f"SOCKS: {socksAddr}:{socksPort} ({socksUsername}:{socksPassword})"
)
self.__sf.socksProxy = proxy
else:
self.__sf.socksProxy = None
# Override the default DNS server
if self.__config['_dnsserver']:
res = dns.resolver.Resolver()
res.nameservers = [self.__config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
self.__config['_useragent'] = self.__sf.optValueToData(
self.__config['_useragent'])
# Get internet TLDs
tlddata = self.__sf.cacheGet("internet_tlds",
self.__config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata is None:
self.__config['_internettlds'] = self.__sf.optValueToData(
self.__config['_internettlds'])
self.__sf.cachePut("internet_tlds", self.__config['_internettlds'])
else:
self.__config["_internettlds"] = tlddata.splitlines()
self.__setStatus("INITIALIZING", time.time() * 1000, None)
if start:
self.__startScan()
def test_init_unsupported_target_type_should_raise(self):
"""
Test __init__(self, targetValue, typeName)
"""
with self.assertRaises(ValueError):
SpiderFootTarget('example target value', 'example target type')
def test_init_invalid_target_value_should_raise(self):
"""
Test __init__(self, targetValue, typeName)
"""
with self.assertRaises(TypeError):
SpiderFootTarget(None, 'IP_ADDRESS')
def test_setAlias_invalid_alias_should_not_set_alias(self):
"""
Test setAlias(self, value, typeName)
"""
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
target.setAlias(None, None)
target.setAlias("", None)
target.setAlias(None, "")
target.setAlias("", "")
target.setAlias("example value", None)
target.setAlias(None, "example type")
target_aliases = target.targetAliases
self.assertIsInstance(target_aliases, list)
self.assertEqual([], target_aliases)