def contentNotify(self, url, httpresult, parentEvent=None):
event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']),
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
store_content = True
headers = httpresult.get('headers')
if headers:
event = SpiderFootEvent("WEBSERVER_HTTPHEADERS",
json.dumps(headers, ensure_ascii=False),
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
ctype = headers.get('content-type')
if ctype:
for mt in self.opts['filtermime']:
if ctype.startswith(mt):
store_content = False
event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype,
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
if store_content:
content = httpresult.get('content')
if content:
event = SpiderFootEvent("TARGET_WEB_CONTENT", str(content),
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
def test_handleEvent_event_data_web_content_not_containing_webframework_string_should_not_create_event(self):
sf = SpiderFoot(self.default_options)
module = sfp_webframework()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
event_type = 'TARGET_WEB_CONTENT'
event_data = 'example data'
event_module = 'example module'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
evt.actualSource = "https://spiderfoot.net/"
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent_should_only_handle_events_within_target_scope(self):
sf = SpiderFoot(self.default_options)
module = sfp_errors()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
event_type = 'TARGET_WEB_CONTENT'
event_data = 'example data Internal Server Error example data'
event_module = 'sfp_spider'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
evt.actualSource = 'https://something.else.entirely/'
result = module.handleEvent(evt)
self.assertIsNone(result)
def contentNotify(self, url, httpresult, parentEvent=None):
sendcontent = True
if httpresult.get('headers'):
ctype = httpresult['headers'].get('content-type')
if not ctype:
sendcontent = True
else:
for mt in self.opts['filtermime']:
if ctype.startswith(mt):
sendcontent = False
if sendcontent:
if httpresult['content'] is not None:
event = SpiderFootEvent("TARGET_WEB_CONTENT",
httpresult['content'], self.__name__,
parentEvent)
event.actualSource = url
self.notifyListeners(event)
hdr = httpresult['headers']
if hdr is not None:
event = SpiderFootEvent("WEBSERVER_HTTPHEADERS",
json.dumps(hdr, ensure_ascii=False),
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']),
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
if not httpresult.get('headers'):
return
ctype = httpresult['headers'].get('content-type')
if ctype:
event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype,
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
def test_actualSource_attribute_should_return_actual_source_as_string(
self):
event_type = 'ROOT'
event_data = 'example event data'
module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, module, source_event)
actual_source = 'example actual source'
evt.actualSource = actual_source
self.assertEqual(actual_source, evt.actualSource)
def test_handleEvent_event_data_containing_error_string_should_return_event(
self):
sf = SpiderFoot(self.default_options)
module = sfp_errors()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
expected = 'ERROR_MESSAGE'
if str(event.eventType) != expected:
raise Exception(f"{event.eventType} != {expected}")
expected = 'Generic Error'
if str(event.data) != expected:
raise Exception(f"{event.data} != {expected}")
raise Exception("OK")
module.notifyListeners = new_notifyListeners.__get__(
module, sfp_errors)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
event_type = 'TARGET_WEB_CONTENT'
event_data = 'example data Internal Server Error example data'
event_module = 'sfp_spider'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
evt.actualSource = 'https://spiderfoot.net/'
with self.assertRaises(Exception) as cm:
module.handleEvent(evt)
self.assertEqual("OK", str(cm.exception))
def test_handleEvent_domain_whois_event_data_containing_webframework_string_should_create_url_web_framework_event(
self):
sf = SpiderFoot(self.default_options)
module = sfp_webframework()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
expected = 'URL_WEB_FRAMEWORK'
if str(event.eventType) != expected:
raise Exception(f"{event.eventType} != {expected}")
expected = "Wordpress"
if str(event.data) != expected:
raise Exception(f"{event.data} != {expected}")
raise Exception("OK")
module.notifyListeners = new_notifyListeners.__get__(
module, sfp_webframework)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
event_type = 'TARGET_WEB_CONTENT'
event_data = 'example data /wp-includes/ example data'
event_module = 'sfp_spider'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
evt.actualSource = "https://spiderfoot.net/"
with self.assertRaises(Exception) as cm:
module.handleEvent(evt)
self.assertEqual("OK", str(cm.exception))
def test_handleEvent_event_data_containing_unusual_header_should_return_event(
self):
sf = SpiderFoot(self.default_options)
module = sfp_strangeheaders()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
expected = 'WEBSERVER_STRANGEHEADER'
if str(event.eventType) != expected:
raise Exception(f"{event.eventType} != {expected}")
expected = 'unusual header: example header value'
if str(event.data) != expected:
raise Exception(f"{event.data} != {expected}")
raise Exception("OK")
module.notifyListeners = new_notifyListeners.__get__(
module, sfp_strangeheaders)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
event_type = 'WEBSERVER_HTTPHEADERS'
event_data = '{"unusual header": "example header value"}'
event_module = 'sfp_spider'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
evt.actualSource = "https://spiderfoot.net/example"
with self.assertRaises(Exception) as cm:
module.handleEvent(evt)
self.assertEqual("OK", str(cm.exception))
def test_handleEvent_event_data_not_containing_unusual_header_should_not_return_event(
self):
sf = SpiderFoot(self.default_options)
module = sfp_strangeheaders()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(
module, sfp_strangeheaders)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
event_type = 'WEBSERVER_HTTPHEADERS'
event_data = '{"server": "example server"}'
event_module = 'sfp_spider'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
evt.actualSource = "https://spiderfoot.net/example"
result = module.handleEvent(evt)
self.assertIsNone(result)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.currentEventSrc = event
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if srcModuleName == "sfp_hackertarget" and eventName == "IP_ADDRESS":
self.debug(f"Ignoring {eventName}, from self.")
return
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return
max_netblock = self.opts['maxnetblock']
net_size = IPNetwork(eventData).prefixlen
if net_size < max_netblock:
self.debug(
f"Network size bigger than permitted: {net_size} > {max_netblock}"
)
return
if eventName == 'DOMAIN_NAME_PARENT':
records = self.zoneTransfer(eventData)
if not records:
return
evt = SpiderFootEvent('RAW_DNS_RECORDS', "\n".join(records),
self.__name__, event)
self.notifyListeners(evt)
# Try and pull out individual records
for row in records:
pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*",
re.IGNORECASE | re.DOTALL)
grps = re.findall(pat, row)
if len(grps) == 0:
continue
hosts = list()
for strdata in grps:
self.debug("Matched: " + strdata)
if strdata.endswith("."):
hosts.append(strdata[:-1])
else:
hosts.append(strdata)
for host in set(hosts):
if self.getTarget().matches(host,
includeChildren=True,
includeParents=True):
evt_type = 'INTERNET_NAME'
else:
evt_type = 'AFFILIATE_INTERNET_NAME'
if self.opts['verify'] and not self.sf.resolveHost(
host) and not self.sf.resolveHost6(host):
self.debug(f"Host {host} could not be resolved")
evt_type += '_UNRESOLVED'
evt = SpiderFootEvent(evt_type, host, self.__name__, event)
self.notifyListeners(evt)
if self.sf.isDomain(host, self.opts['_internettlds']):
if evt_type.startswith('AFFILIATE'):
evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME',
host, self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent('DOMAIN_NAME', host,
self.__name__, event)
self.notifyListeners(evt)
return
qrylist = list()
if eventName.startswith("NETBLOCK_"):
for ipaddr in IPNetwork(eventData):
if str(ipaddr) not in self.results:
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
else:
qrylist.append(eventData)
self.results[eventData] = True
myres = list()
for ip in qrylist:
if self.checkForStop():
return
hosts = self.reverseIpLookup(ip)
if not hosts:
continue
for h in hosts:
if " " in h:
continue
self.info(f"Found something on same IP: {h}")
if not self.opts['cohostsamedomain']:
if self.getTarget().matches(h, includeParents=True):
self.debug(
f"Skipping {h} because it is on the same domain.")
continue
if h not in myres and h != ip:
if self.opts['verify'] and not self.sf.validateIP(h, ip):
self.debug(f"Host {h} no longer resolves to {ip}")
continue
if self.cohostcount < self.opts['maxcohost']:
# Create an IP Address event stemming from the netblock as the
# link to the co-host.
if eventName == "NETBLOCK_OWNER":
ipe = SpiderFootEvent("IP_ADDRESS", ip,
self.__name__, event)
self.notifyListeners(ipe)
evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(),
self.__name__, ipe)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(),
self.__name__, event)
self.notifyListeners(evt)
myres.append(h.lower())
self.cohostcount += 1
# For netblocks, we need to create the IP address event so that
# the threat intel event is more meaningful.
if eventName == 'NETBLOCK_OWNER':
pevent = SpiderFootEvent("IP_ADDRESS", ip, self.__name__,
event)
self.notifyListeners(pevent)
else:
pevent = event
if self.opts.get('http_headers', True):
http_headers = self.httpHeaders(ip)
if http_headers is not None:
e = SpiderFootEvent('WEBSERVER_HTTPHEADERS',
json.dumps(http_headers),
self.__name__, pevent)
e.actualSource = ip
self.notifyListeners(e)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['censys_api_key_uid'] == "" or self.opts[
'censys_api_key_secret'] == "":
self.sf.error(
"You enabled sfp_censys but did not set an API uid/secret!",
False)
self.errorState = True
return None
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
self.results[eventData] = True
qrylist = list()
if eventName.startswith("NETBLOCK_"):
for ipaddr in IPNetwork(eventData):
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
else:
qrylist.append(eventData)
for addr in qrylist:
if self.checkForStop():
return None
if eventName in ["IP_ADDRESS", "NETBLOCK_OWNER"]:
qtype = "ip"
else:
qtype = "host"
rec = self.query(addr, qtype)
if rec is None:
continue
if 'error' in rec:
if rec['error_type'] == "unknown":
self.sf.debug("Censys returned no data for " + addr)
else:
self.sf.error(
"Censys returned an unexpected error: " +
rec['error_type'], False)
continue
self.sf.debug("Found results in Censys.io")
# For netblocks, we need to create the IP address event so that
# the threat intel event is more meaningful.
if eventName.startswith('NETBLOCK_'):
pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__,
event)
self.notifyListeners(pevent)
else:
pevent = event
e = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__,
pevent)
self.notifyListeners(e)
try:
# Date format: 2016-12-24T07:25:35+00:00'
created_dt = datetime.strptime(
rec.get('updated_at', "1970-01-01T00:00:00+00:00"),
'%Y-%m-%dT%H:%M:%S+00:00')
created_ts = int(time.mktime(created_dt.timetuple()))
age_limit_ts = int(
time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts[
'age_limit_days'] > 0 and created_ts < age_limit_ts:
self.sf.debug("Record found but too old, skipping.")
continue
if 'location' in rec:
location = ', '.join([
_f for _f in [
rec['location'].get('city'), rec['location'].
get('province'), rec['location'].get(
'postal_code'), rec['location'].get('country')
] if _f
])
if location:
e = SpiderFootEvent("GEOINFO", location, self.__name__,
pevent)
self.notifyListeners(e)
if 'headers' in rec:
dat = json.dumps(rec['headers'], ensure_ascii=False)
e = SpiderFootEvent("WEBSERVER_HTTPHEADERS", dat,
self.__name__, pevent)
e.actualSource = addr
self.notifyListeners(e)
if 'autonomous_system' in rec:
dat = str(rec['autonomous_system']['asn'])
e = SpiderFootEvent("BGP_AS_MEMBER", dat, self.__name__,
pevent)
self.notifyListeners(e)
dat = rec['autonomous_system']['routed_prefix']
e = SpiderFootEvent("NETBLOCK_MEMBER", dat, self.__name__,
pevent)
self.notifyListeners(e)
if 'protocols' in rec:
for p in rec['protocols']:
if 'ip' not in rec:
continue
dat = rec['ip'] + ":" + p.split("/")[0]
e = SpiderFootEvent("TCP_PORT_OPEN", dat,
self.__name__, pevent)
self.notifyListeners(e)
if 'metadata' in rec:
if 'os_description' in rec['metadata']:
dat = rec['metadata']['os_description']
e = SpiderFootEvent("OPERATING_SYSTEM", dat,
self.__name__, pevent)
self.notifyListeners(e)
except BaseException as e:
self.sf.error(
"Error encountered processing record for " + eventData +
" (" + str(e) + ")", False)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['censys_api_key_uid'] == "" or self.opts[
'censys_api_key_secret'] == "":
self.error(
f"You enabled {self.__class__.__name__} but did not set an API uid/secret!"
)
self.errorState = True
return
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return
max_netblock = self.opts['maxnetblock']
if IPNetwork(eventData).prefixlen < max_netblock:
self.debug(
f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}"
)
return
qrylist = list()
if eventName.startswith("NETBLOCK_"):
for ipaddr in IPNetwork(eventData):
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
else:
qrylist.append(eventData)
for addr in qrylist:
if self.checkForStop():
return
data = self.queryHosts(addr)
if not data:
continue
rec = data.get("result")
if not rec:
self.info(f"Censys.io returned no results for {addr}")
continue
self.debug(f"Found results for {addr} in Censys.io")
# For netblocks, we need to create the IP address event so that
# the threat intel event is more meaningful.
if eventName == 'NETBLOCK_OWNER':
pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__,
event)
self.notifyListeners(pevent)
else:
pevent = event
e = SpiderFootEvent("RAW_RIR_DATA", json.dumps(rec), self.__name__,
pevent)
self.notifyListeners(e)
try:
# Date format: 2021-09-22T16:46:47.623Z
created_dt = datetime.strptime(
rec.get('last_updated_at', "1970-01-01T00:00:00.000Z"),
'%Y-%m-%dT%H:%M:%S.%fZ')
created_ts = int(time.mktime(created_dt.timetuple()))
age_limit_ts = int(
time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts[
'age_limit_days'] > 0 and created_ts < age_limit_ts:
self.debug(
f"Record found but too old ({created_dt}), skipping.")
continue
except Exception as e:
self.error(
f"Error encountered processing last_updated_at record for {eventData} ({e})"
)
try:
location = rec.get('location')
if location:
geoinfo = ', '.join([
_f for _f in [
location.get('city'),
location.get('province'),
location.get('postal_code'),
location.get('country'),
] if _f
])
if geoinfo:
e = SpiderFootEvent("GEOINFO", geoinfo, self.__name__,
pevent)
self.notifyListeners(e)
except Exception as e:
self.error(
f"Error encountered processing location record for {eventData} ({e})"
)
try:
services = rec.get('services')
if services:
for service in services:
port = service.get('port')
if port:
evt = SpiderFootEvent("TCP_PORT_OPEN",
f"{addr}:{port}",
self.__name__, pevent)
self.notifyListeners(evt)
try:
headers = service['http']['response']['headers']
except Exception:
headers = None
if headers:
dat = json.dumps(headers, ensure_ascii=False)
e = SpiderFootEvent("WEBSERVER_HTTPHEADERS", dat,
self.__name__, pevent)
e.actualSource = addr
self.notifyListeners(e)
except Exception as e:
self.error(
f"Error encountered processing services record for {eventData} ({e})"
)
try:
autonomous_system = rec.get('autonomous_system')
if autonomous_system:
asn = autonomous_system.get('asn')
if asn:
e = SpiderFootEvent("BGP_AS_MEMBER", str(asn),
self.__name__, pevent)
self.notifyListeners(e)
bgp_prefix = autonomous_system.get('bgp_prefix')
if bgp_prefix:
e = SpiderFootEvent("NETBLOCK_MEMBER", str(bgp_prefix),
self.__name__, pevent)
self.notifyListeners(e)
except Exception as e:
self.error(
f"Error encountered processing autonomous_system record for {eventData} ({e})"
)
try:
operating_system = rec.get('operating_system')
if operating_system:
vendor = operating_system.get('vendor')
product = operating_system.get('product')
os = ' '.join(filter(None, [vendor, product]))
if os:
e = SpiderFootEvent("OPERATING_SYSTEM", os,
self.__name__, pevent)
self.notifyListeners(e)
except Exception as e:
self.error(
f"Error encountered processing operating_system record for {eventData} ({e})"
)
