def contentNotify(self, url, httpresult, parentEvent=None):
sendcontent = True
if httpresult.get('headers'):
ctype = httpresult['headers'].get('content-type')
if not ctype:
sendcontent = True
else:
for mt in self.opts['filtermime']:
if ctype.startswith(mt):
sendcontent = False
if sendcontent:
if httpresult['content'] != None:
event = SpiderFootEvent("TARGET_WEB_CONTENT",
httpresult['content'], self.__name__,
parentEvent)
event.actualSource = url
self.notifyListeners(event)
hdr = httpresult['headers']
if hdr != None:
event = SpiderFootEvent("WEBSERVER_HTTPHEADERS",
json.dumps(hdr, ensure_ascii=False),
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']),
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
if not httpresult.get('headers'):
return None
ctype = httpresult['headers'].get('content-type')
if ctype:
event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype,
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['censys_api_key_uid'] == "" or self.opts[
'censys_api_key_secret'] == "":
self.sf.error(
"You enabled sfp_censys but did not set an API uid/secret!",
False)
self.errorState = True
return None
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
self.results[eventData] = True
qrylist = list()
if eventName.startswith("NETBLOCK_"):
for ipaddr in IPNetwork(eventData):
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
else:
qrylist.append(eventData)
for addr in qrylist:
if self.checkForStop():
return None
if eventName in ["IP_ADDRESS", "NETBLOCK_OWNER"]:
qtype = "ip"
else:
qtype = "host"
rec = self.query(addr, qtype)
if rec is None:
continue
if 'error' in rec:
if rec['error_type'] == "unknown":
self.sf.debug("Censys returned no data for " + addr)
else:
self.sf.error(
"Censys returned an unexpected error: " +
rec['error_type'], False)
continue
self.sf.debug("Found results in Censys.io")
# For netblocks, we need to create the IP address event so that
# the threat intel event is more meaningful.
if eventName.startswith('NETBLOCK_'):
pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__,
event)
self.notifyListeners(pevent)
else:
pevent = event
e = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__,
pevent)
self.notifyListeners(e)
try:
# Date format: 2016-12-24T07:25:35+00:00'
created_dt = datetime.strptime(
rec.get('updated_at', "1970-01-01T00:00:00+00:00"),
'%Y-%m-%dT%H:%M:%S+00:00')
created_ts = int(time.mktime(created_dt.timetuple()))
age_limit_ts = int(
time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts[
'age_limit_days'] > 0 and created_ts < age_limit_ts:
self.sf.debug("Record found but too old, skipping.")
continue
if 'location' in rec:
location = ', '.join([
_f for _f in [
rec['location'].get('city'), rec['location'].
get('province'), rec['location'].get(
'postal_code'), rec['location'].get('country')
] if _f
])
if location:
e = SpiderFootEvent("GEOINFO", location, self.__name__,
pevent)
self.notifyListeners(e)
if 'headers' in rec:
dat = json.dumps(rec['headers'], ensure_ascii=False)
e = SpiderFootEvent("WEBSERVER_HTTPHEADERS", dat,
self.__name__, pevent)
e.actualSource = addr
self.notifyListeners(e)
if 'autonomous_system' in rec:
dat = str(rec['autonomous_system']['asn'])
e = SpiderFootEvent("BGP_AS_MEMBER", dat, self.__name__,
pevent)
self.notifyListeners(e)
dat = rec['autonomous_system']['routed_prefix']
e = SpiderFootEvent("NETBLOCK_MEMBER", dat, self.__name__,
pevent)
self.notifyListeners(e)
if 'protocols' in rec:
for p in rec['protocols']:
if 'ip' not in rec:
continue
dat = rec['ip'] + ":" + p.split("/")[0]
e = SpiderFootEvent("TCP_PORT_OPEN", dat,
self.__name__, pevent)
self.notifyListeners(e)
if 'metadata' in rec:
if 'os_description' in rec['metadata']:
dat = rec['metadata']['os_description']
e = SpiderFootEvent("OPERATING_SYSTEM", dat,
self.__name__, pevent)
self.notifyListeners(e)
except BaseException as e:
self.sf.error(
"Error encountered processing record for " + eventData +
" (" + str(e) + ")", False)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.currentEventSrc = event
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if srcModuleName == "sfp_hackertarget" and eventName == "IP_ADDRESS":
self.sf.debug("Ignoring " + eventName + ", from self.")
return None
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return None
else:
if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
self.sf.debug("Network size bigger than permitted: " +
str(IPNetwork(eventData).prefixlen) + " > " +
str(self.opts['maxnetblock']))
return None
if eventName == 'DOMAIN_NAME_PARENT':
records = self.zoneTransfer(eventData)
if not records:
return None
evt = SpiderFootEvent('RAW_DNS_RECORDS', "\n".join(records),
self.__name__, event)
self.notifyListeners(evt)
# Try and pull out individual records
for row in records:
pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*",
re.IGNORECASE | re.DOTALL)
grps = re.findall(pat, row)
if len(grps) == 0:
continue
hosts = list()
for strdata in grps:
self.sf.debug("Matched: " + strdata)
if strdata.endswith("."):
hosts.append(strdata[:-1])
else:
hosts.append(strdata)
for host in set(hosts):
if self.getTarget().matches(host,
includeChildren=True,
includeParents=True):
evt_type = 'INTERNET_NAME'
else:
evt_type = 'AFFILIATE_INTERNET_NAME'
if self.opts['verify'] and not self.sf.resolveHost(host):
self.sf.debug(f"Host {host} could not be resolved")
evt_type += '_UNRESOLVED'
evt = SpiderFootEvent(evt_type, host, self.__name__, event)
self.notifyListeners(evt)
if self.sf.isDomain(host, self.opts['_internettlds']):
if evt_type.startswith('AFFILIATE'):
evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME',
host, self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent('DOMAIN_NAME', host,
self.__name__, event)
self.notifyListeners(evt)
return None
qrylist = list()
if eventName.startswith("NETBLOCK_"):
for ipaddr in IPNetwork(eventData):
if str(ipaddr) not in self.results:
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
else:
qrylist.append(eventData)
self.results[eventData] = True
myres = list()
for ip in qrylist:
if self.checkForStop():
return None
hosts = self.reverseIpLookup(ip)
if not hosts:
continue
for h in hosts:
if " " in h:
continue
self.sf.info("Found something on same IP: " + h)
if not self.opts['cohostsamedomain']:
if self.getTarget().matches(h, includeParents=True):
self.sf.debug("Skipping " + h +
" because it is on the same domain.")
continue
if h not in myres and h != ip:
if self.opts['verify'] and not self.sf.validateIP(h, ip):
self.sf.debug("Host " + h + " no longer resolves to " +
ip)
continue
if self.cohostcount < self.opts['maxcohost']:
# Create an IP Address event stemming from the netblock as the
# link to the co-host.
if eventName == "NETBLOCK_OWNER":
ipe = SpiderFootEvent("IP_ADDRESS", ip,
self.__name__, event)
self.notifyListeners(ipe)
evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(),
self.__name__, ipe)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(),
self.__name__, event)
self.notifyListeners(evt)
myres.append(h.lower())
self.cohostcount += 1
# For netblocks, we need to create the IP address event so that
# the threat intel event is more meaningful.
if eventName.startswith('NETBLOCK_'):
pevent = SpiderFootEvent("IP_ADDRESS", ip, self.__name__,
event)
self.notifyListeners(pevent)
else:
pevent = event
if self.opts.get('http_headers', True):
http_headers = self.httpHeaders(ip)
if http_headers is not None:
e = SpiderFootEvent('WEBSERVER_HTTPHEADERS',
json.dumps(http_headers),
self.__name__, pevent)
e.actualSource = ip
self.notifyListeners(e)
if self.opts.get('udp_portscan', True):
udp_ports = self.portScanUDP(ip)
if udp_ports:
for port in udp_ports:
e = SpiderFootEvent("UDP_PORT_OPEN", ip + ":" + port,
self.__name__, pevent)
self.notifyListeners(e)
if self.opts.get('tcp_portscan', True):
tcp_ports = self.portScanTCP(ip)
if tcp_ports:
for port in tcp_ports:
e = SpiderFootEvent("TCP_PORT_OPEN", ip + ":" + port,
self.__name__, pevent)
self.notifyListeners(e)
