def contentNotify(self, url, httpresult, parentEvent=None): event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']), self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) store_content = True headers = httpresult.get('headers') if headers: event = SpiderFootEvent("WEBSERVER_HTTPHEADERS", json.dumps(headers, ensure_ascii=False), self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) ctype = headers.get('content-type') if ctype: for mt in self.opts['filtermime']: if ctype.startswith(mt): store_content = False event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype, self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) if store_content: content = httpresult.get('content') if content: event = SpiderFootEvent("TARGET_WEB_CONTENT", str(content), self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event)
def test_handleEvent_event_data_web_content_not_containing_webframework_string_should_not_create_event(self): sf = SpiderFoot(self.default_options) module = sfp_webframework() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/" result = module.handleEvent(evt) self.assertIsNone(result)
def test_handleEvent_should_only_handle_events_within_target_scope(self): sf = SpiderFoot(self.default_options) module = sfp_errors() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data Internal Server Error example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = 'https://something.else.entirely/' result = module.handleEvent(evt) self.assertIsNone(result)
def contentNotify(self, url, httpresult, parentEvent=None): sendcontent = True if httpresult.get('headers'): ctype = httpresult['headers'].get('content-type') if not ctype: sendcontent = True else: for mt in self.opts['filtermime']: if ctype.startswith(mt): sendcontent = False if sendcontent: if httpresult['content'] is not None: event = SpiderFootEvent("TARGET_WEB_CONTENT", httpresult['content'], self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) hdr = httpresult['headers'] if hdr is not None: event = SpiderFootEvent("WEBSERVER_HTTPHEADERS", json.dumps(hdr, ensure_ascii=False), self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']), self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) if not httpresult.get('headers'): return ctype = httpresult['headers'].get('content-type') if ctype: event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype, self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event)
def test_actualSource_attribute_should_return_actual_source_as_string( self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) actual_source = 'example actual source' evt.actualSource = actual_source self.assertEqual(actual_source, evt.actualSource)
def test_handleEvent_event_data_containing_error_string_should_return_event( self): sf = SpiderFoot(self.default_options) module = sfp_errors() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'ERROR_MESSAGE' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'Generic Error' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__( module, sfp_errors) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data Internal Server Error example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = 'https://spiderfoot.net/' with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception))
def test_handleEvent_domain_whois_event_data_containing_webframework_string_should_create_url_web_framework_event( self): sf = SpiderFoot(self.default_options) module = sfp_webframework() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'URL_WEB_FRAMEWORK' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "Wordpress" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__( module, sfp_webframework) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data /wp-includes/ example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/" with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception))
def test_handleEvent_event_data_containing_unusual_header_should_return_event( self): sf = SpiderFoot(self.default_options) module = sfp_strangeheaders() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'WEBSERVER_STRANGEHEADER' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'unusual header: example header value' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__( module, sfp_strangeheaders) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'WEBSERVER_HTTPHEADERS' event_data = '{"unusual header": "example header value"}' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/example" with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception))
def test_handleEvent_event_data_not_containing_unusual_header_should_not_return_event( self): sf = SpiderFoot(self.default_options) module = sfp_strangeheaders() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__( module, sfp_strangeheaders) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'WEBSERVER_HTTPHEADERS' event_data = '{"server": "example server"}' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/example" result = module.handleEvent(evt) self.assertIsNone(result)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == "sfp_hackertarget" and eventName == "IP_ADDRESS": self.debug(f"Ignoring {eventName}, from self.") return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return max_netblock = self.opts['maxnetblock'] net_size = IPNetwork(eventData).prefixlen if net_size < max_netblock: self.debug( f"Network size bigger than permitted: {net_size} > {max_netblock}" ) return if eventName == 'DOMAIN_NAME_PARENT': records = self.zoneTransfer(eventData) if not records: return evt = SpiderFootEvent('RAW_DNS_RECORDS', "\n".join(records), self.__name__, event) self.notifyListeners(evt) # Try and pull out individual records for row in records: pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL) grps = re.findall(pat, row) if len(grps) == 0: continue hosts = list() for strdata in grps: self.debug("Matched: " + strdata) if strdata.endswith("."): hosts.append(strdata[:-1]) else: hosts.append(strdata) for host in set(hosts): if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost( host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts['_internettlds']): if evt_type.startswith('AFFILIATE'): evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) return qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): if str(ipaddr) not in self.results: qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) self.results[eventData] = True myres = list() for ip in qrylist: if self.checkForStop(): return hosts = self.reverseIpLookup(ip) if not hosts: continue for h in hosts: if " " in h: continue self.info(f"Found something on same IP: {h}") if not self.opts['cohostsamedomain']: if self.getTarget().matches(h, includeParents=True): self.debug( f"Skipping {h} because it is on the same domain.") continue if h not in myres and h != ip: if self.opts['verify'] and not self.sf.validateIP(h, ip): self.debug(f"Host {h} no longer resolves to {ip}") continue if self.cohostcount < self.opts['maxcohost']: # Create an IP Address event stemming from the netblock as the # link to the co-host. if eventName == "NETBLOCK_OWNER": ipe = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(ipe) evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(), self.__name__, ipe) self.notifyListeners(evt) else: evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(), self.__name__, event) self.notifyListeners(evt) myres.append(h.lower()) self.cohostcount += 1 # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(pevent) else: pevent = event if self.opts.get('http_headers', True): http_headers = self.httpHeaders(ip) if http_headers is not None: e = SpiderFootEvent('WEBSERVER_HTTPHEADERS', json.dumps(http_headers), self.__name__, pevent) e.actualSource = ip self.notifyListeners(e)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['censys_api_key_uid'] == "" or self.opts[ 'censys_api_key_secret'] == "": self.sf.error( "You enabled sfp_censys but did not set an API uid/secret!", False) self.errorState = True return None # Don't look up stuff twice if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None self.results[eventData] = True qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) for addr in qrylist: if self.checkForStop(): return None if eventName in ["IP_ADDRESS", "NETBLOCK_OWNER"]: qtype = "ip" else: qtype = "host" rec = self.query(addr, qtype) if rec is None: continue if 'error' in rec: if rec['error_type'] == "unknown": self.sf.debug("Censys returned no data for " + addr) else: self.sf.error( "Censys returned an unexpected error: " + rec['error_type'], False) continue self.sf.debug("Found results in Censys.io") # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName.startswith('NETBLOCK_'): pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event e = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__, pevent) self.notifyListeners(e) try: # Date format: 2016-12-24T07:25:35+00:00' created_dt = datetime.strptime( rec.get('updated_at', "1970-01-01T00:00:00+00:00"), '%Y-%m-%dT%H:%M:%S+00:00') created_ts = int(time.mktime(created_dt.timetuple())) age_limit_ts = int( time.time()) - (86400 * self.opts['age_limit_days']) if self.opts[ 'age_limit_days'] > 0 and created_ts < age_limit_ts: self.sf.debug("Record found but too old, skipping.") continue if 'location' in rec: location = ', '.join([ _f for _f in [ rec['location'].get('city'), rec['location']. get('province'), rec['location'].get( 'postal_code'), rec['location'].get('country') ] if _f ]) if location: e = SpiderFootEvent("GEOINFO", location, self.__name__, pevent) self.notifyListeners(e) if 'headers' in rec: dat = json.dumps(rec['headers'], ensure_ascii=False) e = SpiderFootEvent("WEBSERVER_HTTPHEADERS", dat, self.__name__, pevent) e.actualSource = addr self.notifyListeners(e) if 'autonomous_system' in rec: dat = str(rec['autonomous_system']['asn']) e = SpiderFootEvent("BGP_AS_MEMBER", dat, self.__name__, pevent) self.notifyListeners(e) dat = rec['autonomous_system']['routed_prefix'] e = SpiderFootEvent("NETBLOCK_MEMBER", dat, self.__name__, pevent) self.notifyListeners(e) if 'protocols' in rec: for p in rec['protocols']: if 'ip' not in rec: continue dat = rec['ip'] + ":" + p.split("/")[0] e = SpiderFootEvent("TCP_PORT_OPEN", dat, self.__name__, pevent) self.notifyListeners(e) if 'metadata' in rec: if 'os_description' in rec['metadata']: dat = rec['metadata']['os_description'] e = SpiderFootEvent("OPERATING_SYSTEM", dat, self.__name__, pevent) self.notifyListeners(e) except BaseException as e: self.sf.error( "Error encountered processing record for " + eventData + " (" + str(e) + ")", False)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['censys_api_key_uid'] == "" or self.opts[ 'censys_api_key_secret'] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API uid/secret!" ) self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug( f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}" ) return qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) for addr in qrylist: if self.checkForStop(): return data = self.queryHosts(addr) if not data: continue rec = data.get("result") if not rec: self.info(f"Censys.io returned no results for {addr}") continue self.debug(f"Found results for {addr} in Censys.io") # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event e = SpiderFootEvent("RAW_RIR_DATA", json.dumps(rec), self.__name__, pevent) self.notifyListeners(e) try: # Date format: 2021-09-22T16:46:47.623Z created_dt = datetime.strptime( rec.get('last_updated_at', "1970-01-01T00:00:00.000Z"), '%Y-%m-%dT%H:%M:%S.%fZ') created_ts = int(time.mktime(created_dt.timetuple())) age_limit_ts = int( time.time()) - (86400 * self.opts['age_limit_days']) if self.opts[ 'age_limit_days'] > 0 and created_ts < age_limit_ts: self.debug( f"Record found but too old ({created_dt}), skipping.") continue except Exception as e: self.error( f"Error encountered processing last_updated_at record for {eventData} ({e})" ) try: location = rec.get('location') if location: geoinfo = ', '.join([ _f for _f in [ location.get('city'), location.get('province'), location.get('postal_code'), location.get('country'), ] if _f ]) if geoinfo: e = SpiderFootEvent("GEOINFO", geoinfo, self.__name__, pevent) self.notifyListeners(e) except Exception as e: self.error( f"Error encountered processing location record for {eventData} ({e})" ) try: services = rec.get('services') if services: for service in services: port = service.get('port') if port: evt = SpiderFootEvent("TCP_PORT_OPEN", f"{addr}:{port}", self.__name__, pevent) self.notifyListeners(evt) try: headers = service['http']['response']['headers'] except Exception: headers = None if headers: dat = json.dumps(headers, ensure_ascii=False) e = SpiderFootEvent("WEBSERVER_HTTPHEADERS", dat, self.__name__, pevent) e.actualSource = addr self.notifyListeners(e) except Exception as e: self.error( f"Error encountered processing services record for {eventData} ({e})" ) try: autonomous_system = rec.get('autonomous_system') if autonomous_system: asn = autonomous_system.get('asn') if asn: e = SpiderFootEvent("BGP_AS_MEMBER", str(asn), self.__name__, pevent) self.notifyListeners(e) bgp_prefix = autonomous_system.get('bgp_prefix') if bgp_prefix: e = SpiderFootEvent("NETBLOCK_MEMBER", str(bgp_prefix), self.__name__, pevent) self.notifyListeners(e) except Exception as e: self.error( f"Error encountered processing autonomous_system record for {eventData} ({e})" ) try: operating_system = rec.get('operating_system') if operating_system: vendor = operating_system.get('vendor') product = operating_system.get('product') os = ' '.join(filter(None, [vendor, product])) if os: e = SpiderFootEvent("OPERATING_SYSTEM", os, self.__name__, pevent) self.notifyListeners(e) except Exception as e: self.error( f"Error encountered processing operating_system record for {eventData} ({e})" )