def contentNotify(self, url, httpresult, parentEvent=None): sendcontent = True if httpresult.get('headers'): ctype = httpresult['headers'].get('content-type') if not ctype: sendcontent = True else: for mt in self.opts['filtermime']: if ctype.startswith(mt): sendcontent = False if sendcontent: if httpresult['content'] != None: event = SpiderFootEvent("TARGET_WEB_CONTENT", httpresult['content'], self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) hdr = httpresult['headers'] if hdr != None: event = SpiderFootEvent("WEBSERVER_HTTPHEADERS", json.dumps(hdr, ensure_ascii=False), self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']), self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event) if not httpresult.get('headers'): return None ctype = httpresult['headers'].get('content-type') if ctype: event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype, self.__name__, parentEvent) event.actualSource = url self.notifyListeners(event)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['censys_api_key_uid'] == "" or self.opts[ 'censys_api_key_secret'] == "": self.sf.error( "You enabled sfp_censys but did not set an API uid/secret!", False) self.errorState = True return None # Don't look up stuff twice if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None self.results[eventData] = True qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) for addr in qrylist: if self.checkForStop(): return None if eventName in ["IP_ADDRESS", "NETBLOCK_OWNER"]: qtype = "ip" else: qtype = "host" rec = self.query(addr, qtype) if rec is None: continue if 'error' in rec: if rec['error_type'] == "unknown": self.sf.debug("Censys returned no data for " + addr) else: self.sf.error( "Censys returned an unexpected error: " + rec['error_type'], False) continue self.sf.debug("Found results in Censys.io") # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName.startswith('NETBLOCK_'): pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event e = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__, pevent) self.notifyListeners(e) try: # Date format: 2016-12-24T07:25:35+00:00' created_dt = datetime.strptime( rec.get('updated_at', "1970-01-01T00:00:00+00:00"), '%Y-%m-%dT%H:%M:%S+00:00') created_ts = int(time.mktime(created_dt.timetuple())) age_limit_ts = int( time.time()) - (86400 * self.opts['age_limit_days']) if self.opts[ 'age_limit_days'] > 0 and created_ts < age_limit_ts: self.sf.debug("Record found but too old, skipping.") continue if 'location' in rec: location = ', '.join([ _f for _f in [ rec['location'].get('city'), rec['location']. get('province'), rec['location'].get( 'postal_code'), rec['location'].get('country') ] if _f ]) if location: e = SpiderFootEvent("GEOINFO", location, self.__name__, pevent) self.notifyListeners(e) if 'headers' in rec: dat = json.dumps(rec['headers'], ensure_ascii=False) e = SpiderFootEvent("WEBSERVER_HTTPHEADERS", dat, self.__name__, pevent) e.actualSource = addr self.notifyListeners(e) if 'autonomous_system' in rec: dat = str(rec['autonomous_system']['asn']) e = SpiderFootEvent("BGP_AS_MEMBER", dat, self.__name__, pevent) self.notifyListeners(e) dat = rec['autonomous_system']['routed_prefix'] e = SpiderFootEvent("NETBLOCK_MEMBER", dat, self.__name__, pevent) self.notifyListeners(e) if 'protocols' in rec: for p in rec['protocols']: if 'ip' not in rec: continue dat = rec['ip'] + ":" + p.split("/")[0] e = SpiderFootEvent("TCP_PORT_OPEN", dat, self.__name__, pevent) self.notifyListeners(e) if 'metadata' in rec: if 'os_description' in rec['metadata']: dat = rec['metadata']['os_description'] e = SpiderFootEvent("OPERATING_SYSTEM", dat, self.__name__, pevent) self.notifyListeners(e) except BaseException as e: self.sf.error( "Error encountered processing record for " + eventData + " (" + str(e) + ")", False)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == "sfp_hackertarget" and eventName == "IP_ADDRESS": self.sf.debug("Ignoring " + eventName + ", from self.") return None # Don't look up stuff twice if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return None else: if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']: self.sf.debug("Network size bigger than permitted: " + str(IPNetwork(eventData).prefixlen) + " > " + str(self.opts['maxnetblock'])) return None if eventName == 'DOMAIN_NAME_PARENT': records = self.zoneTransfer(eventData) if not records: return None evt = SpiderFootEvent('RAW_DNS_RECORDS', "\n".join(records), self.__name__, event) self.notifyListeners(evt) # Try and pull out individual records for row in records: pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL) grps = re.findall(pat, row) if len(grps) == 0: continue hosts = list() for strdata in grps: self.sf.debug("Matched: " + strdata) if strdata.endswith("."): hosts.append(strdata[:-1]) else: hosts.append(strdata) for host in set(hosts): if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost(host): self.sf.debug(f"Host {host} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts['_internettlds']): if evt_type.startswith('AFFILIATE'): evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) return None qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): if str(ipaddr) not in self.results: qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) self.results[eventData] = True myres = list() for ip in qrylist: if self.checkForStop(): return None hosts = self.reverseIpLookup(ip) if not hosts: continue for h in hosts: if " " in h: continue self.sf.info("Found something on same IP: " + h) if not self.opts['cohostsamedomain']: if self.getTarget().matches(h, includeParents=True): self.sf.debug("Skipping " + h + " because it is on the same domain.") continue if h not in myres and h != ip: if self.opts['verify'] and not self.sf.validateIP(h, ip): self.sf.debug("Host " + h + " no longer resolves to " + ip) continue if self.cohostcount < self.opts['maxcohost']: # Create an IP Address event stemming from the netblock as the # link to the co-host. if eventName == "NETBLOCK_OWNER": ipe = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(ipe) evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(), self.__name__, ipe) self.notifyListeners(evt) else: evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(), self.__name__, event) self.notifyListeners(evt) myres.append(h.lower()) self.cohostcount += 1 # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName.startswith('NETBLOCK_'): pevent = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(pevent) else: pevent = event if self.opts.get('http_headers', True): http_headers = self.httpHeaders(ip) if http_headers is not None: e = SpiderFootEvent('WEBSERVER_HTTPHEADERS', json.dumps(http_headers), self.__name__, pevent) e.actualSource = ip self.notifyListeners(e) if self.opts.get('udp_portscan', True): udp_ports = self.portScanUDP(ip) if udp_ports: for port in udp_ports: e = SpiderFootEvent("UDP_PORT_OPEN", ip + ":" + port, self.__name__, pevent) self.notifyListeners(e) if self.opts.get('tcp_portscan', True): tcp_ports = self.portScanTCP(ip) if tcp_ports: for port in tcp_ports: e = SpiderFootEvent("TCP_PORT_OPEN", ip + ":" + port, self.__name__, pevent) self.notifyListeners(e)