def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.sf.error("You enabled sfp_securitytrails but did not set an API uid/secret!", False) self.errorState = True return None # Don't look up stuff twice if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None else: self.results[eventData] = True if eventName in ["IP_ADDRESS", "IPV6_ADDRESS", "NETLBLOCK_OWNER"]: ip = eventData rec = self.query(ip, "ipv4") myres = list() hosters = list() if rec is not None: for r in rec: if "host_provider" in r: for dat in r['host_provider']: if dat in hosters: continue e = SpiderFootEvent("PROVIDER_HOSTING", dat, self.__name__, event) self.notifyListeners(e) hosters.append(dat) if "hostname" in r: if self.cohostcount > self.opts['maxcohost']: continue h = r['hostname'] if not self.opts['cohostsamedomain']: if self.getTarget().matches(h, includeParents=True): self.sf.debug("Skipping " + h + " because it is on the same domain.") continue if h not in myres and h != ip: if self.opts['verify'] and not self.sf.validateIP(h, ip): self.sf.debug("Host " + h + " no longer resolves to our IP.") continue myres.append(h.lower()) e = SpiderFootEvent("CO_HOSTED_SITE", h, self.__name__, event) self.notifyListeners(e) self.cohostcount += 1 if eventName in ["EMAILADDR"]: email = eventData rec = self.query(email, "whois_email") myres = list() if rec is not None: for r in rec: if "hostname" in r: if not r['hostname']: continue h = r['hostname'] if h.lower() not in myres: myres.append(h.lower()) else: continue e = SpiderFootEvent("AFFILIATE_INTERNET_NAME", h, self.__name__, event) self.notifyListeners(e) if self.sf.isDomain(h, self.opts['_internettlds']): evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", h, self.__name__, event) self.notifyListeners(evt) if eventName in ["DOMAIN_NAME"]: domain = eventData rec = self.query(domain, "domain") myres = list() if rec is not None: for h in rec: if h == "": continue if h.lower() not in myres: myres.append(h.lower()) else: continue e = SpiderFootEvent("INTERNET_NAME", h + "." + domain, self.__name__, event) self.notifyListeners(e)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return self.results[eventData] = True self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.sf.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return if eventName not in ["DOMAIN_NAME"]: return data = self.queryDomain(eventData) if data is None: self.sf.debug(f"No information found for domain {eventData}") return evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) domains = [] for result in data: raw_domains = result.get('rawDomains') if raw_domains: for domain in raw_domains: domains.append(domain) for domain in set(domains): if self.checkForStop(): return if domain in self.results: continue if not self.getTarget().matches( domain, includeChildren=True, includeParents=True): continue if self.opts['verify'] and not self.sf.resolveHost(domain): self.sf.debug(f"Host {domain} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if self.opts['api_key'] == "": self.error("You enabled sfp_hunter but did not set an API key!") self.errorState = True return data = self.query(eventData, 0, 10) if not data: return if "data" not in data: return # Check if we have more results on further pages if "meta" in data: maxgoal = data['meta'].get('results', 10) else: maxgoal = 10 rescount = len(data['data'].get('emails', list())) while rescount <= maxgoal: for email in data['data'].get('emails', list()): # Notify other modules of what you've found em = email.get('value') if not em: continue if em.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, em, self.__name__, event) self.notifyListeners(e) if 'first_name' in email and 'last_name' in email: if email['first_name'] is not None and email[ 'last_name'] is not None: n = email['first_name'] + " " + email['last_name'] e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + n, self.__name__, event) self.notifyListeners(e) if rescount >= maxgoal: return data = self.query(eventData, rescount, 10) if data is None: return if "data" not in data: return rescount += len(data['data'].get('emails', list()))
def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == 'IP_ADDRESS': malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'NETBLOCK_MEMBER': if not self.opts['subnetlookup']: return max_subnet = self.opts['maxsubnet'] if IPNetwork(eventData).prefixlen < max_subnet: self.debug( f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}" ) return malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" elif eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug( f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}" ) return malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" else: self.debug(f"Unexpected event type {eventName}, skipping") return qrylist = list() if eventName.startswith("NETBLOCK"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) for addr in qrylist: if self.checkForStop(): return data = self.queryIPAddress(addr) if not data: continue # TODO: iterate through hosts and extract co-hosts try: maliciousIP = data[0].get('ip') except Exception: # If ArrayIndex is out of bounds then data doesn't exist continue if not maliciousIP: continue if addr != maliciousIP: self.error( f"Reported address {maliciousIP} doesn't match queried IP address {addr}, skipping" ) continue # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) elif eventName == 'NETBLOCK_MEMBER': pevent = SpiderFootEvent("AFFILIATE_IPADDR", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, pevent) self.notifyListeners(evt) text = f"PhishStats [{addr}]" evt = SpiderFootEvent(blacklist_type, text, self.__name__, pevent) self.notifyListeners(evt) evt = SpiderFootEvent(malicious_type, text, self.__name__, pevent) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.sf.error( "You enabled sfp_builtwith but did not set an API key!") self.errorState = True return None if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None self.results[eventData] = True data = self.queryDomainInfo(eventData) if data is not None: if "Meta" in data: if data['Meta'].get("Names", []): for nb in data['Meta']['Names']: e = SpiderFootEvent( "RAW_RIR_DATA", "Possible full name: " + nb['Name'], self.__name__, event) self.notifyListeners(e) if nb.get('Email', None): if self.sf.validEmail(nb['Email']): if nb['Email'].split("@")[0] in self.opts[ '_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, nb['Email'], self.__name__, event) self.notifyListeners(e) if data['Meta'].get("Emails", []): for email in data['Meta']['Emails']: if self.sf.validEmail(email): if email.split("@")[0] in self.opts[ '_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(e) if data['Meta'].get("Telephones", []): for phone in data['Meta']['Telephones']: phone = phone.replace("-", "").replace("(", "").replace( ")", "").replace(" ", "") e = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event) self.notifyListeners(e) if "Paths" in data.get("Result", []): for p in data["Result"]['Paths']: if p.get("SubDomain", ""): h = p["SubDomain"] + "." + eventData ev = SpiderFootEvent("INTERNET_NAME", h, self.__name__, event) self.notifyListeners(ev) if self.sf.isDomain(h, self.opts['_internettlds']): ev = SpiderFootEvent("DOMAIN_NAME", h, self.__name__, event) self.notifyListeners(ev) else: ev = None # If we have a subdomain, let's get its tech info # and associate it with the subdomain event. for t in p.get("Technologies", []): if ev: src = ev else: src = event agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) if t.get("LastDetected", 0) < agelimit: self.sf.debug("Data found too old, skipping.") continue e = SpiderFootEvent("WEBSERVER_TECHNOLOGY", t["Name"], self.__name__, src) self.notifyListeners(e) data = self.queryRelationships(eventData) if data is None: return None agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) for r in data: if "Domain" not in r or "Identifiers" not in r: self.sf.debug("Data returned not in the format requested.") continue if r['Domain'] != eventData: self.sf.debug( "Data returned doesn't match data requested, skipping.") continue for i in r['Identifiers']: if "Last" not in i or "Type" not in i or "Value" not in i: self.sf.debug("Data returned not in the format requested.") continue if i['Last'] < agelimit: self.sf.debug("Data found too old, skipping.") continue evttype = None # Related through shared IP if i['Type'] == "ip": if self.sf.validIP(i['Value']): val = i['Value'] evttype = "IP_ADDRESS" else: val = i['Value'].strip(".") if self.getTarget.matches(val): evttype = "INTERNET_NAME" else: evttype = "CO_HOSTED_SITE" # Create the name/co-host e = SpiderFootEvent(evttype, val, self.__name__, event) self.notifyListeners(e) continue # Related through shared analytics ID txt = i['Type'] + ": " + str(i['Value']) e = SpiderFootEvent("WEB_ANALYTICS_ID", txt, self.__name__, event) self.notifyListeners(e) if i['Matches']: for m in i['Matches']: if "Domain" not in m: continue evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME", m['Domain'], self.__name__, e) self.notifyListeners(evt) if self.sf.isDomain(m['Domain'], self.opts['_internettlds']): evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", m['Domain'], self.__name__, e) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if eventData in self.results: self.debug(f"Skipping {eventData} as already scanned.") return self.results[eventData] = True if not self.opts['wafw00f_path']: self.error( "You enabled sfp_tool_wafw00f but did not set a path to the tool!" ) self.errorState = True return exe = self.opts['wafw00f_path'] if self.opts['wafw00f_path'].endswith('/'): exe = exe + 'wafw00f' if not os.path.isfile(exe): self.error(f"File does not exist: {exe}") self.errorState = True return url = eventData if not SpiderFootHelpers.sanitiseInput(url): self.error("Invalid input, refusing to run.") return args = [self.opts['python_path'], exe, '-a', '-o-', '-f', 'json', url] try: p = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate(input=None) except Exception as e: self.error(f"Unable to run wafw00f: {e}") return if p.returncode != 0: self.error( f"Unable to read wafw00f output\nstderr: {stderr}\nstdout: {stdout}" ) return if not stdout: self.debug(f"wafw00f returned no output for {eventData}") return try: result_json = json.loads(stdout) except Exception as e: self.error( f"Could not parse wafw00f output as JSON: {e}\nstdout: {stdout}" ) return if not result_json: self.debug(f"wafw00f returned no output for {eventData}") return evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(result_json), self.__name__, event) self.notifyListeners(evt) for waf in result_json: if not waf: continue firewall = waf.get('firewall') if not firewall: continue if firewall == 'Generic': continue manufacturer = waf.get('manufacturer') if not manufacturer: continue software = ' '.join(filter(None, [manufacturer, firewall])) if software: evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', software, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data sourceData = self.sf.hashstring(eventData) if sourceData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return self.results[sourceData] = True self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if event.moduleDataSource: datasource = event.moduleDataSource else: datasource = "Unknown" if eventName == 'TARGET_WEB_CONTENT': # Google Analytics matches = re.findall(r"\bua\-\d{4,10}\-\d{1,4}\b", eventData, re.IGNORECASE) for m in matches: if m.lower().startswith('ua-000000-'): continue if m.lower().startswith('ua-123456-'): continue if m.lower().startswith('ua-12345678'): continue self.sf.debug("Google Analytics match: " + m) evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google Analytics: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Google AdSense matches = re.findall(r"\b(pub-\d{10,20})\b", eventData, re.IGNORECASE) for m in matches: if m.lower().startswith('pub-12345678'): continue self.sf.debug("Google AdSense match: " + m) evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google AdSense: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Google Website Verification # https://developers.google.com/site-verification/v1/getting_started matches = re.findall( r'<meta name="google-site-verification" content="([a-z0-9\-\+_=]{43,44})"', eventData, re.IGNORECASE) for m in matches: self.sf.debug("Google Site Verification match: " + m) evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) matches = re.findall( r'<meta name="verify-v1" content="([a-z0-9\-\+_=]{43,44})"', eventData, re.IGNORECASE) for m in matches: self.sf.debug("Google Site Verification match: " + m) evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Quantcast if '_qevents.push' in eventData: matches = re.findall(r"\bqacct:\"(p-[a-z0-9]+)\"", eventData, re.IGNORECASE) for m in matches: self.sf.debug("Quantcast match: " + m) evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Quantcast: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Ahrefs Site Verification matches = re.findall( r'<meta name="ahrefs-site-verification" content="([a-f0-9]{64})"', eventData, re.IGNORECASE) for m in matches: self.sf.debug("Ahrefs Site Verification match: " + m) evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Ahrefs Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) if eventName == 'DNS_TEXT': # Google Website Verification # https://developers.google.com/site-verification/v1/getting_started matches = re.findall( r'google-site-verification=([a-z0-9\-\+_=]{43,44})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # LogMeIn Domain Verification # https://support.logmeininc.com/openvoice/help/adding-a-txt-record-to-a-dns-server-ov710011 matches = re.findall( r'logmein-domain-confirmation ([A-Z0-9]{24})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "LogMeIn Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) matches = re.findall( r'logmein-verification-code=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "LogMeIn Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # DocuSign Domain Verification # https://support.docusign.com/en/guides/org-admin-guide-domains matches = re.findall( r'docusign=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "DocuSign Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # GlobalSign Site Verification # https://support.globalsign.com/customer/en/portal/articles/2167245-performing-domain-verification---dns-txt-record matches = re.findall( r'globalsign-domain-verification=([a-z0-9\-\+_=]{42,44})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "GlobalSign Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Atlassian Domain Verification # https://confluence.atlassian.com/cloud/verify-a-domain-for-your-organization-873871234.html matches = re.findall( r'atlassian-domain-verification=([a-z0-9\-\+\/_=]{64})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Atlassian Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Adobe IDP Site Verification # https://helpx.adobe.com/au/enterprise/using/verify-domain-ownership.html matches = re.findall( r'adobe-idp-site-verification=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Adobe IDP Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) matches = re.findall( r'adobe-idp-site-verification=([a-f0-9]{64})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Adobe IDP Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Adobe Domain Verification # https://helpx.adobe.com/sign/help/domain_claiming.html matches = re.findall(r'adobe-sign-verification=([a-f0-9]{32})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Adobe Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Stripe Domain Verification # https://stripe.com/docs/apple-pay/web#going-live matches = re.findall(r'stripe-verification=([a-f0-9]{64})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Stripe Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # TeamViewer SSO Verification # https://community.teamviewer.com/t5/Knowledge-Base/Single-Sign-On-SSO/ta-p/30784 matches = re.findall( r'teamviewer-sso-verification=([a-f0-9]{32})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "TeamViewer SSO Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Aliyun Site Verification matches = re.findall( r'aliyun-site-verification=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Aliyun Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Facebook Domain Verification # https://developers.facebook.com/docs/sharing/domain-verification/ matches = re.findall( r'facebook-domain-verification=([a-z0-9]{30})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Facebook Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Citrix Domain Verification matches = re.findall( r'citrix-verification-code=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Citrix Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Dropbox Domain Verification # https://help.dropbox.com/teams-admins/admin/domain-insights-account-capture#verify matches = re.findall( r'dropbox-domain-verification=([a-z0-9]{12})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Dropbox Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Detectify Domain Verification # https://support.detectify.com/customer/en/portal/articles/2836806-verification-with-dns-txt- matches = re.findall(r'detectify-verification=([a-f0-9]{32})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Detectify Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Drift Domain Verification matches = re.findall(r'drift-verification=([a-f0-9]{64})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Drift Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Ahrefs Site Verification # https://help.ahrefs.com/en/articles/1431155-how-do-i-finish-crawling-my-website-faster-in-site-audit matches = re.findall(r'ahrefs-site-verification_([a-f0-9]{64})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Ahrefs Site Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Statuspage.io Domain Verification # https://help.statuspage.io/help/domain-ownership matches = re.findall( r'status-page-domain-verification=([a-z0-9]{12})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Statuspage Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Zoom.us Domain Verification # https://support.zoom.us/hc/en-us/articles/203395207-What-is-Managed-Domain- matches = re.findall(r'ZOOM_verify_([a-z0-9\-\+\/_=]{22})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Zoom.us Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Mail.ru Domain Verification matches = re.findall(r'mailru-verification: ([a-z0-9]{16})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Mail.ru Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Yandex Domain Verification matches = re.findall(r'yandex-verification: ([a-z0-9]{16})$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Yandex Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Brave Ledger Verification # https://support.brave.com/hc/en-us/articles/360021408352-How-do-I-verify-my-channel- matches = re.findall(r'brave-ledger-verification=([a-z0-9]+)$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Brave Ledger Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # have-i-been-pwned Verification matches = re.findall( r'have-i-been-pwned-verification=([a-f0-9]+)$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "have-i-been-pwned Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt) # Cisco Live Domain Verification # https://www.ciscolive.com/c/dam/r/ciscolive/us/docs/2016/pdf/TECCOL-2982.pdf matches = re.findall(r'cisco-ci-domain-verification=([a-f0-9]+)$', eventData.strip(), re.IGNORECASE) for m in matches: evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Cisco Live Domain Verification: " + m, self.__name__, event) evt.moduleDataSource = datasource self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType eventData = event.data if not self.opts['fullnames'] and eventName == 'HUMAN_NAME': return None if eventData in self.results: self.sf.debug("Already did a search for " + eventData + ", skipping.") return None self.results[eventData] = True keepGoing = True page = 1 while keepGoing and page <= int(self.opts['max_pages']): # Check if we've been asked to stop if self.checkForStop(): return None params = { 'search': '"' + eventData.encode('raw_unicode_escape').decode( "ascii", errors='replace') + '"', 'submit': 'Search', 'page': str(page) } # Sites hosted on the domain data = self.sf.fetchUrl( 'https://onionsearchengine.com/search.php?' + urllib.parse.urlencode(params), useragent=self.opts['_useragent'], timeout=self.opts['timeout']) if data is None or not data.get('content'): self.sf.info("No results returned from onionsearchengine.com.") return None page += 1 if "url.php?u=" not in data['content']: # Work around some kind of bug in the site if "you didn't submit a keyword" in data['content']: continue return None if "forward >" not in data['content']: keepGoing = False # Submit the google results for analysis evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'], self.__name__, event) self.notifyListeners(evt) links = re.findall(r"url\.php\?u=(.[^\"\']+)[\"\']", data['content'], re.IGNORECASE | re.DOTALL) for link in links: if self.checkForStop(): return None if link in self.results: continue self.results[link] = True blacklist = False for r in self.opts['blacklist']: if re.match(r, link, re.IGNORECASE): self.sf.debug("Skipping " + link + " as it matches blacklist " + r) blacklist = True if blacklist: continue self.sf.debug("Found a darknet mention: " + link) if not self.sf.urlFQDN(link).endswith(".onion"): continue if not self.opts['fetchlinks']: evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) continue res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False) if res['content'] is None: self.sf.debug("Ignoring " + link + " as no data returned") continue if eventData not in res['content']: self.sf.debug("Ignoring " + link + " as no mention of " + eventData) continue evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) try: startIndex = res['content'].index(eventData) - 120 endIndex = startIndex + len(eventData) + 240 except Exception: self.sf.debug('String "' + eventData + '" not found in content.') continue data = res['content'][startIndex:endIndex] evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...", self.__name__, evt) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return if eventData in self.results: return if self.opts['api_key'] == "": self.error( "You enabled sfp_metadefender but did not set an API key!") self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {event.module}") if eventName == 'IP_ADDRESS': data = self.queryIp(eventData) if data is None: self.debug("No matches found for " + eventData) return geo_info = data.get('geo_info') if geo_info: location = ', '.join([ _f for _f in [ geo_info.get('city').get('name'), geo_info.get('country').get('name') ] if _f ]) evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) res = data.get('lookup_results') if not res: self.debug("No matches found for " + eventData) return sources = res.get('sources') if not sources: self.debug("No matches found for " + eventData) return for m in sources: if not m.get('assessment'): continue provider = m.get('provider') evt = SpiderFootEvent('MALICIOUS_IPADDR', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('BLACKLISTED_IPADDR', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) if eventName == 'INTERNET_NAME': data = self.queryDomain(eventData) if data is None: self.debug("No matches found for " + eventData) return res = data.get('lookup_results') if not res: self.debug("No matches found for " + eventData) return sources = res.get('sources') if not sources: self.debug("No matches found for " + eventData) return for m in sources: if not m.get('assessment'): continue if m['assessment'] == "trustworthy": continue provider = m.get('provider') evt = SpiderFootEvent('MALICIOUS_INTERNET_NAME', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('BLACKLISTED_INTERNET_NAME', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None if srcModuleName == "sfp_open_passive_dns_database": self.sf.debug("Ignoring " + eventName + ", from self.") return None if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None self.results[eventData] = True self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") data = self.query(eventData) if data is None or len(data) == 0: self.sf.info("No passive DNS data found for " + eventData) return None domains = list() for record in data: if self.checkForStop(): return None if self.errorState: return None # first_seen = record[0] # last_seen = record[1] query_html = record[2] answer_type = record[3] answer_html = record[4] # ttl = record[5] # count = record[6] # Extract queries and answers from HTML, and append all in-scope records to the domains list for parsing r = re.findall(r'>(.+?)<', query_html, re.DOTALL) if len(r) == 0: continue query = r[0] if self.getTarget().matches(query, includeChildren=True, includeParents=True): domains.append(query) r = re.findall(r'>(.+?)<', answer_html, re.DOTALL) if len(r) == 0: continue answer = r[0] if self.getTarget().matches(answer, includeChildren=True, includeParents=True): domains.append(answer) # Answers for these records types are ignored, # as all in-scope records have already been added to the domains list for parsing if answer_type in ['PTR', 'NS', 'MX', 'CNAME', 'TXT', 'SOA']: continue if answer in self.results: continue if answer == 'NXDOMAIN': continue if answer_type == 'A': if not self.getTarget().matches( query, includeChildren=True, includeParents=True): continue if not self.sf.validIP(answer): self.sf.debug("Skipping invalid IP address " + answer) continue if self.opts['verify'] and not self.sf.validateIP( query, answer): self.sf.debug("Host " + query + " no longer resolves to " + answer) continue evt = SpiderFootEvent("IP_ADDRESS", answer, self.__name__, event) self.notifyListeners(evt) if answer_type == 'AAAA': if not self.getTarget().matches( query, includeChildren=True, includeParents=True): continue if not self.sf.validIP6(answer): self.sf.debug("Skipping invalid IPv6 address " + answer) continue if self.opts['verify'] and not self.sf.validateIP( query, answer): self.sf.debug("Host " + query + " no longer resolves to " + answer) continue evt = SpiderFootEvent("IPV6_ADDRESS", answer, self.__name__, event) self.notifyListeners(evt) for domain in set(domains): if self.checkForStop(): return None if domain in self.results: continue if not self.getTarget().matches( domain, includeChildren=True, includeParents=True): continue if self.opts['verify'] and not self.sf.resolveHost(domain): self.sf.debug(f"Host {domain} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == 'sfp_fsecure_riddler': self.debug("Ignoring " + eventData + ", from self.") return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.opts['username'] == '' or self.opts['password'] == '': self.error( 'You enabled sfp_fsecure_riddler but did not set an API username/password!' ) self.errorState = True return if not self.token: self.login() self.results[eventData] = True data = None if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']: data = self.query("pld:" + eventData) elif eventName == 'IP_ADDRESS': data = self.query("ip:" + eventData) if not data: self.info("No results found for " + eventData) return e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(e) hosts = list() addrs = list() coords = list() for result in data: host = result.get('host') if not host: continue if not self.getTarget().matches( host, includeChildren=True, includeParents=True): continue hosts.append(host) addr = result.get('addr') if addr: addrs.append(addr) coord = result.get('cordinates') if coord and len(coord) == 2: coords.append(str(coord[0]) + ', ' + str(coord[1])) if self.opts['verify'] and len(hosts) > 0: self.info("Resolving " + str(len(set(hosts))) + " domains ...") for host in set(hosts): if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost( host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts['_internettlds']): if evt_type.startswith('AFFILIATE'): evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) for addr in set(addrs): if self.sf.validIP(addr): evt = SpiderFootEvent('IP_ADDRESS', addr, self.__name__, event) self.notifyListeners(evt) for coord in set(coords): evt = SpiderFootEvent('PHYSICAL_COORDINATES', coord, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event if self.errorState: return self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['google_api_key'] == "" and self.opts['bing_api_key'] == "": self.sf.error("You enabled sfp_socialprofiles but did not set a Google or Bing API key!") self.errorState = True return # Don't look up stuff twice if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if self.keywords is None: self.keywords = self.sf.domainKeywords( self.getTarget().getNames(), self.opts["_internettlds"] ) if len(self.keywords) == 0: self.keywords = None for site in sites: s = str(sites[site][0]).format(name=eventData) searchStr = s.replace(" ", "%20") res = None if self.opts["method"].lower() == "yahoo": self.sf.error( "Yahoo is no longer supported. Please try 'bing' or 'google'." ) return if self.opts["method"].lower() == "google": res = self.sf.googleIterate( searchString=searchStr, opts={ "timeout": self.opts["_fetchtimeout"], "useragent": self.opts["_useragent"], "api_key": self.opts["google_api_key"], "cse_id": self.opts["google_cse_id"], }, ) self.__dataSource__ = "Google" if self.opts["method"].lower() == "bing": res = self.sf.bingIterate( searchString=searchStr, opts={ "timeout": self.opts["_fetchtimeout"], "useragent": self.opts["_useragent"], "count": self.opts["count"], "api_key": self.opts["bing_api_key"], }, ) self.__dataSource__ = "Bing" if res is None: self.sf.info("No data returned from " + self.opts["method"] + ".") continue if self.checkForStop(): return # Submit the results for analysis evt = SpiderFootEvent( "RAW_RIR_DATA", str(res), self.__name__, event ) self.notifyListeners(evt) instances = list() for searchDom in sites[site][1]: # Search both the urls & the search engine web content search_string = " ".join(res["urls"] + [str(res)]) matches = re.findall( searchDom, search_string, re.IGNORECASE | re.MULTILINE ) if not matches: continue for match in matches: self.sf.debug("Match found: " + match) if match in instances: continue else: instances.append(match) if self.checkForStop(): return # Fetch the profile page if we are checking # for a firm relationship. # Keywords might be empty if the target was an IP, subnet or name. if self.opts["tighten"] and self.keywords: match = urllib.parse.unquote(match) self.sf.debug( "Tightening results to look for " + str(self.keywords) ) pres = self.sf.fetchUrl( match, timeout=self.opts["_fetchtimeout"], useragent=self.opts["_useragent"], verify=False ) if pres["content"] is None: continue else: found = False for kw in self.keywords: if re.search( r"[^a-zA-Z\-\_]" + kw + r"[^a-zA-Z\-\_]", pres["content"], re.IGNORECASE, ): found = True if not found: continue self.sf.info("Social Media Profile found at " + site + ": " + match) match = urllib.parse.unquote(match) evt = SpiderFootEvent( "SOCIAL_MEDIA", site + ": <SFURL>" + match + "</SFURL>", self.__name__, event ) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None if eventData in self.results: return None self.results[eventData] = True self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName in ["IP_ADDRESS"]: data = self.queryHost(eventData) if data is None: self.sf.debug("No information found for host " + eventData) return None evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) services = data.get("Services") if services: for service in services: port = service.get('port') if port: evt = SpiderFootEvent("TCP_PORT_OPEN", eventData + ':' + port, self.__name__, event) self.notifyListeners(evt) headers = service.get('headers') if headers: servers = headers.get('Server') if servers: for server in servers: if server: evt = SpiderFootEvent('WEBSERVER_BANNER', server, self.__name__, event) self.notifyListeners(evt) geoip = service.get('geoip') if geoip: location = ', '.join([_f for _f in [geoip.get('city_name'), geoip.get('region_name'), geoip.get('country_name')] if _f]) if location: evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(evt) software = service.get('software') if software: software_version = ' '.join([_f for _f in [software.get('name'), software.get('version')] if _f]) if software_version: evt = SpiderFootEvent("SOFTWARE_USED", software_version, self.__name__, event) self.notifyListeners(evt) os = software.get('os') if os: evt = SpiderFootEvent('OPERATING_SYSTEM', os, self.__name__, event) self.notifyListeners(evt) leaks = data.get("Leaks") if leaks: for leak in leaks: leak_data = leak.get('data') if leak_data: evt = SpiderFootEvent("LEAKSITE_CONTENT", leak_data, self.__name__, event) self.notifyListeners(evt) return None
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return if self.errorState: return if self.opts['api_key'] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") max_pages = int(self.opts['max_pages']) page = 1 last_id = None hosts = list() while page <= max_pages: if self.checkForStop(): break if self.errorState: break data = self.queryIssuances(eventData, last_id) if data is None or len(data) == 0: break page += 1 evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) for result in data: cert_hosts = result.get('dns_names') if cert_hosts: for d in cert_hosts: if d != eventData: hosts.append(d.replace("*.", "")) if result.get('cert') is None: self.debug('Response data contains no certificate data') continue try: rawcert = "-----BEGIN CERTIFICATE-----\n" rawcert += result.get('cert').get('data') rawcert += "\n-----END CERTIFICATE-----\n" cert = self.sf.parseCert(rawcert, eventData, self.opts['certexpiringdays']) except Exception as e: self.info(f"Error parsing certificate: {e}") continue if not cert.get('text'): self.info("Failed to parse the SSL certificate") continue evt = SpiderFootEvent('SSL_CERTIFICATE_RAW', cert['text'], self.__name__, event) self.notifyListeners(evt) if cert.get('issuer'): evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER', cert['issuer'], self.__name__, event) self.notifyListeners(evt) if cert.get('issued'): evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED', cert['issued'], self.__name__, event) self.notifyListeners(evt) for san in set(cert.get('altnames', list())): hosts.append(san.replace("*.", "")) if cert.get('expired'): evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRED", cert.get('expirystr', 'Unknown'), self.__name__, event) self.notifyListeners(evt) continue if cert.get('expiring'): evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRING", cert.get('expirystr', 'Unknown'), self.__name__, event) self.notifyListeners(evt) continue # "To retrieve additional issuances, take the id field of the last issuance and pass it to the issuances endpoint in the after parameter" last_id = data[-1].get('id') if last_id is None: break if not hosts: return if self.opts['verify']: self.info(f"Resolving {len(set(hosts))} hostnames ...") for domain in set(hosts): if self.checkForStop(): return if domain in self.results: continue if self.getTarget().matches(domain, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost( domain) and not self.sf.resolveHost6(domain): self.debug(f"Host {domain} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, domain, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(domain, self.opts['_internettlds']): if evt_type.startswith('AFFILIATE'): evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', domain, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") # Always check if the API key is set and complain if it isn't, then set # self.errorState to avoid this being a continual complaint during the scan. if self.opts['api_key_client_id'] == "" or self.opts['api_key_client_secret'] == "": self.error("You enabled sfp_snov but did not set a Client ID and/or Client Secret") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True # Get access token from Snov IO API accessToken = self.queryAccessToken() if accessToken is None or accessToken == '': self.error("No access token received from snov.io for the provided Client ID and/or Client Secret") self.errorState = True return currentLastId = 0 nextPageHasData = True while nextPageHasData: if self.checkForStop(): return data = self.queryDomainName(eventData, accessToken, currentLastId) if data is None: self.debug("No email address found for target domain") break try: data = json.loads(data) except Exception: self.debug("No email address found for target domain") break evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) records = data.get('emails') lastId = data.get('lastId') if records: for record in records: if record: email = str(record.get('email')) if email: if email in self.results: continue if not SpiderFootHelpers.validEmail(email): continue self.results[email] = True if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) # Determine whether another page of data exists if len(records) < self.limit: nextPageHasData = False currentLastId = lastId
def handleEvent(self, event): eventName = event.eventType eventData = event.data if not self.opts['fullnames'] and eventName == 'HUMAN_NAME': return None if eventData in self.results: self.sf.debug("Already did a search for " + eventData + ", skipping.") return None else: self.results[eventData] = True # Sites hosted on the domain data = self.sf.fetchUrl("https://ahmia.fi/search/?q=" + eventData.replace(" ", "%20"), useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) if data is None or not data.get('content'): self.sf.info("No results returned from ahmia.fi.") return None if "redirect_url=" in data['content']: # Check if we've been asked to stop if self.checkForStop(): return None links = re.findall("redirect_url=(.[^\"]+)\"", data['content'], re.IGNORECASE | re.DOTALL) reported = False for link in links: if link in self.results: continue else: self.results[link] = True self.sf.debug("Found a darknet mention: " + link) if self.sf.urlFQDN(link).endswith(".onion"): if self.checkForStop(): return None if self.opts['fetchlinks']: res = self.sf.fetchUrl( link, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False) if res['content'] is None: self.sf.debug("Ignoring " + link + " as no data returned") continue if eventData not in res['content']: self.sf.debug("Ignoring " + link + " as no mention of " + eventData) continue evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) reported = True try: startIndex = res['content'].index( eventData) - 120 endIndex = startIndex + len(eventData) + 240 except BaseException: self.sf.debug("String not found in content.") continue wdata = res['content'][startIndex:endIndex] evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + wdata + "...", self.__name__, evt) self.notifyListeners(evt) reported = True else: evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) reported = True if reported: # Submit the search results for analysis evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'], self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data ret = None if self.errorState: return None self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") # Ignore messages from myself if srcModuleName == "sfp_riskiq": self.sf.debug("Ignoring " + eventName + ", from self.") return None if self.opts['api_key_login'] == "" or self.opts[ 'api_key_password'] == "": self.sf.error( "You enabled sfp_riskiq but did not set an credentials!") self.errorState = True return None # Don't look up stuff twice if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None else: self.results[eventData] = True if eventName in ['DOMAIN_NAME']: ret = self.query(eventData, "PSSL") if not ret: self.sf.info("No RiskIQ passive SSL data found for " + eventData) if ret: try: # Generate an event for the IP first, and then link the cert # to that event. for res in ret: if res['subjectCommonName'] == eventData: continue if self.getTarget().matches(res['subjectCommonName'], includeChildren=True): if self.sf.resolveHost(res['subjectCommonName']): e = SpiderFootEvent("INTERNET_NAME", res['subjectCommonName'], self.__name__, event) else: e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", res['subjectCommonName'], self.__name__, event) self.notifyListeners(e) if self.sf.isDomain(res['subjectCommonName'], self.opts['_internettlds']): e = SpiderFootEvent("DOMAIN_NAME", res['subjectCommonName'], self.__name__, event) self.notifyListeners(e) except Exception as e: self.sf.error("Invalid response returned from RiskIQ: " + str(e)) if eventName == 'EMAILADDR': ret = self.query(eventData, "WHOIS") if not ret: self.sf.info("No RiskIQ passive DNS data found for " + eventData) return None for r in ret: if not eventData.endswith("@" + r['domain']): if self.sf.validIP(r['domain']): t = "NETBLOCK_OWNER" else: t = "AFFILIATE_INTERNET_NAME" e = SpiderFootEvent(t, r['domain'], self.__name__, event) self.notifyListeners(e) if t == "AFFILIATE_INTERNET_NAME" and self.sf.isDomain( r['domain'], self.opts['_internettlds']): evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", r['domain'], self.__name__, event) self.notifyListeners(evt) return None if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']: ret = self.query(eventData, "PDNS") if not ret: self.sf.info("No RiskIQ passive DNS data found for " + eventData) return None cohosts = list() if eventName == "IP_ADDRESS": for r in ret: if r['focusPoint'].endswith("."): r['focusPoint'] = r['focusPoint'][:-1] # Record could be pointing to our IP, or from our IP if not self.getTarget().matches( r['focusPoint']) and "*" not in r['focusPoint']: # We found a co-host cohosts.append(r['focusPoint']) if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]: # Record could be an A/CNAME of this entity, or something pointing to it for r in ret: if r['focusPoint'].endswith("."): r['focusPoint'] = r['focusPoint'][:-1] if r['focusPoint'] != eventData and "*" not in r[ 'focusPoint']: cohosts.append(r['focusPoint']) for co in cohosts: if co == eventData: continue if eventName == "IP_ADDRESS" and ( self.opts['verify'] and not self.sf.validateIP(co, eventData)): self.sf.debug("Host no longer resolves to our IP.") continue if not self.opts['cohostsamedomain']: if self.getTarget().matches(co, includeParents=True): if self.sf.resolveHost(co): e = SpiderFootEvent("INTERNET_NAME", co, self.__name__, event) else: e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", co, self.__name__, event) self.notifyListeners(e) if self.sf.isDomain(co, self.opts['_internettlds']): e = SpiderFootEvent("DOMAIN_NAME", co, self.__name__, event) self.notifyListeners(e) continue if self.cohostcount < self.opts['maxcohost']: e = SpiderFootEvent("CO_HOSTED_SITE", co, self.__name__, event) self.notifyListeners(e) self.cohostcount += 1
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") # Don't look up stuff twice if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None else: self.results[eventData] = True # BGP AS Owner -> Other Netblocks if eventName == "BGP_AS_OWNER": # Don't report additional netblocks from this AS if we've # already found this AS before. if eventData not in self.nbreported: # Find all the netblocks owned by this AS self.nbreported[eventData] = True netblocks = self.asNetblocks(eventData) if netblocks is not None: for netblock in netblocks: if netblock in self.results: continue # Technically this netblock was identified via the AS, not # the original IP event, so link it to asevt, not event. # Skip IPv6 for now if ":" in netblock: continue evt = SpiderFootEvent("NETBLOCK_OWNER", netblock, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("RAW_RIR_DATA", self.lastContent, self.__name__, event) self.notifyListeners(evt) return None # NETBLOCK -> AS and other owned netblocks if eventName.startswith("NETBLOCK_"): # Get the BGP AS the netblock is a part of asn = self.netblockAs(eventData) if asn is None: self.sf.debug("Could not identify BGP AS for " + eventData) return None if eventName == "NETBLOCK_OWNER" and self.ownsAs(asn): asevt = SpiderFootEvent("BGP_AS_OWNER", asn, self.__name__, event) self.notifyListeners(asevt) evt = SpiderFootEvent("RAW_RIR_DATA", self.lastContent, self.__name__, event) self.notifyListeners(evt) else: asevt = SpiderFootEvent("BGP_AS_MEMBER", asn, self.__name__, event) self.notifyListeners(asevt) return None # IP ADDRESS -> NETBLOCK if eventName == "IP_ADDRESS": # Get the Netblock the IP is a part of prefix = self.ipNetblock(eventData) if prefix is None: self.sf.debug("Could not identify network prefix for " + eventData) return None # Get the BGP AS the netblock is a part of asn = self.netblockAs(prefix) if asn is None: self.sf.debug("Could not identify BGP AS for " + prefix) return None if self.sf.validIpNetwork(prefix): self.sf.info("Netblock found: " + prefix + "(" + asn + ")") evt = SpiderFootEvent("NETBLOCK_MEMBER", prefix, self.__name__, event) self.notifyListeners(evt) return None
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return if self.errorState: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error( "You enabled sfp_grayhatwarfare but did not set an API key!") self.errorState = True return currentIndex = 0 currentPage = 0 maxPages = self.opts['max_pages'] perPage = self.opts['per_page'] keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds']) while currentPage < maxPages: currentIndex = currentPage * perPage if self.checkForStop(): return if self.errorState: break data = self.query(keyword=keyword, start=currentIndex) if not data: return for row in data.get('buckets'): bucketName = row.get('bucket') bucketKeyword = bucketName.split('.')[0] self.debug(bucketKeyword) if bucketKeyword.startswith(keyword) or bucketKeyword.endswith( keyword): evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET', bucketName, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent( 'CLOUD_STORAGE_BUCKET_OPEN', f"{bucketName}: {row.get('fileCount')} files found.", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, event) self.notifyListeners(evt) currentPage += 1 if data.get('buckets_count') < perPage: break
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_shodan but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "DOMAIN_NAME": hosts = self.searchHosts(eventData) if hosts is None: return evt = SpiderFootEvent("RAW_RIR_DATA", str(hosts), self.__name__, event) self.notifyListeners(evt) if eventName == 'WEB_ANALYTICS_ID': try: network = eventData.split(": ")[0] analytics_id = eventData.split(": ")[1] except Exception as e: self.error( f"Unable to parse WEB_ANALYTICS_ID: {eventData} ({e})") return if network not in [ 'Google AdSense', 'Google Analytics', 'Google Site Verification' ]: self.debug(f"Skipping {eventData}, as not supported.") return rec = self.searchHtml(analytics_id) if rec is None: return evt = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__, event) self.notifyListeners(evt) return if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug( f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}" ) return qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) for addr in qrylist: rec = self.queryHost(addr) if rec is None: continue # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event evt = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__, pevent) self.notifyListeners(evt) if self.checkForStop(): return if rec.get('os') is not None: evt = SpiderFootEvent("OPERATING_SYSTEM", f"{rec.get('os')} ({addr})", self.__name__, pevent) self.notifyListeners(evt) if rec.get('devtype') is not None: evt = SpiderFootEvent("DEVICE_TYPE", f"{rec.get('devtype')} ({addr})", self.__name__, pevent) self.notifyListeners(evt) if rec.get('country_name') is not None: location = ', '.join([ _f for _f in [rec.get('city'), rec.get('country_name')] if _f ]) evt = SpiderFootEvent("GEOINFO", location, self.__name__, pevent) self.notifyListeners(evt) if 'data' not in rec: continue self.info(f"Found SHODAN data for {eventData}") ports = list() banners = list() asns = list() products = list() vulnlist = list() for r in rec['data']: port = str(r.get('port')) banner = r.get('banner') asn = r.get('asn') product = r.get('product') vulns = r.get('vulns') if port is not None: cp = addr + ":" + port if cp not in ports: ports.append(cp) evt = SpiderFootEvent("TCP_PORT_OPEN", cp, self.__name__, pevent) self.notifyListeners(evt) if banner is not None: if banner not in banners: banners.append(banner) evt = SpiderFootEvent("TCP_PORT_OPEN_BANNER", banner, self.__name__, pevent) self.notifyListeners(evt) if product is not None: if product not in products: products.append(product) evt = SpiderFootEvent("SOFTWARE_USED", product, self.__name__, pevent) self.notifyListeners(evt) if asn is not None: if asn not in asns: asns.append(asn) evt = SpiderFootEvent("BGP_AS_MEMBER", asn.replace("AS", ""), self.__name__, pevent) self.notifyListeners(evt) if vulns is not None: for vuln in vulns.keys(): if vuln not in vulnlist: vulnlist.append(vuln) etype, cvetext = self.sf.cveInfo(vuln) evt = SpiderFootEvent(etype, cvetext, self.__name__, pevent) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None if eventData in self.results: return None if self.opts['api_key'] == '': self.sf.error("You enabled sfp_networksdb but did not set an API key!", False) self.errorState = True return None self.results[eventData] = True self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName in ["IP_ADDRESS", "IPV6_ADDRESS"]: data = self.queryIpInfo(eventData) if data is None: self.sf.debug("No IP address information found for " + eventData) else: evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) network = data.get('network') if network: cidr = network.get('cidr') if cidr and cidr != 'N/A' and self.sf.validIpNetwork(cidr): evt = SpiderFootEvent('NETBLOCK_MEMBER', cidr, self.__name__, event) self.notifyListeners(evt) data = self.queryIpGeo(eventData) if data is None: self.sf.debug("No IP geolocation information found for " + eventData) else: evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) if data.get('country'): location = ', '.join(filter(None, [data.get('city'), data.get('state'), data.get('country')])) evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) data = self.queryReverseDns(eventData) cohosts = list() if data is None: self.sf.debug("No reverse DNS results for " + eventData) else: evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) results = data.get('results') if results: for domain in results: cohosts.append(domain) for co in set(cohosts): if self.checkForStop(): return None if co in self.results: continue if self.opts['verify'] and not self.sf.validateIP(co, eventData): self.sf.debug("Host " + co + " no longer resolves to " + eventData) continue if not self.opts['cohostsamedomain']: if self.getTarget().matches(co, includeParents=True): evt = SpiderFootEvent('INTERNET_NAME', co, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(co, self.opts['_internettlds']): evt = SpiderFootEvent('DOMAIN_NAME', co, self.__name__, event) self.notifyListeners(evt) continue if self.cohostcount < self.opts['maxcohost']: evt = SpiderFootEvent('CO_HOSTED_SITE', co, self.__name__, event) self.notifyListeners(evt) self.cohostcount += 1 if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]: data = self.queryForwardDns(eventData) if data is None: self.sf.debug("No forward DNS results for " + eventData) return None res = data.get('results') if not res: self.sf.debug("No forward DNS results for " + eventData) return None evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event) self.notifyListeners(evt) for ip in res: if self.sf.validIP(ip): evt = SpiderFootEvent('IP_ADDRESS', ip, self.__name__, event) self.notifyListeners(evt) elif self.sf.validIP6(ip): evt = SpiderFootEvent('IPV6_ADDRESS', ip, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data eventDataHash = self.sf.hashstring(eventData) parentEvent = event self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == "sfp_dnszonexfer": self.debug(f"Ignoring {eventName}, from self.") return if eventDataHash in self.events: self.debug("Skipping duplicate event for " + eventData) return self.events[eventDataHash] = True res = dns.resolver.Resolver() if self.opts.get('_dnsserver', "") != "": res.nameservers = [self.opts['_dnsserver']] # Get the name server's IP. This is to avoid DNS leaks # when attempting to resolve the name server during # the zone transfer. if not self.sf.validIP(eventData) and not self.sf.validIP6(eventData): nsips = self.sf.resolveHost(eventData) if not nsips: return if not nsips: self.error( "Couldn't resolve the name server, so not attempting zone transfer." ) return for n in nsips: if self.sf.validIP(n): nsip = n break else: nsip = eventData for name in self.getTarget().getNames(): self.debug("Trying for name: " + name) try: ret = list() z = dns.zone.from_xfr( dns.query.xfr(nsip, name, timeout=int(self.opts["timeout"]))) names = list(z.nodes.keys()) for n in names: ret.append(z[n].to_text(n)) evt = SpiderFootEvent("RAW_DNS_RECORDS", "\n".join(ret), self.__name__, parentEvent) self.notifyListeners(evt) # Try and pull out individual records for row in ret: pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL) grps = re.findall(pat, row) if len(grps) > 0: for strdata in grps: self.debug("Matched: " + strdata) if strdata.endswith("."): strdata = strdata[:-1] else: strdata = strdata + "." + name evt = SpiderFootEvent("INTERNET_NAME", strdata, self.__name__, parentEvent) self.notifyListeners(evt) except Exception as e: self.info( f"Unable to perform DNS zone transfer for {eventData} ({name}): {e}" )
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if srcModuleName == "sfp_tool_onesixtyone": self.debug("Skipping event from myself.") return if not self.opts['onesixtyone_path']: self.error("You enabled sfp_tool_onesixtyone but did not set a path to the tool!") self.errorState = True return exe = self.opts['onesixtyone_path'] if self.opts['onesixtyone_path'].endswith('/'): exe = f"{exe}onesixtyone" if not os.path.isfile(exe): self.error(f"File does not exist: {exe}") self.errorState = True return if not SpiderFootHelpers.sanitiseInput(eventData, extra=['/']): self.debug("Invalid input, skipping.") return targets = [] try: if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']: net = IPNetwork(eventData) if net.prefixlen < self.opts['netblockscanmax']: self.debug(f"Skipping scanning of {eventData}, too big.") return for addr in net.iter_hosts(): targets.append(str(addr)) except BaseException as e: self.error(f"Strange netblock identified, unable to parse: {eventData} ({e})") return # Don't look up stuff twice, check IP == IP here if eventData in self.results: self.debug(f"Skipping {eventData} as already scanned.") return else: # Might be a subnet within a subnet or IP within a subnet for addr in self.results: if IPNetwork(eventData) in IPNetwork(addr): self.debug(f"Skipping {eventData} as already within a scanned range.") return self.results[eventData] = True # If we weren't passed a netblock, this will be empty if not targets: targets.append(eventData) for target in targets: args = [ exe, "-c", self.communitiesFile, target ] try: p = Popen(args, stdout=PIPE, stderr=PIPE) out, stderr = p.communicate(input=None) stdout = out.decode(sys.stdin.encoding) except Exception as e: self.error(f"Unable to run onesixtyone: {e}") continue if p.returncode != 0: self.error(f"Unable to read onesixtyone output\nstderr: {stderr}\nstdout: {stdout}") continue if not stdout: self.debug(f"onesixtyone returned no output for {target}") continue for result in stdout.split("\n"): srcevent = event if target not in result: continue if target != eventData: srcevent = SpiderFootEvent("IP_ADDRESS", target, self.__name__, event) self.notifyListeners(srcevent) e = SpiderFootEvent('UDP_PORT_OPEN', f"{target}:161", self.__name__, srcevent) self.notifyListeners(e) e = SpiderFootEvent("UDP_PORT_OPEN_INFO", result, self.__name__, e) self.notifyListeners(e)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return None self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.sf.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return None if eventData in self.results: self.sf.debug(f"Skipping {eventData} as already mapped.") return None self.results[eventData] = True data = self.query(event.data) if not data: self.sf.error(f"No data received for {event.data}") return None found = False ipinfo = data.get("ipinfo") if ipinfo and isinstance(ipinfo, dict): for address, ip_data in data["ipinfo"].items(): evt = SpiderFootEvent("IP_ADDRESS", address, self.__name__, event) self.notifyListeners(evt) found = True loc = ip_data.get("loc") if loc and isinstance(loc, str): loc_evt = SpiderFootEvent("PHYSICAL_COORDINATES", loc, self.__name__, evt) self.notifyListeners(loc_evt) found = True geo_info = ', '.join( filter(None, (ip_data.get(k) for k in ("city", "region", "country")))) if geo_info: geo_info_evt = SpiderFootEvent("GEOINFO", geo_info, self.__name__, evt) self.notifyListeners(geo_info_evt) found = True related = data.get("related") if related and isinstance(related, dict): email_section = related.get("email") if email_section and isinstance(email_section, list): for email_data in email_section: if isinstance(email_data, dict): value = email_data["value"] if value and isinstance(value, str): evt = SpiderFootEvent("EMAILADDR", value, self.__name__, event) self.notifyListeners(evt) found = True web = data.get("web") if web and isinstance(web, dict): server = web.get("server") if server and isinstance(server, str): evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", server, self.__name__, event) self.notifyListeners(evt) found = True google_analytics = web.get("googleanalytics") if google_analytics and isinstance(google_analytics, str): evt = SpiderFootEvent("WEB_ANALYTICS_ID", google_analytics, self.__name__, event) self.notifyListeners(evt) found = True title = web.get("title") if title and isinstance(title, str): evt = SpiderFootEvent("DESCRIPTION_ABSTRACT", title, self.__name__, event) self.notifyListeners(evt) found = True if found: evt = SpiderFootEvent("RAW_RIR_DATA", json.dumps(data), self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: return None else: self.results[eventData] = True for fileExt in self.opts['fileexts']: if self.checkForStop(): return None if "." + fileExt.lower() in eventData.lower(): # Fetch the file, allow much more time given that these files are # typically large. ret = self.sf.fetchUrl(eventData, timeout=self.opts['timeout'], useragent=self.opts['_useragent'], dontMangle=True, sizeLimit=10000000, verify=False) if ret['content'] is None: self.sf.error( f"Unable to fetch file for meta analysis: {eventData}") return None if len(ret['content']) < 512: self.sf.error( f"Strange content encountered, size of {len(ret['content'])}" ) return None meta = None data = None # Based on the file extension, handle it if fileExt.lower() == "pdf": try: raw = io.BytesIO(ret['content']) # data = metapdf.MetaPdfReader().read_metadata(raw) pdf = PyPDF2.PdfFileReader(raw, strict=False) data = pdf.getDocumentInfo() meta = str(data) self.sf.debug("Obtained meta data from " + eventData) except Exception as e: self.sf.error( f"Unable to parse meta data from: {eventData} ({e})" ) return None if fileExt.lower() in ["docx"]: try: c = io.BytesIO(ret['content']) doc = docx.Document(c) mtype = mimetypes.guess_type(eventData)[0] self.sf.debug("Office type: " + str(mtype)) a = doc.core_properties.author c = doc.core_properties.comments data = [_f for _f in [a, c] if _f] meta = ", ".join(data) except Exception as e: self.sf.error( f"Unable to process file: {eventData} ({e})") return None if fileExt.lower() in ["pptx"]: try: c = io.BytesIO(ret['content']) doc = pptx.Presentation(c) mtype = mimetypes.guess_type(eventData)[0] self.sf.debug("Office type: " + str(mtype)) a = doc.core_properties.author c = doc.core_properties.comments data = [_f for _f in [a, c] if _f] meta = ", ".join(data) except Exception as e: self.sf.error( f"Unable to process file: {eventData} ({e})") return None if fileExt.lower() in ["jpg", "jpeg", "tiff"]: try: raw = io.BytesIO(ret['content']) data = exifread.process_file(raw) if data is None or len(data) == 0: continue meta = str(data) except Exception as e: self.sf.error( f"Unable to parse meta data from: {eventData} ({e})" ) return None if meta is not None and data is not None: evt = SpiderFootEvent("RAW_FILE_META_DATA", meta, self.__name__, event) self.notifyListeners(evt) val = list() try: if "/Producer" in data: val.append(str(data['/Producer'])) if "/Creator" in data: val.append(str(data['/Creator'])) if "Application" in data: val.append(str(data['Application'])) if "Image Software" in data: val.append(str(data['Image Software'])) except Exception as e: self.sf.error("Failed to parse PDF, " + eventData + ": " + str(e)) return None for v in val: if v and not isinstance(v, PyPDF2.generic.NullObject): self.sf.debug("VAL: " + str(val)) # Strip non-ASCII v = ''.join( [i if ord(i) < 128 else ' ' for i in v]) evt = SpiderFootEvent("SOFTWARE_USED", v, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.opts['affiliatedomains'] and "AFFILIATE_" in eventName: eventData = self.sf.hostDomain(eventData, self.opts['_internettlds']) if not eventData: return None if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None self.results[eventData] = True url = "https://api.duckduckgo.com/?q=" + eventData + "&format=json&pretty=1" res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot") if res['content'] is None: self.sf.error(f"Unable to fetch {url}", False) return None try: ret = json.loads(res['content']) except BaseException as e: self.sf.error(f"Error processing JSON response from DuckDuckGo: {e}", False) return None if not ret['Heading']: self.sf.debug(f"No DuckDuckGo information for {eventData}") return None # Submit the DuckDuckGo results for analysis evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", res['content'], self.__name__, event) self.notifyListeners(evt) abstract_text = ret.get('AbstractText') if abstract_text: event_type = "DESCRIPTION_ABSTRACT" if "AFFILIATE" in eventName: event_type = "AFFILIATE_" + event_type evt = SpiderFootEvent(event_type, str(abstract_text), self.__name__, event) self.notifyListeners(evt) related_topics = ret.get('RelatedTopics') if related_topics: event_type = "DESCRIPTION_CATEGORY" if "AFFILIATE" in eventName: event_type = "AFFILIATE_" + event_type for topic in related_topics: if not isinstance(topic, dict): self.sf.debug("No category text found from DuckDuckGo.") continue category = topic.get('Text') if not category: self.sf.debug("No category text found from DuckDuckGo.") continue evt = SpiderFootEvent(event_type, category, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if eventData in self.results: self.sf.debug("Skipping " + eventData + " as already scanned.") return self.results[eventData] = True if not self.opts['whatweb_path']: self.sf.error("You enabled sfp_tool_whatweb but did not set a path to the tool!") self.errorState = True return exe = self.opts['whatweb_path'] if self.opts['whatweb_path'].endswith('/'): exe = exe + 'whatweb' # If tool is not found, abort if not os.path.isfile(exe): self.sf.error("File does not exist: " + exe) self.errorState = True return # Sanitize domain name. if not self.sf.sanitiseInput(eventData): self.sf.error("Invalid input, refusing to run.") return # Set aggression level try: aggression = int(self.opts['aggression']) if aggression > 4: aggression = 4 if aggression < 1: aggression = 1 except Exception: aggression = 1 # Run WhatWeb args = [ self.opts['ruby_path'], exe, "--quiet", "--aggression=" + str(aggression), "--log-json=/dev/stdout", "--user-agent=Mozilla/5.0", "--follow-redirect=never", eventData ] try: p = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate(input=None) except Exception as e: self.sf.error(f"Unable to run WhatWeb: {e}") return if p.returncode != 0: self.sf.error("Unable to read WhatWeb output.") self.sf.debug("Error running WhatWeb: " + stderr + ", " + stdout) return if not stdout: self.sf.debug(f"WhatWeb returned no output for {eventData}") return try: result_json = json.loads(stdout) except Exception as e: self.sf.error(f"Couldn't parse the JSON output of WhatWeb: {e}") return if len(result_json) == 0: return evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event) self.notifyListeners(evt) blacklist = [ 'Country', 'IP', 'Script', 'Title', 'HTTPServer', 'RedirectLocation', 'UncommonHeaders', 'Via-Proxy', 'Cookies', 'HttpOnly', 'Strict-Transport-Security', 'x-hacker', 'x-machine', 'x-pingback', 'X-Backend', 'X-Cache', 'X-UA-Compatible', 'X-Powered-By', 'X-Forwarded-For', 'X-Frame-Options', 'X-XSS-Protection' ] for result in result_json: plugin_matches = result.get('plugins') if not plugin_matches: continue if plugin_matches.get('HTTPServer'): for w in plugin_matches.get('HTTPServer').get('string'): evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event) self.notifyListeners(evt) if plugin_matches.get('X-Powered-By'): for w in plugin_matches.get('X-Powered-By').get('string'): evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event) self.notifyListeners(evt) for plugin in plugin_matches: if plugin in blacklist: continue evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', plugin, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if self.opts['external']: external = "True" else: external = "" if self.opts['daysback'] is not None and self.opts['daysback'] != 0: newDate = datetime.datetime.now() - datetime.timedelta( days=int(self.opts['daysback'])) maxDate = newDate.strftime("%Y-%m-%d") else: maxDate = "" qdata = eventData.replace(" ", "+") wlurl = "query=%22" + qdata + "%22" + "&released_date_start=" + maxDate + "&include_external_sources=" + external + "&new_search=True&order_by=most_relevant#results" res = self.sf.fetchUrl("https://search.wikileaks.org/?" + wlurl) if res['content'] is None: self.sf.error("Unable to fetch Wikileaks content.") return # Fetch the paste site content links = dict() p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org") if p: links.update(p) p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org") if p: links.update(p) keepGoing = True page = 0 while keepGoing: if not res['content']: break if "page=" not in res['content']: keepGoing = False valid = False for link in links: if "search.wikileaks.org/" in link: continue # We can safely skip search.wikileaks.org and others. if "wikileaks.org/" not in link and "cryptome.org/" not in link: continue else: self.sf.debug("Found a link: " + link) if self.checkForStop(): return # Wikileaks leak links will have a nested folder structure link if link.count('/') >= 4: if not link.endswith(".js") and not link.endswith( ".css"): evt = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event) self.notifyListeners(evt) valid = True if valid: evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", res['content'], self.__name__, event) self.notifyListeners(evt) # Fail-safe to prevent infinite looping if page > 50: break if keepGoing: page += 1 wlurl = "https://search.wikileaks.org/?query=%22" + qdata + "%22" + \ "&released_date_start=" + maxDate + "&include_external_sources=" + \ external + "&new_search=True&order_by=most_relevant&page=" + \ str(page) + "#results" res = self.sf.fetchUrl(wlurl) if not res['content']: break # Fetch the paste site content links = dict() p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org") if p: links.update(p) p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org") if p: links.update(p)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if event.moduleDataSource: moduleDataSource = event.moduleDataSource else: moduleDataSource = "Unknown" self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") eventDataHash = self.sf.hashstring(eventData) if eventDataHash in self.results: self.sf.debug(f"Skipping {eventData}, already checked.") return None self.results[eventDataHash] = True countryNames = list() # Process the event data based on incoming event type if eventName == "PHONE_NUMBER": countryNames.append(self.detectCountryFromPhone(eventData)) elif eventName == "DOMAIN_NAME": countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "AFFILIATE_DOMAIN_NAME" and self.opts["affiliate"]: countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "CO_HOSTED_SITE_DOMAIN" and self.opts["cohosted"]: countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "SIMILARDOMAIN" and self.opts["similardomain"]: countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "IBAN_NUMBER": countryNames.append(self.detectCountryFromIBAN(eventData)) elif eventName in ["DOMAIN_WHOIS", "GEOINFO", "PHYSICAL_ADDRESS"]: countryNames.extend(self.detectCountryFromData(eventData)) elif eventName == "AFFILIATE_DOMAIN_WHOIS" and self.opts["affiliate"]: countryNames.extend(self.detectCountryFromData(eventData)) elif eventName == "CO_HOSTED_SITE_DOMAIN_WHOIS" and self.opts[ "cohosted"]: countryNames.extend(self.detectCountryFromData(eventData)) if not countryNames: self.sf.debug( f"Found no country names associated with {eventName}: {eventData}" ) return None for countryName in set(countryNames): if not countryName: continue self.sf.debug(f"Found country name: {countryName}") evt = SpiderFootEvent("COUNTRY_NAME", countryName, self.__name__, event) evt.moduleDataSource = moduleDataSource self.notifyListeners(evt) return None
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") leis = list() if eventName == "LEI": leis.append(eventData) elif eventName == "COMPANY_NAME": res = self.searchAutocompletions(eventData) if res is None: self.debug(f"Found no results for {eventData}") return e = SpiderFootEvent("RAW_RIR_DATA", str(res), self.__name__, event) self.notifyListeners(e) for record in res: relationships = record.get('relationships') if not relationships: continue lei_records = relationships.get('lei-records') if not lei_records: continue data = lei_records.get('data') if not data: continue lei = data.get('id') if not SpiderFootHelpers.validLEI(lei): continue leis.append(lei) self.info(f"Found {len(leis)} LEIs matching {eventData}") for lei in set(leis): if lei in self.results: continue if not SpiderFootHelpers.validLEI(lei): continue e = SpiderFootEvent("LEI", lei, self.__name__, event) self.notifyListeners(e) self.results[lei] = True res = self.retrieveRecord(lei) if not res: self.debug(f"Found no results for {eventData}") return attributes = res.get('attributes') if not attributes: continue entity = attributes.get('entity') if not entity: continue legal_name = entity.get('legalName') if legal_name: entity_name = legal_name.get('value') if entity_name: e = SpiderFootEvent("COMPANY_NAME", entity_name, self.__name__, event) self.notifyListeners(e) addresses = list() address = entity.get('legalAddress') if address.get('addressLines'): address_lines = ', '.join( filter(None, address.get('addressLines'))) location = ', '.join( filter(None, [ address_lines, address.get('city'), address.get('region'), address.get('country'), address.get('postalCode') ])) if location: addresses.append(location) address = entity.get('headquartersAddress') if address.get('addressLines'): address_lines = ', '.join( filter(None, address.get('addressLines'))) location = ', '.join( filter(None, [ address_lines, address.get('city'), address.get('region'), address.get('country'), address.get('postalCode') ])) if location: addresses.append(location) for address in set(addresses): e = SpiderFootEvent("PHYSICAL_ADDRESS", address, self.__name__, event) self.notifyListeners(e)