def test_valid_email_should_return_a_boolean(self): invalid_types = [None, "", list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): valid_email = SpiderFootHelpers.validEmail(invalid_type) self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('*****@*****.**') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('*****@*****.**') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail( '[email protected]\n.com') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('root@localhost') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('*****@*****.**') self.assertIsInstance(valid_email, bool) self.assertTrue(valid_email)
def queryEmail(self, email): if not SpiderFootHelpers.validEmail(email): return None params = urllib.parse.urlencode({ 'mail': email, 'key': self.opts['api_key'], }) res = self.sf.fetchUrl( f"https://botscout.com/test/?{params}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) return self.parseApiResponse(res)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return if self.errorState: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") position = 0 max_pages = int(self.opts['max_pages']) per_page = int(self.opts['per_page']) emails = list() hashes = list() passwords = list() while position < (per_page * max_pages): if self.checkForStop(): return if self.errorState: break data = self.query(eventData, per_page, position) if not data: return position += per_page # evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) # self.notifyListeners(evt) for row in data: result = row.get('fields') if not result: continue email = result.get('email') # A blank email result should not be possible, as we searched using the 'Email:' filter if not email: continue if not SpiderFootHelpers.validEmail(email): self.debug("Skipping invalid email address: " + email) continue mailDom = email.lower().split('@')[1] # Skip unrelated emails # Scylla sometimes returns broader results than the searched data if not self.getTarget().matches(mailDom): self.debug("Skipped address: " + email) continue breach = result.get('domain', 'Unknown') emails.append(email + " [" + breach + "]") pass_hash = result.get('passhash') if pass_hash: pass_salt = result.get('passsalt') if pass_salt: hashes.append(email + ':' + pass_hash + " (Salt: " + pass_salt + ") [" + breach + "]") else: hashes.append(email + ':' + pass_hash + " [" + breach + "]") password = result.get('password') if password: passwords.append(email + ':' + password + " [" + breach + "]") if len(data) < per_page: break for pass_hash in set(hashes): evt = SpiderFootEvent('HASH_COMPROMISED', pass_hash, self.__name__, event) self.notifyListeners(evt) for email in set(emails): evt = SpiderFootEvent('EMAILADDR_COMPROMISED', email, self.__name__, event) self.notifyListeners(evt) for password in set(passwords): evt = SpiderFootEvent('PASSWORD_COMPROMISED', password, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") # Always check if the API key is set and complain if it isn't, then set # self.errorState to avoid this being a continual complaint during the scan. if self.opts['api_key_client_id'] == "" or self.opts['api_key_client_secret'] == "": self.error("You enabled sfp_snov but did not set a Client ID and/or Client Secret") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True # Get access token from Snov IO API accessToken = self.queryAccessToken() if accessToken is None or accessToken == '': self.error("No access token received from snov.io for the provided Client ID and/or Client Secret") self.errorState = True return currentLastId = 0 nextPageHasData = True while nextPageHasData: if self.checkForStop(): return data = self.queryDomainName(eventData, accessToken, currentLastId) if data is None: self.debug("No email address found for target domain") break try: data = json.loads(data) except Exception: self.debug("No email address found for target domain") break evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) records = data.get('emails') lastId = data.get('lastId') if records: for record in records: if record: email = str(record.get('email')) if email: if email in self.results: continue if not SpiderFootHelpers.validEmail(email): continue self.results[email] = True if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) # Determine whether another page of data exists if len(records) < self.limit: nextPageHasData = False currentLastId = lastId
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_builtwith but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.queryDomainInfo(eventData) if data is not None: if "Meta" in data: if data['Meta'].get("Names", []): for nb in data['Meta']['Names']: e = SpiderFootEvent( "RAW_RIR_DATA", "Possible full name: " + nb['Name'], self.__name__, event) self.notifyListeners(e) if nb.get('Email', None): if SpiderFootHelpers.validEmail(nb['Email']): if nb['Email'].split("@")[0] in self.opts[ '_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, nb['Email'], self.__name__, event) self.notifyListeners(e) if data['Meta'].get("Emails", []): for email in data['Meta']['Emails']: if SpiderFootHelpers.validEmail(email): if email.split("@")[0] in self.opts[ '_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(e) if data['Meta'].get("Telephones", []): for phone in data['Meta']['Telephones']: phone = phone.replace("-", "").replace("(", "").replace( ")", "").replace(" ", "") e = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event) self.notifyListeners(e) if "Paths" in data.get("Result", []): for p in data["Result"]['Paths']: if p.get("SubDomain", ""): h = p["SubDomain"] + "." + eventData ev = SpiderFootEvent("INTERNET_NAME", h, self.__name__, event) self.notifyListeners(ev) if self.sf.isDomain(h, self.opts['_internettlds']): ev = SpiderFootEvent("DOMAIN_NAME", h, self.__name__, event) self.notifyListeners(ev) else: ev = None # If we have a subdomain, let's get its tech info # and associate it with the subdomain event. for t in p.get("Technologies", []): if ev: src = ev else: src = event agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) if t.get("LastDetected", 0) < agelimit: self.debug("Data found too old, skipping.") continue e = SpiderFootEvent("WEBSERVER_TECHNOLOGY", t["Name"], self.__name__, src) self.notifyListeners(e) data = self.queryRelationships(eventData) if data is None: return agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) for r in data: if "Domain" not in r or "Identifiers" not in r: self.debug("Data returned not in the format requested.") continue if r['Domain'] != eventData: self.debug( "Data returned doesn't match data requested, skipping.") continue for i in r['Identifiers']: if "Last" not in i or "Type" not in i or "Value" not in i: self.debug("Data returned not in the format requested.") continue if i['Last'] < agelimit: self.debug("Data found too old, skipping.") continue evttype = None # Related through shared IP if i['Type'] == "ip": if self.sf.validIP(i['Value']): val = i['Value'] evttype = "IP_ADDRESS" else: val = i['Value'].strip(".") if self.getTarget.matches(val): evttype = "INTERNET_NAME" else: evttype = "CO_HOSTED_SITE" # Create the name/co-host e = SpiderFootEvent(evttype, val, self.__name__, event) self.notifyListeners(e) continue # Related through shared analytics ID txt = i['Type'] + ": " + str(i['Value']) e = SpiderFootEvent("WEB_ANALYTICS_ID", txt, self.__name__, event) self.notifyListeners(e) if i['Matches']: for m in i['Matches']: if "Domain" not in m: continue evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME", m['Domain'], self.__name__, e) self.notifyListeners(evt) if self.sf.isDomain(m['Domain'], self.opts['_internettlds']): evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", m['Domain'], self.__name__, e) self.notifyListeners(evt)
def retrieve_domain_info(self, event): eventData = event.data if event.module == "sfp_spyse" and (self.getTarget().targetValue in event.data): return domain_details = self.queryDomainDetails(eventData) if not domain_details: return domain_details_data = domain_details.get("data", {}).get("items", []) if len(domain_details_data) == 0: return domain_item = domain_details_data[0] if domain_item.get("organizations"): for org in domain_item.get("organizations", []): crunchbase = org.get('crunchbase') if crunchbase: if crunchbase.get("is_primary", False): org_name = crunchbase.get("legal_name") if not org_name: org_name = crunchbase.get("name") if org_name: evt = SpiderFootEvent('COMPANY_NAME', org_name, self.__name__, event) self.notifyListeners(evt) domain_dns = domain_item.get("dns_records") if domain_dns: domain_dns_a_records = domain_dns.get("A") if domain_dns_a_records: for dns_A in domain_dns_a_records: if dns_A: evt = SpiderFootEvent('IP_ADDRESS', dns_A, self.__name__, event) self.notifyListeners(evt) domain_dns_aaaa_records = domain_dns.get("AAAA") if domain_dns_aaaa_records: for dns_AAAA in domain_dns_aaaa_records: if dns_AAAA: evt = SpiderFootEvent('IPV6_ADDRESS', dns_AAAA, self.__name__, event) self.notifyListeners(evt) domain_dns_spf_records = domain_dns.get("SPF") if domain_dns_spf_records: for dns_spf in domain_dns_spf_records: if dns_spf: dns_spf_raw = dns_spf.get("raw") if dns_spf_raw: evt = SpiderFootEvent('DNS_SPF', dns_spf_raw, self.__name__, event) self.notifyListeners(evt) domain_dns_txt_records = domain_dns.get("TXT") if domain_dns_txt_records: for dns_txt in domain_dns_txt_records: if dns_txt: evt = SpiderFootEvent('DNS_TEXT', dns_txt, self.__name__, event) self.notifyListeners(evt) domain_dns_ns_records = domain_dns.get("NS") if domain_dns_ns_records: for dns_ns in domain_dns_ns_records: if dns_ns: evt = SpiderFootEvent('PROVIDER_DNS', dns_ns, self.__name__, event) self.notifyListeners(evt) domain_dns_mx_records = domain_dns.get("MX") if domain_dns_mx_records: for dns_mx in domain_dns_mx_records: if dns_mx: evt = SpiderFootEvent('PROVIDER_MAIL', dns_mx, self.__name__, event) self.notifyListeners(evt) hosts_enrichment = domain_item.get("hosts_enrichment") if hosts_enrichment: for host_enrichment in hosts_enrichment: city = host_enrichment.get("city") country = host_enrichment.get("country") location = ', '.join([_f for _f in [city, country] if _f]) if location: evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) domain_technologies = domain_item.get("technologies") if domain_technologies: for tech in domain_technologies: name = tech.get("name") if not name: continue version = tech.get("version") software = ' '.join(filter(None, [name, version])) if software: evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', f"{software}", self.__name__, event) self.notifyListeners(evt) domain_cves = domain_item.get("cve_list") if domain_cves: for cve in domain_cves: cve_id = cve.get('id') if cve_id: etype, cvetext = self.sf.cveInfo(cve_id) evt = SpiderFootEvent(etype, cvetext, self.__name__, event) self.notifyListeners(evt) domain_whois = domain_item.get("whois_parsed") if domain_whois: domain_whois_registrar = domain_whois.get("registrar") if domain_whois_registrar: registrar_name = domain_whois_registrar.get("registrar_name") if registrar_name: evt = SpiderFootEvent('DOMAIN_REGISTRAR', registrar_name, self.__name__, event) self.notifyListeners(evt) domain_http_extract = domain_item.get("http_extract") if domain_http_extract: http_status_code = domain_http_extract.get("http_status_code") if http_status_code: evt = SpiderFootEvent('HTTP_CODE', str(http_status_code), self.__name__, event) self.notifyListeners(evt) http_headers = domain_http_extract.get("http_headers") if http_headers: evt = SpiderFootEvent('WEBSERVER_HTTPHEADERS', str(http_headers), self.__name__, event) self.notifyListeners(evt) domain_emails = domain_http_extract.get("emails") if domain_emails: for email in domain_emails: if SpiderFootHelpers.validEmail(email): evt = SpiderFootEvent('EMAILADDR', email, self.__name__, event) self.notifyListeners(evt) domain_cert_summary = domain_item.get("cert_summary") if domain_cert_summary: domain_cert_summary_subject = domain_cert_summary.get("subject") if domain_cert_summary_subject: cert_issued = domain_cert_summary_subject.get("organization") if cert_issued: evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED', cert_issued, self.__name__, event) self.notifyListeners(evt) domain_cert_summary_issuer = domain_cert_summary.get("issuer") if domain_cert_summary_issuer: cert_issuer = domain_cert_summary_issuer.get("organization") if cert_issuer: evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER', cert_issuer, self.__name__, event) self.notifyListeners(evt) domain_trackers = domain_item.get("trackers") if domain_trackers: google_analytics_key = domain_trackers.get("google_analytics_key") if google_analytics_key: evt = SpiderFootEvent( "WEB_ANALYTICS_ID", f"Google Analytics: {google_analytics_key}", self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") data = self.query(eventData) if data is None: self.debug("No user information found for " + eventData) return evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) if data.get('preferredUsername') is not None: un = data.get('preferredUsername') evt = SpiderFootEvent("USERNAME", un, self.__name__, event) self.notifyListeners(evt) self.reportedUsers[un] = True names = list() if data.get('name') is not None: if type(data.get('name')) != list: names.append(data.get('name')) else: names = data.get('name') for name in names: full_name = name.get('formatted') if full_name: evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event) self.notifyListeners(evt) # TODO: re-enable once location validation is implemented # location can not be trusted # if data.get('currentLocation') is not None: # location = data.get('currentLocation') # if len(location) < 3 or len(location) > 100: # self.debug("Skipping likely invalid location.") # else: # evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) # self.notifyListeners(evt) if data.get('phoneNumbers') is not None: for number in data.get('phoneNumbers'): if number.get('value') is not None: evt = SpiderFootEvent("PHONE_NUMBER", number.get('value'), self.__name__, event) self.notifyListeners(evt) if data.get('emails') is not None: for email in data.get('emails'): em = email.get('value') if not em: continue if SpiderFootHelpers.validEmail(em) and em != eventData: if em.split("@")[0] in self.opts['_genericusers'].split( ","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, em, self.__name__, event) self.notifyListeners(evt) if data.get('ims') is not None: for im in data.get('ims'): v = im.get('value') if v is None: continue t = im.get('type').capitalize() + " (Instant Messenger)\n" + v evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", t, self.__name__, event) self.notifyListeners(evt) if v not in self.reportedUsers: evt = SpiderFootEvent("USERNAME", v, self.__name__, event) self.notifyListeners(evt) self.reportedUsers[v] = True if data.get('accounts') is not None: for account in data.get('accounts'): url = account.get('url') platform = account.get('shortname') if platform is not None and url is not None: t = platform.capitalize() + ": <SFURL>" + url + "</SFURL>" evt = SpiderFootEvent("SOCIAL_MEDIA", t, self.__name__, event) self.notifyListeners(evt)
def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return if self.opts['api_key'] == "": self.error("You enabled sfp_jsonwhoiscom but did not set an API key!") self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") res = self.queryDomain(eventData) if res is None: self.debug(f"No information found for domain {eventData}") return evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event) self.notifyListeners(evt) dns_providers = list() nameservers = res.get('nameservers') if nameservers: for nameserver in nameservers: if nameserver: nameserver_name = nameserver.get('name') if nameserver_name: dns_providers.append(nameserver_name) contacts = list() registrant_contacts = res.get('registrant_contacts') if registrant_contacts: for contact in registrant_contacts: contacts.append(contact) admin_contacts = res.get('admin_contacts') if admin_contacts: for contact in admin_contacts: contacts.append(contact) technical_contacts = res.get('technical_contacts') if technical_contacts: for contact in technical_contacts: contacts.append(contact) emails = list() names = list() phones = list() locations = list() for contact in contacts: email = contact.get('email') if email: if SpiderFootHelpers.validEmail(email): emails.append(email) name = contact.get("name") if name: names.append(name) phone = contact.get('phone') if phone: phone = phone.replace(" ", "").replace("-", "").replace("(", "").replace(")", "").replace(".", "") phones.append(phone) country = SpiderFootHelpers.countryNameFromCountryCode(contact.get('country_code')) location = ', '.join([_f for _f in [contact.get('address'), contact.get('city'), contact.get('state'), contact.get('zip'), country] if _f]) if location: locations.append(location) for email in set(emails): mail_domain = email.lower().split('@')[1] if self.getTarget().matches(mail_domain, includeChildren=True): if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent("AFFILIATE_EMAILADDR", email, self.__name__, event) self.notifyListeners(evt) if eventName in ["DOMAIN_NAME"]: raw = res.get('raw') if raw: evt = SpiderFootEvent("DOMAIN_WHOIS", raw, self.__name__, event) self.notifyListeners(evt) registrar = res.get("registrar") if registrar: registrar_name = registrar.get("name") if registrar_name: evt = SpiderFootEvent("DOMAIN_REGISTRAR", registrar_name, self.__name__, event) self.notifyListeners(evt) for dns_provider in set(dns_providers): evt = SpiderFootEvent("PROVIDER_DNS", dns_provider, self.__name__, event) self.notifyListeners(evt) for name in set(names): evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name {name}", self.__name__, event) self.notifyListeners(evt) for phone in set(phones): evt = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event) self.notifyListeners(evt) for location in set(locations): evt = SpiderFootEvent("PHYSICAL_ADDRESS", location, self.__name__, event) self.notifyListeners(evt) if eventName in ["AFFILIATE_DOMAIN_NAME"]: raw = res.get('raw') if raw: evt = SpiderFootEvent("AFFILIATE_DOMAIN_WHOIS", raw, self.__name__, event) self.notifyListeners(evt) available = res.get('available?') if available: evt = SpiderFootEvent("AFFILIATE_DOMAIN_UNREGISTERED", eventData, self.__name__, event) self.notifyListeners(evt)