def test_valid_email_should_return_a_boolean(self):
invalid_types = [None, "", list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
valid_email = SpiderFootHelpers.validEmail(invalid_type)
self.assertIsInstance(valid_email, bool)
self.assertFalse(valid_email)
valid_email = SpiderFootHelpers.validEmail('*****@*****.**')
self.assertIsInstance(valid_email, bool)
self.assertFalse(valid_email)
valid_email = SpiderFootHelpers.validEmail('*****@*****.**')
self.assertIsInstance(valid_email, bool)
self.assertFalse(valid_email)
valid_email = SpiderFootHelpers.validEmail(
'[email protected]\n.com')
self.assertIsInstance(valid_email, bool)
self.assertFalse(valid_email)
valid_email = SpiderFootHelpers.validEmail('root@localhost')
self.assertIsInstance(valid_email, bool)
self.assertFalse(valid_email)
valid_email = SpiderFootHelpers.validEmail('*****@*****.**')
self.assertIsInstance(valid_email, bool)
self.assertTrue(valid_email)
def queryEmail(self, email):
if not SpiderFootHelpers.validEmail(email):
return None
params = urllib.parse.urlencode({
'mail': email,
'key': self.opts['api_key'],
})
res = self.sf.fetchUrl(
f"https://botscout.com/test/?{params}",
timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'],
)
return self.parseApiResponse(res)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if eventData in self.results:
return
if self.errorState:
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {srcModuleName}")
position = 0
max_pages = int(self.opts['max_pages'])
per_page = int(self.opts['per_page'])
emails = list()
hashes = list()
passwords = list()
while position < (per_page * max_pages):
if self.checkForStop():
return
if self.errorState:
break
data = self.query(eventData, per_page, position)
if not data:
return
position += per_page
# evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
# self.notifyListeners(evt)
for row in data:
result = row.get('fields')
if not result:
continue
email = result.get('email')
# A blank email result should not be possible, as we searched using the 'Email:' filter
if not email:
continue
if not SpiderFootHelpers.validEmail(email):
self.debug("Skipping invalid email address: " + email)
continue
mailDom = email.lower().split('@')[1]
# Skip unrelated emails
# Scylla sometimes returns broader results than the searched data
if not self.getTarget().matches(mailDom):
self.debug("Skipped address: " + email)
continue
breach = result.get('domain', 'Unknown')
emails.append(email + " [" + breach + "]")
pass_hash = result.get('passhash')
if pass_hash:
pass_salt = result.get('passsalt')
if pass_salt:
hashes.append(email + ':' + pass_hash + " (Salt: " + pass_salt + ") [" + breach + "]")
else:
hashes.append(email + ':' + pass_hash + " [" + breach + "]")
password = result.get('password')
if password:
passwords.append(email + ':' + password + " [" + breach + "]")
if len(data) < per_page:
break
for pass_hash in set(hashes):
evt = SpiderFootEvent('HASH_COMPROMISED', pass_hash, self.__name__, event)
self.notifyListeners(evt)
for email in set(emails):
evt = SpiderFootEvent('EMAILADDR_COMPROMISED', email, self.__name__, event)
self.notifyListeners(evt)
for password in set(passwords):
evt = SpiderFootEvent('PASSWORD_COMPROMISED', password, self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
# Always check if the API key is set and complain if it isn't, then set
# self.errorState to avoid this being a continual complaint during the scan.
if self.opts['api_key_client_id'] == "" or self.opts['api_key_client_secret'] == "":
self.error("You enabled sfp_snov but did not set a Client ID and/or Client Secret")
self.errorState = True
return
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
# Get access token from Snov IO API
accessToken = self.queryAccessToken()
if accessToken is None or accessToken == '':
self.error("No access token received from snov.io for the provided Client ID and/or Client Secret")
self.errorState = True
return
currentLastId = 0
nextPageHasData = True
while nextPageHasData:
if self.checkForStop():
return
data = self.queryDomainName(eventData, accessToken, currentLastId)
if data is None:
self.debug("No email address found for target domain")
break
try:
data = json.loads(data)
except Exception:
self.debug("No email address found for target domain")
break
evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event)
self.notifyListeners(evt)
records = data.get('emails')
lastId = data.get('lastId')
if records:
for record in records:
if record:
email = str(record.get('email'))
if email:
if email in self.results:
continue
if not SpiderFootHelpers.validEmail(email):
continue
self.results[email] = True
if email.split("@")[0] in self.opts['_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
# Determine whether another page of data exists
if len(records) < self.limit:
nextPageHasData = False
currentLastId = lastId
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['api_key'] == "":
self.error("You enabled sfp_builtwith but did not set an API key!")
self.errorState = True
return
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
data = self.queryDomainInfo(eventData)
if data is not None:
if "Meta" in data:
if data['Meta'].get("Names", []):
for nb in data['Meta']['Names']:
e = SpiderFootEvent(
"RAW_RIR_DATA",
"Possible full name: " + nb['Name'], self.__name__,
event)
self.notifyListeners(e)
if nb.get('Email', None):
if SpiderFootHelpers.validEmail(nb['Email']):
if nb['Email'].split("@")[0] in self.opts[
'_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
e = SpiderFootEvent(evttype, nb['Email'],
self.__name__, event)
self.notifyListeners(e)
if data['Meta'].get("Emails", []):
for email in data['Meta']['Emails']:
if SpiderFootHelpers.validEmail(email):
if email.split("@")[0] in self.opts[
'_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
e = SpiderFootEvent(evttype, email, self.__name__,
event)
self.notifyListeners(e)
if data['Meta'].get("Telephones", []):
for phone in data['Meta']['Telephones']:
phone = phone.replace("-",
"").replace("(", "").replace(
")", "").replace(" ", "")
e = SpiderFootEvent("PHONE_NUMBER", phone,
self.__name__, event)
self.notifyListeners(e)
if "Paths" in data.get("Result", []):
for p in data["Result"]['Paths']:
if p.get("SubDomain", ""):
h = p["SubDomain"] + "." + eventData
ev = SpiderFootEvent("INTERNET_NAME", h, self.__name__,
event)
self.notifyListeners(ev)
if self.sf.isDomain(h, self.opts['_internettlds']):
ev = SpiderFootEvent("DOMAIN_NAME", h,
self.__name__, event)
self.notifyListeners(ev)
else:
ev = None
# If we have a subdomain, let's get its tech info
# and associate it with the subdomain event.
for t in p.get("Technologies", []):
if ev:
src = ev
else:
src = event
agelimit = int(time.time() *
1000) - (86400000 * self.opts['maxage'])
if t.get("LastDetected", 0) < agelimit:
self.debug("Data found too old, skipping.")
continue
e = SpiderFootEvent("WEBSERVER_TECHNOLOGY", t["Name"],
self.__name__, src)
self.notifyListeners(e)
data = self.queryRelationships(eventData)
if data is None:
return
agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])
for r in data:
if "Domain" not in r or "Identifiers" not in r:
self.debug("Data returned not in the format requested.")
continue
if r['Domain'] != eventData:
self.debug(
"Data returned doesn't match data requested, skipping.")
continue
for i in r['Identifiers']:
if "Last" not in i or "Type" not in i or "Value" not in i:
self.debug("Data returned not in the format requested.")
continue
if i['Last'] < agelimit:
self.debug("Data found too old, skipping.")
continue
evttype = None
# Related through shared IP
if i['Type'] == "ip":
if self.sf.validIP(i['Value']):
val = i['Value']
evttype = "IP_ADDRESS"
else:
val = i['Value'].strip(".")
if self.getTarget.matches(val):
evttype = "INTERNET_NAME"
else:
evttype = "CO_HOSTED_SITE"
# Create the name/co-host
e = SpiderFootEvent(evttype, val, self.__name__, event)
self.notifyListeners(e)
continue
# Related through shared analytics ID
txt = i['Type'] + ": " + str(i['Value'])
e = SpiderFootEvent("WEB_ANALYTICS_ID", txt, self.__name__,
event)
self.notifyListeners(e)
if i['Matches']:
for m in i['Matches']:
if "Domain" not in m:
continue
evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME",
m['Domain'], self.__name__, e)
self.notifyListeners(evt)
if self.sf.isDomain(m['Domain'],
self.opts['_internettlds']):
evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME",
m['Domain'], self.__name__,
e)
self.notifyListeners(evt)
def retrieve_domain_info(self, event):
eventData = event.data
if event.module == "sfp_spyse" and (self.getTarget().targetValue
in event.data):
return
domain_details = self.queryDomainDetails(eventData)
if not domain_details:
return
domain_details_data = domain_details.get("data", {}).get("items", [])
if len(domain_details_data) == 0:
return
domain_item = domain_details_data[0]
if domain_item.get("organizations"):
for org in domain_item.get("organizations", []):
crunchbase = org.get('crunchbase')
if crunchbase:
if crunchbase.get("is_primary", False):
org_name = crunchbase.get("legal_name")
if not org_name:
org_name = crunchbase.get("name")
if org_name:
evt = SpiderFootEvent('COMPANY_NAME', org_name,
self.__name__, event)
self.notifyListeners(evt)
domain_dns = domain_item.get("dns_records")
if domain_dns:
domain_dns_a_records = domain_dns.get("A")
if domain_dns_a_records:
for dns_A in domain_dns_a_records:
if dns_A:
evt = SpiderFootEvent('IP_ADDRESS', dns_A,
self.__name__, event)
self.notifyListeners(evt)
domain_dns_aaaa_records = domain_dns.get("AAAA")
if domain_dns_aaaa_records:
for dns_AAAA in domain_dns_aaaa_records:
if dns_AAAA:
evt = SpiderFootEvent('IPV6_ADDRESS', dns_AAAA,
self.__name__, event)
self.notifyListeners(evt)
domain_dns_spf_records = domain_dns.get("SPF")
if domain_dns_spf_records:
for dns_spf in domain_dns_spf_records:
if dns_spf:
dns_spf_raw = dns_spf.get("raw")
if dns_spf_raw:
evt = SpiderFootEvent('DNS_SPF', dns_spf_raw,
self.__name__, event)
self.notifyListeners(evt)
domain_dns_txt_records = domain_dns.get("TXT")
if domain_dns_txt_records:
for dns_txt in domain_dns_txt_records:
if dns_txt:
evt = SpiderFootEvent('DNS_TEXT', dns_txt,
self.__name__, event)
self.notifyListeners(evt)
domain_dns_ns_records = domain_dns.get("NS")
if domain_dns_ns_records:
for dns_ns in domain_dns_ns_records:
if dns_ns:
evt = SpiderFootEvent('PROVIDER_DNS', dns_ns,
self.__name__, event)
self.notifyListeners(evt)
domain_dns_mx_records = domain_dns.get("MX")
if domain_dns_mx_records:
for dns_mx in domain_dns_mx_records:
if dns_mx:
evt = SpiderFootEvent('PROVIDER_MAIL', dns_mx,
self.__name__, event)
self.notifyListeners(evt)
hosts_enrichment = domain_item.get("hosts_enrichment")
if hosts_enrichment:
for host_enrichment in hosts_enrichment:
city = host_enrichment.get("city")
country = host_enrichment.get("country")
location = ', '.join([_f for _f in [city, country] if _f])
if location:
evt = SpiderFootEvent('GEOINFO', location, self.__name__,
event)
self.notifyListeners(evt)
domain_technologies = domain_item.get("technologies")
if domain_technologies:
for tech in domain_technologies:
name = tech.get("name")
if not name:
continue
version = tech.get("version")
software = ' '.join(filter(None, [name, version]))
if software:
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY',
f"{software}", self.__name__, event)
self.notifyListeners(evt)
domain_cves = domain_item.get("cve_list")
if domain_cves:
for cve in domain_cves:
cve_id = cve.get('id')
if cve_id:
etype, cvetext = self.sf.cveInfo(cve_id)
evt = SpiderFootEvent(etype, cvetext, self.__name__, event)
self.notifyListeners(evt)
domain_whois = domain_item.get("whois_parsed")
if domain_whois:
domain_whois_registrar = domain_whois.get("registrar")
if domain_whois_registrar:
registrar_name = domain_whois_registrar.get("registrar_name")
if registrar_name:
evt = SpiderFootEvent('DOMAIN_REGISTRAR', registrar_name,
self.__name__, event)
self.notifyListeners(evt)
domain_http_extract = domain_item.get("http_extract")
if domain_http_extract:
http_status_code = domain_http_extract.get("http_status_code")
if http_status_code:
evt = SpiderFootEvent('HTTP_CODE', str(http_status_code),
self.__name__, event)
self.notifyListeners(evt)
http_headers = domain_http_extract.get("http_headers")
if http_headers:
evt = SpiderFootEvent('WEBSERVER_HTTPHEADERS',
str(http_headers), self.__name__, event)
self.notifyListeners(evt)
domain_emails = domain_http_extract.get("emails")
if domain_emails:
for email in domain_emails:
if SpiderFootHelpers.validEmail(email):
evt = SpiderFootEvent('EMAILADDR', email,
self.__name__, event)
self.notifyListeners(evt)
domain_cert_summary = domain_item.get("cert_summary")
if domain_cert_summary:
domain_cert_summary_subject = domain_cert_summary.get("subject")
if domain_cert_summary_subject:
cert_issued = domain_cert_summary_subject.get("organization")
if cert_issued:
evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED',
cert_issued, self.__name__, event)
self.notifyListeners(evt)
domain_cert_summary_issuer = domain_cert_summary.get("issuer")
if domain_cert_summary_issuer:
cert_issuer = domain_cert_summary_issuer.get("organization")
if cert_issuer:
evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER',
cert_issuer, self.__name__, event)
self.notifyListeners(evt)
domain_trackers = domain_item.get("trackers")
if domain_trackers:
google_analytics_key = domain_trackers.get("google_analytics_key")
if google_analytics_key:
evt = SpiderFootEvent(
"WEB_ANALYTICS_ID",
f"Google Analytics: {google_analytics_key}", self.__name__,
event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if eventData in self.results:
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {srcModuleName}")
data = self.query(eventData)
if data is None:
self.debug("No user information found for " + eventData)
return
evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event)
self.notifyListeners(evt)
if data.get('preferredUsername') is not None:
un = data.get('preferredUsername')
evt = SpiderFootEvent("USERNAME", un, self.__name__, event)
self.notifyListeners(evt)
self.reportedUsers[un] = True
names = list()
if data.get('name') is not None:
if type(data.get('name')) != list:
names.append(data.get('name'))
else:
names = data.get('name')
for name in names:
full_name = name.get('formatted')
if full_name:
evt = SpiderFootEvent("RAW_RIR_DATA",
f"Possible full name: {full_name}",
self.__name__, event)
self.notifyListeners(evt)
# TODO: re-enable once location validation is implemented
# location can not be trusted
# if data.get('currentLocation') is not None:
# location = data.get('currentLocation')
# if len(location) < 3 or len(location) > 100:
# self.debug("Skipping likely invalid location.")
# else:
# evt = SpiderFootEvent("GEOINFO", location, self.__name__, event)
# self.notifyListeners(evt)
if data.get('phoneNumbers') is not None:
for number in data.get('phoneNumbers'):
if number.get('value') is not None:
evt = SpiderFootEvent("PHONE_NUMBER", number.get('value'),
self.__name__, event)
self.notifyListeners(evt)
if data.get('emails') is not None:
for email in data.get('emails'):
em = email.get('value')
if not em:
continue
if SpiderFootHelpers.validEmail(em) and em != eventData:
if em.split("@")[0] in self.opts['_genericusers'].split(
","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
evt = SpiderFootEvent(evttype, em, self.__name__, event)
self.notifyListeners(evt)
if data.get('ims') is not None:
for im in data.get('ims'):
v = im.get('value')
if v is None:
continue
t = im.get('type').capitalize() + " (Instant Messenger)\n" + v
evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", t,
self.__name__, event)
self.notifyListeners(evt)
if v not in self.reportedUsers:
evt = SpiderFootEvent("USERNAME", v, self.__name__, event)
self.notifyListeners(evt)
self.reportedUsers[v] = True
if data.get('accounts') is not None:
for account in data.get('accounts'):
url = account.get('url')
platform = account.get('shortname')
if platform is not None and url is not None:
t = platform.capitalize() + ": <SFURL>" + url + "</SFURL>"
evt = SpiderFootEvent("SOCIAL_MEDIA", t, self.__name__,
event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
if eventData in self.results:
return
if self.opts['api_key'] == "":
self.error("You enabled sfp_jsonwhoiscom but did not set an API key!")
self.errorState = True
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {srcModuleName}")
res = self.queryDomain(eventData)
if res is None:
self.debug(f"No information found for domain {eventData}")
return
evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event)
self.notifyListeners(evt)
dns_providers = list()
nameservers = res.get('nameservers')
if nameservers:
for nameserver in nameservers:
if nameserver:
nameserver_name = nameserver.get('name')
if nameserver_name:
dns_providers.append(nameserver_name)
contacts = list()
registrant_contacts = res.get('registrant_contacts')
if registrant_contacts:
for contact in registrant_contacts:
contacts.append(contact)
admin_contacts = res.get('admin_contacts')
if admin_contacts:
for contact in admin_contacts:
contacts.append(contact)
technical_contacts = res.get('technical_contacts')
if technical_contacts:
for contact in technical_contacts:
contacts.append(contact)
emails = list()
names = list()
phones = list()
locations = list()
for contact in contacts:
email = contact.get('email')
if email:
if SpiderFootHelpers.validEmail(email):
emails.append(email)
name = contact.get("name")
if name:
names.append(name)
phone = contact.get('phone')
if phone:
phone = phone.replace(" ", "").replace("-", "").replace("(", "").replace(")", "").replace(".", "")
phones.append(phone)
country = SpiderFootHelpers.countryNameFromCountryCode(contact.get('country_code'))
location = ', '.join([_f for _f in [contact.get('address'), contact.get('city'), contact.get('state'), contact.get('zip'), country] if _f])
if location:
locations.append(location)
for email in set(emails):
mail_domain = email.lower().split('@')[1]
if self.getTarget().matches(mail_domain, includeChildren=True):
if email.split("@")[0] in self.opts['_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADR"
evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent("AFFILIATE_EMAILADDR", email, self.__name__, event)
self.notifyListeners(evt)
if eventName in ["DOMAIN_NAME"]:
raw = res.get('raw')
if raw:
evt = SpiderFootEvent("DOMAIN_WHOIS", raw, self.__name__, event)
self.notifyListeners(evt)
registrar = res.get("registrar")
if registrar:
registrar_name = registrar.get("name")
if registrar_name:
evt = SpiderFootEvent("DOMAIN_REGISTRAR", registrar_name, self.__name__, event)
self.notifyListeners(evt)
for dns_provider in set(dns_providers):
evt = SpiderFootEvent("PROVIDER_DNS", dns_provider, self.__name__, event)
self.notifyListeners(evt)
for name in set(names):
evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name {name}", self.__name__, event)
self.notifyListeners(evt)
for phone in set(phones):
evt = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event)
self.notifyListeners(evt)
for location in set(locations):
evt = SpiderFootEvent("PHYSICAL_ADDRESS", location, self.__name__, event)
self.notifyListeners(evt)
if eventName in ["AFFILIATE_DOMAIN_NAME"]:
raw = res.get('raw')
if raw:
evt = SpiderFootEvent("AFFILIATE_DOMAIN_WHOIS", raw, self.__name__, event)
self.notifyListeners(evt)
available = res.get('available?')
if available:
evt = SpiderFootEvent("AFFILIATE_DOMAIN_UNREGISTERED", eventData, self.__name__, event)
self.notifyListeners(evt)
