def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['api_key'] == "":
self.sf.error("You enabled sfp_securitytrails but did not set an API uid/secret!", False)
self.errorState = True
return None
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
else:
self.results[eventData] = True
if eventName in ["IP_ADDRESS", "IPV6_ADDRESS", "NETLBLOCK_OWNER"]:
ip = eventData
rec = self.query(ip, "ipv4")
myres = list()
hosters = list()
if rec is not None:
for r in rec:
if "host_provider" in r:
for dat in r['host_provider']:
if dat in hosters:
continue
e = SpiderFootEvent("PROVIDER_HOSTING", dat,
self.__name__, event)
self.notifyListeners(e)
hosters.append(dat)
if "hostname" in r:
if self.cohostcount > self.opts['maxcohost']:
continue
h = r['hostname']
if not self.opts['cohostsamedomain']:
if self.getTarget().matches(h, includeParents=True):
self.sf.debug("Skipping " + h + " because it is on the same domain.")
continue
if h not in myres and h != ip:
if self.opts['verify'] and not self.sf.validateIP(h, ip):
self.sf.debug("Host " + h + " no longer resolves to our IP.")
continue
myres.append(h.lower())
e = SpiderFootEvent("CO_HOSTED_SITE", h, self.__name__, event)
self.notifyListeners(e)
self.cohostcount += 1
if eventName in ["EMAILADDR"]:
email = eventData
rec = self.query(email, "whois_email")
myres = list()
if rec is not None:
for r in rec:
if "hostname" in r:
if not r['hostname']:
continue
h = r['hostname']
if h.lower() not in myres:
myres.append(h.lower())
else:
continue
e = SpiderFootEvent("AFFILIATE_INTERNET_NAME", h, self.__name__, event)
self.notifyListeners(e)
if self.sf.isDomain(h, self.opts['_internettlds']):
evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", h, self.__name__, event)
self.notifyListeners(evt)
if eventName in ["DOMAIN_NAME"]:
domain = eventData
rec = self.query(domain, "domain")
myres = list()
if rec is not None:
for h in rec:
if h == "":
continue
if h.lower() not in myres:
myres.append(h.lower())
else:
continue
e = SpiderFootEvent("INTERNET_NAME", h + "." + domain,
self.__name__, event)
self.notifyListeners(e)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
if eventData in self.results:
return
self.results[eventData] = True
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts["api_key"] == "":
self.sf.error(
f"You enabled {self.__class__.__name__} but did not set an API key!"
)
self.errorState = True
return
if eventName not in ["DOMAIN_NAME"]:
return
data = self.queryDomain(eventData)
if data is None:
self.sf.debug(f"No information found for domain {eventData}")
return
evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
self.notifyListeners(evt)
domains = []
for result in data:
raw_domains = result.get('rawDomains')
if raw_domains:
for domain in raw_domains:
domains.append(domain)
for domain in set(domains):
if self.checkForStop():
return
if domain in self.results:
continue
if not self.getTarget().matches(
domain, includeChildren=True, includeParents=True):
continue
if self.opts['verify'] and not self.sf.resolveHost(domain):
self.sf.debug(f"Host {domain} could not be resolved")
evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain,
self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__,
event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
if self.opts['api_key'] == "":
self.error("You enabled sfp_hunter but did not set an API key!")
self.errorState = True
return
data = self.query(eventData, 0, 10)
if not data:
return
if "data" not in data:
return
# Check if we have more results on further pages
if "meta" in data:
maxgoal = data['meta'].get('results', 10)
else:
maxgoal = 10
rescount = len(data['data'].get('emails', list()))
while rescount <= maxgoal:
for email in data['data'].get('emails', list()):
# Notify other modules of what you've found
em = email.get('value')
if not em:
continue
if em.split("@")[0] in self.opts['_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
e = SpiderFootEvent(evttype, em, self.__name__, event)
self.notifyListeners(e)
if 'first_name' in email and 'last_name' in email:
if email['first_name'] is not None and email[
'last_name'] is not None:
n = email['first_name'] + " " + email['last_name']
e = SpiderFootEvent("RAW_RIR_DATA",
"Possible full name: " + n,
self.__name__, event)
self.notifyListeners(e)
if rescount >= maxgoal:
return
data = self.query(eventData, rescount, 10)
if data is None:
return
if "data" not in data:
return
rescount += len(data['data'].get('emails', list()))
def handleEvent(self, event):
eventName = event.eventType
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {event.module}")
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
if eventName == 'AFFILIATE_IPADDR':
if not self.opts.get('checkaffiliates', False):
return
malicious_type = "MALICIOUS_AFFILIATE_IPADDR"
blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR"
elif eventName == 'IP_ADDRESS':
malicious_type = "MALICIOUS_IPADDR"
blacklist_type = "BLACKLISTED_IPADDR"
elif eventName == 'NETBLOCK_MEMBER':
if not self.opts['subnetlookup']:
return
max_subnet = self.opts['maxsubnet']
if IPNetwork(eventData).prefixlen < max_subnet:
self.debug(
f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}"
)
return
malicious_type = "MALICIOUS_SUBNET"
blacklist_type = "BLACKLISTED_SUBNET"
elif eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return
max_netblock = self.opts['maxnetblock']
if IPNetwork(eventData).prefixlen < max_netblock:
self.debug(
f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}"
)
return
malicious_type = "MALICIOUS_NETBLOCK"
blacklist_type = "BLACKLISTED_NETBLOCK"
else:
self.debug(f"Unexpected event type {eventName}, skipping")
return
qrylist = list()
if eventName.startswith("NETBLOCK"):
for ipaddr in IPNetwork(eventData):
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
else:
qrylist.append(eventData)
for addr in qrylist:
if self.checkForStop():
return
data = self.queryIPAddress(addr)
if not data:
continue
# TODO: iterate through hosts and extract co-hosts
try:
maliciousIP = data[0].get('ip')
except Exception:
# If ArrayIndex is out of bounds then data doesn't exist
continue
if not maliciousIP:
continue
if addr != maliciousIP:
self.error(
f"Reported address {maliciousIP} doesn't match queried IP address {addr}, skipping"
)
continue
# For netblocks, we need to create the IP address event so that
# the threat intel event is more meaningful.
if eventName == 'NETBLOCK_OWNER':
pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__,
event)
self.notifyListeners(pevent)
elif eventName == 'NETBLOCK_MEMBER':
pevent = SpiderFootEvent("AFFILIATE_IPADDR", addr,
self.__name__, event)
self.notifyListeners(pevent)
else:
pevent = event
evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__,
pevent)
self.notifyListeners(evt)
text = f"PhishStats [{addr}]"
evt = SpiderFootEvent(blacklist_type, text, self.__name__, pevent)
self.notifyListeners(evt)
evt = SpiderFootEvent(malicious_type, text, self.__name__, pevent)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['api_key'] == "":
self.sf.error(
"You enabled sfp_builtwith but did not set an API key!")
self.errorState = True
return None
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
self.results[eventData] = True
data = self.queryDomainInfo(eventData)
if data is not None:
if "Meta" in data:
if data['Meta'].get("Names", []):
for nb in data['Meta']['Names']:
e = SpiderFootEvent(
"RAW_RIR_DATA",
"Possible full name: " + nb['Name'], self.__name__,
event)
self.notifyListeners(e)
if nb.get('Email', None):
if self.sf.validEmail(nb['Email']):
if nb['Email'].split("@")[0] in self.opts[
'_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
e = SpiderFootEvent(evttype, nb['Email'],
self.__name__, event)
self.notifyListeners(e)
if data['Meta'].get("Emails", []):
for email in data['Meta']['Emails']:
if self.sf.validEmail(email):
if email.split("@")[0] in self.opts[
'_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
e = SpiderFootEvent(evttype, email, self.__name__,
event)
self.notifyListeners(e)
if data['Meta'].get("Telephones", []):
for phone in data['Meta']['Telephones']:
phone = phone.replace("-",
"").replace("(", "").replace(
")", "").replace(" ", "")
e = SpiderFootEvent("PHONE_NUMBER", phone,
self.__name__, event)
self.notifyListeners(e)
if "Paths" in data.get("Result", []):
for p in data["Result"]['Paths']:
if p.get("SubDomain", ""):
h = p["SubDomain"] + "." + eventData
ev = SpiderFootEvent("INTERNET_NAME", h, self.__name__,
event)
self.notifyListeners(ev)
if self.sf.isDomain(h, self.opts['_internettlds']):
ev = SpiderFootEvent("DOMAIN_NAME", h,
self.__name__, event)
self.notifyListeners(ev)
else:
ev = None
# If we have a subdomain, let's get its tech info
# and associate it with the subdomain event.
for t in p.get("Technologies", []):
if ev:
src = ev
else:
src = event
agelimit = int(time.time() *
1000) - (86400000 * self.opts['maxage'])
if t.get("LastDetected", 0) < agelimit:
self.sf.debug("Data found too old, skipping.")
continue
e = SpiderFootEvent("WEBSERVER_TECHNOLOGY", t["Name"],
self.__name__, src)
self.notifyListeners(e)
data = self.queryRelationships(eventData)
if data is None:
return None
agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])
for r in data:
if "Domain" not in r or "Identifiers" not in r:
self.sf.debug("Data returned not in the format requested.")
continue
if r['Domain'] != eventData:
self.sf.debug(
"Data returned doesn't match data requested, skipping.")
continue
for i in r['Identifiers']:
if "Last" not in i or "Type" not in i or "Value" not in i:
self.sf.debug("Data returned not in the format requested.")
continue
if i['Last'] < agelimit:
self.sf.debug("Data found too old, skipping.")
continue
evttype = None
# Related through shared IP
if i['Type'] == "ip":
if self.sf.validIP(i['Value']):
val = i['Value']
evttype = "IP_ADDRESS"
else:
val = i['Value'].strip(".")
if self.getTarget.matches(val):
evttype = "INTERNET_NAME"
else:
evttype = "CO_HOSTED_SITE"
# Create the name/co-host
e = SpiderFootEvent(evttype, val, self.__name__, event)
self.notifyListeners(e)
continue
# Related through shared analytics ID
txt = i['Type'] + ": " + str(i['Value'])
e = SpiderFootEvent("WEB_ANALYTICS_ID", txt, self.__name__,
event)
self.notifyListeners(e)
if i['Matches']:
for m in i['Matches']:
if "Domain" not in m:
continue
evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME",
m['Domain'], self.__name__, e)
self.notifyListeners(evt)
if self.sf.isDomain(m['Domain'],
self.opts['_internettlds']):
evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME",
m['Domain'], self.__name__,
e)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
self.results[eventData] = True
if not self.opts['wafw00f_path']:
self.error(
"You enabled sfp_tool_wafw00f but did not set a path to the tool!"
)
self.errorState = True
return
exe = self.opts['wafw00f_path']
if self.opts['wafw00f_path'].endswith('/'):
exe = exe + 'wafw00f'
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
url = eventData
if not SpiderFootHelpers.sanitiseInput(url):
self.error("Invalid input, refusing to run.")
return
args = [self.opts['python_path'], exe, '-a', '-o-', '-f', 'json', url]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
except Exception as e:
self.error(f"Unable to run wafw00f: {e}")
return
if p.returncode != 0:
self.error(
f"Unable to read wafw00f output\nstderr: {stderr}\nstdout: {stdout}"
)
return
if not stdout:
self.debug(f"wafw00f returned no output for {eventData}")
return
try:
result_json = json.loads(stdout)
except Exception as e:
self.error(
f"Could not parse wafw00f output as JSON: {e}\nstdout: {stdout}"
)
return
if not result_json:
self.debug(f"wafw00f returned no output for {eventData}")
return
evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(result_json),
self.__name__, event)
self.notifyListeners(evt)
for waf in result_json:
if not waf:
continue
firewall = waf.get('firewall')
if not firewall:
continue
if firewall == 'Generic':
continue
manufacturer = waf.get('manufacturer')
if not manufacturer:
continue
software = ' '.join(filter(None, [manufacturer, firewall]))
if software:
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', software,
self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
sourceData = self.sf.hashstring(eventData)
if sourceData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return
self.results[sourceData] = True
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if event.moduleDataSource:
datasource = event.moduleDataSource
else:
datasource = "Unknown"
if eventName == 'TARGET_WEB_CONTENT':
# Google Analytics
matches = re.findall(r"\bua\-\d{4,10}\-\d{1,4}\b", eventData,
re.IGNORECASE)
for m in matches:
if m.lower().startswith('ua-000000-'):
continue
if m.lower().startswith('ua-123456-'):
continue
if m.lower().startswith('ua-12345678'):
continue
self.sf.debug("Google Analytics match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Analytics: " + m, self.__name__,
event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Google AdSense
matches = re.findall(r"\b(pub-\d{10,20})\b", eventData,
re.IGNORECASE)
for m in matches:
if m.lower().startswith('pub-12345678'):
continue
self.sf.debug("Google AdSense match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google AdSense: " + m, self.__name__,
event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Google Website Verification
# https://developers.google.com/site-verification/v1/getting_started
matches = re.findall(
r'<meta name="google-site-verification" content="([a-z0-9\-\+_=]{43,44})"',
eventData, re.IGNORECASE)
for m in matches:
self.sf.debug("Google Site Verification match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
matches = re.findall(
r'<meta name="verify-v1" content="([a-z0-9\-\+_=]{43,44})"',
eventData, re.IGNORECASE)
for m in matches:
self.sf.debug("Google Site Verification match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Quantcast
if '_qevents.push' in eventData:
matches = re.findall(r"\bqacct:\"(p-[a-z0-9]+)\"", eventData,
re.IGNORECASE)
for m in matches:
self.sf.debug("Quantcast match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Quantcast: " + m, self.__name__,
event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Ahrefs Site Verification
matches = re.findall(
r'<meta name="ahrefs-site-verification" content="([a-f0-9]{64})"',
eventData, re.IGNORECASE)
for m in matches:
self.sf.debug("Ahrefs Site Verification match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Ahrefs Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
if eventName == 'DNS_TEXT':
# Google Website Verification
# https://developers.google.com/site-verification/v1/getting_started
matches = re.findall(
r'google-site-verification=([a-z0-9\-\+_=]{43,44})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# LogMeIn Domain Verification
# https://support.logmeininc.com/openvoice/help/adding-a-txt-record-to-a-dns-server-ov710011
matches = re.findall(
r'logmein-domain-confirmation ([A-Z0-9]{24})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"LogMeIn Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
matches = re.findall(
r'logmein-verification-code=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"LogMeIn Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# DocuSign Domain Verification
# https://support.docusign.com/en/guides/org-admin-guide-domains
matches = re.findall(
r'docusign=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"DocuSign Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# GlobalSign Site Verification
# https://support.globalsign.com/customer/en/portal/articles/2167245-performing-domain-verification---dns-txt-record
matches = re.findall(
r'globalsign-domain-verification=([a-z0-9\-\+_=]{42,44})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"GlobalSign Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Atlassian Domain Verification
# https://confluence.atlassian.com/cloud/verify-a-domain-for-your-organization-873871234.html
matches = re.findall(
r'atlassian-domain-verification=([a-z0-9\-\+\/_=]{64})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Atlassian Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Adobe IDP Site Verification
# https://helpx.adobe.com/au/enterprise/using/verify-domain-ownership.html
matches = re.findall(
r'adobe-idp-site-verification=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Adobe IDP Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
matches = re.findall(
r'adobe-idp-site-verification=([a-f0-9]{64})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Adobe IDP Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Adobe Domain Verification
# https://helpx.adobe.com/sign/help/domain_claiming.html
matches = re.findall(r'adobe-sign-verification=([a-f0-9]{32})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Adobe Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Stripe Domain Verification
# https://stripe.com/docs/apple-pay/web#going-live
matches = re.findall(r'stripe-verification=([a-f0-9]{64})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Stripe Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# TeamViewer SSO Verification
# https://community.teamviewer.com/t5/Knowledge-Base/Single-Sign-On-SSO/ta-p/30784
matches = re.findall(
r'teamviewer-sso-verification=([a-f0-9]{32})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"TeamViewer SSO Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Aliyun Site Verification
matches = re.findall(
r'aliyun-site-verification=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Aliyun Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Facebook Domain Verification
# https://developers.facebook.com/docs/sharing/domain-verification/
matches = re.findall(
r'facebook-domain-verification=([a-z0-9]{30})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Facebook Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Citrix Domain Verification
matches = re.findall(
r'citrix-verification-code=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Citrix Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Dropbox Domain Verification
# https://help.dropbox.com/teams-admins/admin/domain-insights-account-capture#verify
matches = re.findall(
r'dropbox-domain-verification=([a-z0-9]{12})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Dropbox Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Detectify Domain Verification
# https://support.detectify.com/customer/en/portal/articles/2836806-verification-with-dns-txt-
matches = re.findall(r'detectify-verification=([a-f0-9]{32})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Detectify Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Drift Domain Verification
matches = re.findall(r'drift-verification=([a-f0-9]{64})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Drift Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Ahrefs Site Verification
# https://help.ahrefs.com/en/articles/1431155-how-do-i-finish-crawling-my-website-faster-in-site-audit
matches = re.findall(r'ahrefs-site-verification_([a-f0-9]{64})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Ahrefs Site Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Statuspage.io Domain Verification
# https://help.statuspage.io/help/domain-ownership
matches = re.findall(
r'status-page-domain-verification=([a-z0-9]{12})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Statuspage Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Zoom.us Domain Verification
# https://support.zoom.us/hc/en-us/articles/203395207-What-is-Managed-Domain-
matches = re.findall(r'ZOOM_verify_([a-z0-9\-\+\/_=]{22})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Zoom.us Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Mail.ru Domain Verification
matches = re.findall(r'mailru-verification: ([a-z0-9]{16})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Mail.ru Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Yandex Domain Verification
matches = re.findall(r'yandex-verification: ([a-z0-9]{16})$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Yandex Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Brave Ledger Verification
# https://support.brave.com/hc/en-us/articles/360021408352-How-do-I-verify-my-channel-
matches = re.findall(r'brave-ledger-verification=([a-z0-9]+)$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Brave Ledger Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# have-i-been-pwned Verification
matches = re.findall(
r'have-i-been-pwned-verification=([a-f0-9]+)$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"have-i-been-pwned Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
# Cisco Live Domain Verification
# https://www.ciscolive.com/c/dam/r/ciscolive/us/docs/2016/pdf/TECCOL-2982.pdf
matches = re.findall(r'cisco-ci-domain-verification=([a-f0-9]+)$',
eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Cisco Live Domain Verification: " + m,
self.__name__, event)
evt.moduleDataSource = datasource
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
eventData = event.data
if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':
return None
if eventData in self.results:
self.sf.debug("Already did a search for " + eventData +
", skipping.")
return None
self.results[eventData] = True
keepGoing = True
page = 1
while keepGoing and page <= int(self.opts['max_pages']):
# Check if we've been asked to stop
if self.checkForStop():
return None
params = {
'search':
'"' + eventData.encode('raw_unicode_escape').decode(
"ascii", errors='replace') + '"',
'submit':
'Search',
'page':
str(page)
}
# Sites hosted on the domain
data = self.sf.fetchUrl(
'https://onionsearchengine.com/search.php?' +
urllib.parse.urlencode(params),
useragent=self.opts['_useragent'],
timeout=self.opts['timeout'])
if data is None or not data.get('content'):
self.sf.info("No results returned from onionsearchengine.com.")
return None
page += 1
if "url.php?u=" not in data['content']:
# Work around some kind of bug in the site
if "you didn't submit a keyword" in data['content']:
continue
return None
if "forward >" not in data['content']:
keepGoing = False
# Submit the google results for analysis
evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],
self.__name__, event)
self.notifyListeners(evt)
links = re.findall(r"url\.php\?u=(.[^\"\']+)[\"\']",
data['content'], re.IGNORECASE | re.DOTALL)
for link in links:
if self.checkForStop():
return None
if link in self.results:
continue
self.results[link] = True
blacklist = False
for r in self.opts['blacklist']:
if re.match(r, link, re.IGNORECASE):
self.sf.debug("Skipping " + link +
" as it matches blacklist " + r)
blacklist = True
if blacklist:
continue
self.sf.debug("Found a darknet mention: " + link)
if not self.sf.urlFQDN(link).endswith(".onion"):
continue
if not self.opts['fetchlinks']:
evt = SpiderFootEvent("DARKNET_MENTION_URL", link,
self.__name__, event)
self.notifyListeners(evt)
continue
res = self.sf.fetchUrl(link,
timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'],
verify=False)
if res['content'] is None:
self.sf.debug("Ignoring " + link + " as no data returned")
continue
if eventData not in res['content']:
self.sf.debug("Ignoring " + link + " as no mention of " +
eventData)
continue
evt = SpiderFootEvent("DARKNET_MENTION_URL", link,
self.__name__, event)
self.notifyListeners(evt)
try:
startIndex = res['content'].index(eventData) - 120
endIndex = startIndex + len(eventData) + 240
except Exception:
self.sf.debug('String "' + eventData +
'" not found in content.')
continue
data = res['content'][startIndex:endIndex]
evt = SpiderFootEvent("DARKNET_MENTION_CONTENT",
"..." + data + "...", self.__name__, evt)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
eventData = event.data
if self.errorState:
return
if eventData in self.results:
return
if self.opts['api_key'] == "":
self.error(
"You enabled sfp_metadefender but did not set an API key!")
self.errorState = True
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {event.module}")
if eventName == 'IP_ADDRESS':
data = self.queryIp(eventData)
if data is None:
self.debug("No matches found for " + eventData)
return
geo_info = data.get('geo_info')
if geo_info:
location = ', '.join([
_f for _f in [
geo_info.get('city').get('name'),
geo_info.get('country').get('name')
] if _f
])
evt = SpiderFootEvent('GEOINFO', location, self.__name__,
event)
self.notifyListeners(evt)
res = data.get('lookup_results')
if not res:
self.debug("No matches found for " + eventData)
return
sources = res.get('sources')
if not sources:
self.debug("No matches found for " + eventData)
return
for m in sources:
if not m.get('assessment'):
continue
provider = m.get('provider')
evt = SpiderFootEvent('MALICIOUS_IPADDR',
provider + ' [' + eventData + ']',
self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent('BLACKLISTED_IPADDR',
provider + ' [' + eventData + ']',
self.__name__, event)
self.notifyListeners(evt)
if eventName == 'INTERNET_NAME':
data = self.queryDomain(eventData)
if data is None:
self.debug("No matches found for " + eventData)
return
res = data.get('lookup_results')
if not res:
self.debug("No matches found for " + eventData)
return
sources = res.get('sources')
if not sources:
self.debug("No matches found for " + eventData)
return
for m in sources:
if not m.get('assessment'):
continue
if m['assessment'] == "trustworthy":
continue
provider = m.get('provider')
evt = SpiderFootEvent('MALICIOUS_INTERNET_NAME',
provider + ' [' + eventData + ']',
self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent('BLACKLISTED_INTERNET_NAME',
provider + ' [' + eventData + ']',
self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
if srcModuleName == "sfp_open_passive_dns_database":
self.sf.debug("Ignoring " + eventName + ", from self.")
return None
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
self.results[eventData] = True
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
data = self.query(eventData)
if data is None or len(data) == 0:
self.sf.info("No passive DNS data found for " + eventData)
return None
domains = list()
for record in data:
if self.checkForStop():
return None
if self.errorState:
return None
# first_seen = record[0]
# last_seen = record[1]
query_html = record[2]
answer_type = record[3]
answer_html = record[4]
# ttl = record[5]
# count = record[6]
# Extract queries and answers from HTML, and append all in-scope records to the domains list for parsing
r = re.findall(r'>(.+?)<', query_html, re.DOTALL)
if len(r) == 0:
continue
query = r[0]
if self.getTarget().matches(query,
includeChildren=True,
includeParents=True):
domains.append(query)
r = re.findall(r'>(.+?)<', answer_html, re.DOTALL)
if len(r) == 0:
continue
answer = r[0]
if self.getTarget().matches(answer,
includeChildren=True,
includeParents=True):
domains.append(answer)
# Answers for these records types are ignored,
# as all in-scope records have already been added to the domains list for parsing
if answer_type in ['PTR', 'NS', 'MX', 'CNAME', 'TXT', 'SOA']:
continue
if answer in self.results:
continue
if answer == 'NXDOMAIN':
continue
if answer_type == 'A':
if not self.getTarget().matches(
query, includeChildren=True, includeParents=True):
continue
if not self.sf.validIP(answer):
self.sf.debug("Skipping invalid IP address " + answer)
continue
if self.opts['verify'] and not self.sf.validateIP(
query, answer):
self.sf.debug("Host " + query + " no longer resolves to " +
answer)
continue
evt = SpiderFootEvent("IP_ADDRESS", answer, self.__name__,
event)
self.notifyListeners(evt)
if answer_type == 'AAAA':
if not self.getTarget().matches(
query, includeChildren=True, includeParents=True):
continue
if not self.sf.validIP6(answer):
self.sf.debug("Skipping invalid IPv6 address " + answer)
continue
if self.opts['verify'] and not self.sf.validateIP(
query, answer):
self.sf.debug("Host " + query + " no longer resolves to " +
answer)
continue
evt = SpiderFootEvent("IPV6_ADDRESS", answer, self.__name__,
event)
self.notifyListeners(evt)
for domain in set(domains):
if self.checkForStop():
return None
if domain in self.results:
continue
if not self.getTarget().matches(
domain, includeChildren=True, includeParents=True):
continue
if self.opts['verify'] and not self.sf.resolveHost(domain):
self.sf.debug(f"Host {domain} could not be resolved")
evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain,
self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__,
event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if srcModuleName == 'sfp_fsecure_riddler':
self.debug("Ignoring " + eventData + ", from self.")
return
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
if self.opts['username'] == '' or self.opts['password'] == '':
self.error(
'You enabled sfp_fsecure_riddler but did not set an API username/password!'
)
self.errorState = True
return
if not self.token:
self.login()
self.results[eventData] = True
data = None
if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']:
data = self.query("pld:" + eventData)
elif eventName == 'IP_ADDRESS':
data = self.query("ip:" + eventData)
if not data:
self.info("No results found for " + eventData)
return
e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
self.notifyListeners(e)
hosts = list()
addrs = list()
coords = list()
for result in data:
host = result.get('host')
if not host:
continue
if not self.getTarget().matches(
host, includeChildren=True, includeParents=True):
continue
hosts.append(host)
addr = result.get('addr')
if addr:
addrs.append(addr)
coord = result.get('cordinates')
if coord and len(coord) == 2:
coords.append(str(coord[0]) + ', ' + str(coord[1]))
if self.opts['verify'] and len(hosts) > 0:
self.info("Resolving " + str(len(set(hosts))) + " domains ...")
for host in set(hosts):
if self.getTarget().matches(host,
includeChildren=True,
includeParents=True):
evt_type = 'INTERNET_NAME'
else:
evt_type = 'AFFILIATE_INTERNET_NAME'
if self.opts['verify'] and not self.sf.resolveHost(
host) and not self.sf.resolveHost6(host):
self.debug(f"Host {host} could not be resolved")
evt_type += '_UNRESOLVED'
evt = SpiderFootEvent(evt_type, host, self.__name__, event)
self.notifyListeners(evt)
if self.sf.isDomain(host, self.opts['_internettlds']):
if evt_type.startswith('AFFILIATE'):
evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host,
self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__,
event)
self.notifyListeners(evt)
for addr in set(addrs):
if self.sf.validIP(addr):
evt = SpiderFootEvent('IP_ADDRESS', addr, self.__name__, event)
self.notifyListeners(evt)
for coord in set(coords):
evt = SpiderFootEvent('PHYSICAL_COORDINATES', coord, self.__name__,
event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.currentEventSrc = event
if self.errorState:
return
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['google_api_key'] == "" and self.opts['bing_api_key'] == "":
self.sf.error("You enabled sfp_socialprofiles but did not set a Google or Bing API key!")
self.errorState = True
return
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
if self.keywords is None:
self.keywords = self.sf.domainKeywords(
self.getTarget().getNames(), self.opts["_internettlds"]
)
if len(self.keywords) == 0:
self.keywords = None
for site in sites:
s = str(sites[site][0]).format(name=eventData)
searchStr = s.replace(" ", "%20")
res = None
if self.opts["method"].lower() == "yahoo":
self.sf.error(
"Yahoo is no longer supported. Please try 'bing' or 'google'."
)
return
if self.opts["method"].lower() == "google":
res = self.sf.googleIterate(
searchString=searchStr,
opts={
"timeout": self.opts["_fetchtimeout"],
"useragent": self.opts["_useragent"],
"api_key": self.opts["google_api_key"],
"cse_id": self.opts["google_cse_id"],
},
)
self.__dataSource__ = "Google"
if self.opts["method"].lower() == "bing":
res = self.sf.bingIterate(
searchString=searchStr,
opts={
"timeout": self.opts["_fetchtimeout"],
"useragent": self.opts["_useragent"],
"count": self.opts["count"],
"api_key": self.opts["bing_api_key"],
},
)
self.__dataSource__ = "Bing"
if res is None:
self.sf.info("No data returned from " + self.opts["method"] + ".")
continue
if self.checkForStop():
return
# Submit the results for analysis
evt = SpiderFootEvent(
"RAW_RIR_DATA", str(res), self.__name__, event
)
self.notifyListeners(evt)
instances = list()
for searchDom in sites[site][1]:
# Search both the urls & the search engine web content
search_string = " ".join(res["urls"] + [str(res)])
matches = re.findall(
searchDom, search_string, re.IGNORECASE | re.MULTILINE
)
if not matches:
continue
for match in matches:
self.sf.debug("Match found: " + match)
if match in instances:
continue
else:
instances.append(match)
if self.checkForStop():
return
# Fetch the profile page if we are checking
# for a firm relationship.
# Keywords might be empty if the target was an IP, subnet or name.
if self.opts["tighten"] and self.keywords:
match = urllib.parse.unquote(match)
self.sf.debug(
"Tightening results to look for " + str(self.keywords)
)
pres = self.sf.fetchUrl(
match,
timeout=self.opts["_fetchtimeout"],
useragent=self.opts["_useragent"],
verify=False
)
if pres["content"] is None:
continue
else:
found = False
for kw in self.keywords:
if re.search(
r"[^a-zA-Z\-\_]" + kw + r"[^a-zA-Z\-\_]",
pres["content"],
re.IGNORECASE,
):
found = True
if not found:
continue
self.sf.info("Social Media Profile found at " + site + ": " + match)
match = urllib.parse.unquote(match)
evt = SpiderFootEvent(
"SOCIAL_MEDIA", site + ": <SFURL>" + match + "</SFURL>", self.__name__, event
)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
if eventData in self.results:
return None
self.results[eventData] = True
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventName in ["IP_ADDRESS"]:
data = self.queryHost(eventData)
if data is None:
self.sf.debug("No information found for host " + eventData)
return None
evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
self.notifyListeners(evt)
services = data.get("Services")
if services:
for service in services:
port = service.get('port')
if port:
evt = SpiderFootEvent("TCP_PORT_OPEN", eventData + ':' + port, self.__name__, event)
self.notifyListeners(evt)
headers = service.get('headers')
if headers:
servers = headers.get('Server')
if servers:
for server in servers:
if server:
evt = SpiderFootEvent('WEBSERVER_BANNER', server, self.__name__, event)
self.notifyListeners(evt)
geoip = service.get('geoip')
if geoip:
location = ', '.join([_f for _f in [geoip.get('city_name'), geoip.get('region_name'), geoip.get('country_name')] if _f])
if location:
evt = SpiderFootEvent("GEOINFO", location, self.__name__, event)
self.notifyListeners(evt)
software = service.get('software')
if software:
software_version = ' '.join([_f for _f in [software.get('name'), software.get('version')] if _f])
if software_version:
evt = SpiderFootEvent("SOFTWARE_USED", software_version, self.__name__, event)
self.notifyListeners(evt)
os = software.get('os')
if os:
evt = SpiderFootEvent('OPERATING_SYSTEM', os, self.__name__, event)
self.notifyListeners(evt)
leaks = data.get("Leaks")
if leaks:
for leak in leaks:
leak_data = leak.get('data')
if leak_data:
evt = SpiderFootEvent("LEAKSITE_CONTENT", leak_data, self.__name__, event)
self.notifyListeners(evt)
return None
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if eventData in self.results:
return
if self.errorState:
return
if self.opts['api_key'] == "":
self.error(
f"You enabled {self.__class__.__name__} but did not set an API key!"
)
self.errorState = True
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {srcModuleName}")
max_pages = int(self.opts['max_pages'])
page = 1
last_id = None
hosts = list()
while page <= max_pages:
if self.checkForStop():
break
if self.errorState:
break
data = self.queryIssuances(eventData, last_id)
if data is None or len(data) == 0:
break
page += 1
evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__,
event)
self.notifyListeners(evt)
for result in data:
cert_hosts = result.get('dns_names')
if cert_hosts:
for d in cert_hosts:
if d != eventData:
hosts.append(d.replace("*.", ""))
if result.get('cert') is None:
self.debug('Response data contains no certificate data')
continue
try:
rawcert = "-----BEGIN CERTIFICATE-----\n"
rawcert += result.get('cert').get('data')
rawcert += "\n-----END CERTIFICATE-----\n"
cert = self.sf.parseCert(rawcert, eventData,
self.opts['certexpiringdays'])
except Exception as e:
self.info(f"Error parsing certificate: {e}")
continue
if not cert.get('text'):
self.info("Failed to parse the SSL certificate")
continue
evt = SpiderFootEvent('SSL_CERTIFICATE_RAW', cert['text'],
self.__name__, event)
self.notifyListeners(evt)
if cert.get('issuer'):
evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER',
cert['issuer'], self.__name__, event)
self.notifyListeners(evt)
if cert.get('issued'):
evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED',
cert['issued'], self.__name__, event)
self.notifyListeners(evt)
for san in set(cert.get('altnames', list())):
hosts.append(san.replace("*.", ""))
if cert.get('expired'):
evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRED",
cert.get('expirystr', 'Unknown'),
self.__name__, event)
self.notifyListeners(evt)
continue
if cert.get('expiring'):
evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRING",
cert.get('expirystr', 'Unknown'),
self.__name__, event)
self.notifyListeners(evt)
continue
# "To retrieve additional issuances, take the id field of the last issuance and pass it to the issuances endpoint in the after parameter"
last_id = data[-1].get('id')
if last_id is None:
break
if not hosts:
return
if self.opts['verify']:
self.info(f"Resolving {len(set(hosts))} hostnames ...")
for domain in set(hosts):
if self.checkForStop():
return
if domain in self.results:
continue
if self.getTarget().matches(domain,
includeChildren=True,
includeParents=True):
evt_type = 'INTERNET_NAME'
else:
evt_type = 'AFFILIATE_INTERNET_NAME'
if self.opts['verify'] and not self.sf.resolveHost(
domain) and not self.sf.resolveHost6(domain):
self.debug(f"Host {domain} could not be resolved")
evt_type += '_UNRESOLVED'
evt = SpiderFootEvent(evt_type, domain, self.__name__, event)
self.notifyListeners(evt)
if self.sf.isDomain(domain, self.opts['_internettlds']):
if evt_type.startswith('AFFILIATE'):
evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', domain,
self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__,
event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
# Always check if the API key is set and complain if it isn't, then set
# self.errorState to avoid this being a continual complaint during the scan.
if self.opts['api_key_client_id'] == "" or self.opts['api_key_client_secret'] == "":
self.error("You enabled sfp_snov but did not set a Client ID and/or Client Secret")
self.errorState = True
return
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
# Get access token from Snov IO API
accessToken = self.queryAccessToken()
if accessToken is None or accessToken == '':
self.error("No access token received from snov.io for the provided Client ID and/or Client Secret")
self.errorState = True
return
currentLastId = 0
nextPageHasData = True
while nextPageHasData:
if self.checkForStop():
return
data = self.queryDomainName(eventData, accessToken, currentLastId)
if data is None:
self.debug("No email address found for target domain")
break
try:
data = json.loads(data)
except Exception:
self.debug("No email address found for target domain")
break
evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event)
self.notifyListeners(evt)
records = data.get('emails')
lastId = data.get('lastId')
if records:
for record in records:
if record:
email = str(record.get('email'))
if email:
if email in self.results:
continue
if not SpiderFootHelpers.validEmail(email):
continue
self.results[email] = True
if email.split("@")[0] in self.opts['_genericusers'].split(","):
evttype = "EMAILADDR_GENERIC"
else:
evttype = "EMAILADDR"
evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
# Determine whether another page of data exists
if len(records) < self.limit:
nextPageHasData = False
currentLastId = lastId
def handleEvent(self, event):
eventName = event.eventType
eventData = event.data
if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':
return None
if eventData in self.results:
self.sf.debug("Already did a search for " + eventData +
", skipping.")
return None
else:
self.results[eventData] = True
# Sites hosted on the domain
data = self.sf.fetchUrl("https://ahmia.fi/search/?q=" +
eventData.replace(" ", "%20"),
useragent=self.opts['_useragent'],
timeout=self.opts['_fetchtimeout'])
if data is None or not data.get('content'):
self.sf.info("No results returned from ahmia.fi.")
return None
if "redirect_url=" in data['content']:
# Check if we've been asked to stop
if self.checkForStop():
return None
links = re.findall("redirect_url=(.[^\"]+)\"", data['content'],
re.IGNORECASE | re.DOTALL)
reported = False
for link in links:
if link in self.results:
continue
else:
self.results[link] = True
self.sf.debug("Found a darknet mention: " + link)
if self.sf.urlFQDN(link).endswith(".onion"):
if self.checkForStop():
return None
if self.opts['fetchlinks']:
res = self.sf.fetchUrl(
link,
timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'],
verify=False)
if res['content'] is None:
self.sf.debug("Ignoring " + link +
" as no data returned")
continue
if eventData not in res['content']:
self.sf.debug("Ignoring " + link +
" as no mention of " + eventData)
continue
evt = SpiderFootEvent("DARKNET_MENTION_URL", link,
self.__name__, event)
self.notifyListeners(evt)
reported = True
try:
startIndex = res['content'].index(
eventData) - 120
endIndex = startIndex + len(eventData) + 240
except BaseException:
self.sf.debug("String not found in content.")
continue
wdata = res['content'][startIndex:endIndex]
evt = SpiderFootEvent("DARKNET_MENTION_CONTENT",
"..." + wdata + "...",
self.__name__, evt)
self.notifyListeners(evt)
reported = True
else:
evt = SpiderFootEvent("DARKNET_MENTION_URL", link,
self.__name__, event)
self.notifyListeners(evt)
reported = True
if reported:
# Submit the search results for analysis
evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT",
data['content'], self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
ret = None
if self.errorState:
return None
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
# Ignore messages from myself
if srcModuleName == "sfp_riskiq":
self.sf.debug("Ignoring " + eventName + ", from self.")
return None
if self.opts['api_key_login'] == "" or self.opts[
'api_key_password'] == "":
self.sf.error(
"You enabled sfp_riskiq but did not set an credentials!")
self.errorState = True
return None
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
else:
self.results[eventData] = True
if eventName in ['DOMAIN_NAME']:
ret = self.query(eventData, "PSSL")
if not ret:
self.sf.info("No RiskIQ passive SSL data found for " +
eventData)
if ret:
try:
# Generate an event for the IP first, and then link the cert
# to that event.
for res in ret:
if res['subjectCommonName'] == eventData:
continue
if self.getTarget().matches(res['subjectCommonName'],
includeChildren=True):
if self.sf.resolveHost(res['subjectCommonName']):
e = SpiderFootEvent("INTERNET_NAME",
res['subjectCommonName'],
self.__name__, event)
else:
e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED",
res['subjectCommonName'],
self.__name__, event)
self.notifyListeners(e)
if self.sf.isDomain(res['subjectCommonName'],
self.opts['_internettlds']):
e = SpiderFootEvent("DOMAIN_NAME",
res['subjectCommonName'],
self.__name__, event)
self.notifyListeners(e)
except Exception as e:
self.sf.error("Invalid response returned from RiskIQ: " +
str(e))
if eventName == 'EMAILADDR':
ret = self.query(eventData, "WHOIS")
if not ret:
self.sf.info("No RiskIQ passive DNS data found for " +
eventData)
return None
for r in ret:
if not eventData.endswith("@" + r['domain']):
if self.sf.validIP(r['domain']):
t = "NETBLOCK_OWNER"
else:
t = "AFFILIATE_INTERNET_NAME"
e = SpiderFootEvent(t, r['domain'], self.__name__, event)
self.notifyListeners(e)
if t == "AFFILIATE_INTERNET_NAME" and self.sf.isDomain(
r['domain'], self.opts['_internettlds']):
evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME",
r['domain'], self.__name__,
event)
self.notifyListeners(evt)
return None
if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']:
ret = self.query(eventData, "PDNS")
if not ret:
self.sf.info("No RiskIQ passive DNS data found for " +
eventData)
return None
cohosts = list()
if eventName == "IP_ADDRESS":
for r in ret:
if r['focusPoint'].endswith("."):
r['focusPoint'] = r['focusPoint'][:-1]
# Record could be pointing to our IP, or from our IP
if not self.getTarget().matches(
r['focusPoint']) and "*" not in r['focusPoint']:
# We found a co-host
cohosts.append(r['focusPoint'])
if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]:
# Record could be an A/CNAME of this entity, or something pointing to it
for r in ret:
if r['focusPoint'].endswith("."):
r['focusPoint'] = r['focusPoint'][:-1]
if r['focusPoint'] != eventData and "*" not in r[
'focusPoint']:
cohosts.append(r['focusPoint'])
for co in cohosts:
if co == eventData:
continue
if eventName == "IP_ADDRESS" and (
self.opts['verify']
and not self.sf.validateIP(co, eventData)):
self.sf.debug("Host no longer resolves to our IP.")
continue
if not self.opts['cohostsamedomain']:
if self.getTarget().matches(co, includeParents=True):
if self.sf.resolveHost(co):
e = SpiderFootEvent("INTERNET_NAME", co,
self.__name__, event)
else:
e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", co,
self.__name__, event)
self.notifyListeners(e)
if self.sf.isDomain(co, self.opts['_internettlds']):
e = SpiderFootEvent("DOMAIN_NAME", co,
self.__name__, event)
self.notifyListeners(e)
continue
if self.cohostcount < self.opts['maxcohost']:
e = SpiderFootEvent("CO_HOSTED_SITE", co, self.__name__,
event)
self.notifyListeners(e)
self.cohostcount += 1
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.currentEventSrc = event
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
else:
self.results[eventData] = True
# BGP AS Owner -> Other Netblocks
if eventName == "BGP_AS_OWNER":
# Don't report additional netblocks from this AS if we've
# already found this AS before.
if eventData not in self.nbreported:
# Find all the netblocks owned by this AS
self.nbreported[eventData] = True
netblocks = self.asNetblocks(eventData)
if netblocks is not None:
for netblock in netblocks:
if netblock in self.results:
continue
# Technically this netblock was identified via the AS, not
# the original IP event, so link it to asevt, not event.
# Skip IPv6 for now
if ":" in netblock:
continue
evt = SpiderFootEvent("NETBLOCK_OWNER", netblock,
self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent("RAW_RIR_DATA", self.lastContent,
self.__name__, event)
self.notifyListeners(evt)
return None
# NETBLOCK -> AS and other owned netblocks
if eventName.startswith("NETBLOCK_"):
# Get the BGP AS the netblock is a part of
asn = self.netblockAs(eventData)
if asn is None:
self.sf.debug("Could not identify BGP AS for " + eventData)
return None
if eventName == "NETBLOCK_OWNER" and self.ownsAs(asn):
asevt = SpiderFootEvent("BGP_AS_OWNER", asn, self.__name__,
event)
self.notifyListeners(asevt)
evt = SpiderFootEvent("RAW_RIR_DATA", self.lastContent,
self.__name__, event)
self.notifyListeners(evt)
else:
asevt = SpiderFootEvent("BGP_AS_MEMBER", asn, self.__name__,
event)
self.notifyListeners(asevt)
return None
# IP ADDRESS -> NETBLOCK
if eventName == "IP_ADDRESS":
# Get the Netblock the IP is a part of
prefix = self.ipNetblock(eventData)
if prefix is None:
self.sf.debug("Could not identify network prefix for " +
eventData)
return None
# Get the BGP AS the netblock is a part of
asn = self.netblockAs(prefix)
if asn is None:
self.sf.debug("Could not identify BGP AS for " + prefix)
return None
if self.sf.validIpNetwork(prefix):
self.sf.info("Netblock found: " + prefix + "(" + asn + ")")
evt = SpiderFootEvent("NETBLOCK_MEMBER", prefix, self.__name__,
event)
self.notifyListeners(evt)
return None
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if eventData in self.results:
return
if self.errorState:
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['api_key'] == "":
self.error(
"You enabled sfp_grayhatwarfare but did not set an API key!")
self.errorState = True
return
currentIndex = 0
currentPage = 0
maxPages = self.opts['max_pages']
perPage = self.opts['per_page']
keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
while currentPage < maxPages:
currentIndex = currentPage * perPage
if self.checkForStop():
return
if self.errorState:
break
data = self.query(keyword=keyword, start=currentIndex)
if not data:
return
for row in data.get('buckets'):
bucketName = row.get('bucket')
bucketKeyword = bucketName.split('.')[0]
self.debug(bucketKeyword)
if bucketKeyword.startswith(keyword) or bucketKeyword.endswith(
keyword):
evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET', bucketName,
self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent(
'CLOUD_STORAGE_BUCKET_OPEN',
f"{bucketName}: {row.get('fileCount')} files found.",
self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent('RAW_RIR_DATA', str(row),
self.__name__, event)
self.notifyListeners(evt)
currentPage += 1
if data.get('buckets_count') < perPage:
break
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts['api_key'] == "":
self.error("You enabled sfp_shodan but did not set an API key!")
self.errorState = True
return
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
if eventName == "DOMAIN_NAME":
hosts = self.searchHosts(eventData)
if hosts is None:
return
evt = SpiderFootEvent("RAW_RIR_DATA", str(hosts), self.__name__,
event)
self.notifyListeners(evt)
if eventName == 'WEB_ANALYTICS_ID':
try:
network = eventData.split(": ")[0]
analytics_id = eventData.split(": ")[1]
except Exception as e:
self.error(
f"Unable to parse WEB_ANALYTICS_ID: {eventData} ({e})")
return
if network not in [
'Google AdSense', 'Google Analytics',
'Google Site Verification'
]:
self.debug(f"Skipping {eventData}, as not supported.")
return
rec = self.searchHtml(analytics_id)
if rec is None:
return
evt = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__,
event)
self.notifyListeners(evt)
return
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return
max_netblock = self.opts['maxnetblock']
if IPNetwork(eventData).prefixlen < max_netblock:
self.debug(
f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}"
)
return
qrylist = list()
if eventName.startswith("NETBLOCK_"):
for ipaddr in IPNetwork(eventData):
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
else:
qrylist.append(eventData)
for addr in qrylist:
rec = self.queryHost(addr)
if rec is None:
continue
# For netblocks, we need to create the IP address event so that
# the threat intel event is more meaningful.
if eventName == 'NETBLOCK_OWNER':
pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__,
event)
self.notifyListeners(pevent)
else:
pevent = event
evt = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__,
pevent)
self.notifyListeners(evt)
if self.checkForStop():
return
if rec.get('os') is not None:
evt = SpiderFootEvent("OPERATING_SYSTEM",
f"{rec.get('os')} ({addr})",
self.__name__, pevent)
self.notifyListeners(evt)
if rec.get('devtype') is not None:
evt = SpiderFootEvent("DEVICE_TYPE",
f"{rec.get('devtype')} ({addr})",
self.__name__, pevent)
self.notifyListeners(evt)
if rec.get('country_name') is not None:
location = ', '.join([
_f for _f in [rec.get('city'),
rec.get('country_name')] if _f
])
evt = SpiderFootEvent("GEOINFO", location, self.__name__,
pevent)
self.notifyListeners(evt)
if 'data' not in rec:
continue
self.info(f"Found SHODAN data for {eventData}")
ports = list()
banners = list()
asns = list()
products = list()
vulnlist = list()
for r in rec['data']:
port = str(r.get('port'))
banner = r.get('banner')
asn = r.get('asn')
product = r.get('product')
vulns = r.get('vulns')
if port is not None:
cp = addr + ":" + port
if cp not in ports:
ports.append(cp)
evt = SpiderFootEvent("TCP_PORT_OPEN", cp,
self.__name__, pevent)
self.notifyListeners(evt)
if banner is not None:
if banner not in banners:
banners.append(banner)
evt = SpiderFootEvent("TCP_PORT_OPEN_BANNER", banner,
self.__name__, pevent)
self.notifyListeners(evt)
if product is not None:
if product not in products:
products.append(product)
evt = SpiderFootEvent("SOFTWARE_USED", product,
self.__name__, pevent)
self.notifyListeners(evt)
if asn is not None:
if asn not in asns:
asns.append(asn)
evt = SpiderFootEvent("BGP_AS_MEMBER",
asn.replace("AS", ""),
self.__name__, pevent)
self.notifyListeners(evt)
if vulns is not None:
for vuln in vulns.keys():
if vuln not in vulnlist:
vulnlist.append(vuln)
etype, cvetext = self.sf.cveInfo(vuln)
evt = SpiderFootEvent(etype, cvetext,
self.__name__, pevent)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
if eventData in self.results:
return None
if self.opts['api_key'] == '':
self.sf.error("You enabled sfp_networksdb but did not set an API key!", False)
self.errorState = True
return None
self.results[eventData] = True
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventName in ["IP_ADDRESS", "IPV6_ADDRESS"]:
data = self.queryIpInfo(eventData)
if data is None:
self.sf.debug("No IP address information found for " + eventData)
else:
evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
self.notifyListeners(evt)
network = data.get('network')
if network:
cidr = network.get('cidr')
if cidr and cidr != 'N/A' and self.sf.validIpNetwork(cidr):
evt = SpiderFootEvent('NETBLOCK_MEMBER', cidr, self.__name__, event)
self.notifyListeners(evt)
data = self.queryIpGeo(eventData)
if data is None:
self.sf.debug("No IP geolocation information found for " + eventData)
else:
evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
self.notifyListeners(evt)
if data.get('country'):
location = ', '.join(filter(None, [data.get('city'), data.get('state'), data.get('country')]))
evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)
self.notifyListeners(evt)
data = self.queryReverseDns(eventData)
cohosts = list()
if data is None:
self.sf.debug("No reverse DNS results for " + eventData)
else:
evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
self.notifyListeners(evt)
results = data.get('results')
if results:
for domain in results:
cohosts.append(domain)
for co in set(cohosts):
if self.checkForStop():
return None
if co in self.results:
continue
if self.opts['verify'] and not self.sf.validateIP(co, eventData):
self.sf.debug("Host " + co + " no longer resolves to " + eventData)
continue
if not self.opts['cohostsamedomain']:
if self.getTarget().matches(co, includeParents=True):
evt = SpiderFootEvent('INTERNET_NAME', co, self.__name__, event)
self.notifyListeners(evt)
if self.sf.isDomain(co, self.opts['_internettlds']):
evt = SpiderFootEvent('DOMAIN_NAME', co, self.__name__, event)
self.notifyListeners(evt)
continue
if self.cohostcount < self.opts['maxcohost']:
evt = SpiderFootEvent('CO_HOSTED_SITE', co, self.__name__, event)
self.notifyListeners(evt)
self.cohostcount += 1
if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]:
data = self.queryForwardDns(eventData)
if data is None:
self.sf.debug("No forward DNS results for " + eventData)
return None
res = data.get('results')
if not res:
self.sf.debug("No forward DNS results for " + eventData)
return None
evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event)
self.notifyListeners(evt)
for ip in res:
if self.sf.validIP(ip):
evt = SpiderFootEvent('IP_ADDRESS', ip, self.__name__, event)
self.notifyListeners(evt)
elif self.sf.validIP6(ip):
evt = SpiderFootEvent('IPV6_ADDRESS', ip, self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
eventDataHash = self.sf.hashstring(eventData)
parentEvent = event
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if srcModuleName == "sfp_dnszonexfer":
self.debug(f"Ignoring {eventName}, from self.")
return
if eventDataHash in self.events:
self.debug("Skipping duplicate event for " + eventData)
return
self.events[eventDataHash] = True
res = dns.resolver.Resolver()
if self.opts.get('_dnsserver', "") != "":
res.nameservers = [self.opts['_dnsserver']]
# Get the name server's IP. This is to avoid DNS leaks
# when attempting to resolve the name server during
# the zone transfer.
if not self.sf.validIP(eventData) and not self.sf.validIP6(eventData):
nsips = self.sf.resolveHost(eventData)
if not nsips:
return
if not nsips:
self.error(
"Couldn't resolve the name server, so not attempting zone transfer."
)
return
for n in nsips:
if self.sf.validIP(n):
nsip = n
break
else:
nsip = eventData
for name in self.getTarget().getNames():
self.debug("Trying for name: " + name)
try:
ret = list()
z = dns.zone.from_xfr(
dns.query.xfr(nsip,
name,
timeout=int(self.opts["timeout"])))
names = list(z.nodes.keys())
for n in names:
ret.append(z[n].to_text(n))
evt = SpiderFootEvent("RAW_DNS_RECORDS", "\n".join(ret),
self.__name__, parentEvent)
self.notifyListeners(evt)
# Try and pull out individual records
for row in ret:
pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*",
re.IGNORECASE | re.DOTALL)
grps = re.findall(pat, row)
if len(grps) > 0:
for strdata in grps:
self.debug("Matched: " + strdata)
if strdata.endswith("."):
strdata = strdata[:-1]
else:
strdata = strdata + "." + name
evt = SpiderFootEvent("INTERNET_NAME", strdata,
self.__name__, parentEvent)
self.notifyListeners(evt)
except Exception as e:
self.info(
f"Unable to perform DNS zone transfer for {eventData} ({name}): {e}"
)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if srcModuleName == "sfp_tool_onesixtyone":
self.debug("Skipping event from myself.")
return
if not self.opts['onesixtyone_path']:
self.error("You enabled sfp_tool_onesixtyone but did not set a path to the tool!")
self.errorState = True
return
exe = self.opts['onesixtyone_path']
if self.opts['onesixtyone_path'].endswith('/'):
exe = f"{exe}onesixtyone"
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
if not SpiderFootHelpers.sanitiseInput(eventData, extra=['/']):
self.debug("Invalid input, skipping.")
return
targets = []
try:
if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']:
net = IPNetwork(eventData)
if net.prefixlen < self.opts['netblockscanmax']:
self.debug(f"Skipping scanning of {eventData}, too big.")
return
for addr in net.iter_hosts():
targets.append(str(addr))
except BaseException as e:
self.error(f"Strange netblock identified, unable to parse: {eventData} ({e})")
return
# Don't look up stuff twice, check IP == IP here
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
else:
# Might be a subnet within a subnet or IP within a subnet
for addr in self.results:
if IPNetwork(eventData) in IPNetwork(addr):
self.debug(f"Skipping {eventData} as already within a scanned range.")
return
self.results[eventData] = True
# If we weren't passed a netblock, this will be empty
if not targets:
targets.append(eventData)
for target in targets:
args = [
exe,
"-c",
self.communitiesFile,
target
]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
out, stderr = p.communicate(input=None)
stdout = out.decode(sys.stdin.encoding)
except Exception as e:
self.error(f"Unable to run onesixtyone: {e}")
continue
if p.returncode != 0:
self.error(f"Unable to read onesixtyone output\nstderr: {stderr}\nstdout: {stdout}")
continue
if not stdout:
self.debug(f"onesixtyone returned no output for {target}")
continue
for result in stdout.split("\n"):
srcevent = event
if target not in result:
continue
if target != eventData:
srcevent = SpiderFootEvent("IP_ADDRESS", target, self.__name__, event)
self.notifyListeners(srcevent)
e = SpiderFootEvent('UDP_PORT_OPEN', f"{target}:161", self.__name__, srcevent)
self.notifyListeners(e)
e = SpiderFootEvent("UDP_PORT_OPEN_INFO", result, self.__name__, e)
self.notifyListeners(e)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return None
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts["api_key"] == "":
self.sf.error(
f"You enabled {self.__class__.__name__} but did not set an API key!"
)
self.errorState = True
return None
if eventData in self.results:
self.sf.debug(f"Skipping {eventData} as already mapped.")
return None
self.results[eventData] = True
data = self.query(event.data)
if not data:
self.sf.error(f"No data received for {event.data}")
return None
found = False
ipinfo = data.get("ipinfo")
if ipinfo and isinstance(ipinfo, dict):
for address, ip_data in data["ipinfo"].items():
evt = SpiderFootEvent("IP_ADDRESS", address, self.__name__,
event)
self.notifyListeners(evt)
found = True
loc = ip_data.get("loc")
if loc and isinstance(loc, str):
loc_evt = SpiderFootEvent("PHYSICAL_COORDINATES", loc,
self.__name__, evt)
self.notifyListeners(loc_evt)
found = True
geo_info = ', '.join(
filter(None, (ip_data.get(k)
for k in ("city", "region", "country"))))
if geo_info:
geo_info_evt = SpiderFootEvent("GEOINFO", geo_info,
self.__name__, evt)
self.notifyListeners(geo_info_evt)
found = True
related = data.get("related")
if related and isinstance(related, dict):
email_section = related.get("email")
if email_section and isinstance(email_section, list):
for email_data in email_section:
if isinstance(email_data, dict):
value = email_data["value"]
if value and isinstance(value, str):
evt = SpiderFootEvent("EMAILADDR", value,
self.__name__, event)
self.notifyListeners(evt)
found = True
web = data.get("web")
if web and isinstance(web, dict):
server = web.get("server")
if server and isinstance(server, str):
evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", server,
self.__name__, event)
self.notifyListeners(evt)
found = True
google_analytics = web.get("googleanalytics")
if google_analytics and isinstance(google_analytics, str):
evt = SpiderFootEvent("WEB_ANALYTICS_ID", google_analytics,
self.__name__, event)
self.notifyListeners(evt)
found = True
title = web.get("title")
if title and isinstance(title, str):
evt = SpiderFootEvent("DESCRIPTION_ABSTRACT", title,
self.__name__, event)
self.notifyListeners(evt)
found = True
if found:
evt = SpiderFootEvent("RAW_RIR_DATA", json.dumps(data),
self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventData in self.results:
return None
else:
self.results[eventData] = True
for fileExt in self.opts['fileexts']:
if self.checkForStop():
return None
if "." + fileExt.lower() in eventData.lower():
# Fetch the file, allow much more time given that these files are
# typically large.
ret = self.sf.fetchUrl(eventData,
timeout=self.opts['timeout'],
useragent=self.opts['_useragent'],
dontMangle=True,
sizeLimit=10000000,
verify=False)
if ret['content'] is None:
self.sf.error(
f"Unable to fetch file for meta analysis: {eventData}")
return None
if len(ret['content']) < 512:
self.sf.error(
f"Strange content encountered, size of {len(ret['content'])}"
)
return None
meta = None
data = None
# Based on the file extension, handle it
if fileExt.lower() == "pdf":
try:
raw = io.BytesIO(ret['content'])
# data = metapdf.MetaPdfReader().read_metadata(raw)
pdf = PyPDF2.PdfFileReader(raw, strict=False)
data = pdf.getDocumentInfo()
meta = str(data)
self.sf.debug("Obtained meta data from " + eventData)
except Exception as e:
self.sf.error(
f"Unable to parse meta data from: {eventData} ({e})"
)
return None
if fileExt.lower() in ["docx"]:
try:
c = io.BytesIO(ret['content'])
doc = docx.Document(c)
mtype = mimetypes.guess_type(eventData)[0]
self.sf.debug("Office type: " + str(mtype))
a = doc.core_properties.author
c = doc.core_properties.comments
data = [_f for _f in [a, c] if _f]
meta = ", ".join(data)
except Exception as e:
self.sf.error(
f"Unable to process file: {eventData} ({e})")
return None
if fileExt.lower() in ["pptx"]:
try:
c = io.BytesIO(ret['content'])
doc = pptx.Presentation(c)
mtype = mimetypes.guess_type(eventData)[0]
self.sf.debug("Office type: " + str(mtype))
a = doc.core_properties.author
c = doc.core_properties.comments
data = [_f for _f in [a, c] if _f]
meta = ", ".join(data)
except Exception as e:
self.sf.error(
f"Unable to process file: {eventData} ({e})")
return None
if fileExt.lower() in ["jpg", "jpeg", "tiff"]:
try:
raw = io.BytesIO(ret['content'])
data = exifread.process_file(raw)
if data is None or len(data) == 0:
continue
meta = str(data)
except Exception as e:
self.sf.error(
f"Unable to parse meta data from: {eventData} ({e})"
)
return None
if meta is not None and data is not None:
evt = SpiderFootEvent("RAW_FILE_META_DATA", meta,
self.__name__, event)
self.notifyListeners(evt)
val = list()
try:
if "/Producer" in data:
val.append(str(data['/Producer']))
if "/Creator" in data:
val.append(str(data['/Creator']))
if "Application" in data:
val.append(str(data['Application']))
if "Image Software" in data:
val.append(str(data['Image Software']))
except Exception as e:
self.sf.error("Failed to parse PDF, " + eventData +
": " + str(e))
return None
for v in val:
if v and not isinstance(v, PyPDF2.generic.NullObject):
self.sf.debug("VAL: " + str(val))
# Strip non-ASCII
v = ''.join(
[i if ord(i) < 128 else ' ' for i in v])
evt = SpiderFootEvent("SOFTWARE_USED", v,
self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
eventData = event.data
if self.opts['affiliatedomains'] and "AFFILIATE_" in eventName:
eventData = self.sf.hostDomain(eventData, self.opts['_internettlds'])
if not eventData:
return None
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
self.results[eventData] = True
url = "https://api.duckduckgo.com/?q=" + eventData + "&format=json&pretty=1"
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot")
if res['content'] is None:
self.sf.error(f"Unable to fetch {url}", False)
return None
try:
ret = json.loads(res['content'])
except BaseException as e:
self.sf.error(f"Error processing JSON response from DuckDuckGo: {e}", False)
return None
if not ret['Heading']:
self.sf.debug(f"No DuckDuckGo information for {eventData}")
return None
# Submit the DuckDuckGo results for analysis
evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", res['content'],
self.__name__, event)
self.notifyListeners(evt)
abstract_text = ret.get('AbstractText')
if abstract_text:
event_type = "DESCRIPTION_ABSTRACT"
if "AFFILIATE" in eventName:
event_type = "AFFILIATE_" + event_type
evt = SpiderFootEvent(event_type, str(abstract_text), self.__name__, event)
self.notifyListeners(evt)
related_topics = ret.get('RelatedTopics')
if related_topics:
event_type = "DESCRIPTION_CATEGORY"
if "AFFILIATE" in eventName:
event_type = "AFFILIATE_" + event_type
for topic in related_topics:
if not isinstance(topic, dict):
self.sf.debug("No category text found from DuckDuckGo.")
continue
category = topic.get('Text')
if not category:
self.sf.debug("No category text found from DuckDuckGo.")
continue
evt = SpiderFootEvent(event_type, category, self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if eventData in self.results:
self.sf.debug("Skipping " + eventData + " as already scanned.")
return
self.results[eventData] = True
if not self.opts['whatweb_path']:
self.sf.error("You enabled sfp_tool_whatweb but did not set a path to the tool!")
self.errorState = True
return
exe = self.opts['whatweb_path']
if self.opts['whatweb_path'].endswith('/'):
exe = exe + 'whatweb'
# If tool is not found, abort
if not os.path.isfile(exe):
self.sf.error("File does not exist: " + exe)
self.errorState = True
return
# Sanitize domain name.
if not self.sf.sanitiseInput(eventData):
self.sf.error("Invalid input, refusing to run.")
return
# Set aggression level
try:
aggression = int(self.opts['aggression'])
if aggression > 4:
aggression = 4
if aggression < 1:
aggression = 1
except Exception:
aggression = 1
# Run WhatWeb
args = [
self.opts['ruby_path'],
exe,
"--quiet",
"--aggression=" + str(aggression),
"--log-json=/dev/stdout",
"--user-agent=Mozilla/5.0",
"--follow-redirect=never",
eventData
]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
except Exception as e:
self.sf.error(f"Unable to run WhatWeb: {e}")
return
if p.returncode != 0:
self.sf.error("Unable to read WhatWeb output.")
self.sf.debug("Error running WhatWeb: " + stderr + ", " + stdout)
return
if not stdout:
self.sf.debug(f"WhatWeb returned no output for {eventData}")
return
try:
result_json = json.loads(stdout)
except Exception as e:
self.sf.error(f"Couldn't parse the JSON output of WhatWeb: {e}")
return
if len(result_json) == 0:
return
evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event)
self.notifyListeners(evt)
blacklist = [
'Country', 'IP',
'Script', 'Title',
'HTTPServer', 'RedirectLocation', 'UncommonHeaders', 'Via-Proxy', 'Cookies', 'HttpOnly',
'Strict-Transport-Security', 'x-hacker', 'x-machine', 'x-pingback', 'X-Backend', 'X-Cache',
'X-UA-Compatible', 'X-Powered-By', 'X-Forwarded-For', 'X-Frame-Options', 'X-XSS-Protection'
]
for result in result_json:
plugin_matches = result.get('plugins')
if not plugin_matches:
continue
if plugin_matches.get('HTTPServer'):
for w in plugin_matches.get('HTTPServer').get('string'):
evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event)
self.notifyListeners(evt)
if plugin_matches.get('X-Powered-By'):
for w in plugin_matches.get('X-Powered-By').get('string'):
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event)
self.notifyListeners(evt)
for plugin in plugin_matches:
if plugin in blacklist:
continue
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', plugin, self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.currentEventSrc = event
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
if self.opts['external']:
external = "True"
else:
external = ""
if self.opts['daysback'] is not None and self.opts['daysback'] != 0:
newDate = datetime.datetime.now() - datetime.timedelta(
days=int(self.opts['daysback']))
maxDate = newDate.strftime("%Y-%m-%d")
else:
maxDate = ""
qdata = eventData.replace(" ", "+")
wlurl = "query=%22" + qdata + "%22" + "&released_date_start=" + maxDate + "&include_external_sources=" + external + "&new_search=True&order_by=most_relevant#results"
res = self.sf.fetchUrl("https://search.wikileaks.org/?" + wlurl)
if res['content'] is None:
self.sf.error("Unable to fetch Wikileaks content.")
return
# Fetch the paste site content
links = dict()
p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org")
if p:
links.update(p)
p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org")
if p:
links.update(p)
keepGoing = True
page = 0
while keepGoing:
if not res['content']:
break
if "page=" not in res['content']:
keepGoing = False
valid = False
for link in links:
if "search.wikileaks.org/" in link:
continue
# We can safely skip search.wikileaks.org and others.
if "wikileaks.org/" not in link and "cryptome.org/" not in link:
continue
else:
self.sf.debug("Found a link: " + link)
if self.checkForStop():
return
# Wikileaks leak links will have a nested folder structure link
if link.count('/') >= 4:
if not link.endswith(".js") and not link.endswith(
".css"):
evt = SpiderFootEvent("LEAKSITE_URL", link,
self.__name__, event)
self.notifyListeners(evt)
valid = True
if valid:
evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT",
res['content'], self.__name__, event)
self.notifyListeners(evt)
# Fail-safe to prevent infinite looping
if page > 50:
break
if keepGoing:
page += 1
wlurl = "https://search.wikileaks.org/?query=%22" + qdata + "%22" + \
"&released_date_start=" + maxDate + "&include_external_sources=" + \
external + "&new_search=True&order_by=most_relevant&page=" + \
str(page) + "#results"
res = self.sf.fetchUrl(wlurl)
if not res['content']:
break
# Fetch the paste site content
links = dict()
p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org")
if p:
links.update(p)
p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org")
if p:
links.update(p)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if event.moduleDataSource:
moduleDataSource = event.moduleDataSource
else:
moduleDataSource = "Unknown"
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
eventDataHash = self.sf.hashstring(eventData)
if eventDataHash in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
self.results[eventDataHash] = True
countryNames = list()
# Process the event data based on incoming event type
if eventName == "PHONE_NUMBER":
countryNames.append(self.detectCountryFromPhone(eventData))
elif eventName == "DOMAIN_NAME":
countryNames.append(self.detectCountryFromDomainName(eventData))
elif eventName == "AFFILIATE_DOMAIN_NAME" and self.opts["affiliate"]:
countryNames.append(self.detectCountryFromDomainName(eventData))
elif eventName == "CO_HOSTED_SITE_DOMAIN" and self.opts["cohosted"]:
countryNames.append(self.detectCountryFromDomainName(eventData))
elif eventName == "SIMILARDOMAIN" and self.opts["similardomain"]:
countryNames.append(self.detectCountryFromDomainName(eventData))
elif eventName == "IBAN_NUMBER":
countryNames.append(self.detectCountryFromIBAN(eventData))
elif eventName in ["DOMAIN_WHOIS", "GEOINFO", "PHYSICAL_ADDRESS"]:
countryNames.extend(self.detectCountryFromData(eventData))
elif eventName == "AFFILIATE_DOMAIN_WHOIS" and self.opts["affiliate"]:
countryNames.extend(self.detectCountryFromData(eventData))
elif eventName == "CO_HOSTED_SITE_DOMAIN_WHOIS" and self.opts[
"cohosted"]:
countryNames.extend(self.detectCountryFromData(eventData))
if not countryNames:
self.sf.debug(
f"Found no country names associated with {eventName}: {eventData}"
)
return None
for countryName in set(countryNames):
if not countryName:
continue
self.sf.debug(f"Found country name: {countryName}")
evt = SpiderFootEvent("COUNTRY_NAME", countryName, self.__name__,
event)
evt.moduleDataSource = moduleDataSource
self.notifyListeners(evt)
return None
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
self.debug(f"Received event, {eventName}, from {srcModuleName}")
leis = list()
if eventName == "LEI":
leis.append(eventData)
elif eventName == "COMPANY_NAME":
res = self.searchAutocompletions(eventData)
if res is None:
self.debug(f"Found no results for {eventData}")
return
e = SpiderFootEvent("RAW_RIR_DATA", str(res), self.__name__, event)
self.notifyListeners(e)
for record in res:
relationships = record.get('relationships')
if not relationships:
continue
lei_records = relationships.get('lei-records')
if not lei_records:
continue
data = lei_records.get('data')
if not data:
continue
lei = data.get('id')
if not SpiderFootHelpers.validLEI(lei):
continue
leis.append(lei)
self.info(f"Found {len(leis)} LEIs matching {eventData}")
for lei in set(leis):
if lei in self.results:
continue
if not SpiderFootHelpers.validLEI(lei):
continue
e = SpiderFootEvent("LEI", lei, self.__name__, event)
self.notifyListeners(e)
self.results[lei] = True
res = self.retrieveRecord(lei)
if not res:
self.debug(f"Found no results for {eventData}")
return
attributes = res.get('attributes')
if not attributes:
continue
entity = attributes.get('entity')
if not entity:
continue
legal_name = entity.get('legalName')
if legal_name:
entity_name = legal_name.get('value')
if entity_name:
e = SpiderFootEvent("COMPANY_NAME", entity_name,
self.__name__, event)
self.notifyListeners(e)
addresses = list()
address = entity.get('legalAddress')
if address.get('addressLines'):
address_lines = ', '.join(
filter(None, address.get('addressLines')))
location = ', '.join(
filter(None, [
address_lines,
address.get('city'),
address.get('region'),
address.get('country'),
address.get('postalCode')
]))
if location:
addresses.append(location)
address = entity.get('headquartersAddress')
if address.get('addressLines'):
address_lines = ', '.join(
filter(None, address.get('addressLines')))
location = ', '.join(
filter(None, [
address_lines,
address.get('city'),
address.get('region'),
address.get('country'),
address.get('postalCode')
]))
if location:
addresses.append(location)
for address in set(addresses):
e = SpiderFootEvent("PHYSICAL_ADDRESS", address, self.__name__,
event)
self.notifyListeners(e)
