def test_sanitise_input(self):
"""
Test sanitiseInput(self, cmd)
"""
safe = SpiderFootHelpers.sanitiseInput("example-string")
self.assertIsInstance(safe, bool)
self.assertTrue(safe)
safe = SpiderFootHelpers.sanitiseInput("example-string\n")
self.assertIsInstance(safe, bool)
self.assertFalse(safe)
safe = SpiderFootHelpers.sanitiseInput("example string")
self.assertIsInstance(safe, bool)
self.assertFalse(safe)
safe = SpiderFootHelpers.sanitiseInput("-example-string")
self.assertIsInstance(safe, bool)
self.assertFalse(safe)
safe = SpiderFootHelpers.sanitiseInput("..example-string")
self.assertIsInstance(safe, bool)
self.assertFalse(safe)
safe = SpiderFootHelpers.sanitiseInput("12")
self.assertIsInstance(safe, bool)
self.assertFalse(safe)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if srcModuleName == "sfp_tool_onesixtyone":
self.debug("Skipping event from myself.")
return
if not self.opts['onesixtyone_path']:
self.error("You enabled sfp_tool_onesixtyone but did not set a path to the tool!")
self.errorState = True
return
exe = self.opts['onesixtyone_path']
if self.opts['onesixtyone_path'].endswith('/'):
exe = f"{exe}onesixtyone"
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
if not SpiderFootHelpers.sanitiseInput(eventData, extra=['/']):
self.debug("Invalid input, skipping.")
return
targets = []
try:
if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']:
net = IPNetwork(eventData)
if net.prefixlen < self.opts['netblockscanmax']:
self.debug(f"Skipping scanning of {eventData}, too big.")
return
for addr in net.iter_hosts():
targets.append(str(addr))
except BaseException as e:
self.error(f"Strange netblock identified, unable to parse: {eventData} ({e})")
return
# Don't look up stuff twice, check IP == IP here
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
else:
# Might be a subnet within a subnet or IP within a subnet
for addr in self.results:
if IPNetwork(eventData) in IPNetwork(addr):
self.debug(f"Skipping {eventData} as already within a scanned range.")
return
self.results[eventData] = True
# If we weren't passed a netblock, this will be empty
if not targets:
targets.append(eventData)
for target in targets:
args = [
exe,
"-c",
self.communitiesFile,
target
]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
out, stderr = p.communicate(input=None)
stdout = out.decode(sys.stdin.encoding)
except Exception as e:
self.error(f"Unable to run onesixtyone: {e}")
continue
if p.returncode != 0:
self.error(f"Unable to read onesixtyone output\nstderr: {stderr}\nstdout: {stdout}")
continue
if not stdout:
self.debug(f"onesixtyone returned no output for {target}")
continue
for result in stdout.split("\n"):
srcevent = event
if target not in result:
continue
if target != eventData:
srcevent = SpiderFootEvent("IP_ADDRESS", target, self.__name__, event)
self.notifyListeners(srcevent)
e = SpiderFootEvent('UDP_PORT_OPEN', f"{target}:161", self.__name__, srcevent)
self.notifyListeners(e)
e = SpiderFootEvent("UDP_PORT_OPEN_INFO", result, self.__name__, e)
self.notifyListeners(e)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
self.results[eventData] = True
if not self.opts['wafw00f_path']:
self.error(
"You enabled sfp_tool_wafw00f but did not set a path to the tool!"
)
self.errorState = True
return
exe = self.opts['wafw00f_path']
if self.opts['wafw00f_path'].endswith('/'):
exe = exe + 'wafw00f'
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
url = eventData
if not SpiderFootHelpers.sanitiseInput(url):
self.error("Invalid input, refusing to run.")
return
args = [self.opts['python_path'], exe, '-a', '-o-', '-f', 'json', url]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
except Exception as e:
self.error(f"Unable to run wafw00f: {e}")
return
if p.returncode != 0:
self.error(
f"Unable to read wafw00f output\nstderr: {stderr}\nstdout: {stdout}"
)
return
if not stdout:
self.debug(f"wafw00f returned no output for {eventData}")
return
try:
result_json = json.loads(stdout)
except Exception as e:
self.error(
f"Could not parse wafw00f output as JSON: {e}\nstdout: {stdout}"
)
return
if not result_json:
self.debug(f"wafw00f returned no output for {eventData}")
return
evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(result_json),
self.__name__, event)
self.notifyListeners(evt)
for waf in result_json:
if not waf:
continue
firewall = waf.get('firewall')
if not firewall:
continue
if firewall == 'Generic':
continue
manufacturer = waf.get('manufacturer')
if not manufacturer:
continue
software = ' '.join(filter(None, [manufacturer, firewall]))
if software:
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', software,
self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
self.results[eventData] = True
if not self.opts['cmseekpath']:
self.error("You enabled sfp_tool_cmseek but did not set a path to the tool!")
self.errorState = True
return
# Normalize path
if self.opts['cmseekpath'].endswith('cmseek.py'):
exe = self.opts['cmseekpath']
resultpath = self.opts['cmseekpath'].split("cmseek.py")[0] + "/Result"
elif self.opts['cmseekpath'].endswith('/'):
exe = self.opts['cmseekpath'] + "cmseek.py"
resultpath = self.opts['cmseekpath'] + "Result"
else:
exe = self.opts['cmseekpath'] + "/cmseek.py"
resultpath = self.opts['cmseekpath'] + "/Result"
# If tool is not found, abort
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
# Sanitize domain name.
if not SpiderFootHelpers.sanitiseInput(eventData):
self.error("Invalid input, refusing to run.")
return
args = [
self.opts['pythonpath'],
exe,
'--follow-redirect',
'--batch',
'-u',
eventData
]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
except Exception as e:
self.error(f"Unable to run CMSeeK: {e}")
return
if p.returncode != 0:
self.error(f"Unable to read CMSeeK output\nstderr: {stderr}\nstdout: {stdout}")
return
if b"CMS Detection failed" in stdout:
self.debug(f"Could not detect the CMS for {eventData}")
return
log_path = f"{resultpath}/{eventData}/cms.json"
if not os.path.isfile(log_path):
self.error(f"File does not exist: {log_path}")
return
try:
f = io.open(log_path, encoding='utf-8')
j = json.loads(f.read())
except Exception as e:
self.error(f"Could not parse CMSeeK output file {log_path} as JSON: {e}")
return
cms_name = j.get('cms_name')
if not cms_name:
return
cms_version = j.get('cms_version')
software = ' '.join(filter(None, [cms_name, cms_version]))
if not software:
return
evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", software, self.__name__, event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if srcModuleName == "sfp_tool_nuclei":
return
if not self.opts['nuclei_path'] or not self.opts['template_path']:
self.error(
"You enabled sfp_tool_nuclei but did not set a path to the tool and/or templates!"
)
self.errorState = True
return
exe = self.opts['nuclei_path']
if self.opts['nuclei_path'].endswith('/'):
exe = f"{exe}nuclei"
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
if not SpiderFootHelpers.sanitiseInput(eventData, extra=['/']):
self.debug("Invalid input, skipping.")
return
# Don't look up stuff twice
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
if eventName != "INTERNET_NAME":
# Might be a subnet within a subnet or IP within a subnet
for addr in self.results:
try:
if IPNetwork(eventData) in IPNetwork(addr):
self.debug(
f"Skipping {eventData} as already within a scanned range."
)
return
except BaseException:
# self.results will also contain hostnames
continue
self.results[eventData] = True
timeout = 240
try:
target = eventData
if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']:
target = ""
net = IPNetwork(eventData)
if net.prefixlen < self.opts['netblockscanmax']:
self.debug(f"Skipping scanning of {eventData}, too big.")
return
# Nuclei doesn't support targeting subnets directly,
# so for now work around that by listing each IP.
for addr in IPNetwork(eventData).iter_hosts():
target += str(addr) + "\n"
timeout += 240
except BaseException as e:
self.error(
f"Strange netblock identified, unable to parse: {eventData} ({e})"
)
return
try:
args = [
exe,
"-silent",
"-json",
"-concurrency",
"100",
"-retries",
"1",
"-t",
self.opts["template_path"],
"-no-interactsh",
"-etags",
"dos",
"fuzz",
"misc",
]
p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
try:
stdout, stderr = p.communicate(input=target.encode(
sys.stdin.encoding),
timeout=timeout)
if p.returncode == 0:
content = stdout.decode(sys.stdout.encoding)
else:
self.error("Unable to read Nuclei content.")
self.debug(f"Error running Nuclei: {stderr}, {stdout}")
return
except TimeoutExpired:
p.kill()
stdout, stderr = p.communicate()
self.debug("Timed out waiting for Nuclei to finish")
return
except BaseException as e:
self.error(f"Unable to run Nuclei: {e}")
return
if not content:
return
try:
for line in content.split("\n"):
if not line:
continue
data = json.loads(line)
srcevent = event
host = data['matched-at'].split(":")[0]
if host != eventData:
if self.sf.validIP(host):
srctype = "IP_ADDRESS"
else:
srctype = "INTERNET_NAME"
srcevent = SpiderFootEvent(srctype, host, self.__name__,
event)
self.notifyListeners(srcevent)
matches = re.findall(r"CVE-\d{4}-\d{4,7}", line)
if matches:
for cve in matches:
etype, cvetext = self.sf.cveInfo(cve)
e = SpiderFootEvent(etype, cvetext, self.__name__,
srcevent)
self.notifyListeners(e)
else:
if "matcher-name" in data:
etype = "VULNERABILITY_GENERAL"
if data['info']['severity'] == "info":
etype = "WEBSERVER_TECHNOLOGY"
datatext = f"Template: {data['info']['name']}({data['template-id']})\n"
datatext += f"Matcher: {data['matcher-name']}\n"
datatext += f"Matched at: {data['matched-at']}\n"
if data['info'].get('reference'):
datatext += f"Reference: <SFURL>{data['info']['reference'][0]}</SFURL>"
evt = SpiderFootEvent(
etype,
datatext,
self.__name__,
srcevent,
)
self.notifyListeners(evt)
except (KeyError, ValueError) as e:
self.error(f"Couldn't parse the JSON output of Nuclei: {e}")
self.error(f"Nuclei content: {content}")
return
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
self.results[eventData] = True
if not self.opts['snallygaster_path']:
self.error(
"You enabled sfp_tool_snallygaster but did not set a path to the tool!"
)
self.errorState = True
return
exe = self.opts["snallygaster_path"]
if self.opts["snallygaster_path"].endswith("/"):
exe = f"{exe}snallygaster"
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
if not SpiderFootHelpers.sanitiseInput(eventData):
self.error("Invalid input, refusing to run.")
return
args = [exe, '--nowww', '-j', eventData]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
out, stderr = p.communicate(input=None)
stdout = out.decode(sys.stdin.encoding)
except Exception as e:
self.error(f"Unable to run snallygaster: {e}")
return
if p.returncode != 0:
self.error(
f"Unable to read onesixtyone output\nstderr: {stderr}\nstdout: {stdout}"
)
return
if not stdout:
self.debug(f"snallygaster returned no output for {eventData}")
return
try:
result_json = json.loads(stdout)
except Exception as e:
self.error(
f"Could not parse snallygaster output as JSON: {e}\nstderr: {stderr}\nstdout: {stdout}"
)
return
if not result_json:
self.debug(f"snallygaster returned no output for {eventData}")
return
for res in result_json:
if "cause" not in res:
continue
text = f"Cause: {res['cause']}\nURL: {res['url']}"
if res["misc"]:
text += f"\nAdditional Info: {res['misc']}"
evt = SpiderFootEvent(
"VULNERABILITY_GENERAL",
text,
self.__name__,
event,
)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if not self.opts['wappalyzer_path']:
self.error(
"You enabled sfp_tool_wappalyzer but did not set a path to the tool!"
)
self.errorState = True
return
exe = self.opts['wappalyzer_path']
if self.opts['wappalyzer_path'].endswith('/'):
exe = f"{exe}cli.js"
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
if not SpiderFootHelpers.sanitiseInput(eventData):
self.debug("Invalid input, skipping.")
return
# Don't look up stuff twice
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
self.results[eventData] = True
try:
args = [self.opts["node_path"], exe, f"https://{eventData}"]
p = Popen(args, stdout=PIPE, stderr=PIPE)
try:
stdout, stderr = p.communicate(input=None, timeout=60)
if p.returncode == 0:
content = stdout.decode(sys.stdin.encoding)
else:
self.error("Unable to read Wappalyzer content.")
self.error(f"Error running Wappalyzer: {stderr}, {stdout}")
return
except TimeoutExpired:
p.kill()
stdout, stderr = p.communicate()
self.debug("Timed out waiting for Wappalyzer to finish")
return
except BaseException as e:
self.error(f"Unable to run Wappalyzer: {e}")
return
try:
data = json.loads(content)
for item in data["technologies"]:
for cat in item["categories"]:
if cat["name"] == "Operating systems":
evt = SpiderFootEvent(
"OPERATING_SYSTEM",
item["name"],
self.__name__,
event,
)
elif cat["name"] == "Web servers":
evt = SpiderFootEvent(
"WEBSERVER_TECHNOLOGY",
item["name"],
self.__name__,
event,
)
else:
evt = SpiderFootEvent(
"SOFTWARE_USED",
item["name"],
self.__name__,
event,
)
self.notifyListeners(evt)
except (KeyError, ValueError) as e:
self.error(f"Couldn't parse the JSON output of Wappalyzer: {e}")
self.error(f"Wappalyzer content: {content}")
return
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if srcModuleName == "sfp_tool_testsslsh":
self.debug("Skipping event from myself.")
return
if not self.opts['testsslsh_path']:
self.error(
"You enabled sfp_tool_testsslsh but did not set a path to the tool!"
)
self.errorState = True
return
exe = self.opts['testsslsh_path']
if self.opts['testsslsh_path'].endswith('/'):
exe = f"{exe}testssl.sh"
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
if not SpiderFootHelpers.sanitiseInput(eventData, extra=['/']):
self.debug("Invalid input, skipping.")
return
targets = list()
try:
if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']:
net = IPNetwork(eventData)
if net.prefixlen < self.opts['netblockscanmax']:
self.debug("Skipping scanning of " + eventData +
", too big.")
return
for addr in net.iter_hosts():
targets.append(str(addr))
except BaseException as e:
self.error(
f"Strange netblock identified, unable to parse: {eventData} ({e})"
)
return
# Don't look up stuff twice, check IP == IP here
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
else:
if eventName != "INTERNET_NAME":
# Might be a subnet within a subnet or IP within a subnet
for addr in self.results:
try:
if IPNetwork(eventData) in IPNetwork(addr):
self.debug(
f"Skipping {eventData} as already within a scanned range."
)
return
except BaseException:
# self.results will also contain hostnames
continue
self.results[eventData] = True
# If we weren't passed a netblock, this will be empty
if not targets:
targets.append(eventData)
for target in targets:
# Create a temporary output file
_, fname = tempfile.mkstemp("testssl.json")
args = [
exe, "-U", "--connect-timeout", "5", "--openssl-timeout", "5",
"--jsonfile", fname, target
]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
out, stderr = p.communicate(input=None)
stdout = out.decode(sys.stdin.encoding)
except Exception as e:
self.error(f"Unable to run testssl.sh: {e}")
os.unlink(fname)
continue
if p.returncode != 0:
err = None
if "Unable to open a socket" in stdout:
err = "Unable to connect"
else:
err = "Internal error"
self.error(
f"Unable to read testssl.sh output for {target}: {err}")
os.unlink(fname)
continue
if not stdout:
self.debug(f"testssl.sh returned no output for {target}")
os.unlink(fname)
continue
try:
with open(fname, "r") as f:
result_json = json.loads(f.read())
os.unlink(fname)
except Exception as e:
self.error(
f"Could not parse testssl.sh output as JSON: {e}\nstderr: {stderr}\nstdout: {stdout}"
)
continue
if not result_json:
self.debug(f"testssl.sh returned no output for {target}")
continue
for result in result_json:
if result['finding'] == "not vulnerable":
continue
if result['severity'] not in [
"LOW", "MEDIUM", "HIGH", "CRITICAL"
]:
continue
if 'cve' in result:
for cve in result['cve'].split(" "):
etype, cvetext = self.sf.cveInfo(cve)
evt = SpiderFootEvent(etype, cvetext, self.__name__,
event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent(
"VULNERABILITY_GENERAL",
f"{result['id']} ({result['finding']})", self.__name__,
event)
self.notifyListeners(evt)
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if srcModuleName == "sfp_tool_nbtscan":
self.debug("Skipping event from myself.")
return
if not self.opts['nbtscan_path']:
self.error("You enabled sfp_tool_nbtscan but did not set a path to the tool!")
self.errorState = True
return
exe = self.opts['nbtscan_path']
if self.opts['nbtscan_path'].endswith('/'):
exe = f"{exe}nbtscan"
if not os.path.isfile(exe):
self.error(f"File does not exist: {exe}")
self.errorState = True
return
if not SpiderFootHelpers.sanitiseInput(eventData, extra=['/']):
self.debug("Invalid input, skipping.")
return
try:
if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']:
net = IPNetwork(eventData)
if net.prefixlen < self.opts['netblockscanmax']:
self.debug(f"Skipping scanning of {eventData}, too big.")
return
except BaseException as e:
self.error(f"Strange netblock identified, unable to parse: {eventData} ({e})")
return
# Don't look up stuff twice, check IP == IP here
if eventData in self.results:
self.debug(f"Skipping {eventData} as already scanned.")
return
else:
# Might be a subnet within a subnet or IP within a subnet
for addr in self.results:
if IPNetwork(eventData) in IPNetwork(addr):
self.debug(f"Skipping {eventData} as already within a scanned range.")
return
self.results[eventData] = True
args = [
exe,
"-v",
eventData
]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
out, _ = p.communicate(input=None)
stdout = out.decode(sys.stdin.encoding)
except Exception as e:
self.error(f"Unable to run nbtscan: {e}")
return
if not stdout:
self.debug(f"nbtscan returned no output for {eventData}")
return
inside = False
info = ""
for row in stdout.split("\n"):
if len(row) == 0:
continue
if "NetBIOS Name Table" in row:
inside = True
if "Adapter address" in row:
info += f"{row}\n"
inside = False
if inside:
info += f"{row}\n"
if not inside and len(info) > 0:
srcEvent = event
addr = eventData
if eventName == "NETBLOCK_OWNER":
# Extract the IP from the raw nbtscan output
addr = info.split("\n")[0].split("for Host ")[1].replace(":", "")
srcEvent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event)
self.notifyListeners(srcEvent)
evt = SpiderFootEvent('UDP_PORT_OPEN', f"{addr}:137", self.__name__, srcEvent)
self.notifyListeners(evt)
evt = SpiderFootEvent('UDP_PORT_OPEN_INFO', info, self.__name__, evt)
self.notifyListeners(evt)
info = ""
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if eventData in self.results:
self.debug("Skipping " + eventData + " as already scanned.")
return
self.results[eventData] = True
dnstwistLocation = which('dnstwist')
if dnstwistLocation and Path(dnstwistLocation).is_file():
cmd = ['dnstwist']
else:
if not self.opts['dnstwistpath']:
self.error("You enabled sfp_tool_dnstwist but did not set a path to the tool!")
self.errorState = True
return
# Normalize path
if self.opts['dnstwistpath'].endswith('dnstwist.py'):
exe = self.opts['dnstwistpath']
elif self.opts['dnstwistpath'].endswith('/'):
exe = self.opts['dnstwistpath'] + "dnstwist.py"
else:
exe = self.opts['dnstwistpath'] + "/dnstwist.py"
# If tool is not found, abort
if not Path(exe).is_file():
self.error("File does not exist: " + exe)
self.errorState = True
return
cmd = [self.opts['pythonpath'], exe]
# Sanitize domain name.
if not SpiderFootHelpers.sanitiseInput(eventData):
self.error("Invalid input, refusing to run.")
return
try:
p = Popen(cmd + ["-f", "json", "-r", eventData], stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
if p.returncode == 0:
content = stdout
else:
self.error("Unable to read DNSTwist content.")
self.debug("Error running DNSTwist: " + stderr + ", " + stdout)
return
# For each line in output, generate a SIMILARDOMAIN event
try:
j = json.loads(content)
for r in j:
if self.getTarget().matches(r['domain-name']):
continue
evt = SpiderFootEvent("SIMILARDOMAIN", r['domain-name'],
self.__name__, event)
self.notifyListeners(evt)
except Exception as e:
self.error("Couldn't parse the JSON output of DNSTwist: " + str(e))
return
except Exception as e:
self.error("Unable to run DNSTwist: " + str(e))
return
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.errorState:
return
if eventData in self.results:
self.debug("Skipping " + eventData + " as already scanned.")
return
self.results[eventData] = True
if not self.opts['whatweb_path']:
self.error(
"You enabled sfp_tool_whatweb but did not set a path to the tool!"
)
self.errorState = True
return
exe = self.opts['whatweb_path']
if self.opts['whatweb_path'].endswith('/'):
exe = exe + 'whatweb'
# If tool is not found, abort
if not os.path.isfile(exe):
self.error("File does not exist: " + exe)
self.errorState = True
return
# Sanitize domain name.
if not SpiderFootHelpers.sanitiseInput(eventData):
self.error("Invalid input, refusing to run.")
return
# Set aggression level
try:
aggression = int(self.opts['aggression'])
if aggression > 4:
aggression = 4
if aggression < 1:
aggression = 1
except Exception:
aggression = 1
# Run WhatWeb
args = [
self.opts['ruby_path'], exe, "--quiet",
"--aggression=" + str(aggression), "--log-json=/dev/stdout",
"--user-agent=Mozilla/5.0", "--follow-redirect=never", eventData
]
try:
p = Popen(args, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
except Exception as e:
self.error(f"Unable to run WhatWeb: {e}")
return
if p.returncode != 0:
self.error("Unable to read WhatWeb output.")
self.debug("Error running WhatWeb: " + stderr + ", " + stdout)
return
if not stdout:
self.debug(f"WhatWeb returned no output for {eventData}")
return
try:
result_json = json.loads(stdout)
except Exception as e:
self.error(f"Couldn't parse the JSON output of WhatWeb: {e}")
return
if len(result_json) == 0:
return
blacklist = [
'Country', 'IP', 'Script', 'Title', 'HTTPServer',
'RedirectLocation', 'UncommonHeaders', 'Via-Proxy', 'Cookies',
'HttpOnly', 'Strict-Transport-Security', 'x-hacker', 'x-machine',
'x-pingback', 'X-Backend', 'X-Cache', 'X-UA-Compatible',
'X-Powered-By', 'X-Forwarded-For', 'X-Frame-Options',
'X-XSS-Protection'
]
found = False
for result in result_json:
plugin_matches = result.get('plugins')
if not plugin_matches:
continue
if plugin_matches.get('HTTPServer'):
for w in plugin_matches.get('HTTPServer').get('string'):
evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__,
event)
self.notifyListeners(evt)
found = True
if plugin_matches.get('X-Powered-By'):
for w in plugin_matches.get('X-Powered-By').get('string'):
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w,
self.__name__, event)
self.notifyListeners(evt)
found = True
for plugin in plugin_matches:
if plugin in blacklist:
continue
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', plugin,
self.__name__, event)
self.notifyListeners(evt)
found = True
if found:
evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json),
self.__name__, event)
self.notifyListeners(evt)
