class Module(ModuleTemplate):
name = "Nessus"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Vulnerability = VulnRepository(db, self.name)
self.CVE = CVERepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"--import_file",
help=
"Import separated Nessus files separated by a space. DO NOT USE QUOTES OR COMMAS",
nargs="+",
)
self.options.add_argument(
"--launch",
help=
"Launch Nessus scan using Actively scoped IPs and domains in the database",
action="store_true",
)
self.options.add_argument("--job_name",
help="Job name inside Nessus",
default="Armory Job")
self.options.add_argument("--username", help="Nessus Username")
self.options.add_argument("--password", help="Nessus Password")
self.options.add_argument(
"--host",
help="Hostname:Port of Nessus web interface (ie localhost:8835")
self.options.add_argument("--uuid",
help="UUID of Nessus Policy to run")
self.options.add_argument("--policy_id", help="Policy ID to use")
self.options.add_argument("--folder_id",
help="ID for folder to store job in")
self.options.add_argument(
"--download",
help="Download Nessus job from server and import",
action="store_true",
)
self.options.add_argument("--job_id",
help="Job ID to download and import")
self.options.add_argument(
"--output_path",
help="Path to store downloaded file (Default: Nessus)",
default=self.name,
)
self.options.add_argument(
"--disable_mitre",
help="Disable mitre CVE data gathering.",
action="store_true",
)
def run(self, args):
if args.import_file:
for nFile in args.import_file:
self.process_data(nFile, args)
elif args.launch:
if (not args.username # noqa: W503
and not args.password # noqa: W503
and not args.host # noqa: W503
and not args.uuid # noqa: W503
and not args.policy_id # noqa: W503
and not args.folder_id # noqa: W503
):
display_error(
"You must supply a username, password, and host to launch a Nessus job"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
uuid=args.uuid,
policy_id=args.policy_id,
folder_id=args.folder_id,
)
ips = [
ip.ip_address
for ip in self.IPAddress.all(scope_type="active",
tool=self.name)
]
cidrs = [
cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)
]
domains = [
domain.domain
for domain in self.Domain.all(scope_type="active",
tool=self.name)
]
targets = ", ".join(merge_ranges(ips + cidrs) + domains)
res = n.launch_job(targets, args.job_name)
display("New Nessus job launched with ID {}".format(res))
display(
"Remember this number! You'll need it to download the job once it is done."
)
elif args.download:
if (not args.username # noqa: W503
and not args.password # noqa: W503
and not args.host # noqa: W503
and not args.job_id # noqa: W503
):
display_error(
"You must supply host, username, password and job_id to download a report to import"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
)
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(
output_path,
"Nessus-export-{}.nessus".format(int(time.time())))
n.export_file(args.job_id, output_path)
self.process_data(output_path, args)
def nessCheckPlugin(self, tag):
nessPlugins = [
"10759",
"77026",
"20089",
"56984",
"71049",
"70658",
"40984",
"11411",
]
pluginID = tag.get("pluginID")
if pluginID in nessPlugins:
# print pluginID + " is in the list"
if pluginID == "10759":
if tag.find("plugin_output") is not None:
return (
tag.find("plugin_output").text.split(
"\n\n")[3].strip()
) # returns IP for Web Server HTTP Header INternal IP disclosure
else:
return ""
if pluginID == "77026":
if tag.find("plugin_output") is not None:
return (
tag.find("plugin_output").text.split(
"\n\n")[3].strip()
) # Microsoft Exchange Client Access Server Information Disclosure (IP addy)
else:
return ""
if pluginID == "71049" or pluginID == "70658":
output = ""
if tag.find("plugin_output") is not None:
tmp = tag.find("plugin_output").text.split(":")[
1] # SSH Weak MAC & CBC Algorithms Enabled
# print "#"*5
tmp = tmp.split("\n\n")[1].replace(" ", "")
# print "#"*5
output = tmp.split("\n")
# print ", ".join(output)
return ", ".join(output)
if pluginID == "56984":
if tag.find("plugin_output") is not None:
tmp = (tag.find("plugin_output").text.split(
"This port supports ")[1].strip()
) # SSL / TLS Versions Supported
tmp = tmp.split("/")
bad = []
for i in tmp:
# print i
if "SSLv" in i:
bad.append(i)
elif "TLSv1.0" in i:
bad.append(i)
if bad != []:
return ", ".join(bad).rstrip(".")
else:
return ""
else:
return ""
if pluginID == "40984": # browsable web dirs
if tag.find("plugin_output") is not None:
tmp = (tag.find("plugin_output").text.split(
"The following directories are browsable :")
[1].strip())
directories = tmp.split("\n")
return "\n".join(directories)
if pluginID == "11411": # Backup Files Disclosure
if tag.find("plugin_output") is not None:
urls = []
tmp = (tag.find("plugin_output").text.split(
"It is possible to read the following backup file")
[1].strip())
tmpUrls = tmp.split("\n")
for url in tmpUrls:
if "URL" in url:
urls.append(url.split(":")[1].lstrip())
if urls:
return "\n".join(urls)
else:
return ""
if pluginID == "20089": # F5 cookie
if tag.find("plugin_output") is not None:
f5Output = []
cookieVal = []
output = tag.find("plugin_output").text.strip().split("\n")
for line in output:
# print line
line = line.split(":")
for i, item in enumerate(line):
item = item.strip()
if "Cookie" in item:
line.pop(i) # Pop to remove the first?
tmp = line.pop(i)
tmp.strip()
cookieVal.append(tmp)
else:
item = "".join(item)
f5Output.append(item)
f5Output = " : ".join(f5Output)
f5Output = f5Output.replace(" : : ", ", ")
f5Output += " [" + ", ".join(cookieVal) + "]"
c = 0
tmpF5Output = f5Output.split()
for i, letter in enumerate(tmpF5Output):
if letter == ":":
c += 1
if (c % 2) == 0:
tmpF5Output[i] = " "
return "".join(tmpF5Output).replace("[", " [")
else:
return ""
else:
return False
def getVulns(self, ip, ReportHost):
"""Gets vulns and associated services"""
for tag in ReportHost.iter("ReportItem"):
exploitable = False
cves = []
vuln_refs = {}
proto = tag.get("protocol")
port = tag.get("port")
svc_name = tag.get("svc_name").replace("?", "")
plugin_output = []
tmpPort = proto + "/" + port
if tmpPort.lower() == "tcp/443":
portName = "https"
elif tmpPort.lower() == "tcp/80":
portName = "http"
elif svc_name == "www":
plugin_name = tag.get("pluginName")
if "tls" in plugin_name.lower() or "ssl" in plugin_name.lower(
):
portName = "https"
else:
portName = "http"
else:
portName = svc_name
created, db_port = self.Port.find_or_create(port_number=port,
status="open",
proto=proto,
ip_address_id=ip.id)
if db_port.service_name == "http":
if portName == "https":
db_port.service_name = portName
elif db_port.service_name == "https":
pass
else:
db_port.service_name = portName
db_port.save()
if tag.get("pluginID") == "56984":
severity = 1
elif tag.get("pluginID") == "11411":
severity = 3
else:
severity = int(tag.get("severity"))
findingName = tag.get("pluginName")
description = tag.find("description").text
if tag.find(
"solution") is not None and tag.find("solution") != "n/a":
solution = tag.find("solution").text
else:
solution = "No Remediation From Nessus"
nessCheck = self.nessCheckPlugin(tag)
if nessCheck:
if not db_port.info:
db_port.info = {findingName: nessCheck}
else:
db_port.info[findingName] = nessCheck
db_port.save()
if tag.find("exploit_available") is not None:
exploitable = True
metasploits = tag.findall("metasploit_name")
if metasploits:
vuln_refs["metasploit"] = []
for tmp in metasploits:
vuln_refs["metasploit"].append(tmp.text)
edb_id = tag.findall("edb-id")
if edb_id:
vuln_refs["edb-id"] = []
for tmp in edb_id:
vuln_refs["edb-id"].append(tmp.text)
tmpcves = tag.findall("cve")
for c in tmpcves:
if c.text not in cves:
cves.append(c.text)
cwe_ids = [c.text for c in tag.findall("cwe")]
references = [c.text for c in tag.findall("see_also")]
if not self.Vulnerability.find(name=findingName):
created, db_vuln = self.Vulnerability.find_or_create(
name=findingName,
severity=severity,
description=description,
remediation=solution,
)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if exploitable:
display_new("exploit avalable for " + findingName)
if vuln_refs:
db_vuln.exploit_reference = vuln_refs
else:
db_vuln = self.Vulnerability.find(name=findingName)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if vuln_refs:
if db_vuln.exploit_reference is not None:
for key in vuln_refs.keys():
if key not in db_vuln.exploit_reference.keys():
db_vuln.exploit_reference[key] = vuln_refs[key]
else:
for ref in vuln_refs[key]:
if ref not in db_vuln.exploit_reference[
key]:
db_vuln.exploit_reference[key].append(
ref)
else:
db_vuln.exploit_reference = vuln_refs
db_vuln.meta['CWEs'] = cwe_ids
db_vuln.meta['Refs'] = references
if tag.find("plugin_output") is not None:
plugin_output = tag.find("plugin_output").text
if not db_vuln.meta.get('plugin_output', False):
db_vuln.meta['plugin_output'] = {}
if not db_vuln.meta['plugin_output'].get(ip.ip_address, False):
db_vuln.meta['plugin_output'][ip.ip_address] = {}
if not db_vuln.meta['plugin_output'][ip.ip_address].get(
port, False):
db_vuln.meta['plugin_output'][ip.ip_address][port] = []
if plugin_output not in db_vuln.meta['plugin_output'][
ip.ip_address][port]:
db_vuln.meta['plugin_output'][ip.ip_address][port].append(
plugin_output)
if not self.args.disable_mitre:
for cve in cves:
if not self.CVE.find(name=cve):
try:
url = 'https://nvd.nist.gov/vuln/detail//{}'
res = requests.get(url.format(cve)).text
cveDescription = res.split(
'<p data-testid="vuln-description">')[1].split(
'</p>')[0]
if 'vuln-cvssv3-base-score' in res:
cvss = float(
res.split(
'<span data-testid="vuln-cvssv3-base-score">'
)[1].split('</span>')[0].strip())
else:
cvss = float(
res.split(
'<span data-testid="vuln-cvssv2-base-score">'
)[1].split('</span>')[0].strip())
cveDescription = res["summary"]
cvss = float(res["cvss"])
except Exception:
cveDescription = None
cvss = None
if not self.CVE.find(name=cve):
created, db_cve = self.CVE.find_or_create(
name=cve,
description=cveDescription,
temporal_score=cvss,
)
db_cve.vulnerabilities.append(db_vuln)
else:
db_cve = self.CVE.find(name=cve)
if (db_cve.description is None and
cveDescription is not None # noqa: W503
):
db_cve.description = cveDescription
if db_cve.temporal_score is None and cvss is not None:
db_cve.temporal_score = cvss
db_cve.vulnerabilities.append(db_vuln)
def process_data(self, nFile, args):
display("Reading " + nFile)
tree = ET.parse(nFile)
root = tree.getroot()
self.args = args
for ReportHost in root.iter("ReportHost"):
os = []
hostname = ""
hostIP = ""
for HostProperties in ReportHost.iter("HostProperties"):
for tag in HostProperties:
if tag.get("name") == "host-ip":
hostIP = tag.text
if tag.get("name") == "host-fqdn":
hostname = tag.text.lower()
hostname = hostname.replace("www.", "")
if tag.get("name") == "operating-system":
os = tag.text.split("\n")
if hostIP: # apparently nessus doesn't always have an IP to work with...
if hostname:
display("Gathering Nessus info for {} ( {} )".format(
hostIP, hostname))
else:
display("Gathering Nessus info for {}".format(hostIP))
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
if hostname:
created, domain = self.Domain.find_or_create(
domain=hostname)
if ip not in domain.ip_addresses:
ip.save()
domain.ip_addresses.append(ip)
domain.save()
if os:
for o in os:
if not ip.OS:
ip.OS = o
else:
if o not in ip.OS.split(" OR "):
ip.OS += " OR " + o
self.getVulns(ip, ReportHost)
self.IPAddress.commit()
return
class Module(ModuleTemplate):
"""
The Shodan module will either iterate through Shodan search results from net:<cidr>
for all scoped CIDRs, or a custom search query. The resulting IPs and ports will be
added to the database, along with a dictionary object of the API results.
"""
name = "ShodanImport"
def __init__(self, db):
self.db = db
self.Port = PortRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.ScopeCidr = ScopeCIDRRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-k",
"--api_key",
help="API Key for accessing Shodan")
self.options.add_argument(
"-s", "--search", help="Custom search string (will use credits)")
self.options.add_argument(
"-i",
"--import_db",
help="Import scoped IPs from the database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Rescan CIDRs already processed",
action="store_true")
self.options.add_argument("--fast",
help="Use 'net' filter. (May use credits)",
action="store_true")
self.options.add_argument(
"--cidr_only",
help="Import only CIDRs from database (not individual IPs)",
action="store_true",
)
self.options.add_argument("--target",
"-t",
help="Scan a specific CIDR/IP")
def run(self, args):
ranges = []
cidrs = []
ips = []
search = []
if not args.api_key:
display_error("You must supply an API key to use shodan!")
return
if args.search:
search = [args.search]
if args.import_db:
if args.rescan:
if args.fast:
search += [
"net:{}".format(c.cidr) for c in self.ScopeCidr.all()
]
else:
cidrs += [c.cidr for c in self.ScopeCidr.all()]
if not args.cidr_only:
ips += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active")
]
else:
if args.fast:
search += [
"net:{}".format(c.cidr)
for c in self.ScopeCidr.all(tool=self.name)
]
else:
cidrs += [
c.cidr for c in self.ScopeCidr.all(tool=self.name)
]
if not args.cidr_only:
ips += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active",
tool=self.name)
]
if args.target:
if '/' not in args.target:
ips += [args.target]
elif args.fast:
cidrs += ["net:{}".format(args.target)]
else:
cidrs += [args.target]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
ranges += ips
ranges += search
display(
"Doing a total of {} queries. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(len(ranges), int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for c in cidrs:
ranges = [str(i) for i in IPNetwork(c)]
display(
"Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(c, int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for r in ranges:
self.get_shodan(r, args)
created, cd = self.ScopeCidr.find_or_create(cidr=c)
if created:
cd.delete()
else:
cd.set_tool(self.name)
self.ScopeCidr.commit()
display(
"Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(len(ips), int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for i in ips:
self.get_shodan(i, args)
created, ip = self.IPAddress.find_or_create(ip_address=i)
if created:
ip.delete()
else:
ip.set_tool(self.name)
self.IPAddress.commit()
for s in search:
self.get_shodan(s, args)
if s[:4] == "net:":
created, cd = self.ScopeCidr.find_or_create(cidr=s[4:])
if created:
cd.delete()
else:
cd.set_tool(self.name)
self.ScopeCidr.commit()
def get_shodan(self, r, args):
api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
api_search_url = (
"https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
time.sleep(1)
if ":" in r:
display("Doing Shodan search: {}".format(r))
try:
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
if results.get(
"error") and "request timed out" in results["error"]:
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
total = len(results["matches"])
matches = []
i = 1
# pdb.set_trace()
while total > 0:
display("Adding {} results from page {}".format(total, i))
matches += results["matches"]
i += 1
try:
time.sleep(1)
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
i)).text)
if (results.get("error") and "request timed out"
in results["error"] # noqa: W503
):
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
total = len(results["matches"])
except Exception as e:
display_error("Something went wrong: {}".format(e))
total = 0
pdb.set_trace()
domains = []
for res in matches:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(ip_address=IP,
port_number=port_str,
proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
if res.get("ssl", {}).get('cert', {}).get('extensions'):
for d in res['ssl']['cert']['extensions']:
if d['name'] == 'subjectAltName':
domains += get_domains_from_data(d['name'])
if res.get("ssl", {}).get('cert', {}).get(
'subject', {}
).get('CN') and '*' not in res['ssl']['cert']['subject']['CN']:
domains.append(res['ssl']['cert']['subject']['CN'])
if res.get('hostnames'):
domains += res['hostnames']
for d in list(set(domains)):
display("Adding discovered domain {}".format(only_valid(d)))
created, domain = self.Domain.find_or_create(
domain=only_valid(d))
else:
display("Searching for {}".format(r))
try:
results = json.loads(
requests.get(api_host_url.format(r, args.api_key)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
# pdb.set_trace()
if results.get("data", False):
display("{} results found for: {}".format(
len(results["data"]), r))
domains = []
for res in results["data"]:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP, port_number=port_str, proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
if res.get("ssl", {}).get('cert', {}).get('extensions'):
for d in res['ssl']['cert']['extensions']:
if d['name'] == 'subjectAltName':
domains += get_domains_from_data(d['data'])
display(
"Domains discovered in subjectAltName: {}".
format(", ".join(
get_domains_from_data(d['data']))))
if res.get("ssl", {}).get('cert', {}).get(
'subject', {}).get('CN') and '*' not in res['ssl'][
'cert']['subject']['CN']:
domains.append(res['ssl']['cert']['subject']['CN'])
if res.get('hostnames'):
domains += res['hostnames']
for d in list(set(domains)):
display("Adding discovered domain {}".format(d))
created, domain = self.Domain.find_or_create(domain=d)
class Module(ToolTemplate):
"""
Module for running nmap. Make sure to pass all nmap-specific arguments at the end, after --tool_args
"""
name = "Nmap"
binary_name = "nmap"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Vulnerability = VulnRepository(db, self.name)
self.CVE = CVERepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"--hosts",
help=
"Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS",
nargs="+",
)
self.options.add_argument("--hosts_file", help="File containing hosts")
self.options.add_argument(
"-i",
"--hosts_database",
help="Use unscanned hosts from the database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Overwrite files without asking",
action="store_true")
self.options.add_argument(
"--filename",
help="Output filename. By default will use the current timestamp.",
)
self.options.add_argument(
"--ssl_cert_mode",
help=
"Scan only SSL enabled hosts to collect SSL certs (and domain names)",
action="store_true",
)
self.options.add_argument(
"--filter_ports",
help=
"Comma separated list of protoPort to filter out of results. Useful if firewall returns specific ports open on every host. Ex: t80,u5060"
)
self.options.set_defaults(timeout=None)
self.options.add_argument("--import_file",
help="Import results from an Nmap XML file.")
def get_targets(self, args):
self.args = args
if args.import_file:
args.no_binary = True
return [{"target": "", "output": args.import_file}]
targets = []
if args.hosts:
if type(args.hosts) == list:
for h in args.hosts:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
else:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
if args.hosts_database:
if args.rescan:
targets += [
h.ip_address
for h in self.IPAddress.all(scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all()]
else:
targets += [
h.ip_address
for h in self.IPAddress.all(tool=self.name,
scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)]
if args.hosts_file:
for h in [
l for l in open(args.hosts_file).read().split("\n") if l
]:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
data = []
if args.ssl_cert_mode:
ports = self.Port.all(service_name='https')
data = list(set([i.ip_address.ip_address for i in ports]))
port_numbers = list(set([str(i.port_number) for i in ports]))
args.tool_args += " -sV -p {} --script ssl-cert ".format(
','.join(port_numbers))
else:
# Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs
for t in targets:
ips = [str(i) for i in list(IPNetwork(t))]
data += ips
_, file_name = tempfile.mkstemp()
open(file_name, "w").write("\n".join(list(set(data))))
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
if args.filename:
output_path = os.path.join(self.path, args.filename)
else:
output_path = os.path.join(
self.path,
"nmap-scan-%s.xml" %
datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"),
)
return [{"target": file_name, "output": output_path}]
def build_cmd(self, args):
command = "sudo " + self.binary + " -oX {output} -iL {target} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
self.import_nmap(cmds[0]["output"])
if cmds[0]["target"]:
os.unlink(cmds[0]["target"])
def parseHeaders(self, httpHeaders):
bsHeaders = [
"Pragma",
"Expires",
"Date",
"Transfer-Encoding",
"Connection",
"X-Content-Type-Options",
"Cache-Control",
"X-Frame-Options",
"Content-Type",
"Content-Length",
"(Request type",
]
keepHeaders = {}
for i in range(0, len(httpHeaders)):
if httpHeaders[i].strip() != "" and httpHeaders[i].split(
":")[0].strip() not in " ".join(bsHeaders):
hName = httpHeaders[i].split(":")[0].strip()
hValue = "".join(httpHeaders[i].split(":")[1:]).strip()
keepHeaders[hName] = hValue
if keepHeaders == {}:
keepHeaders = ""
return keepHeaders
def import_nmap(
self,
filename): # domains={}, ips={}, rejects=[] == temp while no db
nFile = filename
# pdb.set_trace()
try:
tree = ET.parse(nFile)
root = tree.getroot()
hosts = root.findall("host")
except Exception as e:
print("Error: {}" % e)
print(nFile + " doesn't exist somehow...skipping")
return
for host in hosts:
hostIP = host.find("address").get("addr")
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
for hostname in host.findall("hostnames/hostname"):
hostname = hostname.get("name")
hostname = hostname.lower().replace("www.", "")
# reHostname = re.search(
# r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname
# ) # attempt to not get PTR record
# if not reHostname:
created, domain = self.Domain.find_or_create(domain=hostname)
if ip not in domain.ip_addresses:
domain.ip_addresses.append(ip)
domain.save()
for port in host.findall("ports/port"):
if port.find("state").get("state"):
portState = port.find("state").get("state")
hostPort = port.get("portid")
portProto = port.get("protocol")
# pdb.set_trace()
if not self.args.filter_ports or int(hostPort) not in [
int(p) for p in self.args.filter_ports.split(',')
]:
created, db_port = self.Port.find_or_create(
port_number=hostPort,
status=portState,
proto=portProto,
ip_address=ip,
)
if port.find("service") is not None:
portName = port.find("service").get("name")
if portName == "http" and hostPort == "443":
portName = "https"
else:
portName = "Unknown"
if created:
db_port.service_name = portName
info = db_port.info
if not info:
info = {}
for script in port.findall(
"script"): # just getting commonName from cert
if script.get("id") == "ssl-cert":
db_port.cert = script.get("output")
cert_domains = self.get_domains_from_cert(
script.get("output"))
for hostname in cert_domains:
hostname = hostname.lower().replace(
"www.", "")
created, domain = self.Domain.find_or_create(
domain=hostname)
if created:
print("New domain found: %s" %
hostname)
elif script.get("id") == "vulners":
print("Gathering vuln info for {} : {}/{}\n".
format(hostIP, portProto, hostPort))
self.parseVulners(script.get("output"),
db_port)
elif script.get("id") == "banner":
info["banner"] = script.get("output")
elif script.get("id") == "http-headers":
httpHeaders = script.get("output")
httpHeaders = httpHeaders.strip().split("\n")
keepHeaders = self.parseHeaders(httpHeaders)
info["http-headers"] = keepHeaders
elif script.get("id") == "http-auth":
info["http-auth"] = script.get("output")
elif script.get("id") == "http-title":
info["http-title"] = script.get("output")
db_port.info = info
db_port.save()
self.IPAddress.commit()
def parseVulners(self, scriptOutput, db_port):
urls = re.findall(r"(https://vulners.com/cve/CVE-\d*-\d*)",
scriptOutput)
for url in urls:
vuln_refs = []
exploitable = False
cve = url.split("/cve/")[1]
vulners = requests.get("https://vulners.com/cve/%s" % cve).text
exploitdb = re.findall(
r"https://www.exploit-db.com/exploits/\d{,7}", vulners)
for edb in exploitdb:
exploitable = True
if edb.split("/exploits/")[1] not in vuln_refs:
vuln_refs.append(edb.split("/exploits/")[1])
if not self.CVE.find(name=cve):
# print "Gathering CVE info for", cve
try:
res = json.loads(
requests.get("http://cve.circl.lu/api/cve/%s" %
cve).text)
cveDescription = res["summary"]
cvss = float(res["cvss"])
findingName = res["oval"][0]["title"]
if int(cvss) <= 3:
severity = 1
elif (int(cvss) / 2) == 5:
severity = 4
else:
severity = int(cvss) / 2
if not self.Vulnerability.find(name=findingName):
# print "Creating", findingName
created, db_vuln = self.Vulnerability.find_or_create(
name=findingName,
severity=severity,
description=cveDescription,
)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if vuln_refs:
db_vuln.exploit_reference = {"edb-id": vuln_refs}
db_vuln.save()
else:
# print "modifying",findingName
db_vuln = self.Vulnerability.find(name=findingName)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if vuln_refs:
db_vuln.exploitable = exploitable
if db_vuln.exploit_reference is not None:
if "edb-id" in db_vuln.exploit_reference:
for ref in vuln_refs:
if (ref not in db_vuln.
exploit_reference["edb-id"]):
db_vuln.exploit_reference[
"edb-id"].append(ref)
else:
db_vuln.exploit_reference[
"edb-id"] = vuln_refs
else:
db_vuln.exploit_reference = {
"edb-id": vuln_refs
}
db_vuln.save()
if not self.CVE.find(name=cve):
created, db_cve = self.CVE.find_or_create(
name=cve,
description=cveDescription,
temporal_score=cvss)
db_cve.vulnerabilities.append(db_vuln)
db_cve.save()
else:
db_cve = self.CVE.find(name=cve)
db_cve.vulnerabilities.append(db_vuln)
db_cve.save()
self.Vulnerability.commit()
self.CVE.commit()
except Exception:
print(
"something went wrong with the vuln/cve info gathering"
)
if vulners:
print(
"Vulners report was found but no exploit-db was discovered"
)
# "Affected vulners items"
# print vulners
print("Affected CVE")
print(cve)
pass
else:
db_cve = self.CVE.find(name=cve)
for db_vulns in db_cve.vulnerabilities:
if db_port not in db_vulns.ports:
db_vulns.ports.append(db_port)
return
def get_domains_from_cert(self, cert):
# Shamelessly lifted regex from stack overflow
regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}"
domains = list(
set([d for d in re.findall(regex, cert) if "*" not in d]))
return domains
class Module(ToolTemplate):
"""
Module for running masscan. Make sure to pass all masscan-specific arguments at the end, after --tool_args
"""
name = "Masscan"
binary_name = "masscan"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Vulnerability = VulnRepository(db, self.name)
self.CVE = CVERepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"--hosts",
help=
"Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS",
nargs="+",
)
self.options.add_argument("--hosts_file", help="File containing hosts")
self.options.add_argument(
"-i",
"--hosts_database",
help="Use unscanned hosts from the database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Overwrite files without asking",
action="store_true")
self.options.add_argument(
"--filename",
help="Output filename. By default will use the current timestamp.",
)
self.options.set_defaults(timeout=None)
self.options.add_argument(
"--import_file",
help="Import results from an Masscan/Nmap XML file.")
def get_targets(self, args):
if args.import_file:
args.no_binary = True
return [{"target": "", "output": args.import_file}]
targets = []
if args.hosts:
if type(args.hosts) == list:
for h in args.hosts:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
else:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
if args.hosts_database:
if args.rescan:
targets += [
h.ip_address
for h in self.IPAddress.all(scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all()]
else:
targets += [
h.ip_address
for h in self.IPAddress.all(tool=self.name,
scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)]
if args.hosts_file:
for h in [
l for l in open(args.hosts_file).read().split("\n") if l
]:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
# Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs
data = []
for t in targets:
ips = [str(i) for i in list(IPNetwork(t))]
data += ips
_, file_name = tempfile.mkstemp()
open(file_name, "w").write("\n".join(set(data)))
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
if args.filename:
output_path = os.path.join(self.path, args.filename)
else:
output_path = os.path.join(
self.path,
"masscan-%s.xml" %
datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"),
)
return [{"target": file_name, "output": output_path}]
def build_cmd(self, args):
command = "sudo " + self.binary + " -oX {output} -iL {target} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
self.import_masscan(cmds[0]["output"])
if cmds[0]["target"]:
os.unlink(cmds[0]["target"])
def import_masscan(
self,
filename): # domains={}, ips={}, rejects=[] == temp while no db
nFile = filename
try:
tree = ET.parse(nFile)
root = tree.getroot()
hosts = root.findall("host")
except Exception:
print(nFile + " doesn't exist somehow...skipping")
return
for host in hosts:
hostIP = host.find("address").get("addr")
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
for hostname in host.findall("hostnames/hostname"):
hostname = hostname.get("name")
hostname = hostname.lower().replace("www.", "")
# reHostname = re.search(
# r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname
# ) # attempt to not get PTR record
# if not reHostname:
created, domain = self.Domain.find_or_create(domain=hostname)
if ip not in domain.ip_addresses:
domain.ip_addresses.append(ip)
domain.save()
for port in host.findall("ports/port"):
if port.find("state").get("state"):
portState = port.find("state").get("state")
hostPort = port.get("portid")
portProto = port.get("protocol")
created, db_port = self.Port.find_or_create(
port_number=hostPort,
status=portState,
proto=portProto,
ip_address=ip,
)
info = db_port.info
if not info:
info = {}
if port.find("service") is not None:
service = port.find("service")
portName = service.get("name")
if portName == "http" and hostPort == "443":
portName = "https"
banner = service.get("banner", None)
if banner:
print("Found banner: {}".format(banner))
info["banner"] = banner
else:
portName = "Unknown"
if created:
db_port.service_name = portName
db_port.info = info
db_port.save()
self.IPAddress.commit()
def get_domains_from_cert(self, cert):
# Shamelessly lifted regex from stack overflow
regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}"
domains = list(
set([d for d in re.findall(regex, cert) if "*" not in d]))
return domains
class Module(ModuleTemplate):
"""
The Shodan module will either iterate through Shodan search results from net:<cidr>
for all scoped CIDRs, or a custom search query. The resulting IPs and ports will be
added to the database, along with a dictionary object of the API results.
"""
name = "ShodanImport"
def __init__(self, db):
self.db = db
self.Port = PortRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.ScopeCidr = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-k",
"--api_key",
help="API Key for accessing Shodan")
self.options.add_argument(
"-s", "--search", help="Custom search string (will use credits)")
self.options.add_argument(
"-i",
"--import_db",
help="Import scoped IPs from the database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Rescan CIDRs already processed",
action="store_true")
self.options.add_argument("--fast",
help="Use 'net' filter. (May use credits)",
action="store_true")
self.options.add_argument(
"--cidr_only",
help="Import only CIDRs from database (not individual IPs)",
action="store_true",
)
def run(self, args):
if not args.api_key:
display_error("You must supply an API key to use shodan!")
return
if args.search:
ranges = [args.search]
if args.import_db:
ranges = []
if args.rescan:
if args.fast:
ranges += [
"net:{}".format(c.cidr) for c in self.ScopeCidr.all()
]
else:
cidrs = [c.cidr for c in self.ScopeCidr.all()]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
if not args.cidr_only:
ranges += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active")
]
else:
if args.fast:
ranges += [
"net:{}".format(c.cidr)
for c in self.ScopeCidr.all(tool=self.name)
]
else:
cidrs = [
c.cidr for c in self.ScopeCidr.all(tool=self.name)
]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
if not args.cidr_only:
ranges += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active",
tool=self.name)
]
api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
api_search_url = (
"https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
for r in ranges:
time.sleep(1)
if ":" in r:
display("Doing Shodan search: {}".format(r))
try:
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
if results.get("error") and "request timed out" in results[
"error"]:
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
total = len(results["matches"])
matches = []
i = 1
while total > 0:
display("Adding {} results from page {}".format(total, i))
matches += results["matches"]
i += 1
try:
time.sleep(1)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
i)).text)
if (results.get("error") and "request timed out"
in results["error"] # noqa: W503
):
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
total = len(results["matches"])
except Exception as e:
display_error("Something went wrong: {}".format(e))
total = 0
pdb.set_trace()
for res in matches:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP, port_number=port_str, proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
else:
try:
results = json.loads(
requests.get(api_host_url.format(r,
args.api_key)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
# pdb.set_trace()
if results.get("data", False):
display("{} results found for: {}".format(
len(results["data"]), r))
for res in results["data"]:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP,
port_number=port_str,
proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
self.IPAddress.commit()
class Report(ReportTemplate):
'''
This report displays the various HTTP header findings we have
'''
name = "HostSummary"
markdown = ['', '-', '--']
def __init__(self, db):
self.db = db
self.IPAddress = IPRepository(db, self.name)
def set_options(self):
super(Report, self).set_options()
self.options.add_argument('--output_html',
help="Output HTML file",
default="output.html")
self.options.add_argument(
'-g',
'--include_gowitness',
help="Include Gowitness data and screenshots",
action="store_true")
self.options.add_argument('-f',
'--include_ffuf',
help="Include FFuF results",
action="store_true")
def run(self, args):
ip_port_result_data = {}
ip_data = {}
gowitness_data = {}
gowitness_databases = []
ffuf_data = {}
if args.include_ffuf:
path = os.path.join(self.base_config["PROJECT"]["base_path"],
'output', 'FFuF')
for (dirpath, dirnames, filenames) in os.walk(path):
for f in filenames:
# pdb.set_trace()
if f[:4] == 'http' and f.count('_') == 4:
proto, _, _, domain, dirty_port = f.split('_')
port = dirty_port.split('-')[0]
data = json.loads(
open(os.path.join(dirpath, f)).read())
url = "{}:{}".format(domain, port)
attack = data['commandline'].split(' -u ')[1].split(
' ')[0]
wordlist = data['commandline'].split(' -w ')[1].split(
' ')[0]
if not ffuf_data.get(url,
False) or not ffuf_data[url].get(
attack, False):
ffuf_data[url] = {
attack: {
'wordlist': wordlist,
'status': {}
}
}
status = ffuf_data[url][attack]['status']
status_count = {}
for r in data['results']:
if not status.get(r['status']):
status[r['status']] = {}
if not status_count.get(r['status'], False):
status_count[r['status']] = 1
status_count[r['status']] += 1
if status_count[r['status']] < 11:
status[r['status']][r['input']] = r['words']
if args.include_gowitness:
path = os.path.join(self.base_config["PROJECT"]["base_path"],
'output', 'Gowitness')
for (dirpath, dirnames, filenames) in os.walk(path):
if 'gowitness.db' in filenames:
gowitness_databases.append(
os.path.join(dirpath, 'gowitness.db'))
for database in gowitness_databases:
data = open(database).read().split('\n')
for d in data:
if '{"url"' in d:
j = json.loads(d)
try:
j['image_data'] = b64encode(
open(j['screenshot_file'],
'rb').read()).decode()
except Exception as e:
print("Could not get image: {}. Error: {}".format(
j['screenshot_file'], e))
url = j['url'].split('/')[-1]
if not gowitness_data.get(url, False):
gowitness_data[url] = []
gowitness_data[url].append(j)
for i in self.IPAddress.all(scope_type="active"):
if i.domains or [p for p in i.ports if p.port_number > 0]:
ip_data[i.ip_address] = {'domains': [], "ports": {}}
domains = [d.domain for d in i.domains]
ip_data[i.ip_address]['domains'] = domains
for p in i.ports:
if p.port_number > 0:
hosts = ["{}:{}".format(i.ip_address, p.port_number)
] + [
"{}:{}".format(d, p.port_number)
for d in domains
]
port_data = {
'vulns': {},
'gw': [],
'service': p.service_name
}
for h in hosts:
if gowitness_data.get(h):
port_data['gw'].append(gowitness_data[h])
ip_data[i.ip_address]['ports'][
p.port_number] = port_data
for v in p.vulnerabilities:
try:
output = v.meta['plugin_output'][i.ip_address][
str(p.port_number)]
except Exception as e:
output = []
port_data['vulns'][v.name] = {
'desc': v.description,
'sev': sev_map[v.severity],
'exploitable': v.exploitable,
"output": list(set(output))
}
template_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'templates')
t = Template(
open(os.path.join(template_path, 'main_template.html')).read())
# ports = Template(open(os.path.join(template_path, 'ports_template.html')).read())
nessus = Template(
open(os.path.join(template_path, 'nessus_template.html')).read())
gowit = Template(
open(os.path.join(template_path,
'gowitness_template.html')).read())
ffuf = Template(
open(os.path.join(template_path, 'ffuf_template.html')).read())
data_path = os.path.join(os.path.dirname(args.output_html), 'data')
static_path = os.path.join(os.path.dirname(args.output_html), 'static')
static_source = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'static')
# data_path = os.path.dirname(args.output_html)
if data_path and not os.path.isdir(data_path):
os.mkdir(data_path)
if static_path and not os.path.isdir(static_path):
os.mkdir(static_path)
static_files = [
'bootstrap.min.css', 'bootstrap.min.css.map', 'bootstrap.min.js',
'jquery-3.3.1.min.js', 'popper.min.js'
]
for s in static_files:
shutil.copyfile(os.path.join(static_source, s),
os.path.join(static_path, s))
new_ips = {}
ip_domains = {}
for ips, data in ip_data.items():
if not new_ips.get(ips, False):
new_ips[ips] = {'ports': {}, 'domains': []}
if data['ports']:
# pdb.set_trace()
port_data = data['ports']
port_res = {}
for p, pdata in port_data.items():
if not port_res.get(p, False):
port_res[p] = {'service': pdata['service'], 'data': []}
if pdata['vulns']:
open(
os.path.join(data_path,
"nessus-{}-{}.html".format(ips, p)),
'w').write(
nessus.render(vulns=pdata['vulns'],
ip_address=ips,
port=p))
port_res[p]['data'].append('nessus')
if pdata['gw']:
open(
os.path.join(data_path,
"gowitness-{}-{}.html".format(ips,
p)),
'w').write(gowit.render(gw_data=pdata['gw']))
port_res[p]['data'].append('gowitness')
ffuf_d = {}
if ffuf_data.get("{}:{}".format(ips, p), False):
f = False
for v in [
v['status'] for k, v in ffuf_data[
"{}:{}".format(ips, p)].items()
]:
if v:
f = True
if f:
ffuf_d[ips] = ffuf_data["{}:{}".format(ips, p)]
for d in data['domains']:
if ffuf_data.get("{}:{}".format(d, p), False):
f = False
for v in [
v['status'] for k, v in ffuf_data[
"{}:{}".format(d, p)].items()
]:
if v:
f = True
if f:
ffuf_d[d] = ffuf_data["{}:{}".format(d, p)]
if ffuf_d:
open(
os.path.join(data_path,
"ffuf-{}-{}.html".format(ips, p)),
'w').write(ffuf.render(ffuf_data=ffuf_d))
port_res[p]['data'].append('ffuf')
# open(os.path.join(data_path, "{}-{}.html".format(ips, p)), 'w').write(ports.render(ip=ips, port = port_res[p], port_num=p))
new_ips[ips]['ports'] = port_res
if data['domains']:
new_ips[ips]['domains'] = data['domains']
open(args.output_html, 'w').write(t.render(ips=new_ips))
class Module(ModuleTemplate):
"""
Ingests domains and IPs. Domains get ip info and cidr info, and IPs get
CIDR info.
"""
name = "Ingestor"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.CIDR = CIDRRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-d",
"--import_domains",
help="Either domain to import or file containing domains to import. One per line",
)
self.options.add_argument(
"-i",
"--import_ips",
help="Either IP/range to import or file containing IPs and ranges, one per line.",
)
self.options.add_argument(
"-a",
"--active",
help="Set scoping on imported data as active",
action="store_true",
)
self.options.add_argument(
"-p",
"--passive",
help="Set scoping on imported data as passive",
action="store_true",
)
self.options.add_argument(
"-sc",
"--scope_cidrs",
help="Cycle through out of scope networks and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument(
"-sb",
"--scope_base_domains",
help="Cycle through out of scope base domains and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument("--descope", help="Descope an IP, domain, or CIDR")
self.options.add_argument(
"-Ii",
"--import_database_ips",
help="Import IPs from database",
action="store_true",
)
self.options.add_argument(
"--force",
help="Force processing again, even if already processed",
action="store_true",
)
def run(self, args):
self.in_scope = args.active
self.passive_scope = args.passive
if args.descope:
if "/" in args.descope:
self.descope_cidr(args.descope)
elif check_string(args.descope):
pass
else:
self.descope_ip(args.descope)
# Check if in ScopeCIDR and remove if found
if args.import_ips:
try:
ips = open(args.import_ips)
for line in ips:
if line.strip():
if "/" in line or "-" in line:
self.process_cidr(line)
else:
self.process_ip(line.strip(), force_scope=True)
self.Domain.commit()
except IOError:
if "/" in args.import_ips or "-" in args.import_ips:
self.process_cidr(args.import_ips)
else:
self.process_ip(args.import_ips.strip(), force_scope=True)
self.Domain.commit()
if args.import_domains:
try:
domains = open(args.import_domains)
for line in domains:
if line.strip():
self.process_domain(line.strip())
self.Domain.commit()
except IOError:
self.process_domain(args.import_domains.strip())
self.Domain.commit()
if args.scope_base_domains:
base_domains = self.BaseDomain.all(in_scope=False, passive_scope=False)
for bd in base_domains:
self.reclassify_domain(bd)
self.BaseDomain.commit()
def get_domain_ips(self, domain):
ips = []
try:
answers = dns.resolver.query(domain, "A")
for a in answers:
ips.append(a.address)
return ips
except Exception:
return []
def process_domain(self, domain_str):
created, domain = self.Domain.find_or_create(
only_tool=True,
domain=domain_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if (
domain.in_scope != self.in_scope
or domain.passive_scope != self.passive_scope # noqa: W503
):
display(
"Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (domain_str, self.in_scope, self.passive_scope)
)
domain.in_scope = self.in_scope
domain.passive_scope = self.passive_scope
domain.update()
if domain.base_domain.domain == domain.domain:
display("Name also matches a base domain. Updating that as well.")
domain.base_domain.in_scope = self.in_scope
domain.base_domain.passive_scope = self.passive_scope
domain.base_domain.update()
def process_ip(self, ip_str, force_scope=True):
created, ip = self.IPAddress.find_or_create(
only_tool=True,
ip_address=ip_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope:
display(
"IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (ip_str, self.in_scope, self.passive_scope)
)
ip.in_scope = self.in_scope
ip.passive_scope = self.passive_scope
ip.update()
return ip
def process_cidr(self, line):
display("Processing %s" % line)
if "/" in line:
created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
if created:
display_new("Adding %s to scoped CIDRs in database" % line.strip())
cidr.in_scope = True
cidr.update()
elif "-" in line:
start_ip, end_ip = line.strip().replace(" ", "").split("-")
if "." not in end_ip:
end_ip = ".".join(start_ip.split(".")[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
if created:
display_new("Adding %s to scoped CIDRs in database" % line.strip())
cidr.in_scope = True
cidr.update()
def reclassify_domain(self, bd):
if bd.meta.get("whois", False):
display_new("Whois data found for {}".format(bd.domain))
print(bd.meta["whois"])
res = six.input(
"Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] "
)
if res.lower() == "a":
bd.in_scope = True
bd.passive_scope = True
elif res.lower() == "p":
bd.in_scope = False
bd.passive_scope = True
else:
bd.in_scope = False
bd.passive_scope = False
bd.save()
else:
display_error(
"Unfortunately, there is no whois information for {}. Please populate it using the Whois module".format(
bd.domain
)
)
def descope_ip(self, ip):
ip = self.IPAddress.all(ip_address=ip)
if ip:
for i in ip:
display("Removing IP {} from scope".format(i.ip_address))
i.in_scope = False
i.passive_scope = False
i.update()
for d in i.domains:
in_scope_ips = [
ipa
for ipa in d.ip_addresses
if ipa.in_scope or ipa.passive_scope
]
if not in_scope_ips:
display(
"Domain {} has no more scoped IPs. Removing from scope.".format(
d.domain
)
)
d.in_scope = False
d.passive_scope = False
self.IPAddress.commit()
def descope_cidr(self, cidr):
CIDR = self.ScopeCIDR.all(cidr=cidr)
if CIDR:
for c in CIDR:
display("Removing {} from ScopeCIDRs".format(c.cidr))
c.delete()
cnet = IPNetwork(cidr)
for ip in self.IPAddress.all():
if IPAddress(ip.ip_address) in cnet:
self.descope_ip(ip.ip_address)