class Module(ModuleTemplate):
"""
The Shodan module will either iterate through Shodan search results from net:<cidr>
for all scoped CIDRs, or a custom search query. The resulting IPs and ports will be
added to the database, along with a dictionary object of the API results.
"""
name = "ShodanImport"
def __init__(self, db):
self.db = db
self.Port = PortRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.ScopeCidr = ScopeCIDRRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-k",
"--api_key",
help="API Key for accessing Shodan")
self.options.add_argument(
"-s", "--search", help="Custom search string (will use credits)")
self.options.add_argument(
"-i",
"--import_db",
help="Import scoped IPs from the database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Rescan CIDRs already processed",
action="store_true")
self.options.add_argument("--fast",
help="Use 'net' filter. (May use credits)",
action="store_true")
self.options.add_argument(
"--cidr_only",
help="Import only CIDRs from database (not individual IPs)",
action="store_true",
)
self.options.add_argument("--target",
"-t",
help="Scan a specific CIDR/IP")
def run(self, args):
ranges = []
cidrs = []
ips = []
search = []
if not args.api_key:
display_error("You must supply an API key to use shodan!")
return
if args.search:
search = [args.search]
if args.import_db:
if args.rescan:
if args.fast:
search += [
"net:{}".format(c.cidr) for c in self.ScopeCidr.all()
]
else:
cidrs += [c.cidr for c in self.ScopeCidr.all()]
if not args.cidr_only:
ips += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active")
]
else:
if args.fast:
search += [
"net:{}".format(c.cidr)
for c in self.ScopeCidr.all(tool=self.name)
]
else:
cidrs += [
c.cidr for c in self.ScopeCidr.all(tool=self.name)
]
if not args.cidr_only:
ips += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active",
tool=self.name)
]
if args.target:
if '/' not in args.target:
ips += [args.target]
elif args.fast:
cidrs += ["net:{}".format(args.target)]
else:
cidrs += [args.target]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
ranges += ips
ranges += search
display(
"Doing a total of {} queries. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(len(ranges), int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for c in cidrs:
ranges = [str(i) for i in IPNetwork(c)]
display(
"Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(c, int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for r in ranges:
self.get_shodan(r, args)
created, cd = self.ScopeCidr.find_or_create(cidr=c)
if created:
cd.delete()
else:
cd.set_tool(self.name)
self.ScopeCidr.commit()
display(
"Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(len(ips), int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for i in ips:
self.get_shodan(i, args)
created, ip = self.IPAddress.find_or_create(ip_address=i)
if created:
ip.delete()
else:
ip.set_tool(self.name)
self.IPAddress.commit()
for s in search:
self.get_shodan(s, args)
if s[:4] == "net:":
created, cd = self.ScopeCidr.find_or_create(cidr=s[4:])
if created:
cd.delete()
else:
cd.set_tool(self.name)
self.ScopeCidr.commit()
def get_shodan(self, r, args):
api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
api_search_url = (
"https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
time.sleep(1)
if ":" in r:
display("Doing Shodan search: {}".format(r))
try:
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
if results.get(
"error") and "request timed out" in results["error"]:
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
total = len(results["matches"])
matches = []
i = 1
# pdb.set_trace()
while total > 0:
display("Adding {} results from page {}".format(total, i))
matches += results["matches"]
i += 1
try:
time.sleep(1)
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
i)).text)
if (results.get("error") and "request timed out"
in results["error"] # noqa: W503
):
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
total = len(results["matches"])
except Exception as e:
display_error("Something went wrong: {}".format(e))
total = 0
pdb.set_trace()
domains = []
for res in matches:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(ip_address=IP,
port_number=port_str,
proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
if res.get("ssl", {}).get('cert', {}).get('extensions'):
for d in res['ssl']['cert']['extensions']:
if d['name'] == 'subjectAltName':
domains += get_domains_from_data(d['name'])
if res.get("ssl", {}).get('cert', {}).get(
'subject', {}
).get('CN') and '*' not in res['ssl']['cert']['subject']['CN']:
domains.append(res['ssl']['cert']['subject']['CN'])
if res.get('hostnames'):
domains += res['hostnames']
for d in list(set(domains)):
display("Adding discovered domain {}".format(only_valid(d)))
created, domain = self.Domain.find_or_create(
domain=only_valid(d))
else:
display("Searching for {}".format(r))
try:
results = json.loads(
requests.get(api_host_url.format(r, args.api_key)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
# pdb.set_trace()
if results.get("data", False):
display("{} results found for: {}".format(
len(results["data"]), r))
domains = []
for res in results["data"]:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP, port_number=port_str, proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
if res.get("ssl", {}).get('cert', {}).get('extensions'):
for d in res['ssl']['cert']['extensions']:
if d['name'] == 'subjectAltName':
domains += get_domains_from_data(d['data'])
display(
"Domains discovered in subjectAltName: {}".
format(", ".join(
get_domains_from_data(d['data']))))
if res.get("ssl", {}).get('cert', {}).get(
'subject', {}).get('CN') and '*' not in res['ssl'][
'cert']['subject']['CN']:
domains.append(res['ssl']['cert']['subject']['CN'])
if res.get('hostnames'):
domains += res['hostnames']
for d in list(set(domains)):
display("Adding discovered domain {}".format(d))
created, domain = self.Domain.find_or_create(domain=d)
class Module(ToolTemplate):
'''
This module uses Gobuster in the DNS brute forcing mode. Gobuster can be installed from:
https://github.com/OJ/gobuster
'''
name = "GobusterDNS"
binary_name = "gobuster"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Domain to brute force")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been brute forced",
action="store_true",
)
self.options.set_defaults(
timeout=600) # Kick the default timeout to 10 minutes
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
if args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
targets.append(d)
if args.import_database:
if args.rescan:
targets += [
b.domain for b in self.BaseDomain.all(scope_type="passive")
]
else:
targets += [
b.domain for b in self.BaseDomain.all(scope_type="passive",
tool=self.name)
]
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
"target":
t,
"output":
os.path.join(output_path,
t.replace("/", "_") + "-dns.txt"),
})
return res
def build_cmd(self, args):
cmd = self.binary + " dns "
cmd += " -o {output} -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for c in cmds:
output_path = c["output"]
if os.path.isfile(output_path):
data = open(output_path).read().split("\n")
for d in data:
if "Found: " in d:
new_domain = d.split(" ")[1].lower()
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
else:
display_error("{} not found.".format(output_path))
created, bd = self.BaseDomain.find_or_create(domain=c['target'])
bd.set_tool(self.name)
self.Domain.commit()
class Module(ToolTemplate):
name = "DNSRecon"
binary_name = "dnsrecon"
"""
This module runs DNSRecon on a domain or set of domains. This will extract found DNS entries.
It can also run over IP ranges, looking for additional domains in the PTR records.
DNSRecon can be installed from https://github.com/darkoperator/dnsrecon
"""
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Target domain for dnsRecon")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument("-r",
"--range",
help="Range to scan for PTR records")
self.options.add_argument(
"-R",
"--import_range",
help="Import CIDRs from in-scope ranges in database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Rescan domains already scanned",
action="store_true")
# self.options.add_argument('--import_output_xml', help="Import XML file")
# self.options.add_argument('--import_output_json', help="Import json file")
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain, passive_scope=True)
targets.append(domain.domain)
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(
domain=d, passive_scope=True)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(scope_type="passive",
tool=self.name)
for domain in domains:
targets.append(domain.domain)
elif args.range:
targets.append(args.range)
elif args.import_range:
if args.rescan:
cidrs = self.ScopeCIDR.all()
else:
cidrs = self.ScopeCIDR.all(tool=self.name)
for cidr in cidrs:
targets.append(cidr.cidr)
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
res = []
for t in targets:
res.append({
"target":
t,
"output":
os.path.join(self.path,
t.replace("/", "_") + ".json"),
})
return res
def build_cmd(self, args):
command = self.binary
if args.domain or args.file or args.import_database:
command += " -d {target} -j {output} "
else:
command += " -s -r {target} -j {output} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
for c in cmds:
target = c["target"]
output_path = c["output"]
try:
res = json.loads(open(output_path).read())
except IOError:
display_error("DnsRecon failed for {}".format(target))
continue
if " -d " in res[0]["arguments"]:
created, dbrec = self.Domain.find_or_create(domain=target)
dbrec.dns = res
dbrec.save()
for record in res:
domain = None
ip = None
if record.get("type") == "A" or record.get("type") == "PTR":
domain = record.get("name").lower().replace("www.", "")
ip = record.get("address")
elif record.get("type") == "MX":
domain = record.get("exchange").lower().replace("www.", "")
elif record.get("type") == "SRV" or record.get("type" == "NS"):
domain = record.get("target").lower().replace("www.", "")
elif record.get("type") == "SOA":
domain = record.get("mname").lower().replace("www.", "")
if domain:
created, domain_obj = self.Domain.find_or_create(
domain=domain)
if ip:
created, ip_obj = self.IPAddress.find_or_create(
ip_address=ip)
domain_obj.ip_addresses.append(ip_obj)
domain_obj.save()
if '/' in target:
created, bd = self.ScopeCIDR.find_or_create(cidr=target)
else:
created, bd = self.BaseDomain.find_or_create(domain=target)
bd.set_tool(self.name)
self.Domain.commit()
class Module(ModuleTemplate):
name = "Nessus"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Vulnerability = VulnRepository(db, self.name)
self.CVE = CVERepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"--import_file",
help=
"Import separated Nessus files separated by a space. DO NOT USE QUOTES OR COMMAS",
nargs="+",
)
self.options.add_argument(
"--launch",
help=
"Launch Nessus scan using Actively scoped IPs and domains in the database",
action="store_true",
)
self.options.add_argument("--job_name",
help="Job name inside Nessus",
default="Armory Job")
self.options.add_argument("--username", help="Nessus Username")
self.options.add_argument("--password", help="Nessus Password")
self.options.add_argument(
"--host",
help="Hostname:Port of Nessus web interface (ie localhost:8835")
self.options.add_argument("--uuid",
help="UUID of Nessus Policy to run")
self.options.add_argument("--policy_id", help="Policy ID to use")
self.options.add_argument("--folder_id",
help="ID for folder to store job in")
self.options.add_argument(
"--download",
help="Download Nessus job from server and import",
action="store_true",
)
self.options.add_argument("--job_id",
help="Job ID to download and import")
self.options.add_argument(
"--output_path",
help="Path to store downloaded file (Default: Nessus)",
default=self.name,
)
self.options.add_argument(
"--disable_mitre",
help="Disable mitre CVE data gathering.",
action="store_true",
)
def run(self, args):
if args.import_file:
for nFile in args.import_file:
self.process_data(nFile, args)
elif args.launch:
if (not args.username # noqa: W503
and not args.password # noqa: W503
and not args.host # noqa: W503
and not args.uuid # noqa: W503
and not args.policy_id # noqa: W503
and not args.folder_id # noqa: W503
):
display_error(
"You must supply a username, password, and host to launch a Nessus job"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
uuid=args.uuid,
policy_id=args.policy_id,
folder_id=args.folder_id,
)
ips = [
ip.ip_address
for ip in self.IPAddress.all(scope_type="active",
tool=self.name)
]
cidrs = [
cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)
]
domains = [
domain.domain
for domain in self.Domain.all(scope_type="active",
tool=self.name)
]
targets = ", ".join(merge_ranges(ips + cidrs) + domains)
res = n.launch_job(targets, args.job_name)
display("New Nessus job launched with ID {}".format(res))
display(
"Remember this number! You'll need it to download the job once it is done."
)
elif args.download:
if (not args.username # noqa: W503
and not args.password # noqa: W503
and not args.host # noqa: W503
and not args.job_id # noqa: W503
):
display_error(
"You must supply host, username, password and job_id to download a report to import"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
)
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(
output_path,
"Nessus-export-{}.nessus".format(int(time.time())))
n.export_file(args.job_id, output_path)
self.process_data(output_path, args)
def nessCheckPlugin(self, tag):
nessPlugins = [
"10759",
"77026",
"20089",
"56984",
"71049",
"70658",
"40984",
"11411",
]
pluginID = tag.get("pluginID")
if pluginID in nessPlugins:
# print pluginID + " is in the list"
if pluginID == "10759":
if tag.find("plugin_output") is not None:
return (
tag.find("plugin_output").text.split(
"\n\n")[3].strip()
) # returns IP for Web Server HTTP Header INternal IP disclosure
else:
return ""
if pluginID == "77026":
if tag.find("plugin_output") is not None:
return (
tag.find("plugin_output").text.split(
"\n\n")[3].strip()
) # Microsoft Exchange Client Access Server Information Disclosure (IP addy)
else:
return ""
if pluginID == "71049" or pluginID == "70658":
output = ""
if tag.find("plugin_output") is not None:
tmp = tag.find("plugin_output").text.split(":")[
1] # SSH Weak MAC & CBC Algorithms Enabled
# print "#"*5
tmp = tmp.split("\n\n")[1].replace(" ", "")
# print "#"*5
output = tmp.split("\n")
# print ", ".join(output)
return ", ".join(output)
if pluginID == "56984":
if tag.find("plugin_output") is not None:
tmp = (tag.find("plugin_output").text.split(
"This port supports ")[1].strip()
) # SSL / TLS Versions Supported
tmp = tmp.split("/")
bad = []
for i in tmp:
# print i
if "SSLv" in i:
bad.append(i)
elif "TLSv1.0" in i:
bad.append(i)
if bad != []:
return ", ".join(bad).rstrip(".")
else:
return ""
else:
return ""
if pluginID == "40984": # browsable web dirs
if tag.find("plugin_output") is not None:
tmp = (tag.find("plugin_output").text.split(
"The following directories are browsable :")
[1].strip())
directories = tmp.split("\n")
return "\n".join(directories)
if pluginID == "11411": # Backup Files Disclosure
if tag.find("plugin_output") is not None:
urls = []
tmp = (tag.find("plugin_output").text.split(
"It is possible to read the following backup file")
[1].strip())
tmpUrls = tmp.split("\n")
for url in tmpUrls:
if "URL" in url:
urls.append(url.split(":")[1].lstrip())
if urls:
return "\n".join(urls)
else:
return ""
if pluginID == "20089": # F5 cookie
if tag.find("plugin_output") is not None:
f5Output = []
cookieVal = []
output = tag.find("plugin_output").text.strip().split("\n")
for line in output:
# print line
line = line.split(":")
for i, item in enumerate(line):
item = item.strip()
if "Cookie" in item:
line.pop(i) # Pop to remove the first?
tmp = line.pop(i)
tmp.strip()
cookieVal.append(tmp)
else:
item = "".join(item)
f5Output.append(item)
f5Output = " : ".join(f5Output)
f5Output = f5Output.replace(" : : ", ", ")
f5Output += " [" + ", ".join(cookieVal) + "]"
c = 0
tmpF5Output = f5Output.split()
for i, letter in enumerate(tmpF5Output):
if letter == ":":
c += 1
if (c % 2) == 0:
tmpF5Output[i] = " "
return "".join(tmpF5Output).replace("[", " [")
else:
return ""
else:
return False
def getVulns(self, ip, ReportHost):
"""Gets vulns and associated services"""
for tag in ReportHost.iter("ReportItem"):
exploitable = False
cves = []
vuln_refs = {}
proto = tag.get("protocol")
port = tag.get("port")
svc_name = tag.get("svc_name").replace("?", "")
plugin_output = []
tmpPort = proto + "/" + port
if tmpPort.lower() == "tcp/443":
portName = "https"
elif tmpPort.lower() == "tcp/80":
portName = "http"
elif svc_name == "www":
plugin_name = tag.get("pluginName")
if "tls" in plugin_name.lower() or "ssl" in plugin_name.lower(
):
portName = "https"
else:
portName = "http"
else:
portName = svc_name
created, db_port = self.Port.find_or_create(port_number=port,
status="open",
proto=proto,
ip_address_id=ip.id)
if db_port.service_name == "http":
if portName == "https":
db_port.service_name = portName
elif db_port.service_name == "https":
pass
else:
db_port.service_name = portName
db_port.save()
if tag.get("pluginID") == "56984":
severity = 1
elif tag.get("pluginID") == "11411":
severity = 3
else:
severity = int(tag.get("severity"))
findingName = tag.get("pluginName")
description = tag.find("description").text
if tag.find(
"solution") is not None and tag.find("solution") != "n/a":
solution = tag.find("solution").text
else:
solution = "No Remediation From Nessus"
nessCheck = self.nessCheckPlugin(tag)
if nessCheck:
if not db_port.info:
db_port.info = {findingName: nessCheck}
else:
db_port.info[findingName] = nessCheck
db_port.save()
if tag.find("exploit_available") is not None:
exploitable = True
metasploits = tag.findall("metasploit_name")
if metasploits:
vuln_refs["metasploit"] = []
for tmp in metasploits:
vuln_refs["metasploit"].append(tmp.text)
edb_id = tag.findall("edb-id")
if edb_id:
vuln_refs["edb-id"] = []
for tmp in edb_id:
vuln_refs["edb-id"].append(tmp.text)
tmpcves = tag.findall("cve")
for c in tmpcves:
if c.text not in cves:
cves.append(c.text)
cwe_ids = [c.text for c in tag.findall("cwe")]
references = [c.text for c in tag.findall("see_also")]
if not self.Vulnerability.find(name=findingName):
created, db_vuln = self.Vulnerability.find_or_create(
name=findingName,
severity=severity,
description=description,
remediation=solution,
)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if exploitable:
display_new("exploit avalable for " + findingName)
if vuln_refs:
db_vuln.exploit_reference = vuln_refs
else:
db_vuln = self.Vulnerability.find(name=findingName)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if vuln_refs:
if db_vuln.exploit_reference is not None:
for key in vuln_refs.keys():
if key not in db_vuln.exploit_reference.keys():
db_vuln.exploit_reference[key] = vuln_refs[key]
else:
for ref in vuln_refs[key]:
if ref not in db_vuln.exploit_reference[
key]:
db_vuln.exploit_reference[key].append(
ref)
else:
db_vuln.exploit_reference = vuln_refs
db_vuln.meta['CWEs'] = cwe_ids
db_vuln.meta['Refs'] = references
if tag.find("plugin_output") is not None:
plugin_output = tag.find("plugin_output").text
if not db_vuln.meta.get('plugin_output', False):
db_vuln.meta['plugin_output'] = {}
if not db_vuln.meta['plugin_output'].get(ip.ip_address, False):
db_vuln.meta['plugin_output'][ip.ip_address] = {}
if not db_vuln.meta['plugin_output'][ip.ip_address].get(
port, False):
db_vuln.meta['plugin_output'][ip.ip_address][port] = []
if plugin_output not in db_vuln.meta['plugin_output'][
ip.ip_address][port]:
db_vuln.meta['plugin_output'][ip.ip_address][port].append(
plugin_output)
if not self.args.disable_mitre:
for cve in cves:
if not self.CVE.find(name=cve):
try:
url = 'https://nvd.nist.gov/vuln/detail//{}'
res = requests.get(url.format(cve)).text
cveDescription = res.split(
'<p data-testid="vuln-description">')[1].split(
'</p>')[0]
if 'vuln-cvssv3-base-score' in res:
cvss = float(
res.split(
'<span data-testid="vuln-cvssv3-base-score">'
)[1].split('</span>')[0].strip())
else:
cvss = float(
res.split(
'<span data-testid="vuln-cvssv2-base-score">'
)[1].split('</span>')[0].strip())
cveDescription = res["summary"]
cvss = float(res["cvss"])
except Exception:
cveDescription = None
cvss = None
if not self.CVE.find(name=cve):
created, db_cve = self.CVE.find_or_create(
name=cve,
description=cveDescription,
temporal_score=cvss,
)
db_cve.vulnerabilities.append(db_vuln)
else:
db_cve = self.CVE.find(name=cve)
if (db_cve.description is None and
cveDescription is not None # noqa: W503
):
db_cve.description = cveDescription
if db_cve.temporal_score is None and cvss is not None:
db_cve.temporal_score = cvss
db_cve.vulnerabilities.append(db_vuln)
def process_data(self, nFile, args):
display("Reading " + nFile)
tree = ET.parse(nFile)
root = tree.getroot()
self.args = args
for ReportHost in root.iter("ReportHost"):
os = []
hostname = ""
hostIP = ""
for HostProperties in ReportHost.iter("HostProperties"):
for tag in HostProperties:
if tag.get("name") == "host-ip":
hostIP = tag.text
if tag.get("name") == "host-fqdn":
hostname = tag.text.lower()
hostname = hostname.replace("www.", "")
if tag.get("name") == "operating-system":
os = tag.text.split("\n")
if hostIP: # apparently nessus doesn't always have an IP to work with...
if hostname:
display("Gathering Nessus info for {} ( {} )".format(
hostIP, hostname))
else:
display("Gathering Nessus info for {}".format(hostIP))
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
if hostname:
created, domain = self.Domain.find_or_create(
domain=hostname)
if ip not in domain.ip_addresses:
ip.save()
domain.ip_addresses.append(ip)
domain.save()
if os:
for o in os:
if not ip.OS:
ip.OS = o
else:
if o not in ip.OS.split(" OR "):
ip.OS += " OR " + o
self.getVulns(ip, ReportHost)
self.IPAddress.commit()
return
class Module(ToolTemplate):
name = "Subfinder"
binary_name = "subfinder"
def __init__(self, db):
self.db = db
self.BaseDomains = BaseDomainRepository(db, self.name)
self.Domains = DomainRepository(db, self.name)
self.IPs = IPRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Domain to run subfinder against.")
self.options.add_argument(
"-dL",
"--domain_list",
help="Read in a list of domains within the given file.",
)
self.options.add_argument(
"-i",
"--db_domains",
help="Import the domains from the database.",
action="store_true",
)
self.options.add_argument("--rescan",
help="Overwrite files without asking",
action="store_true")
def get_targets(self, args):
targets = []
outpath = ""
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
if args.domain:
out_file = os.path.join(outpath,
"{}.subfinder".format(args.domain))
targets.append({
"target": args.domain,
"output": os.path.join(output_path, out_file)
})
if args.db_domains:
if args.rescan:
domains = self.BaseDomains.all(scope_type="passive")
else:
domains = self.BaseDomains.all(tool=self.name,
scope_type="passive")
for d in domains:
out_file = os.path.join(outpath,
"{}.subfinder".format(d.domain))
targets.append({
"target": d.domain,
"output": os.path.join(output_path, out_file)
})
elif args.domain_list:
domains = io.open(args.domain_list,
encoding="utf-8").read().split("\n")
for d in domains:
if d:
targets.append({
"target":
d,
"output":
os.path.join(output_path, "{}.subfinder".format(d)),
})
return targets
def build_cmd(self, args):
if args.binary:
cmd = "{} ".format(args.binary)
else:
cmd = "{} ".format(self.binary_name)
cmd = "{} -o {} -d {}".format(cmd, "{output}", "{target}")
return cmd
def process_output(self, targets):
for target in targets:
try:
with io.open(target["output"], encoding="utf-8") as fd:
for line in fd:
domain = line.strip()
if domain[0] == '.':
domain = domain[1:]
ips = get_domain_ip.run(domain)
ip_obj = None
_, dom = self.Domains.find_or_create(domain=domain)
if ips:
for ip in ips:
_, ip_obj = self.IPs.find_or_create(
ip_address=ip)
if ip_obj:
dom.ip_addresses.append(ip_obj)
dom.save()
except FileNotFoundError:
display_error("File doesn't exist for {}".format(
target["output"]))
self.BaseDomains.commit()
self.IPs.commit()
def post_run(self, args):
# Remove the temporary db file if it was created.
if getattr(self, "db_domain_file", None):
try:
os.unlink(self.db_domain_file)
except IOError as e:
print("Failed to remove the Subfinder db temp file: '{}'.".
format(e))
def __get_tempfile(self, domain=None, args=None):
# Create a temporary file and place all of the current database domains within the file.
from tempfile import NamedTemporaryFile
with NamedTemporaryFile(delete=False) as fd:
if domain:
fd.write("{}\n".format(domain).encode("utf-8"))
else:
# Go through the database and grab the domains adding them to the file.
if args.rescan:
domains = self.BaseDomains.all(passive_scope=True)
else:
domains = self.BaseDomains.all(tool=self.name,
passive_scope=True)
if domains:
for domain in domains:
fd.write("{}\n".format(domain.domain).encode("utf-8"))
else:
return None
return fd.name
class Module(ToolTemplate):
"""
This tool will check various subdomains for domain takeover vulnerabilities.
"""
name = "Tko-subs"
binary_name = "tko-subs"
def __init__(self, db):
self.db = db
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("--data",
help="Path to the providers_data.csv file")
self.options.add_argument("-d",
"--domain",
help="Domain to run the tool against.")
self.options.add_argument(
"-i",
"--importdb",
help="Import subdomains from the database.",
action="store_true",
)
self.options.add_argument("--rescan",
help="Rescan already processed entries",
action="store_true")
def get_targets(self, args):
"""
This module is used to build out a target list and output file list, depending on the arguments. Should return a
list in the format [{'target':target, 'output':output}, {'target':target, 'output':output}, etc, etc]
"""
# Create an empty list to add targets to
domains = []
# Check if a domain has been explicitly passed, and if so add it to targets
if args.domain:
domains.append(args.domain)
# Check if the --import option was passed and if so get data from the domain.
# The scoping is set to "active" since the tool could potentially make a
# request from the server.
if args.importdb:
if args.rescan:
all_domains = self.Domain.all(scope_type="active")
else:
all_domains = self.Domain.all(scope_type="active",
tool=self.name)
for d in all_domains:
domains.append(d.domain)
# Set the output_path base, as a junction of the base_path and the path name supplied
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
# Create the path if it doesn't already exist
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
# Create the final targets list, with output paths added.
for d in domains:
res.append({"target": d, "output": os.path.join(output_path, d)})
return res
def build_cmd(self, args):
"""
Create the actual command that will be executed. Use {target} and {output} as placeholders.
"""
cmd = self.binary + " -domain {target} -output {output} "
# Add in any extra arguments passed in the extra_args parameter
if args.tool_args:
cmd += args.tool_args
# Add that data parameter in there
if not args.data:
args.data = os.path.join(os.path.dirname(self.binary),
'providers-data.csv')
cmd += " -data " + args.data
return cmd
def process_output(self, cmds):
"""
Process the output generated by the earlier commands.
"""
# Cycle through all of the targets we ran earlier
for c in cmds:
output_file = c["output"]
target = c["target"]
# Read file
data = open(output_file).read().split("\n")
# Quick and dirty way to filter out headers and blank lines, as well
# as duplicates
res = list(
set([
d for d in data if "Domain,Cname,Provider" not in d and d
]))
if res:
# Load up the DB entry.
created, subdomain = self.Domain.find_or_create(domain=target)
# Process results
for d in res:
results = d.split(",")
if results[3] == "false":
display_warning(
"Hosting found at {} for {}, not vulnerable.".
format(target, results[2]))
elif results[3] == "true":
display_new("{} vulnerable to {}!".format(
target, results[2]))
if not subdomain.meta[self.name].get(
"vulnerable", False):
subdomain.meta[self.name]["vulnerable"] = []
subdomain.meta[self.name]["vulnerable"].append(d)
else:
display_warning("Not sure of result: {}".format(data))
# This is a little hackish, but needed to get data to save
t = dict(subdomain.meta)
self.Domain.commit()
subdomain.meta = t
self.Domain.commit()
class Module(ToolTemplate):
"""
Module for running nmap. Make sure to pass all nmap-specific arguments at the end, after --tool_args
"""
name = "Nmap"
binary_name = "nmap"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Vulnerability = VulnRepository(db, self.name)
self.CVE = CVERepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"--hosts",
help=
"Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS",
nargs="+",
)
self.options.add_argument("--hosts_file", help="File containing hosts")
self.options.add_argument(
"-i",
"--hosts_database",
help="Use unscanned hosts from the database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Overwrite files without asking",
action="store_true")
self.options.add_argument(
"--filename",
help="Output filename. By default will use the current timestamp.",
)
self.options.add_argument(
"--ssl_cert_mode",
help=
"Scan only SSL enabled hosts to collect SSL certs (and domain names)",
action="store_true",
)
self.options.add_argument(
"--filter_ports",
help=
"Comma separated list of protoPort to filter out of results. Useful if firewall returns specific ports open on every host. Ex: t80,u5060"
)
self.options.set_defaults(timeout=None)
self.options.add_argument("--import_file",
help="Import results from an Nmap XML file.")
def get_targets(self, args):
self.args = args
if args.import_file:
args.no_binary = True
return [{"target": "", "output": args.import_file}]
targets = []
if args.hosts:
if type(args.hosts) == list:
for h in args.hosts:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
else:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
if args.hosts_database:
if args.rescan:
targets += [
h.ip_address
for h in self.IPAddress.all(scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all()]
else:
targets += [
h.ip_address
for h in self.IPAddress.all(tool=self.name,
scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)]
if args.hosts_file:
for h in [
l for l in open(args.hosts_file).read().split("\n") if l
]:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
data = []
if args.ssl_cert_mode:
ports = self.Port.all(service_name='https')
data = list(set([i.ip_address.ip_address for i in ports]))
port_numbers = list(set([str(i.port_number) for i in ports]))
args.tool_args += " -sV -p {} --script ssl-cert ".format(
','.join(port_numbers))
else:
# Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs
for t in targets:
ips = [str(i) for i in list(IPNetwork(t))]
data += ips
_, file_name = tempfile.mkstemp()
open(file_name, "w").write("\n".join(list(set(data))))
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
if args.filename:
output_path = os.path.join(self.path, args.filename)
else:
output_path = os.path.join(
self.path,
"nmap-scan-%s.xml" %
datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"),
)
return [{"target": file_name, "output": output_path}]
def build_cmd(self, args):
command = "sudo " + self.binary + " -oX {output} -iL {target} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
self.import_nmap(cmds[0]["output"])
if cmds[0]["target"]:
os.unlink(cmds[0]["target"])
def parseHeaders(self, httpHeaders):
bsHeaders = [
"Pragma",
"Expires",
"Date",
"Transfer-Encoding",
"Connection",
"X-Content-Type-Options",
"Cache-Control",
"X-Frame-Options",
"Content-Type",
"Content-Length",
"(Request type",
]
keepHeaders = {}
for i in range(0, len(httpHeaders)):
if httpHeaders[i].strip() != "" and httpHeaders[i].split(
":")[0].strip() not in " ".join(bsHeaders):
hName = httpHeaders[i].split(":")[0].strip()
hValue = "".join(httpHeaders[i].split(":")[1:]).strip()
keepHeaders[hName] = hValue
if keepHeaders == {}:
keepHeaders = ""
return keepHeaders
def import_nmap(
self,
filename): # domains={}, ips={}, rejects=[] == temp while no db
nFile = filename
# pdb.set_trace()
try:
tree = ET.parse(nFile)
root = tree.getroot()
hosts = root.findall("host")
except Exception as e:
print("Error: {}" % e)
print(nFile + " doesn't exist somehow...skipping")
return
for host in hosts:
hostIP = host.find("address").get("addr")
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
for hostname in host.findall("hostnames/hostname"):
hostname = hostname.get("name")
hostname = hostname.lower().replace("www.", "")
# reHostname = re.search(
# r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname
# ) # attempt to not get PTR record
# if not reHostname:
created, domain = self.Domain.find_or_create(domain=hostname)
if ip not in domain.ip_addresses:
domain.ip_addresses.append(ip)
domain.save()
for port in host.findall("ports/port"):
if port.find("state").get("state"):
portState = port.find("state").get("state")
hostPort = port.get("portid")
portProto = port.get("protocol")
# pdb.set_trace()
if not self.args.filter_ports or int(hostPort) not in [
int(p) for p in self.args.filter_ports.split(',')
]:
created, db_port = self.Port.find_or_create(
port_number=hostPort,
status=portState,
proto=portProto,
ip_address=ip,
)
if port.find("service") is not None:
portName = port.find("service").get("name")
if portName == "http" and hostPort == "443":
portName = "https"
else:
portName = "Unknown"
if created:
db_port.service_name = portName
info = db_port.info
if not info:
info = {}
for script in port.findall(
"script"): # just getting commonName from cert
if script.get("id") == "ssl-cert":
db_port.cert = script.get("output")
cert_domains = self.get_domains_from_cert(
script.get("output"))
for hostname in cert_domains:
hostname = hostname.lower().replace(
"www.", "")
created, domain = self.Domain.find_or_create(
domain=hostname)
if created:
print("New domain found: %s" %
hostname)
elif script.get("id") == "vulners":
print("Gathering vuln info for {} : {}/{}\n".
format(hostIP, portProto, hostPort))
self.parseVulners(script.get("output"),
db_port)
elif script.get("id") == "banner":
info["banner"] = script.get("output")
elif script.get("id") == "http-headers":
httpHeaders = script.get("output")
httpHeaders = httpHeaders.strip().split("\n")
keepHeaders = self.parseHeaders(httpHeaders)
info["http-headers"] = keepHeaders
elif script.get("id") == "http-auth":
info["http-auth"] = script.get("output")
elif script.get("id") == "http-title":
info["http-title"] = script.get("output")
db_port.info = info
db_port.save()
self.IPAddress.commit()
def parseVulners(self, scriptOutput, db_port):
urls = re.findall(r"(https://vulners.com/cve/CVE-\d*-\d*)",
scriptOutput)
for url in urls:
vuln_refs = []
exploitable = False
cve = url.split("/cve/")[1]
vulners = requests.get("https://vulners.com/cve/%s" % cve).text
exploitdb = re.findall(
r"https://www.exploit-db.com/exploits/\d{,7}", vulners)
for edb in exploitdb:
exploitable = True
if edb.split("/exploits/")[1] not in vuln_refs:
vuln_refs.append(edb.split("/exploits/")[1])
if not self.CVE.find(name=cve):
# print "Gathering CVE info for", cve
try:
res = json.loads(
requests.get("http://cve.circl.lu/api/cve/%s" %
cve).text)
cveDescription = res["summary"]
cvss = float(res["cvss"])
findingName = res["oval"][0]["title"]
if int(cvss) <= 3:
severity = 1
elif (int(cvss) / 2) == 5:
severity = 4
else:
severity = int(cvss) / 2
if not self.Vulnerability.find(name=findingName):
# print "Creating", findingName
created, db_vuln = self.Vulnerability.find_or_create(
name=findingName,
severity=severity,
description=cveDescription,
)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if vuln_refs:
db_vuln.exploit_reference = {"edb-id": vuln_refs}
db_vuln.save()
else:
# print "modifying",findingName
db_vuln = self.Vulnerability.find(name=findingName)
db_vuln.ports.append(db_port)
db_vuln.exploitable = exploitable
if vuln_refs:
db_vuln.exploitable = exploitable
if db_vuln.exploit_reference is not None:
if "edb-id" in db_vuln.exploit_reference:
for ref in vuln_refs:
if (ref not in db_vuln.
exploit_reference["edb-id"]):
db_vuln.exploit_reference[
"edb-id"].append(ref)
else:
db_vuln.exploit_reference[
"edb-id"] = vuln_refs
else:
db_vuln.exploit_reference = {
"edb-id": vuln_refs
}
db_vuln.save()
if not self.CVE.find(name=cve):
created, db_cve = self.CVE.find_or_create(
name=cve,
description=cveDescription,
temporal_score=cvss)
db_cve.vulnerabilities.append(db_vuln)
db_cve.save()
else:
db_cve = self.CVE.find(name=cve)
db_cve.vulnerabilities.append(db_vuln)
db_cve.save()
self.Vulnerability.commit()
self.CVE.commit()
except Exception:
print(
"something went wrong with the vuln/cve info gathering"
)
if vulners:
print(
"Vulners report was found but no exploit-db was discovered"
)
# "Affected vulners items"
# print vulners
print("Affected CVE")
print(cve)
pass
else:
db_cve = self.CVE.find(name=cve)
for db_vulns in db_cve.vulnerabilities:
if db_port not in db_vulns.ports:
db_vulns.ports.append(db_port)
return
def get_domains_from_cert(self, cert):
# Shamelessly lifted regex from stack overflow
regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}"
domains = list(
set([d for d in re.findall(regex, cert) if "*" not in d]))
return domains
class Module(ModuleTemplate):
'''
Uses DomLink from:
https://github.com/vysec/DomLink
By: Vincent Yu
'''
name = "DomLink"
binary_name = "domLink.py"
def __init__(self, db):
self.db = db
self.Domains = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("--binary", help="Path to binary")
self.options.add_argument(
"-o",
"--output_path",
help=
"Path which will contain program output (relative to base_path in config",
default=self.name,
)
self.options.add_argument(
"--tool_args",
help="Additional arguments to be passed to the tool",
nargs=argparse.REMAINDER,
)
self.options.add_argument("-d", "--domain", help="Domain to search.")
self.options.add_argument(
'-s',
'--scope',
help="How to scope results (Default passive)",
choices=["active", "passive", "none"],
default="passive")
self.options.add_argument(
"--no_binary",
help=
"Runs through without actually running the binary. Useful for if you already ran the tool and just want to process the output.",
action="store_true",
)
def run(self, args):
if not args.domain:
display_error("You need to supply a domain to search for.")
return
if not args.binary:
self.binary = which.run(self.binary_name)
else:
self.binary = args.binary
if not self.binary:
display_error(
"{} binary not found. Please explicitly provide path with --binary"
.format(self.binary_name))
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], 'output',
args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], 'output',
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path, "{}.txt".format(args.domain))
command_args = " {} -o {} ".format(args.domain, output_path)
if args.tool_args:
command_args += ' '.join(args.tool_args)
if not args.no_binary:
current_dir = os.getcwd()
new_dir = "/".join(self.binary.split("/")[:-1])
os.chdir(new_dir)
cmd = shlex.split("python2 " + self.binary + command_args)
print("Executing: %s" % " ".join(cmd))
subprocess.Popen(cmd).wait()
os.chdir(current_dir)
results = open(output_path).read().split('\n')
cur_type = None
for r in results:
if r:
if '### Company Names' in r:
cur_type = "company"
elif '### Domain Names' in r:
cur_type = "domain"
elif '### Email Addresses' in r:
cur_type = "email"
else:
if cur_type == "domain":
if args.scope == "active":
created, d = self.Domains.find_or_create(
domain=r, in_scope=True, passive_scope=True)
elif args.scope == "passive":
created, d = self.Domains.find_or_create(
domain=r, in_scope=False, passive_scope=True)
else:
created, d = self.Domains.find_or_create(
domain=r, in_scope=False, passive_scope=False)
self.Domains.commit()
class Module(ToolTemplate):
"""
Module for running masscan. Make sure to pass all masscan-specific arguments at the end, after --tool_args
"""
name = "Masscan"
binary_name = "masscan"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Vulnerability = VulnRepository(db, self.name)
self.CVE = CVERepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"--hosts",
help=
"Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS",
nargs="+",
)
self.options.add_argument("--hosts_file", help="File containing hosts")
self.options.add_argument(
"-i",
"--hosts_database",
help="Use unscanned hosts from the database",
action="store_true",
)
self.options.add_argument("--rescan",
help="Overwrite files without asking",
action="store_true")
self.options.add_argument(
"--filename",
help="Output filename. By default will use the current timestamp.",
)
self.options.set_defaults(timeout=None)
self.options.add_argument(
"--import_file",
help="Import results from an Masscan/Nmap XML file.")
def get_targets(self, args):
if args.import_file:
args.no_binary = True
return [{"target": "", "output": args.import_file}]
targets = []
if args.hosts:
if type(args.hosts) == list:
for h in args.hosts:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
else:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
if args.hosts_database:
if args.rescan:
targets += [
h.ip_address
for h in self.IPAddress.all(scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all()]
else:
targets += [
h.ip_address
for h in self.IPAddress.all(tool=self.name,
scope_type="active")
]
targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)]
if args.hosts_file:
for h in [
l for l in open(args.hosts_file).read().split("\n") if l
]:
if check_if_ip(h):
targets.append(h)
else:
created, domain = self.Domain.find_or_create(domain=h)
targets += [i.ip_address for i in domain.ip_addresses]
# Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs
data = []
for t in targets:
ips = [str(i) for i in list(IPNetwork(t))]
data += ips
_, file_name = tempfile.mkstemp()
open(file_name, "w").write("\n".join(set(data)))
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
if args.filename:
output_path = os.path.join(self.path, args.filename)
else:
output_path = os.path.join(
self.path,
"masscan-%s.xml" %
datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"),
)
return [{"target": file_name, "output": output_path}]
def build_cmd(self, args):
command = "sudo " + self.binary + " -oX {output} -iL {target} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
self.import_masscan(cmds[0]["output"])
if cmds[0]["target"]:
os.unlink(cmds[0]["target"])
def import_masscan(
self,
filename): # domains={}, ips={}, rejects=[] == temp while no db
nFile = filename
try:
tree = ET.parse(nFile)
root = tree.getroot()
hosts = root.findall("host")
except Exception:
print(nFile + " doesn't exist somehow...skipping")
return
for host in hosts:
hostIP = host.find("address").get("addr")
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
for hostname in host.findall("hostnames/hostname"):
hostname = hostname.get("name")
hostname = hostname.lower().replace("www.", "")
# reHostname = re.search(
# r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname
# ) # attempt to not get PTR record
# if not reHostname:
created, domain = self.Domain.find_or_create(domain=hostname)
if ip not in domain.ip_addresses:
domain.ip_addresses.append(ip)
domain.save()
for port in host.findall("ports/port"):
if port.find("state").get("state"):
portState = port.find("state").get("state")
hostPort = port.get("portid")
portProto = port.get("protocol")
created, db_port = self.Port.find_or_create(
port_number=hostPort,
status=portState,
proto=portProto,
ip_address=ip,
)
info = db_port.info
if not info:
info = {}
if port.find("service") is not None:
service = port.find("service")
portName = service.get("name")
if portName == "http" and hostPort == "443":
portName = "https"
banner = service.get("banner", None)
if banner:
print("Found banner: {}".format(banner))
info["banner"] = banner
else:
portName = "Unknown"
if created:
db_port.service_name = portName
db_port.info = info
db_port.save()
self.IPAddress.commit()
def get_domains_from_cert(self, cert):
# Shamelessly lifted regex from stack overflow
regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}"
domains = list(
set([d for d in re.findall(regex, cert) if "*" not in d]))
return domains
class Module(ToolTemplate):
name = "TheHarvester"
binary_name = "theharvester"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d", "--domain", help="Domain to harvest")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
targets.append({"target": args.domain})
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
targets.append({"target": domain.domain})
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in domains:
targets.append({"target": d.domain})
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
t["output"] = os.path.join(
output_path, "%s-theharvester" % t["target"].replace(".", "_"))
return targets
def build_cmd(self, args):
cmd = self.binary + " -f {output} -b default -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
try:
data = xmltodict.parse(open(cmd["output"] + ".xml").read())
except Exception as e:
# display_error("Error with {}: {}".format(cmd["output"], e))
data = None
if data:
if data["theHarvester"].get("email", False):
if type(data["theHarvester"]["email"]) == list:
emails = data["theHarvester"]["email"]
else:
emails = [data["theHarvester"]["email"]]
for e in emails:
display("Processing E-mail: {}".format(e))
created, user = self.User.find_or_create(email=e)
_, domain = self.BaseDomain.find_or_create(
domain=e.split("@")[1])
user.domain = domain
user.update()
if created:
display_new("New email: %s" % e)
if data["theHarvester"].get("host", False):
if type(data["theHarvester"]["host"]) == list:
hosts = data["theHarvester"]["host"]
else:
hosts = [data["theHarvester"]["host"]]
for d in hosts:
created, domain = self.Domain.find_or_create(
domain=d["hostname"])
if data["theHarvester"].get("vhost", False):
if type(data["theHarvester"]["vhost"]) == list:
hosts = data["theHarvester"]["vhost"]
else:
hosts = [data["theHarvester"]["vhost"]]
for d in hosts:
created, domain = self.Domain.find_or_create(
domain=d["hostname"])
self.BaseDomain.commit()
class Module(ToolTemplate):
'''
This module uses Fierce, a Perl based domain brute forcing tool. It can be installed from
https://github.com/davidpepper/fierce-domain-scanner
'''
name = "Fierce"
binary_name = "fierce"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Target domain for Fierce")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"--rescan",
help="Force rescan of already scanned domains",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
if args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
targets.append(d)
if args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for domain in domains:
targets.append(domain.domain)
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
res = []
for t in targets:
res.append({
"target": t,
"output": os.path.join(self.path, t + ".txt")
})
return res
def build_cmd(self, args):
command = self.binary
command += " -dns {target} -file {output} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
for c in cmds:
target = c["target"]
output_path = c["output"]
try:
fierceOutput = open(output_path).read()
except IOError:
display_error(
"The output file for {} was not found. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file."
.format(target))
continue
domains = []
if "Now performing" in fierceOutput:
hosts = re.findall(
r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$",
fierceOutput,
re.MULTILINE,
)
if hosts:
for host in hosts:
# print host
domain = (host.split("\t")[1].lower().replace(
"www.", "").rstrip("."))
if domain not in domains:
domains.append(domain)
elif "Whoah, it worked" in fierceOutput:
print("Zone transfer found!")
hosts = re.findall(
r".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",
fierceOutput,
re.MULTILINE,
)
if hosts:
for host in hosts:
domain = (host.split("\t")[0].lower().replace(
"www.", "").rstrip("."))
if domain not in domains:
domains.append(domain)
else:
display_error(
"No results found in {}. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file."
.format(output_path))
if domains:
for _domain in domains:
created, domain = self.Domain.find_or_create(
domain=_domain)
class Module(ToolTemplate):
name = "Sublist3r"
binary_name = "sublist3r"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Domain to brute force")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
targets.append(d)
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in domains:
targets.append(d.domain)
res = []
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
output = os.path.join(output_path, "%s-sublist3r.txt" % t)
res.append({"target": t, "output": output})
return res
def build_cmd(self, args):
cmd = self.binary + " -o {output} -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
output_path = cmd["output"]
if os.path.isfile(output_path):
data = open(output_path).read().split("\n")
for d in data:
new_domain = d.split(":")[0].lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
else:
display_error("{} not found.".format(output_path))
next
self.Domain.commit()
class Module(ToolTemplate):
'''
This module uses Fuzz Faster U Fool (FFuF) for directory fuzzing. It can be installed from:
https://github.com/ffuf/ffuf
'''
name = "FFuF"
binary_name = "ffuf"
def __init__(self, db):
self.db = db
self.IPAddress = IPRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Url = UrlRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-u", "--url", help="URL to brute force")
self.options.add_argument("--file", help="Import URLs from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import URLs from database",
action="store_true",
)
self.options.add_argument(
"--rescan",
help="Rescan domains that have already been brute forced",
action="store_true",
)
self.options.set_defaults(timeout=0) # Disable the default timeout.
def get_targets(self, args):
targets = []
if args.url:
targets.append(args.url)
if args.file:
urls = open(args.file).read().split("\n")
for u in urls:
if u:
targets.append(u)
if args.import_database:
if args.rescan:
targets += get_urls.run(self.db, scope_type="active")
else:
targets += get_urls.run(self.db, tool=self.name, scope_type="active")
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path[1:],
str(int(time.time())),
)
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path,
str(int(time.time())),
)
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append(
{
"target": t,
"output": os.path.join(
output_path,
t.replace(":", "_")
.replace("/", "_")
.replace("?", "_")
.replace("&", "_")
+ "-dir.txt", # noqa: W503
),
}
)
return res
def build_cmd(self, args):
cmd = self.binary
cmd += " -o {output} -u {target}/FUZZ "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
target = cmd['target']
proto = target.split('/')[0]
url = target.split('/')[2]
if ':' in url:
port_num = url.split(':')[1]
url = url.split(':')[0]
elif proto == 'http':
port_num = "80"
elif proto == 'https':
port_num = "443"
else:
port_num = "0"
try:
[int(i) for i in url.split('.')]
created, ip = self.IPAddress.find_or_create(ip_address=url)
port = [p for p in ip.ports if p.port_number == int(port_num) and p.proto == 'tcp'][0]
port.set_tool(self.name)
except:
display("Domain found: {}".format(url))
created, domain = self.Domain.find_or_create(domain=url)
for ip in domain.ip_addresses:
try:
port = [p for p in ip.ports if p.port_number == int(port_num) and p.proto == 'tcp'][0]
port.set_tool(self.name)
except Exception as e:
print("Error getting ports: {}".format(e))
self.Port.commit()
class Module(ToolTemplate):
'''
This module uses the Ruby version of Aquatone. You can usually install it with "gem install aquatone"
'''
name = "aquatone-discover"
binary_name = "aquatone-discover"
def __init__(self, db):
self.db = db
self.Domain = DomainRepository(db, self.name)
self.BaseDomain = BaseDomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Target domain for aquatone")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-r",
"--rescan",
help="Run aquatone on hosts that have already been processed.",
action="store_true",
)
self.options.set_defaults(timeout=None)
def get_targets(self, args):
"""
This module is used to build out a target list and output file list, depending on the arguments. Should return a
list in the format [(target, output), (target, output), etc, etc]
"""
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.file:
domainsFile = open(args.file).read().split("\n")
for d in domainsFile:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
all_domains = self.BaseDomain.all(scope_type="passive")
else:
all_domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in all_domains:
targets.append(d.domain)
else:
print("You need to supply domain(s).")
output_path = os.path.join(self.base_config["PROJECT"]["base_path"],
"output", "aquatone")
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
"target": t,
"output": "{}/{}/hosts.json".format(output_path, t)
})
return res
def build_cmd(self, args):
"""
Create the actual command that will be executed. Use {target} and {output} as placeholders.
"""
cmd = self.binary + " -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def pre_run(self, args):
output_path = os.path.join(self.base_config["PROJECT"]["base_path"],
"output")
self.orig_home = os.environ["HOME"]
os.environ["HOME"] = output_path
def process_output(self, cmds):
"""
Process the output generated by the earlier commands.
"""
for cmd in cmds:
try:
data2 = json.loads(open(cmd["output"]).read())
for sub, ip in data2.items():
created = False
new_domain = sub.lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
except Exception as e:
display_error("Couldn't find file: {}".format(cmd["output"]))
self.Domain.commit()
def post_run(self, args):
os.environ["HOME"] = self.orig_home
class Module(ModuleTemplate):
name = "HeaderScanner"
def __init__(self, db):
self.db = db
self.Port = PortRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-t",
"--timeout",
help="Connection timeout (default 5)",
default="5")
self.options.add_argument("-u", "--url", help="URL to get headers")
self.options.add_argument("--file", help="Import URLs from file")
self.options.add_argument(
"-i",
"--import_db",
help="Import URLs from the database",
action="store_true",
)
self.options.add_argument("-th",
"--threads",
help="Number of threads to run",
default="10")
self.options.add_argument("--rescan",
help="Rescan URLs already processed",
action="store_true")
def run(self, args):
data = []
if args.url:
service = args.url.split(":")[0]
host = args.url.split("/")[2]
if args.url.count(":") == 2:
port = args.url.split(":")[2].split("/")[0]
elif service == "http":
port = "80"
elif service == "https":
port = "443"
else:
display_error(
"Could not figure out port number for url: {}".format(
args.url))
sys.exit(1)
if check_if_ip(host):
created, ip = self.IPAddress.find_or_create(ip_address=host)
else:
created, domain = self.Domain.find_or_create(domain=host)
ip = domain.ip_addresses[0]
created, service_id = self.Port.find_or_create(ip_address=ip,
port_number=port)
service_id.service_name = service
data.append([service_id.id, [args.url], args.timeout])
if args.file:
url = open(args.file).read().split("\n")
for u in url:
if u:
service = u.split(":")[0]
host = u.split("/")[2]
if u.count(":") == 2:
port = u.split(":")[2].split("/")[0]
elif service == "http":
port = "80"
elif service == "https":
port = "443"
else:
display_error(
"Could not figure out port number for url: {}".
format(args.url))
sys.exit(1)
if check_if_ip(host):
created, ip = self.IPAddress.find_or_create(
ip_address=host)
else:
created, domain = self.Domain.find_or_create(
domain=host)
ip = domain.ip_addresses[0]
created, service_id = self.Port.find_or_create(
ip_address=ip, port_number=port)
service_id.service_name = service
data.append([service_id.id, [u], args.timeout])
if args.import_db:
if args.rescan:
svc = self.Port.all(service_name="http")
svc += self.Port.all(service_name="https")
else:
svc = self.Port.all(service_name="http", tool=self.name)
svc += self.Port.all(service_name="https", tool=self.name)
for s in svc:
if s.ip_address.in_scope:
urls = [
"%s://%s:%s" % (s.service_name,
s.ip_address.ip_address, s.port_number)
]
for d in s.ip_address.domains:
urls.append("%s://%s:%s" %
(s.service_name, d.domain, s.port_number))
data.append([s.id, urls, args.timeout])
if data:
pool = ThreadPool(int(args.threads))
results = pool.map(process_urls, data)
display_new("Adding headers to the database")
for i, headers, cookies in results:
created, svc = self.Port.find_or_create(id=i)
svc.meta["headers"] = headers
svc.meta["cookies"] = cookies
svc.update()
self.Port.commit()
class Module(ModuleTemplate):
"""
Ingests domains and IPs. Domains get ip info and cidr info, and IPs get
CIDR info.
"""
name = "Ingestor"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.CIDR = CIDRRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-d",
"--import_domains",
help="Either domain to import or file containing domains to import. One per line",
)
self.options.add_argument(
"-i",
"--import_ips",
help="Either IP/range to import or file containing IPs and ranges, one per line.",
)
self.options.add_argument(
"-a",
"--active",
help="Set scoping on imported data as active",
action="store_true",
)
self.options.add_argument(
"-p",
"--passive",
help="Set scoping on imported data as passive",
action="store_true",
)
self.options.add_argument(
"-sc",
"--scope_cidrs",
help="Cycle through out of scope networks and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument(
"-sb",
"--scope_base_domains",
help="Cycle through out of scope base domains and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument("--descope", help="Descope an IP, domain, or CIDR")
self.options.add_argument(
"-Ii",
"--import_database_ips",
help="Import IPs from database",
action="store_true",
)
self.options.add_argument(
"--force",
help="Force processing again, even if already processed",
action="store_true",
)
def run(self, args):
self.in_scope = args.active
self.passive_scope = args.passive
if args.descope:
if "/" in args.descope:
self.descope_cidr(args.descope)
elif check_string(args.descope):
pass
else:
self.descope_ip(args.descope)
# Check if in ScopeCIDR and remove if found
if args.import_ips:
try:
ips = open(args.import_ips)
for line in ips:
if line.strip():
if "/" in line or "-" in line:
self.process_cidr(line)
else:
self.process_ip(line.strip(), force_scope=True)
self.Domain.commit()
except IOError:
if "/" in args.import_ips or "-" in args.import_ips:
self.process_cidr(args.import_ips)
else:
self.process_ip(args.import_ips.strip(), force_scope=True)
self.Domain.commit()
if args.import_domains:
try:
domains = open(args.import_domains)
for line in domains:
if line.strip():
self.process_domain(line.strip())
self.Domain.commit()
except IOError:
self.process_domain(args.import_domains.strip())
self.Domain.commit()
if args.scope_base_domains:
base_domains = self.BaseDomain.all(in_scope=False, passive_scope=False)
for bd in base_domains:
self.reclassify_domain(bd)
self.BaseDomain.commit()
def get_domain_ips(self, domain):
ips = []
try:
answers = dns.resolver.query(domain, "A")
for a in answers:
ips.append(a.address)
return ips
except Exception:
return []
def process_domain(self, domain_str):
created, domain = self.Domain.find_or_create(
only_tool=True,
domain=domain_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if (
domain.in_scope != self.in_scope
or domain.passive_scope != self.passive_scope # noqa: W503
):
display(
"Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (domain_str, self.in_scope, self.passive_scope)
)
domain.in_scope = self.in_scope
domain.passive_scope = self.passive_scope
domain.update()
if domain.base_domain.domain == domain.domain:
display("Name also matches a base domain. Updating that as well.")
domain.base_domain.in_scope = self.in_scope
domain.base_domain.passive_scope = self.passive_scope
domain.base_domain.update()
def process_ip(self, ip_str, force_scope=True):
created, ip = self.IPAddress.find_or_create(
only_tool=True,
ip_address=ip_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope:
display(
"IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (ip_str, self.in_scope, self.passive_scope)
)
ip.in_scope = self.in_scope
ip.passive_scope = self.passive_scope
ip.update()
return ip
def process_cidr(self, line):
display("Processing %s" % line)
if "/" in line:
created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
if created:
display_new("Adding %s to scoped CIDRs in database" % line.strip())
cidr.in_scope = True
cidr.update()
elif "-" in line:
start_ip, end_ip = line.strip().replace(" ", "").split("-")
if "." not in end_ip:
end_ip = ".".join(start_ip.split(".")[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
if created:
display_new("Adding %s to scoped CIDRs in database" % line.strip())
cidr.in_scope = True
cidr.update()
def reclassify_domain(self, bd):
if bd.meta.get("whois", False):
display_new("Whois data found for {}".format(bd.domain))
print(bd.meta["whois"])
res = six.input(
"Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] "
)
if res.lower() == "a":
bd.in_scope = True
bd.passive_scope = True
elif res.lower() == "p":
bd.in_scope = False
bd.passive_scope = True
else:
bd.in_scope = False
bd.passive_scope = False
bd.save()
else:
display_error(
"Unfortunately, there is no whois information for {}. Please populate it using the Whois module".format(
bd.domain
)
)
def descope_ip(self, ip):
ip = self.IPAddress.all(ip_address=ip)
if ip:
for i in ip:
display("Removing IP {} from scope".format(i.ip_address))
i.in_scope = False
i.passive_scope = False
i.update()
for d in i.domains:
in_scope_ips = [
ipa
for ipa in d.ip_addresses
if ipa.in_scope or ipa.passive_scope
]
if not in_scope_ips:
display(
"Domain {} has no more scoped IPs. Removing from scope.".format(
d.domain
)
)
d.in_scope = False
d.passive_scope = False
self.IPAddress.commit()
def descope_cidr(self, cidr):
CIDR = self.ScopeCIDR.all(cidr=cidr)
if CIDR:
for c in CIDR:
display("Removing {} from ScopeCIDRs".format(c.cidr))
c.delete()
cnet = IPNetwork(cidr)
for ip in self.IPAddress.all():
if IPAddress(ip.ip_address) in cnet:
self.descope_ip(ip.ip_address)
