def get_targets(self, args):
targets = []
if args.domain:
domain, created = BaseDomain.objects.get_or_create(
name=args.domain, defaults={'passive_scope': True})
targets.append(domain.name)
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
domain, created = BaseDomain.objects.get_or_create(
name=d, defaults={'passive_scope': True})
targets.append(domain.name)
elif args.import_database:
if args.rescan:
domains = BaseDomain.get_set(scope_type="passive")
else:
domains = BaseDomain.get_set(scope_type="passive",
tool=self.name,
args=self.args.tool_args)
for domain in domains:
targets.append(domain.name)
elif args.range:
targets.append(args.range)
elif args.import_range:
if args.rescan:
cidrs = CIDR.get_set(scope_type="active")
else:
cidrs = CIDR.get_set(scope_type="active",
tool=self.name,
args=self.args.tool_args)
for cidr in cidrs:
targets.append(cidr.name)
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
res = []
for t in targets:
res.append({
"target":
t,
"output":
os.path.join(self.path,
t.replace("/", "_") + ".json"),
})
return res
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
if args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
targets.append(d)
if args.import_database:
if args.rescan:
targets += [
b.name for b in BaseDomain.get_set(scope_type="passive")
]
else:
targets += [
b.name
for b in BaseDomain.get_set(scope_type="passive",
tool=self.name,
args=self.args.tool_args)
]
if args.output_path[0] == "/":
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path[1:])
else:
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
"target":
t,
"output":
os.path.join(output_path,
t.replace("/", "_") + "-dns.txt"),
})
return res
def get_targets(self, args):
targets = []
outpath = ""
if args.output_path[0] == "/":
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path[1:])
else:
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
if args.domain:
out_file = os.path.join(outpath,
"{}.subfinder".format(args.domain))
targets.append({
"target": args.domain,
"output": os.path.join(output_path, out_file)
})
if args.db_domains:
if args.rescan:
domains = BaseDomain.get_set(scope_type="passive")
else:
domains = BaseDomain.get_set(tool=self.name,
args=self.args.tool_args,
scope_type="passive")
for d in domains:
out_file = os.path.join(outpath, "{}.subfinder".format(d.name))
targets.append({
"target": d.name,
"output": os.path.join(output_path, out_file)
})
elif args.domain_list:
domains = io.open(args.domain_list,
encoding="utf-8").read().split("\n")
for d in domains:
if d:
targets.append({
"target":
d,
"output":
os.path.join(output_path, "{}.subfinder".format(d)),
})
return targets
def get_targets(self, args):
"""
This module is used to build out a target list and output file list, depending on the arguments. Should return a
list in the format [(target, output), (target, output), etc, etc]
"""
targets = []
if args.domain:
domain, created = BaseDomain.objects.get_or_create(
name=args.domain)
targets.append(domain.name)
elif args.file:
domainsFile = open(args.file).read().split("\n")
for d in domainsFile:
if d:
domain, created = BaseDomain.objects.get_or_create(name=d)
targets.append(domain.name)
elif args.import_database:
if args.rescan:
all_domains = BaseDomain.get_set(scope_type="passive")
else:
all_domains = BaseDomain.get_set(tool=self.name,
args=args.tool_args,
scope_type="passive")
for d in all_domains:
targets.append(d.name)
else:
print("You need to supply domain(s).")
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
"output", "aquatone")
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
"target": t,
"output": "{}/{}/hosts.json".format(output_path, t)
})
return res
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
targets.append(d)
elif args.import_database:
if args.rescan:
domains = BaseDomain.get_set(scope_type="passive")
else:
domains = BaseDomain.get_set(tool=self.name,
scope_type="passive",
args=self.args.tool_args)
for d in domains:
targets.append(d.name)
res = []
if args.output_path[0] == "/":
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path[1:])
else:
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
output = os.path.join(output_path, "%s-sublist3r.txt" % t)
res.append({"target": t, "output": output})
return res
def run(self, args):
self.active_scope = args.active
self.passive_scope = args.passive
if args.descope:
if "/" in args.descope:
self.descope_cidr(args.descope)
elif check_string(args.descope):
pass
else:
self.descope_ip(args.descope)
# Check if in ScopeCIDR and remove if found
if args.import_ips:
try:
ips = open(args.import_ips)
for line in ips:
if line.strip():
# pdb.set_trace()
if "/" in line or "-" in line:
self.process_cidr(line, args.label)
else:
self.process_ip(line.strip(), force_scope=True)
except IOError:
if "/" in args.import_ips or "-" in args.import_ips:
self.process_cidr(args.import_ips, args.label)
else:
self.process_ip(args.import_ips.strip(), force_scope=True)
if args.import_domains:
try:
domains = open(args.import_domains)
for line in domains:
if line.strip():
self.process_domain(line.strip())
except IOError:
self.process_domain(args.import_domains.strip())
if args.scope_base_domains:
base_domains = BaseDomain.all(active_scope=False,
passive_scope=False)
for bd in base_domains:
self.reclassify_domain(bd)
def get_targets(self, args):
targets = []
if args.domain:
targets.append({"target": args.domain})
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
targets.append({"target": d})
elif args.import_database:
if args.rescan:
domains = BaseDomain.get_set(scope_type="passive")
else:
domains = BaseDomain.get_set(tool=self.name,
args=args.tool_args,
scope_type="passive")
for d in domains:
targets.append({"target": d.name})
if args.output_path[0] == "/":
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path[1:])
else:
output_path = os.path.join(self.base_config["ARMORY_BASE_PATH"],
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
t["output"] = os.path.join(
output_path, "%s-theharvester" % t["target"].replace(".", "_"))
return targets
def run(self, args):
# Cidrs = self.CIDR.
results = []
if args.scope != "all":
domains = BaseDomain.get_set(scope_type=args.scope)
else:
domains = BaseDomain.objects.all()
domain_data = {}
for d in domains:
if d.meta.get("whois", False):
domain_data[d.name] = d.meta["whois"]
cidr_data = {}
CIDRs = CIDR.get_set(scope_type=args.scope)
for c in CIDRs:
if c.meta.get("whois", False):
cidr_data[c.name] = c.meta["whois"]
# pdb.set_trace()
domain_blacklist = [
"Please note: ",
"URL of the ICANN WHOIS",
">>>",
"Notes:",
"whitelisting here:",
"NOTICE: ",
"TERMS OF USE: ",
"by the following",
"to: (1) allow",
]
results.append("ARIN Registration")
for c in sorted(cidr_data.keys()):
results.append("\t" + c)
for l in cidr_data[c].split("\n"):
if ": " in l and l[0] != "#":
results.append("\t\t" + l)
results.append("Domain Registration")
for d in sorted(domain_data.keys()):
results.append("\t" + d.upper())
for l in domain_data[d].split("\n"):
if ": " in l:
clean = True
for b in domain_blacklist:
if b in l:
clean = False
if clean:
results.append("\t\t" + l)
self.process_output(results, args)
def run(self, args):
results = []
domains = BaseDomain.get_set(scope_type=args.scope)
for d in domains:
if args.title and d.users:
results.append('{}'.format(d.name))
if args.emails:
emails = sorted(
[u.email.lower() for u in d.user_set.all() if u.email])
if args.title:
results += ['\t{}'.format(e) for e in emails]
else:
results += ['{}'.format(e) for e in emails]
elif args.accounts:
emails = sorted(
[u.email.lower() for u in d.user_set.all() if u.email])
if args.title:
results += ['\t{}'.format(e.split("@")[0]) for e in emails]
else:
results += ['{}'.format(e.split("@")[0]) for e in emails]
elif args.full:
for user in d.user_set.all():
results.append("{}|{}|{}|{}".format(
user.first_name, user.last_name, user.email,
user.job_title))
else:
for user in d.user_set.all():
for cred in user.cred_set.all():
if cred.password and cred.password != "None":
if args.emails_passwords:
txt = "%s:%s" % (user.email, cred.password)
elif args.usernames_passwords:
txt = "%s:%s" % (
user.email.split("@")[0],
cred.password,
)
if txt not in results:
results.append(txt)
self.process_output(results, args)
def get_targets(self, args):
"""
This module is used to build out a target list and output file list, depending on the arguments. Should return a
list in the format [(target, output), (target, output), etc, etc]
"""
targets = []
if args.import_database:
if args.rescan:
all_domains = BaseDomain.get_set(scope_type="passive")
else:
all_domains = BaseDomain.get_set(tool=self.name,
args=args.tool_args,
scope_type="passive")
for d in all_domains:
# We need to find all of the http/https ports and create the json file.
output_path = os.path.join(
self.base_config["ARMORY_BASE_PATH"],
"output",
"aquatone",
d.name,
)
if not os.path.exists(output_path):
os.makedirs(output_path)
hosts_j = {}
hosts = []
open_ports = []
urls = []
targets.append(d.name)
for s in d.domain_set.all():
name = s.name
for ip in s.ip_addresses.all():
hosts_j[name] = ip.ip_address
port_list = []
for p in ip.port_set.all():
if "http" in p.service_name:
hosts.append("{}.{}".format(
name, ip.ip_address))
port_list.append(p.port_number)
urls.append("{}://{}:{}/".format(
p.service_name, name, p.port_number))
urls.append("{}://{}:{}/".format(
p.service_name, ip.ip_address,
p.port_number))
if port_list:
open_ports.append("{},{}".format(
ip.ip_address,
",".join([str(o) for o in port_list])))
open(os.path.join(output_path, "hosts.txt"),
"w").write("\n".join(list(set(hosts))))
open(os.path.join(output_path, "urls.txt"),
"w").write("\n".join(list(set(urls))))
open(os.path.join(output_path, "open_ports.txt"),
"w").write("\n".join(list(set(open_ports))))
open(os.path.join(output_path, "hosts.json"),
"w").write(json.dumps(hosts_j))
else:
display_error("You need to supply domain(s).")
res = []
for t in targets:
res.append({"target": t})
return res
def process_data(self, nFile, args):
display("Reading " + nFile)
tree = ET.parse(nFile)
root = tree.getroot()
current_ips = set([i.ip_address for i in IPAddress.objects.all()])
current_domains = set([d.name for d in Domain.objects.all()])
new_ips = []
new_ip_list = []
new_domains = {}
just_domains = []
self.args = args
print("Preprocessing IPs/Domains")
for ReportHost in root.iter("ReportHost"):
hostname = ""
hostIP = ""
os = ""
for HostProperties in ReportHost.iter("HostProperties"):
for tag in HostProperties:
if tag.get("name") == "host-ip":
hostIP = tag.text
if tag.get("name") == "host-fqdn":
hostname = tag.text.lower()
hostname = hostname.replace("www.", "")
if tag.get("name") == "operating-system":
os = " OR ".join(tag.text.split("\n"))
# pdb.set_trace()
if hostIP and hostIP not in current_ips:
res = validate_ip(hostIP)
if res == "ipv4":
v = 4
else:
v = 6
new_ips.append(
IPAddress(ip_address=hostIP,
active_scope=True,
passive_scope=True,
version=v,
os=os))
new_ip_list.append(hostIP)
current_ips.add(hostIP)
if hostIP and hostname and '.' in hostname: # Filter out the random hostnames that aren't fqdns
if not new_domains.get(hostIP):
new_domains[hostIP] = []
hostname = ''.join([
i for i in hostname.lower()
if i in 'abcdefghijklmnopqrstuvwxyz.-0123456789'
])
if hostname not in current_domains and hostname not in new_domains[
hostIP]:
new_domains[hostIP].append(hostname)
if hostname not in just_domains:
just_domains.append(hostname)
cidrs = {c.name: c.id for c in CIDR.objects.all()}
for instance in new_ips:
found = False
for c, v in cidrs.items():
if instance.ip_address in IPNetwork(c):
instance.cidr__id = v
found = True
break
cidr_data, org_name = get_cidr_info(instance.ip_address)
cidr, created = CIDR.objects.get_or_create(
name=cidr_data, defaults={'org_name': org_name})
instance.cidr = cidr
cidrs[cidr.name] = cidr.id
display("Bulk creating IPs...")
IPAddress.objects.bulk_create(new_ips)
domain_objs = []
base_domains = {bd.name: bd.id for bd in BaseDomain.objects.all()}
for d in just_domains:
base_domain = '.'.join(d.split('.')[-2:])
if not base_domains.get(base_domain):
bd = BaseDomain(name=base_domain,
active_scope=False,
passive_scope=False)
bd.save()
bd_id = bd.id
base_domains[bd.name] = bd.id
else:
bd_id = base_domains[base_domain]
domain_objs.append(Domain(name=d, basedomain_id=bd_id))
display("Bulk creating domains...")
Domain.objects.bulk_create(domain_objs)
current_ips = {i.ip_address: i.id for i in IPAddress.objects.all()}
current_domains = {d.name: d.id for d in Domain.objects.all()}
ThroughModel = Domain.ip_addresses.through
many_objs = []
for i, v in new_domains.items():
ip_id = current_ips[i]
for d in v:
if d in just_domains or i in new_ip_list:
d_id = current_domains[d]
many_objs.append(
ThroughModel(domain_id=d_id, ipaddress_id=ip_id))
display("Bulk gluing them together")
ThroughModel.objects.bulk_create(many_objs)
for ReportHost in root.iter("ReportHost"):
for HostProperties in ReportHost.iter("HostProperties"):
for tag in HostProperties:
if tag.get("name") == "host-ip":
hostIP = tag.text
if hostIP: # apparently nessus doesn't always have an IP to work with...
ip_id = current_ips[hostIP]
self.getVulns(ip_id, ReportHost)
ports = []
current_ports = set([
f"{p.ip_address_id}|{p.port_number}|{p.proto}"
for p in Port.objects.all()
])
for i, v in self.ip_data.items():
for k, data in v.items():
if f"{i}|{k}" not in current_ports:
ports.append(
Port(ip_address_id=i,
port_number=k.split('|')[0],
proto=k.split('|')[1],
service_name=data['service_name']))
current_ports.add(f"{i}|{k}")
display(f"Bulk loading {len(ports)} Ports")
Port.objects.bulk_create(ports)
# Maybe race condition?
# time.sleep(30)
all_ports = {
f"{p.ip_address_id}|{p.port_number}|{p.proto}": p.id
for p in Port.objects.all()
}
display("Gluing in Vulnerabilities")
ThroughModel = Vulnerability.ports.through
vuln_port_current = set([
f"{v.vulnerability_id}|{v.port_id}"
for v in ThroughModel.objects.all()
])
port_vuln_data = []
for d in self.ports:
p = '|'.join(d.split('|')[:3])
v = d.split('|')[-1]
if not all_ports.get(p):
# Weird edge case that seems to happen at random. Relook this one up in DB
print(f"Missing {p}")
p_id, p_num, p_proto = p.split('|')
port_obj = Port.objects.get(ip_address_id=p_id,
port_number=p_num,
proto=p_proto)
all_ports[p] = port_obj.id
if f"{v}|{all_ports[p]}" not in vuln_port_current:
port_vuln_data.append(
ThroughModel(port_id=all_ports[p], vulnerability_id=v))
vuln_port_current.add(f"{v}|{all_ports[p]}")
ThroughModel.objects.bulk_create(port_vuln_data)
vuln_outputs = []
vulnobject_data = set([
f"{v.port.ip_address_id}|{v.port.port_number}|{v.port.proto}|{v.vulnerability_id}"
for v in VulnOutput.objects.all()
])
for k, v in self.vulnobjects.items():
if k not in vulnobject_data:
port_id = all_ports['|'.join(k.split('|')[:3])]
vuln_id = k.split('|')[-1]
vuln_outputs.append(
VulnOutput(port_id=port_id,
vulnerability_id=vuln_id,
data=v))
display(f"Adding in {len(vuln_outputs)} output data")
VulnOutput.objects.bulk_create(vuln_outputs)
display("Attaching CVEs")
cve_current = [c.name for c in CVE.objects.all()]
new_cves = []
for k, v in self.cve_data.items():
if k not in cve_current:
new_cves.append(
CVE(name=k, description=v[0], temporal_score=v[1]))
CVE.objects.bulk_create(new_cves)
display("And finally gluing them to the vulns")
cve_current = {c.name: c.id for c in CVE.objects.all()}
ThroughModel = Vulnerability.cves.through
cve_through = set([
f"{c.cve_id}|{c.vulnerability_id}"
for c in ThroughModel.objects.all()
])
cve_glue = []
for m in list(set(self.cve_map)):
cve_id = cve_current[m.split('|')[0]]
vuln_id = m.split('|')[1]
if f"{cve_id}|{vuln_id}" not in cve_through:
cve_glue.append(
ThroughModel(cve_id=cve_id, vulnerability_id=vuln_id))
ThroughModel.objects.bulk_create(cve_glue)
# Now to add all of this data into the database
return