def run_cmd_noout(cmd_data):
cmd = cmd_data[0]
output = cmd_data[1]
c = cmd[:-1]
timeout = cmd[-1]
display("Executing command: %s" % " ".join(c))
current_time = time.time()
f = open(output, 'w')
if timeout:
process = Popen(c, stdout=f, stderr=STDOUT)
while time.time() < current_time + timeout and process.poll() is None:
time.sleep(5)
if process.poll() is None:
display_error(
"Timeout of %s reached. Aborting thread for command: %s" %
(timeout, " ".join(c)))
process.terminate()
else:
Popen(c, stdout=f, stderr=STDOUT).wait()
f.close()
return cmd_data
def process_output(self, cmds):
for c in cmds:
target = c["target"]
output_path = c["output"]
try:
fierceOutput = open(output_path).read()
except IOError:
display_error(
"The output file for {} was not found. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file.".format(
target
)
)
continue
domains = []
if "Now performing" in fierceOutput:
hosts = re.findall(
r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$",
fierceOutput,
re.MULTILINE,
)
if hosts:
for host in hosts:
# print host
domain = (
host.split("\t")[1].lower().replace("www.", "").rstrip(".")
)
if domain not in domains:
domains.append(domain)
elif "Whoah, it worked" in fierceOutput:
print("Zone transfer found!")
hosts = re.findall(
r".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",
fierceOutput,
re.MULTILINE,
)
if hosts:
for host in hosts:
domain = (
host.split("\t")[0].lower().replace("www.", "").rstrip(".")
)
if domain not in domains:
domains.append(domain)
else:
display_error(
"No results found in {}. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file.".format(
output_path
)
)
if domains:
for _domain in domains:
created, domain = self.Domain.find_or_create(domain=_domain)
def process_output(self, cmds):
for c in cmds:
target = c["target"]
output_path = c["output"]
try:
res = json.loads(open(output_path).read())
except IOError:
display_error("DnsRecon failed for {}".format(target))
continue
if " -d " in res[0]["arguments"]:
created, dbrec = self.Domain.find_or_create(domain=target)
dbrec.dns = res
dbrec.save()
for record in res:
domain = None
ip = None
if record.get("type") == "A" or record.get("type") == "PTR":
domain = record.get("name").lower().replace("www.", "")
ip = record.get("address")
elif record.get("type") == "MX":
domain = record.get("exchange").lower().replace("www.", "")
elif record.get("type") == "SRV" or record.get("type" == "NS"):
domain = record.get("target").lower().replace("www.", "")
elif record.get("type") == "SOA":
domain = record.get("mname").lower().replace("www.", "")
if domain:
created, domain_obj = self.Domain.find_or_create(
domain=domain)
if ip:
created, ip_obj = self.IPAddress.find_or_create(
ip_address=ip)
domain_obj.ip_addresses.append(ip_obj)
domain_obj.save()
if '/' in target:
created, bd = self.ScopeCIDR.find_or_create(cidr=target)
else:
created, bd = self.BaseDomain.find_or_create(domain=target)
bd.set_tool(self.name)
self.Domain.commit()
def run_cmd(cmd):
c = cmd[:-1]
timeout = cmd[-1]
display("Executing command: %s" % " ".join(c))
current_time = time.time()
if timeout:
process = Popen(c)
while time.time() < current_time + timeout and process.poll() is None:
time.sleep(5)
if process.poll() is None:
display_error(
"Timeout of %s reached. Aborting thread for command: %s" %
(timeout, " ".join(c)))
process.terminate()
else:
Popen(c).wait()
def process_output(self, cmds):
for cmd in cmds:
output_path = cmd["output"]
if os.path.isfile(output_path):
data = open(output_path).read().split("\n")
for d in data:
for ds in d.split(
'<BR>'): # Sublist3r is spitting out <BR>s now...
if ds:
new_domain = ds.split(":")[0].lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
else:
display_error("{} not found.".format(output_path))
next
self.Domain.commit()
def run(self, args):
ranges = []
cidrs = []
ips = []
search = []
if not args.api_key:
display_error("You must supply an API key to use shodan!")
return
if args.search:
search = [args.search]
if args.import_db:
if args.rescan:
if args.fast:
search += [
"net:{}".format(c.cidr) for c in self.ScopeCidr.all()
]
else:
cidrs += [c.cidr for c in self.ScopeCidr.all()]
if not args.cidr_only:
ips += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active")
]
else:
if args.fast:
search += [
"net:{}".format(c.cidr)
for c in self.ScopeCidr.all(tool=self.name)
]
else:
cidrs += [
c.cidr for c in self.ScopeCidr.all(tool=self.name)
]
if not args.cidr_only:
ips += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active",
tool=self.name)
]
if args.target:
if '/' not in args.target:
ips += [args.target]
elif args.fast:
cidrs += ["net:{}".format(args.target)]
else:
cidrs += [args.target]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
ranges += ips
ranges += search
display(
"Doing a total of {} queries. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(len(ranges), int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for c in cidrs:
ranges = [str(i) for i in IPNetwork(c)]
display(
"Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(c, int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for r in ranges:
self.get_shodan(r, args)
created, cd = self.ScopeCidr.find_or_create(cidr=c)
if created:
cd.delete()
else:
cd.set_tool(self.name)
self.ScopeCidr.commit()
display(
"Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
.format(len(ips), int(len(ranges) / 24.0 / 60.0 / 60.0),
int(len(ranges) / 60.0 / 60.0) % 60,
int(len(ranges) / 60.0) % 60,
len(ranges) % 60))
for i in ips:
self.get_shodan(i, args)
created, ip = self.IPAddress.find_or_create(ip_address=i)
if created:
ip.delete()
else:
ip.set_tool(self.name)
self.IPAddress.commit()
for s in search:
self.get_shodan(s, args)
if s[:4] == "net:":
created, cd = self.ScopeCidr.find_or_create(cidr=s[4:])
if created:
cd.delete()
else:
cd.set_tool(self.name)
self.ScopeCidr.commit()
def get_shodan(self, r, args):
api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
api_search_url = (
"https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
time.sleep(1)
if ":" in r:
display("Doing Shodan search: {}".format(r))
try:
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
if results.get(
"error") and "request timed out" in results["error"]:
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
total = len(results["matches"])
matches = []
i = 1
# pdb.set_trace()
while total > 0:
display("Adding {} results from page {}".format(total, i))
matches += results["matches"]
i += 1
try:
time.sleep(1)
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
i)).text)
if (results.get("error") and "request timed out"
in results["error"] # noqa: W503
):
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
total = len(results["matches"])
except Exception as e:
display_error("Something went wrong: {}".format(e))
total = 0
pdb.set_trace()
domains = []
for res in matches:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(ip_address=IP,
port_number=port_str,
proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
if res.get("ssl", {}).get('cert', {}).get('extensions'):
for d in res['ssl']['cert']['extensions']:
if d['name'] == 'subjectAltName':
domains += get_domains_from_data(d['name'])
if res.get("ssl", {}).get('cert', {}).get(
'subject', {}
).get('CN') and '*' not in res['ssl']['cert']['subject']['CN']:
domains.append(res['ssl']['cert']['subject']['CN'])
if res.get('hostnames'):
domains += res['hostnames']
for d in list(set(domains)):
display("Adding discovered domain {}".format(only_valid(d)))
created, domain = self.Domain.find_or_create(
domain=only_valid(d))
else:
display("Searching for {}".format(r))
try:
results = json.loads(
requests.get(api_host_url.format(r, args.api_key)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
# pdb.set_trace()
if results.get("data", False):
display("{} results found for: {}".format(
len(results["data"]), r))
domains = []
for res in results["data"]:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP, port_number=port_str, proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.status = "open"
port.meta["shodan_data"] = res
port.save()
if res.get("ssl", {}).get('cert', {}).get('extensions'):
for d in res['ssl']['cert']['extensions']:
if d['name'] == 'subjectAltName':
domains += get_domains_from_data(d['data'])
display(
"Domains discovered in subjectAltName: {}".
format(", ".join(
get_domains_from_data(d['data']))))
if res.get("ssl", {}).get('cert', {}).get(
'subject', {}).get('CN') and '*' not in res['ssl'][
'cert']['subject']['CN']:
domains.append(res['ssl']['cert']['subject']['CN'])
if res.get('hostnames'):
domains += res['hostnames']
for d in list(set(domains)):
display("Adding discovered domain {}".format(d))
created, domain = self.Domain.find_or_create(domain=d)
def run(self, args):
if args.import_file:
for nFile in args.import_file:
self.process_data(nFile, args)
elif args.launch:
if (not args.username # noqa: W503
and not args.password # noqa: W503
and not args.host # noqa: W503
and not args.uuid # noqa: W503
and not args.policy_id # noqa: W503
and not args.folder_id # noqa: W503
):
display_error(
"You must supply a username, password, and host to launch a Nessus job"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
uuid=args.uuid,
policy_id=args.policy_id,
folder_id=args.folder_id,
)
ips = [
ip.ip_address
for ip in self.IPAddress.all(scope_type="active",
tool=self.name)
]
cidrs = [
cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)
]
domains = [
domain.domain
for domain in self.Domain.all(scope_type="active",
tool=self.name)
]
targets = ", ".join(merge_ranges(ips + cidrs) + domains)
res = n.launch_job(targets, args.job_name)
display("New Nessus job launched with ID {}".format(res))
display(
"Remember this number! You'll need it to download the job once it is done."
)
elif args.download:
if (not args.username # noqa: W503
and not args.password # noqa: W503
and not args.host # noqa: W503
and not args.job_id # noqa: W503
):
display_error(
"You must supply host, username, password and job_id to download a report to import"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
)
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"],
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(
output_path,
"Nessus-export-{}.nessus".format(int(time.time())))
n.export_file(args.job_id, output_path)
self.process_data(output_path, args)
def run(self, args):
if not args.keyword:
display_error("You need to supply a keyword to search for.")
return
if not args.binary:
self.binary = which.run(self.binary_name)
else:
self.binary = args.binary
if not self.binary:
display_error(
"Asnlookup binary not found. Please explicitly provide path with --binary"
)
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], 'output',
args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], 'output',
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
command_args = " -o {} --output {} ".format(args.keyword, output_path)
current_dir = os.getcwd()
new_dir = "/".join(self.binary.split("/")[:-1])
os.chdir(new_dir)
cmd = shlex.split("python3 " + self.binary + command_args)
print("Executing: %s" % " ".join(cmd))
subprocess.Popen(cmd).wait()
os.chdir(current_dir)
ip_ranges = open(
os.path.join(output_path, "{}_ipv4.txt".format(
args.keyword))).read().split('\n')
for r in ip_ranges:
if r:
display("Processing {}".format(r))
current_cidrs = [c.cidr for c in self.ScopeCIDRs.all()]
new_cidr = True
for nc in current_cidrs:
if IPNetwork(r) in IPNetwork(nc):
new_cidr = False
if new_cidr:
created, SC = self.ScopeCIDRs.find_or_create(cidr=r)
if created:
display_new(
"New CIDR added to ScopeCIDRS: {}".format(r))
self.ScopeCIDRs.commit()
def run(self, args):
if not args.domain:
display_error("You need to supply a domain to search for.")
return
if not args.binary:
self.binary = which.run(self.binary_name)
else:
self.binary = args.binary
if not self.binary:
display_error(
"{} binary not found. Please explicitly provide path with --binary"
.format(self.binary_name))
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], 'output',
args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], 'output',
args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path, "{}.txt".format(args.domain))
command_args = " {} -o {} ".format(args.domain, output_path)
if args.tool_args:
command_args += ' '.join(args.tool_args)
if not args.no_binary:
current_dir = os.getcwd()
new_dir = "/".join(self.binary.split("/")[:-1])
os.chdir(new_dir)
cmd = shlex.split("python2 " + self.binary + command_args)
print("Executing: %s" % " ".join(cmd))
subprocess.Popen(cmd).wait()
os.chdir(current_dir)
results = open(output_path).read().split('\n')
cur_type = None
for r in results:
if r:
if '### Company Names' in r:
cur_type = "company"
elif '### Domain Names' in r:
cur_type = "domain"
elif '### Email Addresses' in r:
cur_type = "email"
else:
if cur_type == "domain":
if args.scope == "active":
created, d = self.Domains.find_or_create(
domain=r, in_scope=True, passive_scope=True)
elif args.scope == "passive":
created, d = self.Domains.find_or_create(
domain=r, in_scope=False, passive_scope=True)
else:
created, d = self.Domains.find_or_create(
domain=r, in_scope=False, passive_scope=False)
self.Domains.commit()
def run(self, args):
# pdb.set_trace()
if not args.binary:
self.binary = which.run("LinkedInt.py")
else:
self.binary = which.run(args.binary)
if not self.binary:
display_error(
"LinkedInt binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
if args.top:
titles = [
user.job_title.split(" at ")[0] for user in domain.users
if user.job_title
]
words = []
for t in titles:
words += [w.lower() for w in get_words(t)]
word_count = Counter(words).most_common()
display("Using the top %s words:" % args.top)
res = []
for w in word_count[:int(args.top)]:
display("\t{}\t{}".format(w[0], w[1]))
res.append(w[0])
# pdb.set_trace()
args.smart_shuffle = ",".join(res)
if args.auto_keyword:
if not args.top:
display_error(
"You must specify the top number of keywords using --top"
)
else:
if os.path.isfile('/tmp/armory_linkedinsearchqueries'):
blacklist = open('/tmp/armory_linkedinsearchqueries'
).read().split('\n')
else:
blacklist = []
bfile = open('/tmp/armory_linkedinsearchqueries', 'a')
for w in args.smart_shuffle.split(','):
if w not in blacklist:
args.keywords = w
self.process_domain(domain, args)
self.BaseDomain.commit()
bfile.write('{}\n'.format(w))
else:
display(
"Skipped {} due to it already being searched.".
format(w))
bfile.close()
elif args.smart_shuffle:
args.keywords = " OR ".join(
['"{}"'.format(i) for i in args.smart_shuffle.split(",")])
self.process_domain(domain, args)
self.BaseDomain.commit()
args.keywords = " AND ".join(
['-"{}"'.format(i) for i in args.smart_shuffle.split(",")])
self.process_domain(domain, args)
self.BaseDomain.commit()
else:
self.process_domain(domain, args)
self.BaseDomain.commit()
self.BaseDomain.commit()
def find_or_create(self,
ip_str,
only_tool=False,
in_scope=False,
passive_scope=True,
label=None,
force_cidr=None,
**kwargs):
res = False
if label and force_cidr:
res = ([force_cidr, label], )
for cidr in private_subnets:
if IPAddress(ip_str) in cidr:
res = ([str(cidr), "Non-Public Subnet"], )
for cidr in CIDRRepository(self.db, "").all():
if IPAddress(ip_str) in IPNetwork(cidr.cidr):
res = ([str(cidr.cidr), cidr.org_name], )
display("Subnet already in database, not rechecking whois.")
if res:
cidr_data = res
else:
while True:
try:
res = IPWhois(ip_str).lookup_whois(get_referral=True)
except Exception:
try:
res = IPWhois(ip_str).lookup_whois()
except Exception as e:
display_error(
"Error trying to resolve whois: {}".format(e))
res = {}
if res.get('nets', []):
break
else:
display_warning(
"The networks didn't populate from whois. Defaulting to a /24."
)
# again = raw_input("Would you like to try again? [Y/n]").lower()
# if again == 'y':
# time.sleep(5)
# else:
res = {
'nets': [{
'cidr':
'{}.0/24'.format('.'.join(ip_str.split('.')[:3])),
'description':
'Whois failed to resolve.'
}]
}
break
cidr_data = []
for n in res["nets"]:
if "," in n["cidr"]:
for cidr_str in n["cidr"].split(", "):
cidr_data.append([cidr_str, n["description"]])
else:
cidr_data.append([n["cidr"], n["description"]])
cidr_data = [
cidr_d for cidr_d in cidr_data
if IPAddress(ip_str) in IPNetwork(cidr_d[0])
]
if cidr_data:
try:
cidr_len = len(IPNetwork(cidr_data[0][0]))
except Exception:
pdb.set_trace()
matching_cidr = cidr_data[0]
for c in cidr_data:
if len(IPNetwork(c[0])) < cidr_len:
matching_cidr = c
display("Processing CIDR from whois: %s - %s" %
(str(matching_cidr[1]).split('\n')[0], matching_cidr[0]))
created, cidr = super(CIDRRepository,
self).find_or_create(only_tool,
cidr=matching_cidr[0])
if created:
display_new("CIDR %s added to database" % cidr.cidr)
cidr.org_name = str(matching_cidr[1]).split('\n')[0]
cidr.update()
return created, cidr