def process_cidr(self, line):
display("Processing %s" % line)
if "/" in line:
created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
elif "-" in line:
start_ip, end_ip = line.strip().replace(" ", "").split("-")
if "." not in end_ip:
end_ip = ".".join(start_ip.split(".")[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
def process_urls(data):
i, urls, timeout = data
blacklist = [
'Date', 'Connection', 'Content-Type', 'Content-Length', 'Keep-Alive',
'Content-Encoding', 'Vary'
]
new_headers = {}
for u in urls:
display("Processing %s" % u)
try:
res = requests.get(u, timeout=int(timeout), verify=False)
for k in res.headers.keys():
if k not in blacklist:
if not new_headers.get(u, False):
new_headers[u] = []
new_headers[u].append("%s: %s" % (k, res.headers[k]))
except KeyboardInterrupt:
display_warning("Got Ctrl+C, exiting")
sys.exit(1)
except Exception as e:
display_error("%s no good, skipping: %s" % (u, e))
return (i, new_headers)
def process_domain(self, domain_str):
created, domain = self.Domain.find_or_create(only_tool=True, domain=domain_str, in_scope=self.in_scope, passive_scope=self.passive_scope)
if not created:
if domain.in_scope != self.in_scope or domain.passive_scope != self.passive_scope:
display("Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (domain_str, self.in_scope, self.passive_scope))
domain.in_scope = self.in_scope
domain.passive_scope = self.passive_scope
domain.update()
def process_ip(self, ip_str, force_scope=True):
created, ip = self.IPAddress.find_or_create(only_tool=True, ip_address=ip_str, in_scope=in_scope, passive_scope=self.passive_scope)
if not created:
if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope:
display("IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (ip_str, self.in_scope, self.passive_scope))
ip.in_scope = self.in_scope
ip.passive_scope = self.passive_scope
ip.update()
return ip
def run_cmd(cmd):
c = cmd[:-1]
timeout = cmd[-1]
display("Executing command: %s" % ' '.join(c))
try:
Popen(c).wait(timeout=timeout)
except:
display_error(
"Timeout of %s reached. Aborting thread for command: %s" %
(timeout, ' '.join(c)))
def descope_cidr(self, cidr):
CIDR = self.ScopeCIDR.all(cidr=cidr)
if CIDR:
for c in CIDR:
display("Removing {} from ScopeCIDRs".format(c.cidr))
c.delete()
cnet = IPNetwork(cidr)
for ip in self.IPAddress.all():
if IPAddress(ip.ip_address) in cnet:
self.descope_ip(ip.ip_address)
def run(self, args):
# pdb.set_trace()
if not args.binary:
self.binary = which.run("LinkedInt.py")
else:
self.binary = which.run(args.binary)
if not self.binary:
display_error(
"LinkedInt binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(domain=args.domain)
if args.top:
titles = [
user.job_title.split(" at ")[0]
for user in domain.users
if user.job_title
]
words = []
for t in titles:
words += [w.lower() for w in t.split(" ")]
word_count = Counter(words).most_common()
display("Using the top %s words:" % args.top)
res = []
for w in word_count[: int(args.top)]:
display("\t{}\t{}".format(w[0], w[1]))
res.append(w[0])
# pdb.set_trace()
args.smart_shuffle = ",".join(res)
if args.smart_shuffle:
args.keywords = " OR ".join(
['"{}"'.format(i) for i in args.smart_shuffle.split(",")]
)
self.process_domain(domain, args)
args.keywords = " AND ".join(
['-"{}"'.format(i) for i in args.smart_shuffle.split(",")]
)
self.process_domain(domain, args)
else:
self.process_domain(domain, args)
self.BaseDomain.commit()
def run(self, args):
if not args.api_key:
display_error("You must supply an API key to use shodan!")
return
if args.search:
ranges = [args.search]
if args.import_db:
ranges = []
if args.rescan:
ranges += [
"net:{}".format(c.cidr) for c in self.ScopeCidr.all()
]
else:
ranges += [
"net:{}".format(c.cidr)
for c in self.ScopeCidr.all(tool=self.name)
]
api = shodan_api.Shodan(args.api_key)
for r in ranges:
time.sleep(1)
# pdb.set_trace()
results = api.search(r)
display("{} results found for: {}".format(results['total'], r))
for res in results['matches']:
ip_address_str = res['ip_str']
port_str = res['port']
transport = res['transport']
display("Processing IP: {} Port: {}/{}".format(
ip_address_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_address_str)
created, port = self.Port.find_or_create(ip_address=IP,
port_number=port_str,
proto=transport)
port.meta['shodan_data'] = res
port.save()
self.IPAddress.commit()
def run_cmd(cmd):
c = cmd[:-1]
timeout = cmd[-1]
display("Executing command: %s" % " ".join(c))
current_time = time.time()
if timeout:
process = Popen(c)
while time.time() < current_time + timeout and process.poll() is None:
time.sleep(5)
if process.poll() is None:
display_error(
"Timeout of %s reached. Aborting thread for command: %s" %
(timeout, " ".join(c)))
process.terminate()
else:
Popen(c).wait()
def descope_ip(self, ip):
ip = self.IPAddress.all(ip_address=ip)
if ip:
for i in ip:
display("Removing IP {} from scope".format(i.ip_address))
i.in_scope = False
i.passive_scope = False
i.update()
for d in i.domains:
in_scope_ips = [
ipa for ipa in d.ip_addresses
if ipa.in_scope or ipa.passive_scope
]
if not in_scope_ips:
display(
"Domain {} has no more scoped IPs. Removing from scope."
.format(d.domain))
d.in_scope = False
d.passive_scope = False
self.IPAddress.commit()
def process_output(self, cmds):
display("Importing data to database")
for cmd in cmds:
if cmd['cidr']:
_, cidr = self.ScopeCidr.find_or_create(cidr=cmd['cidr'])
cidr.meta['whois'] = open(cmd['output']).read()
display(cidr.meta['whois'])
cidr.update()
elif cmd['domain']:
_, domain = self.BaseDomain.find_or_create(domain=cmd['domain'])
domain.meta['whois'] = open(cmd['output']).read()
display(domain.meta['whois'])
domain.update()
self.BaseDomain.commit()
def run(self, args):
if args.import_file:
for nFile in args.import_file:
self.process_data(nFile, args)
elif args.launch:
if (
not args.username
and not args.password
and not args.host
and not args.uuid
and not args.policy_id
and not args.folder_id
):
display_error(
"You must supply a username, password, and host to launch a Nessus job"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
uuid=args.uuid,
policy_id=args.policy_id,
folder_id=args.folder_id,
)
ips = [
ip.ip_address
for ip in self.IPAddress.all(scope_type="active", tool=self.name)
]
cidrs = [cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)]
domains = [
domain.domain
for domain in self.Domain.all(scope_type="active", tool=self.name)
]
targets = ", ".join(merge_ranges(ips + cidrs) + domains)
res = n.launch_job(targets, args.job_name)
display("New Nessus job launched with ID {}".format(res))
display(
"Remember this number! You'll need it to download the job once it is done."
)
elif args.download:
if (
not args.username
and not args.password
and not args.host
and not args.job_id
):
display_error(
"You must supply host, username, password and job_id to download a report to import"
)
else:
n = NessusRequest(
args.username,
args.password,
args.host,
proxies={"https": "127.0.0.1:8080"},
)
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:]
)
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path
)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(
output_path, "Nessus-export-{}.nessus".format(int(time.time()))
)
n.export_file(args.job_id, output_path)
self.process_data(output_path, args)
def process_data(self, nFile, args):
print("Reading", nFile)
tree = ET.parse(nFile)
root = tree.getroot()
skip = []
for ReportHost in root.iter("ReportHost"):
os = []
hostname = ""
hostIP = ""
for HostProperties in ReportHost.iter("HostProperties"):
for tag in HostProperties:
if tag.get("name") == "host-ip":
hostIP = tag.text
if tag.get("name") == "host-fqdn":
hostname = tag.text.lower()
hostname = hostname.replace("www.", "")
if tag.get("name") == "operating-system":
os = tag.text.split("\n")
if hostIP: # apparently nessus doesn't always have an IP to work with...
if hostname:
display(
"Gathering Nessus info for {} ( {} )".format(hostIP, hostname)
)
else:
display("Gathering Nessus info for {}".format(hostIP))
created, ip = self.IPAddress.find_or_create(ip_address=hostIP)
if hostname:
if not args.internal:
created, domain = self.Domain.find_or_create(domain=hostname)
if ip not in domain.ip_addresses:
ip.save()
domain.ip_addresses.append(ip)
domain.save()
else:
created, domain = self.Domain.find_or_create(domain=hostname)
if ip not in domain.ip_addresses:
domain.ip_addresses.append(ip)
domain.update()
if os:
for o in os:
if not ip.OS:
ip.OS = o
else:
if o not in ip.OS.split(" OR "):
ip.OS += " OR " + o
self.getVulns(ip, ReportHost)
self.IPAddress.commit()
return
def post_run(self, args):
display("Potential takeovers are stored in {}".format(os.environ['HOME']))
os.environ['HOME'] = self.orig_home
def find_or_create(self,
only_tool=False,
in_scope=False,
passive_scope=False,
**kwargs):
created, d = super(DomainRepository,
self).find_or_create(only_tool, **kwargs)
display("Processing %s" % d.domain)
if created:
# If this is a new subdomain, set scoping info based on what is passed to the function initially.
d.in_scope = in_scope
d.passive_scope = passive_scope
base_domain = '.'.join(
[t for t in tldextract.extract(d.domain)[1:] if t])
BaseDomains = BaseDomainRepository(self.db, "")
# If the base domain is new, it'll inherit the same scoping permissions.
created, bd = BaseDomains.find_or_create(
only_tool,
passive_scope=d.passive_scope,
in_scope=in_scope,
domain=base_domain)
if created:
display_new(
"The base domain %s is being added to the database. Active Scope: %s Passive Scope: %s"
% (base_domain, bd.in_scope, bd.passive_scope))
else:
# If the base domain already exists, then the subdomain inherits the scope info from the base domain.
d.passive_scope = bd.passive_scope
d.in_scope = bd.in_scope
d.base_domain = bd
# Get all IPs that this domain resolves to.
ips = []
try:
answers = dns.resolver.query(d.domain, 'A')
for a in answers:
ips.append(a.address)
except:
# If something goes wrong with DNS, we end up here
pass
if not ips:
display_warning("No IPs discovered for %s" % d.domain)
for i in ips:
IPAddresses = IPRepository(self.db, "")
display("Processing IP address %s" % i)
created, ip = IPAddresses.find_or_create(
only_tool,
in_scope=d.in_scope,
passive_scope=d.passive_scope,
ip_address=i)
# If the IP is in scope, then the domain should be
if ip.in_scope:
d.in_scope = ip.in_scope
ip.passive_scope = True
d.passive_scope = True
# display("%s marked active scope due to IP being marked active." % d.domain)
elif ip.passive_scope:
d.passive_scope = ip.passive_scope
d.ip_addresses.append(ip)
display_new(
"%s is being added to the database. Active Scope: %s Passive Scope: %s"
% (d.domain, d.in_scope, d.passive_scope))
# Final sanity check - if a domain is active scoped, it should also be passively scoped.
if d.in_scope:
d.passive_scope = True
return created, d
def run(self, args):
if not args.api_key:
display_error("You must supply an API key to use shodan!")
return
if args.search:
ranges = [args.search]
if args.import_db:
ranges = []
if args.rescan:
if args.fast:
ranges += [
"net:{}".format(c.cidr) for c in self.ScopeCidr.all()
]
else:
cidrs = [c.cidr for c in self.ScopeCidr.all()]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
if not args.cidr_only:
ranges += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active")
]
else:
if args.fast:
ranges += [
"net:{}".format(c.cidr)
for c in self.ScopeCidr.all(tool=self.name)
]
else:
cidrs = [
c.cidr for c in self.ScopeCidr.all(tool=self.name)
]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
if not args.cidr_only:
ranges += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active",
tool=self.name)
]
api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
api_search_url = (
"https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
for r in ranges:
time.sleep(1)
if ":" in r:
display("Doing Shodan search: {}".format(r))
try:
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
if results.get("error") and "request timed out" in results[
"error"]:
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
total = len(results["matches"])
matches = []
i = 1
while total > 0:
display("Adding {} results from page {}".format(total, i))
matches += results["matches"]
i += 1
try:
time.sleep(1)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
i)).text)
if (results.get("error")
and "request timed out" in results["error"]):
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
total = len(results["matches"])
except Exception as e:
display_error("Something went wrong: {}".format(e))
total = 0
pdb.set_trace()
for res in matches:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP, port_number=port_str, proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.meta["shodan_data"] = res
port.save()
else:
try:
results = json.loads(
requests.get(api_host_url.format(r,
args.api_key)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
# pdb.set_trace()
if results.get("data", False):
display("{} results found for: {}".format(
len(results["data"]), r))
for res in results["data"]:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP,
port_number=port_str,
proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.meta["shodan_data"] = res
port.save()
self.IPAddress.commit()
def find_or_create(self,
only_tool=False,
in_scope=False,
passive_scope=True,
**kwargs):
created, ip = super(IPRepository,
self).find_or_create(only_tool, **kwargs)
if created:
# If newly created then will determine scoping based on parent options and if in a scoped cidr.
ip_str = ip.ip_address
ip.passive_scope = passive_scope
# If the parent domain is active scope, then this also is.
if in_scope:
ip.in_scope = in_scope
else:
# Go through ScopeCIDR table and see if this IP is in a CIDR in scope
ScopeCidrs = ScopeCIDRRepository(self.db, "")
addr = IPAddress(ip.ip_address)
cidrs = ScopeCidrs.all()
# pdb.set_trace()
for c in cidrs:
if addr in IPNetwork(c.cidr):
ip.in_scope = True
# Final sanity check - if an IP is active scoped, it should also be passive scoped.
if ip.in_scope:
ip.passive_scope = True
ip.update()
# Build CIDR info - mainly for reporting
res = False
for cidr in private_subnets:
if IPAddress(ip_str) in cidr:
res = ([str(cidr), 'Non-Public Subnet'], )
if res:
cidr_data = res
else:
try:
res = IPWhois(ip_str).lookup_whois(get_referral=True)
except:
res = IPWhois(ip_str).lookup_whois()
cidr_data = []
for n in res['nets']:
if ',' in n['cidr']:
for cidr_str in n['cidr'].split(', '):
cidr_data.append([cidr_str, n['description']])
else:
cidr_data.append([n['cidr'], n['description']])
cidr_data = [
cidr_d for cidr_d in cidr_data
if IPAddress(ip_str) in IPNetwork(cidr_d[0])
]
cidr_len = len(IPNetwork(cidr_data[0][0]))
matching_cidr = cidr_data[0]
for c in cidr_data:
if len(IPNetwork(c[0])) < cidr_len:
matching_cidr = c
display("Processing CIDR from whois: %s - %s" %
(matching_cidr[1], matching_cidr[0]))
CIDR = CIDRRepository(self.db, "")
created, cidr = CIDR.find_or_create(only_tool=True,
cidr=matching_cidr[0])
if created:
display_new("CIDR %s added to database" % cidr.cidr)
cidr.org_name = matching_cidr[1]
cidr.update()
ip.cidr = cidr
ip.update()
display_new(
"IP address %s added to database. Active Scope: %s Passive Scope: %s"
% (ip.ip_address, ip.in_scope, ip.passive_scope))
return created, ip
