async def task_resolve_whois(self, ext, host, ext1, lock):
async with lock:
try:
whois = IPWhois(host).lookup_whois()
if whois is not None:
if 'asn_description' in whois.keys():
_, err = self.exinfodb.updateEx(ext1,\
Whois(host=ext1.host,\
info="{}".format(whois['asn_description'])))
if err is not None:
if self.debug:
log.warning("WARNING: task_resolve_whois unable to update/inser ({0})".format(err))
elif self.debug:
log.warning("WARNING: task_resolve_whois unable to resolve ({0})".format(host))
except Exception as e:
if self.debug:
log.info("task_resolve_whois {}".format(e))
def find_or_create(self,
only_tool=False,
in_scope=False,
passive_scope=True,
**kwargs):
created, ip = super(IPRepository,
self).find_or_create(only_tool, **kwargs)
if created:
# If newly created then will determine scoping based on parent options and if in a scoped cidr.
ip_str = ip.ip_address
ip.passive_scope = passive_scope
# If the parent domain is active scope, then this also is.
if in_scope:
ip.in_scope = in_scope
else:
# Go through ScopeCIDR table and see if this IP is in a CIDR in scope
ScopeCidrs = ScopeCIDRRepository(self.db, "")
addr = IPAddress(ip.ip_address)
cidrs = ScopeCidrs.all()
# pdb.set_trace()
for c in cidrs:
if addr in IPNetwork(c.cidr):
ip.in_scope = True
# Final sanity check - if an IP is active scoped, it should also be passive scoped.
if ip.in_scope:
ip.passive_scope = True
ip.update()
# Build CIDR info - mainly for reporting
res = False
for cidr in private_subnets:
if IPAddress(ip_str) in cidr:
res = ([str(cidr), "Non-Public Subnet"], )
if res:
cidr_data = res
else:
while True:
try:
res = IPWhois(ip_str).lookup_whois(get_referral=True)
except Exception:
try:
res = IPWhois(ip_str).lookup_whois()
except Exception as e:
display_error(
"Error trying to resolve whois: {}".format(e))
res = {}
if "nets" in res.keys():
break
else:
display_warning(
"The networks didn't populate from whois. Usually retrying after a couple of seconds resolves this. Sleeping for 5 seconds and trying again."
)
again = raw_input(
"Would you like to try again? [Y/n]").lower()
if again == 'y':
time.sleep(5)
else:
res = {
'nets': [{
'cidr':
'0.0.0.0/0',
'description':
'Whois failed to resolve.'
}]
}
break
cidr_data = []
for n in res["nets"]:
if "," in n["cidr"]:
for cidr_str in n["cidr"].split(", "):
cidr_data.append([cidr_str, n["description"]])
else:
cidr_data.append([n["cidr"], n["description"]])
cidr_data = [
cidr_d for cidr_d in cidr_data
if IPAddress(ip_str) in IPNetwork(cidr_d[0])
]
try:
cidr_len = len(IPNetwork(cidr_data[0][0]))
except Exception:
pdb.set_trace()
matching_cidr = cidr_data[0]
for c in cidr_data:
if len(IPNetwork(c[0])) < cidr_len:
matching_cidr = c
display("Processing CIDR from whois: %s - %s" %
(matching_cidr[1], matching_cidr[0]))
CIDR = CIDRRepository(self.db, "")
created, cidr = CIDR.find_or_create(only_tool=True,
cidr=matching_cidr[0])
if created:
display_new("CIDR %s added to database" % cidr.cidr)
cidr.org_name = matching_cidr[1]
cidr.update()
ip.cidr = cidr
ip.update()
display_new(
"IP address %s added to database. Active Scope: %s Passive Scope: %s"
% (ip.ip_address, ip.in_scope, ip.passive_scope))
return created, ip
def find_or_create(self,
ip_str,
only_tool=False,
in_scope=False,
passive_scope=True,
label=None,
force_cidr=None,
**kwargs):
res = False
if label and force_cidr:
res = ([force_cidr, label], )
for cidr in private_subnets:
if IPAddress(ip_str) in cidr:
res = ([str(cidr), "Non-Public Subnet"], )
for cidr in CIDRRepository(self.db, "").all():
if IPAddress(ip_str) in IPNetwork(cidr.cidr):
res = ([str(cidr.cidr), cidr.org_name], )
display("Subnet already in database, not rechecking whois.")
if res:
cidr_data = res
else:
while True:
try:
res = IPWhois(ip_str).lookup_whois(get_referral=True)
except Exception:
try:
res = IPWhois(ip_str).lookup_whois()
except Exception as e:
display_error(
"Error trying to resolve whois: {}".format(e))
res = {}
if "nets" in res.keys():
break
else:
display_warning(
"The networks didn't populate from whois. Defaulting to a /24."
)
# again = raw_input("Would you like to try again? [Y/n]").lower()
# if again == 'y':
# time.sleep(5)
# else:
res = {
'nets': [{
'cidr':
'{}.0/24'.format('.'.join(ip_str.split('.')[:3])),
'description':
'Whois failed to resolve.'
}]
}
break
cidr_data = []
for n in res["nets"]:
if "," in n["cidr"]:
for cidr_str in n["cidr"].split(", "):
cidr_data.append([cidr_str, n["description"]])
else:
cidr_data.append([n["cidr"], n["description"]])
cidr_data = [
cidr_d for cidr_d in cidr_data
if IPAddress(ip_str) in IPNetwork(cidr_d[0])
]
if cidr_data:
try:
cidr_len = len(IPNetwork(cidr_data[0][0]))
except Exception:
pdb.set_trace()
matching_cidr = cidr_data[0]
for c in cidr_data:
if len(IPNetwork(c[0])) < cidr_len:
matching_cidr = c
display("Processing CIDR from whois: %s - %s" %
(str(matching_cidr[1]).split('\n')[0], matching_cidr[0]))
created, cidr = super(CIDRRepository,
self).find_or_create(only_tool,
cidr=matching_cidr[0])
if created:
display_new("CIDR %s added to database" % cidr.cidr)
cidr.org_name = str(matching_cidr[1]).split('\n')[0]
cidr.update()
return created, cidr
