async def task_resolve_whois(self, ext, host, ext1, lock): async with lock: try: whois = IPWhois(host).lookup_whois() if whois is not None: if 'asn_description' in whois.keys(): _, err = self.exinfodb.updateEx(ext1,\ Whois(host=ext1.host,\ info="{}".format(whois['asn_description']))) if err is not None: if self.debug: log.warning("WARNING: task_resolve_whois unable to update/inser ({0})".format(err)) elif self.debug: log.warning("WARNING: task_resolve_whois unable to resolve ({0})".format(host)) except Exception as e: if self.debug: log.info("task_resolve_whois {}".format(e))
def find_or_create(self, only_tool=False, in_scope=False, passive_scope=True, **kwargs): created, ip = super(IPRepository, self).find_or_create(only_tool, **kwargs) if created: # If newly created then will determine scoping based on parent options and if in a scoped cidr. ip_str = ip.ip_address ip.passive_scope = passive_scope # If the parent domain is active scope, then this also is. if in_scope: ip.in_scope = in_scope else: # Go through ScopeCIDR table and see if this IP is in a CIDR in scope ScopeCidrs = ScopeCIDRRepository(self.db, "") addr = IPAddress(ip.ip_address) cidrs = ScopeCidrs.all() # pdb.set_trace() for c in cidrs: if addr in IPNetwork(c.cidr): ip.in_scope = True # Final sanity check - if an IP is active scoped, it should also be passive scoped. if ip.in_scope: ip.passive_scope = True ip.update() # Build CIDR info - mainly for reporting res = False for cidr in private_subnets: if IPAddress(ip_str) in cidr: res = ([str(cidr), "Non-Public Subnet"], ) if res: cidr_data = res else: while True: try: res = IPWhois(ip_str).lookup_whois(get_referral=True) except Exception: try: res = IPWhois(ip_str).lookup_whois() except Exception as e: display_error( "Error trying to resolve whois: {}".format(e)) res = {} if "nets" in res.keys(): break else: display_warning( "The networks didn't populate from whois. Usually retrying after a couple of seconds resolves this. Sleeping for 5 seconds and trying again." ) again = raw_input( "Would you like to try again? [Y/n]").lower() if again == 'y': time.sleep(5) else: res = { 'nets': [{ 'cidr': '0.0.0.0/0', 'description': 'Whois failed to resolve.' }] } break cidr_data = [] for n in res["nets"]: if "," in n["cidr"]: for cidr_str in n["cidr"].split(", "): cidr_data.append([cidr_str, n["description"]]) else: cidr_data.append([n["cidr"], n["description"]]) cidr_data = [ cidr_d for cidr_d in cidr_data if IPAddress(ip_str) in IPNetwork(cidr_d[0]) ] try: cidr_len = len(IPNetwork(cidr_data[0][0])) except Exception: pdb.set_trace() matching_cidr = cidr_data[0] for c in cidr_data: if len(IPNetwork(c[0])) < cidr_len: matching_cidr = c display("Processing CIDR from whois: %s - %s" % (matching_cidr[1], matching_cidr[0])) CIDR = CIDRRepository(self.db, "") created, cidr = CIDR.find_or_create(only_tool=True, cidr=matching_cidr[0]) if created: display_new("CIDR %s added to database" % cidr.cidr) cidr.org_name = matching_cidr[1] cidr.update() ip.cidr = cidr ip.update() display_new( "IP address %s added to database. Active Scope: %s Passive Scope: %s" % (ip.ip_address, ip.in_scope, ip.passive_scope)) return created, ip
def find_or_create(self, ip_str, only_tool=False, in_scope=False, passive_scope=True, label=None, force_cidr=None, **kwargs): res = False if label and force_cidr: res = ([force_cidr, label], ) for cidr in private_subnets: if IPAddress(ip_str) in cidr: res = ([str(cidr), "Non-Public Subnet"], ) for cidr in CIDRRepository(self.db, "").all(): if IPAddress(ip_str) in IPNetwork(cidr.cidr): res = ([str(cidr.cidr), cidr.org_name], ) display("Subnet already in database, not rechecking whois.") if res: cidr_data = res else: while True: try: res = IPWhois(ip_str).lookup_whois(get_referral=True) except Exception: try: res = IPWhois(ip_str).lookup_whois() except Exception as e: display_error( "Error trying to resolve whois: {}".format(e)) res = {} if "nets" in res.keys(): break else: display_warning( "The networks didn't populate from whois. Defaulting to a /24." ) # again = raw_input("Would you like to try again? [Y/n]").lower() # if again == 'y': # time.sleep(5) # else: res = { 'nets': [{ 'cidr': '{}.0/24'.format('.'.join(ip_str.split('.')[:3])), 'description': 'Whois failed to resolve.' }] } break cidr_data = [] for n in res["nets"]: if "," in n["cidr"]: for cidr_str in n["cidr"].split(", "): cidr_data.append([cidr_str, n["description"]]) else: cidr_data.append([n["cidr"], n["description"]]) cidr_data = [ cidr_d for cidr_d in cidr_data if IPAddress(ip_str) in IPNetwork(cidr_d[0]) ] if cidr_data: try: cidr_len = len(IPNetwork(cidr_data[0][0])) except Exception: pdb.set_trace() matching_cidr = cidr_data[0] for c in cidr_data: if len(IPNetwork(c[0])) < cidr_len: matching_cidr = c display("Processing CIDR from whois: %s - %s" % (str(matching_cidr[1]).split('\n')[0], matching_cidr[0])) created, cidr = super(CIDRRepository, self).find_or_create(only_tool, cidr=matching_cidr[0]) if created: display_new("CIDR %s added to database" % cidr.cidr) cidr.org_name = str(matching_cidr[1]).split('\n')[0] cidr.update() return created, cidr