def _whois(self, _type, target=None): """ perform whois on domain or ipv4 addr """ res = [] try: if target: if _type == 'domain': res.append(whois.whois(target)) else: obj = IPWhois(target) res.append(obj.lookup_rdap(depth=1)) else: if _type == 'domain': log = self._read_log('domainname') else: log = self._read_log('ipv4addr') for target in log: if target: if _type == 'domain': res.append(whois.whois(target)) else: obj = IPWhois(target) res.append(obj.lookup_rdap(depth=1)) except: pass return res
def asn(ip): try: results = {} target = IPWhois(ip) lookup = target.lookup_rdap(depth=1) if lookup: results["asn"] = { "asn": lookup["asn"], "asn_cidr": lookup["asn_cidr"], "asn_country_code": lookup["asn_country_code"], "asn_date": lookup["asn_date"], "asn_description": lookup["asn_description"], "asn_registry": lookup["asn_registry"], } results["network"] = { "cidr": lookup["network"]["cidr"], "country": lookup["network"]["country"], "handle": lookup["network"]["handle"], "name": lookup["network"]["name"], } return results except Exception as e: tb1 = traceback.TracebackException.from_exception(e) print("".join(tb1.format()))
def _whois(ip_pk): ip = IPv4Whois.objects.get(pk=ip_pk) ip.status = IPv4Whois.STATUS_LOOKING_UP_WHOIS ip.save() obj = IPWhois(ip.address, timeout=9) try: results = obj.lookup_rdap() except Exception as exc: ip.status = IPv4Whois.STATUS_LOOKUP_WHOIS_FAILED ip.save() raise exc ip.whois = json.dumps(results) ip.status = IPv4Whois.STATUS_LOOKUP_WHOIS_SUCCESS ip.save() kafka_msg = { 'IP': ip.address, 'Whois': results, 'Host': settings.EXTERNAL_IP, 'Timestamp': datetime.utcnow().isoformat(), } send_to_kafka('results', kafka_msg)
def whois(ip): obj = IPWhois(ip) results = obj.lookup_rdap(depth=1, retry_count=5, rate_limit_timeout=5) results_stripped = { "asn": results.get("asn"), "asn_cidr": results.get("asn_cidr"), "asn_country_code": results.get("asn_country_code"), "asn_date": results.get("asn_date"), "asn_description": results.get("asn_description"), "asn_registry": results.get("asn_registry"), "entities": results.get("entities"), "net_start_address": results.get("network").get("cidr"), "net_end_address": results.get("network").get("end_address"), "net_cidr": results.get("network").get("cidr"), "net_type": results.get("network").get("type"), "net_organization": results.get("network").get("name"), "net_ref": results.get("network").get("links"), "raw_command": f"whois -h whois.radb.net {ip}" } if results.get("network") != None: if results.get("network").get("events") != None: for event in results.get("network").get("events"): if event.get("action") == "last changed": results_stripped["net_updated"] = event["timestamp"] elif event.get("action") == "registration": results_stripped["registration"] = event["timestamp"] return results_stripped
class Emails(object): ''' Not all of the RIR's store contact info in the same ways at the same levels and an abuse email can be burried. After testing on 700 IPs from various RIRs showed that there is ABSOLUTELY NO consistency as to which level you find a true asbuse contact. So we are parsing multiple levels (depth=x) to get all the emails. In some test cases the first abuse email didn't appear until level 3 ''' def __init__(self, ip): self.WIobj = IPWhois(ip, timeout=20) def get_emails(self, max_depth): email_list = list() seen = set() try: for x in range(max_depth): results = self.WIobj.lookup_rdap(depth=x) for m in results['objects'].items(): if results['objects'] is not None: for x in results['objects']: tmp_em = results['objects'][x]['contact']['email'] if tmp_em is not None: for y in range(len(tmp_em)): if tmp_em is not None: email = tmp_em[y]['value'] if email in seen: continue seen.add(email) email_list.append(email) except: print("Error in the get_emails function") return email_list
def get_info(self): obj = IPWhois(self.ip_address) ip_rir = obj.lookup_rdap(depth=2) # the lookup returns back json object, but the ORG is quite deep inside, so need to get it from the entities dictionary and then use for parsing entities = ip_rir["entities"] #print(entities) org_name = '' for i in entities : if i[0:3] == "ORG": org_name = i rir_info = { 'network' : ip_rir["network"]["cidr"], 'parent_network' : ip_rir["network"]["parent_handle"], 'database': ip_rir["asn_registry"], 'network_name': ip_rir["network"]["name"], 'code': ip_rir["network"]["country"], 'type': ip_rir["network"]["type"], 'org': org_name if (org_name != '') else '', 'company': ip_rir["objects"][org_name]["contact"]["name"] if (org_name != '') else '', 'address': ip_rir["objects"][org_name]["contact"]["address"][0]["value"] if (org_name!='') else '', 'asn' : ip_rir["asn"], 'asn_country': ip_rir["asn_country_code"], 'updated' : ip_rir["network"]["events"][0]["timestamp"] } return rir_info
def run(db_connection, project_name): project = db_connection["projects"].find_one({ "project": project_name}) nets = {} if "asns" in project: for asn in project["asns"]: if "cidrs" in asn: for cidr in asn["cidrs"]: nets[IPNetwork(cidr)] = asn["description"] for ip in db_connection[project_name + ".ips"].find(): if "asn" not in ip: try: if not check_ip_in_net(db_connection, project_name, IPAddress(ip["ip"]), nets): whois_obj = IPWhois(ip["ip"]) whois_results = whois_obj.lookup_rdap() if "asn_description" in whois_results: db_connection[project_name + ".ips"].update_one({"ip": ip["ip"]}, {'$set': { "asn": whois_results["asn_description"]}}) except Exception: pass if "rdns" not in ip: try: rdns, _, _ = socket.gethostbyaddr(ip["ip"]) db_connection[project_name + ".ips"].update_one({"ip": ip["ip"]}, {'$set': { "rdns": rdns}}) except Exception: pass
def lookup(ip): # Import locally as to not interfere with other lookup functions from emojiflags.lookup import lookup ip = IPWhois(ip) # Lookup IP address for domain name output = ip.lookup_rdap(depth=1) # Save JSON data to a local file with open('static/data.txt', 'w', encoding='utf-8') as outfile: json.dump(output, outfile, ensure_ascii=False, indent=2) config = json.loads(open('static/data.txt').read()) # Find the unique website name so it can be used to call specific data from the JSON file name = str(config["entities"]) name = name.replace("[", "") name = name.replace("]", "") name = name.replace("'", "") # Retrieve some relevant data from the JSON file ipaddress = config["query"] country = (config["asn_country_code"]) # Get the emoji flag for the country countryflag = lookup(country) owner = config["objects"][name]["contact"]["name"] description = config["asn_description"] address = config["objects"][name]["contact"]["address"] # Remove unnecessary or irrelevant characters from the address in order to be readable address = str(address[0]).replace("{'type': None, 'value': '", "") address = address.replace("'}", " ") # Combine details into a list to return details = [ipaddress, country + " " + countryflag, owner, description, address] return details
def lookup_full_ip_info(ip): with warnings.catch_warnings(): warnings.simplefilter("ignore") # create a whois instance for the given IP whois = IPWhois(ip) result = whois.lookup_rdap() # iterator object for holding "object's" keys objkeyview = result["objects"].keys() objkeyitr = list(objkeyview) # create a new dictionary object with all cool fields in it asn_description = result["asn_description"] ip_version = result["network"]["ip_version"] if len(objkeyitr) > 0: ip_owner_name = result["objects"][objkeyitr[0]]["contact"]["name"] ip_owner_location = result["objects"][ objkeyitr[0]]["contact"]["address"][0]["value"] ip_contact_name = result["objects"][ objkeyitr[1]]["contact"]["name"] ip_contact_location = result["objects"][ objkeyitr[1]]["contact"]["address"][0]["value"] ip_contact_email = result["objects"][ objkeyitr[1]]["contact"]["email"][0]["value"] ip_contact_phone = result["objects"][ objkeyitr[1]]["contact"]["phone"][0]["value"] else: ip_owner_name = None ip_owner_location = None ip_contact_name = None ip_contact_location = None ip_contact_email = None ip_contact_phone = None result = asn_description + "~" + ip_version + "~" + ip_owner_name + "~" + ip_owner_location + "~" + ip_contact_name + "~" + ip_contact_location + "~" + ip_contact_email + "~" + ip_contact_phone print(result) return result
def get_remote_whos(url): try: obj = IPWhois(DomainCheck.get_remote_IP(url)) results = obj.lookup_rdap(depth=1) pprint(results) except socket.error: print("Error no connection .. WHOS")
def cmd_whois_ip(ip): """Simple whois client to check IP addresses (IPv4 and IPv6). Example: \b $ habu.whois.ip 8.8.8.8 { "nir": null, "asn_registry": "arin", "asn": "15169", "asn_cidr": "8.8.8.0/24", "asn_country_code": "US", "asn_date": "1992-12-01", "asn_description": "GOOGLE - Google LLC, US", "query": "8.8.8.8", ... """ warnings.filterwarnings("ignore") obj = IPWhois(ip) data = obj.lookup_rdap() print(json.dumps(data, indent=4))
def get_whois(self): try: obj = IPWhois(self.address) results = obj.lookup_rdap(depth=1) except: print("whois lookup failed <---") return 1 have_cc = False # Now unpack whois result r, val = self.get_field(results, 'asn_registry') if r and val: self.registry = val print(" registry = %s" % self.registry) r, val = self.get_field(results, 'asn_country_code') if r and val: self.cc = val.lower() have_cc = True print(" cc = %s" % self.cc) if 'network' in results: netwk = results['network'] r, val = self.get_field(netwk, 'name') if r and val: self.name = val print(" name = %s" % val.encode('utf-8', 'replace')) # Print as Bytes if not have_cc: r, val = self.get_field(netwk, 'country') if r and val: self.cc = val.upper() print(" cc = %s" % self.cc) return 0
def main(ip): obj = IPWhois(ip) try: results = obj.lookup_rdap(depth=1) except: results = None return results
def getlatlongIP(ip): result = IPWhois(ip) try: ret = result.lookup_rdap(depth=1) # print(ret) try: # Get the MaxMind geo data for the query. # I do not redistribute the GeoLite2 database, download # GeoLite2-City.mmdb from: # https://dev.maxmind.com/geoip/geoip2/geolite2/ mm_reader = geoip2.database.Reader('GeoLite2-City.mmdb') # Query the database. mm_response = mm_reader.city(ret['query']) lat = mm_response.location.latitude lng = mm_response.location.longitude return lat, lng # Generic exception. Need to determine all raised and update handling. # geoip2.errors.AddressNotFoundError, TypeError, etc. except Exception as e: print(e) pass except: pass
def get_ASN_Infos(ipaddr): """ Get Autonomous System Number informations linked to an ip address :param ipaddr: ip address of the website linked to the certificate common name :return: list of ASN infos: asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email or the same with empty values """ try: warnings.filterwarnings("ignore") obj = IPWhois(ipaddr) results = obj.lookup_rdap(depth=1) asn = results['asn'] asn_cidr = results['asn_cidr'] asn_country_code = results['asn_country_code'] asn_description = results['asn_description'] # parsing of all the entities members of the ASN record. # -> when finding an entity with 'abuse' role, print the email present # in the contact object. try: for entity in results['objects'].values(): if 'abuse' in entity['roles']: asn_abuse_email = entity['contact']['email'][0]['value'] break except Exception as e: asn_abuse_email = "" return asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email except Exception as e: asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email = "", "", "", "", "" return asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email
def whois(): print "[" + t.green("+") + "]Please provide an IP for WHOIS lookup." TARGET = raw_input("\n<" + t.cyan("WHOIS") + ">$ ") obj = IPWhois(TARGET) results = obj.lookup_rdap(depth=1) pprint(results) print "\n[" + t.magenta( "?") + "]Would you like to append the WHOIS record to a text file?\n" logs = raw_input("[Y]es/[N]o: ") format = json.dumps(results, indent=2) if logs == "y": with open("whois.log", "ab") as outfile: outfile.write("Host: " + TARGET + "\n") outfile.write(format) outfile.close() print "[" + t.green( "+") + "]Results saved to whois.log in the current directory.\n" elif logs == "n": print "[" + t.green("+") + "]Returning to main menu.\n" else: print "[" + t.red("!") + "]Unhandled Option.\n"
def do_work_son(sProject, sUniqSelectorId, sUniqTargetId, sDomain, sIp, sProtocol, sOpenTcpPort, sTcpService): getToLogging() try: # ref: https://media.readthedocs.org/pdf/ipwhois/dev/ipwhois.pdf from ipwhois import IPWhois from pprint import pprint from ipwhois import IPDefinedError logging.debug("whois_ip() sIp: " + str(sIp)) try: objIp_Whois = IPWhois(sIp) results = objIp_Whois.lookup_rdap(depth=1, inc_raw=True ) logging.debug("whois_ip() results: " + str(results)) jRawResults = results['raw'] logging.debug("whois_ip() jRawResults: " + str(jRawResults)) except IPDefinedError: logging.debug("whois_ip() Error: Private IP address, skipping IP!") sSourceTool = 'whois_ip' sSeverity = "INFO" jEvent = { "project": sProject, "uniq_selector_id": sUniqSelectorId, "uniq_target_id": sUniqTargetId, "ip": sIp, "whoisip": jRawResults, "severity": sSeverity } splunkEvent(jEvent, sSourceTool) except Exception as e: logging.error("traceback.format_exc(): " + traceback.format_exc()) logging.error("str(Exception): " + str(e)) logging.error("ip_whois() subprocess.Popen failed :\ ")
def who_is(addr): ip = IPWhois(addr) results = ip.lookup_rdap() print('ASN Information: \n') for k,v in results.items(): if k.startswith('asn'): print(f'{k}: {v}')
def whoisProcessor(): print "GroupID: ", groupid consumer = KafkaConsumer(port_scan_source_topic, group_id=groupid, bootstrap_servers=props['kafka']['hosts'], enable_auto_commit=True, auto_offset_reset='smallest') for msg in consumer: node = json.loads(msg.value) ip = str(msg.key) if (len(node['ports']) > 0): print "Node: ", ip try: gir = gi.record_by_addr(ip) w = IPWhois(ip) out = {} out['whois'] = w.lookup_rdap() out['geoip'] = gir out['ports'] = node['ports'] message_out = json.dumps(out) kp.send(port_scan_enrich_topic, message_out, msg.key) except Exception as e: print "Error: %s " % e
def main(): home = os.path.dirname(os.path.abspath(__file__)) os.chdir(home) with io.open("ip-subnets-unique.txt") as in_file: lines = in_file.readlines() with io.open("ip-subnets-unique-whois.txt", "w") as out_file: out_file.write( u"# Attention! All host counters include broadcast and network addresses.\n\n" ) total_hosts = 0 for line in lines: src = line.strip() if not src: out_file.write(u"\n") continue if src.startswith("#"): out_file.write(u"{0}\n".format(src)) continue print(src) (network_address, mask) = src.split("/") whois = IPWhois(network_address) whois_data = whois.lookup_rdap(depth=1) network_name = whois_data["network"]["name"] asn = whois_data["asn"] asn_description = whois_data["asn_description"] hosts = ipaddr.IPv4Network(src).numhosts total_hosts += hosts out_file.write( u"# {0}\n# AS{1} {2}\n# {3} host(s)\n{4}\n\n".format( network_name, asn, asn_description, hosts, src)) out_file.write(u"# {0} host(s)\n".format(total_hosts))
def lookup_ip_info(ip): with warnings.catch_warnings(): warnings.simplefilter("ignore") # create a whois instance for the given IP whois = IPWhois(ip) # create a Google Maps API instance gmap = googlemaps.Client('AIzaSyAJYrRExLKqEW794dHT9QuO82aMt2VO3Yg') # lookup the information for this IP and place in a dictionary object iplookup = whois.lookup_rdap() # need a list here to obtain the first code in the objects dictionary keyview = iplookup["objects"].keys() itr = list(keyview) # continue only if there exist keys if len(itr) > 0: code = itr[0] location = iplookup["objects"][code]["contact"]["address"][0][ "value"] # DEBUG print(location) geocode_result = gmap.geocode(location) # return the location object only if there is a result from Google Maps # send back only if there is a result if len(geocode_result) > 0 and geocode_result[0]["geometry"][ "location"] is not None: # DEBUG: print(pprint.pformat(geocode_result[0]["geometry"]["location"])) return geocode_result[0]["geometry"]["location"] else: return None else: return None
def get_allocated_range(ip): print "Fetching whois report of {}".format(ip) obj = IPWhois(ip) results = obj.lookup_rdap(depth=1) #TODO Confirm that cidr is the relevant field ip_cidr = results["network"]["cidr"].split(",") return ip_cidr
def get_rdap_registry_info(ip_input, rdap_depth, proxies=None): """Gathers registry info in RDAP protocol Arguments: ip_input {string} -- Artifact.value rdap_depth {int} -- 0,1 or 2 Returns: {object} -- Registry info, RDAP Protocol """ try: proxy_opener = make_proxy_opener(proxies) if proxies else None internet_protocol_address_object = IPWhois(ip_input, allow_permutations=True, proxy_opener=proxy_opener) try: rdap_response = internet_protocol_address_object.lookup_rdap( rdap_depth) if internet_protocol_address_object.dns_zone: rdap_response[ "dns_zone"] = internet_protocol_address_object.dns_zone return rdap_response except exceptions.ASNRegistryError as e: logging.error(traceback.format_exc()) except: logging.error(traceback.format_exc())
def whois(input_ip, depth=1): try: result = IPWhois(input_ip) ret = result.lookup_rdap(depth=depth) return ret except: return False
def get_asn_description(ip): try: asn = IPWhois(ip) asn_owner = asn.lookup_rdap(depth=1) return asn_owner['asn_description'] except exceptions.ASNRegistryError: return "Unknown ASN"
def queryarecords(): print("\n") print("[*] Querying A record DNS entries.") print("=========================================================") for key, value in cnamerecords.items(): lookup = key + domain try: answers = dns.resolver.query(lookup, 'A') for rdata in answers: print( "[INFO] \033[33m\"{}\"\033[0;0m Record Found resolving to \033[33m{}\033[0;0m." .format(key[:-1], rdata.address)) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=UserWarning) obj = IPWhois(str(rdata.address)) results = obj.lookup_rdap() print( " └── [INFO] IP Address resolves to an ASN owned by \033[33m{}\033[0;0m" .format(results['asn_description'])) for asnkey, asnvalue in asnproviders.items(): if asnkey in format(results['asn_description']): print( " └── \033[32m[+]\033[0;0m {} Services appear to be provided by \033[33m{}\033[0;0m." .format(value, asnvalue)) except: pass
def check_whois(ip): obj = IPWhois(ip) result = obj.lookup_rdap(depth=1) # print (result) #debug print("[+]debug : " + str(ip)) print("[+]debug : " + str(result['asn_description'])) print("########") return str(result['network']['name'])
def is_ip_asn_country_code_in_set(addr: str, country_codes) -> bool: obj = IPWhois(addr) res = obj.lookup_rdap() asn_country_code = res["asn_country_code"].lower() if asn_country_code in country_codes: return True else: return False
def get_rdap(ipaddr): obj = IPWhois(ipaddr) RdapInfo = obj.lookup_rdap() if RdapInfo is None: return 0 else: return RdapInfo
def Ip_rdap(ip): obj = IPWhois(ip) results = obj.lookup_rdap() print("asn: AS" + results["asn"]) print("asn_cidr: " + results["asn_cidr"]) print("asn_registry: " + results["asn_registry"]) print("asn_country_code: " + results["asn_country_code"]) print("asn_description: " + results["asn_description"])
def get_ipwhois(self, depth=1): # Perform the RDAP lookup for self.addr retrieving all # entities up to depth. obj = IPWhois(self.addr) ret = obj.lookup_rdap(depth=depth) # Set the updated timestamp for cache expiration. ret['updated'] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') return ret
def lookup(ip, rdap=False): obj = IPWhois(ip) if rdap: # TODO: RDAP output includes less relevant info, needs a dedicated formatter return obj.lookup_rdap() else: ret = obj.lookup_whois() # remove some fields that clutter for x in ['raw', 'raw_referral', 'referral']: ret.pop(x, None) return ret
def ip_geography( ip = None , key_file = None ) : """ :param ip: :param key_file: :return: """ key = str() b_ret = False out = str() if ip is None or key_file is None : raise ValueError( 'bad parameter list' ) with open( key_file )as f : key = f.readline().strip() try : req = 'http://api.ipinfodb.com//v3/ip-city/?key=%s&format=json&ip=%s' % \ ( key , ip ) r = requests.get( req ) x = r.json() for item in x : out += '%s : %s\n' % ( item , x[item] ) obj = IPWhois( ip ) results = obj.lookup_rdap(depth=1) s = StringIO() pprint.pprint( results , s ) out += '\n\n\n' out += s.getvalue() b_ret = True except Exception as e : out = e.message return b_ret , str( out )
def _whois(ip, org_names): from ipwhois import IPWhois if type(ip) is not str: ip = _get_flow_ip(ip) if ip not in _whois_cache: whois = IPWhois(ip) try: name = whois.lookup_rdap()['network']['name'] if not name: name = whois.lookup()['nets'][0]['name'] except: print("WHOIS ERROR") name = 'OTHER' _whois_cache[ip] = _clean_netname(org_names, name, ip) return _whois_cache[ip]
def test_lookup_rdap(self): try: from urllib.request import ProxyHandler, build_opener except ImportError: from urllib2 import ProxyHandler, build_opener ips = [ '74.125.225.229', # ARIN '2001:4860:4860::8888', '62.239.237.1', # RIPE '2a00:2381:ffff::1', '210.107.73.73', # APNIC '2001:240:10c:1::ca20:9d1d', '200.57.141.161', # LACNIC '2801:10:c000::', '196.11.240.215', # AFRINIC '2001:43f8:7b0::' ] for ip in ips: log.debug('Testing: {0}'.format(ip)) result = IPWhois(ip) try: self.assertIsInstance(result.lookup_rdap(), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError, HTTPLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) handler = ProxyHandler({'http': 'http://0.0.0.0:80/'}) opener = build_opener(handler) result = IPWhois('74.125.225.229', 0, opener) self.assertRaises(HTTPLookupError, result.lookup_rdap) log.debug('Testing allow_permutations') result = IPWhois('74.125.225.229', timeout=0, allow_permutations=False) self.assertRaises(ASNRegistryError, result.lookup_rdap)
def run_rdap(self,ip_address): """Perform an RDAP lookup for an IP address. An RDAP lookup object is returned. From IPWhois: IPWhois.lookup_rdap() is now the recommended lookup method. RDAP provides a far better data structure than legacy WHOIS and REST lookups (previous implementation). RDAP queries allow for parsing of contact information and details for users, organizations, and groups. RDAP also provides more detailed network information. Parameters: ip_address The IP address to use for the RDAP look-up """ try: with warnings.catch_warnings(): # Hide the 'allow_permutations has been deprecated' warning until ipwhois removes it warnings.filterwarnings("ignore",category=UserWarning) rdapwho = IPWhois(ip_address) results = rdapwho.lookup_rdap(depth=1) return results except Exception as error: click.secho("[!] Failed to collect RDAP information for {}!".format(ip_address),fg="red") click.secho("L.. Details: {}".format(error),fg="red")
def analyze(ip): links = [] results = IPWhois(ip.value) results = results.lookup_rdap() for entity in results['objects']: entity = results['objects'][entity] if entity['contact']['kind'] != 'individual': # Create the company company = Company.get_or_create(name=entity['contact']['name'], rdap=entity) link = Link.connect(ip, company) link.add_history('hosting') links.append(link) # Link it to every email address referenced for email_info in entity['contact']['email']: email = Email.get_or_create(value=email_info['value']) link = Link.connect(company, email) links.append(link) return links
def get_rdap_ip(ip): es = Elasticsearch() does_exist = es.exists(index='rdap', doc_type='ipaddr', id = ip) print does_exist if does_exist is True: status = 200 print "Found it!" get_record = es.get(index='rdap',doc_type='ipaddr', id = ip) results = jsonify(get_record['_source']) else: try: obj = IPWhois(ip) results_raw = obj.lookup_rdap(depth=1) status = 200 results = jsonify(results_raw) es.index(index='rdap', doc_type='ipaddr', id=ip, body=json.dumps(results_raw)) except Exception as e: print e results = jsonify({'status': "not_found"}) status = 404 results = jsonify({'status': "not_found"}) return results,status
def ip_whois(ip): obj = IPWhois(ip) try: results = obj.lookup_rdap(depth=1) except: results = 'notfound' print 'ASN Registry Lookup Failed' if results != 'notfound': print colored(style.BOLD + '\nWhoIS Report for IP: %s\n' + style.END, 'green') % str(ip) print colored(style.BOLD + '--------------- Basic Info ---------------' + style.END, 'blue') print 'ASN ID: %s' % results['asn'] if 'network' in results.keys(): print 'Org. Name: %s' % results['network']['name'] print 'CIDR Range: %s' % results['network']['cidr'] print 'Start Address: %s' % results['network']['start_address'] print 'Parent Handle: %s' % results['network']['parent_handle'] print 'Country: %s' % results['network']['country'] if 'objects' and 'entities' in results.keys(): print colored(style.BOLD + '\n----------- Per Handle Results -----------' + style.END, 'blue') for x in results['entities']: print 'Handle: %s' % x if 'contact' in results['objects'][x].keys(): print '\tKind: %s' % results['objects'][x]['contact']['kind'] if results['objects'][x]['contact']['phone'] is not None: for y in results['objects'][x]['contact']['phone']: print '\tPhone: %s' % y['value'] if results['objects'][x]['contact']['title'] is not None: print results['objects'][x]['contact']['title'] if results['objects'][x]['contact']['role'] is not None: print results['objects'][x]['contact']['role'] if results['objects'][x]['contact']['address'] is not None: for y in results['objects'][x]['contact']['address']: print '\tAddress: %s' % y['value'].replace('\n',',') if results['objects'][x]['contact']['email'] is not None: for y in results['objects'][x]['contact']['email']: print '\tEmail: %s' % y['value']
def whois(self, ip): self.log.debug('Fetching WHOIS data for {}'.format(ip)) w = IPWhois(ip, timeout=10) return w.lookup_rdap(depth=1, retry_count=5, rate_limit_timeout=60)
class IPWhoisCLI: """ The CLI wrapper class for outputting formatted IPWhois results. Args: addr: An IPv4 or IPv6 address as a string, integer, IPv4Address, or IPv6Address. timeout: The default timeout for socket connections in seconds. proxy_http: The urllib.request.ProxyHandler dictionary for proxy HTTP support or None. proxy_https: The urllib.request.ProxyHandler dictionary for proxy HTTPS support or None. allow_permutations: allow net.Net() to use additional methods if DNS lookups to Cymru fail. """ def __init__( self, addr, timeout, proxy_http, proxy_https, allow_permutations ): self.addr = addr self.timeout = timeout handler_dict = None if proxy_http is not None: handler_dict = {'http': proxy_http} if proxy_https is not None: if handler_dict is None: handler_dict = {'https': proxy_https} else: handler_dict['https'] = proxy_https if handler_dict is None: self.opener = None else: handler = ProxyHandler(handler_dict) self.opener = build_opener(handler) self.allow_permutations = allow_permutations self.obj = IPWhois(address=self.addr, timeout=self.timeout, proxy_opener=self.opener, allow_permutations=self.allow_permutations) def generate_output_header(self, query_type='RDAP'): """ The function for generating the CLI output header. Args: query_type: The IPWhois query type. Returns: String: The generated output string. """ output = '\n{0}{1}{2} query for {3}:{4}\n\n'.format( ANSI['ul'], ANSI['b'], query_type, self.obj.address_str, ANSI['end'] ) return output def generate_output_newline(self, line='0', colorize=True): """ The function for generating a CLI output new line. Args: line: The line number (0-4). Determines indentation. colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ return generate_output( line=line, is_parent=True, colorize=colorize ) def generate_output_asn(self, json_data=None, hr=True, show_name=False, colorize=True): """ The function for generating CLI output ASN results. Args: json_data: The data dictionary to process. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ if json_data is None: json_data = {} # Python 2.6 doesn't support set literal expressions, use explicit # set() instead. keys = set(['asn', 'asn_cidr', 'asn_country_code', 'asn_date', 'asn_registry']).intersection(json_data) output = '' for key in keys: output += generate_output( line='0', short=HR_ASN[key]['_short'] if hr else key, name=HR_ASN[key]['_name'] if (hr and show_name) else None, value=(json_data[key] if ( json_data[key] is not None and len(json_data[key]) > 0 and json_data[key] != 'NA') else 'None'), colorize=colorize ) return output def generate_output_entities(self, json_data=None, hr=True, show_name=False, colorize=True): """ The function for generating CLI output RDAP entity results. Args: json_data: The data dictionary to process. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ output = '' short = HR_RDAP['entities']['_short'] if hr else 'entities' name = HR_RDAP['entities']['_name'] if (hr and show_name) else None output += generate_output( line='0', short=short, name=name, is_parent=False if (json_data is None or json_data['entities'] is None) else True, value='None' if (json_data is None or json_data['entities'] is None) else None, colorize=colorize ) if json_data is not None: for ent in json_data['entities']: output += generate_output( line='1', value=ent, colorize=colorize ) return output def generate_output_events(self, source, key, val, line='2', hr=True, show_name=False, colorize=True): """ The function for generating CLI output RDAP events results. Args: source: The parent key (network or objects). key: The event key (events or events_actor). val: The event dictionary. line: The line number (0-4). Determines indentation. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ output = generate_output( line=line, short=HR_RDAP[source][key]['_short'] if hr else key, name=HR_RDAP[source][key]['_name'] if (hr and show_name) else None, is_parent=False if (val is None or len(val) == 0) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) if val is not None: count = 0 for item in val: try: action = item['action'] except KeyError: action = None try: timestamp = item['timestamp'] except KeyError: timestamp = None try: actor = item['actor'] except KeyError: actor = None if count > 0: output += generate_output( line=str(int(line)+1), is_parent=True, colorize=colorize ) output += generate_output( line=str(int(line)+1), short=HR_RDAP_COMMON[key]['action'][ '_short'] if hr else 'action', name=HR_RDAP_COMMON[key]['action'][ '_name'] if (hr and show_name) else None, value=action, colorize=colorize ) output += generate_output( line=str(int(line)+1), short=HR_RDAP_COMMON[key]['timestamp'][ '_short'] if hr else 'timestamp', name=HR_RDAP_COMMON[key]['timestamp'][ '_name'] if (hr and show_name) else None, value=timestamp, colorize=colorize ) output += generate_output( line=str(int(line)+1), short=HR_RDAP_COMMON[key]['actor'][ '_short'] if hr else 'actor', name=HR_RDAP_COMMON[key]['actor'][ '_name'] if (hr and show_name) else None, value=actor, colorize=colorize ) count += 1 return output def generate_output_list(self, source, key, val, line='2', hr=True, show_name=False, colorize=True): """ The function for generating CLI output RDAP list results. Args: source: The parent key (network or objects). key: The event key (events or events_actor). val: The event dictionary. line: The line number (0-4). Determines indentation. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ output = generate_output( line=line, short=HR_RDAP[source][key]['_short'] if hr else key, name=HR_RDAP[source][key]['_name'] if (hr and show_name) else None, is_parent=False if (val is None or len(val) == 0) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) if val is not None: for item in val: output += generate_output( line=str(int(line)+1), value=item, colorize=colorize ) return output def generate_output_notices(self, source, key, val, line='1', hr=True, show_name=False, colorize=True): """ The function for generating CLI output RDAP notices results. Args: source: The parent key (network or objects). key: The event key (events or events_actor). val: The event dictionary. line: The line number (0-4). Determines indentation. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ output = generate_output( line=line, short=HR_RDAP[source][key]['_short'] if hr else key, name=HR_RDAP[source][key]['_name'] if (hr and show_name) else None, is_parent=False if (val is None or len(val) == 0) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) if val is not None: count = 0 for item in val: title = item['title'] description = item['description'] links = item['links'] if count > 0: output += generate_output( line=str(int(line)+1), is_parent=True, colorize=colorize ) output += generate_output( line=str(int(line)+1), short=HR_RDAP_COMMON[key]['title']['_short'] if hr else ( 'title'), name=HR_RDAP_COMMON[key]['title']['_name'] if ( hr and show_name) else None, value=title, colorize=colorize ) output += generate_output( line=str(int(line)+1), short=HR_RDAP_COMMON[key]['description'][ '_short'] if hr else 'description', name=HR_RDAP_COMMON[key]['description'][ '_name'] if (hr and show_name) else None, value=description.replace( '\n', '\n{0}'.format(generate_output(line='3')) ), colorize=colorize ) output += self.generate_output_list( source=source, key='links', val=links, line=str(int(line)+1), hr=hr, show_name=show_name, colorize=colorize ) count += 1 return output def generate_output_network(self, json_data=None, hr=True, show_name=False, colorize=True): """ The function for generating CLI output RDAP network results. Args: json_data: The data dictionary to process. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ if json_data is None: json_data = {} output = generate_output( line='0', short=HR_RDAP['network']['_short'] if hr else 'network', name=HR_RDAP['network']['_name'] if (hr and show_name) else None, is_parent=True, colorize=colorize ) for key, val in json_data['network'].items(): if key in ['links', 'status']: output += self.generate_output_list( source='network', key=key, val=val, line='1', hr=hr, show_name=show_name, colorize=colorize ) elif key in ['notices', 'remarks']: output += self.generate_output_notices( source='network', key=key, val=val, line='1', hr=hr, show_name=show_name, colorize=colorize ) elif key == 'events': output += self.generate_output_events( source='network', key=key, val=val, line='1', hr=hr, show_name=show_name, colorize=colorize ) elif key not in ['raw']: output += generate_output( line='1', short=HR_RDAP['network'][key]['_short'] if hr else key, name=HR_RDAP['network'][key]['_name'] if ( hr and show_name) else None, value=val, colorize=colorize ) return output def generate_output_objects(self, json_data=None, hr=True, show_name=False, colorize=True): """ The function for generating CLI output RDAP object results. Args: json_data: The data dictionary to process. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ if json_data is None: json_data = {} output = generate_output( line='0', short=HR_RDAP['objects']['_short'] if hr else 'objects', name=HR_RDAP['objects']['_name'] if (hr and show_name) else None, is_parent=True, colorize=colorize ) count = 0 for obj_name, obj in json_data['objects'].items(): if count > 0: output += self.generate_output_newline( line='1', colorize=colorize ) count += 1 output += generate_output( line='1', short=obj_name, is_parent=True, colorize=colorize ) for key, val in obj.items(): if key in ['links', 'entities', 'roles', 'status']: output += self.generate_output_list( source='objects', key=key, val=val, line='2', hr=hr, show_name=show_name, colorize=colorize ) elif key in ['notices', 'remarks']: output += self.generate_output_notices( source='objects', key=key, val=val, line='2', hr=hr, show_name=show_name, colorize=colorize ) elif key == 'events': output += self.generate_output_events( source='objects', key=key, val=val, line='2', hr=hr, show_name=show_name, colorize=colorize ) elif key == 'contact': output += generate_output( line='2', short=HR_RDAP['objects']['contact'][ '_short'] if hr else 'contact', name=HR_RDAP['objects']['contact']['_name'] if ( hr and show_name) else None, is_parent=False if (val is None or len(val) == 0) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) if val is not None: for k, v in val.items(): if k in ['phone', 'address', 'email']: output += generate_output( line='3', short=HR_RDAP['objects']['contact'][k][ '_short'] if hr else k, name=HR_RDAP['objects']['contact'][k][ '_name'] if ( hr and show_name) else None, is_parent=False if ( val is None or len(val) == 0 ) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) if v is not None: for item in v: i_type = ', '.join(item['type']) if ( isinstance(item['type'], list) ) else item['type'] i_type = i_type if ( i_type is not None and len(i_type) > 0) else '' i_value = item['value'].replace( '\n', '\n{0}'.format( generate_output( line='4', is_parent=True, colorize=colorize ).replace('\n', '')) ) tmp_out = '{0}{1}{2}'.format( i_type, ': ' if i_type != '' else '', i_value ) output += generate_output( line='4', value=tmp_out, colorize=colorize ) else: output += generate_output( line='3', short=HR_RDAP['objects']['contact'][k][ '_short'] if hr else k, name=HR_RDAP['objects']['contact'][k][ '_name'] if ( hr and show_name) else None, value=v, colorize=colorize ) elif key not in ['raw']: output += generate_output( line='2', short=HR_RDAP['objects'][key]['_short'] if hr else key, name=HR_RDAP['objects'][key]['_name'] if ( hr and show_name) else None, value=val, colorize=colorize ) return output def lookup_rdap(self, hr=True, show_name=False, colorize=True, **kwargs): """ The function for wrapping IPWhois.lookup_rdap() and generating formatted CLI output. Args: hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. kwargs: Arguments to pass to IPWhois.lookup_rdap(). Returns: String: The generated output string. """ # Perform the RDAP lookup ret = self.obj.lookup_rdap(**kwargs) if script_args.json: output = json.dumps(ret) else: # Header output = self.generate_output_header(query_type='RDAP') # ASN output += self.generate_output_asn( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) # Entities output += self.generate_output_entities( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) # Network output += self.generate_output_network( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) # Objects output += self.generate_output_objects( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) if 'nir' in ret: # NIR output += self.generate_output_nir( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) return output def generate_output_whois_nets(self, json_data=None, hr=True, show_name=False, colorize=True): """ The function for generating CLI output Legacy Whois networks results. Args: json_data: The data dictionary to process. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ if json_data is None: json_data = {} output = generate_output( line='0', short=HR_WHOIS['nets']['_short'] if hr else 'nets', name=HR_WHOIS['nets']['_name'] if (hr and show_name) else None, is_parent=True, colorize=colorize ) count = 0 for net in json_data['nets']: if count > 0: output += self.generate_output_newline( line='1', colorize=colorize ) count += 1 output += generate_output( line='1', short=net['handle'], is_parent=True, colorize=colorize ) for key, val in net.items(): if val and '\n' in val: output += generate_output( line='2', short=HR_WHOIS['nets'][key]['_short'] if hr else key, name=HR_WHOIS['nets'][key]['_name'] if ( hr and show_name) else None, is_parent=False if (val is None or len(val) == 0) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) for v in val.split('\n'): output += generate_output( line='3', value=v, colorize=colorize ) else: output += generate_output( line='2', short=HR_WHOIS['nets'][key]['_short'] if hr else key, name=HR_WHOIS['nets'][key]['_name'] if ( hr and show_name) else None, value=val, colorize=colorize ) return output def generate_output_whois_referral(self, json_data=None, hr=True, show_name=False, colorize=True): """ The function for generating CLI output Legacy Whois referral results. Args: json_data: The data dictionary to process. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ if json_data is None: json_data = {} output = generate_output( line='0', short=HR_WHOIS['referral']['_short'] if hr else 'referral', name=HR_WHOIS['referral']['_name'] if (hr and show_name) else None, is_parent=False if json_data['referral'] is None else True, value='None' if json_data['referral'] is None else None, colorize=colorize ) if json_data['referral']: for key, val in json_data['referral'].items(): if val and '\n' in val: output += generate_output( line='1', short=HR_WHOIS['nets'][key]['_short'] if hr else key, name=HR_WHOIS['nets'][key]['_name'] if ( hr and show_name) else None, is_parent=False if (val is None or len(val) == 0) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) for v in val.split('\n'): output += generate_output( line='2', value=v, colorize=colorize ) else: output += generate_output( line='1', short=HR_WHOIS['nets'][key]['_short'] if hr else key, name=HR_WHOIS['nets'][key]['_name'] if ( hr and show_name) else None, value=val, colorize=colorize ) return output def generate_output_nir(self, json_data=None, hr=True, show_name=False, colorize=True): """ The function for generating CLI output NIR network results. Args: json_data: The data dictionary to process. hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. Returns: String: The generated output string. """ if json_data is None: json_data = {} output = generate_output( line='0', short=HR_WHOIS_NIR['nets']['_short'] if hr else 'nir_nets', name=HR_WHOIS_NIR['nets']['_name'] if (hr and show_name) else None, is_parent=True, colorize=colorize ) count = 0 if json_data['nir']: for net in json_data['nir']['nets']: if count > 0: output += self.generate_output_newline( line='1', colorize=colorize ) count += 1 output += generate_output( line='1', short=net['handle'], is_parent=True, colorize=colorize ) for key, val in net.items(): if val and (isinstance(val, dict) or '\n' in val or key == 'nameservers'): output += generate_output( line='2', short=( HR_WHOIS_NIR['nets'][key]['_short'] if ( hr) else key ), name=HR_WHOIS_NIR['nets'][key]['_name'] if ( hr and show_name) else None, is_parent=False if (val is None or len(val) == 0) else True, value='None' if (val is None or len(val) == 0) else None, colorize=colorize ) if key == 'contacts': for k, v in val.items(): if v: output += generate_output( line='3', is_parent=False if ( len(v) == 0) else True, name=k, colorize=colorize ) for contact_key, contact_val in v.items(): if v is not None: tmp_out = '{0}{1}{2}'.format( contact_key, ': ', contact_val ) output += generate_output( line='4', value=tmp_out, colorize=colorize ) elif key == 'nameservers': for v in val: output += generate_output( line='3', value=v, colorize=colorize ) else: for v in val.split('\n'): output += generate_output( line='3', value=v, colorize=colorize ) else: output += generate_output( line='2', short=( HR_WHOIS_NIR['nets'][key]['_short'] if ( hr) else key ), name=HR_WHOIS_NIR['nets'][key]['_name'] if ( hr and show_name) else None, value=val, colorize=colorize ) else: output += 'None' return output def lookup_whois(self, hr=True, show_name=False, colorize=True, **kwargs): """ The function for wrapping IPWhois.lookup_whois() and generating formatted CLI output. Args: hr: Enable human readable key translations. show_name: Show human readable name (default is to only show short). colorize: Colorize the console output with ANSI colors. kwargs: Arguments to pass to IPWhois.lookup_whois(). Returns: String: The generated output string. """ # Perform the RDAP lookup ret = self.obj.lookup_whois(**kwargs) if script_args.json: output = json.dumps(ret) else: # Header output = self.generate_output_header(query_type='Legacy Whois') # ASN output += self.generate_output_asn( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) # Network output += self.generate_output_whois_nets( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) # Referral output += self.generate_output_whois_referral( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) if 'nir' in ret: # NIR output += self.generate_output_nir( json_data=ret, hr=hr, show_name=show_name, colorize=colorize ) output += self.generate_output_newline(colorize=colorize) return output
from ipwhois import IPWhois from pprint import pprint #ip=input("skriv ip:") obj = IPWhois("212.214.217.4") print (type(obj)) results = obj.lookup_rdap(depth=1) print(results['network']['name']) pprint(results)
def extractDNS(ipaddr): obj = IPWhois(ipaddr) results = obj.lookup_rdap(depth=1) pprint(results)
def insert(input_ip='', update=True, expires=7, depth=1): if update: try: # Only update if older than x days. tmp = es.search( index='ipwhois_base', doc_type='base', body={ 'query': { 'bool': { 'must': [{ 'range': { 'updated': { 'gt': 'now-{0}d'.format(expires) } } }, { 'term': { 'query': str(input_ip) } }] } } } ) if len(tmp['hits']['hits']) > 0: return # A generic exception is raised, unfortunately. except Exception as e: print(e) pass # Perform the RDAP lookup for the input IP address retriving all entities # up to depth. result = IPWhois(input_ip) ret = result.lookup_rdap(depth=depth) tmp_objects = ret['objects'].items() for ent_k, ent_v in tmp_objects: if update: try: # Only update if older than 7 days. es_tmp = es.search( index='ipwhois_entity', doc_type='entity', body={ 'query': { 'bool': { 'must': [ { 'range': { 'updated': { 'gt': 'now-{0}d'.format(expires) } } }, { 'term': { 'handle': str(ent_k) } } ] } } } ) if len(es_tmp['hits']['hits']) > 0: continue # A generic exception is raised, unfortunately. except Exception as e: print(e) pass ent = ent_v if sys.version_info >= (2, 7): # Iterate the contact addresses. for addr_k, addr_v in enumerate(ent_v['contact']['address']): try: # Attempt to translate the contact address to geo # coordinates via geopy. location = GEOLOCATOR.geocode(addr_v['value'].replace( '\n', ' ')) # Add the geo coordinates for the contact address. ent['contact']['address'][addr_k]['geo'] = { 'lat': location.latitude, 'lon': location.longitude } except (AttributeError, KeyError, GeocoderQueryError, GeocoderTimedOut): pass # Set the entity updated timestamp. ent['updated'] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') if update: try: ent_search = es.search( index='ipwhois_entity', doc_type='entity', body={ 'query': { 'match': { 'handle': ent['handle'] } } } ) for hit in ent_search['hits']['hits']: es.delete(index='ipwhois_entity', doc_type='entity', id=hit['_id']) except KeyError: pass # Index the entity in elasticsearch. es.index(index='ipwhois_entity', doc_type='entity', body=ent) # Refresh the index for searching duplicates. es.indices.refresh(index='ipwhois_entity') # Don't need the objects key since that data has been entered as the # entities doc_type. del ret['objects'] try: # Get the network ISO country code cc = ret['network']['country'] # Add the geo coordinates for the country, defined in GEO_COORD.json. ret['network']['country_geo'] = { 'lat': GEO_COORD[cc]['latitude'], 'lon': GEO_COORD[cc]['longitude'] } # Set the network country name. ret['network']['country_name'] = COUNTRIES[cc] except KeyError: pass try: # Get the MaxMind geo data for the query. # I do not redistribute the GeoLite2 database, download # GeoLite2-City.mmdb from: # https://dev.maxmind.com/geoip/geoip2/geolite2/ mm_reader = geoip2.database.Reader(str(CUR_DIR) + '/data/GeoLite2-City.mmdb') # Query the database. mm_response = mm_reader.city(ret['query']) # Set the JSON geo data. ret['query_geo'] = { 'lat': mm_response.location.latitude, 'lon': mm_response.location.longitude } ret['query_country_name'] = COUNTRIES[mm_response.country.iso_code] # Generic exception. Need to determine all raised and update handling. # geoip2.errors.AddressNotFoundError, TypeError, etc. except Exception as e: print(e) pass # Set the base updated timestamp. ret['updated'] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') if update: try: ip_search = es.search( index='ipwhois_base', doc_type='base', body={ 'query': { 'match': { 'query': ret['query'] } } } ) for hit in ip_search['hits']['hits']: es.delete(index='ipwhois_base', doc_type='base', id=hit['_id']) except KeyError: pass # Index the base in elasticsearch. es.index(index='ipwhois_base', doc_type='base', body=ret) # Refresh the indices for searching duplicates. es.indices.refresh(index='ipwhois_base') es.indices.refresh(index='ipwhois_entity')