def test_lookup_rdap(self): try: from urllib.request import ProxyHandler, build_opener except ImportError: from urllib2 import ProxyHandler, build_opener ips = [ '74.125.225.229', # ARIN '2001:4860:4860::8888', '62.239.237.1', # RIPE '2a00:2381:ffff::1', '210.107.73.73', # APNIC '2001:240:10c:1::ca20:9d1d', '200.57.141.161', # LACNIC '2801:10:c000::', '196.11.240.215', # AFRINIC '2001:43f8:7b0::', '133.1.2.5', # JPNIC '115.1.2.3' # KRNIC ] for ip in ips: timeout = 5 retry_count = 3 # JPNIC doesn't like my testing if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'): timeout = 15 retry_count = 10 log.debug('Testing: {0}'.format(ip)) result = IPWhois(address=ip, timeout=timeout) try: self.assertIsInstance(result.lookup_rdap( retry_count=retry_count), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError, HTTPLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) handler = ProxyHandler({'http': 'http://0.0.0.0:80/'}) opener = build_opener(handler) result = IPWhois(address='74.125.225.229', timeout=0, proxy_opener=opener) self.assertRaises(ASNRegistryError, result.lookup_rdap) log.debug('Testing allow_permutations') result = IPWhois(address='74.125.225.229', timeout=0, allow_permutations=False) self.assertRaises(ASNRegistryError, result.lookup_rdap)
def test_lookup_rdap(self): try: from urllib.request import ProxyHandler, build_opener except ImportError: from urllib2 import ProxyHandler, build_opener ips = [ '74.125.225.229', # ARIN '2001:4860:4860::8888', '62.239.237.1', # RIPE '2a00:2381:ffff::1', '210.107.73.73', # APNIC '2001:240:10c:1::ca20:9d1d', '200.57.141.161', # LACNIC '2801:10:c000::', '196.11.240.215', # AFRINIC '2001:43f8:7b0::', '133.1.2.5', # JPNIC '115.1.2.3' # KRNIC ] for ip in ips: timeout = 5 retry_count = 3 # JPNIC doesn't like my testing if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'): timeout = 15 retry_count = 10 log.debug('Testing: {0}'.format(ip)) result = IPWhois(address=ip, timeout=timeout) try: self.assertIsInstance( result.lookup_rdap(retry_count=retry_count), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError, HTTPLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) handler = ProxyHandler({'http': 'http://0.0.0.0:80/'}) opener = build_opener(handler) result = IPWhois(address='74.125.225.229', timeout=0, proxy_opener=opener) self.assertRaises(ASNRegistryError, result.lookup_rdap)
def get_ip_whois(self, item, conn): #set our default values, assuming we ever get an answer whois_last_updated = (datetime.now() + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%S") item['_source']['whois_last_updated'] = whois_last_updated item['_source']['whois'] = None ipaddr = item['_source']['ipaddress'] lookup_successful = False try: ans = IPWhois(ipaddr, timeout=5).lookup_rdap(depth=1, rate_limit_timeout=5) if ans: lookup_successful = True whois_last_updated = ( datetime.now()).strftime("%Y-%m-%dT%H:%M:%S") item['_source']['whois_last_updated'] = whois_last_updated item['_source']['whois'] = json.dumps(ans) try: item['_source'][ 'whois_summary'] = self.create_whois_summary(ans) except Exception as e: print(e) pass except Exception as e: pass conn.send([item, lookup_successful]) return
def test_lookup_whois(self): ips = [ '74.125.225.229', # ARIN '2001:4860:4860::8888', '62.239.237.1', # RIPE '2a00:2381:ffff::1', '210.107.73.73', # APNIC '2001:240:10c:1::ca20:9d1d', '200.57.141.161', # LACNIC '2801:10:c000::', '196.11.240.215', # AFRINIC '2001:43f8:7b0::', '133.1.2.5', # JPNIC '115.1.2.3' # KRNIC ] for ip in ips: timeout = 5 retry_count = 3 # JPNIC doesn't like my testing if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'): timeout = 15 retry_count = 10 log.debug('Testing: {0}'.format(ip)) result = IPWhois(address=ip, timeout=timeout) try: # TODO: keep until deprecated lookup is removed, for coverage self.assertIsInstance(result.lookup_whois( retry_count=retry_count), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError, HTTPLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) rwhois_ips = [ '38.113.116.218' # COGNETCO ] for ip in rwhois_ips: result = IPWhois(ip) try: self.assertIsInstance(result.lookup_whois( get_referral=True, ignore_referral_errors=True, inc_raw=True), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) for ip in rwhois_ips: result = IPWhois(ip) try: self.assertIsInstance(result.lookup_whois( get_referral=True, ignore_referral_errors=True, inc_raw=True, extra_blacklist=['rwhois.cogentco.com']), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) try: self.assertIsInstance(result.lookup_whois( get_referral=True, ignore_referral_errors=False, inc_raw=True, extra_blacklist=['rwhois.cogentco.com']), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError, BlacklistError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) break for ip in rwhois_ips: result = IPWhois(ip) try: self.assertIsInstance(result.lookup_whois( get_referral=True, ignore_referral_errors=False, inc_raw=False), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) break
def test_lookup_whois(self): ips = [ '74.125.225.229', # ARIN '2001:4860:4860::8888', '62.239.237.1', # RIPE '2a00:2381:ffff::1', '210.107.73.73', # APNIC '2001:240:10c:1::ca20:9d1d', '200.57.141.161', # LACNIC '2801:10:c000::', '196.11.240.215', # AFRINIC '2001:43f8:7b0::', '133.1.2.5', # JPNIC '115.1.2.3' # KRNIC ] for ip in ips: timeout = 5 retry_count = 3 # JPNIC doesn't like my testing if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'): timeout = 15 retry_count = 10 log.debug('Testing: {0}'.format(ip)) result = IPWhois(address=ip, timeout=timeout) try: # TODO: keep until deprecated lookup is removed, for coverage self.assertIsInstance(result.lookup(retry_count=retry_count), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError, HTTPLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) rwhois_ips = [ '38.113.116.218' # COGNETCO ] for ip in rwhois_ips: result = IPWhois(ip) try: self.assertIsInstance( result.lookup_whois(get_referral=True, ignore_referral_errors=True, inc_raw=True), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) for ip in rwhois_ips: result = IPWhois(ip) try: self.assertIsInstance( result.lookup_whois( get_referral=True, ignore_referral_errors=True, inc_raw=True, extra_blacklist=['rwhois.cogentco.com']), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) try: self.assertIsInstance( result.lookup_whois( get_referral=True, ignore_referral_errors=False, inc_raw=True, extra_blacklist=['rwhois.cogentco.com']), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError, BlacklistError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) break for ip in rwhois_ips: result = IPWhois(ip) try: self.assertIsInstance( result.lookup_whois(get_referral=True, ignore_referral_errors=False, inc_raw=False), dict) except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) break
def test_repr(self): # basic str test log.debug('Basic str test: {0}'.format('74.125.225.229')) obj = IPWhois('74.125.225.229') self.assertIsInstance(repr(obj), str)
def lookup_ip(self, item, conn): thread_timeout = self.process_timeout _start = time.time() updated = "" ans = "" while time.time() - _start <= thread_timeout: ipaddr = item['_source']['ipaddress'] timeout = time.time() try: if not ans: ans = IPWhois(ipaddr, timeout=5).lookup_rdap(depth=1, rate_limit_timeout=5) if ans: #print("breaking out of loop") break else: print('hit timeout') #set last updated so that it will appear in search 6 hours from now whois_last_updated = (datetime.now() - timedelta( weeks=2, hours=6)).strftime("%Y-%m-%dT%H:%M:%S") ans = {"error": "whois lookup timedout"} item['_source']['whois_last_updated'] = whois_last_updated item['_source']['whois'] = json.dumps(ans) conn.send(item) return except Exception as e: up = datetime.now() - timedelta(weeks=2, hours=6) whois_last_updated = up.strftime("%Y-%m-%dT%H:%M:%S") item['_source']['whois_last_updated'] = whois_last_updated if self.verbose_logging: print("Got an exception!: {}".format(e)) print(item) if type(e) == ipwhois.exceptions.HTTPLookupError: ans = {"error": "404"} item['_source']['whois'] = json.dumps(ans) conn.send(item) return elif type(e) == ipwhois.exceptions.IPDefinedError: print("network defined") item['_source']['whois'] = json.dumps(ans) conn.send(item) return elif type(e) == ipwhois.HTTPRateLimitError: conn.send(item) return else: ans = None if ans: try: cidr = ans['network']['cidr'] country = ans['network']['country'] name = ans['network']['name'] k = list(ans['objects'].keys())[0] address = ans['objects'][k]['contact']['address'][0][ 'value'] email = ans['objects'][k]['contact']['email'][0]['value'] whois_summary = "{}, {}, {}, {}".format( name, country, address, email).replace("\n", ", ").replace("\\n", ", ") up = datetime.now() whois_last_updated = up.strftime("%Y-%m-%dT%H:%M:%S") except Exception as e: if self.verbose_logging: print("Got an exception!: {}".format(e)) whois_summary = None up = datetime.now() whois_last_updated = up.strftime("%Y-%m-%dT%H:%M:%S") pass item['_source']['whois'] = json.dumps(ans) item['_source']['whois_last_updated'] = whois_last_updated if whois_summary: item['_source']['whois_summary'] = whois_summary conn.send(item) return print("hit thread timeout") return