def test_lookup_rdap(self):
try:
from urllib.request import ProxyHandler, build_opener
except ImportError:
from urllib2 import ProxyHandler, build_opener
ips = [
'74.125.225.229', # ARIN
'2001:4860:4860::8888',
'62.239.237.1', # RIPE
'2a00:2381:ffff::1',
'210.107.73.73', # APNIC
'2001:240:10c:1::ca20:9d1d',
'200.57.141.161', # LACNIC
'2801:10:c000::',
'196.11.240.215', # AFRINIC
'2001:43f8:7b0::',
'133.1.2.5', # JPNIC
'115.1.2.3' # KRNIC
]
for ip in ips:
timeout = 5
retry_count = 3
# JPNIC doesn't like my testing
if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'):
timeout = 15
retry_count = 10
log.debug('Testing: {0}'.format(ip))
result = IPWhois(address=ip, timeout=timeout)
try:
self.assertIsInstance(result.lookup_rdap(
retry_count=retry_count), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
HTTPLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
handler = ProxyHandler({'http': 'http://0.0.0.0:80/'})
opener = build_opener(handler)
result = IPWhois(address='74.125.225.229', timeout=0,
proxy_opener=opener)
self.assertRaises(ASNRegistryError, result.lookup_rdap)
log.debug('Testing allow_permutations')
result = IPWhois(address='74.125.225.229', timeout=0,
allow_permutations=False)
self.assertRaises(ASNRegistryError, result.lookup_rdap)
def test_lookup_rdap(self):
try:
from urllib.request import ProxyHandler, build_opener
except ImportError:
from urllib2 import ProxyHandler, build_opener
ips = [
'74.125.225.229', # ARIN
'2001:4860:4860::8888',
'62.239.237.1', # RIPE
'2a00:2381:ffff::1',
'210.107.73.73', # APNIC
'2001:240:10c:1::ca20:9d1d',
'200.57.141.161', # LACNIC
'2801:10:c000::',
'196.11.240.215', # AFRINIC
'2001:43f8:7b0::',
'133.1.2.5', # JPNIC
'115.1.2.3' # KRNIC
]
for ip in ips:
timeout = 5
retry_count = 3
# JPNIC doesn't like my testing
if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'):
timeout = 15
retry_count = 10
log.debug('Testing: {0}'.format(ip))
result = IPWhois(address=ip, timeout=timeout)
try:
self.assertIsInstance(
result.lookup_rdap(retry_count=retry_count), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
HTTPLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
handler = ProxyHandler({'http': 'http://0.0.0.0:80/'})
opener = build_opener(handler)
result = IPWhois(address='74.125.225.229',
timeout=0,
proxy_opener=opener)
self.assertRaises(ASNRegistryError, result.lookup_rdap)
def get_ip_whois(self, item, conn):
#set our default values, assuming we ever get an answer
whois_last_updated = (datetime.now() +
timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%S")
item['_source']['whois_last_updated'] = whois_last_updated
item['_source']['whois'] = None
ipaddr = item['_source']['ipaddress']
lookup_successful = False
try:
ans = IPWhois(ipaddr, timeout=5).lookup_rdap(depth=1,
rate_limit_timeout=5)
if ans:
lookup_successful = True
whois_last_updated = (
datetime.now()).strftime("%Y-%m-%dT%H:%M:%S")
item['_source']['whois_last_updated'] = whois_last_updated
item['_source']['whois'] = json.dumps(ans)
try:
item['_source'][
'whois_summary'] = self.create_whois_summary(ans)
except Exception as e:
print(e)
pass
except Exception as e:
pass
conn.send([item, lookup_successful])
return
def test_lookup_whois(self):
ips = [
'74.125.225.229', # ARIN
'2001:4860:4860::8888',
'62.239.237.1', # RIPE
'2a00:2381:ffff::1',
'210.107.73.73', # APNIC
'2001:240:10c:1::ca20:9d1d',
'200.57.141.161', # LACNIC
'2801:10:c000::',
'196.11.240.215', # AFRINIC
'2001:43f8:7b0::',
'133.1.2.5', # JPNIC
'115.1.2.3' # KRNIC
]
for ip in ips:
timeout = 5
retry_count = 3
# JPNIC doesn't like my testing
if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'):
timeout = 15
retry_count = 10
log.debug('Testing: {0}'.format(ip))
result = IPWhois(address=ip, timeout=timeout)
try:
# TODO: keep until deprecated lookup is removed, for coverage
self.assertIsInstance(result.lookup_whois(
retry_count=retry_count), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
HTTPLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
rwhois_ips = [
'38.113.116.218' # COGNETCO
]
for ip in rwhois_ips:
result = IPWhois(ip)
try:
self.assertIsInstance(result.lookup_whois(
get_referral=True,
ignore_referral_errors=True,
inc_raw=True), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
for ip in rwhois_ips:
result = IPWhois(ip)
try:
self.assertIsInstance(result.lookup_whois(
get_referral=True,
ignore_referral_errors=True,
inc_raw=True,
extra_blacklist=['rwhois.cogentco.com']), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
try:
self.assertIsInstance(result.lookup_whois(
get_referral=True,
ignore_referral_errors=False,
inc_raw=True,
extra_blacklist=['rwhois.cogentco.com']), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
BlacklistError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
break
for ip in rwhois_ips:
result = IPWhois(ip)
try:
self.assertIsInstance(result.lookup_whois(
get_referral=True,
ignore_referral_errors=False,
inc_raw=False), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
break
def test_lookup_whois(self):
ips = [
'74.125.225.229', # ARIN
'2001:4860:4860::8888',
'62.239.237.1', # RIPE
'2a00:2381:ffff::1',
'210.107.73.73', # APNIC
'2001:240:10c:1::ca20:9d1d',
'200.57.141.161', # LACNIC
'2801:10:c000::',
'196.11.240.215', # AFRINIC
'2001:43f8:7b0::',
'133.1.2.5', # JPNIC
'115.1.2.3' # KRNIC
]
for ip in ips:
timeout = 5
retry_count = 3
# JPNIC doesn't like my testing
if ip in ('133.1.2.5', '2001:240:10c:1::ca20:9d1d'):
timeout = 15
retry_count = 10
log.debug('Testing: {0}'.format(ip))
result = IPWhois(address=ip, timeout=timeout)
try:
# TODO: keep until deprecated lookup is removed, for coverage
self.assertIsInstance(result.lookup(retry_count=retry_count),
dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
HTTPLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
rwhois_ips = [
'38.113.116.218' # COGNETCO
]
for ip in rwhois_ips:
result = IPWhois(ip)
try:
self.assertIsInstance(
result.lookup_whois(get_referral=True,
ignore_referral_errors=True,
inc_raw=True), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
for ip in rwhois_ips:
result = IPWhois(ip)
try:
self.assertIsInstance(
result.lookup_whois(
get_referral=True,
ignore_referral_errors=True,
inc_raw=True,
extra_blacklist=['rwhois.cogentco.com']), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
try:
self.assertIsInstance(
result.lookup_whois(
get_referral=True,
ignore_referral_errors=False,
inc_raw=True,
extra_blacklist=['rwhois.cogentco.com']), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
BlacklistError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
break
for ip in rwhois_ips:
result = IPWhois(ip)
try:
self.assertIsInstance(
result.lookup_whois(get_referral=True,
ignore_referral_errors=False,
inc_raw=False), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
break
def test_repr(self):
# basic str test
log.debug('Basic str test: {0}'.format('74.125.225.229'))
obj = IPWhois('74.125.225.229')
self.assertIsInstance(repr(obj), str)
def lookup_ip(self, item, conn):
thread_timeout = self.process_timeout
_start = time.time()
updated = ""
ans = ""
while time.time() - _start <= thread_timeout:
ipaddr = item['_source']['ipaddress']
timeout = time.time()
try:
if not ans:
ans = IPWhois(ipaddr,
timeout=5).lookup_rdap(depth=1,
rate_limit_timeout=5)
if ans:
#print("breaking out of loop")
break
else:
print('hit timeout')
#set last updated so that it will appear in search 6 hours from now
whois_last_updated = (datetime.now() - timedelta(
weeks=2, hours=6)).strftime("%Y-%m-%dT%H:%M:%S")
ans = {"error": "whois lookup timedout"}
item['_source']['whois_last_updated'] = whois_last_updated
item['_source']['whois'] = json.dumps(ans)
conn.send(item)
return
except Exception as e:
up = datetime.now() - timedelta(weeks=2, hours=6)
whois_last_updated = up.strftime("%Y-%m-%dT%H:%M:%S")
item['_source']['whois_last_updated'] = whois_last_updated
if self.verbose_logging:
print("Got an exception!: {}".format(e))
print(item)
if type(e) == ipwhois.exceptions.HTTPLookupError:
ans = {"error": "404"}
item['_source']['whois'] = json.dumps(ans)
conn.send(item)
return
elif type(e) == ipwhois.exceptions.IPDefinedError:
print("network defined")
item['_source']['whois'] = json.dumps(ans)
conn.send(item)
return
elif type(e) == ipwhois.HTTPRateLimitError:
conn.send(item)
return
else:
ans = None
if ans:
try:
cidr = ans['network']['cidr']
country = ans['network']['country']
name = ans['network']['name']
k = list(ans['objects'].keys())[0]
address = ans['objects'][k]['contact']['address'][0][
'value']
email = ans['objects'][k]['contact']['email'][0]['value']
whois_summary = "{}, {}, {}, {}".format(
name, country, address,
email).replace("\n", ", ").replace("\\n", ", ")
up = datetime.now()
whois_last_updated = up.strftime("%Y-%m-%dT%H:%M:%S")
except Exception as e:
if self.verbose_logging:
print("Got an exception!: {}".format(e))
whois_summary = None
up = datetime.now()
whois_last_updated = up.strftime("%Y-%m-%dT%H:%M:%S")
pass
item['_source']['whois'] = json.dumps(ans)
item['_source']['whois_last_updated'] = whois_last_updated
if whois_summary:
item['_source']['whois_summary'] = whois_summary
conn.send(item)
return
print("hit thread timeout")
return
