def updateASNinfo(self):
asList = list()
with closing(ASRecord()) as db:
rows = db.dbQueryUnknownCountries()
for row in rows:
(asn, country) = row
asList.append(asn)
print(asList)
client = ip2asn()
for result in client.lookupmany(asList, qType='ASN'):
if not result.asn == None:
print(result.asn[2:], result.owner, result.cc, result.lir)
db.dbUpdateRecord(result.asn[2:], result.owner, result.cc, result.lir)
cymru.ip2asn.dns.testAll()
cymru.ip2asn.whois.testAll()
cymru.mhr.dns.testAll()
cymru.bogon.dns.testAll()
cymru.mhr.whois.testAll()
import sys
sys.exit()
'''
Examples.
'''
import socket
from cymru.ip2asn.dns import DNSClient as ip2asn
client = ip2asn()
ip = socket.gethostbyname("www.google.com")
client.lookup(ip,qType='IP')
client.lookup('15169',qType='ASN')
ip6 = socket.getaddrinfo("www.nerim.net",80,socket.AF_INET6,0,0)[0][4][0]
client.lookup(ip6,qType='IP6')
client.lookupmany(['2001:4860:8010::68','2001:7a8:1:1::76'],qType='IP6')
client.lookupmany(['1515','5005'],qType='ASN')
client.lookup('91.121.224.117',qType='PEER')
import hashlib
from cymru.mhr.dns import DNSClient as mhr
client=mhr()
def getASN(ip_address):
"""Get the ASN Number from the Team Cymru API"""
client = ip2asn()
data = client.lookup(ip_address, qType='IP')
return data.asn
def getASN(ip_address):
"""Get the ASN Number from the Team Cymru API"""
client = ip2asn()
data = client.lookup(ip_address,qType='IP')
return data.asn
uri_scheme = ""
with open(protocolsfile_path, 'r') as scheme_file:
for scheme in scheme_file:
uri_scheme += scheme[:-1]+"|"
uri_scheme = uri_scheme[:-1]
url_regex = "("+uri_scheme+")\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
while True:
if message is not None:
filename, score = message.split()
if prec_filename is None or filename != prec_filename:
domains_list = []
PST = Paste.Paste(filename)
client = ip2asn()
for x in PST.get_regex(url_regex):
matching_url = re.search(url_regex, PST.get_p_content())
url = matching_url.group(0)
to_send = "{} {} {}".format(url, PST._get_p_date(), filename)
p.populate_set_out(to_send, 'Url')
faup.decode(url)
domain = faup.get_domain()
subdomain = faup.get_subdomain()
f1 = None
domains_list.append(domain)
publisher.debug('{} Published'.format(url))
def compute(self, message):
"""
Search for Web links from given message
"""
# Extract item
filename, score = message.split()
if self.prec_filename is None or filename != self.prec_filename:
domains_list = set()
PST = Paste.Paste(filename)
client = ip2asn()
detected_urls = PST.get_regex(self.url_regex)
if len(detected_urls) > 0:
to_print = 'Web;{};{};{};'.format(
PST.p_source, PST.p_date, PST.p_name)
self.redis_logger.info('{}Detected {} URL;{}'.format(
to_print, len(detected_urls), PST.p_rel_path))
for url in detected_urls:
self.redis_logger.debug("match regex: %s" % (url))
# self.redis_logger.debug("match regex search: %s"%(url))
to_send = "{} {} {}".format(url, PST._get_p_date(), filename)
self.process.populate_set_out(to_send, 'Url')
self.redis_logger.debug("url_parsed: %s" % (to_send))
self.faup.decode(url)
domain = self.faup.get_domain()
subdomain = self.faup.get_subdomain()
self.redis_logger.debug('{} Published'.format(url))
if subdomain is not None:
# TODO: # FIXME: remove me
try:
subdomain = subdomain.decode()
except:
pass
if domain is not None:
# TODO: # FIXME: remove me
try:
domain = domain.decode()
except:
pass
domains_list.add(domain)
hostl = self.avoidNone(subdomain) + self.avoidNone(domain)
try:
socket.setdefaulttimeout(1)
ip = socket.gethostbyname(hostl)
# If the resolver is not giving any IPv4 address,
# ASN/CC lookup is skip.
l = client.lookup(ip, qType='IP')
except ipaddress.AddressValueError:
self.redis_logger.debug(
f'ASN/CC lookup failed for IP {ip}')
continue
except:
self.redis_logger.debug(
f'Resolver IPv4 address failed for host {hostl}')
continue
cc = getattr(l, 'cc')
asn = ''
if getattr(l, 'asn') is not None:
asn = getattr(l, 'asn')[2:] # remobe b'
# EU is not an official ISO 3166 code (but used by RIPE
# IP allocation)
if cc is not None and cc != "EU":
self.redis_logger.debug('{};{};{};{}'.format(hostl, asn, cc,
pycountry.countries.get(alpha_2=cc).name))
if cc == self.cc_critical:
to_print = 'Url;{};{};{};Detected {} {}'.format(
PST.p_source, PST.p_date, PST.p_name,
hostl, cc)
self.redis_logger.info(to_print)
else:
self.redis_logger.debug('{};{};{}'.format(hostl, asn, cc))
A_values = lib_refine.checking_A_record(self.r_serv2,
domains_list)
if A_values[0] >= 1:
pprint.pprint(A_values)
# self.redis_logger.info('Url;{};{};{};Checked {} URL;{}'.format(
# PST.p_source, PST.p_date, PST.p_name, A_values[0], PST.p_rel_path))
self.prec_filename = filename