class RsnServer(object):
def __init__(self, server):
self.logger = logging.getLogger('RsnServer')
self.server = server
self.ipv4 = query(self.server, 'A')[0].address
self.ipv6 = query(self.server, 'AAAA')[0].address
self.resolver = Resolver()
self.size_ipv4 = None
self.size_ipv6 = None
self.size_ipv4_dnssec = None
self.size_ipv6_dnssec = None
self.logger.debug('initiate: {} ({}/{})'.format(self.server, self.ipv4, self.ipv6))
self.update_sizes()
def _update_size(self, server, dnssec):
'''get the response size'''
self.resolver.nameservers = [ server ]
if dnssec:
self.resolver.use_edns(0,DO,4096)
else:
self.resolver.use_edns(0,0,4096)
answer = self.resolver.query('.', 'NS')
size = len(answer.response.to_wire())
self.logger.debug('Size:{}:DNSSEC({}):{}'.format(server, dnssec, size))
return size
def update_sizes(self):
self.size_ipv4 = self._update_size(self.ipv4, False)
self.size_ipv6 = self._update_size(self.ipv6, False)
self.size_ipv4_dnssec = self._update_size(self.ipv4, True)
self.size_ipv6_dnssec = self._update_size(self.ipv6, True)
class CustomResolver(object):
def __init__(self):
self.resolver = Resolver()
self.resolver.use_edns(0, 0, 4096)
self.resolver.set_flags(flags.AD + flags.RD)
self.degraded = Resolver()
self.degraded.use_edns(0, 0, 4096)
self.degraded.set_flags(flags.CD + flags.RD)
def query(self, fqdn, rdatatype=rdt.A, degraded=False):
log.debug('Query %s %s', fqdn, rdatatype)
try:
return self.resolver.query(fqdn, rdatatype)
except NoNameservers:
if degraded:
return self.degraded.query(fqdn, rdatatype)
raise
except NXDOMAIN:
if degraded:
return self.degraded.query(fqdn, rdatatype)
return None
def srv(self, name, domainname, proto='tcp'):
fqdn = '_{}._{}.{}'.format(name, proto, domainname)
return self.query(fqdn, rdt.SRV)
def tlsa(self, hostname, port, proto='tcp'):
fqdn = '_{}._{}.{}'.format(port, proto, hostname)
return self.query(fqdn, rdt.TLSA)
def mx(self, domainname):
return self.query(domainname, rdt.MX)
def get_nsset(zone, server, bufsize):
nsset = { 'RRSIG' : False }
resolver = Resolver()
resolver.nameservers = [server]
resolver.use_edns(edns=True, ednsflags=dns.flags.DO, payload=bufsize)
response = resolver.query(zone, 'NS', dns.rdataclass.IN, True).response
for answer in response.answer:
for ans in answer.items:
if ans.rdtype == dns.rdatatype.NS:
nsset[ans.to_text()] = { 'A' : None, 'AAAA' : None, 'RRSIG' : None }
elif ans.rdtype == dns.rdatatype.RRSIG:
nsset['RRSIG'] = True
return nsset
def do_check_host_key(hostname, keytype, keydata) -> str:
resolver = Resolver()
resolver.use_edns(0, flags.DO, 1280)
log("do_check_host_key(%s, %s, ..) resolver=%s", hostname, keytype,
resolver)
key_alg = _key_algorithms.get(keytype)
if key_alg is None:
return "Unsupported key type for SSHFP: %s" % keytype
log("key algorithm for %s: %s", keytype, key_alg)
try:
resp = resolver.query(hostname, "SSHFP")
except (NoAnswer, NoNameservers):
return "could not obtain SSHFP records for host '%s'" % hostname
for item in resp:
try:
alg, fg_type, fg = item.to_text().split()
log("found SSHFP record: %s", (alg, fg_type, fg))
except ValueError:
return "invalid SSHFP record format: %s" % item.to_text()
if alg != key_alg:
log("SSHFP record does not match algorithm")
continue
hash_func = _hash_funcs.get(fg_type)
if not hash_func:
log("unsupported hash function type: %s", fg_type)
continue
fg_expect = hash_func(keydata).hexdigest()
if fg_expect == fg:
log("found valid SSHFP record for host %s", hostname)
if not resp.response.flags & flags.AD:
return "answer matches but does not have a valid DNSSEC signature"
return True
return "no matching SSHFP records found"
def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
EDNS_PAYLOAD = 1232
resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD)
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
if self.option_geoip:
geo = geoip()
while not self.kill_received:
try:
domain = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_a = False
dns_aaaa = False
dns_mx = False
try:
domain['dns-ns'] = self.__answer_to_list(
resolve(domain['domain-name'],
rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
except NoNameservers:
domain['dns-ns'] = ['!ServFail']
except DNSException as e:
self.__debug(e)
if nxdomain is False:
try:
domain['dns-a'] = self.__answer_to_list(
resolve(domain['domain-name'],
rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
domain['dns-a'] = ['!ServFail']
except DNSException as e:
self.__debug(e)
try:
domain['dns-aaaa'] = self.__answer_to_list(
resolve(domain['domain-name'],
rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
domain['dns-aaaa'] = ['!ServFail']
except DNSException as e:
self.__debug(e)
if nxdomain is False and dns_ns is True:
try:
domain['dns-mx'] = self.__answer_to_list(
resolve(domain['domain-name'],
rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
domain['dns-mx'] = ['!ServFail']
except DNSException as e:
self.__debug(e)
else:
try:
ip = socket.getaddrinfo(domain['domain-name'], 80)
except socket.gaierror as e:
if e.errno == -3:
domain['dns-a'] = ['!ServFail']
except Exception as e:
self.__debug(e)
else:
domain['dns-a'] = list()
domain['dns-aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
domain['dns-a'].append(j[4][0])
if ':' in j[4][0]:
domain['dns-aaaa'].append(j[4][0])
domain['dns-a'] = sorted(domain['dns-a'])
domain['dns-aaaa'] = sorted(domain['dns-aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain['domain-name'] != self.domain_init:
if self.__mxcheck(domain['dns-mx'][0],
self.domain_init,
domain['domain-name']):
domain['mx-spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = geo.country_by_addr(domain['dns-a'][0])
except Exception as e:
self.__debug(e)
pass
else:
if country:
domain['geoip-country'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self.__banner_http(domain['dns-a'][0],
domain['domain-name'])
if banner:
domain['banner-http'] = banner
if dns_mx is True:
banner = self.__banner_smtp(domain['dns-mx'][0])
if banner:
domain['banner-smtp'] = banner
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
req = requests.get(
self.uri_scheme + '://' + domain['domain-name'] +
self.uri_path + self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP,
headers={'User-Agent': self.useragent},
verify=False)
except Exception as e:
self.__debug(e)
pass
else:
if req.status_code // 100 == 2 and req.url.split(
'?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(''.join(
req.text.split()).lower())
domain['ssdeep-score'] = ssdeep.compare(
self.ssdeep_init, ssdeep_curr)
self.jobs.task_done()
def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
EDNS_PAYLOAD = 1232
resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD)
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
if self.option_geoip:
geo = geoip()
if self.option_phash:
browser = HeadlessBrowser(useragent=self.useragent)
_answer_to_list = lambda ans: sorted(
[str(x).split(' ')[-1].rstrip('.') for x in ans])
while not self.kill_received:
try:
task = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain = task.get('domain')
dns_a = False
dns_aaaa = False
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_mx = False
try:
task['dns_ns'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
except NoNameservers:
task['dns_ns'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False:
try:
task['dns_a'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
task['dns_a'] = ['!ServFail']
except DNSException as e:
self._debug(e)
try:
task['dns_aaaa'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
task['dns_aaaa'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False and dns_ns is True:
try:
task['dns_mx'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
task['dns_mx'] = ['!ServFail']
except DNSException as e:
self._debug(e)
else:
try:
ip = socket.getaddrinfo(domain, 80)
except socket.gaierror as e:
if e.errno == -3:
task['dns_a'] = ['!ServFail']
except Exception as e:
self._debug(e)
else:
task['dns_a'] = list()
task['dns_aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
task['dns_a'].append(j[4][0])
if ':' in j[4][0]:
task['dns_aaaa'].append(j[4][0])
task['dns_a'] = sorted(task['dns_a'])
task['dns_aaaa'] = sorted(task['dns_aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain != self.url.domain:
if self._mxcheck(task['dns_mx'][0], self.url.domain,
domain):
task['mx_spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = geo.country_by_addr(task['dns_a'][0])
except Exception as e:
self._debug(e)
pass
else:
if country:
task['geoip'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self._banner_http(task['dns_a'][0], domain)
if banner:
task['banner_http'] = banner
if dns_mx is True:
banner = self._banner_smtp(task['dns_mx'][0])
if banner:
task['banner_smtp'] = banner
if self.option_phash or self.screenshot_dir:
if dns_a or dns_aaaa:
try:
browser.get(self.url.full_uri(domain))
screenshot = browser.screenshot()
except Exception as e:
self._debug(e)
else:
if self.option_phash:
phash = pHash(BytesIO(screenshot))
task['phash'] = self.phash_init - phash
if self.screenshot_dir:
filename = os.path.join(
self.screenshot_dir,
'{:08x}_{}.png'.format(self.id, domain))
try:
with open(filename, 'wb') as f:
f.write(screenshot)
except Exception as e:
self._debug(e)
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
r = UrlOpener(
self.url.full_uri(domain),
timeout=REQUEST_TIMEOUT_HTTP,
headers={
'User-Agent': self.useragent,
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9',
'Accept-Encoding': 'gzip,identity',
'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8'
},
verify=False)
except Exception as e:
self._debug(e)
else:
if r.url.split('?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(r.normalized_content)
task['ssdeep'] = ssdeep.compare(
self.ssdeep_init, ssdeep_curr)
self.jobs.task_done()
