def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
EDNS_PAYLOAD = 1232
resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD)
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
if self.option_geoip:
geo = geoip()
while not self.kill_received:
try:
task = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain = task.get('domain')
dns_a = False
dns_aaaa = False
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_mx = False
try:
task['dns_ns'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
except NoNameservers:
task['dns_ns'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False:
try:
task['dns_a'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
task['dns_a'] = ['!ServFail']
except DNSException as e:
self._debug(e)
try:
task['dns_aaaa'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
task['dns_aaaa'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False and dns_ns is True:
try:
task['dns_mx'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
task['dns_mx'] = ['!ServFail']
except DNSException as e:
self._debug(e)
else:
try:
ip = socket.getaddrinfo(domain, 80)
except socket.gaierror as e:
if e.errno == -3:
task['dns_a'] = ['!ServFail']
except Exception as e:
self._debug(e)
else:
task['dns_a'] = list()
task['dns_aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
task['dns_a'].append(j[4][0])
if ':' in j[4][0]:
task['dns_aaaa'].append(j[4][0])
task['dns_a'] = sorted(task['dns_a'])
task['dns_aaaa'] = sorted(task['dns_aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain != self.domain_init:
if self._mxcheck(task['dns_mx'][0], self.domain_init,
domain):
task['mx_spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = geo.country_by_addr(task['dns_a'][0])
except Exception as e:
self._debug(e)
pass
else:
if country:
task['geoip'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self._banner_http(task['dns_a'][0], domain)
if banner:
task['banner_http'] = banner
if dns_mx is True:
banner = self._banner_smtp(task['dns_mx'][0])
if banner:
task['banner_smtp'] = banner
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
req = requests.get(
self.uri_scheme + '://' + domain + self.uri_path +
self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP,
headers={'User-Agent': self.useragent},
verify=False)
except Exception as e:
self._debug(e)
pass
else:
if req.status_code // 100 == 2 and req.url.split(
'?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(''.join(
req.text.split()).lower())
task['ssdeep'] = ssdeep.compare(
self.ssdeep_init, ssdeep_curr)
self.jobs.task_done()
def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
while not self.kill_received:
try:
domain = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain['domain-name'] = domain['domain-name'].encode('idna').decode()
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_a = False
dns_aaaa = False
dns_mx = False
try:
domain['dns-ns'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
pass
except NoNameservers:
domain['dns-ns'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
if nxdomain is False:
try:
domain['dns-a'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
domain['dns-a'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
try:
domain['dns-aaaa'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
domain['dns-aaaa'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
if nxdomain is False and dns_ns is True:
try:
domain['dns-mx'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
domain['dns-mx'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
else:
try:
ip = socket.getaddrinfo(domain['domain-name'], 80)
except socket.gaierror as e:
if e.errno == -3:
domain['dns-a'] = ['!ServFail']
pass
except Exception as e:
self.__debug(e)
pass
else:
domain['dns-a'] = list()
domain['dns-aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
domain['dns-a'].append(j[4][0])
if ':' in j[4][0]:
domain['dns-aaaa'].append(j[4][0])
domain['dns-a'] = sorted(domain['dns-a'])
domain['dns-aaaa'] = sorted(domain['dns-aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain['domain-name'] != self.domain_init:
if self.__mxcheck(domain['dns-mx'][0], self.domain_init, domain['domain-name']):
domain['mx-spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE).country_name_by_addr(domain['dns-a'][0])
except Exception as e:
self.__debug(e)
pass
else:
if country:
domain['geoip-country'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self.__banner_http(domain['dns-a'][0], domain['domain-name'])
if banner:
domain['banner-http'] = banner
if dns_mx is True:
banner = self.__banner_smtp(domain['dns-mx'][0])
if banner:
domain['banner-smtp'] = banner
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
req = requests.get(self.uri_scheme + '://' + domain['domain-name'] + self.uri_path + self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP, headers={'User-Agent': self.useragent}, verify=False)
except Exception as e:
self.__debug(e)
pass
else:
if req.status_code // 100 == 2 and req.url.split('?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(''.join(req.text.split()).lower())
domain['ssdeep-score'] = ssdeep.compare(self.ssdeep_init, ssdeep_curr)
domain['domain-name'] = domain['domain-name'].encode().decode('idna')
self.jobs.task_done()
def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
EDNS_PAYLOAD = 1232
resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD)
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
if self.option_geoip:
geo = geoip()
if self.option_phash:
browser = HeadlessBrowser(useragent=self.useragent)
_answer_to_list = lambda ans: sorted(
[str(x).split(' ')[-1].rstrip('.') for x in ans])
while not self.kill_received:
try:
task = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain = task.get('domain')
dns_a = False
dns_aaaa = False
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_mx = False
try:
task['dns_ns'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
except NoNameservers:
task['dns_ns'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False:
try:
task['dns_a'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
task['dns_a'] = ['!ServFail']
except DNSException as e:
self._debug(e)
try:
task['dns_aaaa'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
task['dns_aaaa'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False and dns_ns is True:
try:
task['dns_mx'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
task['dns_mx'] = ['!ServFail']
except DNSException as e:
self._debug(e)
else:
try:
ip = socket.getaddrinfo(domain, 80)
except socket.gaierror as e:
if e.errno == -3:
task['dns_a'] = ['!ServFail']
except Exception as e:
self._debug(e)
else:
task['dns_a'] = list()
task['dns_aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
task['dns_a'].append(j[4][0])
if ':' in j[4][0]:
task['dns_aaaa'].append(j[4][0])
task['dns_a'] = sorted(task['dns_a'])
task['dns_aaaa'] = sorted(task['dns_aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain != self.url.domain:
if self._mxcheck(task['dns_mx'][0], self.url.domain,
domain):
task['mx_spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = geo.country_by_addr(task['dns_a'][0])
except Exception as e:
self._debug(e)
pass
else:
if country:
task['geoip'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self._banner_http(task['dns_a'][0], domain)
if banner:
task['banner_http'] = banner
if dns_mx is True:
banner = self._banner_smtp(task['dns_mx'][0])
if banner:
task['banner_smtp'] = banner
if self.option_phash or self.screenshot_dir:
if dns_a or dns_aaaa:
try:
browser.get(self.url.full_uri(domain))
screenshot = browser.screenshot()
except Exception as e:
self._debug(e)
else:
if self.option_phash:
phash = pHash(BytesIO(screenshot))
task['phash'] = self.phash_init - phash
if self.screenshot_dir:
filename = os.path.join(
self.screenshot_dir,
'{:08x}_{}.png'.format(self.id, domain))
try:
with open(filename, 'wb') as f:
f.write(screenshot)
except Exception as e:
self._debug(e)
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
r = UrlOpener(
self.url.full_uri(domain),
timeout=REQUEST_TIMEOUT_HTTP,
headers={
'User-Agent': self.useragent,
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9',
'Accept-Encoding': 'gzip,identity',
'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8'
},
verify=False)
except Exception as e:
self._debug(e)
else:
if r.url.split('?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(r.normalized_content)
task['ssdeep'] = ssdeep.compare(
self.ssdeep_init, ssdeep_curr)
self.jobs.task_done()
