def run(self): if self.option_extdns: if self.nameservers: resolv = Resolver(configure=False) resolv.nameservers = self.nameservers else: resolv = Resolver() resolv.search = [] resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS resolv.timeout = REQUEST_TIMEOUT_DNS EDNS_PAYLOAD = 1232 resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD) if hasattr(resolv, 'resolve'): resolve = resolv.resolve else: resolve = resolv.query if self.option_geoip: geo = geoip() while not self.kill_received: try: task = self.jobs.get(block=False) except queue.Empty: self.kill_received = True return domain = task.get('domain') dns_a = False dns_aaaa = False if self.option_extdns: nxdomain = False dns_ns = False dns_mx = False try: task['dns_ns'] = self._answer_to_list( resolve(domain, rdtype=dns.rdatatype.NS)) dns_ns = True except NXDOMAIN: nxdomain = True except NoNameservers: task['dns_ns'] = ['!ServFail'] except DNSException as e: self._debug(e) if nxdomain is False: try: task['dns_a'] = self._answer_to_list( resolve(domain, rdtype=dns.rdatatype.A)) dns_a = True except NoNameservers: task['dns_a'] = ['!ServFail'] except DNSException as e: self._debug(e) try: task['dns_aaaa'] = self._answer_to_list( resolve(domain, rdtype=dns.rdatatype.AAAA)) dns_aaaa = True except NoNameservers: task['dns_aaaa'] = ['!ServFail'] except DNSException as e: self._debug(e) if nxdomain is False and dns_ns is True: try: task['dns_mx'] = self._answer_to_list( resolve(domain, rdtype=dns.rdatatype.MX)) dns_mx = True except NoNameservers: task['dns_mx'] = ['!ServFail'] except DNSException as e: self._debug(e) else: try: ip = socket.getaddrinfo(domain, 80) except socket.gaierror as e: if e.errno == -3: task['dns_a'] = ['!ServFail'] except Exception as e: self._debug(e) else: task['dns_a'] = list() task['dns_aaaa'] = list() for j in ip: if '.' in j[4][0]: task['dns_a'].append(j[4][0]) if ':' in j[4][0]: task['dns_aaaa'].append(j[4][0]) task['dns_a'] = sorted(task['dns_a']) task['dns_aaaa'] = sorted(task['dns_aaaa']) dns_a = True dns_aaaa = True if self.option_mxcheck: if dns_mx is True: if domain != self.domain_init: if self._mxcheck(task['dns_mx'][0], self.domain_init, domain): task['mx_spy'] = True if self.option_geoip: if dns_a is True: try: country = geo.country_by_addr(task['dns_a'][0]) except Exception as e: self._debug(e) pass else: if country: task['geoip'] = country.split(',')[0] if self.option_banners: if dns_a is True: banner = self._banner_http(task['dns_a'][0], domain) if banner: task['banner_http'] = banner if dns_mx is True: banner = self._banner_smtp(task['dns_mx'][0]) if banner: task['banner_smtp'] = banner if self.option_ssdeep: if dns_a is True or dns_aaaa is True: try: req = requests.get( self.uri_scheme + '://' + domain + self.uri_path + self.uri_query, timeout=REQUEST_TIMEOUT_HTTP, headers={'User-Agent': self.useragent}, verify=False) except Exception as e: self._debug(e) pass else: if req.status_code // 100 == 2 and req.url.split( '?')[0] != self.ssdeep_effective_url: ssdeep_curr = ssdeep.hash(''.join( req.text.split()).lower()) task['ssdeep'] = ssdeep.compare( self.ssdeep_init, ssdeep_curr) self.jobs.task_done()
def run(self): if self.option_extdns: if self.nameservers: resolv = Resolver(configure=False) resolv.nameservers = self.nameservers else: resolv = Resolver() resolv.search = [] resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS resolv.timeout = REQUEST_TIMEOUT_DNS if hasattr(resolv, 'resolve'): resolve = resolv.resolve else: resolve = resolv.query while not self.kill_received: try: domain = self.jobs.get(block=False) except queue.Empty: self.kill_received = True return domain['domain-name'] = domain['domain-name'].encode('idna').decode() if self.option_extdns: nxdomain = False dns_ns = False dns_a = False dns_aaaa = False dns_mx = False try: domain['dns-ns'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.NS)) dns_ns = True except NXDOMAIN: nxdomain = True pass except NoNameservers: domain['dns-ns'] = ['!ServFail'] pass except DNSException as e: self.__debug(e) pass if nxdomain is False: try: domain['dns-a'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.A)) dns_a = True except NoNameservers: domain['dns-a'] = ['!ServFail'] pass except DNSException as e: self.__debug(e) pass try: domain['dns-aaaa'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.AAAA)) dns_aaaa = True except NoNameservers: domain['dns-aaaa'] = ['!ServFail'] pass except DNSException as e: self.__debug(e) pass if nxdomain is False and dns_ns is True: try: domain['dns-mx'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.MX)) dns_mx = True except NoNameservers: domain['dns-mx'] = ['!ServFail'] pass except DNSException as e: self.__debug(e) pass else: try: ip = socket.getaddrinfo(domain['domain-name'], 80) except socket.gaierror as e: if e.errno == -3: domain['dns-a'] = ['!ServFail'] pass except Exception as e: self.__debug(e) pass else: domain['dns-a'] = list() domain['dns-aaaa'] = list() for j in ip: if '.' in j[4][0]: domain['dns-a'].append(j[4][0]) if ':' in j[4][0]: domain['dns-aaaa'].append(j[4][0]) domain['dns-a'] = sorted(domain['dns-a']) domain['dns-aaaa'] = sorted(domain['dns-aaaa']) dns_a = True dns_aaaa = True if self.option_mxcheck: if dns_mx is True: if domain['domain-name'] != self.domain_init: if self.__mxcheck(domain['dns-mx'][0], self.domain_init, domain['domain-name']): domain['mx-spy'] = True if self.option_geoip: if dns_a is True: try: country = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE).country_name_by_addr(domain['dns-a'][0]) except Exception as e: self.__debug(e) pass else: if country: domain['geoip-country'] = country.split(',')[0] if self.option_banners: if dns_a is True: banner = self.__banner_http(domain['dns-a'][0], domain['domain-name']) if banner: domain['banner-http'] = banner if dns_mx is True: banner = self.__banner_smtp(domain['dns-mx'][0]) if banner: domain['banner-smtp'] = banner if self.option_ssdeep: if dns_a is True or dns_aaaa is True: try: req = requests.get(self.uri_scheme + '://' + domain['domain-name'] + self.uri_path + self.uri_query, timeout=REQUEST_TIMEOUT_HTTP, headers={'User-Agent': self.useragent}, verify=False) except Exception as e: self.__debug(e) pass else: if req.status_code // 100 == 2 and req.url.split('?')[0] != self.ssdeep_effective_url: ssdeep_curr = ssdeep.hash(''.join(req.text.split()).lower()) domain['ssdeep-score'] = ssdeep.compare(self.ssdeep_init, ssdeep_curr) domain['domain-name'] = domain['domain-name'].encode().decode('idna') self.jobs.task_done()
def run(self): if self.option_extdns: if self.nameservers: resolv = Resolver(configure=False) resolv.nameservers = self.nameservers else: resolv = Resolver() resolv.search = [] resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS resolv.timeout = REQUEST_TIMEOUT_DNS EDNS_PAYLOAD = 1232 resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD) if hasattr(resolv, 'resolve'): resolve = resolv.resolve else: resolve = resolv.query if self.option_geoip: geo = geoip() if self.option_phash: browser = HeadlessBrowser(useragent=self.useragent) _answer_to_list = lambda ans: sorted( [str(x).split(' ')[-1].rstrip('.') for x in ans]) while not self.kill_received: try: task = self.jobs.get(block=False) except queue.Empty: self.kill_received = True return domain = task.get('domain') dns_a = False dns_aaaa = False if self.option_extdns: nxdomain = False dns_ns = False dns_mx = False try: task['dns_ns'] = _answer_to_list( resolve(domain, rdtype=dns.rdatatype.NS)) dns_ns = True except NXDOMAIN: nxdomain = True except NoNameservers: task['dns_ns'] = ['!ServFail'] except DNSException as e: self._debug(e) if nxdomain is False: try: task['dns_a'] = _answer_to_list( resolve(domain, rdtype=dns.rdatatype.A)) dns_a = True except NoNameservers: task['dns_a'] = ['!ServFail'] except DNSException as e: self._debug(e) try: task['dns_aaaa'] = _answer_to_list( resolve(domain, rdtype=dns.rdatatype.AAAA)) dns_aaaa = True except NoNameservers: task['dns_aaaa'] = ['!ServFail'] except DNSException as e: self._debug(e) if nxdomain is False and dns_ns is True: try: task['dns_mx'] = _answer_to_list( resolve(domain, rdtype=dns.rdatatype.MX)) dns_mx = True except NoNameservers: task['dns_mx'] = ['!ServFail'] except DNSException as e: self._debug(e) else: try: ip = socket.getaddrinfo(domain, 80) except socket.gaierror as e: if e.errno == -3: task['dns_a'] = ['!ServFail'] except Exception as e: self._debug(e) else: task['dns_a'] = list() task['dns_aaaa'] = list() for j in ip: if '.' in j[4][0]: task['dns_a'].append(j[4][0]) if ':' in j[4][0]: task['dns_aaaa'].append(j[4][0]) task['dns_a'] = sorted(task['dns_a']) task['dns_aaaa'] = sorted(task['dns_aaaa']) dns_a = True dns_aaaa = True if self.option_mxcheck: if dns_mx is True: if domain != self.url.domain: if self._mxcheck(task['dns_mx'][0], self.url.domain, domain): task['mx_spy'] = True if self.option_geoip: if dns_a is True: try: country = geo.country_by_addr(task['dns_a'][0]) except Exception as e: self._debug(e) pass else: if country: task['geoip'] = country.split(',')[0] if self.option_banners: if dns_a is True: banner = self._banner_http(task['dns_a'][0], domain) if banner: task['banner_http'] = banner if dns_mx is True: banner = self._banner_smtp(task['dns_mx'][0]) if banner: task['banner_smtp'] = banner if self.option_phash or self.screenshot_dir: if dns_a or dns_aaaa: try: browser.get(self.url.full_uri(domain)) screenshot = browser.screenshot() except Exception as e: self._debug(e) else: if self.option_phash: phash = pHash(BytesIO(screenshot)) task['phash'] = self.phash_init - phash if self.screenshot_dir: filename = os.path.join( self.screenshot_dir, '{:08x}_{}.png'.format(self.id, domain)) try: with open(filename, 'wb') as f: f.write(screenshot) except Exception as e: self._debug(e) if self.option_ssdeep: if dns_a is True or dns_aaaa is True: try: r = UrlOpener( self.url.full_uri(domain), timeout=REQUEST_TIMEOUT_HTTP, headers={ 'User-Agent': self.useragent, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9', 'Accept-Encoding': 'gzip,identity', 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8' }, verify=False) except Exception as e: self._debug(e) else: if r.url.split('?')[0] != self.ssdeep_effective_url: ssdeep_curr = ssdeep.hash(r.normalized_content) task['ssdeep'] = ssdeep.compare( self.ssdeep_init, ssdeep_curr) self.jobs.task_done()