def reverse_dns_lookup(input, timeout=3, server=''):
"""Perform a simple reverse DNS lookup, return results in a dictionary"""
assert _REVERSE_DNS_REGEX.search(
input), "Invalid address format: '{0}'".format(input)
resolver = Resolver()
resolver.timeout = float(timeout)
resolver.lifetime = float(timeout)
if server:
resolver.nameservers = [server]
try:
tmp = input.strip().split('.')
tmp.reverse()
inaddr = '.'.join(tmp) + ".in-addr.arpa"
records = resolver.query(inaddr, 'PTR')
return {
'name': records[0].to_text(),
'lookup': inaddr,
'error': '',
'addr': input,
}
except DNSException as e:
return {
'addrs': [],
'lookup': inaddr,
'error': repr(e),
'name': input,
}
def dns6_lookup(input, timeout=3, server=''):
"""Perform a simple DNS lookup, return results in a dictionary"""
resolver = Resolver()
resolver.timeout = float(timeout)
resolver.lifetime = float(timeout)
if server:
resolver.nameservers = [server]
try:
records = resolver.query(input, 'AAAA')
return {'addrs': [ii.address for ii in records],
'error': '',
'name': input,
}
except DNSException as e:
return {'addrs': [],
'error': repr(e),
'name': input,
}
def dns_lookup(input, timeout=3, server=''):
"""Perform a simple DNS lookup, return results in a dictionary"""
resolver = Resolver()
resolver.timeout = float(timeout)
resolver.lifetime = float(timeout)
if server:
resolver.nameservers = [server]
try:
records = resolver.query(input, 'A')
return {
'addrs': [ii.address for ii in records],
'error': '',
'name': input,
}
except DNSException as e:
return {
'addrs': [],
'error': repr(e),
'name': input,
}
def process(self, headers, content):
request = Services.get("request_factory")
hostname = urlparse(request.url).hostname
_ = False
try:
resolver = Resolver(configure=False)
resolver.nameservers = [settings.dns_resolver]
resolver.timeout = 2
resolver.lifetime = 2
dns_query = resolver.query(hostname, "CNAME")
if len(dns_query) > 0:
for answer in dns_query:
_ |= re.search(r"azureedge\.net", str(answer), re.I) is not None
if _:
return "Azure CDN"
except NoAnswer:
pass
except NXDOMAIN:
pass
except Timeout:
pass
def reverse_dns_lookup(input, timeout=3, server=''):
"""Perform a simple reverse DNS lookup, return results in a dictionary"""
assert _REVERSE_DNS_REGEX.search(input), "Invalid address format: '{0}'".format(input)
resolver = Resolver()
resolver.timeout = float(timeout)
resolver.lifetime = float(timeout)
if server:
resolver.nameservers = [server]
try:
tmp = input.strip().split('.')
tmp.reverse()
inaddr = '.'.join(tmp) + ".in-addr.arpa"
records = resolver.query(inaddr, 'PTR')
return {'name': records[0].to_text(),
'lookup': inaddr,
'error': '',
'addr': input,
}
except DNSException as e:
return {'addrs': [],
'lookup': inaddr,
'error': repr(e),
'name': input,
}
def process(self, headers, content):
request = Services.get('request_factory')
hostname = urlparse(request.url).hostname
_ = False
try:
resolver = Resolver(configure=False)
resolver.nameservers = [settings.dns_resolver]
resolver.timeout = 2
resolver.lifetime = 2
dns_query = resolver.query(hostname, 'CNAME')
if len(dns_query) > 0:
for answer in dns_query:
_ |= re.search(r'cloudfront\.net', str(answer),
re.I) is not None
if _:
return "CloudFront CDN (Amazon)"
except NoAnswer:
pass
except NXDOMAIN:
pass
except Timeout:
pass
def check_dns(check, data):
check.addOutput("ScoreEngine: {} Check\n".format(check.getServiceName()))
check.addOutput("EXPECTED: Sucessful and correct query against the DNS server")
check.addOutput("OUTPUT:\n")
# Setup the resolver
resolv = Resolver()
resolv.nameservers = [data["HOST"]]
resolv.timeout = dns_config["timeout"]
resolv.lifetime = dns_config["lifetime"]
check.addOutput("Starting check...")
try:
# Query resolver
check.addOutput("Querying {HOST} for '{LOOKUP}'...".format(**data))
lookup = resolv.query(data["LOOKUP"], data["TYPE"])
found = False
for ans in lookup:
if str(ans) == data["EXPECTED"]:
found = True
else:
check.addOutput("NOTICE: DNS Server returned {}".format(ans))
if not found:
check.addOutput("ERROR: DNS Server did not respond with the correct IP")
return
# We're good!
check.setPassed()
check.addOutput("Check successful!")
except Exception as e:
check.addOutput("ERROR: {}: {}".format(type(e).__name__, e))
return
def check_for_wildcards(args, server, name, rectype, tries=4):
"""
Verify that the DNS server doesn't return wildcard results for domains
which don't exist, it should correctly return NXDOMAIN.
"""
resolver = Resolver()
resolver.timeout = args.timeout
resolver.lifetime = args.timeout
resolver.nameservers = [server]
nx_names = [base64.b32encode(
os.urandom(
random.randint(8, 10))
).strip('=').lower() + name
for _ in range(0, tries)]
correct_result_count = 0
for check_nx_name in nx_names:
try:
result = resolver.query(check_nx_name, rectype)
return False # Any valid response = immediate fail!
except (NXDOMAIN, NoNameservers):
correct_result_count += 1
except DNSException:
continue
return correct_result_count > (tries / 2.0)
def time_resolve(args, server, name, rectype, tries=3):
"""
Time how long it takes to resolve a name using the server
"""
resolver = Resolver()
resolver.timeout = args.timeout
resolver.lifetime = args.timeout
resolver.nameservers = [server]
results = []
while tries > 0:
start = time.time()
try:
result = resolver.query(name, rectype)
except DNSException as ex:
end = time.time()
LOG.debug("%s failed in %.2fs", server, end - start)
result = ex
else:
end = time.time()
LOG.debug("%s resolved %s %s in %.2fs", server, name, rectype, end - start)
results.append((end - start, result))
tries -= 1
return server, check_results(results), results
def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
while not self.kill_received:
try:
domain = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain['domain-name'] = domain['domain-name'].encode('idna').decode()
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_a = False
dns_aaaa = False
dns_mx = False
try:
domain['dns-ns'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
pass
except NoNameservers:
domain['dns-ns'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
if nxdomain is False:
try:
domain['dns-a'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
domain['dns-a'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
try:
domain['dns-aaaa'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
domain['dns-aaaa'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
if nxdomain is False and dns_ns is True:
try:
domain['dns-mx'] = self.__answer_to_list(resolve(domain['domain-name'], rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
domain['dns-mx'] = ['!ServFail']
pass
except DNSException as e:
self.__debug(e)
pass
else:
try:
ip = socket.getaddrinfo(domain['domain-name'], 80)
except socket.gaierror as e:
if e.errno == -3:
domain['dns-a'] = ['!ServFail']
pass
except Exception as e:
self.__debug(e)
pass
else:
domain['dns-a'] = list()
domain['dns-aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
domain['dns-a'].append(j[4][0])
if ':' in j[4][0]:
domain['dns-aaaa'].append(j[4][0])
domain['dns-a'] = sorted(domain['dns-a'])
domain['dns-aaaa'] = sorted(domain['dns-aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain['domain-name'] != self.domain_init:
if self.__mxcheck(domain['dns-mx'][0], self.domain_init, domain['domain-name']):
domain['mx-spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE).country_name_by_addr(domain['dns-a'][0])
except Exception as e:
self.__debug(e)
pass
else:
if country:
domain['geoip-country'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self.__banner_http(domain['dns-a'][0], domain['domain-name'])
if banner:
domain['banner-http'] = banner
if dns_mx is True:
banner = self.__banner_smtp(domain['dns-mx'][0])
if banner:
domain['banner-smtp'] = banner
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
req = requests.get(self.uri_scheme + '://' + domain['domain-name'] + self.uri_path + self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP, headers={'User-Agent': self.useragent}, verify=False)
except Exception as e:
self.__debug(e)
pass
else:
if req.status_code // 100 == 2 and req.url.split('?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(''.join(req.text.split()).lower())
domain['ssdeep-score'] = ssdeep.compare(self.ssdeep_init, ssdeep_curr)
domain['domain-name'] = domain['domain-name'].encode().decode('idna')
self.jobs.task_done()
def getDNSrecords(domain, out_to_json):
print(colored("[*]-Retrieving DNS Records...", "yellow"))
RES = {}
MX = []
NS = []
A = []
AAAA = []
SOA = []
TXT = []
resolver = Resolver()
resolver.timeout = 1
resolver.lifetime = 1
rrtypes = ["A", "MX", "NS", "AAAA", "SOA", "TXT"]
for r in rrtypes:
try:
Aanswer = resolver.query(domain, r)
for answer in Aanswer:
if r == "A":
A.append(answer.address)
RES.update({r: A})
if r == "MX":
MX.append(answer.exchange.to_text()[:-1])
RES.update({r: MX})
if r == "NS":
NS.append(answer.target.to_text()[:-1])
RES.update({r: NS})
if r == "AAAA":
AAAA.append(answer.address)
RES.update({r: AAAA})
if r == "SOA":
SOA.append(answer.mname.to_text()[:-1])
RES.update({r: SOA})
if r == "TXT":
TXT.append(str(answer))
RES.update({r: TXT})
except NXDOMAIN:
pass
except NoAnswer:
pass
except EmptyLabel:
pass
except NoNameservers:
pass
except Timeout:
pass
except DNSException:
pass
for key, value in RES.items():
for record in value:
print(" \__ {0}: {1}".format(colored(key, "cyan"),
colored(record, "yellow")))
if out_to_json:
try:
with open(join("results", domain, "dns.json"), "w") as dns_file:
dns_file.write(dumps(RES))
except OSError:
pass
except IOError:
pass
try:
with open(join("results", domain, "dns.csv"), "w") as dns_file:
for key, value in RES.items():
for record in value:
dns_file.write("{0}|{1}\n".format(key, record))
except OSError:
pass
except IOError:
pass
return NS
def dns_query(input="", query_type="", server="", timeout=2.0):
"""A unified IPv4 & IPv6 DNS lookup interface; this is essentially just a wrapper around dnspython's API. When you query a PTR record, you can use an IPv4 or IPv6 address (which will automatically be converted into an in-addr.arpa name. This wrapper only supports a subset of DNS records: 'A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', and 'TXT'
Kwargs:
- input (str): A string containing the DNS record to lookup
- query_type (str): A string containing the DNS record type (SOA not supported)
- server (str): A string containing the fqdn or IP address of the dns server
- timeout (float): DNS lookup timeout duration (default: 2.0 seconds)
Returns:
A set() of :class:`~ccp_util.DNSResponse` instances
>>> from ciscoconfparse.ccp_util import dns_query
>>> dns_query('www.pennington.net', 'A', '4.2.2.2')
set([<DNSResponse 'A' result_str='65.19.187.2'>])
>>> answer = dns_query('www.pennington.net', 'A', '4.2.2.2')
>>> str(answer.pop())
'65.19.187.2'
>>>
"""
valid_records = set(
['A', 'AAAA', 'AXFR', 'CNAME', 'MX', 'NS', 'PTR', 'TXT'])
query_type = query_type.upper()
assert query_type in valid_records
assert server != ""
assert float(timeout) > 0
assert input != ""
intput = input.strip()
retval = set([])
resolver = Resolver()
resolver.server = [socket.gethostbyname(server)]
resolver.timeout = float(timeout)
resolver.lifetime = float(timeout)
start = time.time()
if (query_type == "A") or (query_type == "AAAA"):
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=input,
result_str=str(result.address))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type == "AXFR":
"""This is a hack: return text of zone transfer, instead of axfr objs"""
_zone = zone.from_xfr(query.xfr(server, input, lifetime=timeout))
return [_zone[node].to_text(node) for node in _zone.nodes.keys()]
elif query_type == "CNAME":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=input,
result_str=str(result.target))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type == "MX":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
input=input,
result_str=str(result.target))
response.preference = int(result.preference)
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type == "NS":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=input,
result_str=str(result.target))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type == "PTR":
if is_valid_ipv4_addr(input) or is_valid_ipv6_addr(input):
inaddr = reversename.from_address(input)
elif 'in-addr.arpa' in input.lower():
inaddr = input
else:
raise ValueError('Cannot query PTR record for "{0}"'.format(input))
try:
answer = resolver.query(inaddr, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=inaddr,
result_str=str(result.target))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type == "TXT":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=inaddr,
result_str=str(result.strings))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
return retval
from domainconnect import *
from string import ascii_lowercase
from multiprocessing.pool import ThreadPool
from threading import Lock, Semaphore
from dns.resolver import Resolver
from dns.name import EmptyLabel
import humanize
import time
import validators
import sys
import pickle
_resolver = Resolver()
_resolver.timeout = 15
_resolver.lifetime = 120
api_url_map = dict()
api_url_map_lck = Lock()
ns_map = dict()
ns_map_lck = Lock()
start = time.time()
class dns_provider_stats:
api_url = None
config = None
cnt = 0
nslist = None # type: dict
def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
EDNS_PAYLOAD = 1232
resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD)
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
if self.option_geoip:
geo = geoip()
while not self.kill_received:
try:
task = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain = task.get('domain')
dns_a = False
dns_aaaa = False
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_mx = False
try:
task['dns_ns'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
except NoNameservers:
task['dns_ns'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False:
try:
task['dns_a'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
task['dns_a'] = ['!ServFail']
except DNSException as e:
self._debug(e)
try:
task['dns_aaaa'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
task['dns_aaaa'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False and dns_ns is True:
try:
task['dns_mx'] = self._answer_to_list(
resolve(domain, rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
task['dns_mx'] = ['!ServFail']
except DNSException as e:
self._debug(e)
else:
try:
ip = socket.getaddrinfo(domain, 80)
except socket.gaierror as e:
if e.errno == -3:
task['dns_a'] = ['!ServFail']
except Exception as e:
self._debug(e)
else:
task['dns_a'] = list()
task['dns_aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
task['dns_a'].append(j[4][0])
if ':' in j[4][0]:
task['dns_aaaa'].append(j[4][0])
task['dns_a'] = sorted(task['dns_a'])
task['dns_aaaa'] = sorted(task['dns_aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain != self.domain_init:
if self._mxcheck(task['dns_mx'][0], self.domain_init,
domain):
task['mx_spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = geo.country_by_addr(task['dns_a'][0])
except Exception as e:
self._debug(e)
pass
else:
if country:
task['geoip'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self._banner_http(task['dns_a'][0], domain)
if banner:
task['banner_http'] = banner
if dns_mx is True:
banner = self._banner_smtp(task['dns_mx'][0])
if banner:
task['banner_smtp'] = banner
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
req = requests.get(
self.uri_scheme + '://' + domain + self.uri_path +
self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP,
headers={'User-Agent': self.useragent},
verify=False)
except Exception as e:
self._debug(e)
pass
else:
if req.status_code // 100 == 2 and req.url.split(
'?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(''.join(
req.text.split()).lower())
task['ssdeep'] = ssdeep.compare(
self.ssdeep_init, ssdeep_curr)
self.jobs.task_done()
def run(self):
if self.option_extdns:
if self.nameservers:
resolv = Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = Resolver()
resolv.search = []
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
EDNS_PAYLOAD = 1232
resolv.use_edns(edns=True, ednsflags=0, payload=EDNS_PAYLOAD)
if hasattr(resolv, 'resolve'):
resolve = resolv.resolve
else:
resolve = resolv.query
if self.option_geoip:
geo = geoip()
if self.option_phash:
browser = HeadlessBrowser(useragent=self.useragent)
_answer_to_list = lambda ans: sorted(
[str(x).split(' ')[-1].rstrip('.') for x in ans])
while not self.kill_received:
try:
task = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain = task.get('domain')
dns_a = False
dns_aaaa = False
if self.option_extdns:
nxdomain = False
dns_ns = False
dns_mx = False
try:
task['dns_ns'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.NS))
dns_ns = True
except NXDOMAIN:
nxdomain = True
except NoNameservers:
task['dns_ns'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False:
try:
task['dns_a'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.A))
dns_a = True
except NoNameservers:
task['dns_a'] = ['!ServFail']
except DNSException as e:
self._debug(e)
try:
task['dns_aaaa'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except NoNameservers:
task['dns_aaaa'] = ['!ServFail']
except DNSException as e:
self._debug(e)
if nxdomain is False and dns_ns is True:
try:
task['dns_mx'] = _answer_to_list(
resolve(domain, rdtype=dns.rdatatype.MX))
dns_mx = True
except NoNameservers:
task['dns_mx'] = ['!ServFail']
except DNSException as e:
self._debug(e)
else:
try:
ip = socket.getaddrinfo(domain, 80)
except socket.gaierror as e:
if e.errno == -3:
task['dns_a'] = ['!ServFail']
except Exception as e:
self._debug(e)
else:
task['dns_a'] = list()
task['dns_aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
task['dns_a'].append(j[4][0])
if ':' in j[4][0]:
task['dns_aaaa'].append(j[4][0])
task['dns_a'] = sorted(task['dns_a'])
task['dns_aaaa'] = sorted(task['dns_aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain != self.url.domain:
if self._mxcheck(task['dns_mx'][0], self.url.domain,
domain):
task['mx_spy'] = True
if self.option_geoip:
if dns_a is True:
try:
country = geo.country_by_addr(task['dns_a'][0])
except Exception as e:
self._debug(e)
pass
else:
if country:
task['geoip'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self._banner_http(task['dns_a'][0], domain)
if banner:
task['banner_http'] = banner
if dns_mx is True:
banner = self._banner_smtp(task['dns_mx'][0])
if banner:
task['banner_smtp'] = banner
if self.option_phash or self.screenshot_dir:
if dns_a or dns_aaaa:
try:
browser.get(self.url.full_uri(domain))
screenshot = browser.screenshot()
except Exception as e:
self._debug(e)
else:
if self.option_phash:
phash = pHash(BytesIO(screenshot))
task['phash'] = self.phash_init - phash
if self.screenshot_dir:
filename = os.path.join(
self.screenshot_dir,
'{:08x}_{}.png'.format(self.id, domain))
try:
with open(filename, 'wb') as f:
f.write(screenshot)
except Exception as e:
self._debug(e)
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
r = UrlOpener(
self.url.full_uri(domain),
timeout=REQUEST_TIMEOUT_HTTP,
headers={
'User-Agent': self.useragent,
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9',
'Accept-Encoding': 'gzip,identity',
'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8'
},
verify=False)
except Exception as e:
self._debug(e)
else:
if r.url.split('?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(r.normalized_content)
task['ssdeep'] = ssdeep.compare(
self.ssdeep_init, ssdeep_curr)
self.jobs.task_done()
def dns_query(input="", query_type="", server="", timeout=2.0):
"""A unified IPv4 & IPv6 DNS lookup interface; this is essentially just a wrapper around dnspython's API. When you query a PTR record, you can use an IPv4 or IPv6 address (which will automatically be converted into an in-addr.arpa name. This wrapper only supports a subset of DNS records: 'A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', and 'TXT'
Kwargs:
- input (str): A string containing the DNS record to lookup
- query_type (str): A string containing the DNS record type (SOA not supported)
- server (str): A string containing the fqdn or IP address of the dns server
- timeout (float): DNS lookup timeout duration (default: 2.0 seconds)
Returns:
A set() of :class:`~ccp_util.DNSResponse` instances
>>> from ciscoconfparse.ccp_util import dns_query
>>> dns_query('www.pennington.net', "A", "4.2.2.2")
set([<DNSResponse "A" result_str="65.19.187.2">])
>>> answer = dns_query('www.pennington.net', 'A', '4.2.2.2')
>>> str(answer.pop())
'65.19.187.2'
>>>
"""
valid_records = set(['A', 'AAAA', 'AXFR', 'CNAME', 'MX', 'NS', 'PTR',
'TXT'])
query_type = query_type.upper()
assert query_type in valid_records
assert server!=""
assert float(timeout)>0
assert input != ""
intput = input.strip()
retval = set([])
resolver = Resolver()
resolver.server = [socket.gethostbyname(server)]
resolver.timeout = float(timeout)
resolver.lifetime = float(timeout)
start = time.time()
if (query_type=="A") or (query_type=="AAAA"):
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=input, result_str = str(result.address))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration, query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type=="AXFR":
"""This is a hack: return text of zone transfer, instead of axfr objs"""
_zone = zone.from_xfr(query.xfr(server, input, lifetime=timeout))
return [_zone[node].to_text(node) for node in _zone.nodes.keys()]
elif query_type=="CNAME":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=input, result_str = str(result.target))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input, duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type=="MX":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
input=input, result_str = str(result.target))
response.preference = int(result.preference)
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input, duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type=="NS":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=input, result_str = str(result.target))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration, query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type=="PTR":
if is_valid_ipv4_addr(input) or is_valid_ipv6_addr(input):
inaddr = reversename.from_address(input)
elif 'in-addr.arpa' in input.lower():
inaddr = input
else:
raise ValueError('Cannot query PTR record for "{0}"'.format(input))
try:
answer = resolver.query(inaddr, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=inaddr, result_str = str(result.target))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration,
query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
elif query_type=="TXT":
try:
answer = resolver.query(input, query_type)
duration = time.time() - start
for result in answer:
response = DNSResponse(query_type=query_type,
duration=duration,
input=inaddr, result_str = str(result.strings))
retval.add(response)
except DNSException as e:
duration = time.time() - start
response = DNSResponse(input=input,
duration=duration, query_type=query_type)
response.has_error = True
response.error_str = e
retval.add(response)
return retval
def retrieveDNSRecords(db, domain):
resolver = Resolver()
resolver.timeout = 1
resolver.lifetime = 1
types = ["A", "MX", "NS", "AAAA", "SOA", "TXT"]
timestamp = int(time())
print(colored("\n[*]-Retrieving DNS Records...", "yellow"))
for type in types:
try:
answers = resolver.query(domain, type)
for answer in answers:
if type == "A":
db.add(
Record(domain=domain,
type=type,
value=answer.address,
timestamp=timestamp))
if type == "MX":
db.add(
Record(domain=domain,
type=type,
value=answer.exchange.to_text()[:-1],
timestamp=timestamp))
if type == "NS":
db.add(
Record(domain=domain,
type=type,
value=answer.target.to_text()[:-1],
timestamp=timestamp))
if type == "AAAA":
db.add(
Record(domain=domain,
type=type,
value=answer.address,
timestamp=timestamp))
if type == "SOA":
db.add(
Record(domain=domain,
type=type,
value=answer.mname.to_text()[:-1],
timestamp=timestamp))
if type == "TXT":
db.add(
Record(domain=domain,
type=type,
value=str(answer),
timestamp=timestamp))
try:
db.commit()
except (IntegrityError, FlushError):
db.rollback()
except Exception as e:
pass
for row in db.query(Record).filter(Record.domain == domain).order_by(
Record.type):
print(" \__ {0}: {1}".format(colored(row.type, "cyan"),
colored(row.value, "yellow")))
