def main(key_start, key_end):
try:
key_file = open(filename, "r")
except:
print "[ERROR]: key file not found. \n run python redisKeyDump.py > redis-backups/236keys.txt"
return
if key_start < 0:
key_start = 0
if key_end > 236733 - 1:
key_end = 236733 - 1
print "range:", key_start, key_end
for i, line in enumerate(key_file):
if i < key_start or i > key_end:
continue
key = line.rstrip("\n")
ip_set = db_dns.smembers(key)
if ip_set == set():
# empty set, we need to look it up
# ip = socket.gethostbyname(key)
# didn't handle those without ip for now
print key
else:
for ip in ip_set:
if ip is not "":
print "[%i] %s, %s, %s" % (i, key, ip, ip2openResolvers(ip))
query(key, ip)
def getTTL(domain, q):
global wrongCnt,additZero,registered
TTL = 0
try:
response = query(domain, name_server)
additCnt = len(response.additional)
except dns.exception.Timeout:
additCnt = 0
wrongCnt += 1
if additCnt == 0:
additZero+=1
else:
addit = response.additional[0]
Serveraddr = addit.items[0].address
try:
response = query(domain, Serveraddr)
ansCnt = len(response.additional)
except dns.exception.Timeout:
ansCnt = 0
wrongCnt += 1
ansCnt = len(response.answer)
if ansCnt != 0:
registered+=1
answer = response.answer[0]
TTL = answer.ttl
# print domain, TTL
# f.write(domain + ' ' + str(TTL) + '\n')
if TTL!=0:
q.put(domain + ' ' + str(TTL))
def dnsHistory(domain):
rows = ''
print("\n-- Checking dns history --")
url = 'http://toolbar.netcraft.com/site_report?url=' + domain
try:
request = urllib.request.urlopen(url)
html = request.read()
except:
html = ''
soup = BeautifulSoup(''.join(html))
tables = soup.findAll(attrs={'class': 'TBtable'})
try:
table = tables[1]
except:
table = '' # Prevents errors if no history returned
rows = ''
if table:
rows = soup.table.findAll('tr') # Need to edit out again
x = -1
try:
for tr in rows:
columns = tr.findAll('td')
for td in columns:
text = ''.join(td.find(text=True))
if x % 5 == 0: # Only ip addresses are checked
if dns.query(text): # Finds last ip thats not CloudFlare
print(output("The last known ip address is: %s" % text))
if text not in iplist:
iplist.append(text)
raise End # Breaks from multiple loops
x += 1
except End:
pass
print("\n#" + "-" * 77 + "#")
def dnsmap(self):
result = subprocess.check_output([dnsmapdir + '/dnsmap', self.domain]) # Executes dnsmap
# Formats output
out = re.split('IP address #.: ', result)
domain = re.split('\n', out)
domain = domain[5] # Stores first domain found
store = '' # Stores previous domain name if >1 ip returned
for i in out:
if i.startswith('\n'):
pass
else:
ip = re.split('\n', i)
if ip[0].startswith('dnsmap'): # Filters out dnsmap messages
pass
else:
if dns.query(ip[0]):
if ip[0] in iplist:
pass
else:
iplist.append(ip[0])
print(output("%s is hosted at %s" % (domain, ip[0])))
# Fixes errors when >1 ip returned for 1 subdomain
try:
if ip[0] == '127.0.0.1': # prevents errors by dnsmap warning
domain = ip[3]
else:
domain = ip[2]
except:
domain = ''
if domain == '':
domain = store
else:
store = ip[2]
def __init__(self, cb, eb, host, address_families = typemap.values()):
self.cb = cb
self.eb = eb
self.host = host
self.result = []
self.queries = [query(self.done, self.lose, host, qtype)
for qtype in self.typemap
if self.typemap[qtype] in address_families]
def fast(self):
print("\n-- Testing common subdomains for misconfiguration --")
for sub in self.subs:
subdomain = sub + '.' + self.domain
response = dns.query(subdomain)
if response: # If subdomain exists
# Prevent errors if >1 ip returned for domain
if type(response) == list:
for ip in response:
self.ips[subdomain] = ip
else:
pass
# Checks if the discovered domains are behind cloudflare
for domain, ip in self.ips.iteritems():
if dns.query(ip):
print(output("%s is hosted at %s" % (domain, ip)))
if ip not in iplist:
iplist.append(ip)
def tld(self):
print("\n-- Testing common tlds --")
tmp = re.split('\..*', self.domain) # Removes current tld
for i in self.tlds:
testDomain = tmp[0] + i
response = dns.query(testDomain)
if response:
if type(response) == list: # If >1 ips returned
for ip in response:
if dns.query(ip):
print(output("%s is hosted at %s" % (testDomain, ip)))
if ip not in iplist:
iplist.append(ip)
else:
if dns.query(ip):
print(output("%s is hosted at %s" % (testDomain, ip)))
if ip not in iplist:
iplist.append(ip)
def recursive_query(resolver, domain, record_type='NS'):
query_domain = str(domain)
query_response = None
try:
while query_response is None:
query_response = query(resolver, query_domain, record_type)
query_domain = query_domain.split('.', 1)[1]
except IndexError:
return None
return query_response
def subdomainWorker(visited):
while True:
subdomain = subdomainQueue.get()
if type(subdomain) is int:
return
url = concatenate_subdomains(domain, [subdomain])
try:
record = query(resolver, url, record_type='A')
except dns.exception.Timeout:
return
if record is None:
continue
try:
ip = ipaddress.IPv4Address(record[0].address)
except TypeError:
return
headers = False
if kwargs.get('connect') and not ip.is_private:
headers = head_request(str(ip))
if kwargs.get("traverse"):
traverse = kwargs.get("traverse")
else:
traverse = 5
if kwargs.get("wide"):
ips = wide_expander(ip)
else:
ips = traverse_expander(ip, traverse)
filter_func = None
if kwargs.get("search"):
filter_func = functools.partial(search_filter, kwargs["search"])
ips = set(ips) - set(visited)
visited |= ips
hosts.put({
str(url): {
'ip': str(ip),
'headers': headers,
'nearby': find_nearby(resolver, ips, filter_func=filter_func)
}
})
if kwargs.get("delay"):
time.sleep(kwargs["delay"])
else:
time.sleep(0)
def check_authority(url, nameservers):
"""check the list of authoritative nameservers for one of our canonical ones"""
owned = 'N/A'
# get a nameserver to resolve for us
nameserver = '8.8.8.8' # Google's
# create a dns name from the URL
subject = dns.name.from_text(url)
def query(nameserver, domain, response_type):
""" generalize queries so we can do SOA, NS, etc; take params, return response """
try:
query = dns.message.make_query(domain, response_type)
return dns.query.udp(query, nameserver, timeout=5)
except:
log.msg('Unable to lookup %s on %s: %s' % (url, nameserver, sys.exc_info()[0]), level=log.WARNING)
# TODO: retry?
response = query(nameserver, subject, dns.rdatatype.SOA)
# handle some errors
rcode = response.rcode()
if rcode == dns.rcode.REFUSED:
log.msg('Nameserver Error: %s refused to lookup %s.' % (nameserver, url), level=log.WARNING)
return owned
elif rcode == dns.rcode.NXDOMAIN:
log.msg('Nameserver cannot find %s.' % url, level=log.DEBUG)
elif rcode != dns.rcode.NOERROR:
log.msg('Nameserver Error: while looking up %s received %s' % (url, dns.rcode.to_text(rcode)), level=log.WARNING)
return owned
# get the dns representation of the response
ns = response.authority if len(response.authority) > 0 else response.answer
# get the text representation of its first record
ns = ns[0][0].to_text()
# get the nameserver out of that
ns = ns.split()[0].rstrip('.')
# check if the ns is in our list of nameservers
owned = ns in nameservers
if owned:
log.msg('->Mozilla is authoritative for %s.' % (url), level=log.DEBUG)
return owned
def query(resolver, domain, record_type='A'):
try:
resp = resolver.query(domain, record_type, raise_on_no_answer=False)
if resp.response.answer:
return resp
# If we don't receive an answer from our current resolver let's
# assume we received information on nameservers we can use and
# perform the same query with those nameservers
if resp.response.additional and resp.response.authority:
ns = [
rdata.address
for additionals in resp.response.additional
for rdata in additionals.items
]
resolver.nameservers = ns
return query(resolver, domain, record_type)
return None
except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
return None
def getAnswer(domain, q, name_server):
global wrongCnt,logsCnt,numofthreads,alllogs,currenttimeinseconds
try:
response = query(domain, name_server)
if len(response.answer) != 0:
ttl = response.answer[0].ttl
if ttl == 0:
ttl = 1
q.put(domain + ',' + name_server + ',' + str(response.answer[0].ttl) + ',' + str(currenttimeinseconds))
# all logs
log = ' ' + '\n'
log+=domain + ' ' + str(currenttimeinseconds) + '\n'
for i in response.answer:
log+= str(i) + '\n'
for i in response.authority:
log+= str(i) + '\n'
for i in response.additional:
log+= str(i) + '\n'
alllogs.append(log)
logsCnt+=1
else:
q.put(domain + ',' + name_server + ',' + str(0) + ',' + str(currenttimeinseconds))
except dns.exception.Timeout:
wrongCnt += 1
except dns.exception.FormError:
pass
#print 'formerror'
except socket.error:
pass
#print 'socket error,connection forced closed'
numofthreads-=1
def __init__(self, qname, qtype = dns.rdatatype.A, qclass = dns.rdataclass.IN): self.qname = qname self.qtype = qtype self.qclass = qclass query(self.done, self.lose, qname, qtype = qtype, qclass = qclass)
def fierce(**kwargs):
resolver = dns.resolver.Resolver()
nameservers = None
if kwargs.get('dns_servers'):
nameservers = kwargs['dns_servers']
elif kwargs.get('dns_file'):
nameservers = [ns.strip() for ns in open(kwargs["dns_file"]).readlines()]
if nameservers:
resolver.nameservers = nameservers
if kwargs.get("range"):
internal_range = ipaddress.IPv4Network(kwargs.get("range"))
find_nearby(resolver, list(internal_range))
if not kwargs.get("domain"):
return
domain = dns.name.from_text(kwargs['domain'])
if not domain.is_absolute():
domain = domain.concatenate(dns.name.root)
ns = query(resolver, domain, record_type='NS')
domain_name_servers = [n.to_text() for n in ns]
print("NS: {}".format(" ".join(domain_name_servers)))
soa = query(resolver, domain, record_type='SOA')
soa_mname = soa[0].mname
master = query(resolver, soa_mname, record_type='A')
master_address = master[0].address
print("SOA: {} ({})".format(soa_mname, master_address))
zone = zone_transfer(master_address, domain)
print("Zone: {}".format("success" if zone else "failure"))
if zone:
pprint.pprint({k: v.to_text(k) for k, v in zone.items()})
return
random_subdomain = str(random.randint(1e10, 1e11))
random_domain = concatenate_subdomains(domain, [random_subdomain])
wildcard = query(resolver, random_domain, record_type='A')
print("Wildcard: {}".format("success" if wildcard else "failure"))
if kwargs.get('subdomains'):
subdomains = kwargs["subdomains"]
else:
subdomains = [sd.strip() for sd in open(kwargs["subdomain_file"]).readlines()]
visited = set()
for subdomain in subdomains:
url = concatenate_subdomains(domain, [subdomain])
record = query(resolver, url, record_type='A')
if record is None:
continue
ip = ipaddress.IPv4Address(record[0].address)
print("Found: {} ({})".format(url, ip))
if kwargs.get('connect') and not ip.is_private:
headers = head_request(str(ip))
if headers:
print("HTTP connected:")
pprint.pprint(headers)
if kwargs.get("wide"):
ips = wide_expander(ip)
elif kwargs.get("traverse"):
ips = traverse_expander(ip, kwargs["traverse"])
else:
continue
filter_func = None
if kwargs.get("search"):
filter_func = functools.partial(search_filter, kwargs["search"])
ips = set(ips) - set(visited)
visited |= ips
find_nearby(resolver, ips, filter_func=filter_func)
if kwargs.get("delay"):
time.sleep(kwargs["delay"])
def reverse_query(resolver, ip):
return query(resolver, dns.reversename.from_address(ip), record_type='PTR')
def test_query_MX():
answers = query("dnspython.org","MX")
for rr in answers:
print(rr.exchange,rr.preference)
assert 0
def test_query_A():
answers = query("www.nist.gov","A")
for rr in answers:
print(dir(rr))
def fierce(**kwargs):
"""
fierce function is directly callable and returns a results dictionary
:param concurrency:
:param domain:
:param print:
:param connect:
:return: python dictionary of results
"""
resolver = dns.resolver.Resolver()
nameservers = []
if kwargs.get('dns_servers'):
nameservers = kwargs['dns_servers']
elif kwargs.get('dns_file'):
nameservers = [ns.strip() for ns in open(kwargs["dns_file"]).readlines()]
if nameservers:
resolver.nameservers = nameservers
reversed_ips = False
if kwargs.get("range"):
internal_range = ipaddress.IPv4Network(kwargs.get("range"))
reversed_ips = find_nearby(resolver, list(internal_range))
if not kwargs.get("concurrency"):
numWorkers = 1
else:
numWorkers = kwargs.get("concurrency")
if kwargs.get("domain"):
domain = dns.name.from_text(kwargs['domain'])
else:
return
if not domain.is_absolute():
domain = domain.concatenate(dns.name.root)
# DNS times out sometimes
domain_name_servers = []
for attempt in range(0, 5):
ns = query(resolver, domain, record_type='NS')
if ns is not None:
domain_name_servers = [n.to_text() for n in ns]
break
time.sleep(3)
soa = query(resolver, domain, record_type='SOA')
zone = {}
soa_mname = ''
if soa:
soa_mname = soa[0].mname
master = query(resolver, soa_mname, record_type='A')
if master:
master_address = master[0].address
zone_dump = zone_transfer(master_address, domain)
if zone_dump is not None:
zone = {(k.to_text() + '.' + domain.to_text()): v.to_text(k) for k, v in zone_dump.items()}
random_subdomain = str(random.randint(1e10, 1e11))
random_domain = concatenate_subdomains(domain, [random_subdomain])
wildcard = query(resolver, random_domain, record_type='A')
if kwargs.get('subdomains'):
subdomains = kwargs["subdomains"]
else:
if kwargs.get("subdomain_file"):
subdomain_file = kwargs.get("subdomain_file")
else:
subdomain_file = find_subdomain_list_file('default.txt')
subdomains = [sd.strip() for sd in open(subdomain_file).readlines()]
visited = set()
hosts = Queue()
def subdomainWorker(visited):
while True:
subdomain = subdomainQueue.get()
if type(subdomain) is int:
return
url = concatenate_subdomains(domain, [subdomain])
try:
record = query(resolver, url, record_type='A')
except dns.exception.Timeout:
return
if record is None:
continue
try:
ip = ipaddress.IPv4Address(record[0].address)
except TypeError:
return
headers = False
if kwargs.get('connect') and not ip.is_private:
headers = head_request(str(ip))
if kwargs.get("traverse"):
traverse = kwargs.get("traverse")
else:
traverse = 5
if kwargs.get("wide"):
ips = wide_expander(ip)
else:
ips = traverse_expander(ip, traverse)
filter_func = None
if kwargs.get("search"):
filter_func = functools.partial(search_filter, kwargs["search"])
ips = set(ips) - set(visited)
visited |= ips
hosts.put({
str(url): {
'ip': str(ip),
'headers': headers,
'nearby': find_nearby(resolver, ips, filter_func=filter_func)
}
})
if kwargs.get("delay"):
time.sleep(kwargs["delay"])
else:
time.sleep(0)
workers = []
subdomainQueue = Queue()
for i in range(numWorkers):
p = Process(target=subdomainWorker, args=(visited,))
workers.append(p)
p.start()
for subdomain in subdomains:
subdomainQueue.put(subdomain)
while True:
if subdomainQueue.empty():
for i in range(numWorkers):
# workers terminate when they encounter an int
subdomainQueue.put(50)
for worker in workers:
# blocks until process terminates
worker.join()
def get_hosts(q):
d = {}
while q.qsize() > 0:
for fqdn, nearby in q.get().items():
d[fqdn] = nearby
return d
results = {'target': domain.to_text(),
'hosts': get_hosts(hosts),
'nameservers': domain_name_servers,
'zone': zone if len(zone) > 0 else False,
'wildcard': bool(wildcard),
'range': reversed_ips
}
results['soa_mname'] = soa_mname.to_text() if soa_mname else ''
if kwargs.get('pretty_print'):
print(json.dumps(results, indent=4))
if kwargs.get('print'):
print(json.dumps(results))
return results
