def test_analyse(self):
"""Test the tool that generates the reports."""
domain = 'a.com'
results = tools.analyse(domain)
self.assertEqual(
'a.com', results[0],
'First item in results should be the original domain')
self.assertEqual(['fuzzy_domains'], results[1].keys(),
'We only return fuzzy domains in report')
self.assertItemsEqual(
{
'domain-name': 'a.com',
'fuzzer': 'Original*',
'hex': 'YS5jb20='
}, results[1]['fuzzy_domains'][0],
'First result is the original domain')
results = map(operator.itemgetter('domain-name'),
results[1]['fuzzy_domains'])
assert sorted(results) == [
'1.com', '2.com', 'a.com', 'aa.com', 'ab.com', 'ac.com',
'acom.com', 'ad.com', 'ae.com', 'af.com', 'ag.com', 'ah.com',
'ai.com', 'aj.com', 'ak.com', 'al.com', 'am.com', 'an.com',
'ao.com', 'ap.com', 'aq.com', 'ar.com', 'as.com', 'at.com',
'au.com', 'av.com', 'aw.com', 'ax.com', 'ay.com', 'az.com',
'c.com', 'e.com', 'i.com', 'o.com', 'q.com', 's.com', 'u.com',
'w.com', 'wwa.com', 'www-a.com', 'wwwa.com', 'y.com', 'z.com'
]
self.assertIs(None, tools.analyse('\\.38iusd-s-da aswd?'),
'Invalid domains return None')
def test_analyse(self):
"""Test the tool that generates the reports."""
domain = Domain('a.com')
results = tools.analyse(domain)
self.assertEqual(
'a.com', results[0],
'First item in results should be the original domain')
self.assertEqual(['fuzzy_domains'], list(results[1].keys()),
'We only return fuzzy domains in report')
assert results[1]['fuzzy_domains'][0] == {
'domain-name': 'a.com',
'fuzzer': 'Original*',
'hex': '612e636f6d'
}
results = map(operator.itemgetter('domain-name'),
results[1]['fuzzy_domains'])
assert sorted(results) == [
'1.com', '2.com', 'a.com', 'aa.com', 'ab.com', 'ac.com',
'acom.com', 'ad.com', 'ae.com', 'af.com', 'ag.com', 'ah.com',
'ai.com', 'aj.com', 'ak.com', 'al.com', 'am.com', 'an.com',
'ao.com', 'ap.com', 'aq.com', 'ar.com', 'as.com', 'at.com',
'au.com', 'av.com', 'aw.com', 'ax.com', 'ay.com', 'az.com',
'c.com', 'e.com', 'i.com', 'o.com', 'q.com', 's.com', 'u.com',
'w.com', 'wwa.com', 'www-a.com', 'wwwa.com', 'y.com', 'z.com'
]
def csv_render(domain):
"""Render and return the csv-formatted report."""
headers = ('Domain', 'Type', 'Tweak', 'IP', 'Error')
reports = dict([tools.analyse(domain)])
def generate():
"""Streaming download generator."""
yield ','.join(headers) + '\n'
for (domain, rept) in reports.items():
for entry in rept['fuzzy_domains']:
ip_addr, error = tools.resolve(entry['domain-name'])
row = (
domain.encode('idna'),
entry['fuzzer'],
entry['domain-name'].encode('idna'),
str(ip_addr),
str(error),
)
# comma not possible in any of the row values.
yield u','.join(row) + '\n'
return flask.Response(
generate(),
headers={
'Content-Disposition': 'attachment; filename=dnstwister_report.csv'
},
mimetype='text/csv',
)
def csv_render(domain):
"""Render and return the csv-formatted report."""
headers = ('Domain', 'Type', 'Tweak', 'IP', 'Error')
csv_filename = 'dnstwister_report_{}.csv'.format(domain.to_ascii())
def local_resolve_candidate(candidate):
domain = Domain(candidate['domain-name'])
ip_addr, error = tools.resolve(domain)
return candidate['fuzzer'], domain, ip_addr, error
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
futures = executor.map(local_resolve_candidate,
tools.analyse(domain)[1]['fuzzy_domains'])
csv = ','.join(headers) + '\n'
for (fuzzer, entry_domain, ip_addr, error) in futures:
row = (
domain.to_ascii(),
fuzzer,
entry_domain.to_ascii(),
str(ip_addr),
str(error),
)
csv += ','.join(row) + '\n'
return flask.Response(
csv,
headers={
'Content-Disposition': 'attachment; filename=' + csv_filename
},
mimetype='text/csv',
)
def json_render(domain):
"""Render and return the json-formatted report.
The hand-assembly is due to the streaming of the response.
"""
reports = dict([tools.analyse(domain)])
json_filename = 'dnstwister_report_{}.json'.format(domain.encode('idna'))
def generate():
"""Streaming download generator."""
indent_size = 4
indent = ' ' * indent_size
yield '{\n'
# TODO: We only have one domain now, simplify this.
for (dom, rept) in reports.items():
yield indent + '"' + dom.encode('idna') + '": {\n'
yield indent * 2 + '"fuzzy_domains": [\n'
fuzzy_domains = rept['fuzzy_domains']
for (j, entry) in enumerate(fuzzy_domains):
ip_addr, error = tools.resolve(entry['domain-name'])
data = {
'domain-name': entry['domain-name'].encode('idna'),
'fuzzer': entry['fuzzer'],
'hex': entry['hex'],
'resolution': {
'error': error,
'ip': ip_addr,
},
}
json_str = json.dumps(data,
sort_keys=True,
indent=indent_size,
separators=(',', ': '))
yield '\n'.join(
[indent * 3 + line for line in json_str.split('\n')])
if j < len(fuzzy_domains) - 1:
yield ','
yield '\n'
yield indent * 2 + ']\n'
yield indent + '}'
yield '\n'
yield '}\n'
return flask.Response(generate(),
headers={
'Content-Disposition':
'attachment; filename=' + json_filename
},
content_type='application/json')
def html_render(domain):
"""Render and return the html report."""
return flask.render_template('www/report.html',
domain=domain,
report=tools.analyse(domain)[1],
exports={
'json': 'json',
'csv': 'csv'
})
def test_analyse(self):
""" Test the tool that generates the reports.
"""
domain = 'a.com'
results = tools.analyse(domain)
self.assertEqual(
'a.com', results[0],
'First item in results should be the original domain'
)
self.assertEqual(
['fuzzy_domains'],
results[1].keys(),
'We only return fuzzy domains in report'
)
self.assertItemsEqual(
{'domain-name': 'a.com', 'fuzzer': 'Original*', 'hex': 'YS5jb20='},
results[1]['fuzzy_domains'][0],
'First result is the original domain'
)
results = map(operator.itemgetter('domain-name'), results[1]['fuzzy_domains'])
assert results == [
'a.com', 'aa.com', 'ab.com', 'ac.com', 'ad.com', 'ae.com',
'af.com', 'ag.com', 'ah.com', 'ai.com', 'aj.com', 'ak.com',
'al.com', 'am.com', 'an.com', 'ao.com', 'ap.com', 'aq.com',
'ar.com', 'as.com', 'at.com', 'au.com', 'av.com', 'aw.com',
'ax.com', 'ay.com', 'az.com', 'c.com', 'e.com', 'i.com',
'q.com', '1.com', 's.com', '2.com', 'w.com', 'y.com', 'z.com',
'u.com', 'o.com', 'wwa.com', 'wwwa.com', 'www-a.com', 'acom.com',
]
self.assertIs(
None, tools.analyse('\\.38iusd-s-da aswd?'),
'Invalid domains return None'
)
def html_render(domain):
"""Render and return the html report."""
reports = dict([tools.analyse(domain)])
return flask.render_template(
'www/report.html',
reports=reports,
atoms=dict([(domain, tools.encode_domain(domain))]),
exports={
'json': 'json',
'csv': 'csv'
},
domain_encoded=tools.encode_domain(domain),
)
def html_render(domain):
"""Render and return the html report."""
reports = dict([tools.analyse(domain)])
return flask.render_template(
'www/report.html',
reports=reports,
atoms=dict([(domain, binascii.hexlify(domain))]),
exports={
'json': 'json',
'csv': 'csv'
},
search=[domain],
)
def json_render(domain):
"""Render and return the json-formatted report."""
json_filename = 'dnstwister_report_{}.json'.format(domain.to_ascii())
def local_resolve_candidate(candidate):
domain = Domain(candidate['domain-name'])
ip_addr, error = tools.resolve(domain)
return candidate['fuzzer'], domain, ip_addr, error
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
futures = executor.map(local_resolve_candidate,
tools.analyse(domain)[1]['fuzzy_domains'])
results = []
for (fuzzer, entry_domain, ip_addr, error) in futures:
results.append({
'domain-name': entry_domain.to_ascii(),
'fuzzer': fuzzer,
'hex': entry_domain.to_hex(),
'resolution': {
'error': error,
'ip': ip_addr
}
})
response = {domain.to_ascii(): {'fuzzy_domains': results}}
return flask.Response(json.dumps(response,
sort_keys=True,
indent=4,
separators=(',', ': ')),
headers={
'Content-Disposition':
'attachment; filename=' + json_filename
},
content_type='application/json')
def process_domain(domain):
"""Process a domain - generating resolution reports and deltas."""
if dnstwist.validate_domain(domain) is None:
print 'Invalid: {}'.format(repr(domain))
repository.unregister_domain(domain)
return
# Unregister long-time unread domains
last_read = repository.delta_report_last_read(domain)
if last_read is None:
repository.mark_delta_report_as_read(domain)
else:
age = datetime.datetime.now() - last_read
if age > datetime.timedelta(seconds=PERIOD * UNREGISTER):
print 'Expired: {}'.format(domain.encode('idna'))
repository.unregister_domain(domain)
return
# Skip domains that have been recently updated
delta_last_updated = repository.delta_report_updated(domain)
if delta_last_updated is not None:
age = datetime.datetime.now() - delta_last_updated
if age < datetime.timedelta(seconds=PERIOD):
print 'Skipping: {}'.format(domain.encode('idna'))
return
start = time.time()
existing_report = repository.get_resolution_report(domain)
if existing_report is None:
existing_report = {}
new_report = {}
for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]:
ip_addr, error = tools.resolve(entry['domain-name'])
if error or not ip_addr or ip_addr is None:
continue
new_report[entry['domain-name']] = {
'ip': ip_addr,
'tweak': entry['fuzzer'],
}
repository.update_resolution_report(domain, new_report)
delta_report = {'new': [], 'updated': [], 'deleted': []}
for (dom, data) in new_report.items():
try:
new_ip = data['ip']
except TypeError:
# handle old-style ip-only reports
new_ip = data
if dom in existing_report.keys():
try:
existing_ip = existing_report[dom]['ip']
except TypeError:
# handle old-style ip-only reports
existing_ip = existing_report[dom]
if new_ip != existing_ip:
delta_report['updated'].append((dom, existing_ip, new_ip))
else:
delta_report['new'].append((dom, new_ip))
for dom in existing_report.keys():
if dom not in new_report.keys():
delta_report['deleted'].append(dom)
repository.update_delta_report(domain, delta_report)
print 'Updated {} in {} seconds'.format(domain.encode('idna'),
time.time() - start)
def process_domain(domain):
"""Process a domain - generating resolution reports and deltas."""
if dnstwist.validate_domain(domain) is None:
print 'Unregistering (invalid) {}'.format(domain)
repository.unregister_domain(domain)
return
# Unregister long-time unread domains
last_read = repository.delta_report_last_read(domain)
if last_read is None:
repository.mark_delta_report_as_read(domain)
else:
age = datetime.datetime.now() - last_read
if age > datetime.timedelta(seconds=PERIOD*UNREGISTER):
print 'Unregistering (not read > 7 days) {}'.format(domain)
repository.unregister_domain(domain)
return
# Skip domains that have been recently updated
delta_last_updated = repository.delta_report_updated(domain)
if delta_last_updated is not None:
age = datetime.datetime.now() - delta_last_updated
if age < datetime.timedelta(seconds=PERIOD):
print 'Skipping (recently updated) {}'.format(domain)
return
start = time.time()
existing_report = repository.get_resolution_report(domain)
if existing_report is None:
existing_report = {}
new_report = {}
for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]:
ip, error = tools.resolve(entry['domain-name'])
if error or not ip or ip is None:
continue
new_report[entry['domain-name']] = {
'ip': ip,
'tweak': entry['fuzzer'],
}
repository.update_resolution_report(domain, new_report)
delta_report = {'new': [], 'updated': [], 'deleted': []}
for (dom, data) in new_report.items():
try:
new_ip = data['ip']
except TypeError:
# handle old-style ip-only reports
new_ip = data
if dom in existing_report.keys():
try:
existing_ip = existing_report[dom]['ip']
except TypeError:
# handle old-style ip-only reports
existing_ip = existing_report[dom]
if new_ip != existing_ip:
delta_report['updated'].append(
(dom, existing_ip, new_ip)
)
else:
delta_report['new'].append((dom, new_ip))
for dom in existing_report.keys():
if dom not in new_report.keys():
delta_report['deleted'].append(dom)
repository.update_delta_report(domain, delta_report)
print 'Updated deltas for {} in {} seconds'.format(
domain, time.time() - start
)
