def test_analyse(self): """Test the tool that generates the reports.""" domain = 'a.com' results = tools.analyse(domain) self.assertEqual( 'a.com', results[0], 'First item in results should be the original domain') self.assertEqual(['fuzzy_domains'], results[1].keys(), 'We only return fuzzy domains in report') self.assertItemsEqual( { 'domain-name': 'a.com', 'fuzzer': 'Original*', 'hex': 'YS5jb20=' }, results[1]['fuzzy_domains'][0], 'First result is the original domain') results = map(operator.itemgetter('domain-name'), results[1]['fuzzy_domains']) assert sorted(results) == [ '1.com', '2.com', 'a.com', 'aa.com', 'ab.com', 'ac.com', 'acom.com', 'ad.com', 'ae.com', 'af.com', 'ag.com', 'ah.com', 'ai.com', 'aj.com', 'ak.com', 'al.com', 'am.com', 'an.com', 'ao.com', 'ap.com', 'aq.com', 'ar.com', 'as.com', 'at.com', 'au.com', 'av.com', 'aw.com', 'ax.com', 'ay.com', 'az.com', 'c.com', 'e.com', 'i.com', 'o.com', 'q.com', 's.com', 'u.com', 'w.com', 'wwa.com', 'www-a.com', 'wwwa.com', 'y.com', 'z.com' ] self.assertIs(None, tools.analyse('\\.38iusd-s-da aswd?'), 'Invalid domains return None')
def test_analyse(self): """Test the tool that generates the reports.""" domain = Domain('a.com') results = tools.analyse(domain) self.assertEqual( 'a.com', results[0], 'First item in results should be the original domain') self.assertEqual(['fuzzy_domains'], list(results[1].keys()), 'We only return fuzzy domains in report') assert results[1]['fuzzy_domains'][0] == { 'domain-name': 'a.com', 'fuzzer': 'Original*', 'hex': '612e636f6d' } results = map(operator.itemgetter('domain-name'), results[1]['fuzzy_domains']) assert sorted(results) == [ '1.com', '2.com', 'a.com', 'aa.com', 'ab.com', 'ac.com', 'acom.com', 'ad.com', 'ae.com', 'af.com', 'ag.com', 'ah.com', 'ai.com', 'aj.com', 'ak.com', 'al.com', 'am.com', 'an.com', 'ao.com', 'ap.com', 'aq.com', 'ar.com', 'as.com', 'at.com', 'au.com', 'av.com', 'aw.com', 'ax.com', 'ay.com', 'az.com', 'c.com', 'e.com', 'i.com', 'o.com', 'q.com', 's.com', 'u.com', 'w.com', 'wwa.com', 'www-a.com', 'wwwa.com', 'y.com', 'z.com' ]
def csv_render(domain): """Render and return the csv-formatted report.""" headers = ('Domain', 'Type', 'Tweak', 'IP', 'Error') reports = dict([tools.analyse(domain)]) def generate(): """Streaming download generator.""" yield ','.join(headers) + '\n' for (domain, rept) in reports.items(): for entry in rept['fuzzy_domains']: ip_addr, error = tools.resolve(entry['domain-name']) row = ( domain.encode('idna'), entry['fuzzer'], entry['domain-name'].encode('idna'), str(ip_addr), str(error), ) # comma not possible in any of the row values. yield u','.join(row) + '\n' return flask.Response( generate(), headers={ 'Content-Disposition': 'attachment; filename=dnstwister_report.csv' }, mimetype='text/csv', )
def csv_render(domain): """Render and return the csv-formatted report.""" headers = ('Domain', 'Type', 'Tweak', 'IP', 'Error') csv_filename = 'dnstwister_report_{}.csv'.format(domain.to_ascii()) def local_resolve_candidate(candidate): domain = Domain(candidate['domain-name']) ip_addr, error = tools.resolve(domain) return candidate['fuzzer'], domain, ip_addr, error with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: futures = executor.map(local_resolve_candidate, tools.analyse(domain)[1]['fuzzy_domains']) csv = ','.join(headers) + '\n' for (fuzzer, entry_domain, ip_addr, error) in futures: row = ( domain.to_ascii(), fuzzer, entry_domain.to_ascii(), str(ip_addr), str(error), ) csv += ','.join(row) + '\n' return flask.Response( csv, headers={ 'Content-Disposition': 'attachment; filename=' + csv_filename }, mimetype='text/csv', )
def json_render(domain): """Render and return the json-formatted report. The hand-assembly is due to the streaming of the response. """ reports = dict([tools.analyse(domain)]) json_filename = 'dnstwister_report_{}.json'.format(domain.encode('idna')) def generate(): """Streaming download generator.""" indent_size = 4 indent = ' ' * indent_size yield '{\n' # TODO: We only have one domain now, simplify this. for (dom, rept) in reports.items(): yield indent + '"' + dom.encode('idna') + '": {\n' yield indent * 2 + '"fuzzy_domains": [\n' fuzzy_domains = rept['fuzzy_domains'] for (j, entry) in enumerate(fuzzy_domains): ip_addr, error = tools.resolve(entry['domain-name']) data = { 'domain-name': entry['domain-name'].encode('idna'), 'fuzzer': entry['fuzzer'], 'hex': entry['hex'], 'resolution': { 'error': error, 'ip': ip_addr, }, } json_str = json.dumps(data, sort_keys=True, indent=indent_size, separators=(',', ': ')) yield '\n'.join( [indent * 3 + line for line in json_str.split('\n')]) if j < len(fuzzy_domains) - 1: yield ',' yield '\n' yield indent * 2 + ']\n' yield indent + '}' yield '\n' yield '}\n' return flask.Response(generate(), headers={ 'Content-Disposition': 'attachment; filename=' + json_filename }, content_type='application/json')
def html_render(domain): """Render and return the html report.""" return flask.render_template('www/report.html', domain=domain, report=tools.analyse(domain)[1], exports={ 'json': 'json', 'csv': 'csv' })
def test_analyse(self): """ Test the tool that generates the reports. """ domain = 'a.com' results = tools.analyse(domain) self.assertEqual( 'a.com', results[0], 'First item in results should be the original domain' ) self.assertEqual( ['fuzzy_domains'], results[1].keys(), 'We only return fuzzy domains in report' ) self.assertItemsEqual( {'domain-name': 'a.com', 'fuzzer': 'Original*', 'hex': 'YS5jb20='}, results[1]['fuzzy_domains'][0], 'First result is the original domain' ) results = map(operator.itemgetter('domain-name'), results[1]['fuzzy_domains']) assert results == [ 'a.com', 'aa.com', 'ab.com', 'ac.com', 'ad.com', 'ae.com', 'af.com', 'ag.com', 'ah.com', 'ai.com', 'aj.com', 'ak.com', 'al.com', 'am.com', 'an.com', 'ao.com', 'ap.com', 'aq.com', 'ar.com', 'as.com', 'at.com', 'au.com', 'av.com', 'aw.com', 'ax.com', 'ay.com', 'az.com', 'c.com', 'e.com', 'i.com', 'q.com', '1.com', 's.com', '2.com', 'w.com', 'y.com', 'z.com', 'u.com', 'o.com', 'wwa.com', 'wwwa.com', 'www-a.com', 'acom.com', ] self.assertIs( None, tools.analyse('\\.38iusd-s-da aswd?'), 'Invalid domains return None' )
def html_render(domain): """Render and return the html report.""" reports = dict([tools.analyse(domain)]) return flask.render_template( 'www/report.html', reports=reports, atoms=dict([(domain, tools.encode_domain(domain))]), exports={ 'json': 'json', 'csv': 'csv' }, domain_encoded=tools.encode_domain(domain), )
def html_render(domain): """Render and return the html report.""" reports = dict([tools.analyse(domain)]) return flask.render_template( 'www/report.html', reports=reports, atoms=dict([(domain, binascii.hexlify(domain))]), exports={ 'json': 'json', 'csv': 'csv' }, search=[domain], )
def json_render(domain): """Render and return the json-formatted report.""" json_filename = 'dnstwister_report_{}.json'.format(domain.to_ascii()) def local_resolve_candidate(candidate): domain = Domain(candidate['domain-name']) ip_addr, error = tools.resolve(domain) return candidate['fuzzer'], domain, ip_addr, error with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: futures = executor.map(local_resolve_candidate, tools.analyse(domain)[1]['fuzzy_domains']) results = [] for (fuzzer, entry_domain, ip_addr, error) in futures: results.append({ 'domain-name': entry_domain.to_ascii(), 'fuzzer': fuzzer, 'hex': entry_domain.to_hex(), 'resolution': { 'error': error, 'ip': ip_addr } }) response = {domain.to_ascii(): {'fuzzy_domains': results}} return flask.Response(json.dumps(response, sort_keys=True, indent=4, separators=(',', ': ')), headers={ 'Content-Disposition': 'attachment; filename=' + json_filename }, content_type='application/json')
def process_domain(domain): """Process a domain - generating resolution reports and deltas.""" if dnstwist.validate_domain(domain) is None: print 'Invalid: {}'.format(repr(domain)) repository.unregister_domain(domain) return # Unregister long-time unread domains last_read = repository.delta_report_last_read(domain) if last_read is None: repository.mark_delta_report_as_read(domain) else: age = datetime.datetime.now() - last_read if age > datetime.timedelta(seconds=PERIOD * UNREGISTER): print 'Expired: {}'.format(domain.encode('idna')) repository.unregister_domain(domain) return # Skip domains that have been recently updated delta_last_updated = repository.delta_report_updated(domain) if delta_last_updated is not None: age = datetime.datetime.now() - delta_last_updated if age < datetime.timedelta(seconds=PERIOD): print 'Skipping: {}'.format(domain.encode('idna')) return start = time.time() existing_report = repository.get_resolution_report(domain) if existing_report is None: existing_report = {} new_report = {} for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]: ip_addr, error = tools.resolve(entry['domain-name']) if error or not ip_addr or ip_addr is None: continue new_report[entry['domain-name']] = { 'ip': ip_addr, 'tweak': entry['fuzzer'], } repository.update_resolution_report(domain, new_report) delta_report = {'new': [], 'updated': [], 'deleted': []} for (dom, data) in new_report.items(): try: new_ip = data['ip'] except TypeError: # handle old-style ip-only reports new_ip = data if dom in existing_report.keys(): try: existing_ip = existing_report[dom]['ip'] except TypeError: # handle old-style ip-only reports existing_ip = existing_report[dom] if new_ip != existing_ip: delta_report['updated'].append((dom, existing_ip, new_ip)) else: delta_report['new'].append((dom, new_ip)) for dom in existing_report.keys(): if dom not in new_report.keys(): delta_report['deleted'].append(dom) repository.update_delta_report(domain, delta_report) print 'Updated {} in {} seconds'.format(domain.encode('idna'), time.time() - start)
def process_domain(domain): """Process a domain - generating resolution reports and deltas.""" if dnstwist.validate_domain(domain) is None: print 'Unregistering (invalid) {}'.format(domain) repository.unregister_domain(domain) return # Unregister long-time unread domains last_read = repository.delta_report_last_read(domain) if last_read is None: repository.mark_delta_report_as_read(domain) else: age = datetime.datetime.now() - last_read if age > datetime.timedelta(seconds=PERIOD*UNREGISTER): print 'Unregistering (not read > 7 days) {}'.format(domain) repository.unregister_domain(domain) return # Skip domains that have been recently updated delta_last_updated = repository.delta_report_updated(domain) if delta_last_updated is not None: age = datetime.datetime.now() - delta_last_updated if age < datetime.timedelta(seconds=PERIOD): print 'Skipping (recently updated) {}'.format(domain) return start = time.time() existing_report = repository.get_resolution_report(domain) if existing_report is None: existing_report = {} new_report = {} for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]: ip, error = tools.resolve(entry['domain-name']) if error or not ip or ip is None: continue new_report[entry['domain-name']] = { 'ip': ip, 'tweak': entry['fuzzer'], } repository.update_resolution_report(domain, new_report) delta_report = {'new': [], 'updated': [], 'deleted': []} for (dom, data) in new_report.items(): try: new_ip = data['ip'] except TypeError: # handle old-style ip-only reports new_ip = data if dom in existing_report.keys(): try: existing_ip = existing_report[dom]['ip'] except TypeError: # handle old-style ip-only reports existing_ip = existing_report[dom] if new_ip != existing_ip: delta_report['updated'].append( (dom, existing_ip, new_ip) ) else: delta_report['new'].append((dom, new_ip)) for dom in existing_report.keys(): if dom not in new_report.keys(): delta_report['deleted'].append(dom) repository.update_delta_report(domain, delta_report) print 'Updated deltas for {} in {} seconds'.format( domain, time.time() - start )