def generate(): """Streaming download generator.""" for (domain, rept) in reports.items(): for entry in rept['fuzzy_domains']: ip, error = tools.resolve(entry['domain-name']) row = map(str, ( domain, entry['fuzzer'], entry['domain-name'], ip, error, )) yield ','.join(row) + '\n'
def json_render(qry_domains): """Render and return the json-formatted report.""" reports = dict(filter(None, map(tools.analyse, qry_domains))) for rept in reports.values(): for entry in rept['fuzzy_domains']: ip, error = tools.resolve(entry['domain-name']) entry['resolution'] = { 'ip': ip, 'error': error, } return flask.json.jsonify(reports)
def generate(): """Streaming download generator.""" yield ','.join(headers) + '\n' for (domain, rept) in reports.items(): for entry in rept['fuzzy_domains']: ip_addr, error = tools.resolve(entry['domain-name']) row = map(str, ( domain, entry['fuzzer'], entry['domain-name'], ip_addr, error, )) yield ','.join(row) + '\n'
def resolve_ip(hexdomain): """Resolves Domains to IPs.""" domain = tools.parse_domain(hexdomain) if domain is None: flask.abort( 400, 'Malformed domain or domain not represented in hexadecimal format.' ) ip_addr, error = tools.resolve(domain) payload = standard_api_values(domain, skip='resolve_ip') payload['ip'] = ip_addr payload['error'] = error return flask.jsonify(payload)
def resolve_ip(hexdomain): """Resolves Domains to IPs.""" domain = tools.parse_domain(hexdomain) if domain is None: flask.abort( 400, 'Malformed domain or domain not represented in hexadecimal format.' ) ip, error = tools.resolve(domain) payload = standard_api_values(domain, skip='resolve_ip') payload['ip'] = ip payload['error'] = error return flask.jsonify(payload)
def generate(): """Streaming download generator.""" yield ','.join(headers) + '\n' for (domain, rept) in reports.items(): for entry in rept['fuzzy_domains']: ip_addr, error = tools.resolve(entry['domain-name']) row = ( domain.encode('idna'), entry['fuzzer'], entry['domain-name'].encode('idna'), str(ip_addr), str(error), ) # comma not possible in any of the row values. yield u','.join(row) + '\n'
def generate(): """Streaming download generator.""" indent_size = 4 indent = ' ' * indent_size yield '{\n' for (i, (dom, rept)) in enumerate(reports.items()): yield indent + '"' + dom + '": {\n' yield indent * 2 + '"fuzzy_domains": [\n' fuzzy_domains = rept['fuzzy_domains'] for (j, entry) in enumerate(fuzzy_domains): ip_addr, error = tools.resolve(entry['domain-name']) data = { 'domain-name': entry['domain-name'], 'fuzzer': entry['fuzzer'], 'hex': entry['hex'], 'resolution': { 'error': error, 'ip': ip_addr, }, } json_str = json.dumps( data, sort_keys=True, indent=indent_size, separators=(',', ': ') ) yield '\n'.join([indent * 3 + line for line in json_str.split('\n')]) if j < len(fuzzy_domains) - 1: yield ',' yield '\n' yield indent * 2 + ']\n' yield indent + '}' if i < len(reports) - 1: yield ',' yield '\n' yield '}\n'
def generate(): """Streaming download generator.""" indent_size = 4 indent = ' ' * indent_size yield '{\n' for (i, (dom, rept)) in enumerate(reports.items()): yield indent + '"' + dom + '": {\n' yield indent * 2 + '"fuzzy_domains": [\n' fuzzy_domains = rept['fuzzy_domains'] for (j, entry) in enumerate(fuzzy_domains): ip_addr, error = tools.resolve(entry['domain-name']) data = { 'domain-name': entry['domain-name'], 'fuzzer': entry['fuzzer'], 'hex': entry['hex'], 'resolution': { 'error': error, 'ip': ip_addr, }, } json_str = json.dumps(data, sort_keys=True, indent=indent_size, separators=(',', ': ')) yield '\n'.join( [indent * 3 + line for line in json_str.split('\n')]) if j < len(fuzzy_domains) - 1: yield ',' yield '\n' yield indent * 2 + ']\n' yield indent + '}' if i < len(reports) - 1: yield ',' yield '\n' yield '}\n'
def process_domain(domain): """Process a domain - generating resolution reports and deltas.""" if dnstwist.validate_domain(domain) is None: print 'Invalid: {}'.format(repr(domain)) repository.unregister_domain(domain) return # Unregister long-time unread domains last_read = repository.delta_report_last_read(domain) if last_read is None: repository.mark_delta_report_as_read(domain) else: age = datetime.datetime.now() - last_read if age > datetime.timedelta(seconds=PERIOD * UNREGISTER): print 'Expired: {}'.format(domain.encode('idna')) repository.unregister_domain(domain) return # Skip domains that have been recently updated delta_last_updated = repository.delta_report_updated(domain) if delta_last_updated is not None: age = datetime.datetime.now() - delta_last_updated if age < datetime.timedelta(seconds=PERIOD): print 'Skipping: {}'.format(domain.encode('idna')) return start = time.time() existing_report = repository.get_resolution_report(domain) if existing_report is None: existing_report = {} new_report = {} for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]: ip_addr, error = tools.resolve(entry['domain-name']) if error or not ip_addr or ip_addr is None: continue new_report[entry['domain-name']] = { 'ip': ip_addr, 'tweak': entry['fuzzer'], } repository.update_resolution_report(domain, new_report) delta_report = {'new': [], 'updated': [], 'deleted': []} for (dom, data) in new_report.items(): try: new_ip = data['ip'] except TypeError: # handle old-style ip-only reports new_ip = data if dom in existing_report.keys(): try: existing_ip = existing_report[dom]['ip'] except TypeError: # handle old-style ip-only reports existing_ip = existing_report[dom] if new_ip != existing_ip: delta_report['updated'].append((dom, existing_ip, new_ip)) else: delta_report['new'].append((dom, new_ip)) for dom in existing_report.keys(): if dom not in new_report.keys(): delta_report['deleted'].append(dom) repository.update_delta_report(domain, delta_report) print 'Updated {} in {} seconds'.format(domain.encode('idna'), time.time() - start)
def test_resolve_validation(): """Resolve validates first.""" assert tools.resolve(',saoi9w3k490q2k4') == (False, True)
def local_resolve_candidate(candidate): domain = Domain(candidate['domain-name']) ip_addr, error = tools.resolve(domain) return candidate['fuzzer'], domain, ip_addr, error
def process_domain(domain): """Process a domain - generating resolution reports and deltas.""" if dnstwist.validate_domain(domain) is None: print 'Unregistering (invalid) {}'.format(domain) repository.unregister_domain(domain) return # Unregister long-time unread domains last_read = repository.delta_report_last_read(domain) if last_read is None: repository.mark_delta_report_as_read(domain) else: age = datetime.datetime.now() - last_read if age > datetime.timedelta(seconds=PERIOD*UNREGISTER): print 'Unregistering (not read > 7 days) {}'.format(domain) repository.unregister_domain(domain) return # Skip domains that have been recently updated delta_last_updated = repository.delta_report_updated(domain) if delta_last_updated is not None: age = datetime.datetime.now() - delta_last_updated if age < datetime.timedelta(seconds=PERIOD): print 'Skipping (recently updated) {}'.format(domain) return start = time.time() existing_report = repository.get_resolution_report(domain) if existing_report is None: existing_report = {} new_report = {} for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]: ip, error = tools.resolve(entry['domain-name']) if error or not ip or ip is None: continue new_report[entry['domain-name']] = { 'ip': ip, 'tweak': entry['fuzzer'], } repository.update_resolution_report(domain, new_report) delta_report = {'new': [], 'updated': [], 'deleted': []} for (dom, data) in new_report.items(): try: new_ip = data['ip'] except TypeError: # handle old-style ip-only reports new_ip = data if dom in existing_report.keys(): try: existing_ip = existing_report[dom]['ip'] except TypeError: # handle old-style ip-only reports existing_ip = existing_report[dom] if new_ip != existing_ip: delta_report['updated'].append( (dom, existing_ip, new_ip) ) else: delta_report['new'].append((dom, new_ip)) for dom in existing_report.keys(): if dom not in new_report.keys(): delta_report['deleted'].append(dom) repository.update_delta_report(domain, delta_report) print 'Updated deltas for {} in {} seconds'.format( domain, time.time() - start )