def generate():
"""Streaming download generator."""
for (domain, rept) in reports.items():
for entry in rept['fuzzy_domains']:
ip, error = tools.resolve(entry['domain-name'])
row = map(str, (
domain,
entry['fuzzer'],
entry['domain-name'],
ip,
error,
))
yield ','.join(row) + '\n'
def json_render(qry_domains):
"""Render and return the json-formatted report."""
reports = dict(filter(None, map(tools.analyse, qry_domains)))
for rept in reports.values():
for entry in rept['fuzzy_domains']:
ip, error = tools.resolve(entry['domain-name'])
entry['resolution'] = {
'ip': ip,
'error': error,
}
return flask.json.jsonify(reports)
def generate():
"""Streaming download generator."""
yield ','.join(headers) + '\n'
for (domain, rept) in reports.items():
for entry in rept['fuzzy_domains']:
ip_addr, error = tools.resolve(entry['domain-name'])
row = map(str, (
domain,
entry['fuzzer'],
entry['domain-name'],
ip_addr,
error,
))
yield ','.join(row) + '\n'
def resolve_ip(hexdomain):
"""Resolves Domains to IPs."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
ip_addr, error = tools.resolve(domain)
payload = standard_api_values(domain, skip='resolve_ip')
payload['ip'] = ip_addr
payload['error'] = error
return flask.jsonify(payload)
def resolve_ip(hexdomain):
"""Resolves Domains to IPs."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
ip, error = tools.resolve(domain)
payload = standard_api_values(domain, skip='resolve_ip')
payload['ip'] = ip
payload['error'] = error
return flask.jsonify(payload)
def generate():
"""Streaming download generator."""
yield ','.join(headers) + '\n'
for (domain, rept) in reports.items():
for entry in rept['fuzzy_domains']:
ip_addr, error = tools.resolve(entry['domain-name'])
row = (
domain.encode('idna'),
entry['fuzzer'],
entry['domain-name'].encode('idna'),
str(ip_addr),
str(error),
)
# comma not possible in any of the row values.
yield u','.join(row) + '\n'
def generate():
"""Streaming download generator."""
indent_size = 4
indent = ' ' * indent_size
yield '{\n'
for (i, (dom, rept)) in enumerate(reports.items()):
yield indent + '"' + dom + '": {\n'
yield indent * 2 + '"fuzzy_domains": [\n'
fuzzy_domains = rept['fuzzy_domains']
for (j, entry) in enumerate(fuzzy_domains):
ip_addr, error = tools.resolve(entry['domain-name'])
data = {
'domain-name': entry['domain-name'],
'fuzzer': entry['fuzzer'],
'hex': entry['hex'],
'resolution': {
'error': error,
'ip': ip_addr,
},
}
json_str = json.dumps(
data,
sort_keys=True,
indent=indent_size,
separators=(',', ': ')
)
yield '\n'.join([indent * 3 + line
for line
in json_str.split('\n')])
if j < len(fuzzy_domains) - 1:
yield ','
yield '\n'
yield indent * 2 + ']\n'
yield indent + '}'
if i < len(reports) - 1:
yield ','
yield '\n'
yield '}\n'
def generate():
"""Streaming download generator."""
indent_size = 4
indent = ' ' * indent_size
yield '{\n'
for (i, (dom, rept)) in enumerate(reports.items()):
yield indent + '"' + dom + '": {\n'
yield indent * 2 + '"fuzzy_domains": [\n'
fuzzy_domains = rept['fuzzy_domains']
for (j, entry) in enumerate(fuzzy_domains):
ip_addr, error = tools.resolve(entry['domain-name'])
data = {
'domain-name': entry['domain-name'],
'fuzzer': entry['fuzzer'],
'hex': entry['hex'],
'resolution': {
'error': error,
'ip': ip_addr,
},
}
json_str = json.dumps(data,
sort_keys=True,
indent=indent_size,
separators=(',', ': '))
yield '\n'.join(
[indent * 3 + line for line in json_str.split('\n')])
if j < len(fuzzy_domains) - 1:
yield ','
yield '\n'
yield indent * 2 + ']\n'
yield indent + '}'
if i < len(reports) - 1:
yield ','
yield '\n'
yield '}\n'
def process_domain(domain):
"""Process a domain - generating resolution reports and deltas."""
if dnstwist.validate_domain(domain) is None:
print 'Invalid: {}'.format(repr(domain))
repository.unregister_domain(domain)
return
# Unregister long-time unread domains
last_read = repository.delta_report_last_read(domain)
if last_read is None:
repository.mark_delta_report_as_read(domain)
else:
age = datetime.datetime.now() - last_read
if age > datetime.timedelta(seconds=PERIOD * UNREGISTER):
print 'Expired: {}'.format(domain.encode('idna'))
repository.unregister_domain(domain)
return
# Skip domains that have been recently updated
delta_last_updated = repository.delta_report_updated(domain)
if delta_last_updated is not None:
age = datetime.datetime.now() - delta_last_updated
if age < datetime.timedelta(seconds=PERIOD):
print 'Skipping: {}'.format(domain.encode('idna'))
return
start = time.time()
existing_report = repository.get_resolution_report(domain)
if existing_report is None:
existing_report = {}
new_report = {}
for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]:
ip_addr, error = tools.resolve(entry['domain-name'])
if error or not ip_addr or ip_addr is None:
continue
new_report[entry['domain-name']] = {
'ip': ip_addr,
'tweak': entry['fuzzer'],
}
repository.update_resolution_report(domain, new_report)
delta_report = {'new': [], 'updated': [], 'deleted': []}
for (dom, data) in new_report.items():
try:
new_ip = data['ip']
except TypeError:
# handle old-style ip-only reports
new_ip = data
if dom in existing_report.keys():
try:
existing_ip = existing_report[dom]['ip']
except TypeError:
# handle old-style ip-only reports
existing_ip = existing_report[dom]
if new_ip != existing_ip:
delta_report['updated'].append((dom, existing_ip, new_ip))
else:
delta_report['new'].append((dom, new_ip))
for dom in existing_report.keys():
if dom not in new_report.keys():
delta_report['deleted'].append(dom)
repository.update_delta_report(domain, delta_report)
print 'Updated {} in {} seconds'.format(domain.encode('idna'),
time.time() - start)
def test_resolve_validation():
"""Resolve validates first."""
assert tools.resolve(',saoi9w3k490q2k4') == (False, True)
def local_resolve_candidate(candidate):
domain = Domain(candidate['domain-name'])
ip_addr, error = tools.resolve(domain)
return candidate['fuzzer'], domain, ip_addr, error
def process_domain(domain):
"""Process a domain - generating resolution reports and deltas."""
if dnstwist.validate_domain(domain) is None:
print 'Unregistering (invalid) {}'.format(domain)
repository.unregister_domain(domain)
return
# Unregister long-time unread domains
last_read = repository.delta_report_last_read(domain)
if last_read is None:
repository.mark_delta_report_as_read(domain)
else:
age = datetime.datetime.now() - last_read
if age > datetime.timedelta(seconds=PERIOD*UNREGISTER):
print 'Unregistering (not read > 7 days) {}'.format(domain)
repository.unregister_domain(domain)
return
# Skip domains that have been recently updated
delta_last_updated = repository.delta_report_updated(domain)
if delta_last_updated is not None:
age = datetime.datetime.now() - delta_last_updated
if age < datetime.timedelta(seconds=PERIOD):
print 'Skipping (recently updated) {}'.format(domain)
return
start = time.time()
existing_report = repository.get_resolution_report(domain)
if existing_report is None:
existing_report = {}
new_report = {}
for entry in tools.analyse(domain)[1]['fuzzy_domains'][1:]:
ip, error = tools.resolve(entry['domain-name'])
if error or not ip or ip is None:
continue
new_report[entry['domain-name']] = {
'ip': ip,
'tweak': entry['fuzzer'],
}
repository.update_resolution_report(domain, new_report)
delta_report = {'new': [], 'updated': [], 'deleted': []}
for (dom, data) in new_report.items():
try:
new_ip = data['ip']
except TypeError:
# handle old-style ip-only reports
new_ip = data
if dom in existing_report.keys():
try:
existing_ip = existing_report[dom]['ip']
except TypeError:
# handle old-style ip-only reports
existing_ip = existing_report[dom]
if new_ip != existing_ip:
delta_report['updated'].append(
(dom, existing_ip, new_ip)
)
else:
delta_report['new'].append((dom, new_ip))
for dom in existing_report.keys():
if dom not in new_report.keys():
delta_report['deleted'].append(dom)
repository.update_delta_report(domain, delta_report)
print 'Updated deltas for {} in {} seconds'.format(
domain, time.time() - start
)