def test_parse_domain(self):
"""Tests of the helper that decodes and validates a domain.
Function returns a valid domain or None.
"""
self.assertIs(None, tools.parse_domain(''),
'Missing hex data should return None')
self.assertIs(None, tools.parse_domain(None),
'Non-hex-decodable data should return None')
self.assertIs(None, tools.parse_domain('he378a -- ?'),
'Non-hex-decodable data should return None')
bad_domain = '\\www.z.comasfff'
self.assertFalse(dnstwist.validate_domain(bad_domain),
'Bad domain should be invalid')
bad_domain_data = binascii.hexlify(bad_domain)
self.assertIs(
None, tools.parse_domain(bad_domain_data),
'hex-decodable (but invalid) domain data should return None')
domain = 'www.example.com'
self.assertTrue(dnstwist.validate_domain(domain),
'Good domain should be valid')
domain_data = binascii.hexlify(domain)
self.assertEqual('www.example.com', tools.parse_domain(domain_data),
'hex-decodable valid domain data should be returned')
def email_subscribe_pending_confirm(hexdomain):
"""Send a confirmation email for a user."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(400, 'Malformed domain or domain not represented in hexadecimal format.')
email_address = flask.request.form['email_address']
if email_address.strip() == '':
return flask.redirect('/email/subscribe/{}/0'.format(hexdomain))
verify_code = tools.random_id()
verify_url = flask.request.url_root + 'email/verify/{}'.format(verify_code)
email_body = email_tools.render_email(
'confirm.html',
domain=domain,
verify_url=verify_url
)
repository.propose_subscription(verify_code, email_address, domain)
emailer.send(
email_address, 'Please verify your subscription', email_body
)
return flask.render_template('www/email/pending_verify.html', domain=domain)
def index(error_arg=None):
"""Main page."""
error = None
error_idx = None
suggestion = None
try:
if error_arg is not None:
error_idx = int(error_arg)
if error_idx >= 0:
error = ERRORS[error_idx]
except (TypeError, ValueError, IndexError):
# This will fail an error index that can't be converted to an
# integer and an error index that can be converted to an integer
# but is not within the range of the tuple of errors.
app.logger.info(
'Invalid value passed for error index: {}'.format(error_idx)
)
if error_idx == 0:
encoded_suggestion = flask.request.args.get('suggestion')
suggestion = tools.parse_domain(encoded_suggestion)
return flask.render_template(
'www/index.html', error=error, suggestion=suggestion
)
def email_subscribe_pending_confirm(hexdomain):
"""Send a confirmation email for a user."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
hide_noisy = bool(flask.request.form.get('hide_noisy'))
email_address = flask.request.form['email_address']
if email_address.strip() == '':
return flask.redirect('/email/subscribe/{}/0?hide_noisy={}'.format(
hexdomain, hide_noisy))
verify_code = tools.random_id()
verify_url = flask.request.url_root + 'email/verify/{}'.format(verify_code)
email_body = email_tools.render_email('confirm.html',
domain=domain,
verify_url=verify_url)
repository.propose_subscription(verify_code, email_address, domain,
hide_noisy)
emailer.send(email_address, 'Please verify your subscription', email_body)
return flask.render_template('www/email/pending_verify.html',
domain=domain)
def email_subscribe_get_email(hexdomain, error=None):
"""Handle subscriptions."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
# Attempt to parse out a validation error.
error_str = None
try:
if error is not None:
error_idx = int(error)
if error_idx >= 0:
error_str = ERRORS[error_idx]
except:
app.logger.info('Invalid error index {}'.format(error))
return flask.render_template(
'www/email/subscribe.html',
domain=domain,
hexdomain=hexdomain,
error=error_str,
hide_noisy=flask.request.args.get('hide_noisy') == 'True')
def domain_to_hex(domain):
"""Helps you convert domains to hex."""
hexdomain = binascii.hexlify(domain)
if tools.parse_domain(hexdomain) is None:
flask.abort(400, 'Malformed domain.')
payload = standard_api_values(domain, skip='domain_to_hex')
payload['domain_as_hexadecimal'] = hexdomain
return flask.jsonify(payload)
def domain_to_hex(domain):
"""Helps you convert domains to hex."""
hexdomain = tools.encode_domain(domain)
if tools.parse_domain(hexdomain) is None:
flask.abort(400, 'Malformed domain.')
payload = standard_api_values(domain, skip='domain_to_hex')
payload['domain_as_hexadecimal'] = hexdomain
return flask.jsonify(payload)
def safebrowsing(hexdomain):
"""Returns number of hits in Google Safe Browsing."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
payload = standard_api_values(domain, skip='safebrowsing')
payload['issue_detected'] = checks.safebrowsing.get_report(domain) != 0
return flask.jsonify(payload)
def safebrowsing_check(hexdomain):
"""Returns number of hits in Google Safe Browsing."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
payload = standard_api_values(domain, skip='safebrowsing')
payload['issue_detected'] = safebrowsing.get_report(domain) != 0
return flask.jsonify(payload)
def analyse(hexdomain):
"""Do a domain analysis."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
return flask.render_template(
'www/analyse.html',
domain=domain,
hexdomain=hexdomain,
)
def resolve_ip(hexdomain):
"""Resolves Domains to IPs."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
ip, error = tools.resolve(domain)
payload = standard_api_values(domain, skip='resolve_ip')
payload['ip'] = ip
payload['error'] = error
return flask.jsonify(payload)
def parked_score(hexdomain):
"""Calculates "parked" scores from 0-1."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
payload = standard_api_values(domain, skip='parked_score')
score, score_text, redirects, dest = parked.get_score(domain)
payload['score'] = score
payload['score_text'] = score_text
payload['redirects'] = redirects
payload['redirects_to'] = dest
return flask.jsonify(payload)
def resolve_ip(hexdomain):
"""Resolves Domains to IPs."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
ip_addr, error = tools.resolve(domain)
payload = standard_api_values(domain, skip='resolve_ip')
payload['ip'] = ip_addr
payload['error'] = error
return flask.jsonify(payload)
def search_async():
"""Chunked endpoint supporting search."""
encoded_domain_parameter = flask.request.args.get('ed')
domain_parameter = tools.parse_domain(encoded_domain_parameter)
if domain_parameter is None:
return handle_invalid_domain(encoded_domain_parameter)
return flask.render_template(
'www/search.html',
domain=domain_parameter,
exports={
'json': 'json',
'csv': 'csv'
},
)
def parked_score(hexdomain):
"""Calculates "parked" scores from 0-1."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
payload = standard_api_values(domain, skip='parked_score')
score, score_text, redirects, dressed, dest = parked.get_score(domain)
payload['score'] = score
payload['score_text'] = score_text
payload['redirects'] = redirects
payload['redirects_to'] = dest
payload['dressed'] = dressed
return flask.jsonify(payload)
def fuzz_chunked(hexdomain):
"""Return a chunked json fuzz based on jsonpipe by eBay."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
def generate():
for result in tools.fuzzy_domains_iter(domain):
domain_result = result.domain
yield json.dumps({
'd': domain_result,
'ed': tools.encode_domain(domain_result),
'pd': domain_result.encode('idna')
}) + '\n\n'
return flask.Response(generate())
def fuzz(hexdomain):
"""Calculates the dnstwist "fuzzy domains" for a domain."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
fuzz_result = tools.fuzzy_domains(domain)
fuzz_payload = []
for result in fuzz_result:
result_payload = standard_api_values(result['domain-name'], skip='url')
result_payload['fuzzer'] = result['fuzzer']
fuzz_payload.append(result_payload)
payload = standard_api_values(domain, skip='fuzz')
payload['fuzzy_domains'] = fuzz_payload
return flask.jsonify(payload)
def whois(hexdomain):
"""Returns whois information."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
payload = standard_api_values(domain, skip='whois')
try:
payload['whois_text'] = whois_mod.whois(domain).text.strip()
if payload['whois_text'] == '':
raise Exception('No whois data retrieved')
except Exception as ex:
current_app.logger.error(
'Unable to retrieve whois info for domain: {}'.format(ex))
flask.abort(500, 'Unable to retrieve whois info')
return flask.jsonify(payload)
def fuzz(hexdomain):
"""Calculates the dnstwist "fuzzy domains" for a domain."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
fuzz_result = tools.fuzzy_domains(domain)
fuzz_payload = []
for result in fuzz_result:
result_payload = standard_api_values(result['domain-name'], skip='url')
result_payload['fuzzer'] = result['fuzzer']
fuzz_payload.append(result_payload)
payload = standard_api_values(domain, skip='fuzz')
payload['fuzzy_domains'] = fuzz_payload
return flask.jsonify(payload)
def test_parse_domain(self):
"""Tests of the helper that decodes and validates a domain.
Function returns a valid domain or None.
"""
self.assertIs(
None, tools.parse_domain(''),
'Missing hex data should return None'
)
self.assertIs(
None, tools.parse_domain(None),
'Non-hex-decodable data should return None'
)
self.assertIs(
None, tools.parse_domain('he378a -- ?'),
'Non-hex-decodable data should return None'
)
bad_domain = '\\www.z.comasfff'
self.assertFalse(
dnstwist.validate_domain(bad_domain),
'Bad domain should be invalid'
)
bad_domain_data = binascii.hexlify(bad_domain)
self.assertIs(
None, tools.parse_domain(bad_domain_data),
'hex-decodable (but invalid) domain data should return None'
)
domain = 'www.example.com'
self.assertTrue(
dnstwist.validate_domain(domain),
'Good domain should be valid'
)
domain_data = binascii.hexlify(domain)
self.assertEqual(
'www.example.com',
tools.parse_domain(domain_data),
'hex-decodable valid domain data should be returned'
)
domain_data = base64.b64encode(domain)
self.assertEqual(
'www.example.com',
tools.parse_domain(domain_data),
'Old b64-style domain data is also processable.'
)
def search_async():
"""New endpoint supporting async search."""
encoded_domain_parameter = flask.request.args.get('ed')
domain_parameter = tools.parse_domain(encoded_domain_parameter)
if domain_parameter is None:
return handle_invalid_domain(encoded_domain_parameter)
if not features.enable_async_search():
return flask.redirect('/search/{}'.format(encoded_domain_parameter))
return flask.render_template(
'www/search.html',
domain=domain_parameter,
exports={
'json': 'json',
'csv': 'csv'
},
allow_email_subs=features.enable_emails(),
)
def whois(hexdomain):
"""Returns whois information."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
payload = standard_api_values(domain, skip='whois')
try:
payload['whois_text'] = whois_mod.whois(domain).text
if payload['whois_text'].strip() == '':
raise Exception('No whois data retrieved')
except Exception as ex:
current_app.logger.error(
'Unable to retrieve whois info for domain: {}'.format(ex)
)
flask.abort(500, 'Unable to retrieve whois info')
return flask.jsonify(payload)
def email_subscribe_get_email(hexdomain, error=None):
"""Handle subscriptions."""
domain = tools.parse_domain(hexdomain)
if domain is None:
flask.abort(400, 'Malformed domain or domain not represented in hexadecimal format.')
# Attempt to parse out a validation error.
error_str = None
try:
if error is not None:
error_idx = int(error)
assert error_idx >= 0
error_str = ERRORS[error_idx]
except:
app.logger.info(
'Invalid error index {}'.format(error)
)
return flask.render_template(
'www/email/subscribe.html',
domain=domain,
hexdomain=hexdomain,
error=error_str,
)
def view(hexdomain):
"""Return new atom items for changes in resolved domains."""
# Parse out the requested domain
domain = tools.parse_domain(hexdomain)
# Redirect old base64 requests to the new format.
if domain is None:
redirect_url = _base64_redirect(hexdomain)
if redirect_url is not None:
return flask.redirect(redirect_url, code=302)
flask.abort(
400,
'Malformed domain or domain not represented in hexadecimal format.'
)
# Prepare a feed
feed = werkzeug.contrib.atom.AtomFeed(
title=u'dnstwister report for {}'.format(
template_tools.domain_renderer(domain)
),
feed_url='{}atom/{}'.format(flask.request.url_root, hexdomain),
url='{}search/{}'.format(flask.request.url_root, hexdomain),
)
# The publish/update date for the placeholder is locked to 00:00:00.000
# (midnight UTC) on the current day.
today = datetime.datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0
)
# Ensure the domain is registered.
if not repository.is_domain_registered(domain):
repository.register_domain(domain)
# Retrieve the delta report
delta_report = repository.get_delta_report(domain)
# If we don't have a delta report yet, show the placeholder.
if delta_report is None:
feed.add(
title=u'No report yet for {}'.format(template_tools.domain_renderer(domain)),
title_type='text',
content=flask.render_template(
'syndication/atom/placeholder.html', domain=domain
),
content_type='html',
author='dnstwister',
updated=today,
published=today,
id=u'waiting:{}'.format(template_tools.domain_renderer(domain)),
url=feed.url,
)
else:
# If there is a delta report, generate the feed and return it. We use
# the actual date of generation here.
updated = repository.delta_report_updated(domain)
if updated is None:
updated = today
# Setting the ID to be epoch seconds, floored per 24 hours, ensure the
# updates are only every 24 hours max.
id_24hr = (updated - datetime.datetime(1970, 1, 1)).total_seconds()
common_kwargs = {
'title_type': 'text',
'content_type': 'html',
'author': 'dnstwister',
'updated': updated,
'published': updated,
'url': feed.url,
}
for (dom, ip) in delta_report['new']:
feed.add(
title=u'NEW: {}'.format(template_tools.domain_renderer(dom)),
content=flask.render_template(
'syndication/atom/new.html',
ip=ip, hexdomain=tools.encode_domain(dom)
),
id='new:{}:{}:{}'.format(dom.encode('idna'), ip, id_24hr),
**common_kwargs
)
for (dom, old_ip, new_ip) in delta_report['updated']:
feed.add(
title=u'UPDATED: {}'.format(template_tools.domain_renderer(dom)),
content=flask.render_template(
'syndication/atom/updated.html',
new_ip=new_ip, old_ip=old_ip,
hexdomain=tools.encode_domain(dom),
),
id='updated:{}:{}:{}:{}'.format(
dom.encode('idna'),
old_ip,
new_ip,
id_24hr
),
**common_kwargs
)
for dom in delta_report['deleted']:
feed.add(
title=u'DELETED: {}'.format(template_tools.domain_renderer(dom)),
content=flask.render_template(
'syndication/atom/deleted.html',
),
id='deleted:{}:{}'.format(dom.encode('idna'), id_24hr),
**common_kwargs
)
feed_response = feed.get_response()
repository.mark_delta_report_as_read(domain)
return feed_response
