def test_dont_send_too_often(capsys, monkeypatch):
"""Test that emails are not sent more than every 24 hours."""
# Patches
monkeypatch.setattr(
'dnstwister.repository.db',
patches.SimpleKVDatabase()
)
monkeypatch.setattr(
'dnstwister.repository.statistics.db',
patches.SimpleKVDatabase()
)
monkeypatch.setattr(
'dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer
)
monkeypatch.setattr(
'dnstwister.tools.resolve',
lambda domain: ('999.999.999.999', False)
)
emailer = patches.NoEmailer()
monkeypatch.setattr(
'workers.email.emailer',
emailer
)
repository = dnstwister.repository
# Settings
domain = 'www.example.com'
sub_id = '1234'
email = '*****@*****.**'
# Subscribe a new user.
repository.subscribe_email(sub_id, email, domain)
# Do a delta report.
workers.deltas.process_domain(domain)
# Process the subscription.
sub_data = repository.db.data['email_sub:{}'.format(sub_id)]
workers.email.process_sub(sub_id, sub_data)
# And we've sent an email.
assert len(emailer.sent_emails) == 1
# Re-process immediately
workers.email.process_sub(sub_id, sub_data)
# Check we haven't sent two emails
assert len(emailer.sent_emails) == 1
def test_updated_ip_is_noted_in_delta(capsys, monkeypatch):
"""IP updates are marked as 'updated'."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
repository = dnstwister.repository
domain = u'www.\u0454xample.com'
workers.deltas.process_domain(domain)
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('000.999.999.999', False))
# Mark the entry as 'old' to ensure it is re-run
old_date = datetime.datetime.now() - datetime.timedelta(days=10)
db_key = u'delta_report_updated:{}'.format(domain)
repository.db._data[db_key] = old_date.strftime('%Y-%m-%dT%H:%M:%SZ')
workers.deltas.process_domain(domain)
delta_report = repository.get_delta_report(domain)
assert delta_report == {
'deleted': [],
'updated':
[(u'www.\u0454xample.co', '999.999.999.999', '000.999.999.999')],
'new': []
}
def test_dont_send_when_no_changes(capsys, monkeypatch):
"""Test that emails are not sent when there are no changes."""
# Patches
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr(
'dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer
)
# Return no IPs
monkeypatch.setattr(
'dnstwister.tools.resolve', lambda domain: (False, False)
)
emailer = patches.NoEmailer()
monkeypatch.setattr('workers.email.emailer', emailer)
repository = dnstwister.repository
# Settings
domain = 'www.example.com'
sub_id = '1234'
email = '*****@*****.**'
# Subscribe a new user.
repository.subscribe_email(sub_id, email, domain)
# Do a delta report.
workers.deltas.process_domain(domain)
# Process the subscription.
sub_data = repository.db.data['email_sub:{}'.format(sub_id)]
workers.email.process_sub(sub_id, sub_data)
# We've not sent any emails as there were no changes (no IPs resolved at
# all).
assert len(emailer.sent_emails) == 0
def test_all_changes_are_new_on_first_delta(capsys, monkeypatch):
"""The first delta marks all resolved domains as 'new'."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
repository = dnstwister.repository
domain = u'www.\u0454xample.com'
workers.deltas.process_domain(domain)
assert capsys.readouterr()[0] != ''
resolution_report = repository.get_resolution_report(domain)
assert resolution_report == {
u'www.\u0454xample.co': {
'ip': '999.999.999.999',
'tweak': 'Pretend'
}
}
delta_report = repository.get_delta_report(domain)
assert delta_report == {
'deleted': [],
'new': [(u'www.\u0454xample.co', '999.999.999.999')],
'updated': []
}
def set_up_mocks(monkeypatch):
mock_db = patches.SimpleKVDatabase()
monkeypatch.setattr('dnstwister.repository.statistics.db', mock_db)
monkeypatch.setattr('dnstwister.repository.db', mock_db)
data_repository = dnstwister.repository
statistics_repository = dnstwister.repository.statistics
return data_repository, statistics_repository
def test_pre_noisy_domain_subscriptions_default_to_off(capsys, monkeypatch):
"""Test that subscriptions prior to noisy domains existing are supported
still but default to 'off'.
"""
# Patches
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
# Enable noisy domains functionality
monkeypatch.setenv('feature.noisy_domains', 'true')
# Ensure the fake redis will work.
monkeypatch.setenv('REDIS_URL', '')
# Return a result
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
emailer = patches.NoEmailer()
monkeypatch.setattr('workers.email.emailer', emailer)
repository = dnstwister.repository
# Settings
domain = 'www.example.com'
sub_id = '1234'
email = '*****@*****.**'
# Mark the found domain as also noisy.
store = dnstwister.stats_store
for _ in range(5):
store.note('www.example.co')
# Subscribe a new user, make them look like the pre-feature style.
repository.subscribe_email(sub_id, email, domain, True)
del (repository.db._data['email_sub:1234']['hide_noisy'])
# Do a delta report.
workers.deltas.process_domain(domain)
# Process the subscription.
sub_data = repository.db.data['email_sub:{}'.format(sub_id)]
workers.email.process_sub(sub_id, sub_data)
# We have an email.
assert len(emailer.sent_emails) == 1
# And there is no reference to noisy domains in the email.
assert 'noisy' not in emailer.sent_emails[0][2]
def test_dont_send_when_only_noisy(capsys, monkeypatch):
"""Test that emails are not sent when no non-noisy domains exist in
delta report.
"""
# Patches
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
# Ensure the fake redis will work.
monkeypatch.setenv('REDIS_URL', '')
# Return a result
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
# Enable noisy domains functionality
monkeypatch.setenv('feature.noisy_domains', 'true')
emailer = patches.NoEmailer()
monkeypatch.setattr('workers.email.emailer', emailer)
repository = dnstwister.repository
# Settings
domain = 'www.example.com'
sub_id = '1234'
email = '*****@*****.**'
# Mark the found domain as also noisy.
store = dnstwister.stats_store
for _ in range(5):
store.note('www.example.co')
# Subscribe a new user, with noisy filtering enabled.
repository.subscribe_email(sub_id, email, domain, True)
# Do a delta report.
workers.deltas.process_domain(domain)
# Process the subscription.
sub_data = repository.db.data['email_sub:{}'.format(sub_id)]
workers.email.process_sub(sub_id, sub_data)
# We've not sent any emails as there were no changes (no IPs resolved at
# all).
assert len(emailer.sent_emails) == 0
def test_domains_iter_lists_all_domains(capsys, monkeypatch):
"""Test the repo can return all domains registered."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
domain = u'www.\u0454xample.com'
repository = dnstwister.repository
assert list(repository.iregistered_domains()) == []
repository.register_domain(domain)
assert list(repository.iregistered_domains()) == [domain]
def test_new_domain_is_marked_as_read(capsys, monkeypatch):
"""New domain is marked as read at first pass."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
repository = dnstwister.repository
domain = u'www.\u0454xample.com'
assert repository.delta_report_updated(domain) is None
workers.deltas.process_domain(domain)
assert repository.delta_report_updated(domain) is not None
assert capsys.readouterr()[0] != ''
def test_invalid_domain_is_unregistered(capsys, monkeypatch):
"""Invalid domains are tidied up."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
repository = dnstwister.repository
invalid_domain = '3j88??ASd'
assert not repository.is_domain_registered(invalid_domain)
repository.register_domain(invalid_domain)
assert repository.is_domain_registered(invalid_domain)
workers.deltas.process_domain(invalid_domain)
assert not repository.is_domain_registered(invalid_domain)
expected_output = 'Invalid: \'{}\'\n'.format(invalid_domain.encode('idna'))
assert capsys.readouterr()[0] == expected_output
def test_old_domain_is_unregistered(capsys, monkeypatch):
"""Long-time unread domains are unregistered."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
repository = dnstwister.repository
domain = u'www.\u0454xample.com'
old_date = datetime.datetime.now() - datetime.timedelta(days=10)
assert repository.delta_report_last_read(domain) is None
repository.mark_delta_report_as_read(domain, old_date)
assert repository.delta_report_last_read(domain) == old_date.replace(
microsecond=0)
workers.deltas.process_domain(domain)
assert repository.delta_report_last_read(domain) is None
expected_output = 'Expired: {}\n'.format(domain.encode('idna'))
assert capsys.readouterr()[0] == expected_output
def test_domains_are_checked_once_a_day(capsys, monkeypatch):
"""Test domains are not checked constantly."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
repository = dnstwister.repository
domain = u'www.\u0454xample.com'
workers.deltas.process_domain(domain)
last_updated_db_key = u'delta_report_updated:{}'.format(domain)
last_updated = repository.db._data[last_updated_db_key]
# Process again not long after.
time.sleep(2)
workers.deltas.process_domain(domain)
# Ensure that we didn't updated the last-updated date
assert repository.db._data[last_updated_db_key] == last_updated
def test_old_style_resolution_reports_are_updated(capsys, monkeypatch):
"""Test the migration to the more feature-rich reports works."""
monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase())
monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer)
monkeypatch.setattr('dnstwister.tools.resolve', lambda domain:
('999.999.999.999', False))
repository = dnstwister.repository
domain = u'www.\u0454xample.com'
# Pre-load an old-style resolution report
db_key = u'resolution_report:{}'.format(domain)
repository.db._data[db_key] = {u'www.\u0454xample.co': '127.0.0.1'}
workers.deltas.process_domain(domain)
delta_report = repository.get_delta_report(domain)
assert delta_report == {
'deleted': [],
'updated': [(u'www.\u0454xample.co', '127.0.0.1', '999.999.999.999')],
'new': []
}
"""Tests of the email subscription mechanism."""
import binascii
import flask_webtest
import mock
import pytest
import webtest.app
import dnstwister
import dnstwister.tools
import patches
@mock.patch('dnstwister.views.www.email.emailer', patches.NoEmailer())
@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_bad_domains_fail(webapp):
"""Test the email views check domain validity."""
with pytest.raises(webtest.app.AppError) as err:
webapp.get('/email/subscribe/3234jskdnfsdf7y34')
assert '400 BAD REQUEST' in err.value.message
with pytest.raises(webtest.app.AppError) as err:
webapp.post('/email/pending_verify/3234jskdnfsdf7y34')
assert '400 BAD REQUEST' in err.value.message
def test_bad_error_codes(webapp):
"""Test the email error codes being weird doesn't break the page."""
normal_html = webapp.get(
'/email/subscribe/7777772e6578616d706c652e636f6d').html
assert webapp.get('/email/subscribe/7777772e6578616d706c652e636f6d/9',
def test_subscription_email_timing(capsys, monkeypatch):
"""Test that email subscriptions and delta reporting are in sync.
A bug was found where, because signing up registers for delta reporting
and the email is sent as soon as the report is generated, it is possible
to send 2 emails between delta reports.
"""
# Patch away
monkeypatch.setattr(
'dnstwister.repository.db',
patches.SimpleKVDatabase()
)
monkeypatch.setattr(
'dnstwister.repository.statistics.db',
patches.SimpleKVDatabase()
)
monkeypatch.setattr(
'dnstwister.tools.dnstwist.DomainFuzzer',
patches.SimpleFuzzer
)
monkeypatch.setattr(
'dnstwister.tools.resolve',
lambda domain: ('999.999.999.999', False)
)
emailer = patches.NoEmailer()
monkeypatch.setattr(
'workers.email.emailer',
emailer
)
repository = dnstwister.repository
# Settings
domain = 'www.example.com'
sub_id = '1234'
email = '*****@*****.**'
# We start with an unregistered domain.
assert not repository.is_domain_registered(domain)
# Subscribe a new user.
repository.subscribe_email(sub_id, email, domain)
# Subscribing a user does not register the domain.
assert not repository.is_domain_registered(domain)
# Process the subscription.
sub_data = repository.db.data['email_sub:{}'.format(sub_id)]
workers.email.process_sub(sub_id, sub_data)
# We won't have sent any emails.
assert emailer.sent_emails == []
# But the domain is now registered for delta reporting.
assert repository.is_domain_registered(domain)
# So let's do a delta report.
workers.deltas.process_domain(domain)
# Process the subscription again.
sub_data = repository.db.data['email_sub:{}'.format(sub_id)]
workers.email.process_sub(sub_id, sub_data)
# And we've sent an email.
assert len(emailer.sent_emails) == 1
# Now we "let" a bit over 24 hours pass since the email was sent and the
# report was updated.
passed_time = datetime.timedelta(hours=24, minutes=1)
repository.update_last_email_sub_sent_date(
sub_id,
datetime.datetime.now() - passed_time
)
delta_report = repository.get_delta_report(domain)
repository.update_delta_report(
domain, delta_report, datetime.datetime.now() - passed_time
)
# Now we run the email worker for the sub *before* the delta report.
workers.email.process_sub(sub_id, sub_data)
# We've not sent an extra email because it's more than 23 hours since the
# last delta report.
assert len(emailer.sent_emails) == 1
# As soon as the delta report is ran again we can send another email.
monkeypatch.setattr(
'dnstwister.tools.resolve', lambda domain: ('999.999.999.222', False)
)
workers.deltas.process_domain(domain)
workers.email.process_sub(sub_id, sub_data)
assert len(emailer.sent_emails) == 2
# And the emails are different.
assert emailer.sent_emails[0] != emailer.sent_emails[1]
class TestAtom(unittest.TestCase):
"""Tests of the atom feed behaviour."""
def setUp(self):
"""Set up the app for testing."""
# Create a webtest Test App for use
self.app = flask.ext.webtest.TestApp(dnstwister.app)
# Clear the webapp cache
dnstwister.cache.clear()
@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_new_feed(self):
"""Tests the registration of a new feed."""
repository = dnstwister.repository
# We need a domain to get the feed for.
domain = 'www.example.com'
# A feed is registered by trying to load it (and it not already being
# registered).
res = self.app.get('/atom/{}'.format(
base64.b64encode(domain))).follow()
# And only returns a single placeholder item.
assert str(res) == textwrap.dedent("""
Response: 200 OK
Content-Type: application/atom+xml; charset=utf-8
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title type="text">dnstwister report for www.example.com</title>
<id>http://localhost:80/atom/7777772e6578616d706c652e636f6d</id>
<updated>{date_today}</updated>
<link href="http://*****:*****@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_deleted_items_appear_in_rss(self):
"""Tests that deleted items in delta reports appear in the RSS.
"""
repository = dnstwister.repository
# We need a domain to get the feed for.
domain = 'www.example.com'
# We can calculate a delta though.
update_date = datetime.datetime(2016, 2, 28, 11, 10, 34)
repository.update_delta_report(
domain, {
'new': [('www.examp1e.com', '127.0.0.1')],
'updated': [('wwwexa.mple.com', '127.0.0.1', '127.0.0.2')],
'deleted': ['www.eeexample.com', 'www2.example.com.au'],
}, update_date)
# Clear the webapp cache
dnstwister.cache.clear()
res = self.app.get('/atom/{}'.format(
base64.b64encode(domain))).follow()
assert str(res) == textwrap.dedent("""
Response: 200 OK
Content-Type: application/atom+xml; charset=utf-8
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title type="text">dnstwister report for www.example.com</title>
<id>http://localhost:80/atom/7777772e6578616d706c652e636f6d</id>
<updated>2016-02-28T11:10:34Z</updated>
<link href="http://*****:*****@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_feed_reading_is_tracked(self):
"""Tests that reading a feed is logged."""
repository = dnstwister.repository
domain = 'www.example.com'
b64domain = base64.b64encode(domain)
# Read dates are None by default
read_date = repository.delta_report_last_read(domain)
assert read_date is None
# Registering a feed will update the read date
self.app.get('/atom/{}'.format(b64domain)).follow()
read_date = repository.delta_report_last_read(domain)
assert type(read_date) is datetime.datetime
# Manually set the date to an older date so we don't have to 'sleep'
# in the test.
repository.mark_delta_report_as_read(
domain, datetime.datetime(2000, 1, 1, 0, 0, 0))
# Clear the webapp cache
dnstwister.cache.clear()
# Reading a feed will update the read date
read_date = repository.delta_report_last_read(domain)
self.app.get('/atom/{}'.format(b64domain)).follow()
read_date2 = repository.delta_report_last_read(domain)
assert read_date2 > read_date
@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_unregister_tidies_database(self):
"""Tests that you can unregister domains."""
repository = dnstwister.repository
domain = 'www.example.com'
b64domain = base64.b64encode(domain)
assert not repository.is_domain_registered(domain)
assert repository.db.data == {}
self.app.get('/atom/{}'.format(b64domain)).follow()
repository.update_delta_report(
domain,
{
'new': [('www.examp1e.com', '127.0.0.1')],
'updated': [],
'deleted': [],
},
)
assert repository.is_domain_registered(domain)
assert repository.db.data != {}
repository.unregister_domain(domain)
assert not repository.is_domain_registered(domain)
assert repository.db.data == {}
class TestAtom(unittest.TestCase):
"""Tests of the atom feed behaviour."""
def setUp(self):
"""Set up the app for testing."""
# Create a webtest Test App for use
self.app = flask.ext.webtest.TestApp(dnstwister.app)
# Clear the webapp cache
dnstwister.cache.clear()
@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_new_feed(self):
"""Tests the registration of a new feed - currently disabled."""
repository = dnstwister.repository
# We need a domain to get the feed for.
domain = 'www.example.com'
# A feed is registered by trying to load it (and it not already being
# registered).
with pytest.raises(webtest.app.AppError) as err:
res = self.app.get('/atom/{}'.format(
base64.b64encode(domain))).follow()
assert '404 NOT FOUND' in err.value.message
assert 'New RSS feed generation currently disabled.' in err.value.message
@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_deleted_items_appear_in_rss(self):
"""Tests that deleted items in delta reports appear in the RSS.
"""
repository = dnstwister.repository
# We need a domain to get the feed for.
domain = 'www.example.com'
repository.register_domain(domain)
# We can calculate a delta though.
update_date = datetime.datetime(2016, 2, 28, 11, 10, 34)
repository.update_delta_report(
domain, {
'new': [('www.examp1e.com', '127.0.0.1')],
'updated': [('wwwexa.mple.com', '127.0.0.1', '127.0.0.2')],
'deleted': ['www.eeexample.com', 'www2.example.com.au'],
}, update_date)
# Clear the webapp cache
dnstwister.cache.clear()
res = self.app.get('/atom/{}'.format(
base64.b64encode(domain))).follow()
assert str(res) == textwrap.dedent("""
Response: 200 OK
Content-Type: application/atom+xml; charset=utf-8
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title type="text">dnstwister report for www.example.com</title>
<id>http://localhost:80/atom/7777772e6578616d706c652e636f6d</id>
<updated>2016-02-28T11:10:34Z</updated>
<link href="http://*****:*****@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_feed_reading_is_tracked(self):
"""Tests that reading a feed is logged."""
repository = dnstwister.repository
domain = 'www.example.com'
b64domain = base64.b64encode(domain)
# Read dates are None by default
read_date = repository.delta_report_last_read(domain)
assert read_date is None
# Registering a feed will update the read date
repository.register_domain(domain)
self.app.get('/atom/{}'.format(b64domain)).follow()
read_date = repository.delta_report_last_read(domain)
assert type(read_date) is datetime.datetime
# Manually set the date to an older date so we don't have to 'sleep'
# in the test.
repository.mark_delta_report_as_read(
domain, datetime.datetime(2000, 1, 1, 0, 0, 0))
# Clear the webapp cache
dnstwister.cache.clear()
# Reading a feed will update the read date
read_date = repository.delta_report_last_read(domain)
self.app.get('/atom/{}'.format(b64domain)).follow()
read_date2 = repository.delta_report_last_read(domain)
assert read_date2 > read_date
@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase())
def test_unregister_tidies_database(self):
"""Tests that you can unregister domains."""
repository = dnstwister.repository
domain = 'www.example.com'
b64domain = base64.b64encode(domain)
assert not repository.is_domain_registered(domain)
assert repository.db.data == {}
repository.register_domain(domain)
self.app.get('/atom/{}'.format(b64domain)).follow()
repository.update_delta_report(
domain,
{
'new': [('www.examp1e.com', '127.0.0.1')],
'updated': [],
'deleted': [],
},
)
assert repository.is_domain_registered(domain)
assert repository.db.data != {}
repository.unregister_domain(domain)
assert not repository.is_domain_registered(domain)
assert repository.db.data == {}
