def test_dont_send_too_often(capsys, monkeypatch): """Test that emails are not sent more than every 24 hours.""" # Patches monkeypatch.setattr( 'dnstwister.repository.db', patches.SimpleKVDatabase() ) monkeypatch.setattr( 'dnstwister.repository.statistics.db', patches.SimpleKVDatabase() ) monkeypatch.setattr( 'dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer ) monkeypatch.setattr( 'dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False) ) emailer = patches.NoEmailer() monkeypatch.setattr( 'workers.email.emailer', emailer ) repository = dnstwister.repository # Settings domain = 'www.example.com' sub_id = '1234' email = '*****@*****.**' # Subscribe a new user. repository.subscribe_email(sub_id, email, domain) # Do a delta report. workers.deltas.process_domain(domain) # Process the subscription. sub_data = repository.db.data['email_sub:{}'.format(sub_id)] workers.email.process_sub(sub_id, sub_data) # And we've sent an email. assert len(emailer.sent_emails) == 1 # Re-process immediately workers.email.process_sub(sub_id, sub_data) # Check we haven't sent two emails assert len(emailer.sent_emails) == 1
def test_updated_ip_is_noted_in_delta(capsys, monkeypatch): """IP updates are marked as 'updated'.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) repository = dnstwister.repository domain = u'www.\u0454xample.com' workers.deltas.process_domain(domain) monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('000.999.999.999', False)) # Mark the entry as 'old' to ensure it is re-run old_date = datetime.datetime.now() - datetime.timedelta(days=10) db_key = u'delta_report_updated:{}'.format(domain) repository.db._data[db_key] = old_date.strftime('%Y-%m-%dT%H:%M:%SZ') workers.deltas.process_domain(domain) delta_report = repository.get_delta_report(domain) assert delta_report == { 'deleted': [], 'updated': [(u'www.\u0454xample.co', '999.999.999.999', '000.999.999.999')], 'new': [] }
def test_dont_send_when_no_changes(capsys, monkeypatch): """Test that emails are not sent when there are no changes.""" # Patches monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr( 'dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer ) # Return no IPs monkeypatch.setattr( 'dnstwister.tools.resolve', lambda domain: (False, False) ) emailer = patches.NoEmailer() monkeypatch.setattr('workers.email.emailer', emailer) repository = dnstwister.repository # Settings domain = 'www.example.com' sub_id = '1234' email = '*****@*****.**' # Subscribe a new user. repository.subscribe_email(sub_id, email, domain) # Do a delta report. workers.deltas.process_domain(domain) # Process the subscription. sub_data = repository.db.data['email_sub:{}'.format(sub_id)] workers.email.process_sub(sub_id, sub_data) # We've not sent any emails as there were no changes (no IPs resolved at # all). assert len(emailer.sent_emails) == 0
def test_all_changes_are_new_on_first_delta(capsys, monkeypatch): """The first delta marks all resolved domains as 'new'.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) repository = dnstwister.repository domain = u'www.\u0454xample.com' workers.deltas.process_domain(domain) assert capsys.readouterr()[0] != '' resolution_report = repository.get_resolution_report(domain) assert resolution_report == { u'www.\u0454xample.co': { 'ip': '999.999.999.999', 'tweak': 'Pretend' } } delta_report = repository.get_delta_report(domain) assert delta_report == { 'deleted': [], 'new': [(u'www.\u0454xample.co', '999.999.999.999')], 'updated': [] }
def set_up_mocks(monkeypatch): mock_db = patches.SimpleKVDatabase() monkeypatch.setattr('dnstwister.repository.statistics.db', mock_db) monkeypatch.setattr('dnstwister.repository.db', mock_db) data_repository = dnstwister.repository statistics_repository = dnstwister.repository.statistics return data_repository, statistics_repository
def test_pre_noisy_domain_subscriptions_default_to_off(capsys, monkeypatch): """Test that subscriptions prior to noisy domains existing are supported still but default to 'off'. """ # Patches monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) # Enable noisy domains functionality monkeypatch.setenv('feature.noisy_domains', 'true') # Ensure the fake redis will work. monkeypatch.setenv('REDIS_URL', '') # Return a result monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) emailer = patches.NoEmailer() monkeypatch.setattr('workers.email.emailer', emailer) repository = dnstwister.repository # Settings domain = 'www.example.com' sub_id = '1234' email = '*****@*****.**' # Mark the found domain as also noisy. store = dnstwister.stats_store for _ in range(5): store.note('www.example.co') # Subscribe a new user, make them look like the pre-feature style. repository.subscribe_email(sub_id, email, domain, True) del (repository.db._data['email_sub:1234']['hide_noisy']) # Do a delta report. workers.deltas.process_domain(domain) # Process the subscription. sub_data = repository.db.data['email_sub:{}'.format(sub_id)] workers.email.process_sub(sub_id, sub_data) # We have an email. assert len(emailer.sent_emails) == 1 # And there is no reference to noisy domains in the email. assert 'noisy' not in emailer.sent_emails[0][2]
def test_dont_send_when_only_noisy(capsys, monkeypatch): """Test that emails are not sent when no non-noisy domains exist in delta report. """ # Patches monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) # Ensure the fake redis will work. monkeypatch.setenv('REDIS_URL', '') # Return a result monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) # Enable noisy domains functionality monkeypatch.setenv('feature.noisy_domains', 'true') emailer = patches.NoEmailer() monkeypatch.setattr('workers.email.emailer', emailer) repository = dnstwister.repository # Settings domain = 'www.example.com' sub_id = '1234' email = '*****@*****.**' # Mark the found domain as also noisy. store = dnstwister.stats_store for _ in range(5): store.note('www.example.co') # Subscribe a new user, with noisy filtering enabled. repository.subscribe_email(sub_id, email, domain, True) # Do a delta report. workers.deltas.process_domain(domain) # Process the subscription. sub_data = repository.db.data['email_sub:{}'.format(sub_id)] workers.email.process_sub(sub_id, sub_data) # We've not sent any emails as there were no changes (no IPs resolved at # all). assert len(emailer.sent_emails) == 0
def test_domains_iter_lists_all_domains(capsys, monkeypatch): """Test the repo can return all domains registered.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) domain = u'www.\u0454xample.com' repository = dnstwister.repository assert list(repository.iregistered_domains()) == [] repository.register_domain(domain) assert list(repository.iregistered_domains()) == [domain]
def test_new_domain_is_marked_as_read(capsys, monkeypatch): """New domain is marked as read at first pass.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) repository = dnstwister.repository domain = u'www.\u0454xample.com' assert repository.delta_report_updated(domain) is None workers.deltas.process_domain(domain) assert repository.delta_report_updated(domain) is not None assert capsys.readouterr()[0] != ''
def test_invalid_domain_is_unregistered(capsys, monkeypatch): """Invalid domains are tidied up.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) repository = dnstwister.repository invalid_domain = '3j88??ASd' assert not repository.is_domain_registered(invalid_domain) repository.register_domain(invalid_domain) assert repository.is_domain_registered(invalid_domain) workers.deltas.process_domain(invalid_domain) assert not repository.is_domain_registered(invalid_domain) expected_output = 'Invalid: \'{}\'\n'.format(invalid_domain.encode('idna')) assert capsys.readouterr()[0] == expected_output
def test_old_domain_is_unregistered(capsys, monkeypatch): """Long-time unread domains are unregistered.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) repository = dnstwister.repository domain = u'www.\u0454xample.com' old_date = datetime.datetime.now() - datetime.timedelta(days=10) assert repository.delta_report_last_read(domain) is None repository.mark_delta_report_as_read(domain, old_date) assert repository.delta_report_last_read(domain) == old_date.replace( microsecond=0) workers.deltas.process_domain(domain) assert repository.delta_report_last_read(domain) is None expected_output = 'Expired: {}\n'.format(domain.encode('idna')) assert capsys.readouterr()[0] == expected_output
def test_domains_are_checked_once_a_day(capsys, monkeypatch): """Test domains are not checked constantly.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) repository = dnstwister.repository domain = u'www.\u0454xample.com' workers.deltas.process_domain(domain) last_updated_db_key = u'delta_report_updated:{}'.format(domain) last_updated = repository.db._data[last_updated_db_key] # Process again not long after. time.sleep(2) workers.deltas.process_domain(domain) # Ensure that we didn't updated the last-updated date assert repository.db._data[last_updated_db_key] == last_updated
def test_old_style_resolution_reports_are_updated(capsys, monkeypatch): """Test the migration to the more feature-rich reports works.""" monkeypatch.setattr('dnstwister.repository.db', patches.SimpleKVDatabase()) monkeypatch.setattr('dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer) monkeypatch.setattr('dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False)) repository = dnstwister.repository domain = u'www.\u0454xample.com' # Pre-load an old-style resolution report db_key = u'resolution_report:{}'.format(domain) repository.db._data[db_key] = {u'www.\u0454xample.co': '127.0.0.1'} workers.deltas.process_domain(domain) delta_report = repository.get_delta_report(domain) assert delta_report == { 'deleted': [], 'updated': [(u'www.\u0454xample.co', '127.0.0.1', '999.999.999.999')], 'new': [] }
"""Tests of the email subscription mechanism.""" import binascii import flask_webtest import mock import pytest import webtest.app import dnstwister import dnstwister.tools import patches @mock.patch('dnstwister.views.www.email.emailer', patches.NoEmailer()) @mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_bad_domains_fail(webapp): """Test the email views check domain validity.""" with pytest.raises(webtest.app.AppError) as err: webapp.get('/email/subscribe/3234jskdnfsdf7y34') assert '400 BAD REQUEST' in err.value.message with pytest.raises(webtest.app.AppError) as err: webapp.post('/email/pending_verify/3234jskdnfsdf7y34') assert '400 BAD REQUEST' in err.value.message def test_bad_error_codes(webapp): """Test the email error codes being weird doesn't break the page.""" normal_html = webapp.get( '/email/subscribe/7777772e6578616d706c652e636f6d').html assert webapp.get('/email/subscribe/7777772e6578616d706c652e636f6d/9',
def test_subscription_email_timing(capsys, monkeypatch): """Test that email subscriptions and delta reporting are in sync. A bug was found where, because signing up registers for delta reporting and the email is sent as soon as the report is generated, it is possible to send 2 emails between delta reports. """ # Patch away monkeypatch.setattr( 'dnstwister.repository.db', patches.SimpleKVDatabase() ) monkeypatch.setattr( 'dnstwister.repository.statistics.db', patches.SimpleKVDatabase() ) monkeypatch.setattr( 'dnstwister.tools.dnstwist.DomainFuzzer', patches.SimpleFuzzer ) monkeypatch.setattr( 'dnstwister.tools.resolve', lambda domain: ('999.999.999.999', False) ) emailer = patches.NoEmailer() monkeypatch.setattr( 'workers.email.emailer', emailer ) repository = dnstwister.repository # Settings domain = 'www.example.com' sub_id = '1234' email = '*****@*****.**' # We start with an unregistered domain. assert not repository.is_domain_registered(domain) # Subscribe a new user. repository.subscribe_email(sub_id, email, domain) # Subscribing a user does not register the domain. assert not repository.is_domain_registered(domain) # Process the subscription. sub_data = repository.db.data['email_sub:{}'.format(sub_id)] workers.email.process_sub(sub_id, sub_data) # We won't have sent any emails. assert emailer.sent_emails == [] # But the domain is now registered for delta reporting. assert repository.is_domain_registered(domain) # So let's do a delta report. workers.deltas.process_domain(domain) # Process the subscription again. sub_data = repository.db.data['email_sub:{}'.format(sub_id)] workers.email.process_sub(sub_id, sub_data) # And we've sent an email. assert len(emailer.sent_emails) == 1 # Now we "let" a bit over 24 hours pass since the email was sent and the # report was updated. passed_time = datetime.timedelta(hours=24, minutes=1) repository.update_last_email_sub_sent_date( sub_id, datetime.datetime.now() - passed_time ) delta_report = repository.get_delta_report(domain) repository.update_delta_report( domain, delta_report, datetime.datetime.now() - passed_time ) # Now we run the email worker for the sub *before* the delta report. workers.email.process_sub(sub_id, sub_data) # We've not sent an extra email because it's more than 23 hours since the # last delta report. assert len(emailer.sent_emails) == 1 # As soon as the delta report is ran again we can send another email. monkeypatch.setattr( 'dnstwister.tools.resolve', lambda domain: ('999.999.999.222', False) ) workers.deltas.process_domain(domain) workers.email.process_sub(sub_id, sub_data) assert len(emailer.sent_emails) == 2 # And the emails are different. assert emailer.sent_emails[0] != emailer.sent_emails[1]
class TestAtom(unittest.TestCase): """Tests of the atom feed behaviour.""" def setUp(self): """Set up the app for testing.""" # Create a webtest Test App for use self.app = flask.ext.webtest.TestApp(dnstwister.app) # Clear the webapp cache dnstwister.cache.clear() @mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_new_feed(self): """Tests the registration of a new feed.""" repository = dnstwister.repository # We need a domain to get the feed for. domain = 'www.example.com' # A feed is registered by trying to load it (and it not already being # registered). res = self.app.get('/atom/{}'.format( base64.b64encode(domain))).follow() # And only returns a single placeholder item. assert str(res) == textwrap.dedent(""" Response: 200 OK Content-Type: application/atom+xml; charset=utf-8 <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom"> <title type="text">dnstwister report for www.example.com</title> <id>http://localhost:80/atom/7777772e6578616d706c652e636f6d</id> <updated>{date_today}</updated> <link href="http://*****:*****@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_deleted_items_appear_in_rss(self): """Tests that deleted items in delta reports appear in the RSS. """ repository = dnstwister.repository # We need a domain to get the feed for. domain = 'www.example.com' # We can calculate a delta though. update_date = datetime.datetime(2016, 2, 28, 11, 10, 34) repository.update_delta_report( domain, { 'new': [('www.examp1e.com', '127.0.0.1')], 'updated': [('wwwexa.mple.com', '127.0.0.1', '127.0.0.2')], 'deleted': ['www.eeexample.com', 'www2.example.com.au'], }, update_date) # Clear the webapp cache dnstwister.cache.clear() res = self.app.get('/atom/{}'.format( base64.b64encode(domain))).follow() assert str(res) == textwrap.dedent(""" Response: 200 OK Content-Type: application/atom+xml; charset=utf-8 <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom"> <title type="text">dnstwister report for www.example.com</title> <id>http://localhost:80/atom/7777772e6578616d706c652e636f6d</id> <updated>2016-02-28T11:10:34Z</updated> <link href="http://*****:*****@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_feed_reading_is_tracked(self): """Tests that reading a feed is logged.""" repository = dnstwister.repository domain = 'www.example.com' b64domain = base64.b64encode(domain) # Read dates are None by default read_date = repository.delta_report_last_read(domain) assert read_date is None # Registering a feed will update the read date self.app.get('/atom/{}'.format(b64domain)).follow() read_date = repository.delta_report_last_read(domain) assert type(read_date) is datetime.datetime # Manually set the date to an older date so we don't have to 'sleep' # in the test. repository.mark_delta_report_as_read( domain, datetime.datetime(2000, 1, 1, 0, 0, 0)) # Clear the webapp cache dnstwister.cache.clear() # Reading a feed will update the read date read_date = repository.delta_report_last_read(domain) self.app.get('/atom/{}'.format(b64domain)).follow() read_date2 = repository.delta_report_last_read(domain) assert read_date2 > read_date @mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_unregister_tidies_database(self): """Tests that you can unregister domains.""" repository = dnstwister.repository domain = 'www.example.com' b64domain = base64.b64encode(domain) assert not repository.is_domain_registered(domain) assert repository.db.data == {} self.app.get('/atom/{}'.format(b64domain)).follow() repository.update_delta_report( domain, { 'new': [('www.examp1e.com', '127.0.0.1')], 'updated': [], 'deleted': [], }, ) assert repository.is_domain_registered(domain) assert repository.db.data != {} repository.unregister_domain(domain) assert not repository.is_domain_registered(domain) assert repository.db.data == {}
class TestAtom(unittest.TestCase): """Tests of the atom feed behaviour.""" def setUp(self): """Set up the app for testing.""" # Create a webtest Test App for use self.app = flask.ext.webtest.TestApp(dnstwister.app) # Clear the webapp cache dnstwister.cache.clear() @mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_new_feed(self): """Tests the registration of a new feed - currently disabled.""" repository = dnstwister.repository # We need a domain to get the feed for. domain = 'www.example.com' # A feed is registered by trying to load it (and it not already being # registered). with pytest.raises(webtest.app.AppError) as err: res = self.app.get('/atom/{}'.format( base64.b64encode(domain))).follow() assert '404 NOT FOUND' in err.value.message assert 'New RSS feed generation currently disabled.' in err.value.message @mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_deleted_items_appear_in_rss(self): """Tests that deleted items in delta reports appear in the RSS. """ repository = dnstwister.repository # We need a domain to get the feed for. domain = 'www.example.com' repository.register_domain(domain) # We can calculate a delta though. update_date = datetime.datetime(2016, 2, 28, 11, 10, 34) repository.update_delta_report( domain, { 'new': [('www.examp1e.com', '127.0.0.1')], 'updated': [('wwwexa.mple.com', '127.0.0.1', '127.0.0.2')], 'deleted': ['www.eeexample.com', 'www2.example.com.au'], }, update_date) # Clear the webapp cache dnstwister.cache.clear() res = self.app.get('/atom/{}'.format( base64.b64encode(domain))).follow() assert str(res) == textwrap.dedent(""" Response: 200 OK Content-Type: application/atom+xml; charset=utf-8 <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom"> <title type="text">dnstwister report for www.example.com</title> <id>http://localhost:80/atom/7777772e6578616d706c652e636f6d</id> <updated>2016-02-28T11:10:34Z</updated> <link href="http://*****:*****@mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_feed_reading_is_tracked(self): """Tests that reading a feed is logged.""" repository = dnstwister.repository domain = 'www.example.com' b64domain = base64.b64encode(domain) # Read dates are None by default read_date = repository.delta_report_last_read(domain) assert read_date is None # Registering a feed will update the read date repository.register_domain(domain) self.app.get('/atom/{}'.format(b64domain)).follow() read_date = repository.delta_report_last_read(domain) assert type(read_date) is datetime.datetime # Manually set the date to an older date so we don't have to 'sleep' # in the test. repository.mark_delta_report_as_read( domain, datetime.datetime(2000, 1, 1, 0, 0, 0)) # Clear the webapp cache dnstwister.cache.clear() # Reading a feed will update the read date read_date = repository.delta_report_last_read(domain) self.app.get('/atom/{}'.format(b64domain)).follow() read_date2 = repository.delta_report_last_read(domain) assert read_date2 > read_date @mock.patch('dnstwister.repository.db', patches.SimpleKVDatabase()) def test_unregister_tidies_database(self): """Tests that you can unregister domains.""" repository = dnstwister.repository domain = 'www.example.com' b64domain = base64.b64encode(domain) assert not repository.is_domain_registered(domain) assert repository.db.data == {} repository.register_domain(domain) self.app.get('/atom/{}'.format(b64domain)).follow() repository.update_delta_report( domain, { 'new': [('www.examp1e.com', '127.0.0.1')], 'updated': [], 'deleted': [], }, ) assert repository.is_domain_registered(domain) assert repository.db.data != {} repository.unregister_domain(domain) assert not repository.is_domain_registered(domain) assert repository.db.data == {}