def test_save_to_cache(self, track_return_values): """Test that the form data can be saved to the cache.""" tracker = track_return_values(file_form, 'token_urlsafe') file = make_csv_file_from_dicts( *make_matched_rows(1), filename='cache-test.csv', ) form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file.getvalue()), }, ) assert form.is_valid() form.save_to_cache() assert len(tracker.return_values) == 1 token = tracker.return_values[0] contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) file.seek(0) assert gzip.decompress(cache.get(contents_key)) == file.read() assert cache.get(name_key) == file.name
def test_saves_file_contents_and_name(self): """Test that the file is saved in the cache.""" contents = b'file-contents' name = 'file-name' token = 'test-token' save_file_contents_and_name(token, contents, name) contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) assert cache.get(name_key) == name saved_contents = gzip.decompress(cache.get(contents_key)) assert saved_contents == contents
def test_loads_contents_and_name(self): """Test load_file_contents_and_name().""" token = 'test-token' contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) contents = b'file-contents' name = 'file-name' cache.set(contents_key, gzip.compress(contents)) cache.set(name_key, name) loaded_contents, loaded_name = load_file_contents_and_name(token) assert loaded_contents == contents assert loaded_name == name
def test_saves_results(self): """Test that counts by matching status are saved in the cache.""" num_matching = 3 num_unmatched = 2 num_multiple_matches = 1 token = 'test-token' _create_file_in_cache(token, num_matching, num_unmatched, num_multiple_matches) url = reverse( import_save_urlname, kwargs={'token': token}, ) response = self.client.post(url) assert response.status_code == status.HTTP_302_FOUND expected_redirect_url = reverse( import_complete_urlname, kwargs={'token': token}, ) assert response.url == expected_redirect_url cache_key = _cache_key_for_token(token, CacheKeyType.result_counts_by_status) matching_counts = cache.get(cache_key) assert matching_counts == { ContactMatchingStatus.matched: num_matching, ContactMatchingStatus.unmatched: num_unmatched, ContactMatchingStatus.multiple_matches: num_multiple_matches, }
def test_loads_unmatched_rows(self): """Test that the file is loaded from the cache.""" contents = b'file-contents' token = 'test-token' key = _cache_key_for_token(token, CacheKeyType.unmatched_rows) cache.set(key, gzip.compress(contents)) assert load_unmatched_rows_csv_contents(token) == contents
def test_saves_unmatched_rows(self): """Test that the file is saved to the cache.""" contents = b'file-contents' token = 'test-token' save_unmatched_rows_csv_contents(token, contents) key = _cache_key_for_token(token, CacheKeyType.unmatched_rows) saved_contents = gzip.decompress(cache.get(key)) assert saved_contents == contents
def _create_file_in_cache(token, num_matching, num_unmatched, num_multiple_matches): matched_rows = make_matched_rows(num_matching) unmatched_rows = make_unmatched_rows(num_unmatched) multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches) file = make_csv_file_from_dicts( *matched_rows, *unmatched_rows, *multiple_matches_rows, filename='cache-test.csv', ) with file: compressed_contents = gzip.compress(file.read()) contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) cache.set(contents_key, compressed_contents) cache.set(name_key, file.name) return matched_rows
class TestLoadFileContentsAndName: """Tests for load_file_contents_and_name().""" def test_loads_contents_and_name(self): """Test load_file_contents_and_name().""" token = 'test-token' contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) contents = b'file-contents' name = 'file-name' cache.set(contents_key, gzip.compress(contents)) cache.set(name_key, name) loaded_contents, loaded_name = load_file_contents_and_name(token) assert loaded_contents == contents assert loaded_name == name @pytest.mark.parametrize( 'cache_data', ( # only the file contents { _cache_key_for_token('test-token', CacheKeyType.file_contents): b'data' }, # only the file name { _cache_key_for_token('test-token', CacheKeyType.file_name): 'name' }, # nothing {}, ), ) def test_returns_none_if_any_key_not_found(self, cache_data): """Test that load_file_contents_and_name() returns None if a cache key is missing.""" cache.set_many(cache_data) assert load_file_contents_and_name('test-token') is None
def test_from_token_with_valid_token(self): """Test that a form can be restored from the cache.""" token = 'test-token' contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) file = make_csv_file_from_dicts( *make_matched_rows(1), filename='cache-test.csv', ) compressed_data = gzip.compress(file.read()) cache.set(contents_key, compressed_data) cache.set(name_key, file.name) form = InteractionCSVForm.from_token(token) assert form.is_valid() file.seek(0) assert file.read() == form.cleaned_data['csv_file'].read() assert file.name == form.cleaned_data['csv_file'].name
def test_raises_error_if_file_in_cache_is_invalid(self): """ Test that if the file in the cache fails InteractionCSVForm re-validation, a DataHubException is raised. (This should not happen in normal circumstances.) """ token = 'test-token' compressed_conents = gzip.compress(b'invalid') contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) cache.set(contents_key, compressed_conents) cache.set(name_key, 'test.csv') url = reverse( import_save_urlname, kwargs={'token': token}, ) with pytest.raises(DataHubException): self.client.post(url)
def test_key_expires(self): """Test that the key expires after the expiry period.""" contents = b'file-contents' token = 'test-token' base_datetime = datetime(2019, 2, 3) with freeze_time(base_datetime): save_unmatched_rows_csv_contents(token, contents) key = _cache_key_for_token(token, CacheKeyType.unmatched_rows) with freeze_time(base_datetime + CACHE_VALUE_TIMEOUT + timedelta(minutes=1)): assert cache.get(key) is None
def test_can_download_unmatched_rows(self, num_matching, num_unmatched, num_multiple_matches): """Test that unmatched rows can be downloaded.""" token = 'test-token' file_contents = 'unmatched-rows'.encode('utf-8-sig') cache_key = _cache_key_for_token(token, CacheKeyType.unmatched_rows) cache.set(cache_key, gzip.compress(file_contents)) url = reverse( import_download_unmatched_urlname, kwargs={'token': token}, ) response = self.client.get(url) assert response.status_code == status.HTTP_200_OK assert parse_header(response['Content-Type']) == ('text/csv', {'charset': 'utf-8'}) assert parse_header(response['Content-Disposition']) == ( 'attachment', {'filename': 'Unmatched interactions - 2019-05-10-12-13-14.csv'}, ) assert response.getvalue() == file_contents
def test_displays_counts_by_status(self, num_matching, num_unmatched, num_multiple_matches): """Test that counts are displayed for each matching status.""" token = 'test-token' cache_key = _cache_key_for_token(token, CacheKeyType.result_counts_by_status) cache_value = { ContactMatchingStatus.matched: num_matching, ContactMatchingStatus.unmatched: num_unmatched, ContactMatchingStatus.multiple_matches: num_multiple_matches, } cache.set(cache_key, cache_value) url = reverse( import_complete_urlname, kwargs={'token': token}, ) response = self.client.get(url) assert response.status_code == status.HTTP_200_OK assert response.context['num_matched'] == num_matching assert response.context['num_unmatched'] == num_unmatched assert response.context['num_multiple_matches'] == num_multiple_matches
def test_raises_error_on_invalid_input(self, token, type_, error): """Test that an error is raised if an invalid token or type_ is provided.""" with pytest.raises(error): _cache_key_for_token(token, type_)
def test_generates_keys(self, token, type_, key): """Test that the expected keys are generated.""" assert _cache_key_for_token(token, type_) == key
class TestInteractionCSVForm: """Tests for InteractionCSVForm.""" def test_get_row_errors_with_duplicate_rows(self): """Test that duplicate rows are tracked and errors returned when encountered.""" matched_rows = make_matched_rows(5) file = make_csv_file_from_dicts( # Duplicate the first row matched_rows[0], *matched_rows, *make_unmatched_rows(5), *make_multiple_matches_rows(5), ) form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file.getvalue()), }, ) assert form.is_valid() row_errors = list(form.get_row_error_iterator()) assert row_errors == [ CSVRowError(1, NON_FIELD_ERRORS, '', DUPLICATE_OF_ANOTHER_ROW_MESSAGE), ] @pytest.mark.parametrize( 'num_matching,num_unmatched,num_multiple_matches,max_returned_rows', ( (4, 3, 2, 4), (4, 3, 2, 2), (4, 3, 2, 8), (4, 0, 0, 4), (0, 2, 2, 4), (0, 0, 2, 4), ), ) def test_get_matching_summary( self, num_matching, num_unmatched, num_multiple_matches, max_returned_rows, ): """Test get_matching_summary() with various inputs.""" input_matched_rows = make_matched_rows(num_matching) unmatched_rows = make_unmatched_rows(num_unmatched) multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches) file = make_csv_file_from_dicts( *input_matched_rows, *unmatched_rows, *multiple_matches_rows, ) form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file.getvalue()), }, ) assert form.is_valid() matching_counts, returned_matched_rows = form.get_matching_summary(max_returned_rows) assert matching_counts == { ContactMatchingStatus.matched: num_matching, ContactMatchingStatus.unmatched: num_unmatched, ContactMatchingStatus.multiple_matches: num_multiple_matches, } expected_num_returned_rows = min(num_matching, max_returned_rows) assert len(returned_matched_rows) == expected_num_returned_rows # Check the the rows returned are the ones we expect expected_contact_emails = [ row['contact_email'] for row in input_matched_rows[:expected_num_returned_rows] ] actual_contact_emails = [row['contacts'][0].email for row in returned_matched_rows] assert expected_contact_emails == actual_contact_emails def test_get_matching_summary_with_invalid_rows(self): """ Test that get_matching_summary() raises an exception if one of the CSV rows fails validation. """ file = make_csv_file_from_dicts( { 'theme': 'invalid', 'kind': 'invalid', 'date': 'invalid', 'adviser_1': 'invalid', 'contact_email': 'invalid', 'service': 'invalid', 'communication_channel': 'invalid', }, ) form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file.getvalue()), }, ) assert form.is_valid() with pytest.raises(DataHubError): form.get_matching_summary(50) @pytest.mark.parametrize('num_unmatched', (0, 2)) @pytest.mark.parametrize('num_multiple_matches', (0, 2)) @pytest.mark.usefixtures('local_memory_cache') def test_save_returns_correct_counts(self, num_unmatched, num_multiple_matches): """Test that save() returns the expected counts for each matching status.""" num_matching = 1 matched_rows = make_matched_rows(num_matching) unmatched_rows = make_unmatched_rows(num_unmatched) multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches) user = AdviserFactory(first_name='Admin', last_name='User') file = make_csv_file_from_dicts( *matched_rows, *unmatched_rows, *multiple_matches_rows, ) file_contents = file.getvalue() form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file_contents), }, ) assert form.is_valid() matching_counts, _ = form.save(user) assert matching_counts == { ContactMatchingStatus.matched: num_matching, ContactMatchingStatus.unmatched: num_unmatched, ContactMatchingStatus.multiple_matches: num_multiple_matches, } @pytest.mark.parametrize('num_unmatched', (0, 2)) @pytest.mark.parametrize('num_multiple_matches', (0, 2)) @pytest.mark.usefixtures('local_memory_cache') def test_save_returns_unmatched_rows(self, num_unmatched, num_multiple_matches): """Test that save() returns an UnmatchedRowCollector with the expected rows.""" num_matching = 2 matched_rows = make_matched_rows(num_matching) unmatched_rows = make_unmatched_rows(num_unmatched) multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches) user = AdviserFactory(first_name='Admin', last_name='User') file = make_csv_file_from_dicts( *matched_rows, *unmatched_rows, *multiple_matches_rows, ) file_contents = file.getvalue() form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file_contents), }, ) assert form.is_valid() _, unmatched_row_collector = form.save(user) assert unmatched_row_collector.rows == [ *unmatched_rows, *multiple_matches_rows, ] @pytest.mark.parametrize('num_unmatched', (0, 2)) @pytest.mark.parametrize('num_multiple_matches', (0, 2)) def test_save_creates_interactions(self, num_unmatched, num_multiple_matches): """Test that save() creates interactions.""" num_matching = 3 matched_rows = make_matched_rows(num_matching) unmatched_rows = make_unmatched_rows(num_unmatched) multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches) user = AdviserFactory(first_name='Admin', last_name='User') file = make_csv_file_from_dicts( *matched_rows, *unmatched_rows, *multiple_matches_rows, ) file_contents = file.getvalue() form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file_contents), }, ) assert form.is_valid() form.save(user) created_interactions = list(Interaction.objects.all()) assert len(created_interactions) == num_matching expected_contact_emails = {row['contact_email'] for row in matched_rows} actual_contact_emails = { interaction.contacts.first().email for interaction in created_interactions } # Make sure the test was correctly set up with unique contact emails assert len(actual_contact_emails) == num_matching # Check that the interactions created are the ones we expect # Note: the full saving logic for a row is tested in the InteractionCSVRowForm tests assert expected_contact_emails == actual_contact_emails expected_source = { 'file': { 'name': file.name, 'size': len(file_contents), 'sha256': hashlib.sha256(file_contents).hexdigest(), }, } # `source` has been set (list used rather than a generator for useful failure messages) assert all([ interaction.source == expected_source for interaction in created_interactions ]) def test_save_creates_versions(self): """Test that save() creates versions using django-reversion.""" num_matching = 5 matched_rows = make_matched_rows(num_matching) user = AdviserFactory(first_name='Admin', last_name='User') file = make_csv_file_from_dicts(*matched_rows) file_contents = file.getvalue() form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file_contents), }, ) assert form.is_valid() form.save(user) created_interactions = list(Interaction.objects.all()) assert len(created_interactions) == num_matching # Single revision created assert Revision.objects.count() == 1 assert Revision.objects.first().get_comment() == REVISION_COMMENT # Versions were created (list used rather than a generator for useful failure messages) assert all([ Version.objects.get_for_object(interaction).count() == 1 for interaction in created_interactions ]) def test_save_rolls_back_on_error(self): """Test that save() rolls back if one row can't be saved.""" user = AdviserFactory(first_name='Admin', last_name='User') file = make_csv_file_from_dicts( *make_matched_rows(5), # an invalid row {}, ) file_contents = file.getvalue() form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file_contents), }, ) assert form.is_valid() with pytest.raises(DataHubError): form.save(user) assert not Interaction.objects.count() @pytest.mark.usefixtures('local_memory_cache') def test_save_to_cache(self, track_return_values): """Test that the form data can be saved to the cache.""" tracker = track_return_values(file_form, 'token_urlsafe') file = make_csv_file_from_dicts( *make_matched_rows(1), filename='cache-test.csv', ) form = InteractionCSVForm( files={ 'csv_file': SimpleUploadedFile(file.name, file.getvalue()), }, ) assert form.is_valid() form.save_to_cache() assert len(tracker.return_values) == 1 token = tracker.return_values[0] contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) file.seek(0) assert gzip.decompress(cache.get(contents_key)) == file.read() assert cache.get(name_key) == file.name @pytest.mark.usefixtures('local_memory_cache') def test_from_token_with_valid_token(self): """Test that a form can be restored from the cache.""" token = 'test-token' contents_key = _cache_key_for_token(token, CacheKeyType.file_contents) name_key = _cache_key_for_token(token, CacheKeyType.file_name) file = make_csv_file_from_dicts( *make_matched_rows(1), filename='cache-test.csv', ) compressed_data = gzip.compress(file.read()) cache.set(contents_key, compressed_data) cache.set(name_key, file.name) form = InteractionCSVForm.from_token(token) assert form.is_valid() file.seek(0) assert file.read() == form.cleaned_data['csv_file'].read() assert file.name == form.cleaned_data['csv_file'].name @pytest.mark.usefixtures('local_memory_cache') @pytest.mark.parametrize( 'cache_data', ( # only the file contents {_cache_key_for_token('test-token', CacheKeyType.file_contents): b'data'}, # only the file name {_cache_key_for_token('test-token', CacheKeyType.file_name): 'name'}, # nothing {}, ), ) def test_from_token_with_invalid_token(self, cache_data): """ Test that from_token() returns None if there is incomplete data for the token in the cache. """ cache.set_many(cache_data) form = InteractionCSVForm.from_token('test-token') assert form is None