def test_save_to_cache(self, track_return_values):
"""Test that the form data can be saved to the cache."""
tracker = track_return_values(file_form, 'token_urlsafe')
file = make_csv_file_from_dicts(
*make_matched_rows(1),
filename='cache-test.csv',
)
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file.getvalue()),
},
)
assert form.is_valid()
form.save_to_cache()
assert len(tracker.return_values) == 1
token = tracker.return_values[0]
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
file.seek(0)
assert gzip.decompress(cache.get(contents_key)) == file.read()
assert cache.get(name_key) == file.name
def test_saves_file_contents_and_name(self):
"""Test that the file is saved in the cache."""
contents = b'file-contents'
name = 'file-name'
token = 'test-token'
save_file_contents_and_name(token, contents, name)
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
assert cache.get(name_key) == name
saved_contents = gzip.decompress(cache.get(contents_key))
assert saved_contents == contents
def test_loads_contents_and_name(self):
"""Test load_file_contents_and_name()."""
token = 'test-token'
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
contents = b'file-contents'
name = 'file-name'
cache.set(contents_key, gzip.compress(contents))
cache.set(name_key, name)
loaded_contents, loaded_name = load_file_contents_and_name(token)
assert loaded_contents == contents
assert loaded_name == name
def test_saves_results(self):
"""Test that counts by matching status are saved in the cache."""
num_matching = 3
num_unmatched = 2
num_multiple_matches = 1
token = 'test-token'
_create_file_in_cache(token, num_matching, num_unmatched, num_multiple_matches)
url = reverse(
import_save_urlname,
kwargs={'token': token},
)
response = self.client.post(url)
assert response.status_code == status.HTTP_302_FOUND
expected_redirect_url = reverse(
import_complete_urlname,
kwargs={'token': token},
)
assert response.url == expected_redirect_url
cache_key = _cache_key_for_token(token, CacheKeyType.result_counts_by_status)
matching_counts = cache.get(cache_key)
assert matching_counts == {
ContactMatchingStatus.matched: num_matching,
ContactMatchingStatus.unmatched: num_unmatched,
ContactMatchingStatus.multiple_matches: num_multiple_matches,
}
def test_loads_unmatched_rows(self):
"""Test that the file is loaded from the cache."""
contents = b'file-contents'
token = 'test-token'
key = _cache_key_for_token(token, CacheKeyType.unmatched_rows)
cache.set(key, gzip.compress(contents))
assert load_unmatched_rows_csv_contents(token) == contents
def test_saves_unmatched_rows(self):
"""Test that the file is saved to the cache."""
contents = b'file-contents'
token = 'test-token'
save_unmatched_rows_csv_contents(token, contents)
key = _cache_key_for_token(token, CacheKeyType.unmatched_rows)
saved_contents = gzip.decompress(cache.get(key))
assert saved_contents == contents
def _create_file_in_cache(token, num_matching, num_unmatched, num_multiple_matches):
matched_rows = make_matched_rows(num_matching)
unmatched_rows = make_unmatched_rows(num_unmatched)
multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches)
file = make_csv_file_from_dicts(
*matched_rows,
*unmatched_rows,
*multiple_matches_rows,
filename='cache-test.csv',
)
with file:
compressed_contents = gzip.compress(file.read())
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
cache.set(contents_key, compressed_contents)
cache.set(name_key, file.name)
return matched_rows
class TestLoadFileContentsAndName:
"""Tests for load_file_contents_and_name()."""
def test_loads_contents_and_name(self):
"""Test load_file_contents_and_name()."""
token = 'test-token'
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
contents = b'file-contents'
name = 'file-name'
cache.set(contents_key, gzip.compress(contents))
cache.set(name_key, name)
loaded_contents, loaded_name = load_file_contents_and_name(token)
assert loaded_contents == contents
assert loaded_name == name
@pytest.mark.parametrize(
'cache_data',
(
# only the file contents
{
_cache_key_for_token('test-token', CacheKeyType.file_contents):
b'data'
},
# only the file name
{
_cache_key_for_token('test-token', CacheKeyType.file_name):
'name'
},
# nothing
{},
),
)
def test_returns_none_if_any_key_not_found(self, cache_data):
"""Test that load_file_contents_and_name() returns None if a cache key is missing."""
cache.set_many(cache_data)
assert load_file_contents_and_name('test-token') is None
def test_from_token_with_valid_token(self):
"""Test that a form can be restored from the cache."""
token = 'test-token'
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
file = make_csv_file_from_dicts(
*make_matched_rows(1),
filename='cache-test.csv',
)
compressed_data = gzip.compress(file.read())
cache.set(contents_key, compressed_data)
cache.set(name_key, file.name)
form = InteractionCSVForm.from_token(token)
assert form.is_valid()
file.seek(0)
assert file.read() == form.cleaned_data['csv_file'].read()
assert file.name == form.cleaned_data['csv_file'].name
def test_raises_error_if_file_in_cache_is_invalid(self):
"""
Test that if the file in the cache fails InteractionCSVForm re-validation,
a DataHubException is raised.
(This should not happen in normal circumstances.)
"""
token = 'test-token'
compressed_conents = gzip.compress(b'invalid')
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
cache.set(contents_key, compressed_conents)
cache.set(name_key, 'test.csv')
url = reverse(
import_save_urlname,
kwargs={'token': token},
)
with pytest.raises(DataHubException):
self.client.post(url)
def test_key_expires(self):
"""Test that the key expires after the expiry period."""
contents = b'file-contents'
token = 'test-token'
base_datetime = datetime(2019, 2, 3)
with freeze_time(base_datetime):
save_unmatched_rows_csv_contents(token, contents)
key = _cache_key_for_token(token, CacheKeyType.unmatched_rows)
with freeze_time(base_datetime + CACHE_VALUE_TIMEOUT +
timedelta(minutes=1)):
assert cache.get(key) is None
def test_can_download_unmatched_rows(self, num_matching, num_unmatched, num_multiple_matches):
"""Test that unmatched rows can be downloaded."""
token = 'test-token'
file_contents = 'unmatched-rows'.encode('utf-8-sig')
cache_key = _cache_key_for_token(token, CacheKeyType.unmatched_rows)
cache.set(cache_key, gzip.compress(file_contents))
url = reverse(
import_download_unmatched_urlname,
kwargs={'token': token},
)
response = self.client.get(url)
assert response.status_code == status.HTTP_200_OK
assert parse_header(response['Content-Type']) == ('text/csv', {'charset': 'utf-8'})
assert parse_header(response['Content-Disposition']) == (
'attachment',
{'filename': 'Unmatched interactions - 2019-05-10-12-13-14.csv'},
)
assert response.getvalue() == file_contents
def test_displays_counts_by_status(self, num_matching, num_unmatched, num_multiple_matches):
"""Test that counts are displayed for each matching status."""
token = 'test-token'
cache_key = _cache_key_for_token(token, CacheKeyType.result_counts_by_status)
cache_value = {
ContactMatchingStatus.matched: num_matching,
ContactMatchingStatus.unmatched: num_unmatched,
ContactMatchingStatus.multiple_matches: num_multiple_matches,
}
cache.set(cache_key, cache_value)
url = reverse(
import_complete_urlname,
kwargs={'token': token},
)
response = self.client.get(url)
assert response.status_code == status.HTTP_200_OK
assert response.context['num_matched'] == num_matching
assert response.context['num_unmatched'] == num_unmatched
assert response.context['num_multiple_matches'] == num_multiple_matches
def test_raises_error_on_invalid_input(self, token, type_, error):
"""Test that an error is raised if an invalid token or type_ is provided."""
with pytest.raises(error):
_cache_key_for_token(token, type_)
def test_generates_keys(self, token, type_, key):
"""Test that the expected keys are generated."""
assert _cache_key_for_token(token, type_) == key
class TestInteractionCSVForm:
"""Tests for InteractionCSVForm."""
def test_get_row_errors_with_duplicate_rows(self):
"""Test that duplicate rows are tracked and errors returned when encountered."""
matched_rows = make_matched_rows(5)
file = make_csv_file_from_dicts(
# Duplicate the first row
matched_rows[0],
*matched_rows,
*make_unmatched_rows(5),
*make_multiple_matches_rows(5),
)
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file.getvalue()),
},
)
assert form.is_valid()
row_errors = list(form.get_row_error_iterator())
assert row_errors == [
CSVRowError(1, NON_FIELD_ERRORS, '', DUPLICATE_OF_ANOTHER_ROW_MESSAGE),
]
@pytest.mark.parametrize(
'num_matching,num_unmatched,num_multiple_matches,max_returned_rows',
(
(4, 3, 2, 4),
(4, 3, 2, 2),
(4, 3, 2, 8),
(4, 0, 0, 4),
(0, 2, 2, 4),
(0, 0, 2, 4),
),
)
def test_get_matching_summary(
self,
num_matching,
num_unmatched,
num_multiple_matches,
max_returned_rows,
):
"""Test get_matching_summary() with various inputs."""
input_matched_rows = make_matched_rows(num_matching)
unmatched_rows = make_unmatched_rows(num_unmatched)
multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches)
file = make_csv_file_from_dicts(
*input_matched_rows,
*unmatched_rows,
*multiple_matches_rows,
)
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file.getvalue()),
},
)
assert form.is_valid()
matching_counts, returned_matched_rows = form.get_matching_summary(max_returned_rows)
assert matching_counts == {
ContactMatchingStatus.matched: num_matching,
ContactMatchingStatus.unmatched: num_unmatched,
ContactMatchingStatus.multiple_matches: num_multiple_matches,
}
expected_num_returned_rows = min(num_matching, max_returned_rows)
assert len(returned_matched_rows) == expected_num_returned_rows
# Check the the rows returned are the ones we expect
expected_contact_emails = [
row['contact_email'] for row in input_matched_rows[:expected_num_returned_rows]
]
actual_contact_emails = [row['contacts'][0].email for row in returned_matched_rows]
assert expected_contact_emails == actual_contact_emails
def test_get_matching_summary_with_invalid_rows(self):
"""
Test that get_matching_summary() raises an exception if one of the CSV rows fails
validation.
"""
file = make_csv_file_from_dicts(
{
'theme': 'invalid',
'kind': 'invalid',
'date': 'invalid',
'adviser_1': 'invalid',
'contact_email': 'invalid',
'service': 'invalid',
'communication_channel': 'invalid',
},
)
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file.getvalue()),
},
)
assert form.is_valid()
with pytest.raises(DataHubError):
form.get_matching_summary(50)
@pytest.mark.parametrize('num_unmatched', (0, 2))
@pytest.mark.parametrize('num_multiple_matches', (0, 2))
@pytest.mark.usefixtures('local_memory_cache')
def test_save_returns_correct_counts(self, num_unmatched, num_multiple_matches):
"""Test that save() returns the expected counts for each matching status."""
num_matching = 1
matched_rows = make_matched_rows(num_matching)
unmatched_rows = make_unmatched_rows(num_unmatched)
multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches)
user = AdviserFactory(first_name='Admin', last_name='User')
file = make_csv_file_from_dicts(
*matched_rows,
*unmatched_rows,
*multiple_matches_rows,
)
file_contents = file.getvalue()
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file_contents),
},
)
assert form.is_valid()
matching_counts, _ = form.save(user)
assert matching_counts == {
ContactMatchingStatus.matched: num_matching,
ContactMatchingStatus.unmatched: num_unmatched,
ContactMatchingStatus.multiple_matches: num_multiple_matches,
}
@pytest.mark.parametrize('num_unmatched', (0, 2))
@pytest.mark.parametrize('num_multiple_matches', (0, 2))
@pytest.mark.usefixtures('local_memory_cache')
def test_save_returns_unmatched_rows(self, num_unmatched, num_multiple_matches):
"""Test that save() returns an UnmatchedRowCollector with the expected rows."""
num_matching = 2
matched_rows = make_matched_rows(num_matching)
unmatched_rows = make_unmatched_rows(num_unmatched)
multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches)
user = AdviserFactory(first_name='Admin', last_name='User')
file = make_csv_file_from_dicts(
*matched_rows,
*unmatched_rows,
*multiple_matches_rows,
)
file_contents = file.getvalue()
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file_contents),
},
)
assert form.is_valid()
_, unmatched_row_collector = form.save(user)
assert unmatched_row_collector.rows == [
*unmatched_rows,
*multiple_matches_rows,
]
@pytest.mark.parametrize('num_unmatched', (0, 2))
@pytest.mark.parametrize('num_multiple_matches', (0, 2))
def test_save_creates_interactions(self, num_unmatched, num_multiple_matches):
"""Test that save() creates interactions."""
num_matching = 3
matched_rows = make_matched_rows(num_matching)
unmatched_rows = make_unmatched_rows(num_unmatched)
multiple_matches_rows = make_multiple_matches_rows(num_multiple_matches)
user = AdviserFactory(first_name='Admin', last_name='User')
file = make_csv_file_from_dicts(
*matched_rows,
*unmatched_rows,
*multiple_matches_rows,
)
file_contents = file.getvalue()
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file_contents),
},
)
assert form.is_valid()
form.save(user)
created_interactions = list(Interaction.objects.all())
assert len(created_interactions) == num_matching
expected_contact_emails = {row['contact_email'] for row in matched_rows}
actual_contact_emails = {
interaction.contacts.first().email for interaction in created_interactions
}
# Make sure the test was correctly set up with unique contact emails
assert len(actual_contact_emails) == num_matching
# Check that the interactions created are the ones we expect
# Note: the full saving logic for a row is tested in the InteractionCSVRowForm tests
assert expected_contact_emails == actual_contact_emails
expected_source = {
'file': {
'name': file.name,
'size': len(file_contents),
'sha256': hashlib.sha256(file_contents).hexdigest(),
},
}
# `source` has been set (list used rather than a generator for useful failure messages)
assert all([
interaction.source == expected_source for interaction in created_interactions
])
def test_save_creates_versions(self):
"""Test that save() creates versions using django-reversion."""
num_matching = 5
matched_rows = make_matched_rows(num_matching)
user = AdviserFactory(first_name='Admin', last_name='User')
file = make_csv_file_from_dicts(*matched_rows)
file_contents = file.getvalue()
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file_contents),
},
)
assert form.is_valid()
form.save(user)
created_interactions = list(Interaction.objects.all())
assert len(created_interactions) == num_matching
# Single revision created
assert Revision.objects.count() == 1
assert Revision.objects.first().get_comment() == REVISION_COMMENT
# Versions were created (list used rather than a generator for useful failure messages)
assert all([
Version.objects.get_for_object(interaction).count() == 1
for interaction in created_interactions
])
def test_save_rolls_back_on_error(self):
"""Test that save() rolls back if one row can't be saved."""
user = AdviserFactory(first_name='Admin', last_name='User')
file = make_csv_file_from_dicts(
*make_matched_rows(5),
# an invalid row
{},
)
file_contents = file.getvalue()
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file_contents),
},
)
assert form.is_valid()
with pytest.raises(DataHubError):
form.save(user)
assert not Interaction.objects.count()
@pytest.mark.usefixtures('local_memory_cache')
def test_save_to_cache(self, track_return_values):
"""Test that the form data can be saved to the cache."""
tracker = track_return_values(file_form, 'token_urlsafe')
file = make_csv_file_from_dicts(
*make_matched_rows(1),
filename='cache-test.csv',
)
form = InteractionCSVForm(
files={
'csv_file': SimpleUploadedFile(file.name, file.getvalue()),
},
)
assert form.is_valid()
form.save_to_cache()
assert len(tracker.return_values) == 1
token = tracker.return_values[0]
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
file.seek(0)
assert gzip.decompress(cache.get(contents_key)) == file.read()
assert cache.get(name_key) == file.name
@pytest.mark.usefixtures('local_memory_cache')
def test_from_token_with_valid_token(self):
"""Test that a form can be restored from the cache."""
token = 'test-token'
contents_key = _cache_key_for_token(token, CacheKeyType.file_contents)
name_key = _cache_key_for_token(token, CacheKeyType.file_name)
file = make_csv_file_from_dicts(
*make_matched_rows(1),
filename='cache-test.csv',
)
compressed_data = gzip.compress(file.read())
cache.set(contents_key, compressed_data)
cache.set(name_key, file.name)
form = InteractionCSVForm.from_token(token)
assert form.is_valid()
file.seek(0)
assert file.read() == form.cleaned_data['csv_file'].read()
assert file.name == form.cleaned_data['csv_file'].name
@pytest.mark.usefixtures('local_memory_cache')
@pytest.mark.parametrize(
'cache_data',
(
# only the file contents
{_cache_key_for_token('test-token', CacheKeyType.file_contents): b'data'},
# only the file name
{_cache_key_for_token('test-token', CacheKeyType.file_name): 'name'},
# nothing
{},
),
)
def test_from_token_with_invalid_token(self, cache_data):
"""
Test that from_token() returns None if there is incomplete data for the token in
the cache.
"""
cache.set_many(cache_data)
form = InteractionCSVForm.from_token('test-token')
assert form is None
