def test_recipient_whitelist(file_contents, template_type, whitelist,
count_of_rows_with_errors):
recipients = RecipientCSV(file_contents,
template_type=template_type,
whitelist=whitelist)
if count_of_rows_with_errors:
assert not recipients.allowed_to_send_to
else:
assert recipients.allowed_to_send_to
# Make sure the whitelist isn’t emptied by reading it. If it’s an iterator then
# there’s a risk that it gets emptied after being read once
recipients.whitelist = (str(fake_number)
for fake_number in range(7700900888, 7700900898))
list(recipients.whitelist)
assert not recipients.allowed_to_send_to
assert recipients.has_errors
# An empty whitelist is treated as no whitelist at all
recipients.whitelist = []
assert recipients.allowed_to_send_to
recipients.whitelist = itertools.chain()
assert recipients.allowed_to_send_to
def test_recipient_whitelist(file_contents, template_type, whitelist, count_of_rows_with_errors):
recipients = RecipientCSV(
file_contents,
template_type=template_type,
whitelist=whitelist
)
if count_of_rows_with_errors:
assert not recipients.allowed_to_send_to
else:
assert recipients.allowed_to_send_to
# Make sure the whitelist isn’t emptied by reading it. If it’s an iterator then
# there’s a risk that it gets emptied after being read once
recipients.whitelist = (str(fake_number) for fake_number in range(7700900888, 7700900898))
list(recipients.whitelist)
assert not recipients.allowed_to_send_to
assert recipients.has_errors
# An empty whitelist is treated as no whitelist at all
recipients.whitelist = []
assert recipients.allowed_to_send_to
recipients.whitelist = itertools.chain()
assert recipients.allowed_to_send_to
def test_get_rows_does_no_error_checking_of_rows_or_cells(mocker):
has_error_mock = mocker.patch.object(Row, 'has_error')
has_bad_recipient_mock = mocker.patch.object(Row, 'has_bad_recipient')
has_missing_data_mock = mocker.patch.object(Row, 'has_missing_data')
cell_recipient_error_mock = mocker.patch.object(Cell, 'recipient_error')
recipients = RecipientCSV("""
email address, name
[email protected],
[email protected], My Name
[email protected],
""",
template_type='email',
placeholders=['name'],
max_errors_shown=3)
rows = recipients.get_rows()
for i in range(3):
assert next(rows).recipient == '*****@*****.**'
assert has_error_mock.called is False
assert has_bad_recipient_mock.called is False
assert has_missing_data_mock.called is False
assert cell_recipient_error_mock.called is False
def recipients(self):
return RecipientCSV(
self.contents,
template=get_sample_template(self.template_type),
allow_international_sms=True,
max_initial_rows_shown=50,
)
def test_detects_rows_which_result_in_overly_long_messages():
template = SMSMessageTemplate(
{
'content': '((placeholder))',
'template_type': 'sms'
},
sender=None,
prefix=None,
)
recipients = RecipientCSV("""
phone number,placeholder
07700900460,1
07700900461,{one_under}
07700900462,{exactly}
07700900463,{one_over}
""".format(
one_under='a' * (SMS_CHAR_COUNT_LIMIT - 1),
exactly='a' * SMS_CHAR_COUNT_LIMIT,
one_over='a' * (SMS_CHAR_COUNT_LIMIT + 1),
),
template_type=template.template_type,
template=template)
assert _index_rows(recipients.rows_with_errors) == {3}
assert _index_rows(recipients.rows_with_message_too_long) == {3}
assert recipients.has_errors
def test_dont_error_if_too_many_recipients_not_specified():
recipients = RecipientCSV(
'phone number,\n07700900460,\n07700900460,\n07700900460,',
template=_sample_template('sms'),
)
assert not recipients.has_errors
assert not recipients.more_rows_than_can_send
def test_dont_error_if_too_many_recipients_not_specified():
recipients = RecipientCSV(
'phone number,\n07700900460,\n07700900460,\n07700900460,',
placeholders=['phone_number'],
template_type='sms')
assert not recipients.has_errors
assert not recipients.more_rows_than_can_send
def process_incomplete_job(job_id):
job = dao_get_job_by_id(job_id)
last_notification_added = dao_get_last_notification_added_for_job_id(job_id)
if last_notification_added:
resume_from_row = last_notification_added.job_row_number
else:
resume_from_row = -1 # The first row in the csv with a number is row 0
current_app.logger.info("Resuming job {} from row {}".format(job_id, resume_from_row))
db_template = dao_get_template_by_id(job.template_id, job.template_version)
TemplateClass = get_template_class(db_template.template_type)
template = TemplateClass(db_template.__dict__)
for row in RecipientCSV(
s3.get_job_from_s3(str(job.service_id), str(job.id)),
template_type=template.template_type,
placeholders=template.placeholders
).rows:
if row.index > resume_from_row:
process_row(row, template, job, job.service)
job_complete(job, resumed=True)
def test_multiple_sms_recipient_columns_with_missing_data(column_name):
recipients = RecipientCSV(
"""
names, phone number, {}
"Joanna and Steve", 07900 900111
""".format(column_name),
template_type='sms',
international_sms=True,
)
expected_column_headers = ['names', 'phone number']
if column_name != "phone number":
expected_column_headers.append(column_name)
assert recipients.column_headers == expected_column_headers
assert recipients.column_headers_as_column_keys == dict(phonenumber='',
names='').keys()
# A piece of weirdness uncovered: since rows are created before spaces in column names are normalised, when
# there are duplicate recipient columns and there is data for only one of the columns, if the columns have the same
# spacing, phone number data will be a list of this one phone number and None, while if the spacing style differs
# between two duplicate column names, the phone number data will be None. If there are no duplicate columns
# then our code finds the phone number well regardless of the spacing, so this should not affect our users.
phone_number_data = None
if column_name == "phone number":
phone_number_data = ['07900 900111', None]
assert recipients.rows[0]['phonenumber'].data == phone_number_data
assert recipients.rows[0].get('phone number').error is None
expected_duplicated_columns = ['phone number']
if column_name != "phone number":
expected_duplicated_columns.append(column_name)
assert recipients.duplicate_recipient_column_headers == OrderedSet(
expected_duplicated_columns)
assert recipients.has_errors
def test_error_if_too_many_recipients():
recipients = RecipientCSV(
'phone number,\n07700900460,\n07700900460,\n07700900460,',
template=_sample_template('sms'),
remaining_messages=2)
assert recipients.has_errors
assert recipients.more_rows_than_can_send
def get_recipient_csv(job: Job, template: Template) -> RecipientCSV:
return RecipientCSV(
s3.get_job_from_s3(str(job.service_id), str(job.id)),
template_type=template.template_type,
placeholders=template.placeholders,
max_rows=get_csv_max_rows(job.service_id),
)
def generate_notifications_csv(**kwargs):
from app import notification_api_client
from app.s3_client.s3_csv_client import s3download
if 'page' not in kwargs:
kwargs['page'] = 1
if kwargs.get('job_id'):
original_file_contents = s3download(kwargs['service_id'],
kwargs['job_id'])
original_upload = RecipientCSV(
original_file_contents,
template_type=kwargs['template_type'],
)
original_column_headers = original_upload.column_headers
fieldnames = ['Row number'] + original_column_headers + [
'Template', 'Type', 'Job', 'Status', 'Time'
]
else:
fieldnames = [
'Recipient', 'Template', 'Type', 'Sent by', 'Sent by email', 'Job',
'Status', 'Time'
]
yield ','.join(fieldnames) + '\n'
while kwargs['page']:
notifications_resp = notification_api_client.get_notifications_for_service(
**kwargs)
for notification in notifications_resp['notifications']:
if kwargs.get('job_id'):
values = [
notification['row_number'],
] + [
original_upload[notification['row_number'] -
1].get(header).data
for header in original_column_headers
] + [
notification['template_name'],
notification['template_type'],
notification['job_name'],
notification['status'],
notification['created_at'],
]
else:
values = [
notification['recipient'], notification['template_name'],
notification['template_type'],
notification['created_by_name'] or '',
notification['created_by_email_address'] or '',
notification['job_name'] or '', notification['status'],
notification['created_at']
]
yield Spreadsheet.from_rows([map(str, values)]).as_csv_data
if notifications_resp['links'].get('next'):
kwargs['page'] += 1
else:
return
raise Exception("Should never reach here")
def get_recipient_csv_and_template_and_sender_id(job):
db_template = dao_get_template_by_id(job.template_id, job.template_version)
template = db_template._as_utils_template()
contents, meta_data = s3.get_job_and_metadata_from_s3(service_id=str(job.service_id), job_id=str(job.id))
recipient_csv = RecipientCSV(contents, template=template)
return recipient_csv, template, meta_data.get("sender_id")
def test_column_headers(file_contents, template_type, expected,
expected_missing):
recipients = RecipientCSV(file_contents,
template=_sample_template(
template_type, '((name))'))
assert recipients.column_headers == expected
assert recipients.missing_column_headers == expected_missing
assert recipients.has_errors == bool(expected_missing)
def test_errors_when_too_many_rows():
recipients = RecipientCSV("email address\n" +
("[email protected]\n" * (RecipientCSV.max_rows + 1)),
template_type='email')
assert RecipientCSV.max_rows == 50000
assert recipients.too_many_rows is True
assert recipients.has_errors is True
assert recipients.annotated_rows == []
def test_get_rows(file_contents, template_type, expected):
rows = list(RecipientCSV(file_contents, template_type=template_type).rows)
if not expected:
assert rows == expected
for index, row in enumerate(expected):
assert len(rows[index].items()) == len(row)
for key, value in row:
assert rows[index].get(key).data == value
def test_column_headers(file_contents, template_type, expected,
expected_missing):
recipients = RecipientCSV(file_contents,
template_type=template_type,
placeholders=['name'])
assert recipients.column_headers == expected
assert recipients.missing_column_headers == expected_missing
assert recipients.has_errors == bool(expected_missing)
def test_get_recipient(file_contents, template_type, placeholders, expected_recipients, expected_personalisation):
recipients = RecipientCSV(file_contents, template_type=template_type, placeholders=placeholders)
for index, row in enumerate(expected_personalisation):
for key, value in row.items():
assert recipients[index].recipient == expected_recipients[index]
assert recipients[index].personalisation.get(key) == value
def test_error_if_too_many_recipients():
recipients = RecipientCSV(
'phone number,\n07700900460,\n07700900460,\n07700900460,',
placeholders=['phone_number'],
template_type='sms',
remaining_messages=2)
assert recipients.has_errors
assert recipients.more_rows_than_can_send
def test_international_recipients(file_contents, rows_with_bad_recipients):
recipients = RecipientCSV(
file_contents,
template=_sample_template('sms'),
allow_international_sms=True,
)
assert _index_rows(
recipients.rows_with_bad_recipients) == rows_with_bad_recipients
def test_multi_line_placeholders_work():
recipients = RecipientCSV("""
email address, data
[email protected], "a\nb\n\nc"
""",
template_type='email',
placeholders=['data'])
assert recipients.rows[0].personalisation['data'] == 'a\nb\n\nc'
def test_errors_when_too_many_rows():
recipients = RecipientCSV("email address\n" + ("[email protected]\n" * (50001)),
template_type='email')
assert recipients.max_rows == 50000
assert recipients.too_many_rows is True
assert recipients.has_errors is True
assert recipients.rows[49000]['email_address'].data == '*****@*****.**'
# We stop processing subsequent rows
assert recipients.rows[50000] is None
def test_get_annotated_rows(file_contents, template_type, expected):
recipients = RecipientCSV(file_contents,
template_type=template_type,
placeholders=['name'],
max_initial_rows_shown=1)
assert list(recipients.annotated_rows) == expected
assert len(list(recipients.annotated_rows)) == 2
assert len(list(recipients.initial_annotated_rows)) == 1
assert not recipients.has_errors
def test_multi_line_placeholders_work():
recipients = RecipientCSV(
"""
email address, data
[email protected], "a\nb\n\nc"
""",
template=_sample_template('email', '((data))'),
)
assert recipients.rows[0].personalisation['data'] == 'a\nb\n\nc'
def test_ignores_spaces_and_case_in_placeholders(key, expected):
recipients = RecipientCSV(
"""
phone number,FIRSTNAME, Last Name
07700900460, Jo, Bloggs
""",
placeholders=['phone_number', 'First Name', 'lastname'],
template_type='sms')
first_row = recipients[0]
assert first_row.get(key).data == expected
assert first_row[key].data == expected
assert first_row.recipient == '07700900460'
assert len(first_row.items()) == 3
assert not recipients.has_errors
assert recipients.missing_column_headers == set()
recipients.placeholders = {'one', 'TWO', 'Thirty_Three'}
assert recipients.missing_column_headers == {'one', 'TWO', 'Thirty_Three'}
assert recipients.has_errors
def test_processing_a_big_list():
process = Mock()
for row in RecipientCSV(
"phone_number\n" + ("07900900900\n" * RecipientCSV.max_rows),
template=_sample_template('sms'),
).get_rows():
process()
assert process.call_count == 50000
def test_get_recipient_respects_order(file_contents, template_type,
placeholders, expected_recipients,
expected_personalisation):
recipients = RecipientCSV(file_contents,
template_type=template_type,
placeholders=placeholders)
recipients_gen = recipients.enumerated_recipients_and_personalisation
for row, email in expected_personalisation:
assert next(recipients_gen) == (row, email, [])
def test_ignores_spaces_and_case_in_placeholders(key, expected):
recipients = RecipientCSV(
"""
phone number,FIRSTNAME, Last Name
07700900460, Jo, Bloggs
""",
placeholders=['phone_number', 'First Name', 'lastname'],
template_type='sms')
first_row = list(recipients.annotated_rows)[0]
assert first_row['columns'].get(key)['data'] == expected
assert first_row['columns'][key]['data'] == expected
assert list(recipients.personalisation)[0][key] == expected
assert list(recipients.recipients) == ['07700900460']
assert len(first_row['columns'].items()) == 3
assert not recipients.has_errors
assert recipients.missing_column_headers == set()
recipients.placeholders = {'one', 'TWO', 'Thirty_Three'}
assert recipients.missing_column_headers == {'one', 'TWO', 'Thirty_Three'}
assert recipients.has_errors
def test_recipients_can_be_accessed_by_index(index, expected_row):
recipients = RecipientCSV("""
phone number, colour
07700 90000 1, red
07700 90000 2, green
07700 90000 3, blue
""",
placeholders=['phone_number'],
template_type='sms')
for key, value in expected_row.items():
assert recipients[index][key].data == value
def test_big_list():
big_csv = RecipientCSV(
"email address,name\n" + ("[email protected]\n" * RecipientCSV.max_rows),
template=_sample_template('email', 'hello ((name))'),
max_errors_shown=100,
max_initial_rows_shown=3,
whitelist=["*****@*****.**"])
assert len(list(big_csv.initial_rows)) == 3
assert len(list(big_csv.initial_rows_with_errors)) == 100
assert len(list(big_csv.rows)) == RecipientCSV.max_rows
assert big_csv.has_errors
def test_big_list():
big_csv = RecipientCSV("email address,name\n" + ("[email protected]\n" * 50000),
template_type='email',
placeholders=['name'],
max_errors_shown=100,
max_initial_rows_shown=3,
safelist=["*****@*****.**"])
assert len(list(big_csv.initial_rows)) == 3
assert len(list(big_csv.initial_rows_with_errors)) == 100
assert len(list(big_csv.rows)) == big_csv.max_rows
assert big_csv.has_errors
def test_ignores_spaces_and_case_in_placeholders(key, expected):
recipients = RecipientCSV(
"""
phone number,FIRSTNAME, Last Name
07700900460, Jo, Bloggs
""",
placeholders=['phone_number', 'First Name', 'lastname'],
template_type='sms'
)
first_row = list(recipients.annotated_rows)[0]
assert first_row['columns'].get(key)['data'] == expected
assert first_row['columns'][key]['data'] == expected
assert list(recipients.personalisation)[0][key] == expected
assert list(recipients.recipients) == ['07700900460']
assert len(first_row['columns'].items()) == 3
assert not recipients.has_errors
assert recipients.missing_column_headers == set()
recipients.placeholders = {'one', 'TWO', 'Thirty_Three'}
assert recipients.missing_column_headers == {'one', 'TWO', 'Thirty_Three'}
assert recipients.has_errors
