def test_recipient_whitelist(file_contents, template_type, whitelist, count_of_rows_with_errors): recipients = RecipientCSV(file_contents, template_type=template_type, whitelist=whitelist) if count_of_rows_with_errors: assert not recipients.allowed_to_send_to else: assert recipients.allowed_to_send_to # Make sure the whitelist isn’t emptied by reading it. If it’s an iterator then # there’s a risk that it gets emptied after being read once recipients.whitelist = (str(fake_number) for fake_number in range(7700900888, 7700900898)) list(recipients.whitelist) assert not recipients.allowed_to_send_to assert recipients.has_errors # An empty whitelist is treated as no whitelist at all recipients.whitelist = [] assert recipients.allowed_to_send_to recipients.whitelist = itertools.chain() assert recipients.allowed_to_send_to
def test_recipient_whitelist(file_contents, template_type, whitelist, count_of_rows_with_errors): recipients = RecipientCSV( file_contents, template_type=template_type, whitelist=whitelist ) if count_of_rows_with_errors: assert not recipients.allowed_to_send_to else: assert recipients.allowed_to_send_to # Make sure the whitelist isn’t emptied by reading it. If it’s an iterator then # there’s a risk that it gets emptied after being read once recipients.whitelist = (str(fake_number) for fake_number in range(7700900888, 7700900898)) list(recipients.whitelist) assert not recipients.allowed_to_send_to assert recipients.has_errors # An empty whitelist is treated as no whitelist at all recipients.whitelist = [] assert recipients.allowed_to_send_to recipients.whitelist = itertools.chain() assert recipients.allowed_to_send_to
def test_get_rows_does_no_error_checking_of_rows_or_cells(mocker): has_error_mock = mocker.patch.object(Row, 'has_error') has_bad_recipient_mock = mocker.patch.object(Row, 'has_bad_recipient') has_missing_data_mock = mocker.patch.object(Row, 'has_missing_data') cell_recipient_error_mock = mocker.patch.object(Cell, 'recipient_error') recipients = RecipientCSV(""" email address, name [email protected], [email protected], My Name [email protected], """, template_type='email', placeholders=['name'], max_errors_shown=3) rows = recipients.get_rows() for i in range(3): assert next(rows).recipient == '*****@*****.**' assert has_error_mock.called is False assert has_bad_recipient_mock.called is False assert has_missing_data_mock.called is False assert cell_recipient_error_mock.called is False
def recipients(self): return RecipientCSV( self.contents, template=get_sample_template(self.template_type), allow_international_sms=True, max_initial_rows_shown=50, )
def test_detects_rows_which_result_in_overly_long_messages(): template = SMSMessageTemplate( { 'content': '((placeholder))', 'template_type': 'sms' }, sender=None, prefix=None, ) recipients = RecipientCSV(""" phone number,placeholder 07700900460,1 07700900461,{one_under} 07700900462,{exactly} 07700900463,{one_over} """.format( one_under='a' * (SMS_CHAR_COUNT_LIMIT - 1), exactly='a' * SMS_CHAR_COUNT_LIMIT, one_over='a' * (SMS_CHAR_COUNT_LIMIT + 1), ), template_type=template.template_type, template=template) assert _index_rows(recipients.rows_with_errors) == {3} assert _index_rows(recipients.rows_with_message_too_long) == {3} assert recipients.has_errors
def test_dont_error_if_too_many_recipients_not_specified(): recipients = RecipientCSV( 'phone number,\n07700900460,\n07700900460,\n07700900460,', template=_sample_template('sms'), ) assert not recipients.has_errors assert not recipients.more_rows_than_can_send
def test_dont_error_if_too_many_recipients_not_specified(): recipients = RecipientCSV( 'phone number,\n07700900460,\n07700900460,\n07700900460,', placeholders=['phone_number'], template_type='sms') assert not recipients.has_errors assert not recipients.more_rows_than_can_send
def process_incomplete_job(job_id): job = dao_get_job_by_id(job_id) last_notification_added = dao_get_last_notification_added_for_job_id(job_id) if last_notification_added: resume_from_row = last_notification_added.job_row_number else: resume_from_row = -1 # The first row in the csv with a number is row 0 current_app.logger.info("Resuming job {} from row {}".format(job_id, resume_from_row)) db_template = dao_get_template_by_id(job.template_id, job.template_version) TemplateClass = get_template_class(db_template.template_type) template = TemplateClass(db_template.__dict__) for row in RecipientCSV( s3.get_job_from_s3(str(job.service_id), str(job.id)), template_type=template.template_type, placeholders=template.placeholders ).rows: if row.index > resume_from_row: process_row(row, template, job, job.service) job_complete(job, resumed=True)
def test_multiple_sms_recipient_columns_with_missing_data(column_name): recipients = RecipientCSV( """ names, phone number, {} "Joanna and Steve", 07900 900111 """.format(column_name), template_type='sms', international_sms=True, ) expected_column_headers = ['names', 'phone number'] if column_name != "phone number": expected_column_headers.append(column_name) assert recipients.column_headers == expected_column_headers assert recipients.column_headers_as_column_keys == dict(phonenumber='', names='').keys() # A piece of weirdness uncovered: since rows are created before spaces in column names are normalised, when # there are duplicate recipient columns and there is data for only one of the columns, if the columns have the same # spacing, phone number data will be a list of this one phone number and None, while if the spacing style differs # between two duplicate column names, the phone number data will be None. If there are no duplicate columns # then our code finds the phone number well regardless of the spacing, so this should not affect our users. phone_number_data = None if column_name == "phone number": phone_number_data = ['07900 900111', None] assert recipients.rows[0]['phonenumber'].data == phone_number_data assert recipients.rows[0].get('phone number').error is None expected_duplicated_columns = ['phone number'] if column_name != "phone number": expected_duplicated_columns.append(column_name) assert recipients.duplicate_recipient_column_headers == OrderedSet( expected_duplicated_columns) assert recipients.has_errors
def test_error_if_too_many_recipients(): recipients = RecipientCSV( 'phone number,\n07700900460,\n07700900460,\n07700900460,', template=_sample_template('sms'), remaining_messages=2) assert recipients.has_errors assert recipients.more_rows_than_can_send
def get_recipient_csv(job: Job, template: Template) -> RecipientCSV: return RecipientCSV( s3.get_job_from_s3(str(job.service_id), str(job.id)), template_type=template.template_type, placeholders=template.placeholders, max_rows=get_csv_max_rows(job.service_id), )
def generate_notifications_csv(**kwargs): from app import notification_api_client from app.s3_client.s3_csv_client import s3download if 'page' not in kwargs: kwargs['page'] = 1 if kwargs.get('job_id'): original_file_contents = s3download(kwargs['service_id'], kwargs['job_id']) original_upload = RecipientCSV( original_file_contents, template_type=kwargs['template_type'], ) original_column_headers = original_upload.column_headers fieldnames = ['Row number'] + original_column_headers + [ 'Template', 'Type', 'Job', 'Status', 'Time' ] else: fieldnames = [ 'Recipient', 'Template', 'Type', 'Sent by', 'Sent by email', 'Job', 'Status', 'Time' ] yield ','.join(fieldnames) + '\n' while kwargs['page']: notifications_resp = notification_api_client.get_notifications_for_service( **kwargs) for notification in notifications_resp['notifications']: if kwargs.get('job_id'): values = [ notification['row_number'], ] + [ original_upload[notification['row_number'] - 1].get(header).data for header in original_column_headers ] + [ notification['template_name'], notification['template_type'], notification['job_name'], notification['status'], notification['created_at'], ] else: values = [ notification['recipient'], notification['template_name'], notification['template_type'], notification['created_by_name'] or '', notification['created_by_email_address'] or '', notification['job_name'] or '', notification['status'], notification['created_at'] ] yield Spreadsheet.from_rows([map(str, values)]).as_csv_data if notifications_resp['links'].get('next'): kwargs['page'] += 1 else: return raise Exception("Should never reach here")
def get_recipient_csv_and_template_and_sender_id(job): db_template = dao_get_template_by_id(job.template_id, job.template_version) template = db_template._as_utils_template() contents, meta_data = s3.get_job_and_metadata_from_s3(service_id=str(job.service_id), job_id=str(job.id)) recipient_csv = RecipientCSV(contents, template=template) return recipient_csv, template, meta_data.get("sender_id")
def test_column_headers(file_contents, template_type, expected, expected_missing): recipients = RecipientCSV(file_contents, template=_sample_template( template_type, '((name))')) assert recipients.column_headers == expected assert recipients.missing_column_headers == expected_missing assert recipients.has_errors == bool(expected_missing)
def test_errors_when_too_many_rows(): recipients = RecipientCSV("email address\n" + ("[email protected]\n" * (RecipientCSV.max_rows + 1)), template_type='email') assert RecipientCSV.max_rows == 50000 assert recipients.too_many_rows is True assert recipients.has_errors is True assert recipients.annotated_rows == []
def test_get_rows(file_contents, template_type, expected): rows = list(RecipientCSV(file_contents, template_type=template_type).rows) if not expected: assert rows == expected for index, row in enumerate(expected): assert len(rows[index].items()) == len(row) for key, value in row: assert rows[index].get(key).data == value
def test_column_headers(file_contents, template_type, expected, expected_missing): recipients = RecipientCSV(file_contents, template_type=template_type, placeholders=['name']) assert recipients.column_headers == expected assert recipients.missing_column_headers == expected_missing assert recipients.has_errors == bool(expected_missing)
def test_get_recipient(file_contents, template_type, placeholders, expected_recipients, expected_personalisation): recipients = RecipientCSV(file_contents, template_type=template_type, placeholders=placeholders) for index, row in enumerate(expected_personalisation): for key, value in row.items(): assert recipients[index].recipient == expected_recipients[index] assert recipients[index].personalisation.get(key) == value
def test_error_if_too_many_recipients(): recipients = RecipientCSV( 'phone number,\n07700900460,\n07700900460,\n07700900460,', placeholders=['phone_number'], template_type='sms', remaining_messages=2) assert recipients.has_errors assert recipients.more_rows_than_can_send
def test_international_recipients(file_contents, rows_with_bad_recipients): recipients = RecipientCSV( file_contents, template=_sample_template('sms'), allow_international_sms=True, ) assert _index_rows( recipients.rows_with_bad_recipients) == rows_with_bad_recipients
def test_multi_line_placeholders_work(): recipients = RecipientCSV(""" email address, data [email protected], "a\nb\n\nc" """, template_type='email', placeholders=['data']) assert recipients.rows[0].personalisation['data'] == 'a\nb\n\nc'
def test_errors_when_too_many_rows(): recipients = RecipientCSV("email address\n" + ("[email protected]\n" * (50001)), template_type='email') assert recipients.max_rows == 50000 assert recipients.too_many_rows is True assert recipients.has_errors is True assert recipients.rows[49000]['email_address'].data == '*****@*****.**' # We stop processing subsequent rows assert recipients.rows[50000] is None
def test_get_annotated_rows(file_contents, template_type, expected): recipients = RecipientCSV(file_contents, template_type=template_type, placeholders=['name'], max_initial_rows_shown=1) assert list(recipients.annotated_rows) == expected assert len(list(recipients.annotated_rows)) == 2 assert len(list(recipients.initial_annotated_rows)) == 1 assert not recipients.has_errors
def test_multi_line_placeholders_work(): recipients = RecipientCSV( """ email address, data [email protected], "a\nb\n\nc" """, template=_sample_template('email', '((data))'), ) assert recipients.rows[0].personalisation['data'] == 'a\nb\n\nc'
def test_ignores_spaces_and_case_in_placeholders(key, expected): recipients = RecipientCSV( """ phone number,FIRSTNAME, Last Name 07700900460, Jo, Bloggs """, placeholders=['phone_number', 'First Name', 'lastname'], template_type='sms') first_row = recipients[0] assert first_row.get(key).data == expected assert first_row[key].data == expected assert first_row.recipient == '07700900460' assert len(first_row.items()) == 3 assert not recipients.has_errors assert recipients.missing_column_headers == set() recipients.placeholders = {'one', 'TWO', 'Thirty_Three'} assert recipients.missing_column_headers == {'one', 'TWO', 'Thirty_Three'} assert recipients.has_errors
def test_processing_a_big_list(): process = Mock() for row in RecipientCSV( "phone_number\n" + ("07900900900\n" * RecipientCSV.max_rows), template=_sample_template('sms'), ).get_rows(): process() assert process.call_count == 50000
def test_get_recipient_respects_order(file_contents, template_type, placeholders, expected_recipients, expected_personalisation): recipients = RecipientCSV(file_contents, template_type=template_type, placeholders=placeholders) recipients_gen = recipients.enumerated_recipients_and_personalisation for row, email in expected_personalisation: assert next(recipients_gen) == (row, email, [])
def test_ignores_spaces_and_case_in_placeholders(key, expected): recipients = RecipientCSV( """ phone number,FIRSTNAME, Last Name 07700900460, Jo, Bloggs """, placeholders=['phone_number', 'First Name', 'lastname'], template_type='sms') first_row = list(recipients.annotated_rows)[0] assert first_row['columns'].get(key)['data'] == expected assert first_row['columns'][key]['data'] == expected assert list(recipients.personalisation)[0][key] == expected assert list(recipients.recipients) == ['07700900460'] assert len(first_row['columns'].items()) == 3 assert not recipients.has_errors assert recipients.missing_column_headers == set() recipients.placeholders = {'one', 'TWO', 'Thirty_Three'} assert recipients.missing_column_headers == {'one', 'TWO', 'Thirty_Three'} assert recipients.has_errors
def test_recipients_can_be_accessed_by_index(index, expected_row): recipients = RecipientCSV(""" phone number, colour 07700 90000 1, red 07700 90000 2, green 07700 90000 3, blue """, placeholders=['phone_number'], template_type='sms') for key, value in expected_row.items(): assert recipients[index][key].data == value
def test_big_list(): big_csv = RecipientCSV( "email address,name\n" + ("[email protected]\n" * RecipientCSV.max_rows), template=_sample_template('email', 'hello ((name))'), max_errors_shown=100, max_initial_rows_shown=3, whitelist=["*****@*****.**"]) assert len(list(big_csv.initial_rows)) == 3 assert len(list(big_csv.initial_rows_with_errors)) == 100 assert len(list(big_csv.rows)) == RecipientCSV.max_rows assert big_csv.has_errors
def test_big_list(): big_csv = RecipientCSV("email address,name\n" + ("[email protected]\n" * 50000), template_type='email', placeholders=['name'], max_errors_shown=100, max_initial_rows_shown=3, safelist=["*****@*****.**"]) assert len(list(big_csv.initial_rows)) == 3 assert len(list(big_csv.initial_rows_with_errors)) == 100 assert len(list(big_csv.rows)) == big_csv.max_rows assert big_csv.has_errors
def test_ignores_spaces_and_case_in_placeholders(key, expected): recipients = RecipientCSV( """ phone number,FIRSTNAME, Last Name 07700900460, Jo, Bloggs """, placeholders=['phone_number', 'First Name', 'lastname'], template_type='sms' ) first_row = list(recipients.annotated_rows)[0] assert first_row['columns'].get(key)['data'] == expected assert first_row['columns'][key]['data'] == expected assert list(recipients.personalisation)[0][key] == expected assert list(recipients.recipients) == ['07700900460'] assert len(first_row['columns'].items()) == 3 assert not recipients.has_errors assert recipients.missing_column_headers == set() recipients.placeholders = {'one', 'TWO', 'Thirty_Three'} assert recipients.missing_column_headers == {'one', 'TWO', 'Thirty_Three'} assert recipients.has_errors