def test_get_errors_for_csv(app_, rows_with_bad_recipients,
rows_with_missing_data, template_type,
expected_errors):
with app_.test_request_context():
assert get_errors_for_csv(
MockRecipients(rows_with_bad_recipients, rows_with_missing_data),
template_type) == expected_errors
def test_get_errors_for_csv(rows_with_bad_recipients, rows_with_missing_data,
rows_with_message_too_long,
rows_with_empty_message, template_type,
expected_errors):
assert get_errors_for_csv(
MockRecipients(rows_with_bad_recipients, rows_with_missing_data,
rows_with_message_too_long, rows_with_empty_message),
template_type) == expected_errors
def test_get_errors_for_csv(
rows_with_bad_recipients, rows_with_missing_data,
template_type,
expected_errors
):
assert get_errors_for_csv(
MockRecipients(rows_with_bad_recipients, rows_with_missing_data),
template_type
) == expected_errors
def _check_messages(service_id,
template_id,
upload_id,
preview_row,
letters_as_pdf=False):
try:
# The happy path is that the job doesn’t already exist, so the
# API will return a 404 and the client will raise HTTPError.
job_api_client.get_job(service_id, upload_id)
# the job exists already - so go back to the templates page
# If we just return a `redirect` (302) object here, we'll get
# errors when we try and unpack in the check_messages route.
# Rasing a werkzeug.routing redirect means that doesn't happen.
raise PermanentRedirect(
url_for('.send_messages',
service_id=service_id,
template_id=template_id))
except HTTPError as e:
if e.status_code != 404:
raise
statistics = service_api_client.get_service_statistics(service_id,
today_only=True)
remaining_messages = (current_service.message_limit -
sum(stat['requested']
for stat in statistics.values()))
contents = s3download(service_id, upload_id)
db_template = current_service.get_template_with_user_permission_or_403(
template_id, current_user)
email_reply_to = None
sms_sender = None
if db_template['template_type'] == 'email':
email_reply_to = get_email_reply_to_address_from_session()
elif db_template['template_type'] == 'sms':
sms_sender = get_sms_sender_from_session()
template = get_template(
db_template,
current_service,
show_recipient=True,
letter_preview_url=url_for(
'.check_messages_preview',
service_id=service_id,
template_id=template_id,
upload_id=upload_id,
filetype='png',
row_index=preview_row,
) if not letters_as_pdf else None,
email_reply_to=email_reply_to,
sms_sender=sms_sender,
page_count=get_page_count_for_letter(db_template),
)
recipients = RecipientCSV(
contents,
template_type=template.template_type,
placeholders=template.placeholders,
max_initial_rows_shown=50,
max_errors_shown=50,
whitelist=itertools.chain.from_iterable(
[user.name, user.mobile_number, user.email_address]
for user in Users(service_id))
if current_service.trial_mode else None,
remaining_messages=remaining_messages,
international_sms=current_service.has_permission('international_sms'),
)
if request.args.get('from_test'):
# only happens if generating a letter preview test
back_link = url_for('.send_test',
service_id=service_id,
template_id=template.id)
choose_time_form = None
else:
back_link = url_for('.send_messages',
service_id=service_id,
template_id=template.id)
choose_time_form = ChooseTimeForm()
if preview_row < 2:
abort(404)
if preview_row < len(recipients) + 2:
template.values = recipients[preview_row -
2].recipient_and_personalisation
elif preview_row > 2:
abort(404)
return dict(
recipients=recipients,
template=template,
errors=recipients.has_errors,
row_errors=get_errors_for_csv(recipients, template.template_type),
count_of_recipients=len(recipients),
count_of_displayed_recipients=len(list(recipients.displayed_rows)),
original_file_name=request.args.get('original_file_name', ''),
upload_id=upload_id,
form=CsvUploadForm(),
remaining_messages=remaining_messages,
choose_time_form=choose_time_form,
back_link=back_link,
help=get_help_argument(),
trying_to_send_letters_in_trial_mode=all((
current_service.trial_mode,
template.template_type == 'letter',
)),
required_recipient_columns=OrderedSet(
recipients.recipient_column_headers) - optional_address_columns,
preview_row=preview_row,
sent_previously=job_api_client.has_sent_previously(
service_id, template.id, db_template['version'],
request.args.get('original_file_name', '')))
def check_messages(service_id, template_type, upload_id):
if not session.get('upload_data'):
return redirect(
url_for('main.choose_template',
service_id=service_id,
template_type=template_type))
users = user_api_client.get_users_for_service(service_id=service_id)
statistics = service_api_client.get_detailed_service_for_today(
service_id)['data']['statistics']
remaining_messages = (current_service['message_limit'] -
sum(stat['requested']
for stat in statistics.values()))
contents = s3download(service_id, upload_id)
if not contents:
flash('There was a problem reading your upload file')
template = Template(service_api_client.get_service_template(
service_id, session['upload_data'].get('template_id'))['data'],
prefix=current_service['name'],
sms_sender=current_service['sms_sender'])
recipients = RecipientCSV(
contents,
template_type=template.template_type,
placeholders=template.placeholders,
max_initial_rows_shown=50,
max_errors_shown=50,
whitelist=itertools.chain.from_iterable(
[user.mobile_number, user.email_address]
for user in users) if current_service['restricted'] else None,
remaining_messages=remaining_messages)
if request.args.get('from_test'):
extra_args = {
'help': 1
} if request.args.get('help', '0') != '0' else {}
if len(template.placeholders):
back_link = url_for('.send_test',
service_id=service_id,
template_id=template.id,
**extra_args)
else:
back_link = url_for('.choose_template',
service_id=service_id,
template_type=template.template_type,
**extra_args)
choose_time_form = None
else:
back_link = url_for('.send_messages',
service_id=service_id,
template_id=template.id)
choose_time_form = ChooseTimeForm()
with suppress(StopIteration):
template.values = next(recipients.rows)
first_recipient = template.values.get(
recipients.recipient_column_header, '')
session['upload_data']['notification_count'] = len(list(recipients.rows))
session['upload_data']['valid'] = not recipients.has_errors
return render_template(
'views/check.html',
recipients=recipients,
first_recipient=first_recipient,
template=template,
errors=recipients.has_errors,
row_errors=get_errors_for_csv(recipients, template.template_type),
count_of_recipients=session['upload_data']['notification_count'],
count_of_displayed_recipients=(
len(list(recipients.initial_annotated_rows_with_errors))
if any(recipients.rows_with_errors)
and not recipients.missing_column_headers else len(
list(recipients.initial_annotated_rows))),
original_file_name=session['upload_data'].get('original_file_name'),
upload_id=upload_id,
form=CsvUploadForm(),
remaining_messages=remaining_messages,
choose_time_form=choose_time_form,
back_link=back_link,
help=get_help_argument())
def _check_messages(service_id, template_type, upload_id, preview_row, letters_as_pdf=False):
if not session.get('upload_data'):
# if we just return a `redirect` (302) object here, we'll get errors when we try and unpack in the
# check_messages route - so raise a werkzeug.routing redirect to ensure that doesn't happen.
# NOTE: this is a 301 MOVED PERMANENTLY (httpstatus.es/301), so the browser will cache this redirect, and it'll
# *always* happen for that browser. _check_messages is only used by endpoints that contain `upload_id`, which
# is a one-time-use id (that ties to a given file in S3 that is already deleted if it's not in the session)
raise RequestRedirect(url_for('main.choose_template', service_id=service_id))
users = user_api_client.get_users_for_service(service_id=service_id)
statistics = service_api_client.get_detailed_service_for_today(service_id)['data']['statistics']
remaining_messages = (current_service['message_limit'] - sum(stat['requested'] for stat in statistics.values()))
contents = s3download(service_id, upload_id)
email_reply_to = None
sms_sender = None
if template_type == 'email':
email_reply_to = get_email_reply_to_address_from_session(service_id)
elif template_type == 'sms':
sms_sender = get_sms_sender_from_session(service_id)
template = get_template(
service_api_client.get_service_template(
service_id,
session['upload_data'].get('template_id')
)['data'],
current_service,
show_recipient=True,
letter_preview_url=url_for(
'.check_messages_preview',
service_id=service_id,
template_type=template_type,
upload_id=upload_id,
filetype='png',
row_index=preview_row,
) if not letters_as_pdf else None,
email_reply_to=email_reply_to,
sms_sender=sms_sender
)
recipients = RecipientCSV(
contents,
template_type=template.template_type,
placeholders=template.placeholders,
max_initial_rows_shown=50,
max_errors_shown=50,
whitelist=itertools.chain.from_iterable(
[user.name, user.mobile_number, user.email_address] for user in users
) if current_service['restricted'] else None,
remaining_messages=remaining_messages,
international_sms='international_sms' in current_service['permissions'],
)
if request.args.get('from_test'):
# only happens if generating a letter preview test
back_link = url_for('.send_test', service_id=service_id, template_id=template.id)
choose_time_form = None
else:
back_link = url_for('.send_messages', service_id=service_id, template_id=template.id)
choose_time_form = ChooseTimeForm()
count_of_recipients = len(list(recipients.rows))
if preview_row < 2:
abort(404)
if preview_row < count_of_recipients + 2:
template.values = recipients[preview_row - 2]
elif preview_row > 2:
abort(404)
session['upload_data']['notification_count'] = count_of_recipients
session['upload_data']['valid'] = not recipients.has_errors
return dict(
recipients=recipients,
template=template,
errors=recipients.has_errors,
row_errors=get_errors_for_csv(recipients, template.template_type),
count_of_recipients=count_of_recipients,
count_of_displayed_recipients=(
len(list(recipients.initial_annotated_rows_with_errors))
if any(recipients.rows_with_errors) and not recipients.missing_column_headers else
len(list(recipients.initial_annotated_rows))
),
original_file_name=session['upload_data'].get('original_file_name'),
upload_id=upload_id,
form=CsvUploadForm(),
remaining_messages=remaining_messages,
choose_time_form=choose_time_form,
back_link=back_link,
help=get_help_argument(),
trying_to_send_letters_in_trial_mode=all((
current_service['restricted'],
template.template_type == 'letter',
not request.args.get('from_test'),
)),
required_recipient_columns=OrderedSet(recipients.recipient_column_headers) - optional_address_columns,
preview_row=preview_row,
)
def check_contact_list(service_id, upload_id):
form = CsvUploadForm()
contents = ContactList.download(service_id, upload_id)
first_row = contents.splitlines()[0].strip().rstrip(
',') if contents else ''
original_file_name = ContactList.get_metadata(service_id, upload_id).get(
'original_file_name', '')
template_type = {
'emailaddress': 'email',
'phonenumber': 'sms',
}.get(Columns.make_key(first_row))
recipients = RecipientCSV(
contents,
template=get_sample_template(template_type or 'sms'),
whitelist=itertools.chain.from_iterable(
[user.name, user.mobile_number, user.email_address]
for user in current_service.active_users)
if current_service.trial_mode else None,
allow_international_sms=current_service.has_permission(
'international_sms'),
max_initial_rows_shown=50,
max_errors_shown=50,
)
non_empty_column_headers = list(filter(None, recipients.column_headers))
if len(non_empty_column_headers
) > 1 or not template_type or not recipients:
return render_template(
'views/uploads/contact-list/too-many-columns.html',
recipients=recipients,
original_file_name=original_file_name,
template_type=template_type,
form=form,
allowed_file_extensions=Spreadsheet.ALLOWED_FILE_EXTENSIONS)
if recipients.too_many_rows or not len(recipients):
return render_template(
'views/uploads/contact-list/column-errors.html',
recipients=recipients,
original_file_name=original_file_name,
form=form,
allowed_file_extensions=Spreadsheet.ALLOWED_FILE_EXTENSIONS)
row_errors = get_errors_for_csv(recipients, template_type)
if row_errors:
return render_template(
'views/uploads/contact-list/row-errors.html',
recipients=recipients,
original_file_name=original_file_name,
row_errors=row_errors,
form=form,
allowed_file_extensions=Spreadsheet.ALLOWED_FILE_EXTENSIONS)
if recipients.has_errors:
return render_template(
'views/uploads/contact-list/column-errors.html',
recipients=recipients,
original_file_name=original_file_name,
form=form,
allowed_file_extensions=Spreadsheet.ALLOWED_FILE_EXTENSIONS)
metadata_kwargs = {
'row_count': len(recipients),
'valid': True,
'original_file_name': original_file_name,
'template_type': template_type
}
ContactList.set_metadata(service_id, upload_id, **metadata_kwargs)
return render_template(
'views/uploads/contact-list/ok.html',
recipients=recipients,
original_file_name=original_file_name,
upload_id=upload_id,
)
def check_messages(service_id, template_type, upload_id):
if not session.get('upload_data'):
return redirect(url_for('main.choose_template', service_id=service_id, template_type=template_type))
users = user_api_client.get_users_for_service(service_id=service_id)
today = datetime.utcnow().date().strftime('%Y-%m-%d')
statistics = statistics_api_client.get_statistics_for_service_for_day(service_id, today)
if not statistics:
statistics = {}
contents = s3download(service_id, upload_id)
if not contents:
flash('There was a problem reading your upload file')
template = service_api_client.get_service_template(
service_id,
session['upload_data'].get('template_id')
)['data']
template = Template(
template,
prefix=current_service['name']
)
recipients = RecipientCSV(
contents,
template_type=template.template_type,
placeholders=template.placeholders,
max_initial_rows_shown=50,
max_errors_shown=50,
whitelist=itertools.chain.from_iterable(
[user.mobile_number, user.email_address] for user in users
) if current_service['restricted'] else None
)
if request.args.get('from_test'):
extra_args = {'help': 1} if request.args.get('help', '0') != '0' else {}
if len(template.placeholders):
back_link = url_for(
'.send_test', service_id=service_id, template_id=template.id, **extra_args
)
else:
back_link = url_for(
'.choose_template', service_id=service_id, template_type=template.template_type, **extra_args
)
else:
back_link = url_for('.send_messages', service_id=service_id, template_id=template.id)
with suppress(StopIteration):
template.values = next(recipients.rows)
first_recipient = template.values.get(recipients.recipient_column_header, '')
session['upload_data']['notification_count'] = len(list(recipients.rows))
session['upload_data']['valid'] = not recipients.has_errors
return render_template(
'views/check.html',
recipients=recipients,
first_recipient=first_recipient,
template=template,
errors=recipients.has_errors,
row_errors=get_errors_for_csv(recipients, template.template_type),
count_of_recipients=session['upload_data']['notification_count'],
count_of_displayed_recipients=(
len(list(recipients.initial_annotated_rows_with_errors))
if any(recipients.rows_with_errors) and not recipients.missing_column_headers else
len(list(recipients.initial_annotated_rows))
),
original_file_name=session['upload_data'].get('original_file_name'),
upload_id=upload_id,
form=CsvUploadForm(),
statistics=statistics,
back_link=back_link,
help=get_help_argument()
)
def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_pdf=False):
users = user_api_client.get_users_for_service(service_id=service_id)
statistics = service_api_client.get_detailed_service_for_today(service_id)['data']['statistics']
remaining_messages = (current_service['message_limit'] - sum(stat['requested'] for stat in statistics.values()))
contents = s3download(service_id, upload_id)
db_template = service_api_client.get_service_template(
service_id,
str(template_id),
)['data']
email_reply_to = None
sms_sender = None
if db_template['template_type'] == 'email':
email_reply_to = get_email_reply_to_address_from_session(service_id)
elif db_template['template_type'] == 'sms':
sms_sender = get_sms_sender_from_session(service_id)
template = get_template(
service_api_client.get_service_template(
service_id,
str(template_id),
)['data'],
current_service,
show_recipient=True,
letter_preview_url=url_for(
'.check_messages_preview',
service_id=service_id,
template_id=template_id,
upload_id=upload_id,
filetype='png',
row_index=preview_row,
) if not letters_as_pdf else None,
email_reply_to=email_reply_to,
sms_sender=sms_sender,
)
recipients = RecipientCSV(
contents,
template_type=template.template_type,
placeholders=template.placeholders,
max_initial_rows_shown=50,
max_errors_shown=50,
whitelist=itertools.chain.from_iterable(
[user.name, user.mobile_number, user.email_address] for user in users
) if current_service['restricted'] else None,
remaining_messages=remaining_messages,
international_sms='international_sms' in current_service['permissions'],
)
if request.args.get('from_test'):
# only happens if generating a letter preview test
back_link = url_for('.send_test', service_id=service_id, template_id=template.id)
choose_time_form = None
else:
back_link = url_for('.send_messages', service_id=service_id, template_id=template.id)
choose_time_form = ChooseTimeForm()
if preview_row < 2:
abort(404)
if preview_row < len(recipients) + 2:
template.values = recipients[preview_row - 2].recipient_and_personalisation
elif preview_row > 2:
abort(404)
if 'file_uploads' not in session:
session['file_uploads'] = {}
session['file_uploads'][upload_id] = {}
if any(recipients) and not recipients.has_errors:
session['file_uploads'][upload_id]['notification_count'] = len(recipients)
session['file_uploads'][upload_id]['template_id'] = str(template_id)
session['file_uploads'][upload_id]['valid'] = True
else:
session['file_uploads'].pop(upload_id)
return dict(
recipients=recipients,
template=template,
errors=recipients.has_errors,
row_errors=get_errors_for_csv(recipients, template.template_type),
count_of_recipients=len(recipients),
count_of_displayed_recipients=len(list(recipients.displayed_rows)),
original_file_name=request.args.get('original_file_name'),
upload_id=upload_id,
form=CsvUploadForm(),
remaining_messages=remaining_messages,
choose_time_form=choose_time_form,
back_link=back_link,
help=get_help_argument(),
trying_to_send_letters_in_trial_mode=all((
current_service['restricted'],
template.template_type == 'letter',
not request.args.get('from_test'),
)),
required_recipient_columns=OrderedSet(recipients.recipient_column_headers) - optional_address_columns,
preview_row=preview_row,
)
