def test_should_send_personalised_template_to_correct_email_provider_and_persist(
notify_db,
notify_db_session,
sample_email_template_with_placeholders,
mocker
):
db_notification = sample_notification(
notify_db=notify_db, notify_db_session=notify_db_session,
template=sample_email_template_with_placeholders,
to_field="*****@*****.**",
personalisation={'name': 'Jo'}
)
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
mocker.patch('app.aws_ses_client.get_name', return_value="ses")
send_to_providers.send_email_to_provider(
db_notification
)
app.aws_ses_client.send_email.assert_called_once_with(
'"Sample service" <*****@*****.**>',
'*****@*****.**',
'Jo',
body='Hello Jo\nThis is an email from GOV.\u200bUK',
html_body=ANY,
reply_to_address=None
)
assert '<!DOCTYPE html' in app.aws_ses_client.send_email.call_args[1]['html_body']
notification = Notification.query.filter_by(id=db_notification.id).one()
assert notification.status == 'sending'
assert notification.sent_at <= datetime.utcnow()
assert notification.sent_by == 'ses'
assert notification.personalisation == {"name": "Jo"}
def test_should_send_personalised_template_to_correct_email_provider_and_persist(
sample_email_template_with_html, mock_email_client,
mocked_build_ga_pixel_url):
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={'name': 'Jo'})
send_to_providers.send_email_to_provider(db_notification)
mock_email_client.send_email.assert_called_once_with(
'"Sample service" <sample.service@{}>'.format(
current_app.config['NOTIFY_EMAIL_DOMAIN']),
'*****@*****.**',
'Jo <em>some HTML</em>',
body=
'Hello Jo\nThis is an email from GOV.\u200bUK with <em>some HTML</em>\n',
html_body=ANY,
reply_to_address=None,
attachments=[])
assert '<!DOCTYPE html' in mock_email_client.send_email.call_args[1][
'html_body']
assert '<em>some HTML</em>' in mock_email_client.send_email.call_args[
1]['html_body']
notification = Notification.query.filter_by(id=db_notification.id).one()
assert notification.status == 'sending'
assert notification.sent_at <= datetime.utcnow()
assert notification.sent_by == mock_email_client.get_name()
assert notification.personalisation == {"name": "Jo"}
def test_should_send_personalised_template_to_correct_email_provider_and_persist(
sample_email_template_with_html, mocker):
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={'name': 'Jo'})
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
send_to_providers.send_email_to_provider(db_notification)
app.aws_ses_client.send_email.assert_called_once_with(
'"Sample service" <*****@*****.**>',
'*****@*****.**',
'Jo <em>some HTML</em>',
body=
'Hello Jo\nThis is an email from GOV.\u200bUK with <em>some HTML</em>\n',
html_body=ANY,
reply_to_address=None)
assert '<!DOCTYPE html' in app.aws_ses_client.send_email.call_args[1][
'html_body']
assert '<em>some HTML</em>' in app.aws_ses_client.send_email.call_args[
1]['html_body']
notification = Notification.query.filter_by(id=db_notification.id).one()
assert notification.status == 'sending'
assert notification.sent_at <= datetime.utcnow()
assert notification.sent_by == 'ses'
assert notification.personalisation == {"name": "Jo"}
def test_should_use_custom_sending_domain_and_email_from(
sample_service,
mock_email_client,
sample_email_template_with_html
):
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={'name': 'Jo'}
)
sample_service.sending_domain = "foo.bar"
sample_service.email_from = "custom-email-from"
send_to_providers.send_email_to_provider(db_notification)
mock_email_client.send_email.assert_called_once_with(
'"Sample service" <*****@*****.**>',
'*****@*****.**',
'Jo <em>some HTML</em>',
body='Hello Jo\nThis is an email from GOV.\u200bUK with <em>some HTML</em>\n',
html_body=ANY,
reply_to_address=None,
attachments=[]
)
def test_should_use_default_sending_domain_and_email_from(
sample_service,
mock_email_client,
sample_email_template_with_html,
notify_api
):
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={'name': 'Jo'}
)
sample_service.sending_domain = None
sample_service.email_from = None
with set_config_values(notify_api, {
'NOTIFY_EMAIL_DOMAIN': 'default.email.domain',
'NOTIFY_EMAIL_FROM': 'default-email-from'
}):
send_to_providers.send_email_to_provider(db_notification)
mock_email_client.send_email.assert_called_once_with(
'"Sample service" <*****@*****.**>',
'*****@*****.**',
'Jo <em>some HTML</em>',
body='Hello Jo\nThis is an email from GOV.\u200bUK with <em>some HTML</em>\n',
html_body=ANY,
reply_to_address=None,
attachments=[]
)
def test_send_email_to_provider_should_call_research_mode_task_response_task_if_research_mode(
sample_service, sample_email_template, mocker, research_mode, key_type
):
notification = create_notification(
template=sample_email_template,
to_field="*****@*****.**",
key_type=key_type,
billable_units=0,
)
sample_service.research_mode = research_mode
reference = str(uuid.uuid4())
mocker.patch("app.aws_ses_client.send_email")
mocker.patch("app.delivery.send_to_providers.send_email_response", return_value=reference)
send_to_providers.send_email_to_provider(notification)
assert not app.aws_ses_client.send_email.called
app.delivery.send_to_providers.send_email_response.assert_called_once_with("*****@*****.**")
persisted_notification = Notification.query.filter_by(id=notification.id).one()
assert persisted_notification.to == "*****@*****.**"
assert persisted_notification.template_id == sample_email_template.id
assert persisted_notification.status == "sending"
assert persisted_notification.sent_at <= datetime.utcnow()
assert persisted_notification.created_at <= datetime.utcnow()
assert persisted_notification.sent_by == "ses"
assert persisted_notification.reference == str(reference)
assert persisted_notification.billable_units == 0
def deliver_email(self, notification_id):
try:
current_app.logger.info(
"Start sending email for notification id: {}".format(
notification_id))
notification = notifications_dao.get_notification_by_id(
notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_email_to_provider(notification)
except EmailClientNonRetryableException as e:
current_app.logger.exception(
f"Email notification {notification_id} failed: {e}")
update_notification_status_by_id(notification_id, 'technical-failure')
except Exception as e:
try:
if isinstance(e, AwsSesClientThrottlingSendRateException):
current_app.logger.warning(
f"RETRY: Email notification {notification_id} was rate limited by SES"
)
else:
current_app.logger.exception(
f"RETRY: Email notification {notification_id} failed")
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task send_email_to_provider failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification_id)
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_send_email_to_provider_should_call_research_mode_task_response_task_if_research_mode(
sample_service,
sample_email_template,
mocker,
mock_email_client,
mocked_build_ga_pixel_url,
research_mode,
key_type):
notification = create_notification(
template=sample_email_template,
to_field="*****@*****.**",
key_type=key_type,
billable_units=0
)
sample_service.research_mode = research_mode
reference = uuid.uuid4()
mocker.patch('app.uuid.uuid4', return_value=reference)
mocker.patch('app.delivery.send_to_providers.send_email_response')
send_to_providers.send_email_to_provider(
notification
)
assert not mock_email_client.send_email.called
app.delivery.send_to_providers.send_email_response.assert_called_once_with(str(reference), '*****@*****.**')
persisted_notification = Notification.query.filter_by(id=notification.id).one()
assert persisted_notification.to == '*****@*****.**'
assert persisted_notification.template_id == sample_email_template.id
assert persisted_notification.status == 'sending'
assert persisted_notification.sent_at <= datetime.utcnow()
assert persisted_notification.created_at <= datetime.utcnow()
assert persisted_notification.sent_by == mock_email_client.get_name()
assert persisted_notification.reference == str(reference)
assert persisted_notification.billable_units == 0
def test_notification_raises_sets_notification_to_virus_found_if_mlwr_score_above_500(
sample_email_template, mocker):
send_mock = mocker.patch("app.aws_ses_client.send_email",
return_value='reference')
mocker.patch('app.delivery.send_to_providers.check_mlwr',
return_value={
"state": "completed",
"submission": {
"max_score": 501
}
})
personalisation = {
"file": {
"document": {
"mlwr_sid": "foo",
"direct_file_url": "http://foo.bar",
"url": "http://foo.bar"
}
}
}
db_notification = create_notification(template=sample_email_template,
personalisation=personalisation)
with pytest.raises(NotificationTechnicalFailureException) as e:
send_to_providers.send_email_to_provider(db_notification)
assert db_notification.id in e.value
send_mock.assert_not_called()
assert Notification.query.get(
db_notification.id).status == 'virus-scan-failed'
def deliver_email(self, notification_id):
try:
current_app.logger.info(
"Start sending email for notification id: {}".format(
notification_id))
notification = notifications_dao.get_notification_by_id(
notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_email_to_provider(notification)
except InvalidEmailError as e:
current_app.logger.exception(e)
update_notification_status_by_id(notification_id, 'technical-failure')
except Exception as e:
try:
current_app.logger.exception(
"RETRY: Email notification {} failed".format(notification_id))
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task send_email_to_provider failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification_id)
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_notification_raises_a_retry_exception_if_mlwr_state_is_not_complete(sample_email_template, mocker):
mocker.patch("app.aws_ses_client.send_email", return_value="reference")
mocker.patch("app.delivery.send_to_providers.check_mlwr", return_value={"state": "foo"})
personalisation = {"file": document_download_response()}
db_notification = create_notification(template=sample_email_template, personalisation=personalisation)
with pytest.raises(MalwarePendingException):
send_to_providers.send_email_to_provider(
db_notification,
)
def test_should_not_send_email_message_when_service_is_inactive_notification_is_in_tech_failure(
sample_service, sample_notification, mock_email_client,
mocked_build_ga_pixel_url):
sample_service.active = False
with pytest.raises(NotificationTechnicalFailureException) as e:
send_to_providers.send_email_to_provider(sample_notification)
assert str(sample_notification.id) in str(e.value)
mock_email_client.send_email.assert_not_called()
assert Notification.query.get(
sample_notification.id).status == 'technical-failure'
def test_should_not_send_email_message_when_service_is_inactive_notifcation_is_in_tech_failure(
sample_service, sample_notification, mocker
):
sample_service.active = False
send_mock = mocker.patch("app.aws_ses_client.send_email", return_value="reference")
with pytest.raises(NotificationTechnicalFailureException) as e:
send_to_providers.send_email_to_provider(sample_notification)
assert str(sample_notification.id) in str(e.value)
send_mock.assert_not_called()
assert Notification.query.get(sample_notification.id).status == "technical-failure"
def test_send_email_to_provider_should_format_email_address(
sample_email_notification,
mock_email_client,
mocked_build_ga_pixel_url
):
sample_email_notification.to = '[email protected]\t'
send_to_providers.send_email_to_provider(sample_email_notification)
_, kwargs = mock_email_client.send_email.call_args
assert kwargs['to_addresses'] == '*****@*****.**'
def test_send_email_to_provider_should_format_reply_to_email_address(
sample_email_template,
mock_email_client,
mocked_build_ga_pixel_url
):
db_notification = create_notification(template=sample_email_template, reply_to_text="[email protected]\t")
send_to_providers.send_email_to_provider(db_notification)
_, kwargs = mock_email_client.send_email.call_args
assert kwargs['reply_to_address'] == "*****@*****.**"
def test_notification_raises_a_retry_exception_if_mlwr_state_is_missing(sample_email_template, mocker):
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
mocker.patch('app.delivery.send_to_providers.check_mlwr', return_value={})
personalisation = {
"file": {"document": {"mlwr_sid": "foo", "direct_file_url": "http://foo.bar", "url": "http://foo.bar"}}}
db_notification = create_notification(template=sample_email_template, personalisation=personalisation)
with pytest.raises(MalwarePendingException):
send_to_providers.send_email_to_provider(
db_notification,
)
def test_send_email_should_use_service_reply_to_email(
sample_service,
sample_email_template,
mock_email_client,
mocked_build_ga_pixel_url
):
db_notification = create_notification(template=sample_email_template, reply_to_text='*****@*****.**')
create_reply_to_email(service=sample_service, email_address='*****@*****.**')
send_to_providers.send_email_to_provider(db_notification)
_, kwargs = mock_email_client.send_email.call_args
assert kwargs['reply_to_address'] == '*****@*****.**'
def test_send_email_should_use_service_reply_to_email(sample_service,
sample_email_template,
mocker):
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
db_notification = create_notification(template=sample_email_template,
reply_to_text='*****@*****.**')
create_reply_to_email(service=sample_service, email_address='*****@*****.**')
send_to_providers.send_email_to_provider(db_notification, )
app.aws_ses_client.send_email.assert_called_once_with(
ANY, ANY, ANY, body=ANY, html_body=ANY, reply_to_address='*****@*****.**')
def test_notification_passes_if_message_contains_sin_pii_that_fails_luhn(
sample_email_template_with_html, mock_email_client,
mocked_build_ga_pixel_url):
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={'name': '123-456-789'})
send_to_providers.send_email_to_provider(db_notification)
mock_email_client.send_email.assert_called()
assert Notification.query.get(db_notification.id).status == 'sending'
def test_notification_passes_if_message_contains_phone_number(sample_email_template_with_html, mock_email_client):
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={'name': '123-456-7890'}
)
send_to_providers.send_email_to_provider(db_notification)
mock_email_client.send_email.assert_called()
assert Notification.query.get(db_notification.id).status == 'sending'
def test_notification_can_have_document_attachment_if_mlwr_sid_is_false(sample_email_template, mocker):
send_mock = mocker.patch("app.aws_ses_client.send_email", return_value="reference")
mlwr_mock = mocker.patch("app.delivery.send_to_providers.check_mlwr")
personalisation = {"file": document_download_response({"mlwr_sid": "false"})}
db_notification = create_notification(template=sample_email_template, personalisation=personalisation)
send_to_providers.send_email_to_provider(
db_notification,
)
send_mock.assert_called()
mlwr_mock.assert_not_called()
def test_notification_passes_if_message_contains_phone_number(sample_email_template_with_html, mocker):
send_mock = mocker.patch("app.aws_ses_client.send_email", return_value="reference")
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={"name": "123-456-7890"},
)
send_to_providers.send_email_to_provider(db_notification)
send_mock.assert_called()
assert Notification.query.get(db_notification.id).status == "sending"
def test_notification_passes_if_message_contains_sin_pii_that_fails_luhn(
sample_email_template_with_html, mocker, notify_api):
send_mock = mocker.patch("app.aws_ses_client.send_email",
return_value='reference')
db_notification = create_notification(
template=sample_email_template_with_html,
to_field="*****@*****.**",
personalisation={'name': '123-456-789'})
send_to_providers.send_email_to_provider(db_notification)
send_mock.assert_called()
assert Notification.query.get(db_notification.id).status == 'sending'
def test_send_email_to_provider_should_format_reply_to_email_address(
sample_email_template, mock_email_client, mocked_build_ga_pixel_url):
db_notification = create_notification(template=sample_email_template,
reply_to_text="[email protected]\t")
send_to_providers.send_email_to_provider(db_notification, )
mock_email_client.send_email.assert_called_once_with(
ANY,
ANY,
ANY,
body=ANY,
html_body=ANY,
reply_to_address="*****@*****.**",
attachments=[])
def test_notification_can_have_document_attachment_without_mlwr_sid(sample_email_template, mocker):
send_mock = mocker.patch("app.aws_ses_client.send_email", return_value="reference")
mlwr_mock = mocker.patch("app.delivery.send_to_providers.check_mlwr")
response = document_download_response()
del response["document"]["mlwr_sid"]
personalisation = {"file": response}
db_notification = create_notification(template=sample_email_template, personalisation=personalisation)
send_to_providers.send_email_to_provider(
db_notification,
)
send_mock.assert_called()
mlwr_mock.assert_not_called()
def test_send_email_to_provider_should_format_email_address(sample_email_notification, mock_email_client):
sample_email_notification.to = '[email protected]\t'
send_to_providers.send_email_to_provider(sample_email_notification)
# to_addresses
mock_email_client.send_email.assert_called_once_with(
ANY,
# to_addresses
'*****@*****.**',
ANY,
body=ANY,
html_body=ANY,
reply_to_address=ANY,
attachments=[]
)
def test_notification_raises_sets_notification_to_virus_found_if_mlwr_score_above_500(sample_email_template, mocker):
send_mock = mocker.patch("app.aws_ses_client.send_email", return_value="reference")
mocker.patch(
"app.delivery.send_to_providers.check_mlwr",
return_value={"state": "completed", "submission": {"max_score": 501}},
)
personalisation = {"file": document_download_response()}
db_notification = create_notification(template=sample_email_template, personalisation=personalisation)
with pytest.raises(NotificationTechnicalFailureException) as e:
send_to_providers.send_email_to_provider(db_notification)
assert db_notification.id in e.value
send_mock.assert_not_called()
assert Notification.query.get(db_notification.id).status == "virus-scan-failed"
def test_send_email_to_provider_should_format_email_address(sample_email_notification, mocker):
sample_email_notification.to = '[email protected]\t'
send_mock = mocker.patch('app.aws_ses_client.send_email', return_value='reference')
send_to_providers.send_email_to_provider(sample_email_notification)
# to_addresses
send_mock.assert_called_once_with(
ANY,
# to_addresses
'*****@*****.**',
ANY,
body=ANY,
html_body=ANY,
reply_to_address=ANY,
)
def test_send_email_to_provider_should_user_normalised_to(
mocker, client, sample_email_template):
send_mock = mocker.patch('app.aws_ses_client.send_email',
return_value='reference')
notification = create_notification(template=sample_email_template,
to_field='*****@*****.**',
normalised_to='*****@*****.**')
send_to_providers.send_email_to_provider(notification)
send_mock.assert_called_once_with(
ANY,
notification.normalised_to,
ANY,
body=ANY,
html_body=ANY,
reply_to_address=notification.reply_to_text)
def test_send_email_to_provider_should_format_reply_to_email_address(
sample_email_template, mocker):
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
db_notification = create_notification(template=sample_email_template,
reply_to_text="[email protected]\t")
send_to_providers.send_email_to_provider(db_notification, )
app.aws_ses_client.send_email.assert_called_once_with(
ANY,
ANY,
ANY,
body=ANY,
html_body=ANY,
reply_to_address="*****@*****.**")
def deliver_email(self, notification_id):
try:
current_app.logger.info(
"Start sending email for notification id: {}".format(
notification_id))
notification = notifications_dao.get_notification_by_id(
notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_email_to_provider(notification)
current_app.logger.info(
f"Successfully sent email for notification id: {notification_id}")
except InvalidEmailError as e:
current_app.logger.exception(
f"Email notification {notification_id} failed: {str(e)}")
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(str(e))
except MalwarePendingException:
current_app.logger.info(
f"RETRY number {self.request.retries}: Email notification {notification_id} is pending malware scans"
)
self.retry(queue=QueueNames.RETRY, countdown=60)
except InvalidProviderException as e:
current_app.logger.exception(
f"Invalid provider for {notification_id}: {str(e)}")
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(str(e))
except Exception as e:
try:
if isinstance(e, AwsSesClientThrottlingSendRateException):
current_app.logger.warning(
f"RETRY number {self.request.retries}: Email notification {notification_id} was rate limited by SES"
)
else:
current_app.logger.exception(
f"RETRY number {self.request.retries}: Email notification {notification_id} failed"
)
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task send_email_to_provider failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification_id)
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_send_email_to_provider_should_call_research_mode_task_response_task_if_research_mode(
notify_db,
notify_db_session,
sample_service,
sample_email_template,
ses_provider,
mocker,
research_mode,
key_type):
notification = sample_notification(notify_db=notify_db, notify_db_session=notify_db_session,
template=sample_email_template,
to_field="*****@*****.**",
key_type=key_type
)
reference = uuid.uuid4()
mocker.patch('app.uuid.uuid4', return_value=reference)
mocker.patch('app.aws_ses_client.send_email')
mocker.patch('app.aws_ses_client.get_name', return_value="ses")
mocker.patch('app.celery.research_mode_tasks.send_email_response.apply_async')
if research_mode:
sample_service.research_mode = True
notify_db.session.add(sample_service)
notify_db.session.commit()
send_to_providers.send_email_to_provider(
notification
)
assert not app.aws_ses_client.send_email.called
send_to_providers.send_email_response.apply_async.assert_called_once_with(
('ses', str(reference), '*****@*****.**'), queue="research-mode"
)
persisted_notification = Notification.query.filter_by(id=notification.id).one()
assert persisted_notification.to == '*****@*****.**'
assert persisted_notification.template_id == sample_email_template.id
assert persisted_notification.status == 'sending'
assert persisted_notification.sent_at <= datetime.utcnow()
assert persisted_notification.created_at <= datetime.utcnow()
assert persisted_notification.sent_by == 'ses'
assert persisted_notification.reference == str(reference)
assert persisted_notification.billable_units == 0
def deliver_email(self, notification_id):
try:
notification = notifications_dao.get_notification_by_id(notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_email_to_provider(notification)
except InvalidEmailError as e:
current_app.logger.exception(e)
update_notification_status_by_id(notification_id, 'technical-failure')
except Exception as e:
try:
current_app.logger.exception(
"RETRY: Email notification {} failed".format(notification_id)
)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id)
)
update_notification_status_by_id(notification_id, 'technical-failure')
def test_send_email_should_use_service_reply_to_email(
notify_db, notify_db_session,
sample_service,
sample_email_template,
mocker):
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
mocker.patch('app.aws_ses_client.get_name', return_value="ses")
db_notification = sample_notification(notify_db, notify_db_session, template=sample_email_template)
sample_service.reply_to_email_address = '*****@*****.**'
send_to_providers.send_email_to_provider(
db_notification,
)
app.aws_ses_client.send_email.assert_called_once_with(
ANY,
ANY,
ANY,
body=ANY,
html_body=ANY,
reply_to_address=sample_service.reply_to_email_address
)