def test_ses_callback_should_not_set_status_once_status_is_delivered(notify_api,
notify_db,
notify_db_session,
sample_email_template):
with notify_api.test_request_context():
with notify_api.test_client() as client:
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='delivered',
sent_at=datetime.utcnow()
)
assert get_notification_by_id(notification.id).status == 'delivered'
response = client.post(
path='/notifications/email/ses',
data=ses_soft_bounce_callback(),
headers=[('Content-Type', 'text/plain; charset=UTF-8')]
)
json_resp = json.loads(response.get_data(as_text=True))
assert response.status_code == 404
assert json_resp['result'] == 'error'
assert json_resp['message'] == 'SES callback failed: notification either not found or already updated from sending. Status temporary-failure' # noqa
assert get_notification_by_id(notification.id).status == 'delivered'
def test_ses_callback_should_set_status_to_permanent_failure(notify_api,
notify_db,
notify_db_session,
sample_email_template):
with notify_api.test_request_context():
with notify_api.test_client() as client:
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow()
)
assert get_notification_by_id(notification.id).status == 'sending'
response = client.post(
path='/notifications/email/ses',
data=ses_hard_bounce_callback(),
headers=[('Content-Type', 'text/plain; charset=UTF-8')]
)
json_resp = json.loads(response.get_data(as_text=True))
assert response.status_code == 200
assert json_resp['result'] == 'success'
assert json_resp['message'] == 'SES callback succeeded'
assert get_notification_by_id(notification.id).status == 'permanent-failure'
def test_firetext_callback_should_update_notification_status_failed(
notify_db, notify_db_session, client, sample_template, mocker
):
mocker.patch('app.statsd_client.incr')
mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow())
original = get_notification_by_id(notification.id)
assert original.status == 'sending'
data = 'mobile=61412345678&status=1&time=2016-03-10 14:17:00&reference={}'.format(
notification.id)
response = firetext_post(client, data)
json_resp = json.loads(response.get_data(as_text=True))
assert response.status_code == 200
assert json_resp['result'] == 'success'
assert json_resp['message'] == 'Firetext callback succeeded. reference {} updated'.format(
notification.id
)
assert get_notification_by_id(notification.id).status == 'permanent-failure'
def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry(
client,
notify_db,
notify_db_session,
sample_email_template,
mocker):
with freeze_time('2001-01-01T12:00:00'):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow()
)
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results_task(ses_notification_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'delivered'
send_mock.assert_not_called()
def test_ses_callback_should_update_notification_status(
notify_db, notify_db_session, sample_email_template, mocker):
with freeze_time("2001-01-01T12:00:00"):
mocker.patch("app.statsd_client.incr")
mocker.patch("app.statsd_client.timing_with_dates")
send_mock = mocker.patch(
"app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async"
)
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
reference="ref",
status="sending",
sent_at=datetime.utcnow(),
)
callback_api = create_service_callback_api(
service=sample_email_template.service,
url="https://original_url.com")
assert get_notification_by_id(notification.id).status == "sending"
assert process_ses_results(ses_notification_callback(reference="ref"))
notification = get_notification_by_id(notification.id)
assert notification.status == "delivered"
assert notification.provider_response is None
statsd_client.timing_with_dates.assert_any_call(
"callback.ses.elapsed-time", datetime.utcnow(),
notification.sent_at)
statsd_client.incr.assert_any_call("callback.ses.delivered")
updated_notification = Notification.query.get(notification.id)
encrypted_data = create_delivery_status_callback_data(
updated_notification, callback_api)
send_mock.assert_called_once_with(
[str(notification.id), encrypted_data], queue="service-callbacks")
def test_process_sns_results_calls_service_callback(sample_template,
notify_db_session,
notify_db, mocker):
with freeze_time('2021-01-01T12:00:00'):
mocker.patch('app.statsd_client.incr')
mocker.patch('app.statsd_client.timing_with_dates')
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(notify_db,
notify_db_session,
template=sample_template,
reference='ref',
status=NOTIFICATION_SENT,
sent_by='sns',
sent_at=datetime.utcnow())
callback_api = create_service_callback_api(
service=sample_template.service, url="https://example.com")
assert get_notification_by_id(
notification.id).status == NOTIFICATION_SENT
assert process_sns_results(sns_success_callback(reference='ref'))
assert get_notification_by_id(
notification.id).status == NOTIFICATION_DELIVERED
statsd_client.timing_with_dates.assert_any_call(
"callback.sns.elapsed-time", datetime.utcnow(),
notification.sent_at)
statsd_client.incr.assert_any_call("callback.sns.delivered")
updated_notification = get_notification_by_id(notification.id)
encrypted_data = create_delivery_status_callback_data(
updated_notification, callback_api)
send_mock.assert_called_once_with(
[str(notification.id), encrypted_data], queue="service-callbacks")
def test_process_sns_results_failed(
sample_template,
notify_db,
notify_db_session,
mocker,
provider_response,
expected_status,
should_log_warning,
):
mock_logger = mocker.patch(
'app.celery.process_sns_receipts_tasks.current_app.logger.info')
mock_warning_logger = mocker.patch(
'app.celery.process_sns_receipts_tasks.current_app.logger.warning')
notification = create_sample_notification(notify_db,
notify_db_session,
template=sample_template,
reference='ref',
status=NOTIFICATION_SENT,
sent_by='sns',
sent_at=datetime.utcnow())
assert get_notification_by_id(notification.id).status == NOTIFICATION_SENT
assert process_sns_results(
sns_failed_callback(provider_response=provider_response,
reference='ref'))
assert get_notification_by_id(notification.id).status == expected_status
mock_logger.assert_called_once_with((
f'SNS delivery failed: notification id {notification.id} and reference ref has error found. '
f'Provider response: {provider_response}'))
assert mock_warning_logger.call_count == int(should_log_warning)
def test_ses_callback_should_update_notification_status(
client, notify_db_session, sample_email_template, mocker):
with freeze_time('2001-01-01T12:00:00'):
mocker.patch('app.statsd_client.incr')
mocker.patch('app.statsd_client.timing_with_dates')
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_notification(
template=sample_email_template,
status='sending',
reference='ref',
)
callback_api = create_service_callback_api(
service=sample_email_template.service,
url="https://original_url.com")
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results(ses_notification_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'delivered'
statsd_client.timing_with_dates.assert_any_call(
"callback.ses.elapsed-time", datetime.utcnow(),
notification.sent_at)
statsd_client.incr.assert_any_call("callback.ses.delivered")
updated_notification = Notification.query.get(notification.id)
encrypted_data = create_delivery_status_callback_data(
updated_notification, callback_api)
send_mock.assert_called_once_with(
[str(notification.id), encrypted_data], queue="service-callbacks")
def test_firetext_callback_should_update_notification_status(
client, mocker, sample_notification):
mocker.patch('app.statsd_client.incr')
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
sample_notification.status = 'sending'
original = get_notification_by_id(sample_notification.id)
assert original.status == 'sending'
data = 'mobile=441234123123&status=0&time=2016-03-10 14:17:00&reference={}'.format(
sample_notification.id)
response = firetext_post(client, data)
json_resp = json.loads(response.get_data(as_text=True))
assert response.status_code == 200
assert json_resp['result'] == 'success'
assert json_resp[
'message'] == 'Firetext callback succeeded. reference {} updated'.format(
sample_notification.id)
updated = get_notification_by_id(sample_notification.id)
assert updated.status == 'delivered'
assert get_notification_by_id(sample_notification.id).status == 'delivered'
assert send_mock.called_once_with([sample_notification.id],
queue="notify-internal-tasks")
def test_firetext_callback_should_update_notification_status_failed(
notify_db, notify_db_session, notify_api, sample_template, mocker
):
with notify_api.test_request_context():
with notify_api.test_client() as client:
mocker.patch('app.statsd_client.incr')
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow())
original = get_notification_by_id(notification.id)
assert original.status == 'sending'
response = client.post(
path='/notifications/sms/firetext',
data='mobile=441234123123&status=1&time=2016-03-10 14:17:00&reference={}'.format(
notification.id
),
headers=[('Content-Type', 'application/x-www-form-urlencoded')])
json_resp = json.loads(response.get_data(as_text=True))
assert response.status_code == 200
assert json_resp['result'] == 'success'
assert json_resp['message'] == 'Firetext callback succeeded. reference {} updated'.format(
notification.id
)
assert get_notification_by_id(notification.id).status == 'permanent-failure'
def test_ses_callback_should_set_status_to_permanent_failure(
notify_db,
notify_db_session,
sample_email_template,
mocker,
bounce_subtype,
provider_response,
):
send_mock = mocker.patch(
"app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async"
)
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
reference="ref",
status="sending",
sent_at=datetime.utcnow(),
)
create_service_callback_api(service=sample_email_template.service,
url="https://original_url.com")
assert get_notification_by_id(notification.id).status == "sending"
assert process_ses_results(
ses_hard_bounce_callback(reference="ref",
bounce_subtype=bounce_subtype))
notification = get_notification_by_id(notification.id)
assert notification.status == "permanent-failure"
assert notification.provider_response == provider_response
assert send_mock.called
def test_process_mmg_response_status_4_updates_notification_with_temporary_failed(
notify_db, notify_db_session, client, mocker):
mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(notify_db,
notify_db_session,
status='sending',
sent_at=datetime.utcnow())
data = json.dumps({
"reference": "mmg_reference",
"CID": str(notification.id),
"MSISDN": "447777349060",
"status": 4
})
response = mmg_post(client, data)
assert response.status_code == 200
json_data = json.loads(response.data)
assert json_data['result'] == 'success'
assert json_data[
'message'] == 'MMG callback succeeded. reference {} updated'.format(
notification.id)
assert get_notification_by_id(
notification.id).status == 'temporary-failure'
def cancel_notification_for_service(service_id, notification_id):
notification = notifications_dao.get_notification_by_id(
notification_id, service_id)
if not notification:
raise InvalidRequest('Notification not found', status_code=404)
elif notification.notification_type != LETTER_TYPE:
raise InvalidRequest(
'Notification cannot be cancelled - only letters can be cancelled',
status_code=400)
elif not letter_can_be_cancelled(notification.status,
notification.created_at):
print_day = letter_print_day(notification.created_at)
raise InvalidRequest(
"It’s too late to cancel this letter. Printing started {} at 5.30pm"
.format(print_day),
status_code=400)
updated_notification = notifications_dao.update_notification_status_by_id(
notification_id,
NOTIFICATION_CANCELLED,
)
return jsonify(
notification_with_template_schema.dump(updated_notification).data), 200
def deliver_email(self, notification_id):
try:
current_app.logger.info(
"Start sending email for notification id: {}".format(
notification_id))
notification = notifications_dao.get_notification_by_id(
notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_email_to_provider(notification)
except InvalidEmailError as e:
current_app.logger.exception(e)
update_notification_status_by_id(notification_id, 'technical-failure')
except Exception as e:
try:
current_app.logger.exception(
"RETRY: Email notification {} failed".format(notification_id))
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task send_email_to_provider failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification_id)
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_should_persist_notification(notify_api, sample_template,
sample_email_template,
template_type,
fake_uuid, mocker):
with notify_api.test_request_context(), notify_api.test_client() as client:
mocked = mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type))
mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
template = sample_template if template_type == 'sms' else sample_email_template
to = sample_template.service.created_by.mobile_number if template_type == 'sms' \
else sample_email_template.service.created_by.email_address
data = {
'to': to,
'template': template.id
}
api_key = ApiKey(
service=template.service,
name='team_key',
created_by=template.created_by,
key_type=KEY_TYPE_TEAM)
save_model_api_key(api_key)
auth_header = create_jwt_token(secret=api_key.unsigned_secret, client_id=str(api_key.service_id))
response = client.post(
path='/notifications/{}'.format(template_type),
data=json.dumps(data),
headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type))
assert response.status_code == 201
notification = notifications_dao.get_notification_by_id(fake_uuid)
assert notification.to == to
assert notification.template_id == template.id
assert notification.notification_type == template_type
def test_should_call_send_sms_response_task_if_research_mode(notify_db, sample_service, sample_notification, mocker,
research_mode, key_type):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.mmg_client.get_name', return_value="mmg")
mocker.patch('app.celery.research_mode_tasks.send_sms_response.apply_async')
if research_mode:
sample_service.research_mode = True
notify_db.session.add(sample_service)
notify_db.session.commit()
sample_notification.key_type = key_type
send_to_providers.send_sms_to_provider(
sample_notification
)
assert not mmg_client.send_sms.called
send_to_providers.send_sms_response.apply_async.assert_called_once_with(
('mmg', str(sample_notification.id), sample_notification.to), queue='research-mode'
)
persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
assert persisted_notification.to == sample_notification.to
assert persisted_notification.template_id == sample_notification.template_id
assert persisted_notification.status == 'sending'
assert persisted_notification.sent_at <= datetime.utcnow()
assert persisted_notification.sent_by == 'mmg'
assert not persisted_notification.personalisation
def test_send_sms_should_use_template_version_from_notification_not_latest(
sample_template, mocker):
db_notification = create_notification(
template=sample_template,
to_field='+447234123123',
status='created',
reply_to_text=sample_template.service.get_default_sms_sender())
mocker.patch('app.mmg_client.send_sms')
version_on_notification = sample_template.version
# Change the template
from app.dao.templates_dao import dao_update_template, dao_get_template_by_id
sample_template.content = sample_template.content + " another version of the template"
dao_update_template(sample_template)
t = dao_get_template_by_id(sample_template.id)
assert t.version > version_on_notification
send_to_providers.send_sms_to_provider(db_notification)
mmg_client.send_sms.assert_called_once_with(
to=validate_and_format_phone_number("+447234123123"),
content="Sample service: This is a template:\nwith a newline",
reference=str(db_notification.id),
sender=current_app.config['FROM_NUMBER'])
persisted_notification = notifications_dao.get_notification_by_id(
db_notification.id)
assert persisted_notification.to == db_notification.to
assert persisted_notification.template_id == sample_template.id
assert persisted_notification.template_version == version_on_notification
assert persisted_notification.template_version != sample_template.version
assert persisted_notification.status == 'sending'
assert not persisted_notification.personalisation
def get_pdf_for_notification(notification_id):
_data = {"notification_id": notification_id}
validate(_data, notification_by_id)
notification = notifications_dao.get_notification_by_id(
notification_id, authenticated_service.id, _raise=True)
if notification.notification_type != LETTER_TYPE:
raise BadRequestError(message="Notification is not a letter")
if notification.status == NOTIFICATION_VIRUS_SCAN_FAILED:
raise BadRequestError(message='Document did not pass the virus scan')
if notification.status == NOTIFICATION_TECHNICAL_FAILURE:
raise BadRequestError(
message='PDF not available for letters in status {}'.format(
notification.status))
if notification.status == NOTIFICATION_PENDING_VIRUS_CHECK:
raise PDFNotReadyError()
try:
pdf_data, metadata = get_letter_pdf_and_metadata(notification)
except Exception:
raise PDFNotReadyError()
return send_file(filename_or_fp=BytesIO(pdf_data),
mimetype='application/pdf')
def test_should_persist_notification(client, sample_template,
sample_email_template, fake_uuid, mocker,
template_type, queue_name):
mocked = mocker.patch(
'app.celery.provider_tasks.deliver_{}.apply_async'.format(
template_type))
mocker.patch('app.notifications.process_notifications.uuid.uuid4',
return_value=fake_uuid)
template = sample_template if template_type == SMS_TYPE else sample_email_template
to = sample_template.service.created_by.mobile_number if template_type == SMS_TYPE \
else sample_email_template.service.created_by.email_address
data = {'to': to, 'template': template.id}
api_key = ApiKey(service=template.service,
name='team_key',
created_by=template.created_by,
key_type=KEY_TYPE_TEAM)
save_model_api_key(api_key)
auth_header = create_jwt_token(secret=api_key.secret,
client_id=str(api_key.service_id))
response = client.post(path='/notifications/{}'.format(template_type),
data=json.dumps(data),
headers=[('Content-Type', 'application/json'),
('Authorization',
'Bearer {}'.format(auth_header))])
mocked.assert_called_once_with([fake_uuid], queue=queue_name)
assert response.status_code == 201
notification = notifications_dao.get_notification_by_id(fake_uuid)
assert notification.to == to
assert notification.template_id == template.id
assert notification.notification_type == template_type
def create_letters_pdf(self, notification_id):
try:
notification = get_notification_by_id(notification_id, _raise=True)
pdf_data, billable_units = get_letters_pdf(
notification.template,
contact_block=notification.reply_to_text,
filename=notification.service.letter_branding
and notification.service.letter_branding.filename,
values=notification.personalisation)
upload_letter_pdf(notification, pdf_data)
if notification.key_type != KEY_TYPE_TEST:
notification.billable_units = billable_units
dao_update_notification(notification)
current_app.logger.info(
'Letter notification reference {reference}: billable units set to {billable_units}'
.format(reference=str(notification.reference),
billable_units=billable_units))
except (RequestException, BotoClientError):
try:
current_app.logger.exception(
"Letters PDF notification creation for id: {} failed".format(
notification_id))
self.retry(queue=QueueNames.RETRY)
except MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task create_letters_pdf failed for notification {}"
.format(notification_id), )
update_notification_status_by_id(notification_id,
'technical-failure')
def test_should_call_send_sms_response_task_if_research_mode(
notify_db, sample_service, sample_notification, mocker, research_mode,
key_type):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.send_sms_response')
if research_mode:
sample_service.research_mode = True
notify_db.session.add(sample_service)
notify_db.session.commit()
sample_notification.key_type = key_type
send_to_providers.send_sms_to_provider(sample_notification)
assert not mmg_client.send_sms.called
app.delivery.send_to_providers.send_sms_response.assert_called_once_with(
'mmg', str(sample_notification.id), sample_notification.to)
persisted_notification = notifications_dao.get_notification_by_id(
sample_notification.id)
assert persisted_notification.to == sample_notification.to
assert persisted_notification.template_id == sample_notification.template_id
assert persisted_notification.status == 'sending'
assert persisted_notification.sent_at <= datetime.utcnow()
assert persisted_notification.sent_by == 'mmg'
assert not persisted_notification.personalisation
def deliver_email(self, notification_id):
try:
current_app.logger.info(
"Start sending email for notification id: {}".format(
notification_id))
notification = notifications_dao.get_notification_by_id(
notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_email_to_provider(notification)
except EmailClientNonRetryableException as e:
current_app.logger.exception(
f"Email notification {notification_id} failed: {e}")
update_notification_status_by_id(notification_id, 'technical-failure')
except Exception as e:
try:
if isinstance(e, AwsSesClientThrottlingSendRateException):
current_app.logger.warning(
f"RETRY: Email notification {notification_id} was rate limited by SES"
)
else:
current_app.logger.exception(
f"RETRY: Email notification {notification_id} failed")
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task send_email_to_provider failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification_id)
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_should_delete_notification_and_return_error_if_sqs_fails(
client, sample_email_template, sample_template, fake_uuid, mocker,
template_type, queue_name):
mocked = mocker.patch(
'app.celery.provider_tasks.deliver_{}.apply_async'.format(
template_type),
side_effect=Exception("failed to talk to SQS"))
mocker.patch('app.notifications.process_notifications.uuid.uuid4',
return_value=fake_uuid)
template = sample_template if template_type == SMS_TYPE else sample_email_template
to = sample_template.service.created_by.mobile_number if template_type == SMS_TYPE \
else sample_email_template.service.created_by.email_address
data = {'to': to, 'template': template.id}
api_key = ApiKey(service=template.service,
name='team_key',
created_by=template.created_by,
key_type=KEY_TYPE_TEAM)
save_model_api_key(api_key)
auth_header = create_jwt_token(secret=api_key.secret,
client_id=str(api_key.service_id))
with pytest.raises(Exception) as e:
client.post(path='/notifications/{}'.format(template_type),
data=json.dumps(data),
headers=[('Content-Type', 'application/json'),
('Authorization', 'Bearer {}'.format(auth_header))
])
assert str(e.value) == 'failed to talk to SQS'
mocked.assert_called_once_with([fake_uuid], queue=queue_name)
assert not notifications_dao.get_notification_by_id(fake_uuid)
assert not NotificationHistory.query.get(fake_uuid)
def deliver_sms(self, notification_id):
try:
current_app.logger.info(
"Start sending SMS for notification id: {}".format(
notification_id))
notification = notifications_dao.get_notification_by_id(
notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_sms_to_provider(notification)
except Exception as e:
if isinstance(e, SmsClientResponseException):
current_app.logger.warning(
"SMS notification delivery for id: {} failed".format(
notification_id))
else:
current_app.logger.exception(
"SMS notification delivery for id: {} failed".format(
notification_id))
try:
if self.request.retries == 0:
self.retry(queue=QueueNames.RETRY, countdown=0)
else:
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification_id)
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_should_call_send_sms_response_task_if_research_mode(
notify_db, sample_service, sample_notification, mocker, research_mode, key_type
):
reference = str(uuid.uuid4())
mocker.patch("app.aws_sns_client.send_sms")
mocker.patch("app.delivery.send_to_providers.send_sms_response", return_value=reference)
if research_mode:
sample_service.research_mode = True
notify_db.session.add(sample_service)
notify_db.session.commit()
sample_notification.key_type = key_type
send_to_providers.send_sms_to_provider(sample_notification)
assert not aws_sns_client.send_sms.called
app.delivery.send_to_providers.send_sms_response.assert_called_once_with("sns", sample_notification.to)
persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
assert persisted_notification.to == sample_notification.to
assert persisted_notification.template_id == sample_notification.template_id
assert persisted_notification.status == "sent"
assert persisted_notification.sent_at <= datetime.utcnow()
assert persisted_notification.sent_by == "sns"
assert persisted_notification.reference == reference
assert not persisted_notification.personalisation
def test_ses_callback_should_set_status_to_temporary_failure(
client, notify_db_session, sample_email_template, mocker):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_notification(
template=sample_email_template,
status='sending',
reference='ref',
)
create_service_callback_api(service=notification.service,
url="https://original_url.com")
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results(ses_soft_bounce_callback(reference='ref'))
assert get_notification_by_id(
notification.id).status == 'temporary-failure'
assert send_mock.called
def test_ses_callback_should_not_set_status_once_status_is_delivered(
sample_email_template):
notification = create_notification(
sample_email_template,
status='delivered',
)
assert get_notification_by_id(notification.id).status == 'delivered'
def test_ses_callback_should_update_notification_status_when_receiving_new_delivery_receipt(
sample_email_template, mocker):
notification = create_notification(template=sample_email_template,
reference="ref",
status="delivered")
assert process_ses_results(ses_hard_bounce_callback(reference="ref"))
assert get_notification_by_id(
notification.id).status == "permanent-failure"
def lookup_va_profile_id(self, notification_id):
current_app.logger.info(
f"Retrieving VA Profile ID from MPI for notification {notification_id}"
)
notification = notifications_dao.get_notification_by_id(notification_id)
try:
va_profile_id = mpi_client.get_va_profile_id(notification)
notification.recipient_identifiers.set(
RecipientIdentifier(notification_id=notification.id,
id_type=IdentifierType.VA_PROFILE_ID.value,
id_value=va_profile_id))
notifications_dao.dao_update_notification(notification)
current_app.logger.info(
f"Successfully updated notification {notification_id} with VA PROFILE ID {va_profile_id}"
)
except MpiRetryableException as e:
current_app.logger.warning(
f"Received {str(e)} for notification {notification_id}.")
try:
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
f"The task lookup_va_profile_id failed for notification {notification_id}. " \
"Notification has been updated to technical-failure"
notifications_dao.update_notification_status_by_id(
notification_id,
NOTIFICATION_TECHNICAL_FAILURE,
status_reason=e.failure_reason)
raise NotificationTechnicalFailureException(message) from e
except (BeneficiaryDeceasedException, IdentifierNotFound,
MultipleActiveVaProfileIdsException) as e:
message = f"{e.__class__.__name__} - {str(e)}: " \
f"Can't proceed after querying MPI for VA Profile ID for {notification_id}. " \
"Stopping execution of following tasks. Notification has been updated to permanent-failure."
current_app.logger.warning(message)
self.request.chain = None
notifications_dao.update_notification_status_by_id(
notification_id,
NOTIFICATION_PERMANENT_FAILURE,
status_reason=e.failure_reason)
except Exception as e:
message = f"Failed to retrieve VA Profile ID from MPI for notification: {notification_id} " \
"Notification has been updated to technical-failure"
current_app.logger.exception(message)
status_reason = e.failure_reason if hasattr(
e, 'failure_reason') else 'Unknown error from MPI'
notifications_dao.update_notification_status_by_id(
notification_id,
NOTIFICATION_TECHNICAL_FAILURE,
status_reason=status_reason)
raise NotificationTechnicalFailureException(message) from e
def test_ses_callback_should_not_update_notification_status_if_already_delivered(sample_email_template, mocker):
mock_dup = mocker.patch('app.notifications.notifications_ses_callback.notifications_dao._duplicate_update_warning')
mock_upd = mocker.patch('app.notifications.notifications_ses_callback.notifications_dao._update_notification_status')
notification = create_notification(template=sample_email_template, reference='ref', status='delivered')
assert process_ses_results_task(ses_notification_callback(reference='ref')) is None
assert get_notification_by_id(notification.id).status == 'delivered'
mock_dup.assert_called_once_with(notification, 'delivered')
assert mock_upd.call_count == 0
def test_ses_callback_should_set_status_to_permanent_failure(
client, notify_db, notify_db_session, sample_email_template, mocker):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow())
create_service_callback_api(service=sample_email_template.service,
url="https://original_url.com")
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results_task(ses_hard_bounce_callback(reference='ref'))
assert get_notification_by_id(
notification.id).status == 'permanent-failure'
assert send_mock.called
def test_ses_callback_should_not_set_status_once_status_is_delivered(
client, notify_db, notify_db_session, sample_email_template, mocker):
notification = create_sample_notification(notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='delivered',
sent_at=datetime.utcnow())
assert get_notification_by_id(notification.id).status == 'delivered'
def test_sap_callback_should_update_notification_status(
notify_db, notify_db_session, client, sample_sap_oauth2_client,
sample_email_template, mocker):
mocker.patch('app.statsd_client.incr')
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow())
original = get_notification_by_id(notification.id)
assert original.status == 'sending'
data = {
"messageId": "1234",
"status": "DELIVERED",
"recipient": "+61412345678",
"message": "message"
}
response = sap_post(client, sample_sap_oauth2_client, notification.id,
data)
json_resp = json.loads(response.get_data(as_text=True))
assert response.status_code == 200
assert json_resp['result'] == 'success'
assert json_resp[
'message'] == 'sap callback succeeded. reference {} updated'.format(
notification.id)
updated = get_notification_by_id(notification.id)
assert updated.status == 'delivered'
assert get_notification_by_id(notification.id).status == 'delivered'
assert send_mock.called_once_with([notification.id],
queue="notify-internal-tasks")
def test_process_sns_results_delivered(sample_template, notify_db,
notify_db_session, mocker):
mock_logger = mocker.patch(
'app.celery.process_sns_receipts_tasks.current_app.logger.info')
notification = create_sample_notification(notify_db,
notify_db_session,
template=sample_template,
reference='ref',
status=NOTIFICATION_SENT,
sent_by='sns',
sent_at=datetime.utcnow())
assert get_notification_by_id(notification.id).status == NOTIFICATION_SENT
assert process_sns_results(sns_success_callback(reference='ref'))
assert get_notification_by_id(
notification.id).status == NOTIFICATION_DELIVERED
mock_logger.assert_called_once_with(
f'SNS callback return status of delivered for notification: {notification.id}'
)
def test_should_not_update_notification_if_research_mode_on_exception(sample_service, sample_notification, mocker):
mock_send_sms = mocker.patch("app.delivery.send_to_providers.send_sms_response", side_effect=Exception())
sample_service.research_mode = True
sample_notification.billable_units = 0
with pytest.raises(Exception):
send_to_providers.send_sms_to_provider(sample_notification)
persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
assert persisted_notification.billable_units == 0
assert mock_send_sms.called
def send_sms(
self, service_id, notification_id, encrypted_notification, created_at, api_key_id=None, key_type=KEY_TYPE_NORMAL
):
notification = encryption.decrypt(encrypted_notification)
service = dao_fetch_service_by_id(service_id)
if not service_allowed_to_send_to(notification["to"], service, key_type):
current_app.logger.info("SMS {} failed as restricted service".format(notification_id))
return
try:
saved_notification = persist_notification(
template_id=notification["template"],
template_version=notification["template_version"],
recipient=notification["to"],
service_id=service.id,
personalisation=notification.get("personalisation"),
notification_type=SMS_TYPE,
api_key_id=api_key_id,
key_type=key_type,
created_at=created_at,
job_id=notification.get("job", None),
job_row_number=notification.get("row_number", None),
notification_id=notification_id,
)
provider_tasks.deliver_sms.apply_async(
[str(saved_notification.id)], queue="send-sms" if not service.research_mode else "research-mode"
)
current_app.logger.info(
"SMS {} created at {} for job {}".format(saved_notification.id, created_at, notification.get("job", None))
)
except SQLAlchemyError as e:
if not get_notification_by_id(notification_id):
# Sometimes, SQS plays the same message twice. We should be able to catch an IntegrityError, but it seems
# SQLAlchemy is throwing a FlushError. So we check if the notification id already exists then do not
# send to the retry queue.
current_app.logger.error(
"RETRY: send_sms notification for job {} row number {} and notification id {}".format(
notification.get("job", None), notification.get("row_number", None), notification_id
)
)
current_app.logger.exception(e)
try:
raise self.retry(queue="retry", exc=e)
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: send_sms notification for job {} row number {} and notification id {}".format(
notification.get("job", None), notification.get("row_number", None), notification_id
)
)
current_app.logger.exception(e)
def test_ses_callback_should_update_notification_status(
notify_api,
notify_db,
notify_db_session,
sample_email_template,
mocker):
with notify_api.test_request_context():
with notify_api.test_client() as client:
with freeze_time('2001-01-01T12:00:00'):
mocker.patch('app.statsd_client.incr')
mocker.patch('app.statsd_client.timing_with_dates')
notification = create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow()
)
assert get_notification_by_id(notification.id).status == 'sending'
response = client.post(
path='/notifications/email/ses',
data=ses_notification_callback(),
headers=[('Content-Type', 'text/plain; charset=UTF-8')]
)
json_resp = json.loads(response.get_data(as_text=True))
assert response.status_code == 200
assert json_resp['result'] == 'success'
assert json_resp['message'] == 'SES callback succeeded'
assert get_notification_by_id(notification.id).status == 'delivered'
app.statsd_client.timing_with_dates.assert_any_call(
"callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at
)
app.statsd_client.incr.assert_any_call("callback.ses.delivered")
def test_should_not_set_billable_units_if_research_mode(notify_db, sample_service, sample_notification, mocker):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.mmg_client.get_name', return_value="mmg")
mocker.patch('app.celery.research_mode_tasks.send_sms_response.apply_async')
sample_service.research_mode = True
notify_db.session.add(sample_service)
notify_db.session.commit()
send_to_providers.send_sms_to_provider(
sample_notification
)
persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
assert persisted_notification.billable_units == 0
def test_should_set_billable_units_to_zero_in_research_mode_or_test_key(
notify_db, sample_service, sample_notification, mocker, research_mode, key_type):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.mmg_client.get_name', return_value="mmg")
mocker.patch('app.celery.research_mode_tasks.send_sms_response.apply_async')
if research_mode:
sample_service.research_mode = True
notify_db.session.add(sample_service)
notify_db.session.commit()
sample_notification.key_type = key_type
send_to_providers.send_sms_to_provider(
sample_notification
)
assert notifications_dao.get_notification_by_id(sample_notification.id).billable_units == 0
def deliver_sms(self, notification_id):
try:
notification = notifications_dao.get_notification_by_id(notification_id)
if not notification:
raise NoResultFound()
send_to_providers.send_sms_to_provider(notification)
except Exception as e:
try:
current_app.logger.exception(
"RETRY: SMS notification {} failed".format(notification_id)
)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError:
current_app.logger.exception(
"RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id),
)
update_notification_status_by_id(notification_id, 'technical-failure')
def send_notification_to_provider(notification_id):
notification = notifications_dao.get_notification_by_id(notification_id)
if not notification:
return jsonify({"result": "error", "message": "No result found"}), 404
if notification.notification_type == EMAIL_TYPE:
send_response(
send_to_providers.send_email_to_provider,
provider_tasks.deliver_email,
notification,
'send-email')
else:
send_response(
send_to_providers.send_sms_to_provider,
provider_tasks.deliver_sms,
notification,
'send-sms')
return jsonify({}), 204
def test_process_mmg_response_status_2_updates_notification_with_permanently_failed(
notify_db, notify_db_session, notify_api
):
with notify_api.test_client() as client:
notification = create_sample_notification(
notify_db, notify_db_session, status='sending', sent_at=datetime.utcnow()
)
data = json.dumps({"reference": "mmg_reference",
"CID": str(notification.id),
"MSISDN": "447777349060",
"status": 2})
response = client.post(path='notifications/sms/mmg',
data=data,
headers=[('Content-Type', 'application/json')])
assert response.status_code == 200
json_data = json.loads(response.data)
assert json_data['result'] == 'success'
assert json_data['message'] == 'MMG callback succeeded. reference {} updated'.format(notification.id)
assert get_notification_by_id(notification.id).status == 'permanent-failure'
def test_process_mmg_response_returns_200_when_cid_is_valid_notification_id(
notify_db, notify_db_session, notify_api
):
with notify_api.test_client() as client:
notification = create_sample_notification(
notify_db, notify_db_session, status='sending', sent_at=datetime.utcnow()
)
data = json.dumps({"reference": "mmg_reference",
"CID": str(notification.id),
"MSISDN": "447777349060",
"status": "3",
"deliverytime": "2016-04-05 16:01:07"})
response = client.post(path='notifications/sms/mmg',
data=data,
headers=[('Content-Type', 'application/json')])
assert response.status_code == 200
json_data = json.loads(response.data)
assert json_data['result'] == 'success'
assert json_data['message'] == 'MMG callback succeeded. reference {} updated'.format(notification.id)
assert get_notification_by_id(notification.id).status == 'delivered'
def test_send_sms_should_use_template_version_from_notification_not_latest(
notify_db,
notify_db_session,
sample_template,
mocker):
db_notification = sample_notification(notify_db, notify_db_session,
template=sample_template, to_field='+447234123123',
status='created')
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.mmg_client.get_name', return_value="mmg")
version_on_notification = sample_template.version
# Change the template
from app.dao.templates_dao import dao_update_template, dao_get_template_by_id
sample_template.content = sample_template.content + " another version of the template"
dao_update_template(sample_template)
t = dao_get_template_by_id(sample_template.id)
assert t.version > version_on_notification
send_to_providers.send_sms_to_provider(
db_notification
)
mmg_client.send_sms.assert_called_once_with(
to=format_phone_number(validate_phone_number("+447234123123")),
content="Sample service: This is a template:\nwith a newline",
reference=str(db_notification.id),
sender=None
)
persisted_notification = notifications_dao.get_notification_by_id(db_notification.id)
assert persisted_notification.to == db_notification.to
assert persisted_notification.template_id == sample_template.id
assert persisted_notification.template_version == version_on_notification
assert persisted_notification.template_version != sample_template.version
assert persisted_notification.status == 'sending'
assert not persisted_notification.personalisation