def process_job(job_id):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != "pending":
return
service = job.service
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = "in progress"
dao_update_job(job)
template = Template(dao_get_template_by_id(job.template_id, job.template_version).__dict__)
for row_number, recipient, personalisation in RecipientCSV(
s3.get_job_from_s3(str(service.id), str(job_id)),
template_type=template.template_type,
placeholders=template.placeholders,
).enumerated_recipients_and_personalisation:
encrypted = encryption.encrypt(
{
"template": str(template.id),
"template_version": job.template_version,
"job": str(job.id),
"to": recipient,
"row_number": row_number,
"personalisation": dict(personalisation),
}
)
if template.template_type == SMS_TYPE:
send_sms.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-sms" if not service.research_mode else "research-mode",
)
if template.template_type == EMAIL_TYPE:
send_email.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-email" if not service.research_mode else "research-mode",
)
finished = datetime.utcnow()
job.job_status = "finished"
job.processing_started = start
job.processing_finished = finished
dao_update_job(job)
current_app.logger.info(
"Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_task(mocker):
mocker.patch('app.delivery.send_to_providers.send_email_to_provider')
mocker.patch('app.celery.provider_tasks.deliver_email.retry')
notification_id = app.create_uuid()
deliver_email(notification_id)
app.delivery.send_to_providers.send_email_to_provider.assert_not_called()
app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry", countdown=10)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task(
notify_db_session, mocker):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
notification_id = app.create_uuid()
deliver_sms(notification_id)
app.delivery.send_to_providers.send_sms_to_provider.assert_not_called()
app.celery.provider_tasks.deliver_sms.retry.assert_called_with(
queue="retry-tasks", countdown=0)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_task(
mocker):
mocker.patch('app.delivery.send_to_providers.send_email_to_provider')
mocker.patch('app.celery.provider_tasks.deliver_email.retry')
notification_id = app.create_uuid()
deliver_email(notification_id)
app.delivery.send_to_providers.send_email_to_provider.assert_not_called()
app.celery.provider_tasks.deliver_email.retry.assert_called_with(
queue="retry-tasks")
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(EMAIL_TYPE, notification.id)
current_app.logger.debug(
"Starting sending EMAIL {} to provider at {}".format(
notification.id, datetime.utcnow()))
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(template_dict,
values=notification.personalisation,
**get_html_email_options(service))
plain_text_email = PlainTextEmailTemplate(
template_dict, values=notification.personalisation)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
reference = str(create_uuid())
notification.billable_units = 0
notification.reference = reference
update_notification(notification, provider)
send_email_response(reference, notification.to)
else:
from_address = '"{}" <{}@{}>'.format(
service.name, service.email_from,
current_app.config['NOTIFY_EMAIL_DOMAIN'])
email_reply_to = notification.reply_to_text
reference, status = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(
email_reply_to) if email_reply_to else None,
)
notification.reference = reference
update_notification(notification, provider, status=status)
current_app.logger.debug("SENT_MAIL: {} -- {}".format(
validate_and_format_email_address(notification.to),
str(plain_text_email)))
current_app.logger.debug("Email {} sent to provider at {}".format(
notification.id, notification.sent_at))
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("email.total-time", delta_milliseconds)
def test_should_reject_if_notification_id_cannot_be_found(notify_api):
with notify_api.test_request_context():
with notify_api.test_client() as client:
auth = create_authorization_header()
response = client.post(
'/deliver/notification/{}'.format(app.create_uuid()),
headers=[auth]
)
body = json.loads(response.get_data(as_text=True))
assert response.status_code == 404
assert body['message'] == 'No result found'
assert body['result'] == 'error'
def test_should_reject_if_notification_id_cannot_be_found(notify_db, notify_api):
with notify_api.test_request_context():
with notify_api.test_client() as client:
auth = create_authorization_header()
response = client.post(
'/deliver/notification/{}'.format(app.create_uuid()),
headers=[auth]
)
body = json.loads(response.get_data(as_text=True))
assert response.status_code == 404
assert body['message'] == 'No result found'
assert body['result'] == 'error'
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(EMAIL_TYPE)
template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(
template_dict,
values=notification.personalisation,
**get_html_email_options(service)
)
plain_text_email = PlainTextEmailTemplate(
template_dict,
values=notification.personalisation
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = str(create_uuid())
update_notification_to_sending(notification, provider)
send_email_response(notification.reference, notification.to)
else:
from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
current_app.config['NOTIFY_EMAIL_DOMAIN'])
email_reply_to = notification.reply_to_text
reference = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
)
notification.reference = reference
update_notification_to_sending(notification, provider)
delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds()
if notification.key_type == KEY_TYPE_TEST:
statsd_client.timing("email.test-key.total-time", delta_seconds)
else:
statsd_client.timing("email.live-key.total-time", delta_seconds)
if str(service.id) in current_app.config.get('HIGH_VOLUME_SERVICE'):
statsd_client.timing("email.live-key.high-volume.total-time", delta_seconds)
else:
statsd_client.timing("email.live-key.not-high-volume.total-time", delta_seconds)
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(SMS_TYPE, notification,
notification.international)
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = create_uuid()
update_notification_to_sending(notification, provider)
send_sms_response(provider.get_name(), str(notification.id),
notification.to, notification.reference)
else:
try:
reference = provider.send_sms(
to=validate_and_format_phone_number(
notification.to,
international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.billable_units = template.fragment_count
notification.reference = reference
update_notification_to_sending(notification, provider)
current_app.logger.info(
f"Saved provider reference: {reference} for notification id: {notification.id}"
)
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("sms.total-time", delta_milliseconds)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task(
notify_db_session,
mocker,
sms_method,
sms_method_name,
):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
mocker.patch(f'app.celery.provider_tasks.{sms_method_name}.retry')
notification_id = app.create_uuid()
sms_method(notification_id)
app.delivery.send_to_providers.send_sms_to_provider.assert_not_called()
getattr(app.celery.provider_tasks,
sms_method_name).retry.assert_called_with(queue="retry-tasks",
countdown=0)
def send_email_to_provider(notification):
service = dao_fetch_service_by_id(notification.service_id)
provider = provider_to_use(EMAIL_TYPE, notification.id)
if notification.status == 'created':
template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(
template_dict,
values=notification.personalisation,
**get_html_email_options(service)
)
plain_text_email = PlainTextEmailTemplate(
template_dict,
values=notification.personalisation
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
reference = str(create_uuid())
send_email_response.apply_async(
(provider.get_name(), reference, notification.to), queue='research-mode'
)
notification.billable_units = 0
else:
from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
current_app.config['NOTIFY_EMAIL_DOMAIN'])
reference = provider.send_email(
from_address,
notification.to,
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=service.reply_to_email_address,
)
notification.reference = reference
notification.sent_at = datetime.utcnow()
notification.sent_by = provider.get_name(),
notification.status = 'sending'
dao_update_notification(notification)
current_app.logger.info(
"Email {} sent to provider at {}".format(notification.id, notification.sent_at)
)
delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000
statsd_client.timing("email.total-time", delta_milliseconds)
def process_row(row: Row, template: Template, job: Job, service: Service):
template_type = template.template_type
encrypted = encryption.encrypt({
"api_key":
job.api_key_id and str(job.api_key_id),
"template":
str(template.id),
"template_version":
job.template_version,
"job":
str(job.id),
"to":
row.recipient,
"row_number":
row.index,
"personalisation":
dict(row.personalisation),
"queue":
queue_to_use(job.notification_count),
})
sender_id = str(job.sender_id) if job.sender_id else None
send_fns = {
SMS_TYPE: save_sms,
EMAIL_TYPE: save_email,
LETTER_TYPE: save_letter
}
send_fn = send_fns[template_type]
task_kwargs = {}
if sender_id:
task_kwargs["sender_id"] = sender_id
send_fn.apply_async(
(
str(service.id),
create_uuid(),
encrypted,
),
task_kwargs,
queue=QueueNames.DATABASE
if not service.research_mode else QueueNames.RESEARCH_MODE,
)
class TestProcessSNSDeliveryStatus:
@pytest.mark.skip(reason="Endpoint disabled and slated for removal")
@pytest.mark.parametrize('data', [
payload_with_missing_message_id(),
payload_with_missing_status(),
get_sns_delivery_status_payload(create_uuid(), "NOT_A_VALID_STATE"),
get_sns_delivery_status_payload("not-uuid", SNS_STATUS_SUCCESS)
])
def test_returns_bad_request_on_schema_validation_errors(
self, client, data):
response = post(client, data)
assert response.status_code == 400
@pytest.mark.skip(reason="Endpoint disabled and slated for removal")
def test_loads_notification_by_reference(
self, client, mock_notification,
mock_dao_get_notification_by_reference,
mock_update_notification_status, mock_process_service_callback):
mock_dao_get_notification_by_reference.return_value = mock_notification
post(
client,
get_sns_delivery_status_payload(mock_notification.reference,
SNS_STATUS_SUCCESS))
mock_dao_get_notification_by_reference.assert_called_with(
mock_notification.reference)
@pytest.mark.skip(reason="Endpoint disabled and slated for removal")
@pytest.mark.parametrize(
"exception",
[MultipleResultsFound(), NoResultFound()])
def test_returns_404_when_unable_to_load_notification(
self, client, mock_notification,
mock_dao_get_notification_by_reference, exception):
mock_dao_get_notification_by_reference.side_effect = exception
response = post(
client,
get_sns_delivery_status_payload(mock_notification.reference,
SNS_STATUS_SUCCESS))
assert response.status_code == 404
@pytest.mark.skip(reason="Endpoint disabled and slated for removal")
@pytest.mark.parametrize("sns_status, status",
[(SNS_STATUS_SUCCESS, NOTIFICATION_SENT),
(SNS_STATUS_FAILURE, NOTIFICATION_FAILED)])
def test_should_update_notification_status(
self, client, mock_notification,
mock_dao_get_notification_by_reference,
mock_update_notification_status, mock_process_service_callback,
sns_status, status):
mock_dao_get_notification_by_reference.return_value = mock_notification
post(
client,
get_sns_delivery_status_payload(mock_notification.reference,
sns_status))
mock_update_notification_status.assert_called_with(
mock_notification, status)
@pytest.mark.skip(reason="Endpoint disabled and slated for removal")
def test_should_process_service_callback(
self,
client,
mock_notification,
mock_dao_get_notification_by_reference,
mock_update_notification_status,
mock_process_service_callback,
):
mock_dao_get_notification_by_reference.return_value = mock_notification
mock_update_notification_status.return_value = mock_notification
post(
client,
get_sns_delivery_status_payload(mock_notification.reference,
SNS_STATUS_SUCCESS))
mock_process_service_callback.assert_called_with(mock_notification)
@pytest.mark.skip(reason="Endpoint disabled and slated for removal")
def test_should_send_callback_metrics(
self, client, mock_notification,
mock_dao_get_notification_by_reference,
mock_update_notification_status, mock_process_service_callback,
mock_send_callback_metrics):
mock_dao_get_notification_by_reference.return_value = mock_notification
mock_update_notification_status.return_value = mock_notification
post(
client,
get_sns_delivery_status_payload(mock_notification.reference,
SNS_STATUS_SUCCESS))
mock_send_callback_metrics.assert_called_with(mock_notification)
@pytest.mark.skip(reason="Endpoint disabled and slated for removal")
def test_returns_204(
self,
client,
mock_notification,
mock_dao_get_notification_by_reference,
mock_update_notification_status,
mock_process_service_callback,
):
mock_dao_get_notification_by_reference.return_value = mock_notification
mock_update_notification_status.return_value = mock_notification
response = post(
client,
get_sns_delivery_status_payload(mock_notification.reference,
SNS_STATUS_SUCCESS))
assert response.status_code == 204
def mock_notification(mocker):
notification = mocker.Mock(Notification)
notification.id = create_uuid()
notification.reference = create_uuid()
notification.sent_at = datetime.utcnow()
return notification
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(EMAIL_TYPE, notification.id)
# Extract any file objects from the personalization
file_keys = [
k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and 'document' in v
]
attachments = []
personalisation_data = notification.personalisation.copy()
for key in file_keys:
# Check if a MLWR sid exists
if (current_app.config["MLWR_HOST"] and
'mlwr_sid' in personalisation_data[key]['document'] and
personalisation_data[key]['document']['mlwr_sid'] != "false"):
mlwr_result = check_mlwr(personalisation_data[key]['document']['mlwr_sid'])
if "state" in mlwr_result and mlwr_result["state"] == "completed":
# Update notification that it contains malware
if "submission" in mlwr_result and mlwr_result["submission"]['max_score'] >= 500:
malware_failure(notification=notification)
return
else:
# Throw error so celery will retry in sixty seconds
raise MalwarePendingException
try:
req = urllib.request.Request(personalisation_data[key]['document']['direct_file_url'])
with urllib.request.urlopen(req) as response:
buffer = response.read()
mime_type = magic.from_buffer(buffer, mime=True)
if mime_type == 'application/pdf':
attachments.append({"name": "{}.pdf".format(key), "data": buffer})
except Exception:
current_app.logger.error(
"Could not download and attach {}".format(personalisation_data[key]['document']['direct_file_url'])
)
personalisation_data[key] = personalisation_data[key]['document']['url']
template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(
template_dict,
values=personalisation_data,
**get_html_email_options(service)
)
plain_text_email = PlainTextEmailTemplate(
template_dict,
values=personalisation_data
)
if current_app.config["SCAN_FOR_PII"]:
contains_pii(notification, str(plain_text_email))
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = str(create_uuid())
update_notification_to_sending(notification, provider)
send_email_response(notification.reference, notification.to)
else:
if service.sending_domain is None or service.sending_domain.strip() == "":
sending_domain = current_app.config['NOTIFY_EMAIL_DOMAIN']
else:
sending_domain = service.sending_domain
from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
sending_domain)
email_reply_to = notification.reply_to_text
reference = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
attachments=attachments
)
notification.reference = reference
update_notification_to_sending(notification, provider)
delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000
statsd_client.timing("email.total-time", delta_milliseconds)
def process_sms_or_email_notification(*, form, notification_type, api_key, template, service, reply_to_text=None):
form_send_to = form["email_address"] if notification_type == EMAIL_TYPE else form["phone_number"]
send_to = validate_and_format_recipient(
send_to=form_send_to,
key_type=api_key.key_type,
service=service,
notification_type=notification_type,
)
# Do not persist or send notification to the queue if it is a simulated recipient
simulated = simulated_recipient(send_to, notification_type)
personalisation = process_document_uploads(form.get("personalisation"), service, simulated, template.id)
notification = {
"id": create_uuid(),
"template": str(template.id),
"template_version": str(template.version),
"to": form_send_to,
"personalisation": personalisation,
"simulated": simulated,
"api_key": str(api_key.id),
"key_type": str(api_key.key_type),
"client_reference": form.get("reference", None),
}
encrypted_notification_data = encryption.encrypt(notification)
scheduled_for = form.get("scheduled_for", None)
if scheduled_for:
notification = persist_notification(
template_id=template.id,
template_version=template.version,
recipient=form_send_to,
service=service,
personalisation=personalisation,
notification_type=notification_type,
api_key_id=api_key.id,
key_type=api_key.key_type,
client_reference=form.get("reference", None),
simulated=simulated,
reply_to_text=reply_to_text,
)
persist_scheduled_notification(notification.id, form["scheduled_for"])
elif current_app.config["FF_NOTIFICATION_CELERY_PERSISTENCE"] and not simulated:
# depending on the type route to the appropriate save task
if notification_type == EMAIL_TYPE:
current_app.logger.info("calling save email task")
save_email.apply_async(
(authenticated_service.id, create_uuid(), encrypted_notification_data),
queue=QueueNames.DATABASE if not authenticated_service.research_mode else QueueNames.RESEARCH_MODE,
)
elif notification_type == SMS_TYPE:
save_sms.apply_async(
(authenticated_service.id, create_uuid(), encrypted_notification_data),
queue=QueueNames.DATABASE if not authenticated_service.research_mode else QueueNames.RESEARCH_MODE,
)
else:
notification = persist_notification(
template_id=template.id,
template_version=template.version,
recipient=form_send_to,
service=service,
personalisation=personalisation,
notification_type=notification_type,
api_key_id=api_key.id,
key_type=api_key.key_type,
client_reference=form.get("reference", None),
simulated=simulated,
reply_to_text=reply_to_text,
)
if not simulated:
send_notification_to_queue(
notification=notification,
research_mode=service.research_mode,
queue=template.queue_to_use(),
)
else:
current_app.logger.debug("POST simulated notification for id: {}".format(notification.id))
if not isinstance(notification, Notification):
notification["template_id"] = notification["template"]
notification["api_key_id"] = notification["api_key"]
notification["template_version"] = template.version
notification["service"] = service
notification["service_id"] = service.id
notification["reply_to_text"] = reply_to_text
del notification["template"]
del notification["api_key"]
del notification["simulated"]
notification = Notification(**notification)
return notification
def save_service_inbound_api(service_inbound_api):
service_inbound_api.id = create_uuid()
service_inbound_api.created_at = datetime.utcnow()
db.session.add(service_inbound_api)
def test_should_reject_if_not_authenticated(notify_api):
with notify_api.test_request_context():
with notify_api.test_client() as client:
response = client.post('/deliver/notification/{}'.format(app.create_uuid()))
assert response.status_code == 401
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
# TODO: no else - replace with if statement raising error / logging when not 'created'
if notification.status == 'created':
provider = provider_to_use(notification)
# TODO: remove that code or extract attachment handling to separate method
# Extract any file objects from the personalization
file_keys = [
k for k, v in (notification.personalisation or {}).items()
if isinstance(v, dict) and 'document' in v
]
attachments = []
personalisation_data = notification.personalisation.copy()
for key in file_keys:
# Check if a MLWR sid exists
if (current_app.config["MLWR_HOST"]
and 'mlwr_sid' in personalisation_data[key]['document']
and personalisation_data[key]['document']['mlwr_sid'] !=
"false"):
mlwr_result = check_mlwr(
personalisation_data[key]['document']['mlwr_sid'])
if "state" in mlwr_result and mlwr_result[
"state"] == "completed":
# Update notification that it contains malware
if "submission" in mlwr_result and mlwr_result[
"submission"]['max_score'] >= 500:
malware_failure(notification=notification)
return
else:
# Throw error so celery will retry in sixty seconds
raise MalwarePendingException
try:
response = requests.get(
personalisation_data[key]['document']['direct_file_url'])
if response.headers['Content-Type'] == 'application/pdf':
attachments.append({
"name": "{}.pdf".format(key),
"data": response.content
})
except Exception:
current_app.logger.error(
"Could not download and attach {}".format(
personalisation_data[key]['document']
['direct_file_url']))
personalisation_data[key] = personalisation_data[key]['document'][
'url']
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(template_dict,
values=personalisation_data,
**get_html_email_options(
notification, provider))
plain_text_email = PlainTextEmailTemplate(template_dict,
values=personalisation_data)
if current_app.config["SCAN_FOR_PII"]:
contains_pii(notification, str(plain_text_email))
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = str(create_uuid())
update_notification_to_sending(notification, provider)
send_email_response(notification.reference, notification.to)
else:
email_reply_to = notification.reply_to_text
reference = provider.send_email(
source=compute_source_email_address(service, provider),
to_addresses=validate_and_format_email_address(
notification.to),
subject=plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(
email_reply_to) if email_reply_to else None,
attachments=attachments)
notification.reference = reference
update_notification_to_sending(notification, provider)
current_app.logger.info(
f"Saved provider reference: {reference} for notification id: {notification.id}"
)
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("email.total-time", delta_milliseconds)
def test_should_reject_if_not_authenticated(notify_api):
with notify_api.test_request_context():
with notify_api.test_client() as client:
response = client.post('/deliver/notification/{}'.format(app.create_uuid()))
assert response.status_code == 401
