def _process_for_status(notification_status, client_name, provider_reference):
notification = notifications_dao.update_notification_status_by_id(
provider_reference, notification_status)
if not notification:
current_app.logger.warning(
"{} callback failed: notification {} either not found or already updated "
"from sending. Status {}".format(client_name, provider_reference,
notification_status))
return
statsd_client.incr('callback.{}.{}'.format(client_name,
notification_status))
if not notification.sent_by:
set_notification_sent_by(notification, client_name)
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name), datetime.utcnow(),
notification.sent_at)
check_for_callback_and_send_delivery_status_to_service(notification)
success = "{} callback succeeded. reference {} updated".format(
client_name, provider_reference)
return success
def process_sendgrid_response():
data = json.loads(request.data)
try:
for obj in data:
notification_status = get_sendgrid_responses(obj["event"])
reference = obj['sg_message_id'].split(".")[0]
notification = notifications_dao.dao_get_notification_by_reference(reference)
notifications_dao._update_notification_status(notification=notification, status=notification_status)
current_app.logger.info('SendGird callback return status of {} for notification: {}'.format(
notification_status, notification.id
))
statsd_client.incr('callback.sendgrid.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.sendgrid.elapsed-time', datetime.utcnow(), notification.sent_at)
except Exception as e:
current_app.logger.exception('Error processing SendGrid results: {}'.format(type(e)))
raise InvalidRequest(message="Error processing SendGrid results", status_code=400)
else:
return jsonify(result='success'), 200
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == "created":
provider = provider_to_use(
SMS_TYPE,
notification.id,
notification.international,
notification.reply_to_text,
)
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
template = SMSMessageTemplate(
template_dict,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = send_sms_response(provider.get_name(),
notification.to)
update_notification_to_sending(notification, provider)
else:
try:
reference = provider.send_sms(
to=validate_and_format_phone_number(
notification.to,
international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text,
)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.reference = reference
notification.billable_units = template.fragment_count
update_notification_to_sending(notification, provider)
# Record StatsD stats to compute SLOs
statsd_client.timing_with_dates("sms.total-time", notification.sent_at,
notification.created_at)
statsd_key = f"sms.process_type-{template_dict['process_type']}"
statsd_client.timing_with_dates(statsd_key, notification.sent_at,
notification.created_at)
statsd_client.incr(statsd_key)
def process_ses_results(response):
try:
ses_message = json.loads(response['Message'])
notification_type = ses_message['notificationType']
if notification_type == 'Bounce':
notification_type = determine_notification_bounce_type(notification_type, ses_message)
elif notification_type == 'Complaint':
_check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
return True, False
aws_response_dict = get_aws_responses(notification_type)
notification_status = aws_response_dict['notification_status']
reference = ses_message['mail']['messageId']
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
return None, True
current_app.logger.warning(
"notification not found for reference: {} (update to {})".format(reference, notification_status)
)
return None, False
if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
notifications_dao._duplicate_update_warning(notification, notification_status)
return None, False
notifications_dao._update_notification_status(notification=notification, status=notification_status)
if not aws_response_dict['success']:
current_app.logger.info(
"SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
notification.id, reference, aws_response_dict['message']
)
)
else:
current_app.logger.info('SES callback returned status of {} for notification: {}'.format(
notification_status, notification.id
))
statsd_client.incr('callback.ses.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True, False
except Exception as e:
current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
return None, True
def process_sms_client_response(status, reference, client_name):
success = None
errors = None
# validate reference
if reference == 'send-sms-code':
success = "{} callback succeeded: send-sms-code".format(client_name)
return success, errors
try:
uuid.UUID(reference, version=4)
except ValueError:
message = "{} callback with invalid reference {}".format(client_name, reference)
return success, message
try:
response_parser = sms_response_mapper[client_name]
except KeyError:
return success, 'unknown sms client: {}'.format(client_name)
# validate status
try:
response_dict = response_parser(status)
current_app.logger.info('{} callback return status of {} for reference: {}'.format(client_name,
status, reference))
except KeyError:
msg = "{} callback failed: status {} not found.".format(client_name, status)
return success, msg
notification_status = response_dict['notification_status']
notification_status_message = response_dict['message']
notification_success = response_dict['success']
# record stats
notification = notifications_dao.update_notification_status_by_id(reference, notification_status)
if not notification:
status_error = "{} callback failed: notification {} either not found or already updated " \
"from sending. Status {}".format(client_name,
reference,
notification_status_message)
return success, status_error
if not notification_success:
current_app.logger.info(
"{} delivery failed: notification {} has error found. Status {}".format(client_name,
reference,
notification_status_message))
statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name.lower()),
datetime.utcnow(),
notification.sent_at
)
success = "{} callback succeeded. reference {} updated".format(client_name, reference)
return success, errors
def process_pinpoint_results(self, response):
if not is_feature_enabled(FeatureFlag.PINPOINT_RECEIPTS_ENABLED):
current_app.logger.info(
'Pinpoint receipts toggle is disabled, skipping callback task')
return True
try:
pinpoint_message = json.loads(base64.b64decode(response['Message']))
reference = pinpoint_message['attributes']['message_id']
event_type = pinpoint_message.get('event_type')
record_status = pinpoint_message['attributes']['record_status']
current_app.logger.info(
f'received callback from Pinpoint with event_type of {event_type} and record_status of {record_status}'
f'with reference {reference}')
notification_status = get_notification_status(event_type,
record_status, reference)
notification, should_retry, should_exit = attempt_to_get_notification(
reference, notification_status,
pinpoint_message['event_timestamp'])
if should_retry:
self.retry(queue=QueueNames.RETRY)
if should_exit:
return
update_notification_status_by_id(notification.id, notification_status)
current_app.logger.info(
f"Pinpoint callback return status of {notification_status} for notification: {notification.id}"
)
statsd_client.incr(f"callback.pinpoint.{notification_status}")
if notification.sent_at:
statsd_client.timing_with_dates('callback.pinpoint.elapsed-time',
datetime.datetime.utcnow(),
notification.sent_at)
check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(
f"Error processing Pinpoint results: {type(e)}")
self.retry(queue=QueueNames.RETRY)
def _process_for_status(notification_status,
client_name,
provider_reference,
detailed_status_code=None):
# record stats
if client_name == 'Twilio':
notification = notifications_dao.update_notification_status_by_reference(
reference=provider_reference, status=notification_status)
else:
notification = notifications_dao.update_notification_status_by_id(
notification_id=provider_reference,
status=notification_status,
sent_by=client_name.lower(),
detailed_status_code=detailed_status_code)
if not notification:
return
statsd_client.incr('callback.{}.{}'.format(client_name.lower(),
notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name.lower()),
datetime.utcnow(), notification.sent_at)
if notification.billable_units == 0:
service = notification.service
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
notification.billable_units = template.fragment_count
notifications_dao.dao_update_notification(notification)
if notification_status != NOTIFICATION_PENDING:
service_callback_api = get_service_delivery_status_callback_api_for_service(
service_id=notification.service_id)
# queue callback task only if the service_callback_api exists
if service_callback_api:
encrypted_notification = create_delivery_status_callback_data(
notification, service_callback_api)
send_delivery_status_to_service.apply_async(
[str(notification.id), encrypted_notification],
queue=QueueNames.CALLBACKS)
def process_govdelivery_response():
try:
data = validate(request.form, govdelivery_webhook_schema)
sid = data['sid']
reference = data['message_url'].split("/")[-1]
govdelivery_status = data['status']
notify_status = govdelivery_status_map[govdelivery_status]
except ValidationError as e:
raise e
except Exception as e:
raise InvalidRequest(f'Error processing Govdelivery callback: {e}',
400)
else:
try:
notification = notifications_dao.dao_get_notification_by_reference(
reference)
except (MultipleResultsFound, NoResultFound) as e:
exception_type = type(e).__name__
current_app.logger.exception(
f'Govdelivery callback with sid {sid} for reference {reference} '
f'did not find exactly one notification: {exception_type}')
statsd_client.incr(
f'callback.govdelivery.failure.{exception_type}')
else:
current_app.logger.info(
f'Govdelivery callback for notification {notification.id} has status {govdelivery_status},'
f' which maps to notification-api status {notify_status}')
if data.get('error_message'):
current_app.logger.info(
f"Govdelivery error_message for notification {notification.id}: "
f"{data['error_message']}")
notifications_dao._update_notification_status(
notification, notify_status)
statsd_client.incr(f'callback.govdelivery.{notify_status}')
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.govdelivery.elapsed-time', datetime.utcnow(),
notification.sent_at)
if govdelivery_status == 'blacklisted':
complaint = create_complaint(data, notification)
publish_complaint(complaint, notification, notification.to)
return jsonify(result='success'), 200
def process_job(job_id):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != JOB_STATUS_PENDING:
return
service = job.service
if not service.active:
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
current_app.logger.warning(
"Job {} has been cancelled, service {} is inactive".format(
job_id, service.id))
return
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = JOB_STATUS_IN_PROGRESS
job.processing_started = start
dao_update_job(job)
# Record StatsD stats to compute SLOs
job_start = job.scheduled_for or job.created_at
statsd_client.timing_with_dates("job.processing-start-delay",
job.processing_started, job_start)
db_template = dao_get_template_by_id(job.template_id, job.template_version)
TemplateClass = get_template_class(db_template.template_type)
template = TemplateClass(db_template.__dict__)
current_app.logger.debug(
"Starting job {} processing {} notifications".format(
job_id, job.notification_count))
csv = get_recipient_csv(job, template)
for row in csv.get_rows():
process_row(row, template, job, service)
job_complete(job, start=start)
def process_govdelivery_response():
try:
data = request.form
reference = data['message_url'].split("/")[-1]
govdelivery_status = data['status']
notify_status = map_govdelivery_status_to_notify_status(
govdelivery_status)
except Exception as e:
raise InvalidRequest(
'Error processing Govdelivery callback: {}'.format(e), 400)
else:
try:
notification = notifications_dao.dao_get_notification_by_reference(
reference)
except (MultipleResultsFound, NoResultFound) as e:
current_app.logger.exception(
'Govdelivery callback for reference {} did not find exactly one notification: {}'
.format(reference, type(e)))
pass
else:
current_app.logger.info(
'Govdelivery callback for notification {} has status "{}", which maps to notification-api status "{}"'
.format(notification.id, govdelivery_status, notify_status))
notifications_dao._update_notification_status(
notification, notify_status)
statsd_client.incr('callback.govdelivery.{}'.format(notify_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.govdelivery.elapsed-time', datetime.utcnow(),
notification.sent_at)
return jsonify(result='success'), 200
def _process_for_status(notification_status, client_name, provider_reference):
# record stats
notification = notifications_dao.update_notification_status_by_id(
provider_reference, notification_status)
if not notification:
current_app.logger.warning(
"{} callback failed: notification {} either not found or already updated "
"from sending. Status {}".format(client_name, provider_reference,
notification_status))
return
statsd_client.incr('callback.{}.{}'.format(client_name.lower(),
notification_status))
if not notification.sent_by:
set_notification_sent_by(notification, client_name.lower())
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name.lower()),
datetime.utcnow(), notification.sent_at)
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_delivery_status_callback_api_for_service(
service_id=notification.service_id)
if service_callback_api:
encrypted_notification = create_encrypted_callback_data(
notification, service_callback_api)
send_delivery_status_to_service.apply_async(
[str(notification.id), encrypted_notification],
queue=QueueNames.CALLBACKS)
success = "{} callback succeeded. reference {} updated".format(
client_name, provider_reference)
return success
def process_ses_results(self, response):
try:
ses_message = json.loads(response["Message"])
notification_type = ses_message["notificationType"]
if notification_type == "Complaint":
_check_and_queue_complaint_callback_task(
*handle_complaint(ses_message))
return True
aws_response_dict = get_aws_responses(ses_message)
notification_status = aws_response_dict["notification_status"]
reference = ses_message["mail"]["messageId"]
try:
notification = notifications_dao.dao_get_notification_by_reference(
reference)
except NoResultFound:
message_time = iso8601.parse_date(
ses_message["mail"]["timestamp"]).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
"notification not found for reference: {} (update to {})".
format(reference, notification_status))
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status,
provider_response=aws_response_dict["provider_response"],
)
if not aws_response_dict["success"]:
current_app.logger.info(
"SES delivery failed: notification id {} and reference {} has error found. Status {}"
.format(notification.id, reference,
aws_response_dict["message"]))
else:
current_app.logger.info(
"SES callback return status of {} for notification: {}".format(
notification_status, notification.id))
statsd_client.incr("callback.ses.{}".format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates("callback.ses.elapsed-time",
datetime.utcnow(),
notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception("Error processing SES results: {}".format(
type(e)))
self.retry(queue=QueueNames.RETRY)
def process_ses_results(self, response):
try:
ses_message = json.loads(response['Message'])
notification_type = ses_message['notificationType']
bounce_message = None
if notification_type == 'Bounce':
notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message)
elif notification_type == 'Complaint':
_check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
return True
aws_response_dict = get_aws_responses(notification_type)
notification_status = aws_response_dict['notification_status']
reference = ses_message['mail']['messageId']
try:
notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
except NoResultFound:
message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
current_app.logger.info(
f"notification not found for reference: {reference} (update to {notification_status}). "
f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
)
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
f"notification not found for reference: {reference} (update to {notification_status})"
)
return
if bounce_message:
current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
notifications_dao._duplicate_update_warning(
notification=notification,
status=notification_status
)
return
else:
notifications_dao.dao_update_notifications_by_reference(
references=[reference],
update_dict={'status': notification_status}
)
statsd_client.incr('callback.ses.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
self.retry(queue=QueueNames.RETRY)
def process_sns_results(self, response):
try:
# Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html
sns_message = json.loads(response['Message'])
reference = sns_message['notification']['messageId']
status = sns_message['status']
provider_response = sns_message['delivery']['providerResponse']
# See all the possible provider responses
# https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html#sms_stats_delivery_fail_reasons
reasons = {
'Blocked as spam by phone carrier': NOTIFICATION_TECHNICAL_FAILURE,
'Destination is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE,
'Invalid phone number': NOTIFICATION_TECHNICAL_FAILURE,
'Message body is invalid': NOTIFICATION_TECHNICAL_FAILURE,
'Phone carrier has blocked this message': NOTIFICATION_TECHNICAL_FAILURE,
'Phone carrier is currently unreachable/unavailable': NOTIFICATION_TEMPORARY_FAILURE,
'Phone has blocked SMS': NOTIFICATION_TECHNICAL_FAILURE,
'Phone is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE,
'Phone is currently unreachable/unavailable': NOTIFICATION_PERMANENT_FAILURE,
'Phone number is opted out': NOTIFICATION_TECHNICAL_FAILURE,
'This delivery would exceed max price': NOTIFICATION_TECHNICAL_FAILURE,
'Unknown error attempting to reach phone': NOTIFICATION_TECHNICAL_FAILURE,
}
if status == "SUCCESS":
notification_status = NOTIFICATION_DELIVERED
else:
if provider_response not in reasons:
current_app.logger.warning(
f"unhandled provider response for reference {reference}, received '{provider_response}'"
)
notification_status = reasons.get(provider_response, NOTIFICATION_TECHNICAL_FAILURE)
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(sns_message['notification']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
f"notification not found for reference: {reference} (update to {notification_status})"
)
return
if notification.sent_by != SNS_PROVIDER:
current_app.logger.exception(f'SNS callback handled notification {notification.id} not sent by SNS')
return
if notification.status != NOTIFICATION_SENT:
notifications_dao._duplicate_update_warning(notification, notification_status)
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status
)
if notification_status != NOTIFICATION_DELIVERED:
current_app.logger.info((
f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. "
f"Provider response: {sns_message['delivery']['providerResponse']}"
))
else:
current_app.logger.info(
f'SNS callback return status of {notification_status} for notification: {notification.id}'
)
statsd_client.incr(f'callback.sns.{notification_status}')
if notification.sent_at:
statsd_client.timing_with_dates('callback.sns.elapsed-time', datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(f'Error processing SNS results: {str(e)}')
self.retry(queue=QueueNames.RETRY)
def process_sns_results(self, response):
try:
# Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html
sns_message = json.loads(response["Message"])
reference = sns_message["notification"]["messageId"]
sns_status = sns_message["status"]
provider_response = sns_message["delivery"]["providerResponse"]
try:
notification_status = determine_status(sns_status, provider_response)
except KeyError:
current_app.logger.warning(f"unhandled provider response for reference {reference}, received '{provider_response}'")
notification_status = NOTIFICATION_TECHNICAL_FAILURE
provider_response = None
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(sns_message["notification"]["timestamp"]).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(f"notification not found for reference: {reference} (update to {notification_status})")
return
if notification.sent_by != SNS_PROVIDER:
current_app.logger.exception(f"SNS callback handled notification {notification.id} not sent by SNS")
return
if notification.status != NOTIFICATION_SENT:
notifications_dao._duplicate_update_warning(notification, notification_status)
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status,
provider_response=provider_response if notification_status == NOTIFICATION_TECHNICAL_FAILURE else None,
)
if notification_status != NOTIFICATION_DELIVERED:
current_app.logger.info(
(
f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. "
f"Provider response: {sns_message['delivery']['providerResponse']}"
)
)
else:
current_app.logger.info(f"SNS callback return status of {notification_status} for notification: {notification.id}")
statsd_client.incr(f"callback.sns.{notification_status}")
if notification.sent_at:
statsd_client.timing_with_dates("callback.sns.elapsed-time", datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(f"Error processing SNS results: {str(e)}")
self.retry(queue=QueueNames.RETRY)
def send_callback_metrics(notification):
statsd_client.incr(f"callback.sns.{notification.status}")
if notification.sent_at:
statsd_client.timing_with_dates("callback.sns.elapsed-time", datetime.utcnow(), notification.sent_at)
def process_ses_response(ses_request):
client_name = 'SES'
try:
errors = validate_callback_data(data=ses_request,
fields=['Message'],
client_name=client_name)
if errors:
return errors
ses_message = json.loads(ses_request['Message'])
errors = validate_callback_data(data=ses_message,
fields=['notificationType'],
client_name=client_name)
if errors:
return errors
notification_type = ses_message['notificationType']
if notification_type == 'Bounce':
notification_type = determine_notification_bounce_type(
notification_type, ses_message)
elif notification_type == 'Complaint':
complaint, notification, recipient = handle_complaint(ses_message)
_check_and_queue_complaint_callback_task(complaint, notification,
recipient)
return
try:
aws_response_dict = get_aws_responses(notification_type)
except KeyError:
error = "{} callback failed: status {} not found".format(
client_name, notification_type)
return error
notification_status = aws_response_dict['notification_status']
try:
reference = ses_message['mail']['messageId']
notification = notifications_dao.update_notification_status_by_reference(
reference, notification_status)
if not notification:
warning = "SES callback failed: notification either not found or already updated " \
"from sending. Status {} for notification reference {}".format(notification_status, reference)
current_app.logger.warning(warning)
return
if not aws_response_dict['success']:
current_app.logger.info(
"SES delivery failed: notification id {} and reference {} has error found. Status {}"
.format(notification.id, reference,
aws_response_dict['message']))
else:
current_app.logger.info(
'{} callback return status of {} for notification: {}'.
format(client_name, notification_status, notification.id))
statsd_client.incr('callback.ses.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.ses.elapsed-time'.format(client_name.lower()),
datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return
except KeyError:
error = "SES callback failed: messageId missing"
return error
except ValueError:
error = "{} callback failed: invalid json".format(client_name)
return error
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == "created":
provider = provider_to_use(EMAIL_TYPE, notification.id)
# Extract any file objects from the personalization
file_keys = [
k for k, v in (notification.personalisation or {}).items()
if isinstance(v, dict) and "document" in v
]
attachments = []
personalisation_data = notification.personalisation.copy()
for key in file_keys:
sending_method = personalisation_data[key]["document"].get(
"sending_method")
# Check if a MLWR sid exists
if (current_app.config["MLWR_HOST"]
and "mlwr_sid" in personalisation_data[key]["document"]
and personalisation_data[key]["document"]["mlwr_sid"] !=
"false"):
mlwr_result = check_mlwr(
personalisation_data[key]["document"]["mlwr_sid"])
if "state" in mlwr_result and mlwr_result[
"state"] == "completed":
# Update notification that it contains malware
if "submission" in mlwr_result and mlwr_result[
"submission"]["max_score"] >= 500:
malware_failure(notification=notification)
return
else:
# Throw error so celery will retry in sixty seconds
raise MalwarePendingException
if sending_method == "attach":
try:
req = urllib.request.Request(
personalisation_data[key]["document"]
["direct_file_url"])
with urllib.request.urlopen(req) as response:
buffer = response.read()
filename = personalisation_data[key]["document"].get(
"filename")
mime_type = personalisation_data[key]["document"].get(
"mime_type")
attachments.append({
"name": filename,
"data": buffer,
"mime_type": mime_type,
})
except Exception:
current_app.logger.error(
"Could not download and attach {}".format(
personalisation_data[key]["document"]
["direct_file_url"]))
del personalisation_data[key]
else:
personalisation_data[key] = personalisation_data[key][
"document"]["url"]
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
# Local Jinja support - Add USE_LOCAL_JINJA_TEMPLATES=True to .env
# Add a folder to the project root called 'jinja_templates'
# with a copy of 'email_template.jinja2' from notification-utils repo
debug_template_path = (
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if os.environ.get("USE_LOCAL_JINJA_TEMPLATES") == "True" else None)
html_email = HTMLEmailTemplate(
template_dict,
values=personalisation_data,
jinja_path=debug_template_path,
**get_html_email_options(service),
)
plain_text_email = PlainTextEmailTemplate(template_dict,
values=personalisation_data)
if current_app.config["SCAN_FOR_PII"]:
contains_pii(notification, str(plain_text_email))
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = send_email_response(notification.to)
update_notification_to_sending(notification, provider)
else:
if service.sending_domain is None or service.sending_domain.strip(
) == "":
sending_domain = current_app.config["NOTIFY_EMAIL_DOMAIN"]
else:
sending_domain = service.sending_domain
from_address = '"{}" <{}@{}>'.format(service.name,
service.email_from,
sending_domain)
email_reply_to = notification.reply_to_text
reference = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(
email_reply_to) if email_reply_to else None,
attachments=attachments,
)
notification.reference = reference
update_notification_to_sending(notification, provider)
# Record StatsD stats to compute SLOs
statsd_client.timing_with_dates("email.total-time",
notification.sent_at,
notification.created_at)
attachments_category = "with-attachments" if attachments else "no-attachments"
statsd_key = f"email.{attachments_category}.process_type-{template_dict['process_type']}"
statsd_client.timing_with_dates(statsd_key, notification.sent_at,
notification.created_at)
statsd_client.incr(statsd_key)
