def process_ses_results(self, response):
try:
ses_message = json.loads(response['Message'])
notification_type = ses_message['notificationType']
if notification_type == 'Bounce':
notification_type = determine_notification_bounce_type(
notification_type, ses_message)
elif notification_type == 'Complaint':
_check_and_queue_complaint_callback_task(
*handle_complaint(ses_message))
return True
aws_response_dict = get_aws_responses(notification_type)
notification_status = aws_response_dict['notification_status']
reference = ses_message['mail']['messageId']
try:
notification = notifications_dao.dao_get_notification_by_reference(
reference)
except NoResultFound:
message_time = iso8601.parse_date(
ses_message['mail']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
"notification not found for reference: {} (update to {})".
format(reference, notification_status))
return
if notification.status not in {
NOTIFICATION_SENDING, NOTIFICATION_PENDING
}:
notifications_dao._duplicate_update_warning(
notification, notification_status)
return
notifications_dao._update_notification_status(
notification=notification, status=notification_status)
if not aws_response_dict['success']:
current_app.logger.info(
"SES delivery failed: notification id {} and reference {} has error found. Status {}"
.format(notification.id, reference,
aws_response_dict['message']))
else:
current_app.logger.info(
'SES callback return status of {} for notification: {}'.format(
notification_status, notification.id))
statsd_client.incr('callback.ses.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates('callback.ses.elapsed-time',
datetime.utcnow(),
notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception('Error processing SES results: {}'.format(
type(e)))
self.retry(queue=QueueNames.RETRY)
def process_sns_results(self, response):
try:
# Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html
sns_message = json.loads(response['Message'])
reference = sns_message['notification']['messageId']
status = sns_message['status']
provider_response = sns_message['delivery']['providerResponse']
# See all the possible provider responses
# https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html#sms_stats_delivery_fail_reasons
reasons = {
'Blocked as spam by phone carrier': NOTIFICATION_TECHNICAL_FAILURE,
'Destination is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE,
'Invalid phone number': NOTIFICATION_TECHNICAL_FAILURE,
'Message body is invalid': NOTIFICATION_TECHNICAL_FAILURE,
'Phone carrier has blocked this message': NOTIFICATION_TECHNICAL_FAILURE,
'Phone carrier is currently unreachable/unavailable': NOTIFICATION_TEMPORARY_FAILURE,
'Phone has blocked SMS': NOTIFICATION_TECHNICAL_FAILURE,
'Phone is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE,
'Phone is currently unreachable/unavailable': NOTIFICATION_PERMANENT_FAILURE,
'Phone number is opted out': NOTIFICATION_TECHNICAL_FAILURE,
'This delivery would exceed max price': NOTIFICATION_TECHNICAL_FAILURE,
'Unknown error attempting to reach phone': NOTIFICATION_TECHNICAL_FAILURE,
}
if status == "SUCCESS":
notification_status = NOTIFICATION_DELIVERED
else:
if provider_response not in reasons:
current_app.logger.warning(
f"unhandled provider response for reference {reference}, received '{provider_response}'"
)
notification_status = reasons.get(provider_response, NOTIFICATION_TECHNICAL_FAILURE)
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(sns_message['notification']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
f"notification not found for reference: {reference} (update to {notification_status})"
)
return
if notification.sent_by != SNS_PROVIDER:
current_app.logger.exception(f'SNS callback handled notification {notification.id} not sent by SNS')
return
if notification.status != NOTIFICATION_SENT:
notifications_dao._duplicate_update_warning(notification, notification_status)
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status
)
if notification_status != NOTIFICATION_DELIVERED:
current_app.logger.info((
f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. "
f"Provider response: {sns_message['delivery']['providerResponse']}"
))
else:
current_app.logger.info(
f'SNS callback return status of {notification_status} for notification: {notification.id}'
)
statsd_client.incr(f'callback.sns.{notification_status}')
if notification.sent_at:
statsd_client.timing_with_dates('callback.sns.elapsed-time', datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(f'Error processing SNS results: {str(e)}')
self.retry(queue=QueueNames.RETRY)
def process_sns_results(self, response):
try:
# Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html
sns_message = json.loads(response["Message"])
reference = sns_message["notification"]["messageId"]
sns_status = sns_message["status"]
provider_response = sns_message["delivery"]["providerResponse"]
try:
notification_status = determine_status(sns_status, provider_response)
except KeyError:
current_app.logger.warning(f"unhandled provider response for reference {reference}, received '{provider_response}'")
notification_status = NOTIFICATION_TECHNICAL_FAILURE
provider_response = None
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(sns_message["notification"]["timestamp"]).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(f"notification not found for reference: {reference} (update to {notification_status})")
return
if notification.sent_by != SNS_PROVIDER:
current_app.logger.exception(f"SNS callback handled notification {notification.id} not sent by SNS")
return
if notification.status != NOTIFICATION_SENT:
notifications_dao._duplicate_update_warning(notification, notification_status)
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status,
provider_response=provider_response if notification_status == NOTIFICATION_TECHNICAL_FAILURE else None,
)
if notification_status != NOTIFICATION_DELIVERED:
current_app.logger.info(
(
f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. "
f"Provider response: {sns_message['delivery']['providerResponse']}"
)
)
else:
current_app.logger.info(f"SNS callback return status of {notification_status} for notification: {notification.id}")
statsd_client.incr(f"callback.sns.{notification_status}")
if notification.sent_at:
statsd_client.timing_with_dates("callback.sns.elapsed-time", datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(f"Error processing SNS results: {str(e)}")
self.retry(queue=QueueNames.RETRY)
def process_virus_scan_passed(self, filename):
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
current_app.logger.info("notification id {} Virus scan passed: {}".format(
notification.id, filename))
is_test_key = notification.key_type == KEY_TYPE_TEST
scan_pdf_object = s3.get_s3_object(
current_app.config["LETTERS_SCAN_BUCKET_NAME"], filename)
old_pdf = scan_pdf_object.get()["Body"].read()
try:
billable_units = get_page_count(old_pdf)
except PdfReadError:
current_app.logger.exception(
msg="Invalid PDF received for notification_id: {}".format(
notification.id))
_move_invalid_letter_and_update_status(notification, filename,
scan_pdf_object)
return
sanitise_response = _sanitise_precompiled_pdf(self, notification, old_pdf)
if not sanitise_response:
new_pdf = None
else:
sanitise_response = sanitise_response.json()
try:
new_pdf = base64.b64decode(sanitise_response["file"].encode())
except JSONDecodeError:
new_pdf = sanitise_response.content
redaction_failed_message = sanitise_response.get(
"redaction_failed_message")
if redaction_failed_message and not is_test_key:
current_app.logger.info("{} for notification id {} ({})".format(
redaction_failed_message, notification.id, filename))
copy_redaction_failed_pdf(filename)
# TODO: Remove this once CYSP update their template to not cross over the margins
if notification.service_id == UUID(
"fe44178f-3b45-4625-9f85-2264a36dd9ec"): # CYSP
# Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
new_pdf = old_pdf
if not new_pdf:
current_app.logger.info(
"Invalid precompiled pdf received {} ({})".format(
notification.id, filename))
_move_invalid_letter_and_update_status(notification, filename,
scan_pdf_object)
return
else:
current_app.logger.info(
"Validation was successful for precompiled pdf {} ({})".format(
notification.id, filename))
current_app.logger.info(
"notification id {} ({}) sanitised and ready to send".format(
notification.id, filename))
try:
_upload_pdf_to_test_or_live_pdf_bucket(new_pdf,
filename,
is_test_letter=is_test_key)
update_letter_pdf_status(
reference=reference,
status=NOTIFICATION_DELIVERED
if is_test_key else NOTIFICATION_CREATED,
billable_units=billable_units,
)
scan_pdf_object.delete()
except BotoClientError:
current_app.logger.exception(
"Error uploading letter to live pdf bucket for notification: {}".
format(notification.id))
update_notification_status_by_id(notification.id,
NOTIFICATION_TECHNICAL_FAILURE)