def receive_firetext_sms():
post_data = request.form
auth = request.authorization
if not auth:
current_app.logger.warning("Inbound sms (Firetext) no auth header")
abort(401)
elif auth.username != 'notify' or auth.password not in current_app.config[
'FIRETEXT_INBOUND_SMS_AUTH']:
current_app.logger.warning(
"Inbound sms (Firetext) incorrect username ({}) or password".
format(auth.username))
abort(403)
inbound_number = strip_leading_forty_four(post_data['destination'])
service = fetch_potential_service(inbound_number, 'firetext')
if not service:
return jsonify({"status": "ok"}), 200
inbound = create_inbound_sms_object(service=service,
content=post_data["message"],
from_number=post_data['source'],
provider_ref=None,
date_received=post_data['time'],
provider_name="firetext")
statsd_client.incr('inbound.firetext.successful')
tasks.send_inbound_sms_to_service.apply_async(
[str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug(
'{} received inbound SMS with reference {} from Firetext'.format(
service.id, inbound.provider_reference))
return jsonify({"status": "ok"}), 200
def process_sendgrid_response():
data = json.loads(request.data)
try:
for obj in data:
notification_status = get_sendgrid_responses(obj["event"])
reference = obj['sg_message_id'].split(".")[0]
notification = notifications_dao.dao_get_notification_by_reference(reference)
notifications_dao._update_notification_status(notification=notification, status=notification_status)
current_app.logger.info('SendGird callback return status of {} for notification: {}'.format(
notification_status, notification.id
))
statsd_client.incr('callback.sendgrid.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.sendgrid.elapsed-time', datetime.utcnow(), notification.sent_at)
except Exception as e:
current_app.logger.exception('Error processing SendGrid results: {}'.format(type(e)))
raise InvalidRequest(message="Error processing SendGrid results", status_code=400)
else:
return jsonify(result='success'), 200
def _process_for_status(notification_status, client_name, provider_reference):
notification = notifications_dao.update_notification_status_by_id(
provider_reference, notification_status)
if not notification:
current_app.logger.warning(
"{} callback failed: notification {} either not found or already updated "
"from sending. Status {}".format(client_name, provider_reference,
notification_status))
return
statsd_client.incr('callback.{}.{}'.format(client_name,
notification_status))
if not notification.sent_by:
set_notification_sent_by(notification, client_name)
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name), datetime.utcnow(),
notification.sent_at)
check_for_callback_and_send_delivery_status_to_service(notification)
success = "{} callback succeeded. reference {} updated".format(
client_name, provider_reference)
return success
def _receive_sap_sms(provider_name, data):
response = MessagingResponse()
service = fetch_potential_service(data['originatingAddress'],
provider_name)
if not service:
# Since this is an issue with our service <-> number mapping, or no
# inbound_sms service permission we should still tell SAP that we
# received it successfully.
return response
statsd_client.incr(f'inbound.{provider_name}.successful')
inbound = create_inbound_sms_object(service,
content=data["message"],
from_number=data['msisdn'],
provider_ref=data["messageId"],
provider_name=provider_name)
tasks.send_inbound_sms_to_service.apply_async(
[str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug(
'{} received inbound SMS with reference {} from SAP:{}'.format(
service.id,
inbound.provider_reference,
provider_name,
))
return response
def logout():
_assert_github_login_toggle_enabled()
response = make_response(redirect(f"{current_app.config['UI_HOST_NAME']}"))
response.delete_cookie(current_app.config['JWT_ACCESS_COOKIE_NAME'])
statsd_client.incr('oauth.logout.success')
return response
def attempt_to_get_notification(
reference: str, notification_status: str,
event_timestamp: str) -> Tuple[Notification, bool, bool]:
should_retry = False
notification = None
try:
notification = dao_get_notification_by_reference(reference)
should_exit = check_notification_status(notification,
notification_status)
except NoResultFound:
message_time = iso8601.parse_date(event_timestamp).replace(tzinfo=None)
if datetime.datetime.utcnow() - message_time < datetime.timedelta(
minutes=5):
should_retry = True
else:
current_app.logger.warning(
f'notification not found for reference: {reference} (update to {notification_status})'
)
statsd_client.incr('callback.pinpoint.no_notification_found')
should_exit = True
except MultipleResultsFound:
current_app.logger.warning(
f'multiple notifications found for reference: {reference} (update to {notification_status})'
)
statsd_client.incr('callback.pinpoint.multiple_notifications_found')
should_exit = True
return notification, should_retry, should_exit
def process_pinpoint_inbound_sms(self, event: CeleryEvent):
provider_name = 'pinpoint'
if not is_feature_enabled(FeatureFlag.PINPOINT_INBOUND_SMS_ENABLED):
current_app.logger.info(
'Pinpoint inbound SMS toggle is disabled, skipping task')
return True
pinpoint_message: PinpointInboundSmsMessage = json.loads(event['Message'])
service = fetch_potential_service(pinpoint_message['destinationNumber'],
provider_name)
statsd_client.incr(f"inbound.{provider_name}.successful")
inbound_sms = create_inbound_sms_object(
service=service,
content=pinpoint_message['messageBody'],
notify_number=pinpoint_message['destinationNumber'],
from_number=pinpoint_message['originationNumber'],
provider_ref=pinpoint_message['inboundMessageId'],
date_received=datetime.utcnow(),
provider_name=provider_name)
send_inbound_sms_to_service.apply_async([inbound_sms.id, service.id],
queue=QueueNames.NOTIFY)
def get_notification_status(event_type: str, record_status: str,
reference: str) -> str:
if event_type_is_optout(event_type, reference):
statsd_client.incr(f"callback.pinpoint.optout")
notification_status = NOTIFICATION_PERMANENT_FAILURE
else:
notification_status = _map_record_status_to_notification_status(
record_status)
return notification_status
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == "created":
provider = provider_to_use(
SMS_TYPE,
notification.id,
notification.international,
notification.reply_to_text,
)
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
template = SMSMessageTemplate(
template_dict,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = send_sms_response(provider.get_name(),
notification.to)
update_notification_to_sending(notification, provider)
else:
try:
reference = provider.send_sms(
to=validate_and_format_phone_number(
notification.to,
international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text,
)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.reference = reference
notification.billable_units = template.fragment_count
update_notification_to_sending(notification, provider)
# Record StatsD stats to compute SLOs
statsd_client.timing_with_dates("sms.total-time", notification.sent_at,
notification.created_at)
statsd_key = f"sms.process_type-{template_dict['process_type']}"
statsd_client.timing_with_dates(statsd_key, notification.sent_at,
notification.created_at)
statsd_client.incr(statsd_key)
def process_sms_client_response(status, reference, client_name):
success = None
errors = None
# validate reference
if reference == 'send-sms-code':
success = "{} callback succeeded: send-sms-code".format(client_name)
return success, errors
try:
uuid.UUID(reference, version=4)
except ValueError:
message = "{} callback with invalid reference {}".format(client_name, reference)
return success, message
try:
response_parser = sms_response_mapper[client_name]
except KeyError:
return success, 'unknown sms client: {}'.format(client_name)
# validate status
try:
response_dict = response_parser(status)
current_app.logger.info('{} callback return status of {} for reference: {}'.format(client_name,
status, reference))
except KeyError:
msg = "{} callback failed: status {} not found.".format(client_name, status)
return success, msg
notification_status = response_dict['notification_status']
notification_status_message = response_dict['message']
notification_success = response_dict['success']
# record stats
notification = notifications_dao.update_notification_status_by_id(reference, notification_status)
if not notification:
status_error = "{} callback failed: notification {} either not found or already updated " \
"from sending. Status {}".format(client_name,
reference,
notification_status_message)
return success, status_error
if not notification_success:
current_app.logger.info(
"{} delivery failed: notification {} has error found. Status {}".format(client_name,
reference,
notification_status_message))
statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name.lower()),
datetime.utcnow(),
notification.sent_at
)
success = "{} callback succeeded. reference {} updated".format(client_name, reference)
return success, errors
def process_ses_results(response):
try:
ses_message = json.loads(response['Message'])
notification_type = ses_message['notificationType']
if notification_type == 'Bounce':
notification_type = determine_notification_bounce_type(notification_type, ses_message)
elif notification_type == 'Complaint':
_check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
return True, False
aws_response_dict = get_aws_responses(notification_type)
notification_status = aws_response_dict['notification_status']
reference = ses_message['mail']['messageId']
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
return None, True
current_app.logger.warning(
"notification not found for reference: {} (update to {})".format(reference, notification_status)
)
return None, False
if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
notifications_dao._duplicate_update_warning(notification, notification_status)
return None, False
notifications_dao._update_notification_status(notification=notification, status=notification_status)
if not aws_response_dict['success']:
current_app.logger.info(
"SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
notification.id, reference, aws_response_dict['message']
)
)
else:
current_app.logger.info('SES callback returned status of {} for notification: {}'.format(
notification_status, notification.id
))
statsd_client.incr('callback.ses.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True, False
except Exception as e:
current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
return None, True
def publish_complaint(complaint: Complaint, notification: Notification,
recipient_email: str) -> bool:
provider_name = notification.sent_by
_check_and_queue_complaint_callback_task(complaint, notification,
recipient_email)
send_complaint_to_vanotify.apply_async(
[str(complaint.id), notification.template.name],
queue=QueueNames.NOTIFY)
statsd_client.incr(f'callback.{provider_name}.complaint_count')
return True
def receive_twilio_sms():
response = MessagingResponse()
auth = request.authorization
if not auth:
current_app.logger.warning("Inbound sms (Twilio) no auth header")
abort(401)
elif auth.username not in current_app.config[
'TWILIO_INBOUND_SMS_USERNAMES'] or auth.password not in current_app.config[
'TWILIO_INBOUND_SMS_PASSWORDS']:
current_app.logger.warning(
"Inbound sms (Twilio) incorrect username ({}) or password".format(
auth.username))
abort(403)
# Locally, when using ngrok the URL comes in without HTTPS so force it
# otherwise the Twilio signature validator will fail.
url = request.url.replace("http://", "https://")
post_data = request.form
twilio_signature = request.headers.get('X-Twilio-Signature')
validator = RequestValidator(os.getenv('TWILIO_AUTH_TOKEN'))
if not validator.validate(url, post_data, twilio_signature):
current_app.logger.warning(
"Inbound sms (Twilio) signature did not match request")
abort(400)
service = fetch_potential_service(post_data['To'], 'twilio')
if not service:
# Since this is an issue with our service <-> number mapping, or no
# inbound_sms service permission we should still tell Twilio that we
# received it successfully.
return str(response), 200
statsd_client.incr('inbound.twilio.successful')
inbound = create_inbound_sms_object(service,
content=post_data["Body"],
from_number=post_data['From'],
provider_ref=post_data["MessageSid"],
provider_name="twilio")
tasks.send_inbound_sms_to_service.apply_async(
[str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug(
'{} received inbound SMS with reference {} from Twilio'.format(
service.id,
inbound.provider_reference,
))
return str(response), 200
def receive_twilio_sms():
response = MessagingResponse()
auth = request.authorization
if not auth:
current_app.logger.warning("Inbound sms (Twilio) no auth header")
abort(401)
elif auth.username not in current_app.config['TWILIO_INBOUND_SMS_USERNAMES'] \
or auth.password not in current_app.config['TWILIO_INBOUND_SMS_PASSWORDS']:
current_app.logger.warning(
"Inbound sms (Twilio) incorrect username ({}) or password".format(
auth.username))
abort(403)
url = request.url
post_data = request.form
twilio_signature = request.headers.get('X-Twilio-Signature')
validator = RequestValidator(current_app.config['TWILIO_AUTH_TOKEN'])
if not validator.validate(url, post_data, twilio_signature):
current_app.logger.warning(
"Inbound sms (Twilio) signature did not match request")
abort(400)
try:
service = fetch_potential_service(post_data['To'], 'twilio')
except NoSuitableServiceForInboundSms:
# Since this is an issue with our service <-> number mapping, or no
# inbound_sms service permission we should still tell Twilio that we
# received it successfully.
return str(response), 200
statsd_client.incr('inbound.twilio.successful')
inbound = create_inbound_sms_object(service,
content=post_data["Body"],
notify_number=post_data['To'],
from_number=post_data['From'],
provider_ref=post_data["MessageSid"],
date_received=datetime.utcnow(),
provider_name="twilio")
send_inbound_sms_to_service.apply_async(
[str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug(
'{} received inbound SMS with reference {} from Twilio'.format(
service.id,
inbound.provider_reference,
))
return str(response), 200
def receive_mmg_sms():
"""
{
'MSISDN': '447123456789'
'Number': '40604',
'Message': 'some+uri+encoded+message%3A',
'ID': 'SOME-MMG-SPECIFIC-ID',
'DateRecieved': '2017-05-21+11%3A56%3A11'
}
"""
post_data = request.get_json()
auth = request.authorization
if not auth:
current_app.logger.warning("Inbound sms (MMG) no auth header")
abort(401)
elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \
or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']:
current_app.logger.warning(
"Inbound sms (MMG) incorrect username ({}) or password".format(
auth.username))
abort(403)
inbound_number = strip_leading_forty_four(post_data['Number'])
try:
service = fetch_potential_service(inbound_number, 'mmg')
except NoSuitableServiceForInboundSms:
# since this is an issue with our service <-> number mapping, or no inbound_sms service permission
# we should still tell MMG that we received it successfully
return 'RECEIVED', 200
statsd_client.incr('inbound.mmg.successful')
inbound = create_inbound_sms_object(
service,
content=format_mmg_message(post_data["Message"]),
notify_number=inbound_number,
from_number=post_data['MSISDN'],
provider_ref=post_data["ID"],
date_received=format_mmg_datetime(post_data.get('DateRecieved')),
provider_name="mmg")
send_inbound_sms_to_service.apply_async(
[str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug(
'{} received inbound SMS with reference {} from MMG'.format(
service.id, inbound.provider_reference))
return jsonify({"status": "ok"}), 200
def process_pinpoint_results(self, response):
if not is_feature_enabled(FeatureFlag.PINPOINT_RECEIPTS_ENABLED):
current_app.logger.info(
'Pinpoint receipts toggle is disabled, skipping callback task')
return True
try:
pinpoint_message = json.loads(base64.b64decode(response['Message']))
reference = pinpoint_message['attributes']['message_id']
event_type = pinpoint_message.get('event_type')
record_status = pinpoint_message['attributes']['record_status']
current_app.logger.info(
f'received callback from Pinpoint with event_type of {event_type} and record_status of {record_status}'
f'with reference {reference}')
notification_status = get_notification_status(event_type,
record_status, reference)
notification, should_retry, should_exit = attempt_to_get_notification(
reference, notification_status,
pinpoint_message['event_timestamp'])
if should_retry:
self.retry(queue=QueueNames.RETRY)
if should_exit:
return
update_notification_status_by_id(notification.id, notification_status)
current_app.logger.info(
f"Pinpoint callback return status of {notification_status} for notification: {notification.id}"
)
statsd_client.incr(f"callback.pinpoint.{notification_status}")
if notification.sent_at:
statsd_client.timing_with_dates('callback.pinpoint.elapsed-time',
datetime.datetime.utcnow(),
notification.sent_at)
check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(
f"Error processing Pinpoint results: {type(e)}")
self.retry(queue=QueueNames.RETRY)
def _process_for_status(notification_status,
client_name,
provider_reference,
detailed_status_code=None):
# record stats
if client_name == 'Twilio':
notification = notifications_dao.update_notification_status_by_reference(
reference=provider_reference, status=notification_status)
else:
notification = notifications_dao.update_notification_status_by_id(
notification_id=provider_reference,
status=notification_status,
sent_by=client_name.lower(),
detailed_status_code=detailed_status_code)
if not notification:
return
statsd_client.incr('callback.{}.{}'.format(client_name.lower(),
notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name.lower()),
datetime.utcnow(), notification.sent_at)
if notification.billable_units == 0:
service = notification.service
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
notification.billable_units = template.fragment_count
notifications_dao.dao_update_notification(notification)
if notification_status != NOTIFICATION_PENDING:
service_callback_api = get_service_delivery_status_callback_api_for_service(
service_id=notification.service_id)
# queue callback task only if the service_callback_api exists
if service_callback_api:
encrypted_notification = create_delivery_status_callback_data(
notification, service_callback_api)
send_delivery_status_to_service.apply_async(
[str(notification.id), encrypted_notification],
queue=QueueNames.CALLBACKS)
def process_govdelivery_response():
try:
data = validate(request.form, govdelivery_webhook_schema)
sid = data['sid']
reference = data['message_url'].split("/")[-1]
govdelivery_status = data['status']
notify_status = govdelivery_status_map[govdelivery_status]
except ValidationError as e:
raise e
except Exception as e:
raise InvalidRequest(f'Error processing Govdelivery callback: {e}',
400)
else:
try:
notification = notifications_dao.dao_get_notification_by_reference(
reference)
except (MultipleResultsFound, NoResultFound) as e:
exception_type = type(e).__name__
current_app.logger.exception(
f'Govdelivery callback with sid {sid} for reference {reference} '
f'did not find exactly one notification: {exception_type}')
statsd_client.incr(
f'callback.govdelivery.failure.{exception_type}')
else:
current_app.logger.info(
f'Govdelivery callback for notification {notification.id} has status {govdelivery_status},'
f' which maps to notification-api status {notify_status}')
if data.get('error_message'):
current_app.logger.info(
f"Govdelivery error_message for notification {notification.id}: "
f"{data['error_message']}")
notifications_dao._update_notification_status(
notification, notify_status)
statsd_client.incr(f'callback.govdelivery.{notify_status}')
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.govdelivery.elapsed-time', datetime.utcnow(),
notification.sent_at)
if govdelivery_status == 'blacklisted':
complaint = create_complaint(data, notification)
publish_complaint(complaint, notification, notification.to)
return jsonify(result='success'), 200
def wrapper(*args, **kwargs):
start_time = monotonic()
res = func(*args, **kwargs)
elapsed_time = monotonic() - start_time
current_app.logger.info(
"{namespace} call {func} took {time}".format(
namespace=namespace, func=func.__name__, time="{0:.4f}".format(elapsed_time)
)
)
statsd_client.incr('{namespace}.{func}'.format(
namespace=namespace, func=func.__name__)
)
statsd_client.timing('{namespace}.{func}'.format(
namespace=namespace, func=func.__name__), elapsed_time
)
return res
def fetch_potential_service(inbound_number, provider_name):
service = dao_fetch_service_by_inbound_number(inbound_number)
if not service:
current_app.logger.error(
'Inbound number "{}" from {} not associated with a service'.format(
inbound_number, provider_name))
statsd_client.incr('inbound.{}.failed'.format(provider_name))
return False
if not has_inbound_sms_permissions(service.permissions):
current_app.logger.error(
'Service "{}" does not allow inbound SMS'.format(service.id))
return False
return service
def fetch_potential_service(inbound_number: str,
provider_name: str) -> Service:
service = dao_fetch_service_by_inbound_number(inbound_number)
if not service:
statsd_client.incr(f"inbound.{provider_name}.failed")
message = f'Inbound number "{inbound_number}" from {provider_name} not associated with a service'
current_app.logger.error(message)
raise NoSuitableServiceForInboundSms(message)
elif not has_inbound_sms_permissions(service.permissions):
statsd_client.incr(f"inbound.{provider_name}.failed")
message = f'Service "{service.id}" does not allow inbound SMS'
current_app.logger.error(message)
raise NoSuitableServiceForInboundSms(message)
else:
return service
def process_govdelivery_response():
try:
data = request.form
reference = data['message_url'].split("/")[-1]
govdelivery_status = data['status']
notify_status = map_govdelivery_status_to_notify_status(
govdelivery_status)
except Exception as e:
raise InvalidRequest(
'Error processing Govdelivery callback: {}'.format(e), 400)
else:
try:
notification = notifications_dao.dao_get_notification_by_reference(
reference)
except (MultipleResultsFound, NoResultFound) as e:
current_app.logger.exception(
'Govdelivery callback for reference {} did not find exactly one notification: {}'
.format(reference, type(e)))
pass
else:
current_app.logger.info(
'Govdelivery callback for notification {} has status "{}", which maps to notification-api status "{}"'
.format(notification.id, govdelivery_status, notify_status))
notifications_dao._update_notification_status(
notification, notify_status)
statsd_client.incr('callback.govdelivery.{}'.format(notify_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.govdelivery.elapsed-time', datetime.utcnow(),
notification.sent_at)
return jsonify(result='success'), 200
def _process_for_status(notification_status, client_name, provider_reference):
# record stats
notification = notifications_dao.update_notification_status_by_id(
provider_reference, notification_status)
if not notification:
current_app.logger.warning(
"{} callback failed: notification {} either not found or already updated "
"from sending. Status {}".format(client_name, provider_reference,
notification_status))
return
statsd_client.incr('callback.{}.{}'.format(client_name.lower(),
notification_status))
if not notification.sent_by:
set_notification_sent_by(notification, client_name.lower())
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name.lower()),
datetime.utcnow(), notification.sent_at)
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_delivery_status_callback_api_for_service(
service_id=notification.service_id)
if service_callback_api:
encrypted_notification = create_encrypted_callback_data(
notification, service_callback_api)
send_delivery_status_to_service.apply_async(
[str(notification.id), encrypted_notification],
queue=QueueNames.CALLBACKS)
success = "{} callback succeeded. reference {} updated".format(
client_name, provider_reference)
return success
def authorize():
_assert_github_login_toggle_enabled()
try:
github_token = oauth_registry.github.authorize_access_token()
make_github_get_request(
'/user/memberships/orgs/department-of-veterans-affairs',
github_token)
email_resp = make_github_get_request('/user/emails', github_token)
user_resp = make_github_get_request('/user', github_token)
verified_email, verified_user_id, verified_name = _extract_github_user_info(
email_resp, user_resp)
user = create_or_retrieve_user(
email_address=verified_email,
identity_provider_user_id=verified_user_id,
name=verified_name)
except OAuthError as e:
current_app.logger.error(f'User denied authorization: {e}')
statsd_client.incr('oauth.authorization.denied')
return make_response(
redirect(
f"{current_app.config['UI_HOST_NAME']}/login/failure?denied_authorization"
))
except (OAuthException, HTTPError) as e:
current_app.logger.error(f"Authorization exception raised:\n{e}\n")
statsd_client.incr('oauth.authorization.failure')
return make_response(
redirect(f"{current_app.config['UI_HOST_NAME']}/login/failure"))
except IncorrectGithubIdException as e:
current_app.logger.error(e)
statsd_client.incr('oauth.authorization.github_id_mismatch')
return make_response(
redirect(f"{current_app.config['UI_HOST_NAME']}/login/failure"))
else:
response = make_response(
redirect(f"{current_app.config['UI_HOST_NAME']}/login/success"))
response.set_cookie(
current_app.config['JWT_ACCESS_COOKIE_NAME'],
create_access_token(identity=user),
httponly=True,
secure=current_app.config['SESSION_COOKIE_SECURE'],
samesite=current_app.config['SESSION_COOKIE_SAMESITE'])
statsd_client.incr('oauth.authorization.success')
return response
def process_sns_results(self, response):
try:
# Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html
sns_message = json.loads(response["Message"])
reference = sns_message["notification"]["messageId"]
sns_status = sns_message["status"]
provider_response = sns_message["delivery"]["providerResponse"]
try:
notification_status = determine_status(sns_status, provider_response)
except KeyError:
current_app.logger.warning(f"unhandled provider response for reference {reference}, received '{provider_response}'")
notification_status = NOTIFICATION_TECHNICAL_FAILURE
provider_response = None
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(sns_message["notification"]["timestamp"]).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(f"notification not found for reference: {reference} (update to {notification_status})")
return
if notification.sent_by != SNS_PROVIDER:
current_app.logger.exception(f"SNS callback handled notification {notification.id} not sent by SNS")
return
if notification.status != NOTIFICATION_SENT:
notifications_dao._duplicate_update_warning(notification, notification_status)
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status,
provider_response=provider_response if notification_status == NOTIFICATION_TECHNICAL_FAILURE else None,
)
if notification_status != NOTIFICATION_DELIVERED:
current_app.logger.info(
(
f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. "
f"Provider response: {sns_message['delivery']['providerResponse']}"
)
)
else:
current_app.logger.info(f"SNS callback return status of {notification_status} for notification: {notification.id}")
statsd_client.incr(f"callback.sns.{notification_status}")
if notification.sent_at:
statsd_client.timing_with_dates("callback.sns.elapsed-time", datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(f"Error processing SNS results: {str(e)}")
self.retry(queue=QueueNames.RETRY)
def process_ses_results(self, response):
try:
ses_message = json.loads(response["Message"])
notification_type = ses_message["notificationType"]
if notification_type == "Complaint":
_check_and_queue_complaint_callback_task(
*handle_complaint(ses_message))
return True
aws_response_dict = get_aws_responses(ses_message)
notification_status = aws_response_dict["notification_status"]
reference = ses_message["mail"]["messageId"]
try:
notification = notifications_dao.dao_get_notification_by_reference(
reference)
except NoResultFound:
message_time = iso8601.parse_date(
ses_message["mail"]["timestamp"]).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
"notification not found for reference: {} (update to {})".
format(reference, notification_status))
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status,
provider_response=aws_response_dict["provider_response"],
)
if not aws_response_dict["success"]:
current_app.logger.info(
"SES delivery failed: notification id {} and reference {} has error found. Status {}"
.format(notification.id, reference,
aws_response_dict["message"]))
else:
current_app.logger.info(
"SES callback return status of {} for notification: {}".format(
notification_status, notification.id))
statsd_client.incr("callback.ses.{}".format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates("callback.ses.elapsed-time",
datetime.utcnow(),
notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception("Error processing SES results: {}".format(
type(e)))
self.retry(queue=QueueNames.RETRY)
def process_ses_results(self, response):
try:
ses_message = json.loads(response['Message'])
notification_type = ses_message['notificationType']
bounce_message = None
if notification_type == 'Bounce':
notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message)
elif notification_type == 'Complaint':
_check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
return True
aws_response_dict = get_aws_responses(notification_type)
notification_status = aws_response_dict['notification_status']
reference = ses_message['mail']['messageId']
try:
notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
except NoResultFound:
message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
current_app.logger.info(
f"notification not found for reference: {reference} (update to {notification_status}). "
f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
)
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
f"notification not found for reference: {reference} (update to {notification_status})"
)
return
if bounce_message:
current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
notifications_dao._duplicate_update_warning(
notification=notification,
status=notification_status
)
return
else:
notifications_dao.dao_update_notifications_by_reference(
references=[reference],
update_dict={'status': notification_status}
)
statsd_client.incr('callback.ses.{}'.format(notification_status))
if notification.sent_at:
statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
self.retry(queue=QueueNames.RETRY)
def process_ses_smtp_results(self, response):
try:
ses_message = json.loads(response['Message'])
notification_type = ses_message['notificationType']
headers = ses_message['mail']['commonHeaders']
source = ses_message['mail']['source']
recipients = ses_message['mail']['destination']
if notification_type == 'Bounce':
notification_type = determine_notification_bounce_type(
notification_type, ses_message)
aws_response_dict = get_aws_responses(notification_type)
notification_status = aws_response_dict['notification_status']
try:
# Get service based on SMTP name
service = services_dao.dao_services_by_partial_smtp_name(
source.split("@")[-1])
# Create a sent notification based on details from the payload
template = templates_dao.dao_get_template_by_id(
current_app.config['SMTP_TEMPLATE_ID'])
for recipient in recipients:
message = "".join((
'A message was sent from: \n', # noqa: E126
source,
'\n\n to: \n',
recipient,
'\n\n on: \n',
headers["date"],
'\n\n with the subject: \n',
headers["subject"]))
notification = process_notifications.persist_notification(
template_id=template.id,
template_version=template.version,
recipient=recipient,
service=service,
personalisation={
'subject': headers["subject"],
'message': message
},
notification_type=EMAIL_TYPE,
api_key_id=None,
key_type=KEY_TYPE_NORMAL,
reply_to_text=recipient,
created_at=headers["date"],
status=notification_status,
reference=ses_message['mail']['messageId'])
if notification_type == 'Complaint':
_check_and_queue_complaint_callback_task(
*handle_smtp_complaint(ses_message))
else:
_check_and_queue_callback_task(notification)
except NoResultFound:
reference = ses_message['mail']['messageId']
current_app.logger.warning(
"SMTP service not found for reference: {} (update to {})".
format(reference, notification_status))
return
statsd_client.incr('callback.ses-smtp.{}'.format(notification_status))
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(
'Error processing SES SMTP results: {}'.format(type(e)))
self.retry(queue=QueueNames.RETRY)
def send_callback_metrics(notification):
statsd_client.incr(f"callback.sns.{notification.status}")
if notification.sent_at:
statsd_client.timing_with_dates("callback.sns.elapsed-time", datetime.utcnow(), notification.sent_at)
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == "created":
provider = provider_to_use(EMAIL_TYPE, notification.id)
# Extract any file objects from the personalization
file_keys = [
k for k, v in (notification.personalisation or {}).items()
if isinstance(v, dict) and "document" in v
]
attachments = []
personalisation_data = notification.personalisation.copy()
for key in file_keys:
sending_method = personalisation_data[key]["document"].get(
"sending_method")
# Check if a MLWR sid exists
if (current_app.config["MLWR_HOST"]
and "mlwr_sid" in personalisation_data[key]["document"]
and personalisation_data[key]["document"]["mlwr_sid"] !=
"false"):
mlwr_result = check_mlwr(
personalisation_data[key]["document"]["mlwr_sid"])
if "state" in mlwr_result and mlwr_result[
"state"] == "completed":
# Update notification that it contains malware
if "submission" in mlwr_result and mlwr_result[
"submission"]["max_score"] >= 500:
malware_failure(notification=notification)
return
else:
# Throw error so celery will retry in sixty seconds
raise MalwarePendingException
if sending_method == "attach":
try:
req = urllib.request.Request(
personalisation_data[key]["document"]
["direct_file_url"])
with urllib.request.urlopen(req) as response:
buffer = response.read()
filename = personalisation_data[key]["document"].get(
"filename")
mime_type = personalisation_data[key]["document"].get(
"mime_type")
attachments.append({
"name": filename,
"data": buffer,
"mime_type": mime_type,
})
except Exception:
current_app.logger.error(
"Could not download and attach {}".format(
personalisation_data[key]["document"]
["direct_file_url"]))
del personalisation_data[key]
else:
personalisation_data[key] = personalisation_data[key][
"document"]["url"]
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
# Local Jinja support - Add USE_LOCAL_JINJA_TEMPLATES=True to .env
# Add a folder to the project root called 'jinja_templates'
# with a copy of 'email_template.jinja2' from notification-utils repo
debug_template_path = (
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if os.environ.get("USE_LOCAL_JINJA_TEMPLATES") == "True" else None)
html_email = HTMLEmailTemplate(
template_dict,
values=personalisation_data,
jinja_path=debug_template_path,
**get_html_email_options(service),
)
plain_text_email = PlainTextEmailTemplate(template_dict,
values=personalisation_data)
if current_app.config["SCAN_FOR_PII"]:
contains_pii(notification, str(plain_text_email))
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = send_email_response(notification.to)
update_notification_to_sending(notification, provider)
else:
if service.sending_domain is None or service.sending_domain.strip(
) == "":
sending_domain = current_app.config["NOTIFY_EMAIL_DOMAIN"]
else:
sending_domain = service.sending_domain
from_address = '"{}" <{}@{}>'.format(service.name,
service.email_from,
sending_domain)
email_reply_to = notification.reply_to_text
reference = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(
email_reply_to) if email_reply_to else None,
attachments=attachments,
)
notification.reference = reference
update_notification_to_sending(notification, provider)
# Record StatsD stats to compute SLOs
statsd_client.timing_with_dates("email.total-time",
notification.sent_at,
notification.created_at)
attachments_category = "with-attachments" if attachments else "no-attachments"
statsd_key = f"email.{attachments_category}.process_type-{template_dict['process_type']}"
statsd_client.timing_with_dates(statsd_key, notification.sent_at,
notification.created_at)
statsd_client.incr(statsd_key)
def save_stats_for_attachments(files_data, service_id, template_id):
nb_files = len(files_data)
statsd_client.incr(f"attachments.nb-attachments.count-{nb_files}")
statsd_client.incr("attachments.nb-attachments", count=nb_files)
statsd_client.incr(f"attachments.services.{service_id}", count=nb_files)
statsd_client.incr(f"attachments.templates.{template_id}", count=nb_files)
for document in [f["document"] for f in files_data]:
statsd_client.incr(f"attachments.sending-method.{document['sending_method']}")
statsd_client.incr(f"attachments.file-type.{document['mime_type']}")
# File size is in bytes, convert to whole megabytes
nb_mb = document["file_size"] // (1_024 * 1_024)
file_size_bucket = f"{nb_mb}-{nb_mb + 1}mb"
statsd_client.incr(f"attachments.file-size.{file_size_bucket}")
def process_sns_results(self, response):
try:
# Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html
sns_message = json.loads(response['Message'])
reference = sns_message['notification']['messageId']
status = sns_message['status']
provider_response = sns_message['delivery']['providerResponse']
# See all the possible provider responses
# https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html#sms_stats_delivery_fail_reasons
reasons = {
'Blocked as spam by phone carrier': NOTIFICATION_TECHNICAL_FAILURE,
'Destination is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE,
'Invalid phone number': NOTIFICATION_TECHNICAL_FAILURE,
'Message body is invalid': NOTIFICATION_TECHNICAL_FAILURE,
'Phone carrier has blocked this message': NOTIFICATION_TECHNICAL_FAILURE,
'Phone carrier is currently unreachable/unavailable': NOTIFICATION_TEMPORARY_FAILURE,
'Phone has blocked SMS': NOTIFICATION_TECHNICAL_FAILURE,
'Phone is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE,
'Phone is currently unreachable/unavailable': NOTIFICATION_PERMANENT_FAILURE,
'Phone number is opted out': NOTIFICATION_TECHNICAL_FAILURE,
'This delivery would exceed max price': NOTIFICATION_TECHNICAL_FAILURE,
'Unknown error attempting to reach phone': NOTIFICATION_TECHNICAL_FAILURE,
}
if status == "SUCCESS":
notification_status = NOTIFICATION_DELIVERED
else:
if provider_response not in reasons:
current_app.logger.warning(
f"unhandled provider response for reference {reference}, received '{provider_response}'"
)
notification_status = reasons.get(provider_response, NOTIFICATION_TECHNICAL_FAILURE)
try:
notification = notifications_dao.dao_get_notification_by_reference(reference)
except NoResultFound:
message_time = iso8601.parse_date(sns_message['notification']['timestamp']).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
f"notification not found for reference: {reference} (update to {notification_status})"
)
return
if notification.sent_by != SNS_PROVIDER:
current_app.logger.exception(f'SNS callback handled notification {notification.id} not sent by SNS')
return
if notification.status != NOTIFICATION_SENT:
notifications_dao._duplicate_update_warning(notification, notification_status)
return
notifications_dao._update_notification_status(
notification=notification,
status=notification_status
)
if notification_status != NOTIFICATION_DELIVERED:
current_app.logger.info((
f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. "
f"Provider response: {sns_message['delivery']['providerResponse']}"
))
else:
current_app.logger.info(
f'SNS callback return status of {notification_status} for notification: {notification.id}'
)
statsd_client.incr(f'callback.sns.{notification_status}')
if notification.sent_at:
statsd_client.timing_with_dates('callback.sns.elapsed-time', datetime.utcnow(), notification.sent_at)
_check_and_queue_callback_task(notification)
return True
except Retry:
raise
except Exception as e:
current_app.logger.exception(f'Error processing SNS results: {str(e)}')
self.retry(queue=QueueNames.RETRY)
