def _process_for_status(notification_status, client_name, provider_reference): notification = notifications_dao.update_notification_status_by_id( provider_reference, notification_status) if not notification: current_app.logger.warning( "{} callback failed: notification {} either not found or already updated " "from sending. Status {}".format(client_name, provider_reference, notification_status)) return statsd_client.incr('callback.{}.{}'.format(client_name, notification_status)) if not notification.sent_by: set_notification_sent_by(notification, client_name) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name), datetime.utcnow(), notification.sent_at) check_for_callback_and_send_delivery_status_to_service(notification) success = "{} callback succeeded. reference {} updated".format( client_name, provider_reference) return success
def process_sendgrid_response(): data = json.loads(request.data) try: for obj in data: notification_status = get_sendgrid_responses(obj["event"]) reference = obj['sg_message_id'].split(".")[0] notification = notifications_dao.dao_get_notification_by_reference(reference) notifications_dao._update_notification_status(notification=notification, status=notification_status) current_app.logger.info('SendGird callback return status of {} for notification: {}'.format( notification_status, notification.id )) statsd_client.incr('callback.sendgrid.{}'.format(notification_status)) if notification.sent_at: statsd_client.timing_with_dates( 'callback.sendgrid.elapsed-time', datetime.utcnow(), notification.sent_at) except Exception as e: current_app.logger.exception('Error processing SendGrid results: {}'.format(type(e))) raise InvalidRequest(message="Error processing SendGrid results", status_code=400) else: return jsonify(result='success'), 200
def send_sms_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == "created": provider = provider_to_use( SMS_TYPE, notification.id, notification.international, notification.reply_to_text, ) template_dict = dao_get_template_by_id( notification.template_id, notification.template_version).__dict__ template = SMSMessageTemplate( template_dict, values=notification.personalisation, prefix=service.name, show_prefix=service.prefix_sms, ) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = send_sms_response(provider.get_name(), notification.to) update_notification_to_sending(notification, provider) else: try: reference = provider.send_sms( to=validate_and_format_phone_number( notification.to, international=notification.international), content=str(template), reference=str(notification.id), sender=notification.reply_to_text, ) except Exception as e: notification.billable_units = template.fragment_count dao_update_notification(notification) dao_toggle_sms_provider(provider.name) raise e else: notification.reference = reference notification.billable_units = template.fragment_count update_notification_to_sending(notification, provider) # Record StatsD stats to compute SLOs statsd_client.timing_with_dates("sms.total-time", notification.sent_at, notification.created_at) statsd_key = f"sms.process_type-{template_dict['process_type']}" statsd_client.timing_with_dates(statsd_key, notification.sent_at, notification.created_at) statsd_client.incr(statsd_key)
def process_ses_results(response): try: ses_message = json.loads(response['Message']) notification_type = ses_message['notificationType'] if notification_type == 'Bounce': notification_type = determine_notification_bounce_type(notification_type, ses_message) elif notification_type == 'Complaint': _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) return True, False aws_response_dict = get_aws_responses(notification_type) notification_status = aws_response_dict['notification_status'] reference = ses_message['mail']['messageId'] try: notification = notifications_dao.dao_get_notification_by_reference(reference) except NoResultFound: message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): return None, True current_app.logger.warning( "notification not found for reference: {} (update to {})".format(reference, notification_status) ) return None, False if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}: notifications_dao._duplicate_update_warning(notification, notification_status) return None, False notifications_dao._update_notification_status(notification=notification, status=notification_status) if not aws_response_dict['success']: current_app.logger.info( "SES delivery failed: notification id {} and reference {} has error found. Status {}".format( notification.id, reference, aws_response_dict['message'] ) ) else: current_app.logger.info('SES callback returned status of {} for notification: {}'.format( notification_status, notification.id )) statsd_client.incr('callback.ses.{}'.format(notification_status)) if notification.sent_at: statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at) _check_and_queue_callback_task(notification) return True, False except Exception as e: current_app.logger.exception('Error processing SES results: {}'.format(type(e))) return None, True
def process_sms_client_response(status, reference, client_name): success = None errors = None # validate reference if reference == 'send-sms-code': success = "{} callback succeeded: send-sms-code".format(client_name) return success, errors try: uuid.UUID(reference, version=4) except ValueError: message = "{} callback with invalid reference {}".format(client_name, reference) return success, message try: response_parser = sms_response_mapper[client_name] except KeyError: return success, 'unknown sms client: {}'.format(client_name) # validate status try: response_dict = response_parser(status) current_app.logger.info('{} callback return status of {} for reference: {}'.format(client_name, status, reference)) except KeyError: msg = "{} callback failed: status {} not found.".format(client_name, status) return success, msg notification_status = response_dict['notification_status'] notification_status_message = response_dict['message'] notification_success = response_dict['success'] # record stats notification = notifications_dao.update_notification_status_by_id(reference, notification_status) if not notification: status_error = "{} callback failed: notification {} either not found or already updated " \ "from sending. Status {}".format(client_name, reference, notification_status_message) return success, status_error if not notification_success: current_app.logger.info( "{} delivery failed: notification {} has error found. Status {}".format(client_name, reference, notification_status_message)) statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status)) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name.lower()), datetime.utcnow(), notification.sent_at ) success = "{} callback succeeded. reference {} updated".format(client_name, reference) return success, errors
def process_pinpoint_results(self, response): if not is_feature_enabled(FeatureFlag.PINPOINT_RECEIPTS_ENABLED): current_app.logger.info( 'Pinpoint receipts toggle is disabled, skipping callback task') return True try: pinpoint_message = json.loads(base64.b64decode(response['Message'])) reference = pinpoint_message['attributes']['message_id'] event_type = pinpoint_message.get('event_type') record_status = pinpoint_message['attributes']['record_status'] current_app.logger.info( f'received callback from Pinpoint with event_type of {event_type} and record_status of {record_status}' f'with reference {reference}') notification_status = get_notification_status(event_type, record_status, reference) notification, should_retry, should_exit = attempt_to_get_notification( reference, notification_status, pinpoint_message['event_timestamp']) if should_retry: self.retry(queue=QueueNames.RETRY) if should_exit: return update_notification_status_by_id(notification.id, notification_status) current_app.logger.info( f"Pinpoint callback return status of {notification_status} for notification: {notification.id}" ) statsd_client.incr(f"callback.pinpoint.{notification_status}") if notification.sent_at: statsd_client.timing_with_dates('callback.pinpoint.elapsed-time', datetime.datetime.utcnow(), notification.sent_at) check_and_queue_callback_task(notification) return True except Retry: raise except Exception as e: current_app.logger.exception( f"Error processing Pinpoint results: {type(e)}") self.retry(queue=QueueNames.RETRY)
def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None): # record stats if client_name == 'Twilio': notification = notifications_dao.update_notification_status_by_reference( reference=provider_reference, status=notification_status) else: notification = notifications_dao.update_notification_status_by_id( notification_id=provider_reference, status=notification_status, sent_by=client_name.lower(), detailed_status_code=detailed_status_code) if not notification: return statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status)) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name.lower()), datetime.utcnow(), notification.sent_at) if notification.billable_units == 0: service = notification.service template_model = dao_get_template_by_id(notification.template_id, notification.template_version) template = SMSMessageTemplate( template_model.__dict__, values=notification.personalisation, prefix=service.name, show_prefix=service.prefix_sms, ) notification.billable_units = template.fragment_count notifications_dao.dao_update_notification(notification) if notification_status != NOTIFICATION_PENDING: service_callback_api = get_service_delivery_status_callback_api_for_service( service_id=notification.service_id) # queue callback task only if the service_callback_api exists if service_callback_api: encrypted_notification = create_delivery_status_callback_data( notification, service_callback_api) send_delivery_status_to_service.apply_async( [str(notification.id), encrypted_notification], queue=QueueNames.CALLBACKS)
def process_govdelivery_response(): try: data = validate(request.form, govdelivery_webhook_schema) sid = data['sid'] reference = data['message_url'].split("/")[-1] govdelivery_status = data['status'] notify_status = govdelivery_status_map[govdelivery_status] except ValidationError as e: raise e except Exception as e: raise InvalidRequest(f'Error processing Govdelivery callback: {e}', 400) else: try: notification = notifications_dao.dao_get_notification_by_reference( reference) except (MultipleResultsFound, NoResultFound) as e: exception_type = type(e).__name__ current_app.logger.exception( f'Govdelivery callback with sid {sid} for reference {reference} ' f'did not find exactly one notification: {exception_type}') statsd_client.incr( f'callback.govdelivery.failure.{exception_type}') else: current_app.logger.info( f'Govdelivery callback for notification {notification.id} has status {govdelivery_status},' f' which maps to notification-api status {notify_status}') if data.get('error_message'): current_app.logger.info( f"Govdelivery error_message for notification {notification.id}: " f"{data['error_message']}") notifications_dao._update_notification_status( notification, notify_status) statsd_client.incr(f'callback.govdelivery.{notify_status}') if notification.sent_at: statsd_client.timing_with_dates( 'callback.govdelivery.elapsed-time', datetime.utcnow(), notification.sent_at) if govdelivery_status == 'blacklisted': complaint = create_complaint(data, notification) publish_complaint(complaint, notification, notification.to) return jsonify(result='success'), 200
def process_job(job_id): start = datetime.utcnow() job = dao_get_job_by_id(job_id) if job.job_status != JOB_STATUS_PENDING: return service = job.service if not service.active: job.job_status = JOB_STATUS_CANCELLED dao_update_job(job) current_app.logger.warning( "Job {} has been cancelled, service {} is inactive".format( job_id, service.id)) return if __sending_limits_for_job_exceeded(service, job, job_id): return job.job_status = JOB_STATUS_IN_PROGRESS job.processing_started = start dao_update_job(job) # Record StatsD stats to compute SLOs job_start = job.scheduled_for or job.created_at statsd_client.timing_with_dates("job.processing-start-delay", job.processing_started, job_start) db_template = dao_get_template_by_id(job.template_id, job.template_version) TemplateClass = get_template_class(db_template.template_type) template = TemplateClass(db_template.__dict__) current_app.logger.debug( "Starting job {} processing {} notifications".format( job_id, job.notification_count)) csv = get_recipient_csv(job, template) for row in csv.get_rows(): process_row(row, template, job, service) job_complete(job, start=start)
def process_govdelivery_response(): try: data = request.form reference = data['message_url'].split("/")[-1] govdelivery_status = data['status'] notify_status = map_govdelivery_status_to_notify_status( govdelivery_status) except Exception as e: raise InvalidRequest( 'Error processing Govdelivery callback: {}'.format(e), 400) else: try: notification = notifications_dao.dao_get_notification_by_reference( reference) except (MultipleResultsFound, NoResultFound) as e: current_app.logger.exception( 'Govdelivery callback for reference {} did not find exactly one notification: {}' .format(reference, type(e))) pass else: current_app.logger.info( 'Govdelivery callback for notification {} has status "{}", which maps to notification-api status "{}"' .format(notification.id, govdelivery_status, notify_status)) notifications_dao._update_notification_status( notification, notify_status) statsd_client.incr('callback.govdelivery.{}'.format(notify_status)) if notification.sent_at: statsd_client.timing_with_dates( 'callback.govdelivery.elapsed-time', datetime.utcnow(), notification.sent_at) return jsonify(result='success'), 200
def _process_for_status(notification_status, client_name, provider_reference): # record stats notification = notifications_dao.update_notification_status_by_id( provider_reference, notification_status) if not notification: current_app.logger.warning( "{} callback failed: notification {} either not found or already updated " "from sending. Status {}".format(client_name, provider_reference, notification_status)) return statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status)) if not notification.sent_by: set_notification_sent_by(notification, client_name.lower()) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name.lower()), datetime.utcnow(), notification.sent_at) # queue callback task only if the service_callback_api exists service_callback_api = get_service_delivery_status_callback_api_for_service( service_id=notification.service_id) if service_callback_api: encrypted_notification = create_encrypted_callback_data( notification, service_callback_api) send_delivery_status_to_service.apply_async( [str(notification.id), encrypted_notification], queue=QueueNames.CALLBACKS) success = "{} callback succeeded. reference {} updated".format( client_name, provider_reference) return success
def process_ses_results(self, response): try: ses_message = json.loads(response["Message"]) notification_type = ses_message["notificationType"] if notification_type == "Complaint": _check_and_queue_complaint_callback_task( *handle_complaint(ses_message)) return True aws_response_dict = get_aws_responses(ses_message) notification_status = aws_response_dict["notification_status"] reference = ses_message["mail"]["messageId"] try: notification = notifications_dao.dao_get_notification_by_reference( reference) except NoResultFound: message_time = iso8601.parse_date( ses_message["mail"]["timestamp"]).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( "notification not found for reference: {} (update to {})". format(reference, notification_status)) return notifications_dao._update_notification_status( notification=notification, status=notification_status, provider_response=aws_response_dict["provider_response"], ) if not aws_response_dict["success"]: current_app.logger.info( "SES delivery failed: notification id {} and reference {} has error found. Status {}" .format(notification.id, reference, aws_response_dict["message"])) else: current_app.logger.info( "SES callback return status of {} for notification: {}".format( notification_status, notification.id)) statsd_client.incr("callback.ses.{}".format(notification_status)) if notification.sent_at: statsd_client.timing_with_dates("callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at) _check_and_queue_callback_task(notification) return True except Retry: raise except Exception as e: current_app.logger.exception("Error processing SES results: {}".format( type(e))) self.retry(queue=QueueNames.RETRY)
def process_ses_results(self, response): try: ses_message = json.loads(response['Message']) notification_type = ses_message['notificationType'] bounce_message = None if notification_type == 'Bounce': notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message) elif notification_type == 'Complaint': _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) return True aws_response_dict = get_aws_responses(notification_type) notification_status = aws_response_dict['notification_status'] reference = ses_message['mail']['messageId'] try: notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference) except NoResultFound: message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): current_app.logger.info( f"notification not found for reference: {reference} (update to {notification_status}). " f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue" ) self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( f"notification not found for reference: {reference} (update to {notification_status})" ) return if bounce_message: current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}") if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]: notifications_dao._duplicate_update_warning( notification=notification, status=notification_status ) return else: notifications_dao.dao_update_notifications_by_reference( references=[reference], update_dict={'status': notification_status} ) statsd_client.incr('callback.ses.{}'.format(notification_status)) if notification.sent_at: statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at) _check_and_queue_callback_task(notification) return True except Retry: raise except Exception as e: current_app.logger.exception('Error processing SES results: {}'.format(type(e))) self.retry(queue=QueueNames.RETRY)
def process_sns_results(self, response): try: # Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html sns_message = json.loads(response['Message']) reference = sns_message['notification']['messageId'] status = sns_message['status'] provider_response = sns_message['delivery']['providerResponse'] # See all the possible provider responses # https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html#sms_stats_delivery_fail_reasons reasons = { 'Blocked as spam by phone carrier': NOTIFICATION_TECHNICAL_FAILURE, 'Destination is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE, 'Invalid phone number': NOTIFICATION_TECHNICAL_FAILURE, 'Message body is invalid': NOTIFICATION_TECHNICAL_FAILURE, 'Phone carrier has blocked this message': NOTIFICATION_TECHNICAL_FAILURE, 'Phone carrier is currently unreachable/unavailable': NOTIFICATION_TEMPORARY_FAILURE, 'Phone has blocked SMS': NOTIFICATION_TECHNICAL_FAILURE, 'Phone is on a blocked list': NOTIFICATION_TECHNICAL_FAILURE, 'Phone is currently unreachable/unavailable': NOTIFICATION_PERMANENT_FAILURE, 'Phone number is opted out': NOTIFICATION_TECHNICAL_FAILURE, 'This delivery would exceed max price': NOTIFICATION_TECHNICAL_FAILURE, 'Unknown error attempting to reach phone': NOTIFICATION_TECHNICAL_FAILURE, } if status == "SUCCESS": notification_status = NOTIFICATION_DELIVERED else: if provider_response not in reasons: current_app.logger.warning( f"unhandled provider response for reference {reference}, received '{provider_response}'" ) notification_status = reasons.get(provider_response, NOTIFICATION_TECHNICAL_FAILURE) try: notification = notifications_dao.dao_get_notification_by_reference(reference) except NoResultFound: message_time = iso8601.parse_date(sns_message['notification']['timestamp']).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( f"notification not found for reference: {reference} (update to {notification_status})" ) return if notification.sent_by != SNS_PROVIDER: current_app.logger.exception(f'SNS callback handled notification {notification.id} not sent by SNS') return if notification.status != NOTIFICATION_SENT: notifications_dao._duplicate_update_warning(notification, notification_status) return notifications_dao._update_notification_status( notification=notification, status=notification_status ) if notification_status != NOTIFICATION_DELIVERED: current_app.logger.info(( f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. " f"Provider response: {sns_message['delivery']['providerResponse']}" )) else: current_app.logger.info( f'SNS callback return status of {notification_status} for notification: {notification.id}' ) statsd_client.incr(f'callback.sns.{notification_status}') if notification.sent_at: statsd_client.timing_with_dates('callback.sns.elapsed-time', datetime.utcnow(), notification.sent_at) _check_and_queue_callback_task(notification) return True except Retry: raise except Exception as e: current_app.logger.exception(f'Error processing SNS results: {str(e)}') self.retry(queue=QueueNames.RETRY)
def process_sns_results(self, response): try: # Payload details: https://docs.aws.amazon.com/sns/latest/dg/sms_stats_cloudwatch.html sns_message = json.loads(response["Message"]) reference = sns_message["notification"]["messageId"] sns_status = sns_message["status"] provider_response = sns_message["delivery"]["providerResponse"] try: notification_status = determine_status(sns_status, provider_response) except KeyError: current_app.logger.warning(f"unhandled provider response for reference {reference}, received '{provider_response}'") notification_status = NOTIFICATION_TECHNICAL_FAILURE provider_response = None try: notification = notifications_dao.dao_get_notification_by_reference(reference) except NoResultFound: message_time = iso8601.parse_date(sns_message["notification"]["timestamp"]).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning(f"notification not found for reference: {reference} (update to {notification_status})") return if notification.sent_by != SNS_PROVIDER: current_app.logger.exception(f"SNS callback handled notification {notification.id} not sent by SNS") return if notification.status != NOTIFICATION_SENT: notifications_dao._duplicate_update_warning(notification, notification_status) return notifications_dao._update_notification_status( notification=notification, status=notification_status, provider_response=provider_response if notification_status == NOTIFICATION_TECHNICAL_FAILURE else None, ) if notification_status != NOTIFICATION_DELIVERED: current_app.logger.info( ( f"SNS delivery failed: notification id {notification.id} and reference {reference} has error found. " f"Provider response: {sns_message['delivery']['providerResponse']}" ) ) else: current_app.logger.info(f"SNS callback return status of {notification_status} for notification: {notification.id}") statsd_client.incr(f"callback.sns.{notification_status}") if notification.sent_at: statsd_client.timing_with_dates("callback.sns.elapsed-time", datetime.utcnow(), notification.sent_at) _check_and_queue_callback_task(notification) return True except Retry: raise except Exception as e: current_app.logger.exception(f"Error processing SNS results: {str(e)}") self.retry(queue=QueueNames.RETRY)
def send_callback_metrics(notification): statsd_client.incr(f"callback.sns.{notification.status}") if notification.sent_at: statsd_client.timing_with_dates("callback.sns.elapsed-time", datetime.utcnow(), notification.sent_at)
def process_ses_response(ses_request): client_name = 'SES' try: errors = validate_callback_data(data=ses_request, fields=['Message'], client_name=client_name) if errors: return errors ses_message = json.loads(ses_request['Message']) errors = validate_callback_data(data=ses_message, fields=['notificationType'], client_name=client_name) if errors: return errors notification_type = ses_message['notificationType'] if notification_type == 'Bounce': notification_type = determine_notification_bounce_type( notification_type, ses_message) elif notification_type == 'Complaint': complaint, notification, recipient = handle_complaint(ses_message) _check_and_queue_complaint_callback_task(complaint, notification, recipient) return try: aws_response_dict = get_aws_responses(notification_type) except KeyError: error = "{} callback failed: status {} not found".format( client_name, notification_type) return error notification_status = aws_response_dict['notification_status'] try: reference = ses_message['mail']['messageId'] notification = notifications_dao.update_notification_status_by_reference( reference, notification_status) if not notification: warning = "SES callback failed: notification either not found or already updated " \ "from sending. Status {} for notification reference {}".format(notification_status, reference) current_app.logger.warning(warning) return if not aws_response_dict['success']: current_app.logger.info( "SES delivery failed: notification id {} and reference {} has error found. Status {}" .format(notification.id, reference, aws_response_dict['message'])) else: current_app.logger.info( '{} callback return status of {} for notification: {}'. format(client_name, notification_status, notification.id)) statsd_client.incr('callback.ses.{}'.format(notification_status)) if notification.sent_at: statsd_client.timing_with_dates( 'callback.ses.elapsed-time'.format(client_name.lower()), datetime.utcnow(), notification.sent_at) _check_and_queue_callback_task(notification) return except KeyError: error = "SES callback failed: messageId missing" return error except ValueError: error = "{} callback failed: invalid json".format(client_name) return error
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == "created": provider = provider_to_use(EMAIL_TYPE, notification.id) # Extract any file objects from the personalization file_keys = [ k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and "document" in v ] attachments = [] personalisation_data = notification.personalisation.copy() for key in file_keys: sending_method = personalisation_data[key]["document"].get( "sending_method") # Check if a MLWR sid exists if (current_app.config["MLWR_HOST"] and "mlwr_sid" in personalisation_data[key]["document"] and personalisation_data[key]["document"]["mlwr_sid"] != "false"): mlwr_result = check_mlwr( personalisation_data[key]["document"]["mlwr_sid"]) if "state" in mlwr_result and mlwr_result[ "state"] == "completed": # Update notification that it contains malware if "submission" in mlwr_result and mlwr_result[ "submission"]["max_score"] >= 500: malware_failure(notification=notification) return else: # Throw error so celery will retry in sixty seconds raise MalwarePendingException if sending_method == "attach": try: req = urllib.request.Request( personalisation_data[key]["document"] ["direct_file_url"]) with urllib.request.urlopen(req) as response: buffer = response.read() filename = personalisation_data[key]["document"].get( "filename") mime_type = personalisation_data[key]["document"].get( "mime_type") attachments.append({ "name": filename, "data": buffer, "mime_type": mime_type, }) except Exception: current_app.logger.error( "Could not download and attach {}".format( personalisation_data[key]["document"] ["direct_file_url"])) del personalisation_data[key] else: personalisation_data[key] = personalisation_data[key][ "document"]["url"] template_dict = dao_get_template_by_id( notification.template_id, notification.template_version).__dict__ # Local Jinja support - Add USE_LOCAL_JINJA_TEMPLATES=True to .env # Add a folder to the project root called 'jinja_templates' # with a copy of 'email_template.jinja2' from notification-utils repo debug_template_path = ( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if os.environ.get("USE_LOCAL_JINJA_TEMPLATES") == "True" else None) html_email = HTMLEmailTemplate( template_dict, values=personalisation_data, jinja_path=debug_template_path, **get_html_email_options(service), ) plain_text_email = PlainTextEmailTemplate(template_dict, values=personalisation_data) if current_app.config["SCAN_FOR_PII"]: contains_pii(notification, str(plain_text_email)) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = send_email_response(notification.to) update_notification_to_sending(notification, provider) else: if service.sending_domain is None or service.sending_domain.strip( ) == "": sending_domain = current_app.config["NOTIFY_EMAIL_DOMAIN"] else: sending_domain = service.sending_domain from_address = '"{}" <{}@{}>'.format(service.name, service.email_from, sending_domain) email_reply_to = notification.reply_to_text reference = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address( email_reply_to) if email_reply_to else None, attachments=attachments, ) notification.reference = reference update_notification_to_sending(notification, provider) # Record StatsD stats to compute SLOs statsd_client.timing_with_dates("email.total-time", notification.sent_at, notification.created_at) attachments_category = "with-attachments" if attachments else "no-attachments" statsd_key = f"email.{attachments_category}.process_type-{template_dict['process_type']}" statsd_client.timing_with_dates(statsd_key, notification.sent_at, notification.created_at) statsd_client.incr(statsd_key)