def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(EMAIL_TYPE, notification.id)
current_app.logger.debug(
"Starting sending EMAIL {} to provider at {}".format(notification.id, datetime.utcnow())
)
template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(
template_dict,
values=notification.personalisation,
**get_html_email_options(service)
)
plain_text_email = PlainTextEmailTemplate(
template_dict,
values=notification.personalisation
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
reference = str(create_uuid())
notification.billable_units = 0
notification.reference = reference
update_notification(notification, provider)
send_email_response(reference, notification.to)
else:
from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
current_app.config['NOTIFY_EMAIL_DOMAIN'])
email_reply_to = notification.reply_to_text
reference, status = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
)
notification.reference = reference
update_notification(notification, provider, status=status)
current_app.logger.debug("SENT_MAIL: {} -- {}".format(
validate_and_format_email_address(notification.to),
str(plain_text_email))
)
current_app.logger.debug(
"Email {} sent to provider at {}".format(notification.id, notification.sent_at)
)
delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000
statsd_client.timing("email.total-time", delta_milliseconds)
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(EMAIL_TYPE)
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(template_dict,
values=notification.personalisation,
**get_html_email_options(service))
plain_text_email = PlainTextEmailTemplate(
template_dict, values=notification.personalisation)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = str(create_uuid())
update_notification_to_sending(notification, provider)
send_email_response(notification.reference, notification.to)
else:
from_address = '"{}" <{}@{}>'.format(
service.name, service.email_from,
current_app.config['NOTIFY_EMAIL_DOMAIN'])
email_reply_to = notification.reply_to_text
reference = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(
email_reply_to) if email_reply_to else None,
)
notification.reference = reference
update_notification_to_sending(notification, provider)
delta_seconds = (datetime.utcnow() -
notification.created_at).total_seconds()
if notification.key_type == KEY_TYPE_TEST:
statsd_client.timing("email.test-key.total-time", delta_seconds)
else:
statsd_client.timing("email.live-key.total-time", delta_seconds)
if str(service.id) in current_app.config.get(
'HIGH_VOLUME_SERVICE'):
statsd_client.timing("email.live-key.high-volume.total-time",
delta_seconds)
else:
statsd_client.timing(
"email.live-key.not-high-volume.total-time", delta_seconds)
def validate_and_format_recipient(send_to,
key_type,
service,
notification_type,
allow_safelisted_recipients=True):
if send_to is None:
raise BadRequestError(message="Recipient can't be empty")
service_can_send_to_recipient(send_to, key_type, service,
allow_safelisted_recipients)
if notification_type == SMS_TYPE:
international_phone_info = get_international_phone_info(send_to)
if international_phone_info.international and INTERNATIONAL_SMS_TYPE not in [
p.permission for p in service.permissions
]:
raise BadRequestError(
message="Cannot send to international mobile numbers")
return validate_and_format_phone_number(
number=send_to,
international=international_phone_info.international)
elif notification_type == EMAIL_TYPE:
return validate_and_format_email_address(email_address=send_to)
def dao_get_notifications_by_recipient_or_reference(
service_id,
search_term,
notification_type=None,
statuses=None,
page=1,
page_size=None,
):
if notification_type == SMS_TYPE:
normalised = try_validate_and_format_phone_number(search_term)
for character in {'(', ')', ' ', '-'}:
normalised = normalised.replace(character, '')
normalised = normalised.lstrip('+0')
elif notification_type == EMAIL_TYPE:
try:
normalised = validate_and_format_email_address(search_term)
except InvalidEmailError:
normalised = search_term.lower()
elif notification_type in {LETTER_TYPE, None}:
# For letters, we store the address without spaces, so we need
# to removes spaces from the search term to match. We also do
# this when a notification type isn’t provided (this will
# happen if a user doesn’t have permission to see the dashboard)
# because email addresses and phone numbers will never be stored
# with spaces either.
normalised = ''.join(search_term.split()).lower()
else:
raise TypeError(
f'Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, {LETTER_TYPE} or None'
)
normalised = escape_special_characters(normalised)
search_term = escape_special_characters(search_term)
filters = [
Notification.service_id == service_id,
or_(
Notification.normalised_to.like("%{}%".format(normalised)),
Notification.client_reference.ilike("%{}%".format(search_term)),
),
Notification.key_type != KEY_TYPE_TEST,
]
if statuses:
filters.append(Notification.status.in_(statuses))
if notification_type:
filters.append(Notification.notification_type == notification_type)
results = db.session.query(Notification)\
.filter(*filters)\
.order_by(desc(Notification.created_at))\
.paginate(page=page, per_page=page_size)
return results
def validate_and_format_recipient(send_to, key_type, service, notification_type, allow_guest_list_recipients=True):
if send_to is None:
raise BadRequestError(message="Recipient can't be empty")
service_can_send_to_recipient(send_to, key_type, service, allow_guest_list_recipients)
if notification_type == SMS_TYPE:
international_phone_info = check_if_service_can_send_to_number(service, send_to)
return validate_and_format_phone_number(
number=send_to,
international=international_phone_info.international
)
elif notification_type == EMAIL_TYPE:
return validate_and_format_email_address(email_address=send_to)
def dao_get_notifications_by_to_field(service_id,
search_term,
notification_type=None,
statuses=None):
if notification_type is None:
notification_type = guess_notification_type(search_term)
if notification_type == SMS_TYPE:
normalised = try_validate_and_format_phone_number(search_term)
for character in {'(', ')', ' ', '-'}:
normalised = normalised.replace(character, '')
normalised = normalised.lstrip('+0')
elif notification_type == EMAIL_TYPE:
try:
normalised = validate_and_format_email_address(search_term)
except InvalidEmailError:
normalised = search_term.lower()
else:
raise InvalidRequest("Only email and SMS can use search by recipient",
400)
normalised = escape_special_characters(normalised)
filters = [
Notification.service_id == service_id,
Notification.normalised_to.like("%{}%".format(normalised)),
Notification.key_type != KEY_TYPE_TEST,
]
if statuses:
filters.append(Notification.status.in_(statuses))
if notification_type:
filters.append(Notification.notification_type == notification_type)
results = db.session.query(Notification).filter(*filters).order_by(
desc(Notification.created_at)).all()
return results
def test_simulated_recipient(notify_api, to_address, notification_type, expected):
"""
The values where the expected = 'research-mode' are listed in the config['SIMULATED_EMAIL_ADDRESSES']
and config['SIMULATED_SMS_NUMBERS']. These values should result in using the research mode queue.
SIMULATED_EMAIL_ADDRESSES = (
'*****@*****.**',
'*****@*****.**',
'*****@*****.**'
)
SIMULATED_SMS_NUMBERS = ('6132532222', '+16132532222', '+16132532223')
"""
formatted_address = None
if notification_type == 'email':
formatted_address = validate_and_format_email_address(to_address)
else:
formatted_address = validate_and_format_phone_number(to_address)
is_simulated_address = simulated_recipient(formatted_address, notification_type)
assert is_simulated_address == expected
def test_simulated_recipient(notify_api, to_address, notification_type,
expected):
"""
The values where the expected = 'research-mode' are listed in the config['SIMULATED_EMAIL_ADDRESSES']
and config['SIMULATED_SMS_NUMBERS']. These values should result in using the research mode queue.
SIMULATED_EMAIL_ADDRESSES = (
'*****@*****.**',
'*****@*****.**',
'*****@*****.**'
)
SIMULATED_SMS_NUMBERS = ('+61400900000', '+61400900111', '+61400900222', '+61426305772', '+61426305773', '+61426305774')
"""
formatted_address = None
if notification_type == 'email':
formatted_address = validate_and_format_email_address(to_address)
else:
formatted_address = validate_and_format_phone_number_and_allow_international(
to_address)
is_simulated_address = simulated_recipient(formatted_address,
notification_type)
assert is_simulated_address == expected
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
# TODO: no else - replace with if statement raising error / logging when not 'created'
if notification.status == 'created':
provider = provider_to_use(notification)
# TODO: remove that code or extract attachment handling to separate method
# Extract any file objects from the personalization
file_keys = [
k for k, v in (notification.personalisation or {}).items()
if isinstance(v, dict) and 'document' in v
]
attachments = []
personalisation_data = notification.personalisation.copy()
for key in file_keys:
# Check if a MLWR sid exists
if (current_app.config["MLWR_HOST"]
and 'mlwr_sid' in personalisation_data[key]['document']
and personalisation_data[key]['document']['mlwr_sid'] !=
"false"):
mlwr_result = check_mlwr(
personalisation_data[key]['document']['mlwr_sid'])
if "state" in mlwr_result and mlwr_result[
"state"] == "completed":
# Update notification that it contains malware
if "submission" in mlwr_result and mlwr_result[
"submission"]['max_score'] >= 500:
malware_failure(notification=notification)
return
else:
# Throw error so celery will retry in sixty seconds
raise MalwarePendingException
try:
response = requests.get(
personalisation_data[key]['document']['direct_file_url'])
if response.headers['Content-Type'] == 'application/pdf':
attachments.append({
"name": "{}.pdf".format(key),
"data": response.content
})
except Exception:
current_app.logger.error(
"Could not download and attach {}".format(
personalisation_data[key]['document']
['direct_file_url']))
personalisation_data[key] = personalisation_data[key]['document'][
'url']
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(template_dict,
values=personalisation_data,
**get_html_email_options(
notification, provider))
plain_text_email = PlainTextEmailTemplate(template_dict,
values=personalisation_data)
if current_app.config["SCAN_FOR_PII"]:
contains_pii(notification, str(plain_text_email))
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = str(create_uuid())
update_notification_to_sending(notification, provider)
send_email_response(notification.reference, notification.to)
else:
email_reply_to = notification.reply_to_text
reference = provider.send_email(
source=compute_source_email_address(service, provider),
to_addresses=validate_and_format_email_address(
notification.to),
subject=plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(
email_reply_to) if email_reply_to else None,
attachments=attachments)
notification.reference = reference
update_notification_to_sending(notification, provider)
current_app.logger.info(
f"Saved provider reference: {reference} for notification id: {notification.id}"
)
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("email.total-time", delta_milliseconds)
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(EMAIL_TYPE, notification.id)
# Extract any file objects from the personalization
file_keys = [
k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and 'document' in v
]
attachments = []
personalisation_data = notification.personalisation.copy()
for key in file_keys:
# Check if a MLWR sid exists
if (current_app.config["MLWR_HOST"] and
'mlwr_sid' in personalisation_data[key]['document'] and
personalisation_data[key]['document']['mlwr_sid'] != "false"):
mlwr_result = check_mlwr(personalisation_data[key]['document']['mlwr_sid'])
if "state" in mlwr_result and mlwr_result["state"] == "completed":
# Update notification that it contains malware
if "submission" in mlwr_result and mlwr_result["submission"]['max_score'] >= 500:
malware_failure(notification=notification)
return
else:
# Throw error so celery will retry in sixty seconds
raise MalwarePendingException
try:
req = urllib.request.Request(personalisation_data[key]['document']['direct_file_url'])
with urllib.request.urlopen(req) as response:
buffer = response.read()
mime_type = magic.from_buffer(buffer, mime=True)
if mime_type == 'application/pdf':
attachments.append({"name": "{}.pdf".format(key), "data": buffer})
except Exception:
current_app.logger.error(
"Could not download and attach {}".format(personalisation_data[key]['document']['direct_file_url'])
)
personalisation_data[key] = personalisation_data[key]['document']['url']
template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__
html_email = HTMLEmailTemplate(
template_dict,
values=personalisation_data,
**get_html_email_options(service)
)
plain_text_email = PlainTextEmailTemplate(
template_dict,
values=personalisation_data
)
if current_app.config["SCAN_FOR_PII"]:
contains_pii(notification, str(plain_text_email))
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = str(create_uuid())
update_notification_to_sending(notification, provider)
send_email_response(notification.reference, notification.to)
else:
if service.sending_domain is None or service.sending_domain.strip() == "":
sending_domain = current_app.config['NOTIFY_EMAIL_DOMAIN']
else:
sending_domain = service.sending_domain
from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
sending_domain)
email_reply_to = notification.reply_to_text
reference = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
attachments=attachments
)
notification.reference = reference
update_notification_to_sending(notification, provider)
delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000
statsd_client.timing("email.total-time", delta_milliseconds)
def send_email_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == "created":
provider = provider_to_use(EMAIL_TYPE, notification.id)
# Extract any file objects from the personalization
file_keys = [
k for k, v in (notification.personalisation or {}).items()
if isinstance(v, dict) and "document" in v
]
attachments = []
personalisation_data = notification.personalisation.copy()
for key in file_keys:
sending_method = personalisation_data[key]["document"].get(
"sending_method")
# Check if a MLWR sid exists
if (current_app.config["MLWR_HOST"]
and "mlwr_sid" in personalisation_data[key]["document"]
and personalisation_data[key]["document"]["mlwr_sid"] !=
"false"):
mlwr_result = check_mlwr(
personalisation_data[key]["document"]["mlwr_sid"])
if "state" in mlwr_result and mlwr_result[
"state"] == "completed":
# Update notification that it contains malware
if "submission" in mlwr_result and mlwr_result[
"submission"]["max_score"] >= 500:
malware_failure(notification=notification)
return
else:
# Throw error so celery will retry in sixty seconds
raise MalwarePendingException
if sending_method == "attach":
try:
req = urllib.request.Request(
personalisation_data[key]["document"]
["direct_file_url"])
with urllib.request.urlopen(req) as response:
buffer = response.read()
filename = personalisation_data[key]["document"].get(
"filename")
mime_type = personalisation_data[key]["document"].get(
"mime_type")
attachments.append({
"name": filename,
"data": buffer,
"mime_type": mime_type,
})
except Exception:
current_app.logger.error(
"Could not download and attach {}".format(
personalisation_data[key]["document"]
["direct_file_url"]))
del personalisation_data[key]
else:
personalisation_data[key] = personalisation_data[key][
"document"]["url"]
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
# Local Jinja support - Add USE_LOCAL_JINJA_TEMPLATES=True to .env
# Add a folder to the project root called 'jinja_templates'
# with a copy of 'email_template.jinja2' from notification-utils repo
debug_template_path = (
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if os.environ.get("USE_LOCAL_JINJA_TEMPLATES") == "True" else None)
html_email = HTMLEmailTemplate(
template_dict,
values=personalisation_data,
jinja_path=debug_template_path,
**get_html_email_options(service),
)
plain_text_email = PlainTextEmailTemplate(template_dict,
values=personalisation_data)
if current_app.config["SCAN_FOR_PII"]:
contains_pii(notification, str(plain_text_email))
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = send_email_response(notification.to)
update_notification_to_sending(notification, provider)
else:
if service.sending_domain is None or service.sending_domain.strip(
) == "":
sending_domain = current_app.config["NOTIFY_EMAIL_DOMAIN"]
else:
sending_domain = service.sending_domain
from_address = '"{}" <{}@{}>'.format(service.name,
service.email_from,
sending_domain)
email_reply_to = notification.reply_to_text
reference = provider.send_email(
from_address,
validate_and_format_email_address(notification.to),
plain_text_email.subject,
body=str(plain_text_email),
html_body=str(html_email),
reply_to_address=validate_and_format_email_address(
email_reply_to) if email_reply_to else None,
attachments=attachments,
)
notification.reference = reference
update_notification_to_sending(notification, provider)
# Record StatsD stats to compute SLOs
statsd_client.timing_with_dates("email.total-time",
notification.sent_at,
notification.created_at)
attachments_category = "with-attachments" if attachments else "no-attachments"
statsd_key = f"email.{attachments_category}.process_type-{template_dict['process_type']}"
statsd_client.timing_with_dates(statsd_key, notification.sent_at,
notification.created_at)
statsd_client.incr(statsd_key)
