def test_send_sms_should_use_template_version_from_notification_not_latest(
sample_template, mocker):
db_notification = create_notification(
template=sample_template,
to_field='+447234123123',
status='created',
reply_to_text=sample_template.service.get_default_sms_sender())
mocker.patch('app.mmg_client.send_sms')
version_on_notification = sample_template.version
# Change the template
from app.dao.templates_dao import dao_update_template, dao_get_template_by_id
sample_template.content = sample_template.content + " another version of the template"
dao_update_template(sample_template)
t = dao_get_template_by_id(sample_template.id)
assert t.version > version_on_notification
send_to_providers.send_sms_to_provider(db_notification)
mmg_client.send_sms.assert_called_once_with(
to=validate_and_format_phone_number("+447234123123"),
content="Sample service: This is a template:\nwith a newline",
reference=str(db_notification.id),
sender=current_app.config['FROM_NUMBER'])
persisted_notification = notifications_dao.get_notification_by_id(
db_notification.id)
assert persisted_notification.to == db_notification.to
assert persisted_notification.template_id == sample_template.id
assert persisted_notification.template_version == version_on_notification
assert persisted_notification.template_version != sample_template.version
assert persisted_notification.status == 'sending'
assert not persisted_notification.personalisation
def validate_and_format_recipient(send_to,
key_type,
service,
notification_type,
allow_safelisted_recipients=True):
if send_to is None:
raise BadRequestError(message="Recipient can't be empty")
service_can_send_to_recipient(send_to, key_type, service,
allow_safelisted_recipients)
if notification_type == SMS_TYPE:
international_phone_info = get_international_phone_info(send_to)
if international_phone_info.international and INTERNATIONAL_SMS_TYPE not in [
p.permission for p in service.permissions
]:
raise BadRequestError(
message="Cannot send to international mobile numbers")
return validate_and_format_phone_number(
number=send_to,
international=international_phone_info.international)
elif notification_type == EMAIL_TYPE:
return validate_and_format_email_address(email_address=send_to)
def test_should_send_personalised_template_to_correct_sms_provider_and_persist(
sample_sms_template_with_html, mocker):
db_notification = create_notification(
template=sample_sms_template_with_html,
to_field="+447234123123",
personalisation={"name": "Jo"},
status='created',
reply_to_text=sample_sms_template_with_html.service.
get_default_sms_sender())
mocker.patch('app.mmg_client.send_sms')
send_to_providers.send_sms_to_provider(db_notification)
mmg_client.send_sms.assert_called_once_with(
to=validate_and_format_phone_number("+447234123123"),
content=
"Sample service: Hello Jo\nHere is <em>some HTML</em> & entities",
reference=str(db_notification.id),
sender=current_app.config['FROM_NUMBER'])
notification = Notification.query.filter_by(id=db_notification.id).one()
assert notification.status == 'sending'
assert notification.sent_at <= datetime.utcnow()
assert notification.sent_by == 'mmg'
assert notification.billable_units == 1
assert notification.personalisation == {"name": "Jo"}
def test_should_send_personalised_template_to_correct_sms_provider_and_persist(sample_sms_template_with_html, mocker):
db_notification = create_notification(
template=sample_sms_template_with_html,
to_field="+16502532222",
personalisation={"name": "Jo"},
status="created",
reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender(),
)
statsd_mock = mocker.patch("app.delivery.send_to_providers.statsd_client")
mocker.patch("app.aws_sns_client.send_sms", return_value="message_id_from_sns")
send_to_providers.send_sms_to_provider(db_notification)
aws_sns_client.send_sms.assert_called_once_with(
to=validate_and_format_phone_number("+16502532222"),
content="Sample service: Hello Jo\nHere is <em>some HTML</em> & entities",
reference=str(db_notification.id),
sender=current_app.config["FROM_NUMBER"],
)
notification = Notification.query.filter_by(id=db_notification.id).one()
assert notification.status == "sent"
assert notification.sent_at <= datetime.utcnow()
assert notification.sent_by == "sns"
assert notification.billable_units == 1
assert notification.personalisation == {"name": "Jo"}
assert notification.reference == "message_id_from_sns"
statsd_timing_calls = statsd_mock.timing_with_dates.call_args_list
assert call("sms.total-time", notification.sent_at, notification.created_at) in statsd_timing_calls
assert call("sms.process_type-normal", notification.sent_at, notification.created_at) in statsd_timing_calls
assert call("sms.process_type-normal") in statsd_mock.incr.call_args_list
def send_sms_to_provider(notification):
service = dao_fetch_service_by_id(notification.service_id)
provider = provider_to_use(SMS_TYPE, notification.id)
if notification.status == 'created':
template_model = dao_get_template_by_id(notification.template_id, notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
sender=service.sms_sender
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
send_sms_response.apply_async(
(provider.get_name(), str(notification.id), notification.to), queue='research-mode'
)
notification.billable_units = 0
else:
provider.send_sms(
to=validate_and_format_phone_number(notification.to),
content=str(template),
reference=str(notification.id),
sender=service.sms_sender
)
notification.billable_units = template.fragment_count
notification.sent_at = datetime.utcnow()
notification.sent_by = provider.get_name()
notification.status = 'sending'
dao_update_notification(notification)
current_app.logger.info(
"SMS {} sent to provider at {}".format(notification.id, notification.sent_at)
)
delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000
statsd_client.timing("sms.total-time", delta_milliseconds)
def simulated_recipient(to_address, notification_type):
if notification_type == SMS_TYPE:
formatted_simulated_numbers = [
validate_and_format_phone_number(number) for number in current_app.config["SIMULATED_SMS_NUMBERS"]
]
return to_address in formatted_simulated_numbers
else:
return to_address in current_app.config["SIMULATED_EMAIL_ADDRESSES"]
def get_example_csv_rows(template, use_example_as_example=True, submitted_fields=False):
return [
{
'email': '*****@*****.**' if use_example_as_example else current_user.email_address,
'sms': '07700 900321' if use_example_as_example else validate_and_format_phone_number(
current_user.mobile_number, human_readable=True
)
}[template.template_type]
] + get_example_csv_fields(template.placeholders, use_example_as_example, submitted_fields)
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(SMS_TYPE, notification.international)
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
update_notification_to_sending(notification, provider)
send_sms_response(provider.get_name(), str(notification.id),
notification.to)
else:
try:
provider.send_sms(to=validate_and_format_phone_number(
notification.to, international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_reduce_sms_provider_priority(
provider.get_name(), time_threshold=timedelta(minutes=1))
raise e
else:
notification.billable_units = template.fragment_count
update_notification_to_sending(notification, provider)
delta_seconds = (datetime.utcnow() -
notification.created_at).total_seconds()
statsd_client.timing("sms.total-time", delta_seconds)
if notification.key_type == KEY_TYPE_TEST:
statsd_client.timing("sms.test-key.total-time", delta_seconds)
else:
statsd_client.timing("sms.live-key.total-time", delta_seconds)
if str(service.id) in current_app.config.get(
'HIGH_VOLUME_SERVICE'):
statsd_client.timing("sms.live-key.high-volume.total-time",
delta_seconds)
else:
statsd_client.timing("sms.live-key.not-high-volume.total-time",
delta_seconds)
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(SMS_TYPE, notification.id,
notification.international)
current_app.logger.debug(
"Starting sending SMS {} to provider at {}".format(
notification.id, datetime.utcnow()))
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.billable_units = 0
update_notification(notification, provider)
try:
send_sms_response(provider.get_name(), str(notification.id),
notification.to)
except HTTPError:
# when we retry, we only do anything if the notification is in created - it's currently in sending,
# so set it back so that we actually attempt the callback again
notification.sent_at = None
notification.sent_by = None
notification.status = NOTIFICATION_CREATED
dao_update_notification(notification)
raise
else:
try:
provider.send_sms(to=validate_and_format_phone_number(
notification.to, international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
except Exception as e:
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.billable_units = template.fragment_count
update_notification(notification, provider,
notification.international)
current_app.logger.debug("SMS {} sent to provider {} at {}".format(
notification.id, provider.get_name(), notification.sent_at))
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("sms.total-time", delta_milliseconds)
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == "created":
provider = provider_to_use(
SMS_TYPE,
notification.id,
notification.international,
notification.reply_to_text,
)
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
template = SMSMessageTemplate(
template_dict,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = send_sms_response(provider.get_name(),
notification.to)
update_notification_to_sending(notification, provider)
else:
try:
reference = provider.send_sms(
to=validate_and_format_phone_number(
notification.to,
international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text,
)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.reference = reference
notification.billable_units = template.fragment_count
update_notification_to_sending(notification, provider)
# Record StatsD stats to compute SLOs
statsd_client.timing_with_dates("sms.total-time", notification.sent_at,
notification.created_at)
statsd_key = f"sms.process_type-{template_dict['process_type']}"
statsd_client.timing_with_dates(statsd_key, notification.sent_at,
notification.created_at)
statsd_client.incr(statsd_key)
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(SMS_TYPE, notification,
notification.international)
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = create_uuid()
update_notification_to_sending(notification, provider)
send_sms_response(provider.get_name(), str(notification.id),
notification.to, notification.reference)
else:
try:
reference = provider.send_sms(
to=validate_and_format_phone_number(
notification.to,
international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.billable_units = template.fragment_count
notification.reference = reference
update_notification_to_sending(notification, provider)
current_app.logger.info(
f"Saved provider reference: {reference} for notification id: {notification.id}"
)
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("sms.total-time", delta_milliseconds)
def get_example_csv_rows(template,
use_example_as_example=True,
submitted_fields=False):
return [{
'email':
'*****@*****.**'
if use_example_as_example else current_user.email_address,
'sms':
'07700 900321' if use_example_as_example else
validate_and_format_phone_number(current_user.mobile_number,
human_readable=True)
}[template.template_type]] + get_example_csv_fields(
template.placeholders, use_example_as_example, submitted_fields)
def validate_and_format_recipient(send_to, key_type, service, notification_type, allow_guest_list_recipients=True):
if send_to is None:
raise BadRequestError(message="Recipient can't be empty")
service_can_send_to_recipient(send_to, key_type, service, allow_guest_list_recipients)
if notification_type == SMS_TYPE:
international_phone_info = check_if_service_can_send_to_number(service, send_to)
return validate_and_format_phone_number(
number=send_to,
international=international_phone_info.international
)
elif notification_type == EMAIL_TYPE:
return validate_and_format_email_address(email_address=send_to)
def test_simulated_recipient(notify_api, to_address, notification_type, expected):
"""
The values where the expected = 'research-mode' are listed in the config['SIMULATED_EMAIL_ADDRESSES']
and config['SIMULATED_SMS_NUMBERS']. These values should result in using the research mode queue.
SIMULATED_EMAIL_ADDRESSES = (
'*****@*****.**',
'*****@*****.**',
'*****@*****.**'
)
SIMULATED_SMS_NUMBERS = ('6132532222', '+16132532222', '+16132532223')
"""
formatted_address = None
if notification_type == 'email':
formatted_address = validate_and_format_email_address(to_address)
else:
formatted_address = validate_and_format_phone_number(to_address)
is_simulated_address = simulated_recipient(formatted_address, notification_type)
assert is_simulated_address == expected
def test_send_sms_should_use_template_version_from_notification_not_latest(sample_template, mocker):
db_notification = create_notification(
template=sample_template,
to_field="+16502532222",
status="created",
reply_to_text=sample_template.service.get_default_sms_sender(),
)
mocker.patch("app.aws_sns_client.send_sms", return_value="message_id_from_sns")
version_on_notification = sample_template.version
# Change the template
from app.dao.templates_dao import dao_get_template_by_id, dao_update_template
sample_template.content = sample_template.content + " another version of the template"
dao_update_template(sample_template)
t = dao_get_template_by_id(sample_template.id)
assert t.version > version_on_notification
send_to_providers.send_sms_to_provider(db_notification)
aws_sns_client.send_sms.assert_called_once_with(
to=validate_and_format_phone_number("+16502532222"),
content="Sample service: This is a template:\nwith a newline",
reference=str(db_notification.id),
sender=current_app.config["FROM_NUMBER"],
)
persisted_notification = notifications_dao.get_notification_by_id(db_notification.id)
assert persisted_notification.to == db_notification.to
assert persisted_notification.template_id == sample_template.id
assert persisted_notification.template_version == version_on_notification
assert persisted_notification.template_version != sample_template.version
assert persisted_notification.status == "sent"
assert persisted_notification.reference == "message_id_from_sns"
assert not persisted_notification.personalisation
def test_valid_international_phone_number_can_be_formatted_consistently(phone_number, expected_formatted):
assert validate_and_format_phone_number(
phone_number, international=True
) == expected_formatted
def test_valid_uk_phone_number_can_be_formatted_consistently(phone_number):
assert validate_and_format_phone_number(phone_number) == '447123456789'
def format_phone_number(self, item):
item['to'] = validate_and_format_phone_number(item['to'])
return item
def test_valid_phone_number_can_be_formatted_consistently(phone_number):
assert format_phone_number(validate_phone_number(phone_number)) == '+447123456789'
assert validate_and_format_phone_number(phone_number) == '+447123456789'
assert validate_and_format_phone_number(phone_number, human_readable=True) == '07123 456 789'
def format_phone_number(self, item):
item['to'] = validate_and_format_phone_number(item['to'],
international=True)
return item
def persist_notification(*,
template_id,
template_version,
recipient=None,
service,
personalisation,
notification_type,
api_key_id,
key_type,
created_at=None,
job_id=None,
job_row_number=None,
reference=None,
client_reference=None,
notification_id=None,
simulated=False,
created_by_id=None,
status=NOTIFICATION_CREATED,
reply_to_text=None,
billable_units=None,
postage=None,
template_postage=None,
recipient_identifier=None):
notification_created_at = created_at or datetime.utcnow()
if not notification_id:
notification_id = uuid.uuid4()
notification = Notification(id=notification_id,
template_id=template_id,
template_version=template_version,
to=recipient,
service_id=service.id,
service=service,
personalisation=personalisation,
notification_type=notification_type,
api_key_id=api_key_id,
key_type=key_type,
created_at=notification_created_at,
job_id=job_id,
job_row_number=job_row_number,
client_reference=client_reference,
reference=reference,
created_by_id=created_by_id,
status=status,
reply_to_text=reply_to_text,
billable_units=billable_units)
if accept_recipient_identifiers_enabled() and recipient_identifier:
_recipient_identifier = RecipientIdentifier(
notification_id=notification_id,
id_type=recipient_identifier['id_type'],
id_value=recipient_identifier['id_value'])
notification.recipient_identifiers.set(_recipient_identifier)
if notification_type == SMS_TYPE and notification.to:
formatted_recipient = validate_and_format_phone_number(
recipient, international=True)
recipient_info = get_international_phone_info(formatted_recipient)
notification.normalised_to = formatted_recipient
notification.international = recipient_info.international
notification.phone_prefix = recipient_info.country_prefix
notification.rate_multiplier = recipient_info.billable_units
elif notification_type == EMAIL_TYPE and notification.to:
notification.normalised_to = format_email_address(notification.to)
elif notification_type == LETTER_TYPE:
notification.postage = postage or template_postage
# if simulated create a Notification model to return but do not persist the Notification to the dB
if not simulated:
dao_create_notification(notification)
if key_type != KEY_TYPE_TEST:
if redis_store.get(redis.daily_limit_cache_key(service.id)):
redis_store.incr(redis.daily_limit_cache_key(service.id))
current_app.logger.info("{} {} created at {}".format(
notification_type, notification_id, notification_created_at))
return notification
def persist_notification(*,
template_id,
template_version,
recipient,
service,
personalisation,
notification_type,
api_key_id,
key_type,
created_at=None,
job_id=None,
job_row_number=None,
reference=None,
client_reference=None,
notification_id=None,
simulated=False,
created_by_id=None,
status=NOTIFICATION_CREATED,
reply_to_text=None,
billable_units=None,
postage=None,
document_download_count=None,
updated_at=None):
notification_created_at = created_at or datetime.utcnow()
if not notification_id:
notification_id = uuid.uuid4()
notification = Notification(
id=notification_id,
template_id=template_id,
template_version=template_version,
to=recipient,
service_id=service.id,
personalisation=personalisation,
notification_type=notification_type,
api_key_id=api_key_id,
key_type=key_type,
created_at=notification_created_at,
job_id=job_id,
job_row_number=job_row_number,
client_reference=client_reference,
reference=reference,
created_by_id=created_by_id,
status=status,
reply_to_text=reply_to_text,
billable_units=billable_units,
document_download_count=document_download_count,
updated_at=updated_at)
if notification_type == SMS_TYPE:
formatted_recipient = validate_and_format_phone_number(
recipient, international=True)
recipient_info = get_international_phone_info(formatted_recipient)
notification.normalised_to = formatted_recipient
notification.international = recipient_info.international
notification.phone_prefix = recipient_info.country_prefix
notification.rate_multiplier = recipient_info.billable_units
elif notification_type == EMAIL_TYPE:
notification.normalised_to = format_email_address(notification.to)
elif notification_type == LETTER_TYPE:
notification.postage = postage
notification.international = postage in INTERNATIONAL_POSTAGE_TYPES
notification.normalised_to = ''.join(notification.to.split()).lower()
# if simulated create a Notification model to return but do not persist the Notification to the dB
if not simulated:
dao_create_notification(notification)
# Only keep track of the daily limit for trial mode services.
if service.restricted and key_type != KEY_TYPE_TEST:
if redis_store.get(redis.daily_limit_cache_key(service.id)):
redis_store.incr(redis.daily_limit_cache_key(service.id))
current_app.logger.info("{} {} created at {}".format(
notification_type, notification_id, notification_created_at))
return notification
def test_valid_local_phone_number_can_be_formatted_consistently(phone_number):
assert validate_and_format_phone_number(phone_number) == '+16502532222'
