def test_get_character_count_of_content(content, prefix, expected_length,
expected_replaced_length):
template = SMSMessageTemplate({'content': content}, )
template.prefix = prefix
template.sender = None
assert template.content_count == expected_length
template.values = {'placeholder': '123'}
assert template.content_count == expected_replaced_length
def _content_count_greater_than_limit(content, template_type):
if template_type != SMS_TYPE:
return False
template = SMSMessageTemplate({
'content': content,
'template_type': template_type
})
return template.is_message_too_long()
def fix_billable_units():
query = Notification.query.filter(
Notification.notification_type == SMS_TYPE,
Notification.status != NOTIFICATION_CREATED,
Notification.sent_at == None, # noqa
Notification.billable_units == 0,
Notification.key_type != KEY_TYPE_TEST,
)
for notification in query.all():
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=notification.service.name,
show_prefix=notification.service.prefix_sms,
)
print("Updating notification: {} with {} billable_units".format(
notification.id, template.fragment_count))
Notification.query.filter(Notification.id == notification.id).update(
{"billable_units": template.fragment_count})
db.session.commit()
print("End fix_billable_units")
def test_detects_rows_which_result_in_overly_long_messages():
template = SMSMessageTemplate(
{
'content': '((placeholder))',
'template_type': 'sms'
},
sender=None,
prefix=None,
)
recipients = RecipientCSV("""
phone number,placeholder
07700900460,1
07700900461,{one_under}
07700900462,{exactly}
07700900463,{one_over}
""".format(
one_under='a' * (SMS_CHAR_COUNT_LIMIT - 1),
exactly='a' * SMS_CHAR_COUNT_LIMIT,
one_over='a' * (SMS_CHAR_COUNT_LIMIT + 1),
),
template_type=template.template_type,
template=template)
assert _index_rows(recipients.rows_with_errors) == {3}
assert _index_rows(recipients.rows_with_message_too_long) == {3}
assert recipients.has_errors
def create_content_for_notification(template, personalisation):
if template.template_type == EMAIL_TYPE:
template_object = PlainTextEmailTemplate(
{
'content': template.content,
'subject': template.subject,
'template_type': template.template_type,
},
personalisation,
)
if template.template_type == SMS_TYPE:
template_object = SMSMessageTemplate(
{
'content': template.content,
'template_type': template.template_type,
},
personalisation,
)
if template.template_type == LETTER_TYPE:
template_object = LetterPrintTemplate(
{
'content': template.content,
'subject': template.subject,
'template_type': template.template_type,
},
personalisation,
contact_block=template.reply_to_text,
)
check_placeholders(template_object)
return template_object
def test_sms_fragment_count(char_count, expected_sms_fragment_count):
with patch(
'notifications_utils.template.SMSMessageTemplate.content_count',
new_callable=PropertyMock
) as mocked:
mocked.return_value = char_count
template = SMSMessageTemplate({'content': 'faked', 'template_type': 'sms'})
assert template.fragment_count == expected_sms_fragment_count
def _content_count_greater_than_limit(content, template_type):
if template_type != SMS_TYPE:
return False
template = SMSMessageTemplate({
'content': content,
'template_type': template_type
})
return template.content_count > SMS_CHAR_COUNT_LIMIT
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(SMS_TYPE, notification.international)
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
update_notification_to_sending(notification, provider)
send_sms_response(provider.get_name(), str(notification.id),
notification.to)
else:
try:
provider.send_sms(to=validate_and_format_phone_number(
notification.to, international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_reduce_sms_provider_priority(
provider.get_name(), time_threshold=timedelta(minutes=1))
raise e
else:
notification.billable_units = template.fragment_count
update_notification_to_sending(notification, provider)
delta_seconds = (datetime.utcnow() -
notification.created_at).total_seconds()
statsd_client.timing("sms.total-time", delta_seconds)
if notification.key_type == KEY_TYPE_TEST:
statsd_client.timing("sms.test-key.total-time", delta_seconds)
else:
statsd_client.timing("sms.live-key.total-time", delta_seconds)
if str(service.id) in current_app.config.get(
'HIGH_VOLUME_SERVICE'):
statsd_client.timing("sms.live-key.high-volume.total-time",
delta_seconds)
else:
statsd_client.timing("sms.live-key.not-high-volume.total-time",
delta_seconds)
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(SMS_TYPE, notification.id,
notification.international)
current_app.logger.debug(
"Starting sending SMS {} to provider at {}".format(
notification.id, datetime.utcnow()))
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.billable_units = 0
update_notification(notification, provider)
try:
send_sms_response(provider.get_name(), str(notification.id),
notification.to)
except HTTPError:
# when we retry, we only do anything if the notification is in created - it's currently in sending,
# so set it back so that we actually attempt the callback again
notification.sent_at = None
notification.sent_by = None
notification.status = NOTIFICATION_CREATED
dao_update_notification(notification)
raise
else:
try:
provider.send_sms(to=validate_and_format_phone_number(
notification.to, international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
except Exception as e:
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.billable_units = template.fragment_count
update_notification(notification, provider,
notification.international)
current_app.logger.debug("SMS {} sent to provider {} at {}".format(
notification.id, provider.get_name(), notification.sent_at))
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("sms.total-time", delta_milliseconds)
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == "created":
provider = provider_to_use(
SMS_TYPE,
notification.id,
notification.international,
notification.reply_to_text,
)
template_dict = dao_get_template_by_id(
notification.template_id, notification.template_version).__dict__
template = SMSMessageTemplate(
template_dict,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = send_sms_response(provider.get_name(),
notification.to)
update_notification_to_sending(notification, provider)
else:
try:
reference = provider.send_sms(
to=validate_and_format_phone_number(
notification.to,
international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text,
)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.reference = reference
notification.billable_units = template.fragment_count
update_notification_to_sending(notification, provider)
# Record StatsD stats to compute SLOs
statsd_client.timing_with_dates("sms.total-time", notification.sent_at,
notification.created_at)
statsd_key = f"sms.process_type-{template_dict['process_type']}"
statsd_client.timing_with_dates(statsd_key, notification.sent_at,
notification.created_at)
statsd_client.incr(statsd_key)
def test_sms_fragment_count_unicode_encoding(char_count,
expected_sms_fragment_count):
with patch('notifications_utils.template.SMSMessageTemplate.content_count',
new_callable=PropertyMock) as mocked:
mocked.return_value = char_count
template = SMSMessageTemplate({
'content': 'This is â mêssâgê with Ŵêlsh chârâctêrs',
'template_type': 'sms'
})
assert template.fragment_count == expected_sms_fragment_count
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
provider = provider_to_use(SMS_TYPE, notification,
notification.international)
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.reference = create_uuid()
update_notification_to_sending(notification, provider)
send_sms_response(provider.get_name(), str(notification.id),
notification.to, notification.reference)
else:
try:
reference = provider.send_sms(
to=validate_and_format_phone_number(
notification.to,
international=notification.international),
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
except Exception as e:
notification.billable_units = template.fragment_count
dao_update_notification(notification)
dao_toggle_sms_provider(provider.name)
raise e
else:
notification.billable_units = template.fragment_count
notification.reference = reference
update_notification_to_sending(notification, provider)
current_app.logger.info(
f"Saved provider reference: {reference} for notification id: {notification.id}"
)
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("sms.total-time", delta_milliseconds)
def _process_for_status(notification_status,
client_name,
provider_reference,
detailed_status_code=None):
# record stats
if client_name == 'Twilio':
notification = notifications_dao.update_notification_status_by_reference(
reference=provider_reference, status=notification_status)
else:
notification = notifications_dao.update_notification_status_by_id(
notification_id=provider_reference,
status=notification_status,
sent_by=client_name.lower(),
detailed_status_code=detailed_status_code)
if not notification:
return
statsd_client.incr('callback.{}.{}'.format(client_name.lower(),
notification_status))
if notification.sent_at:
statsd_client.timing_with_dates(
'callback.{}.elapsed-time'.format(client_name.lower()),
datetime.utcnow(), notification.sent_at)
if notification.billable_units == 0:
service = notification.service
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
notification.billable_units = template.fragment_count
notifications_dao.dao_update_notification(notification)
if notification_status != NOTIFICATION_PENDING:
service_callback_api = get_service_delivery_status_callback_api_for_service(
service_id=notification.service_id)
# queue callback task only if the service_callback_api exists
if service_callback_api:
encrypted_notification = create_delivery_status_callback_data(
notification, service_callback_api)
send_delivery_status_to_service.apply_async(
[str(notification.id), encrypted_notification],
queue=QueueNames.CALLBACKS)
def test_detects_rows_which_result_in_empty_messages():
template = SMSMessageTemplate(
{
'content': '((show??content))',
'template_type': 'sms'
},
sender=None,
prefix=None,
)
recipients = RecipientCSV("""
phone number,show
07700900460,yes
07700900462,no
07700900463,yes
""",
template=template)
assert _index_rows(recipients.rows_with_errors) == {1}
assert _index_rows(recipients.rows_with_empty_message) == {1}
assert recipients.has_errors
assert recipients[0].has_error_spanning_multiple_cells is False
assert recipients[1].has_error_spanning_multiple_cells is True
assert recipients[2].has_error_spanning_multiple_cells is False
def test_detects_rows_which_result_in_overly_long_messages():
template = SMSMessageTemplate(
{
'content': '((placeholder))',
'template_type': 'sms'
},
sender=None,
prefix=None,
)
recipients = RecipientCSV("""
phone number,placeholder
07700900460,1
07700900461,1234567890
07700900462,12345678901
07700900463,123456789012345678901234567890
""",
template_type=template.template_type,
template=template,
sms_character_limit=10)
assert recipients.rows_with_errors == {2, 3}
assert recipients.rows_with_message_too_long == {2, 3}
assert recipients.has_errors
def _sample_template(template_type, content='foo'):
return {
'email':
EmailPreviewTemplate({
'content': content,
'subject': 'bar',
'template_type': 'email'
}),
'sms':
SMSMessageTemplate({
'content': content,
'template_type': 'sms'
}),
'letter':
LetterImageTemplate(
{
'content': content,
'subject': 'bar',
'template_type': 'letter'
},
image_url='https://example.com',
page_count=1,
),
}.get(template_type)
def test_sms_message_adds_prefix(prefix, body, expected):
template = SMSMessageTemplate({'content': body})
template.prefix = prefix
template.sender = None
assert str(template) == expected
def send_sms_to_provider(notification):
service = notification.service
if not service.active:
technical_failure(notification=notification)
return
if notification.status == 'created':
# TODO: issue is that this does not get the provider based on who owns
# the inbound number. The notification.reply_to_text below is the phone
# number that we should send from, but we need to look at that and see
# who the provider is.
# TODO: now that we do get the right provider, the issue is that the
# reply to text could be different because the service is able to choose
# the sender when sending a message. So we need to check if the sender
# ID that was chosen is also an inbound number.
provider = None
preferred_provider = get_preferred_sms_provider(service)
if preferred_provider:
provider = get_sms_provider_client(preferred_provider,
notification.id)
else:
provider = provider_to_use(SMS_TYPE, notification.id,
notification.international)
current_app.logger.debug(
"Starting sending SMS {} to provider at {}".format(
notification.id, datetime.utcnow()))
template_model = dao_get_template_by_id(notification.template_id,
notification.template_version)
template = SMSMessageTemplate(
template_model.__dict__,
values=notification.personalisation,
prefix=service.name,
show_prefix=service.prefix_sms,
)
if service.research_mode or notification.key_type == KEY_TYPE_TEST:
notification.billable_units = 0
update_notification(notification, provider)
try:
send_sms_response(provider.get_name(), str(notification.id),
notification.to)
except HTTPError:
# when we retry, we only do anything if the notification is in created - it's currently in sending,
# so set it back so that we actually attempt the callback again
notification.sent_at = None
notification.sent_by = None
notification.status = NOTIFICATION_CREATED
dao_update_notification(notification)
raise
else:
status = None
try:
reference, status = provider.send_sms(
to=notification.normalised_to,
content=str(template),
reference=str(notification.id),
sender=notification.reply_to_text)
notification.reference = reference
notification.billable_units = template.fragment_count
# An international notification (i.e. a text message with an
# abroad recipient phone number) instantly get marked as "sent".
# It might later get marked as "delivered" when the provider
# status callback is triggered.
if notification.international:
status = NOTIFICATION_SENT
except Exception as e:
dao_toggle_sms_provider(provider.name)
raise e
else:
update_notification(notification, provider, status=status)
current_app.logger.debug("SMS {} sent to provider {} at {}".format(
notification.id, provider.get_name(), notification.sent_at))
delta_milliseconds = (datetime.utcnow() -
notification.created_at).total_seconds() * 1000
statsd_client.timing("sms.total-time", delta_milliseconds)
def test_basic_templates_return_markup():
template_dict = {'content': 'content', 'subject': 'subject'}
for output in [
str(Template(template_dict)),
str(WithSubjectTemplate(template_dict)),
WithSubjectTemplate(template_dict).subject,
]:
assert isinstance(output, Markup)
@pytest.mark.parametrize('template_instance, expected_placeholders', [
(
SMSMessageTemplate(
{"content": "((content))", "subject": "((subject))"},
),
['content'],
),
(
SMSPreviewTemplate(
{"content": "((content))", "subject": "((subject))"},
),
['content'],
),
(
PlainTextEmailTemplate(
{"content": "((content))", "subject": "((subject))"},
),
['content', 'subject'],
),
