def test_should_send_template_to_correct_sms_task_and_persist(sample_template_with_placeholders, mocker):
notification = _notification_json(sample_template_with_placeholders,
to="+447234123123", personalisation={"name": "Jo"})
mocked_deliver_sms = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
send_sms(
sample_template_with_placeholders.service_id,
uuid.uuid4(),
encryption.encrypt(notification),
datetime.utcnow().strftime(DATETIME_FORMAT)
)
persisted_notification = Notification.query.one()
assert persisted_notification.to == '+447234123123'
assert persisted_notification.template_id == sample_template_with_placeholders.id
assert persisted_notification.template_version == sample_template_with_placeholders.version
assert persisted_notification.status == 'created'
assert persisted_notification.created_at <= datetime.utcnow()
assert not persisted_notification.sent_at
assert not persisted_notification.sent_by
assert not persisted_notification.job_id
assert persisted_notification.personalisation == {'name': 'Jo'}
assert persisted_notification._personalisation == encryption.encrypt({"name": "Jo"})
assert persisted_notification.notification_type == 'sms'
mocked_deliver_sms.assert_called_once_with(
[str(persisted_notification.id)],
queue="send-sms"
)
def _ensure_admin(self):
"""
For development use only. Creates a default admin user
if one does not yet exist.
"""
with Session(self._engine, future=True) as session:
accounts = session.query(User).all()
if len(accounts):
log.info("Found an existing account.")
return
log.info("Creating a default admin account.")
# cur_user = session.query(User).filter(User.username == 'admin').one()
# session.delete(cur_user)
# session.commit()
user = User(
id=str(uuid4()),
username="******",
password=encrypt("admin"),
f_name="tha",
l_name="admin",
email="*****@*****.**",
type="admin",
confirmed=True,
deactivated=False,
)
session.add(user)
session.commit()
def update_user(
self, token: str, user_details: dict,
credentials: Optional[dict[str,
str]]) -> Tuple[Token, UserDetails]:
"""Update a user's data"""
if not credentials:
credentials = {}
user_details = {**user_details} # don't mutate original object
user_id, token = validate_token(token)
if "password" in user_details or "username" in user_details:
# Must login again to change access credentials
token = self.login( # will raise an error if login fails
username=credentials["username"],
password=credentials["password"])
if "password" in user_details: # encrypt new password
password = user_details["password"]
user_details["password"] = encrypt(password)
with Session(self._engine, future=True) as session:
user = self._get_user_by_id(user_id, session)
for key, val in user_details.items():
if key in PROTECTED_FIELDS:
if not user.is_admin:
raise UnauthorizedUserError
setattr(user, key, val)
session.add(user)
session.commit()
return str(token), user.to_dict()
def test_process_sanitised_letter_retries_if_there_is_an_exception(
mocker,
sample_letter_notification,
):
mocker.patch('app.celery.letters_pdf_tasks.update_letter_pdf_status',
side_effect=Exception())
mock_celery_retry = mocker.patch(
'app.celery.letters_pdf_tasks.process_sanitised_letter.retry')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
encrypted_data = encryption.encrypt({
'page_count':
2,
'message':
None,
'invalid_pages':
None,
'validation_status':
'passed',
'filename':
'NOTIFY.{}'.format(sample_letter_notification.reference),
'notification_id':
str(sample_letter_notification.id),
'address':
None
})
process_sanitised_letter(encrypted_data)
mock_celery_retry.assert_called_once_with(queue='retry-tasks')
def save_email_to_queue(*,
notification_id,
form,
notification_type,
api_key,
template,
service_id,
personalisation,
document_download_count,
reply_to_text=None):
data = {
"id": notification_id,
"template_id": str(template.id),
"template_version": template.version,
"to": form['email_address'],
"service_id": str(service_id),
"personalisation": personalisation,
"notification_type": notification_type,
"api_key_id": str(api_key.id),
"key_type": api_key.key_type,
"client_reference": form.get('reference', None),
"reply_to_text": reply_to_text,
"document_download_count": document_download_count,
"status": NOTIFICATION_CREATED,
"created_at": datetime.utcnow().strftime(DATETIME_FORMAT),
}
encrypted = encryption.encrypt(data)
save_api_email.apply_async([encrypted], queue=QueueNames.SAVE_API_EMAIL)
return Notification(**data)
def test_should_put_send_email_task_in_research_mode_queue_if_research_mode_service(
notify_db, notify_db_session, mocker
):
service = sample_service(notify_db, notify_db_session)
service.research_mode = True
services_dao.dao_update_service(service)
template = sample_email_template(notify_db, notify_db_session, service=service)
notification = _notification_json(template, to="*****@*****.**")
mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
notification_id = uuid.uuid4()
send_email(
template.service_id,
notification_id,
encryption.encrypt(notification),
datetime.utcnow().strftime(DATETIME_FORMAT)
)
persisted_notification = Notification.query.one()
provider_tasks.deliver_email.apply_async.assert_called_once_with(
[str(persisted_notification.id)],
queue="research-mode"
)
def test_process_sanitised_letter_puts_letter_into_tech_failure_for_boto_errors(
sample_letter_notification,
mocker,
):
mocker.patch('app.celery.letters_pdf_tasks.s3.get_s3_object',
side_effect=ClientError({}, 'operation_name'))
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
encrypted_data = encryption.encrypt({
'page_count':
2,
'message':
None,
'invalid_pages':
None,
'validation_status':
'passed',
'filename':
'NOTIFY.{}'.format(sample_letter_notification.reference),
'notification_id':
str(sample_letter_notification.id),
'address':
None
})
with pytest.raises(NotificationTechnicalFailureException):
process_sanitised_letter(encrypted_data)
assert sample_letter_notification.status == NOTIFICATION_TECHNICAL_FAILURE
def test_get_pdf_for_templated_letter_happy_path(mocker, sample_letter_notification, branding_name, logo_filename):
if branding_name:
letter_branding = create_letter_branding(name=branding_name, filename=logo_filename)
sample_letter_notification.service.letter_branding = letter_branding
mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task')
mocker.patch('app.celery.letters_pdf_tasks.get_letter_pdf_filename', return_value='LETTER.PDF')
get_pdf_for_templated_letter(sample_letter_notification.id)
letter_data = {
'letter_contact_block': sample_letter_notification.reply_to_text,
'template': {
"subject": sample_letter_notification.template.subject,
"content": sample_letter_notification.template.content,
"template_type": sample_letter_notification.template.template_type
},
'values': sample_letter_notification.personalisation,
'logo_filename': logo_filename,
'letter_filename': 'LETTER.PDF',
"notification_id": str(sample_letter_notification.id),
'key_type': sample_letter_notification.key_type
}
encrypted_data = encryption.encrypt(letter_data)
mock_celery.assert_called_once_with(
name=TaskNames.CREATE_PDF_FOR_TEMPLATED_LETTER,
args=(encrypted_data,),
queue=QueueNames.SANITISE_LETTERS
)
def test_should_use_email_template_and_persist_without_personalisation(sample_email_template, mocker):
notification = _notification_json(sample_email_template, "my_email@my_email.com")
mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
notification_id = uuid.uuid4()
now = datetime.utcnow()
send_email(
sample_email_template.service_id,
notification_id,
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT)
)
persisted_notification = Notification.query.one()
assert persisted_notification.to == 'my_email@my_email.com'
assert persisted_notification.template_id == sample_email_template.id
assert persisted_notification.created_at == now
assert not persisted_notification.sent_at
assert persisted_notification.status == 'created'
assert not persisted_notification.sent_by
assert not persisted_notification.personalisation
assert not persisted_notification.reference
assert persisted_notification.notification_type == 'email'
provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)],
queue='send-email')
def test_process_sanitised_letter_when_letter_status_is_not_pending_virus_scan(
sample_letter_notification,
mocker,
):
mock_s3 = mocker.patch('app.celery.letters_pdf_tasks.s3')
sample_letter_notification.status = NOTIFICATION_CREATED
encrypted_data = encryption.encrypt({
'page_count':
2,
'message':
None,
'invalid_pages':
None,
'validation_status':
'passed',
'filename':
'NOTIFY.{}'.format(sample_letter_notification.reference),
'notification_id':
str(sample_letter_notification.id),
'address':
None
})
process_sanitised_letter(encrypted_data)
assert not mock_s3.called
def test_process_sanitised_letter_puts_letter_into_technical_failure_if_max_retries_exceeded(
mocker,
sample_letter_notification,
):
mocker.patch('app.celery.letters_pdf_tasks.update_letter_pdf_status',
side_effect=Exception())
mocker.patch('app.celery.letters_pdf_tasks.process_sanitised_letter.retry',
side_effect=MaxRetriesExceededError())
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
encrypted_data = encryption.encrypt({
'page_count':
2,
'message':
None,
'invalid_pages':
None,
'validation_status':
'passed',
'filename':
'NOTIFY.{}'.format(sample_letter_notification.reference),
'notification_id':
str(sample_letter_notification.id),
'address':
None
})
with pytest.raises(NotificationTechnicalFailureException):
process_sanitised_letter(encrypted_data)
assert sample_letter_notification.status == NOTIFICATION_TECHNICAL_FAILURE
def process_row(row, template, job, service):
template_type = template.template_type
encrypted = encryption.encrypt({
'template': str(template.id),
'template_version': job.template_version,
'job': str(job.id),
'to': row.recipient,
'row_number': row.index,
'personalisation': dict(row.personalisation)
})
send_fns = {
SMS_TYPE: save_sms,
EMAIL_TYPE: save_email,
LETTER_TYPE: save_letter
}
send_fn = send_fns[template_type]
send_fn.apply_async(
(
str(service.id),
create_uuid(),
encrypted,
),
queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE
)
def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notify_db_session, mocker):
user = sample_user(notify_db, notify_db_session, mobile_numnber="07700 900890")
service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = sample_template(notify_db, notify_db_session, service=service)
notification = _notification_json(template, "+447700900890") # The user’s own number, but in a different format
mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
notification_id = uuid.uuid4()
encrypt_notification = encryption.encrypt(notification)
send_sms(
service.id,
notification_id,
encrypt_notification,
datetime.utcnow().strftime(DATETIME_FORMAT)
)
persisted_notification = Notification.query.one()
assert persisted_notification.to == '+447700900890'
assert persisted_notification.template_id == template.id
assert persisted_notification.template_version == template.version
assert persisted_notification.status == 'created'
assert persisted_notification.created_at <= datetime.utcnow()
assert not persisted_notification.sent_at
assert not persisted_notification.sent_by
assert not persisted_notification.job_id
assert not persisted_notification.personalisation
assert persisted_notification.notification_type == 'sms'
provider_tasks.deliver_sms.apply_async.assert_called_once_with(
[str(persisted_notification.id)],
queue="send-sms"
)
def test_send_email_should_use_template_version_from_job_not_latest(sample_email_template, mocker):
notification = _notification_json(sample_email_template, 'my_email@my_email.com')
version_on_notification = sample_email_template.version
# Change the template
from app.dao.templates_dao import dao_update_template, dao_get_template_by_id
sample_email_template.content = sample_email_template.content + " another version of the template"
mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
dao_update_template(sample_email_template)
t = dao_get_template_by_id(sample_email_template.id)
assert t.version > version_on_notification
now = datetime.utcnow()
send_email(
sample_email_template.service_id,
uuid.uuid4(),
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT)
)
persisted_notification = Notification.query.one()
assert persisted_notification.to == 'my_email@my_email.com'
assert persisted_notification.template_id == sample_email_template.id
assert persisted_notification.template_version == version_on_notification
assert persisted_notification.created_at == now
assert not persisted_notification.sent_at
assert persisted_notification.status == 'created'
assert not persisted_notification.sent_by
assert persisted_notification.notification_type == 'email'
provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)],
queue='send-email')
def test_process_sanitised_letter_with_valid_letter(
sample_letter_notification,
key_type,
destination_bucket,
expected_status,
postage,
destination_filename,
):
# We save the letter as if it's 2nd class initially, and the task changes the filename to have the correct postage
filename = 'NOTIFY.FOO.D.2.C.C.20180701120000.PDF'
scan_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
template_preview_bucket_name = current_app.config['LETTER_SANITISE_BUCKET_NAME']
destination_bucket_name = current_app.config[destination_bucket]
conn = boto3.resource('s3', region_name='eu-west-1')
scan_bucket = conn.create_bucket(
Bucket=scan_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'}
)
template_preview_bucket = conn.create_bucket(
Bucket=template_preview_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'}
)
destination_bucket = conn.create_bucket(
Bucket=destination_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'}
)
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=scan_bucket_name, Key=filename, Body=b'original_pdf_content')
s3.put_object(Bucket=template_preview_bucket_name, Key=filename, Body=b'sanitised_pdf_content')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
sample_letter_notification.billable_units = 1
sample_letter_notification.created_at = datetime(2018, 7, 1, 12)
sample_letter_notification.postage = postage
encrypted_data = encryption.encrypt({
'page_count': 2,
'message': None,
'invalid_pages': None,
'validation_status': 'passed',
'filename': filename,
'notification_id': str(sample_letter_notification.id),
'address': 'A. User\nThe house on the corner'
})
process_sanitised_letter(encrypted_data)
assert sample_letter_notification.status == expected_status
assert sample_letter_notification.billable_units == 1
assert sample_letter_notification.to == 'A. User\nThe house on the corner'
assert not [x for x in scan_bucket.objects.all()]
assert not [x for x in template_preview_bucket.objects.all()]
assert len([x for x in destination_bucket.objects.all()]) == 1
file_contents = conn.Object(destination_bucket_name, destination_filename).get()['Body'].read().decode('utf-8')
assert file_contents == 'sanitised_pdf_content'
def create_delivery_status_callback_data(notification, service_callback_api):
data = {
"notification_id":
str(notification.id),
"notification_client_reference":
notification.client_reference,
"notification_to":
notification.to,
"notification_status":
notification.status,
"notification_created_at":
notification.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at":
notification.updated_at.strftime(DATETIME_FORMAT)
if notification.updated_at else None,
"notification_sent_at":
notification.sent_at.strftime(DATETIME_FORMAT)
if notification.sent_at else None,
"notification_type":
notification.notification_type,
"service_callback_api_url":
service_callback_api.url,
"service_callback_api_bearer_token":
service_callback_api.bearer_token,
}
return encryption.encrypt(data)
def test_should_send_email_if_restricted_service_and_non_team_email_address_with_test_key(notify_db,
notify_db_session,
mocker):
user = sample_user(notify_db, notify_db_session)
service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = sample_template(
notify_db, notify_db_session, service=service, template_type='email', subject_line='Hello'
)
notification = _notification_json(template, to="*****@*****.**")
mocked_deliver_email = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
notification_id = uuid.uuid4()
send_email(
service.id,
notification_id,
encryption.encrypt(notification),
datetime.utcnow().strftime(DATETIME_FORMAT),
key_type=KEY_TYPE_TEST
)
persisted_notification = Notification.query.one()
mocked_deliver_email.assert_called_once_with(
[str(persisted_notification.id)],
queue="send-email"
)
def test_should_send_sms_template_to_and_persist_with_job_id(sample_job, sample_api_key, mocker):
notification = _notification_json(
sample_job.template,
to="+447234123123",
job_id=sample_job.id,
row_number=2)
mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
notification_id = uuid.uuid4()
now = datetime.utcnow()
send_sms(
sample_job.service.id,
notification_id,
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT),
api_key_id=str(sample_api_key.id),
key_type=KEY_TYPE_NORMAL
)
persisted_notification = Notification.query.one()
assert persisted_notification.to == '+447234123123'
assert persisted_notification.job_id == sample_job.id
assert persisted_notification.template_id == sample_job.template.id
assert persisted_notification.status == 'created'
assert not persisted_notification.sent_at
assert persisted_notification.created_at <= now
assert not persisted_notification.sent_by
assert persisted_notification.job_row_number == 2
assert persisted_notification.api_key_id == sample_api_key.id
assert persisted_notification.key_type == KEY_TYPE_NORMAL
assert persisted_notification.notification_type == 'sms'
provider_tasks.deliver_sms.apply_async.assert_called_once_with(
[str(persisted_notification.id)],
queue="send-sms"
)
def test_should_not_send_email_if_team_key_and_recipient_not_in_team(sample_email_template_with_placeholders,
sample_team_api_key,
mocker):
notification = _notification_json(
sample_email_template_with_placeholders,
"my_email@my_email.com",
{"name": "Jo"},
row_number=1)
apply_async = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
notification_id = uuid.uuid4()
team_members = [user.email_address for user in sample_email_template_with_placeholders.service.users]
assert "my_email@my_email.com" not in team_members
with freeze_time("2016-01-01 11:09:00.00000"):
now = datetime.utcnow()
send_email(
sample_email_template_with_placeholders.service_id,
notification_id,
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT),
api_key_id=str(sample_team_api_key.id),
key_type=KEY_TYPE_TEAM
)
assert Notification.query.count() == 0
apply_async.not_called()
def _set_up_data_for_status_update(callback_api, notification):
data = {
"notification_id":
str(notification.id),
"notification_client_reference":
notification.client_reference,
"notification_to":
notification.to,
"notification_status":
notification.status,
"notification_created_at":
notification.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at":
notification.updated_at.strftime(DATETIME_FORMAT)
if notification.updated_at else None,
"notification_sent_at":
notification.sent_at.strftime(DATETIME_FORMAT)
if notification.sent_at else None,
"notification_type":
notification.notification_type,
"service_callback_api_url":
callback_api.url,
"service_callback_api_bearer_token":
callback_api.bearer_token,
}
encrypted_status_update = encryption.encrypt(data)
return encrypted_status_update
def add_new_user(login, password):
hashed, salt = enc.encrypt(password)
cursor = connection.cursor(buffered=True)
cursor.execute(
"INSERT INTO users(username, passwd, salt) VALUES (%s, %s, %s)",
(login, hashed, salt))
connection.commit()
cursor.close()
def test_process_sanitised_letter_with_invalid_letter(
sample_letter_notification, key_type):
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
scan_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
template_preview_bucket_name = current_app.config[
'LETTER_SANITISE_BUCKET_NAME']
invalid_letter_bucket_name = current_app.config['INVALID_PDF_BUCKET_NAME']
conn = boto3.resource('s3', region_name='eu-west-1')
scan_bucket = conn.create_bucket(
Bucket=scan_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
template_preview_bucket = conn.create_bucket(
Bucket=template_preview_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
invalid_letter_bucket = conn.create_bucket(
Bucket=invalid_letter_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=scan_bucket_name,
Key=filename,
Body=b'original_pdf_content')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
sample_letter_notification.billable_units = 1
sample_letter_notification.created_at = datetime(2018, 7, 1, 12)
encrypted_data = encryption.encrypt({
'page_count':
2,
'message':
'content-outside-printable-area',
'invalid_pages': [1],
'validation_status':
'failed',
'filename':
filename,
'notification_id':
str(sample_letter_notification.id),
'address':
None,
})
process_sanitised_letter(encrypted_data)
assert sample_letter_notification.status == NOTIFICATION_VALIDATION_FAILED
assert sample_letter_notification.billable_units == 0
assert not [x for x in scan_bucket.objects.all()]
assert not [x for x in template_preview_bucket.objects.all()]
assert len([x for x in invalid_letter_bucket.objects.all()]) == 1
file_contents = conn.Object(invalid_letter_bucket_name,
filename).get()['Body'].read().decode('utf-8')
assert file_contents == 'original_pdf_content'
def test_process_sanitised_letter_sets_postage_international(
sample_letter_notification, expected_postage, expected_international,
address):
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
scan_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
template_preview_bucket_name = current_app.config[
'LETTER_SANITISE_BUCKET_NAME']
destination_bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
conn = boto3.resource('s3', region_name='eu-west-1')
conn.create_bucket(
Bucket=scan_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
conn.create_bucket(
Bucket=template_preview_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
conn.create_bucket(
Bucket=destination_bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=scan_bucket_name,
Key=filename,
Body=b'original_pdf_content')
s3.put_object(Bucket=template_preview_bucket_name,
Key=filename,
Body=b'sanitised_pdf_content')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.billable_units = 1
sample_letter_notification.created_at = datetime(2018, 7, 1, 12)
encrypted_data = encryption.encrypt({
'page_count':
2,
'message':
None,
'invalid_pages':
None,
'validation_status':
'passed',
'filename':
filename,
'notification_id':
str(sample_letter_notification.id),
'address':
address
})
process_sanitised_letter(encrypted_data)
assert sample_letter_notification.status == 'created'
assert sample_letter_notification.billable_units == 1
assert sample_letter_notification.to == address
assert sample_letter_notification.postage == expected_postage
assert sample_letter_notification.international == expected_international
def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
data = {
"complaint_id": str(complaint.id),
"notification_id": str(notification.id),
"reference": notification.client_reference,
"to": recipient,
"complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT),
"service_callback_api_url": service_callback_api.url,
"service_callback_api_bearer_token": service_callback_api.bearer_token,
}
return encryption.encrypt(data)
def loaded_db(db, user_details, unconfirmed_user):
"""An active database instance with two users -- one admin, one contrib"""
encrypted_user_details = {
**user_details,
"password": encrypt(user_details["password"]),
}
with Session(db._engine, future=True) as session:
session.add(User(**encrypted_user_details))
session.add(User(**unconfirmed_user))
session.commit()
return db
def db(bare_db, admin_user_details):
"""An active database instance with one admin user"""
# must start with an admin user
encrypted_admin_details = {
**admin_user_details,
"password": encrypt(admin_user_details["password"]),
}
with Session(bare_db._engine, future=True) as session:
session.add(User(**encrypted_admin_details))
session.commit()
return bare_db
def process_job(job_id):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != "pending":
return
service = job.service
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = "in progress"
dao_update_job(job)
template = Template(dao_get_template_by_id(job.template_id, job.template_version).__dict__)
for row_number, recipient, personalisation in RecipientCSV(
s3.get_job_from_s3(str(service.id), str(job_id)),
template_type=template.template_type,
placeholders=template.placeholders,
).enumerated_recipients_and_personalisation:
encrypted = encryption.encrypt(
{
"template": str(template.id),
"template_version": job.template_version,
"job": str(job.id),
"to": recipient,
"row_number": row_number,
"personalisation": dict(personalisation),
}
)
if template.template_type == SMS_TYPE:
send_sms.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-sms" if not service.research_mode else "research-mode",
)
if template.template_type == EMAIL_TYPE:
send_email.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-email" if not service.research_mode else "research-mode",
)
finished = datetime.utcnow()
job.job_status = "finished"
job.processing_started = start
job.processing_finished = finished
dao_update_job(job)
current_app.logger.info(
"Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
)
def test_should_use_email_template_and_persist(sample_email_template_with_placeholders, sample_api_key, mocker):
with freeze_time("2016-01-01 12:00:00.000000"):
notification = _notification_json(
sample_email_template_with_placeholders,
'my_email@my_email.com',
{"name": "Jo"},
row_number=1)
mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
notification_id = uuid.uuid4()
with freeze_time("2016-01-01 11:09:00.00000"):
now = datetime.utcnow()
with freeze_time("2016-01-01 11:10:00.00000"):
send_email(
sample_email_template_with_placeholders.service_id,
notification_id,
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT),
api_key_id=str(sample_api_key.id),
key_type=sample_api_key.key_type
)
persisted_notification = Notification.query.one()
assert persisted_notification.to == 'my_email@my_email.com'
assert persisted_notification.template_id == sample_email_template_with_placeholders.id
assert persisted_notification.template_version == sample_email_template_with_placeholders.version
assert persisted_notification.created_at == now
assert not persisted_notification.sent_at
assert persisted_notification.status == 'created'
assert not persisted_notification.sent_by
assert persisted_notification.job_row_number == 1
assert persisted_notification.personalisation == {'name': 'Jo'}
assert persisted_notification._personalisation == encryption.encrypt({"name": "Jo"})
assert persisted_notification.api_key_id == sample_api_key.id
assert persisted_notification.key_type == KEY_TYPE_NORMAL
assert persisted_notification.notification_type == 'email'
provider_tasks.deliver_email.apply_async.assert_called_once_with(
[str(persisted_notification.id)], queue='send-email')
def _set_up_data_for_complaint(callback_api, complaint, notification):
data = {
"complaint_id": str(complaint.id),
"notification_id": str(notification.id),
"reference": notification.client_reference,
"to": notification.to,
"complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT),
"service_callback_api_url": callback_api.url,
"service_callback_api_bearer_token": callback_api.bearer_token,
}
obscured_status_update = encryption.encrypt(data)
return obscured_status_update
def get_pdf_for_templated_letter(self, notification_id):
try:
notification = get_notification_by_id(notification_id, _raise=True)
letter_filename = get_letter_pdf_filename(
reference=notification.reference,
crown=notification.service.crown,
sending_date=notification.created_at,
dont_use_sending_date=notification.key_type == KEY_TYPE_TEST,
postage=notification.postage)
letter_data = {
'letter_contact_block':
notification.reply_to_text,
'template': {
"subject": notification.template.subject,
"content": notification.template.content,
"template_type": notification.template.template_type
},
'values':
notification.personalisation,
'logo_filename':
notification.service.letter_branding
and notification.service.letter_branding.filename,
'letter_filename':
letter_filename,
"notification_id":
str(notification_id),
'key_type':
notification.key_type
}
encrypted_data = encryption.encrypt(letter_data)
notify_celery.send_task(name=TaskNames.CREATE_PDF_FOR_TEMPLATED_LETTER,
args=(encrypted_data, ),
queue=QueueNames.SANITISE_LETTERS)
except Exception:
try:
current_app.logger.exception(
f"RETRY: calling create-letter-pdf task for notification {notification_id} failed"
)
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = f"RETRY FAILED: Max retries reached. " \
f"The task create-letter-pdf failed for notification id {notification_id}. " \
f"Notification has been updated to technical-failure"
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def replay_service_callbacks(file_name, service_id):
print("Start send service callbacks for service: ", service_id)
callback_api = get_service_delivery_status_callback_api_for_service(
service_id=service_id)
if not callback_api:
print("Callback api was not found for service: {}".format(service_id))
return
errors = []
notifications = []
file = open(file_name)
for ref in file:
try:
notification = Notification.query.filter_by(
client_reference=ref.strip()).one()
notifications.append(notification)
except NoResultFound:
errors.append(
"Reference: {} was not found in notifications.".format(ref))
for e in errors:
print(e)
if errors:
raise Exception(
"Some notifications for the given references were not found")
for n in notifications:
data = {
"notification_id": str(n.id),
"notification_client_reference": n.client_reference,
"notification_to": n.to,
"notification_status": n.status,
"notification_created_at": n.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT),
"notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT),
"notification_type": n.notification_type,
"service_callback_api_url": callback_api.url,
"service_callback_api_bearer_token": callback_api.bearer_token,
}
encrypted_status_update = encryption.encrypt(data)
send_delivery_status_to_service.apply_async(
[str(n.id), encrypted_status_update], queue=QueueNames.CALLBACKS)
print(
"Replay service status for service: {}. Sent {} notification status updates to the queue"
.format(service_id, len(notifications)))
def test_create_invited_user(notify_api, sample_service, mocker, invitation_email_template):
with notify_api.test_request_context():
with notify_api.test_client() as client:
mocker.patch('uuid.uuid4', return_value='some_uuid') # for the notification id
mocker.patch('app.celery.tasks.send_email.apply_async')
mocker.patch('notifications_utils.url_safe_token.generate_token', return_value='the-token')
email_address = '*****@*****.**'
invite_from = sample_service.users[0]
data = {
'service': str(sample_service.id),
'email_address': email_address,
'from_user': str(invite_from.id),
'permissions': 'send_messages,manage_service,manage_api_keys'
}
auth_header = create_authorization_header()
response = client.post(
'/service/{}/invite'.format(sample_service.id),
headers=[('Content-Type', 'application/json'), auth_header],
data=json.dumps(data)
)
assert response.status_code == 201
json_resp = json.loads(response.get_data(as_text=True))
assert json_resp['data']['service'] == str(sample_service.id)
assert json_resp['data']['email_address'] == email_address
assert json_resp['data']['from_user'] == str(invite_from.id)
assert json_resp['data']['permissions'] == 'send_messages,manage_service,manage_api_keys'
assert json_resp['data']['id']
message = {
'template': str(invitation_email_template.id),
'template_version': invitation_email_template.version,
'to': email_address,
'personalisation': {
'user_name': invite_from.name,
'service_name': sample_service.name,
'url': '{0}/invitation/{1}'.format(current_app.config['ADMIN_BASE_URL'], 'the-token')
}
}
app.celery.tasks.send_email.apply_async.assert_called_once_with(
(str(current_app.config['NOTIFY_SERVICE_ID']),
'some_uuid',
encryption.encrypt(message),
"2016-01-01T11:09:00.061258Z"),
queue="notify")
def test_send_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample_notification, mocker):
json = _notification_json(sample_notification.template, sample_notification.to, job_id=uuid.uuid4(), row_number=1)
deliver_sms = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
retry = mocker.patch('app.celery.tasks.send_sms.retry', side_effect=Exception())
now = datetime.utcnow()
notification_id = sample_notification.id
send_sms(
sample_notification.service_id,
notification_id,
encryption.encrypt(json),
now.strftime(DATETIME_FORMAT)
)
assert Notification.query.count() == 1
assert not deliver_sms.called
assert not retry.called
def test_should_not_send_sms_if_restricted_service_and_invalid_number(notify_db, notify_db_session, mocker):
user = sample_user(notify_db, notify_db_session, mobile_numnber="07700 900205")
service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = sample_template(notify_db, notify_db_session, service=service)
notification = _notification_json(template, "07700 900849")
mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
notification_id = uuid.uuid4()
send_sms(
service.id,
notification_id,
encryption.encrypt(notification),
datetime.utcnow().strftime(DATETIME_FORMAT)
)
assert provider_tasks.deliver_sms.apply_async.called is False
assert Notification.query.count() == 0
def test_should_not_send_email_if_restricted_service_and_invalid_email_address(notify_db, notify_db_session, mocker):
user = sample_user(notify_db, notify_db_session)
service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = sample_template(
notify_db, notify_db_session, service=service, template_type='email', subject_line='Hello'
)
notification = _notification_json(template, to="*****@*****.**")
notification_id = uuid.uuid4()
send_email(
service.id,
notification_id,
encryption.encrypt(notification),
datetime.utcnow().strftime(DATETIME_FORMAT)
)
assert Notification.query.count() == 0
def process_row(row: Row, template: Template, job: Job, service: Service):
template_type = template.template_type
encrypted = encryption.encrypt({
"api_key":
job.api_key_id and str(job.api_key_id),
"template":
str(template.id),
"template_version":
job.template_version,
"job":
str(job.id),
"to":
row.recipient,
"row_number":
row.index,
"personalisation":
dict(row.personalisation),
"queue":
queue_to_use(job.notification_count),
})
sender_id = str(job.sender_id) if job.sender_id else None
send_fns = {
SMS_TYPE: save_sms,
EMAIL_TYPE: save_email,
LETTER_TYPE: save_letter
}
send_fn = send_fns[template_type]
task_kwargs = {}
if sender_id:
task_kwargs["sender_id"] = sender_id
send_fn.apply_async(
(
str(service.id),
create_uuid(),
encrypted,
),
task_kwargs,
queue=QueueNames.DATABASE
if not service.research_mode else QueueNames.RESEARCH_MODE,
)
def send_user_reset_password():
email, errors = email_data_request_schema.load(request.get_json())
user_to_send_to = get_user_by_email(email['email'])
template = dao_get_template_by_id(current_app.config['PASSWORD_RESET_TEMPLATE_ID'])
message = {
'template': str(template.id),
'template_version': template.version,
'to': user_to_send_to.email_address,
'personalisation': {
'user_name': user_to_send_to.name,
'url': _create_reset_password_url(user_to_send_to.email_address)
}
}
send_email.apply_async([current_app.config['NOTIFY_SERVICE_ID'],
str(uuid.uuid4()),
encryption.encrypt(message),
datetime.utcnow().strftime(DATETIME_FORMAT)], queue='notify')
return jsonify({}), 204
def add_user(self, token: Token, user_details: Dict) -> UserDetails:
"""Adds a user to the database"""
user_id, token = validate_token(token)
if not self.check_if_username_is_unique(user_details["username"]):
raise DuplicateUsernameError
with Session(self._engine, future=True) as session:
self._require_admin_user(
user_id=user_id,
session=session,
)
for field in user_details.keys():
if field in PROTECTED_FIELDS:
raise UnauthorizedUserError
# don't mutate original dict
user_details = {
**user_details,
"id": str(uuid4()),
"type": "contrib",
"confirmed": False,
"deactivated": False,
}
# handle password
password = user_details["password"]
user_details["password"] = encrypt(password)
# create user object
new_user = User(**user_details)
session.add(new_user)
session.commit()
# TODO: when email service is enabled, add call here to send a token to
# the provided email address.
new_user_token = get_token(new_user.id)
log.info(f"New User Token is: {new_user_token}")
return token, new_user.to_dict()
def send_already_registered_email(user_id):
to, errors = email_data_request_schema.load(request.get_json())
template = dao_get_template_by_id(current_app.config['ALREADY_REGISTERED_EMAIL_TEMPLATE_ID'])
message = {
'template': str(template.id),
'template_version': template.version,
'to': to['email'],
'personalisation': {
'signin_url': current_app.config['ADMIN_BASE_URL'] + '/sign-in',
'forgot_password_url': current_app.config['ADMIN_BASE_URL'] + '/forgot-password',
'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/feedback'
}
}
send_email.apply_async((
current_app.config['NOTIFY_SERVICE_ID'],
str(uuid.uuid4()),
encryption.encrypt(message),
datetime.utcnow().strftime(DATETIME_FORMAT)
), queue='notify')
return jsonify({}), 204
def test_send_email_should_go_to_retry_queue_if_database_errors(sample_email_template, mocker):
notification = _notification_json(sample_email_template, "*****@*****.**")
expected_exception = SQLAlchemyError()
mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
mocker.patch('app.celery.tasks.send_email.retry', side_effect=Exception())
mocker.patch('app.notifications.process_notifications.dao_create_notification', side_effect=expected_exception)
now = datetime.utcnow()
notification_id = uuid.uuid4()
with pytest.raises(Exception):
send_email(
sample_email_template.service_id,
notification_id,
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT)
)
assert not provider_tasks.deliver_email.apply_async.called
tasks.send_email.retry.assert_called_with(exc=expected_exception, queue='retry')
assert Notification.query.count() == 0
def test_should_send_sms_if_restricted_service_and_non_team_number_with_test_key(notify_db,
notify_db_session,
mocker):
user = sample_user(notify_db, notify_db_session, mobile_numnber="07700 900205")
service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = sample_template(notify_db, notify_db_session, service=service)
notification = _notification_json(template, "07700 900849")
mocked_deliver_sms = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
notification_id = uuid.uuid4()
send_sms(
service.id,
notification_id,
encryption.encrypt(notification),
datetime.utcnow().strftime(DATETIME_FORMAT),
key_type=KEY_TYPE_TEST
)
persisted_notification = Notification.query.one()
mocked_deliver_sms.assert_called_once_with(
[str(persisted_notification.id)],
queue="send-sms"
)
def get_api_key_by_secret(secret):
return db.on_reader().query(ApiKey).filter_by(
_secret=encryption.encrypt(str(secret))).options(
joinedload('service')).one()
def personalisation(self, personalisation):
if personalisation:
self._personalisation = encryption.encrypt(personalisation)
def test_encrypt():
value = "some random value"
encrypted_val = encrypt(value)
assert value != encrypted_val
def test_check_password():
password = "******"
encrypted_password = encrypt(password)
assert password != encrypted_password
assert check_password(password, encrypted_password)
def process_sms_or_email_notification(*, form, notification_type, api_key, template, service, reply_to_text=None):
form_send_to = form["email_address"] if notification_type == EMAIL_TYPE else form["phone_number"]
send_to = validate_and_format_recipient(
send_to=form_send_to,
key_type=api_key.key_type,
service=service,
notification_type=notification_type,
)
# Do not persist or send notification to the queue if it is a simulated recipient
simulated = simulated_recipient(send_to, notification_type)
personalisation = process_document_uploads(form.get("personalisation"), service, simulated, template.id)
notification = {
"id": create_uuid(),
"template": str(template.id),
"template_version": str(template.version),
"to": form_send_to,
"personalisation": personalisation,
"simulated": simulated,
"api_key": str(api_key.id),
"key_type": str(api_key.key_type),
"client_reference": form.get("reference", None),
}
encrypted_notification_data = encryption.encrypt(notification)
scheduled_for = form.get("scheduled_for", None)
if scheduled_for:
notification = persist_notification(
template_id=template.id,
template_version=template.version,
recipient=form_send_to,
service=service,
personalisation=personalisation,
notification_type=notification_type,
api_key_id=api_key.id,
key_type=api_key.key_type,
client_reference=form.get("reference", None),
simulated=simulated,
reply_to_text=reply_to_text,
)
persist_scheduled_notification(notification.id, form["scheduled_for"])
elif current_app.config["FF_NOTIFICATION_CELERY_PERSISTENCE"] and not simulated:
# depending on the type route to the appropriate save task
if notification_type == EMAIL_TYPE:
current_app.logger.info("calling save email task")
save_email.apply_async(
(authenticated_service.id, create_uuid(), encrypted_notification_data),
queue=QueueNames.DATABASE if not authenticated_service.research_mode else QueueNames.RESEARCH_MODE,
)
elif notification_type == SMS_TYPE:
save_sms.apply_async(
(authenticated_service.id, create_uuid(), encrypted_notification_data),
queue=QueueNames.DATABASE if not authenticated_service.research_mode else QueueNames.RESEARCH_MODE,
)
else:
notification = persist_notification(
template_id=template.id,
template_version=template.version,
recipient=form_send_to,
service=service,
personalisation=personalisation,
notification_type=notification_type,
api_key_id=api_key.id,
key_type=api_key.key_type,
client_reference=form.get("reference", None),
simulated=simulated,
reply_to_text=reply_to_text,
)
if not simulated:
send_notification_to_queue(
notification=notification,
research_mode=service.research_mode,
queue=template.queue_to_use(),
)
else:
current_app.logger.debug("POST simulated notification for id: {}".format(notification.id))
if not isinstance(notification, Notification):
notification["template_id"] = notification["template"]
notification["api_key_id"] = notification["api_key"]
notification["template_version"] = template.version
notification["service"] = service
notification["service_id"] = service.id
notification["reply_to_text"] = reply_to_text
del notification["template"]
del notification["api_key"]
del notification["simulated"]
notification = Notification(**notification)
return notification
def test_notification_personalisation_getter_always_returns_empty_dict():
noti = Notification()
noti._personalisation = encryption.encrypt({})
assert noti.personalisation == {}
def test_notification_personalisation_setter_always_sets_empty_dict(
input_value):
noti = Notification()
noti.personalisation = input_value
assert noti._personalisation == encryption.encrypt({})
