def test_delete_notifications_updates_notification_history(
sample_email_template, mocker):
mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
notification = create_notification(template=sample_email_template,
created_at=datetime.utcnow() -
timedelta(days=8))
Notification.query.filter_by(id=notification.id).update({
"status":
"delivered",
"reference":
"ses_reference",
"billable_units":
1, # I know we don't update this for emails but this is a unit test
"updated_at":
datetime.utcnow(),
"sent_at":
datetime.utcnow(),
"sent_by":
"ses"
})
delete_notifications_older_than_retention_by_type("email")
history = NotificationHistory.query.all()
assert len(history) == 1
assert history[0].status == 'delivered'
assert history[0].reference == 'ses_reference'
assert history[0].billable_units == 1
assert history[0].updated_at
assert history[0].sent_by == 'ses'
def test_delete_notifications_inserts_notification_history(sample_service):
create_test_data('sms', sample_service)
assert Notification.query.count() == 9
delete_notifications_older_than_retention_by_type('sms')
assert Notification.query.count() == 7
assert NotificationHistory.query.count() == 2
def test_delete_notifications_delete_notification_type_for_default_time_if_no_days_of_retention_for_type(
sample_service, ):
create_service_data_retention(service=sample_service,
notification_type="sms",
days_of_retention=15)
email_template, letter_template, sms_template = _create_templates(
sample_service)
create_notification(template=email_template, status="delivered")
create_notification(template=sms_template, status="permanent-failure")
create_notification(template=letter_template, status="temporary-failure")
create_notification(
template=email_template,
status="delivered",
created_at=datetime.utcnow() - timedelta(days=14),
)
create_notification(
template=sms_template,
status="permanent-failure",
created_at=datetime.utcnow() - timedelta(days=14),
)
create_notification(
template=letter_template,
status="temporary-failure",
created_at=datetime.utcnow() - timedelta(days=14),
)
assert Notification.query.count() == 6
delete_notifications_older_than_retention_by_type("email")
assert Notification.query.count() == 5
assert Notification.query.filter_by(notification_type="email").count() == 1
def test_delete_notifications_with_test_keys(sample_template, mocker):
mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
create_notification(template=sample_template,
key_type='test',
created_at=datetime.utcnow() - timedelta(days=8))
delete_notifications_older_than_retention_by_type('sms')
assert Notification.query.count() == 0
def test_delete_notifications_calls_subquery_multiple_times(sample_template):
create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8))
create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8))
create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8))
assert Notification.query.count() == 3
delete_notifications_older_than_retention_by_type('sms', qry_limit=1)
assert Notification.query.count() == 0
def test_delete_notifications_does_try_to_delete_from_s3_when_letter_has_not_been_sent(sample_service, mocker):
mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
letter_template = create_template(service=sample_service, template_type='letter')
create_notification(template=letter_template, status='sending',
reference='LETTER_REF')
delete_notifications_older_than_retention_by_type('email', qry_limit=1)
mock_get_s3.assert_not_called()
def test_delete_notifications_keep_data_for_days_of_retention_is_longer(
sample_service):
create_test_data('sms', sample_service, 15)
assert Notification.query.count() == 9
delete_notifications_older_than_retention_by_type('sms')
assert Notification.query.count() == 8
assert Notification.query.filter(
Notification.notification_type == 'sms').count() == 2
def test_should_delete_notifications_by_type_after_seven_days(
sample_service,
mocker,
month,
delete_run_time,
notification_type,
expected_sms_count,
expected_email_count,
expected_letter_count,
):
mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
email_template, letter_template, sms_template = _create_templates(
sample_service)
# create one notification a day between 1st and 10th from 11:00 to 19:00 of each type
for i in range(1, 11):
past_date = "2016-0{0}-{1:02d} {1:02d}:00:00.000000".format(month, i)
with freeze_time(past_date):
create_notification(
template=email_template,
created_at=datetime.utcnow(),
status="permanent-failure",
)
create_notification(template=sms_template,
created_at=datetime.utcnow(),
status="delivered")
create_notification(
template=letter_template,
created_at=datetime.utcnow(),
status="temporary-failure",
)
assert Notification.query.count() == 30
# Records from before 3rd should be deleted
with freeze_time(delete_run_time):
delete_notifications_older_than_retention_by_type(notification_type)
remaining_sms_notifications = Notification.query.filter_by(
notification_type="sms").all()
remaining_letter_notifications = Notification.query.filter_by(
notification_type="letter").all()
remaining_email_notifications = Notification.query.filter_by(
notification_type="email").all()
assert len(remaining_sms_notifications) == expected_sms_count
assert len(remaining_email_notifications) == expected_email_count
assert len(remaining_letter_notifications) == expected_letter_count
if notification_type == "sms":
notifications_to_check = remaining_sms_notifications
if notification_type == "email":
notifications_to_check = remaining_email_notifications
if notification_type == "letter":
notifications_to_check = remaining_letter_notifications
for notification in notifications_to_check:
assert notification.created_at.date() >= date(2016, month, 3)
def test_should_not_delete_notification_history(sample_service, mocker):
mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
with freeze_time('2016-01-01 12:00'):
email_template, letter_template, sms_template = _create_templates(sample_service)
create_notification(template=email_template, status='permanent-failure')
create_notification(template=sms_template, status='permanent-failure')
create_notification(template=letter_template, status='permanent-failure')
assert Notification.query.count() == 3
delete_notifications_older_than_retention_by_type('sms')
assert Notification.query.count() == 2
assert NotificationHistory.query.count() == 1
def test_delete_notifications_for_days_of_retention(sample_service,
notification_type, mocker):
mock_get_s3 = mocker.patch(
"app.dao.notifications_dao.get_s3_bucket_objects")
create_test_data(notification_type, sample_service)
assert Notification.query.count() == 9
delete_notifications_older_than_retention_by_type(notification_type)
assert Notification.query.count() == 7
assert Notification.query.filter_by(
notification_type=notification_type).count() == 1
if notification_type == 'letter':
assert mock_get_s3.call_count == 2
else:
mock_get_s3.assert_not_called()
def test_delete_notifications_for_days_of_retention(sample_service,
notification_type, mocker):
mock_s3_object = mocker.patch(
'app.dao.notifications_dao.find_letter_pdf_in_s3').return_value
create_test_data(notification_type, sample_service)
assert Notification.query.count() == 9
delete_notifications_older_than_retention_by_type(notification_type)
assert Notification.query.count() == 7
assert Notification.query.filter_by(
notification_type=notification_type).count() == 1
if notification_type == 'letter':
assert mock_s3_object.delete.call_count == 2
else:
mock_s3_object.delete.assert_not_called()
def test_delete_notifications_for_days_of_retention(sample_service, notification_type, mocker):
mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
create_test_data(notification_type, sample_service)
assert Notification.query.count() == 9
delete_notifications_older_than_retention_by_type(notification_type)
assert Notification.query.count() == 7
assert Notification.query.filter_by(notification_type=notification_type).count() == 1
if notification_type == 'letter':
mock_get_s3.assert_called_with(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder="{}/NOTIFY.LETTER_REF.D.2.C.C".format(str(datetime.utcnow().date()))
)
assert mock_get_s3.call_count == 2
else:
mock_get_s3.assert_not_called()
def test_delete_notifications_does_nothing_if_notification_history_row_already_exists(
sample_email_template, mocker):
notification = create_notification(template=sample_email_template,
created_at=datetime.utcnow() -
timedelta(days=8),
status='temporary-failure')
create_notification_history(id=notification.id,
template=sample_email_template,
created_at=datetime.utcnow() -
timedelta(days=8),
status='delivered')
delete_notifications_older_than_retention_by_type("email")
history = NotificationHistory.query.all()
assert len(history) == 1
assert history[0].status == 'delivered'
def test_delete_notifications_deletes_letters_from_s3(sample_letter_template,
mocker):
mock_get_s3 = mocker.patch(
"app.dao.notifications_dao.get_s3_bucket_objects")
eight_days_ago = datetime.utcnow() - timedelta(days=8)
create_notification(template=sample_letter_template,
status='delivered',
reference='LETTER_REF',
created_at=eight_days_ago,
sent_at=eight_days_ago)
delete_notifications_older_than_retention_by_type(
notification_type='letter')
mock_get_s3.assert_called_once_with(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder="{}/NOTIFY.LETTER_REF.D.2.C.C.{}.PDF".format(
str(eight_days_ago.date()),
eight_days_ago.strftime('%Y%m%d%H%M%S')))
def test_should_not_delete_notification_if_history_does_not_exist(
sample_service, mocker):
mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
mocker.patch(
"app.dao.notifications_dao.insert_update_notification_history")
with freeze_time("2016-01-01 12:00"):
email_template, letter_template, sms_template = _create_templates(
sample_service)
create_notification(template=email_template,
status="permanent-failure")
create_notification(template=sms_template, status="delivered")
create_notification(template=letter_template,
status="temporary-failure")
assert Notification.query.count() == 3
delete_notifications_older_than_retention_by_type("sms")
assert Notification.query.count() == 3
assert NotificationHistory.query.count() == 0
def test_delete_notifications_deletes_letters_not_sent_and_in_final_state_from_table_but_not_s3(
sample_service, mocker, notification_status):
mock_s3_object = mocker.patch(
"app.dao.notifications_dao.find_letter_pdf_in_s3").return_value
letter_template = create_template(service=sample_service,
template_type='letter')
create_notification(template=letter_template,
status=notification_status,
reference='LETTER_REF',
created_at=datetime.utcnow() - timedelta(days=14))
assert Notification.query.count() == 1
assert NotificationHistory.query.count() == 0
delete_notifications_older_than_retention_by_type('letter')
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 1
mock_s3_object.assert_not_called()
def test_delete_notifications_delete_notification_type_for_default_time_if_no_days_of_retention_for_type(
sample_service
):
create_service_data_retention(service=sample_service, notification_type='sms',
days_of_retention=15)
email_template, letter_template, sms_template = _create_templates(sample_service)
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='permanent-failure')
create_notification(template=letter_template, status='temporary-failure')
create_notification(template=email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=14))
create_notification(template=sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=14))
create_notification(template=letter_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=14))
assert Notification.query.count() == 6
delete_notifications_older_than_retention_by_type('email')
assert Notification.query.count() == 5
assert Notification.query.filter_by(notification_type='email').count() == 1
def delete_letter_notifications_older_than_retention():
try:
start = datetime.utcnow()
deleted = delete_notifications_older_than_retention_by_type('letter')
current_app.logger.info(
"Delete {} job started {} finished {} deleted {} letter notifications"
.format('letter', start, datetime.utcnow(), deleted))
except SQLAlchemyError:
current_app.logger.exception("Failed to delete letter notifications")
raise
def test_delete_notifications_returns_sum_correctly(sample_template):
create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8))
create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8))
s2 = create_service(service_name='s2')
t2 = create_template(s2, template_type='sms')
create_notification(template=t2, created_at=datetime.now() - timedelta(days=8))
create_notification(template=t2, created_at=datetime.now() - timedelta(days=8))
ret = delete_notifications_older_than_retention_by_type('sms', qry_limit=1)
assert ret == 4
def test_delete_notifications_does_not_delete_letters_not_yet_in_final_state(
sample_service, mocker, notification_status):
mock_get_s3 = mocker.patch(
"app.dao.notifications_dao.get_s3_bucket_objects")
letter_template = create_template(service=sample_service,
template_type='letter')
create_notification(
template=letter_template,
status=notification_status,
reference='LETTER_REF',
created_at=datetime.utcnow() - timedelta(days=8),
)
assert Notification.query.count() == 1
assert NotificationHistory.query.count() == 0
delete_notifications_older_than_retention_by_type('letter')
assert Notification.query.count() == 1
assert NotificationHistory.query.count() == 0
mock_get_s3.assert_not_called()
def test_delete_notifications_deletes_letters_from_s3(sample_letter_template,
mocker):
s3 = boto3.client('s3', region_name='eu-west-1')
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
s3.create_bucket(
Bucket=bucket_name,
CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
eight_days_ago = datetime.utcnow() - timedelta(days=8)
create_notification(template=sample_letter_template,
status='delivered',
reference='LETTER_REF',
created_at=eight_days_ago,
sent_at=eight_days_ago)
filename = "{}/NOTIFY.LETTER_REF.D.2.C.{}.PDF".format(
str(eight_days_ago.date()), eight_days_ago.strftime('%Y%m%d%H%M%S'))
s3.put_object(Bucket=bucket_name, Key=filename, Body=b'foo')
delete_notifications_older_than_retention_by_type(
notification_type='letter')
with pytest.raises(s3.exceptions.NoSuchKey):
s3.get_object(Bucket=bucket_name, Key=filename)
def test_should_delete_notification_and_recipient_identifiers_when_bulk_deleting(
month, delete_run_time, notification_type, expected_count, sample_job,
sample_api_key, mocker):
mocker.patch(
'app.notifications.process_notifications.accept_recipient_identifiers_enabled',
return_value=True)
# create one notification a day between 1st and 10th from 11:00 to 19:00 of each type
for i in range(1, 11):
past_date = '2016-0{0}-{1:02d} {1:02d}:00:00.000000'.format(month, i)
with freeze_time(past_date):
recipient_identifier = {
"id_type": IdentifierType.VA_PROFILE_ID.value,
"id_value": "foo"
}
persist_notification(template_id=sample_job.template.id,
template_version=sample_job.template.version,
service=sample_job.service,
personalisation=None,
notification_type=notification_type,
api_key_id=sample_api_key.id,
key_type=sample_api_key.key_type,
recipient_identifier=recipient_identifier,
created_at=datetime.utcnow())
assert Notification.query.count() == 10
assert RecipientIdentifier.query.count() == 10
# Records from before 3rd should be deleted
with freeze_time(delete_run_time):
delete_notifications_older_than_retention_by_type(notification_type)
remaining_notifications = Notification.query.filter_by(
notification_type=notification_type).all()
assert len(remaining_notifications) == expected_count
remaining_recipient_identifiers = len(RecipientIdentifier.query.all())
assert remaining_recipient_identifiers == expected_count
def test_delete_notifications_deletes_letters_sent_and_in_final_state_from_table_and_s3(
sample_service, mocker, notification_status):
mock_get_s3 = mocker.patch(
"app.dao.notifications_dao.get_s3_bucket_objects")
letter_template = create_template(service=sample_service,
template_type='letter')
eight_days_ago = datetime.utcnow() - timedelta(days=8)
create_notification(template=letter_template,
status=notification_status,
reference='LETTER_REF',
created_at=eight_days_ago,
sent_at=eight_days_ago)
assert Notification.query.count() == 1
assert NotificationHistory.query.count() == 0
delete_notifications_older_than_retention_by_type('letter')
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 1
mock_get_s3.assert_called_once_with(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder="{}/NOTIFY.LETTER_REF.D.2.C.C.{}.PDF".format(
str(eight_days_ago.date()),
eight_days_ago.strftime('%Y%m%d%H%M%S')))
def test_delete_notifications_with_test_keys(sample_template, mocker):
create_notification(template=sample_template,
key_type='test',
created_at=datetime.utcnow() - timedelta(days=8))
delete_notifications_older_than_retention_by_type('sms')
assert Notification.query.count() == 0
