def test_fetch_returned_letters_from_notifications_and_notification_history(sample_letter_template):
today = datetime.now()
last_month = datetime.now() - timedelta(days=30)
letter_1 = create_notification(template=sample_letter_template, client_reference='letter_1',
status=NOTIFICATION_RETURNED_LETTER,
created_at=datetime.utcnow() - timedelta(days=1))
returned_letter_1 = create_returned_letter(service=sample_letter_template.service, reported_at=today,
notification_id=letter_1.id)
letter_2 = create_notification_history(template=sample_letter_template, client_reference='letter_2',
status=NOTIFICATION_RETURNED_LETTER, created_at=datetime.utcnow())
returned_letter_2 = create_returned_letter(service=sample_letter_template.service, reported_at=today,
notification_id=letter_2.id)
letter_3 = create_notification_history(template=sample_letter_template, client_reference='letter_3',
status=NOTIFICATION_RETURNED_LETTER)
create_returned_letter(service=sample_letter_template.service, reported_at=last_month,
notification_id=letter_3.id)
results = fetch_returned_letters(service_id=sample_letter_template.service_id, report_date=today.date())
assert len(results) == 2
assert results[0] == (letter_2.id, returned_letter_2.reported_at, letter_2.client_reference, letter_2.created_at,
sample_letter_template.name, letter_2.template_id, letter_2.template_version, False, None,
None, None, None, None, None)
assert results[1] == (letter_1.id, returned_letter_1.reported_at, letter_1.client_reference, letter_1.created_at,
sample_letter_template.name, letter_1.template_id, letter_1.template_version, False,
letter_1.api_key_id, None, None, None, None, None)
def test_fetch_stats_ignores_historical_notification_data(sample_template):
create_notification_history(template=sample_template)
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 1
stats = dao_fetch_stats_for_service(sample_template.service_id, 7)
assert len(stats) == 0
def test_update_fact_notification_status(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
second_template = create_template(service=second_service,
template_type='email')
third_service = create_service(service_name='third Service')
third_template = create_template(service=third_service,
template_type='letter')
create_notification(template=first_template, status='delivered')
create_notification(template=first_template,
created_at=datetime.utcnow() - timedelta(days=1))
# simulate a service with data retention - data has been moved to history and does not exist in notifications
create_notification_history(template=second_template,
status='temporary-failure')
create_notification_history(template=second_template,
created_at=datetime.utcnow() -
timedelta(days=1))
create_notification(template=third_template, status='created')
create_notification(template=third_template,
created_at=datetime.utcnow() - timedelta(days=1))
process_day = datetime.utcnow()
data = fetch_notification_status_for_day(process_day=process_day)
update_fact_notification_status(data=data, process_day=process_day.date())
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type).all()
assert len(new_fact_data) == 3
assert new_fact_data[0].bst_date == process_day.date()
assert new_fact_data[0].template_id == second_template.id
assert new_fact_data[0].service_id == second_service.id
assert new_fact_data[0].job_id == UUID(
'00000000-0000-0000-0000-000000000000')
assert new_fact_data[0].notification_type == 'email'
assert new_fact_data[0].notification_status == 'temporary-failure'
assert new_fact_data[0].notification_count == 1
assert new_fact_data[1].bst_date == process_day.date()
assert new_fact_data[1].template_id == third_template.id
assert new_fact_data[1].service_id == third_service.id
assert new_fact_data[1].job_id == UUID(
'00000000-0000-0000-0000-000000000000')
assert new_fact_data[1].notification_type == 'letter'
assert new_fact_data[1].notification_status == 'created'
assert new_fact_data[1].notification_count == 1
assert new_fact_data[2].bst_date == process_day.date()
assert new_fact_data[2].template_id == first_template.id
assert new_fact_data[2].service_id == first_service.id
assert new_fact_data[2].job_id == UUID(
'00000000-0000-0000-0000-000000000000')
assert new_fact_data[2].notification_type == 'sms'
assert new_fact_data[2].notification_status == 'delivered'
assert new_fact_data[2].notification_count == 1
def test_update_fact_notification_status(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
second_template = create_template(service=second_service, template_type='email')
third_service = create_service(service_name='third Service')
third_template = create_template(service=third_service, template_type='letter')
create_service_data_retention(second_service, 'email', days_of_retention=3)
process_day = date.today() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.min)):
create_notification(template=first_template, status='delivered')
# 2nd service email has 3 day data retention - data has been moved to history and doesn't exist in notifications
create_notification_history(template=second_template, status='temporary-failure')
create_notification(template=third_template, status='sending')
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.min)):
create_notification(template=first_template)
create_notification_history(template=second_template)
create_notification(template=third_template)
for notification_type in ('letter', 'sms', 'email'):
data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
update_fact_notification_status(data=data, process_day=process_day, notification_type=notification_type)
new_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 3
assert new_fact_data[0].bst_date == process_day
assert new_fact_data[0].template_id == second_template.id
assert new_fact_data[0].service_id == second_service.id
assert new_fact_data[0].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[0].notification_type == 'email'
assert new_fact_data[0].notification_status == 'temporary-failure'
assert new_fact_data[0].notification_count == 1
assert new_fact_data[1].bst_date == process_day
assert new_fact_data[1].template_id == third_template.id
assert new_fact_data[1].service_id == third_service.id
assert new_fact_data[1].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[1].notification_type == 'letter'
assert new_fact_data[1].notification_status == 'sending'
assert new_fact_data[1].notification_count == 1
assert new_fact_data[2].bst_date == process_day
assert new_fact_data[2].template_id == first_template.id
assert new_fact_data[2].service_id == first_service.id
assert new_fact_data[2].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[2].notification_type == 'sms'
assert new_fact_data[2].notification_status == 'delivered'
assert new_fact_data[2].notification_count == 1
def test_insert_or_update_returned_letters_for_history_only(sample_letter_template):
history_1 = create_notification_history(template=sample_letter_template,
reference='ref1')
history_2 = create_notification_history(template=sample_letter_template,
reference='ref2')
assert ReturnedLetter.query.count() == 0
insert_or_update_returned_letters(['ref1', 'ref2'])
returned_letters = ReturnedLetter.query.all()
assert len(returned_letters) == 2
for x in returned_letters:
assert x.notification_id in [history_1.id, history_2.id]
def test_fetch_billing_data_for_day_uses_notification_history(notify_db_session):
service = create_service()
sms_template = create_template(service=service, template_type='sms')
create_notification_history(template=sms_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=8))
create_notification_history(template=sms_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=8))
Notification.query.delete()
db.session.commit()
results = fetch_billing_data_for_day(process_day=datetime.utcnow() - timedelta(days=8), service_id=service.id)
assert len(results) == 1
assert results[0].notifications_sent == 2
def test_fetch_billing_data_for_day_uses_correct_table(notify_db_session):
service = create_service()
create_service_data_retention(service, notification_type='email', days_of_retention=3)
sms_template = create_template(service=service, template_type='sms')
email_template = create_template(service=service, template_type='email')
five_days_ago = datetime.utcnow() - timedelta(days=5)
create_notification(template=sms_template, status='delivered', created_at=five_days_ago)
create_notification_history(template=email_template, status='delivered', created_at=five_days_ago)
results = fetch_billing_data_for_day(process_day=five_days_ago.date(), service_id=service.id)
assert len(results) == 2
assert results[0].notification_type == 'sms'
assert results[0].notifications_sent == 1
assert results[1].notification_type == 'email'
assert results[1].notifications_sent == 1
def test_insert_or_update_returned_letters_updates(sample_letter_template):
notification = create_notification(template=sample_letter_template,
reference='ref1')
history = create_notification_history(template=sample_letter_template,
reference='ref2')
assert ReturnedLetter.query.count() == 0
with freeze_time('2019-12-09 13:30'):
insert_or_update_returned_letters(['ref1', 'ref2'])
returned_letters = ReturnedLetter.query.all()
assert len(returned_letters) == 2
for x in returned_letters:
assert x.reported_at == date(2019, 12, 9)
assert x.created_at == datetime(2019, 12, 9, 13, 30)
assert not x.updated_at
assert x.notification_id in [notification.id, history.id]
with freeze_time('2019-12-10 14:20'):
insert_or_update_returned_letters(['ref1', 'ref2'])
returned_letters = ReturnedLetter.query.all()
assert len(returned_letters) == 2
for x in returned_letters:
assert x.reported_at == date(2019, 12, 10)
assert x.created_at == datetime(2019, 12, 9, 13, 30)
assert x.updated_at == datetime(2019, 12, 10, 14, 20)
assert x.notification_id in [notification.id, history.id]
def test_delete_notifications_does_nothing_if_notification_history_row_already_exists(
sample_email_template, mocker):
notification = create_notification(template=sample_email_template,
created_at=datetime.utcnow() -
timedelta(days=8),
status='temporary-failure')
create_notification_history(id=notification.id,
template=sample_email_template,
created_at=datetime.utcnow() -
timedelta(days=8),
status='delivered')
delete_notifications_older_than_retention_by_type("email")
history = NotificationHistory.query.all()
assert len(history) == 1
assert history[0].status == 'delivered'
def test_process_ses_results_in_complaint_if_notification_does_not_exist(
sample_email_template, ):
notification = create_notification_history(template=sample_email_template,
reference="ref1")
handle_complaint(json.loads(ses_complaint_callback()["Message"]))
complaints = Complaint.query.all()
assert len(complaints) == 1
assert complaints[0].notification_id == notification.id
def test_fetch_billing_data_for_day_uses_notification_history(notify_db_session):
local_now = convert_utc_to_local_timezone(datetime.utcnow())
service = create_service()
sms_template = create_template(service=service, template_type="sms")
create_notification_history(
template=sms_template,
status="delivered",
created_at=datetime.utcnow() - timedelta(days=8),
)
create_notification_history(
template=sms_template,
status="delivered",
created_at=datetime.utcnow() - timedelta(days=8),
)
Notification.query.delete()
db.session.commit()
results = fetch_billing_data_for_day(process_day=local_now - timedelta(days=8), service_id=service.id)
assert len(results) == 1
assert results[0].notifications_sent == 2
def test_fetch_returned_letters_with_jobs(sample_letter_job):
today = datetime.now()
letter_1 = create_notification_history(template=sample_letter_job.template, client_reference='letter_1',
status=NOTIFICATION_RETURNED_LETTER,
job=sample_letter_job, job_row_number=20,
created_at=datetime.utcnow() - timedelta(minutes=1))
returned_letter_1 = create_returned_letter(service=sample_letter_job.service, reported_at=today,
notification_id=letter_1.id)
results = fetch_returned_letters(service_id=sample_letter_job.service_id, report_date=today.date())
assert len(results) == 1
assert results[0] == (letter_1.id, returned_letter_1.reported_at, letter_1.client_reference, letter_1.created_at,
sample_letter_job.template.name, letter_1.template_id, letter_1.template_version, False, None,
None, None, None, sample_letter_job.original_file_name, 21)
def test_update_letter_notification_statuses_when_notification_does_not_exist_updates_notification_history(
sample_letter_template, mocker):
valid_file = 'ref-foo|Sent|1|Unsorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
notification = create_notification_history(sample_letter_template,
reference='ref-foo',
status=NOTIFICATION_SENDING,
billable_units=1)
update_letter_notifications_statuses(
filename="NOTIFY-20170823160812-RSP.TXT")
updated_history = NotificationHistory.query.filter_by(
id=notification.id).one()
assert updated_history.status == NOTIFICATION_DELIVERED
def test_fetch_returned_letters_with_create_by_user(sample_letter_template):
today = datetime.now()
letter_1 = create_notification_history(template=sample_letter_template, client_reference='letter_1',
status=NOTIFICATION_RETURNED_LETTER,
created_at=datetime.utcnow() - timedelta(minutes=1),
created_by_id=sample_letter_template.service.users[0].id)
returned_letter_1 = create_returned_letter(service=sample_letter_template.service, reported_at=today,
notification_id=letter_1.id)
results = fetch_returned_letters(service_id=sample_letter_template.service_id, report_date=today.date())
assert len(results) == 1
assert results[0] == (letter_1.id, returned_letter_1.reported_at, letter_1.client_reference, letter_1.created_at,
sample_letter_template.name, letter_1.template_id, letter_1.template_version, False, None,
letter_1.created_by_id, sample_letter_template.service.users[0].name,
sample_letter_template.service.users[0].email_address, None, None)
def test_insert_or_update_returned_letters_inserts(sample_letter_template):
notification = create_notification(template=sample_letter_template,
reference='ref1')
history = create_notification_history(template=sample_letter_template,
reference='ref2')
assert ReturnedLetter.query.count() == 0
insert_or_update_returned_letters(['ref1', 'ref2'])
returned_letters = ReturnedLetter.query.all()
assert len(returned_letters) == 2
returned_letters_ = [x.notification_id for x in returned_letters]
assert notification.id in returned_letters_
assert history.id in returned_letters_
