def test_delete_notifications_updates_notification_history( sample_email_template, mocker): mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects") notification = create_notification(template=sample_email_template, created_at=datetime.utcnow() - timedelta(days=8)) Notification.query.filter_by(id=notification.id).update({ "status": "delivered", "reference": "ses_reference", "billable_units": 1, # I know we don't update this for emails but this is a unit test "updated_at": datetime.utcnow(), "sent_at": datetime.utcnow(), "sent_by": "ses" }) delete_notifications_older_than_retention_by_type("email") history = NotificationHistory.query.all() assert len(history) == 1 assert history[0].status == 'delivered' assert history[0].reference == 'ses_reference' assert history[0].billable_units == 1 assert history[0].updated_at assert history[0].sent_by == 'ses'
def test_delete_notifications_inserts_notification_history(sample_service): create_test_data('sms', sample_service) assert Notification.query.count() == 9 delete_notifications_older_than_retention_by_type('sms') assert Notification.query.count() == 7 assert NotificationHistory.query.count() == 2
def test_delete_notifications_delete_notification_type_for_default_time_if_no_days_of_retention_for_type( sample_service, ): create_service_data_retention(service=sample_service, notification_type="sms", days_of_retention=15) email_template, letter_template, sms_template = _create_templates( sample_service) create_notification(template=email_template, status="delivered") create_notification(template=sms_template, status="permanent-failure") create_notification(template=letter_template, status="temporary-failure") create_notification( template=email_template, status="delivered", created_at=datetime.utcnow() - timedelta(days=14), ) create_notification( template=sms_template, status="permanent-failure", created_at=datetime.utcnow() - timedelta(days=14), ) create_notification( template=letter_template, status="temporary-failure", created_at=datetime.utcnow() - timedelta(days=14), ) assert Notification.query.count() == 6 delete_notifications_older_than_retention_by_type("email") assert Notification.query.count() == 5 assert Notification.query.filter_by(notification_type="email").count() == 1
def test_delete_notifications_with_test_keys(sample_template, mocker): mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects") create_notification(template=sample_template, key_type='test', created_at=datetime.utcnow() - timedelta(days=8)) delete_notifications_older_than_retention_by_type('sms') assert Notification.query.count() == 0
def test_delete_notifications_calls_subquery_multiple_times(sample_template): create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8)) create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8)) create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8)) assert Notification.query.count() == 3 delete_notifications_older_than_retention_by_type('sms', qry_limit=1) assert Notification.query.count() == 0
def test_delete_notifications_does_try_to_delete_from_s3_when_letter_has_not_been_sent(sample_service, mocker): mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects") letter_template = create_template(service=sample_service, template_type='letter') create_notification(template=letter_template, status='sending', reference='LETTER_REF') delete_notifications_older_than_retention_by_type('email', qry_limit=1) mock_get_s3.assert_not_called()
def test_delete_notifications_keep_data_for_days_of_retention_is_longer( sample_service): create_test_data('sms', sample_service, 15) assert Notification.query.count() == 9 delete_notifications_older_than_retention_by_type('sms') assert Notification.query.count() == 8 assert Notification.query.filter( Notification.notification_type == 'sms').count() == 2
def test_should_delete_notifications_by_type_after_seven_days( sample_service, mocker, month, delete_run_time, notification_type, expected_sms_count, expected_email_count, expected_letter_count, ): mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects") email_template, letter_template, sms_template = _create_templates( sample_service) # create one notification a day between 1st and 10th from 11:00 to 19:00 of each type for i in range(1, 11): past_date = "2016-0{0}-{1:02d} {1:02d}:00:00.000000".format(month, i) with freeze_time(past_date): create_notification( template=email_template, created_at=datetime.utcnow(), status="permanent-failure", ) create_notification(template=sms_template, created_at=datetime.utcnow(), status="delivered") create_notification( template=letter_template, created_at=datetime.utcnow(), status="temporary-failure", ) assert Notification.query.count() == 30 # Records from before 3rd should be deleted with freeze_time(delete_run_time): delete_notifications_older_than_retention_by_type(notification_type) remaining_sms_notifications = Notification.query.filter_by( notification_type="sms").all() remaining_letter_notifications = Notification.query.filter_by( notification_type="letter").all() remaining_email_notifications = Notification.query.filter_by( notification_type="email").all() assert len(remaining_sms_notifications) == expected_sms_count assert len(remaining_email_notifications) == expected_email_count assert len(remaining_letter_notifications) == expected_letter_count if notification_type == "sms": notifications_to_check = remaining_sms_notifications if notification_type == "email": notifications_to_check = remaining_email_notifications if notification_type == "letter": notifications_to_check = remaining_letter_notifications for notification in notifications_to_check: assert notification.created_at.date() >= date(2016, month, 3)
def test_should_not_delete_notification_history(sample_service, mocker): mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects") with freeze_time('2016-01-01 12:00'): email_template, letter_template, sms_template = _create_templates(sample_service) create_notification(template=email_template, status='permanent-failure') create_notification(template=sms_template, status='permanent-failure') create_notification(template=letter_template, status='permanent-failure') assert Notification.query.count() == 3 delete_notifications_older_than_retention_by_type('sms') assert Notification.query.count() == 2 assert NotificationHistory.query.count() == 1
def test_delete_notifications_for_days_of_retention(sample_service, notification_type, mocker): mock_get_s3 = mocker.patch( "app.dao.notifications_dao.get_s3_bucket_objects") create_test_data(notification_type, sample_service) assert Notification.query.count() == 9 delete_notifications_older_than_retention_by_type(notification_type) assert Notification.query.count() == 7 assert Notification.query.filter_by( notification_type=notification_type).count() == 1 if notification_type == 'letter': assert mock_get_s3.call_count == 2 else: mock_get_s3.assert_not_called()
def test_delete_notifications_for_days_of_retention(sample_service, notification_type, mocker): mock_s3_object = mocker.patch( 'app.dao.notifications_dao.find_letter_pdf_in_s3').return_value create_test_data(notification_type, sample_service) assert Notification.query.count() == 9 delete_notifications_older_than_retention_by_type(notification_type) assert Notification.query.count() == 7 assert Notification.query.filter_by( notification_type=notification_type).count() == 1 if notification_type == 'letter': assert mock_s3_object.delete.call_count == 2 else: mock_s3_object.delete.assert_not_called()
def test_delete_notifications_for_days_of_retention(sample_service, notification_type, mocker): mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects") create_test_data(notification_type, sample_service) assert Notification.query.count() == 9 delete_notifications_older_than_retention_by_type(notification_type) assert Notification.query.count() == 7 assert Notification.query.filter_by(notification_type=notification_type).count() == 1 if notification_type == 'letter': mock_get_s3.assert_called_with(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder="{}/NOTIFY.LETTER_REF.D.2.C.C".format(str(datetime.utcnow().date())) ) assert mock_get_s3.call_count == 2 else: mock_get_s3.assert_not_called()
def test_delete_notifications_does_nothing_if_notification_history_row_already_exists( sample_email_template, mocker): notification = create_notification(template=sample_email_template, created_at=datetime.utcnow() - timedelta(days=8), status='temporary-failure') create_notification_history(id=notification.id, template=sample_email_template, created_at=datetime.utcnow() - timedelta(days=8), status='delivered') delete_notifications_older_than_retention_by_type("email") history = NotificationHistory.query.all() assert len(history) == 1 assert history[0].status == 'delivered'
def test_delete_notifications_deletes_letters_from_s3(sample_letter_template, mocker): mock_get_s3 = mocker.patch( "app.dao.notifications_dao.get_s3_bucket_objects") eight_days_ago = datetime.utcnow() - timedelta(days=8) create_notification(template=sample_letter_template, status='delivered', reference='LETTER_REF', created_at=eight_days_ago, sent_at=eight_days_ago) delete_notifications_older_than_retention_by_type( notification_type='letter') mock_get_s3.assert_called_once_with( bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder="{}/NOTIFY.LETTER_REF.D.2.C.C.{}.PDF".format( str(eight_days_ago.date()), eight_days_ago.strftime('%Y%m%d%H%M%S')))
def test_should_not_delete_notification_if_history_does_not_exist( sample_service, mocker): mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects") mocker.patch( "app.dao.notifications_dao.insert_update_notification_history") with freeze_time("2016-01-01 12:00"): email_template, letter_template, sms_template = _create_templates( sample_service) create_notification(template=email_template, status="permanent-failure") create_notification(template=sms_template, status="delivered") create_notification(template=letter_template, status="temporary-failure") assert Notification.query.count() == 3 delete_notifications_older_than_retention_by_type("sms") assert Notification.query.count() == 3 assert NotificationHistory.query.count() == 0
def test_delete_notifications_deletes_letters_not_sent_and_in_final_state_from_table_but_not_s3( sample_service, mocker, notification_status): mock_s3_object = mocker.patch( "app.dao.notifications_dao.find_letter_pdf_in_s3").return_value letter_template = create_template(service=sample_service, template_type='letter') create_notification(template=letter_template, status=notification_status, reference='LETTER_REF', created_at=datetime.utcnow() - timedelta(days=14)) assert Notification.query.count() == 1 assert NotificationHistory.query.count() == 0 delete_notifications_older_than_retention_by_type('letter') assert Notification.query.count() == 0 assert NotificationHistory.query.count() == 1 mock_s3_object.assert_not_called()
def test_delete_notifications_delete_notification_type_for_default_time_if_no_days_of_retention_for_type( sample_service ): create_service_data_retention(service=sample_service, notification_type='sms', days_of_retention=15) email_template, letter_template, sms_template = _create_templates(sample_service) create_notification(template=email_template, status='delivered') create_notification(template=sms_template, status='permanent-failure') create_notification(template=letter_template, status='temporary-failure') create_notification(template=email_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=14)) create_notification(template=sms_template, status='permanent-failure', created_at=datetime.utcnow() - timedelta(days=14)) create_notification(template=letter_template, status='temporary-failure', created_at=datetime.utcnow() - timedelta(days=14)) assert Notification.query.count() == 6 delete_notifications_older_than_retention_by_type('email') assert Notification.query.count() == 5 assert Notification.query.filter_by(notification_type='email').count() == 1
def delete_letter_notifications_older_than_retention(): try: start = datetime.utcnow() deleted = delete_notifications_older_than_retention_by_type('letter') current_app.logger.info( "Delete {} job started {} finished {} deleted {} letter notifications" .format('letter', start, datetime.utcnow(), deleted)) except SQLAlchemyError: current_app.logger.exception("Failed to delete letter notifications") raise
def test_delete_notifications_returns_sum_correctly(sample_template): create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8)) create_notification(template=sample_template, created_at=datetime.now() - timedelta(days=8)) s2 = create_service(service_name='s2') t2 = create_template(s2, template_type='sms') create_notification(template=t2, created_at=datetime.now() - timedelta(days=8)) create_notification(template=t2, created_at=datetime.now() - timedelta(days=8)) ret = delete_notifications_older_than_retention_by_type('sms', qry_limit=1) assert ret == 4
def test_delete_notifications_does_not_delete_letters_not_yet_in_final_state( sample_service, mocker, notification_status): mock_get_s3 = mocker.patch( "app.dao.notifications_dao.get_s3_bucket_objects") letter_template = create_template(service=sample_service, template_type='letter') create_notification( template=letter_template, status=notification_status, reference='LETTER_REF', created_at=datetime.utcnow() - timedelta(days=8), ) assert Notification.query.count() == 1 assert NotificationHistory.query.count() == 0 delete_notifications_older_than_retention_by_type('letter') assert Notification.query.count() == 1 assert NotificationHistory.query.count() == 0 mock_get_s3.assert_not_called()
def test_delete_notifications_deletes_letters_from_s3(sample_letter_template, mocker): s3 = boto3.client('s3', region_name='eu-west-1') bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME'] s3.create_bucket( Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'}) eight_days_ago = datetime.utcnow() - timedelta(days=8) create_notification(template=sample_letter_template, status='delivered', reference='LETTER_REF', created_at=eight_days_ago, sent_at=eight_days_ago) filename = "{}/NOTIFY.LETTER_REF.D.2.C.{}.PDF".format( str(eight_days_ago.date()), eight_days_ago.strftime('%Y%m%d%H%M%S')) s3.put_object(Bucket=bucket_name, Key=filename, Body=b'foo') delete_notifications_older_than_retention_by_type( notification_type='letter') with pytest.raises(s3.exceptions.NoSuchKey): s3.get_object(Bucket=bucket_name, Key=filename)
def test_should_delete_notification_and_recipient_identifiers_when_bulk_deleting( month, delete_run_time, notification_type, expected_count, sample_job, sample_api_key, mocker): mocker.patch( 'app.notifications.process_notifications.accept_recipient_identifiers_enabled', return_value=True) # create one notification a day between 1st and 10th from 11:00 to 19:00 of each type for i in range(1, 11): past_date = '2016-0{0}-{1:02d} {1:02d}:00:00.000000'.format(month, i) with freeze_time(past_date): recipient_identifier = { "id_type": IdentifierType.VA_PROFILE_ID.value, "id_value": "foo" } persist_notification(template_id=sample_job.template.id, template_version=sample_job.template.version, service=sample_job.service, personalisation=None, notification_type=notification_type, api_key_id=sample_api_key.id, key_type=sample_api_key.key_type, recipient_identifier=recipient_identifier, created_at=datetime.utcnow()) assert Notification.query.count() == 10 assert RecipientIdentifier.query.count() == 10 # Records from before 3rd should be deleted with freeze_time(delete_run_time): delete_notifications_older_than_retention_by_type(notification_type) remaining_notifications = Notification.query.filter_by( notification_type=notification_type).all() assert len(remaining_notifications) == expected_count remaining_recipient_identifiers = len(RecipientIdentifier.query.all()) assert remaining_recipient_identifiers == expected_count
def test_delete_notifications_deletes_letters_sent_and_in_final_state_from_table_and_s3( sample_service, mocker, notification_status): mock_get_s3 = mocker.patch( "app.dao.notifications_dao.get_s3_bucket_objects") letter_template = create_template(service=sample_service, template_type='letter') eight_days_ago = datetime.utcnow() - timedelta(days=8) create_notification(template=letter_template, status=notification_status, reference='LETTER_REF', created_at=eight_days_ago, sent_at=eight_days_ago) assert Notification.query.count() == 1 assert NotificationHistory.query.count() == 0 delete_notifications_older_than_retention_by_type('letter') assert Notification.query.count() == 0 assert NotificationHistory.query.count() == 1 mock_get_s3.assert_called_once_with( bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder="{}/NOTIFY.LETTER_REF.D.2.C.C.{}.PDF".format( str(eight_days_ago.date()), eight_days_ago.strftime('%Y%m%d%H%M%S')))
def test_delete_notifications_with_test_keys(sample_template, mocker): create_notification(template=sample_template, key_type='test', created_at=datetime.utcnow() - timedelta(days=8)) delete_notifications_older_than_retention_by_type('sms') assert Notification.query.count() == 0