def test_get_list_of_files_by_suffix(notify_api, mocker, suffix_str, days_before, returned_no): paginator_mock = mocker.patch("app.aws.s3.client") multiple_pages_s3_object = [ { "Contents": [ single_s3_object_stub("bar/foo.ACK.txt", datetime_in_past(1, 0)), ] }, { "Contents": [ single_s3_object_stub("bar/foo1.rs.txt", datetime_in_past(2, 0)), ] }, ] paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object if days_before: key = get_list_of_files_by_suffix( "foo-bucket", subfolder="bar", suffix=suffix_str, last_modified=datetime.now(tz=pytz.utc) - timedelta(days=days_before), ) else: key = get_list_of_files_by_suffix("foo-bucket", subfolder="bar", suffix=suffix_str) assert sum(1 for x in key) == returned_no for k in key: assert k == "bar/foo.ACK.txt"
def test_filter_s3_bucket_objects_within_date_range_filters_by_date_range( notify_api, mocker): START_DATE = datetime_in_past(days=9) JUST_BEFORE_START_DATE = START_DATE - timedelta(seconds=1) JUST_AFTER_START_DATE = START_DATE + timedelta(seconds=1) END_DATE = datetime_in_past(days=7) JUST_BEFORE_END_DATE = END_DATE - timedelta(seconds=1) JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1) s3_objects_stub = [ single_s3_object_stub("bar/", JUST_BEFORE_START_DATE), single_s3_object_stub("bar/foo.txt", START_DATE), single_s3_object_stub("bar/foo2.txt", JUST_AFTER_START_DATE), single_s3_object_stub("bar/foo3.txt", JUST_BEFORE_END_DATE), single_s3_object_stub("bar/foo4.txt", END_DATE), single_s3_object_stub("bar/foo5.txt", JUST_AFTER_END_DATE), ] filtered_items = filter_s3_bucket_objects_within_date_range( s3_objects_stub) assert len(filtered_items) == 2 assert filtered_items[0]["Key"] == "bar/foo2.txt" assert filtered_items[0]["LastModified"] == JUST_AFTER_START_DATE assert filtered_items[1]["Key"] == "bar/foo3.txt" assert filtered_items[1]["LastModified"] == JUST_BEFORE_END_DATE
def test_get_s3_bucket_objects_removes_redundant_root_object( notify_api, mocker): AFTER_SEVEN_DAYS = datetime_in_past(days=8) s3_objects_stub = [ single_s3_object_stub("bar/", AFTER_SEVEN_DAYS), single_s3_object_stub("bar/foo.txt", AFTER_SEVEN_DAYS), ] filtered_items = filter_s3_bucket_objects_within_date_range( s3_objects_stub) assert len(filtered_items) == 1 assert filtered_items[0]["Key"] == "bar/foo.txt" assert filtered_items[0]["LastModified"] == datetime_in_past(days=8)
def test_delete_dvla_response_files_older_than_seven_days_removes_old_files( notify_api, mocker): AFTER_SEVEN_DAYS = datetime_in_past(days=8) single_page_s3_objects = [{ "Contents": [ single_s3_object_stub("bar/foo1.txt", AFTER_SEVEN_DAYS), single_s3_object_stub("bar/foo2.txt", AFTER_SEVEN_DAYS), ] }] mocker.patch( "app.celery.nightly_tasks.s3.get_s3_bucket_objects", return_value=single_page_s3_objects[0]["Contents"], ) remove_s3_mock = mocker.patch( "app.celery.nightly_tasks.s3.remove_s3_object") delete_dvla_response_files_older_than_seven_days() remove_s3_mock.assert_has_calls([ call( current_app.config["DVLA_RESPONSE_BUCKET_NAME"], single_page_s3_objects[0]["Contents"][0]["Key"], ), call( current_app.config["DVLA_RESPONSE_BUCKET_NAME"], single_page_s3_objects[0]["Contents"][1]["Key"], ), ])
def test_get_s3_bucket_objects_does_not_return_outside_of_date_range( notify_api, mocker): START_DATE = datetime_in_past(days=9) JUST_BEFORE_START_DATE = START_DATE - timedelta(seconds=1) END_DATE = datetime_in_past(days=7) JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1) s3_objects_stub = [ single_s3_object_stub("bar/", JUST_BEFORE_START_DATE), single_s3_object_stub("bar/foo1.txt", START_DATE), single_s3_object_stub("bar/foo2.txt", END_DATE), single_s3_object_stub("bar/foo3.txt", JUST_AFTER_END_DATE), ] filtered_items = filter_s3_bucket_objects_within_date_range( s3_objects_stub) assert len(filtered_items) == 0
def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(notify_api, mocker): START_DATE = datetime_in_past(days=9) JUST_BEFORE_START_DATE = datetime_in_past(days=9, seconds=1) END_DATE = datetime_in_past(days=7) JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1) single_page_s3_objects = [{ "Contents": [ single_s3_object_stub('bar/foo1.txt', JUST_BEFORE_START_DATE), single_s3_object_stub('bar/foo2.txt', START_DATE), single_s3_object_stub('bar/foo3.txt', END_DATE), single_s3_object_stub('bar/foo4.txt', JUST_AFTER_END_DATE), ] }] mocker.patch( 'app.celery.nightly_tasks.s3.get_s3_bucket_objects', return_value=single_page_s3_objects[0]["Contents"] ) remove_s3_mock = mocker.patch('app.celery.nightly_tasks.s3.remove_s3_object') delete_dvla_response_files_older_than_seven_days() remove_s3_mock.assert_not_called()
def test_get_s3_bucket_objects_builds_objects_list_from_paginator( notify_api, mocker): AFTER_SEVEN_DAYS = datetime_in_past(days=8) paginator_mock = mocker.patch('app.aws.s3.client') multiple_pages_s3_object = [{ "Contents": [ single_s3_object_stub('bar/foo.txt', AFTER_SEVEN_DAYS), ] }, { "Contents": [ single_s3_object_stub('bar/foo1.txt', AFTER_SEVEN_DAYS), ] }] paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object bucket_objects = get_s3_bucket_objects('foo-bucket', subfolder='bar') assert len(bucket_objects) == 2 assert set(bucket_objects[0].keys()) == set( ['ETag', 'Key', 'LastModified'])
def test_get_s3_bucket_objects_builds_objects_list_from_paginator( notify_api, mocker): AFTER_SEVEN_DAYS = datetime_in_past(days=8) paginator_mock = mocker.patch("app.aws.s3.client") multiple_pages_s3_object = [ { "Contents": [ single_s3_object_stub("bar/foo.txt", AFTER_SEVEN_DAYS), ] }, { "Contents": [ single_s3_object_stub("bar/foo1.txt", AFTER_SEVEN_DAYS), ] }, ] paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object bucket_objects = get_s3_bucket_objects("foo-bucket", subfolder="bar") assert len(bucket_objects) == 2 assert set(bucket_objects[0].keys()) == set( ["ETag", "Key", "LastModified"])