def test_get_list_of_files_by_suffix(notify_api, mocker, suffix_str,
days_before, returned_no):
paginator_mock = mocker.patch("app.aws.s3.client")
multiple_pages_s3_object = [
{
"Contents": [
single_s3_object_stub("bar/foo.ACK.txt",
datetime_in_past(1, 0)),
]
},
{
"Contents": [
single_s3_object_stub("bar/foo1.rs.txt",
datetime_in_past(2, 0)),
]
},
]
paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object
if days_before:
key = get_list_of_files_by_suffix(
"foo-bucket",
subfolder="bar",
suffix=suffix_str,
last_modified=datetime.now(tz=pytz.utc) -
timedelta(days=days_before),
)
else:
key = get_list_of_files_by_suffix("foo-bucket",
subfolder="bar",
suffix=suffix_str)
assert sum(1 for x in key) == returned_no
for k in key:
assert k == "bar/foo.ACK.txt"
def test_filter_s3_bucket_objects_within_date_range_filters_by_date_range(
notify_api, mocker):
START_DATE = datetime_in_past(days=9)
JUST_BEFORE_START_DATE = START_DATE - timedelta(seconds=1)
JUST_AFTER_START_DATE = START_DATE + timedelta(seconds=1)
END_DATE = datetime_in_past(days=7)
JUST_BEFORE_END_DATE = END_DATE - timedelta(seconds=1)
JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1)
s3_objects_stub = [
single_s3_object_stub("bar/", JUST_BEFORE_START_DATE),
single_s3_object_stub("bar/foo.txt", START_DATE),
single_s3_object_stub("bar/foo2.txt", JUST_AFTER_START_DATE),
single_s3_object_stub("bar/foo3.txt", JUST_BEFORE_END_DATE),
single_s3_object_stub("bar/foo4.txt", END_DATE),
single_s3_object_stub("bar/foo5.txt", JUST_AFTER_END_DATE),
]
filtered_items = filter_s3_bucket_objects_within_date_range(
s3_objects_stub)
assert len(filtered_items) == 2
assert filtered_items[0]["Key"] == "bar/foo2.txt"
assert filtered_items[0]["LastModified"] == JUST_AFTER_START_DATE
assert filtered_items[1]["Key"] == "bar/foo3.txt"
assert filtered_items[1]["LastModified"] == JUST_BEFORE_END_DATE
def test_get_s3_bucket_objects_removes_redundant_root_object(
notify_api, mocker):
AFTER_SEVEN_DAYS = datetime_in_past(days=8)
s3_objects_stub = [
single_s3_object_stub("bar/", AFTER_SEVEN_DAYS),
single_s3_object_stub("bar/foo.txt", AFTER_SEVEN_DAYS),
]
filtered_items = filter_s3_bucket_objects_within_date_range(
s3_objects_stub)
assert len(filtered_items) == 1
assert filtered_items[0]["Key"] == "bar/foo.txt"
assert filtered_items[0]["LastModified"] == datetime_in_past(days=8)
def test_delete_dvla_response_files_older_than_seven_days_removes_old_files(
notify_api, mocker):
AFTER_SEVEN_DAYS = datetime_in_past(days=8)
single_page_s3_objects = [{
"Contents": [
single_s3_object_stub("bar/foo1.txt", AFTER_SEVEN_DAYS),
single_s3_object_stub("bar/foo2.txt", AFTER_SEVEN_DAYS),
]
}]
mocker.patch(
"app.celery.nightly_tasks.s3.get_s3_bucket_objects",
return_value=single_page_s3_objects[0]["Contents"],
)
remove_s3_mock = mocker.patch(
"app.celery.nightly_tasks.s3.remove_s3_object")
delete_dvla_response_files_older_than_seven_days()
remove_s3_mock.assert_has_calls([
call(
current_app.config["DVLA_RESPONSE_BUCKET_NAME"],
single_page_s3_objects[0]["Contents"][0]["Key"],
),
call(
current_app.config["DVLA_RESPONSE_BUCKET_NAME"],
single_page_s3_objects[0]["Contents"][1]["Key"],
),
])
def test_get_s3_bucket_objects_does_not_return_outside_of_date_range(
notify_api, mocker):
START_DATE = datetime_in_past(days=9)
JUST_BEFORE_START_DATE = START_DATE - timedelta(seconds=1)
END_DATE = datetime_in_past(days=7)
JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1)
s3_objects_stub = [
single_s3_object_stub("bar/", JUST_BEFORE_START_DATE),
single_s3_object_stub("bar/foo1.txt", START_DATE),
single_s3_object_stub("bar/foo2.txt", END_DATE),
single_s3_object_stub("bar/foo3.txt", JUST_AFTER_END_DATE),
]
filtered_items = filter_s3_bucket_objects_within_date_range(
s3_objects_stub)
assert len(filtered_items) == 0
def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(notify_api, mocker):
START_DATE = datetime_in_past(days=9)
JUST_BEFORE_START_DATE = datetime_in_past(days=9, seconds=1)
END_DATE = datetime_in_past(days=7)
JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1)
single_page_s3_objects = [{
"Contents": [
single_s3_object_stub('bar/foo1.txt', JUST_BEFORE_START_DATE),
single_s3_object_stub('bar/foo2.txt', START_DATE),
single_s3_object_stub('bar/foo3.txt', END_DATE),
single_s3_object_stub('bar/foo4.txt', JUST_AFTER_END_DATE),
]
}]
mocker.patch(
'app.celery.nightly_tasks.s3.get_s3_bucket_objects', return_value=single_page_s3_objects[0]["Contents"]
)
remove_s3_mock = mocker.patch('app.celery.nightly_tasks.s3.remove_s3_object')
delete_dvla_response_files_older_than_seven_days()
remove_s3_mock.assert_not_called()
def test_get_s3_bucket_objects_builds_objects_list_from_paginator(
notify_api, mocker):
AFTER_SEVEN_DAYS = datetime_in_past(days=8)
paginator_mock = mocker.patch('app.aws.s3.client')
multiple_pages_s3_object = [{
"Contents": [
single_s3_object_stub('bar/foo.txt', AFTER_SEVEN_DAYS),
]
}, {
"Contents": [
single_s3_object_stub('bar/foo1.txt', AFTER_SEVEN_DAYS),
]
}]
paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object
bucket_objects = get_s3_bucket_objects('foo-bucket', subfolder='bar')
assert len(bucket_objects) == 2
assert set(bucket_objects[0].keys()) == set(
['ETag', 'Key', 'LastModified'])
def test_get_s3_bucket_objects_builds_objects_list_from_paginator(
notify_api, mocker):
AFTER_SEVEN_DAYS = datetime_in_past(days=8)
paginator_mock = mocker.patch("app.aws.s3.client")
multiple_pages_s3_object = [
{
"Contents": [
single_s3_object_stub("bar/foo.txt", AFTER_SEVEN_DAYS),
]
},
{
"Contents": [
single_s3_object_stub("bar/foo1.txt", AFTER_SEVEN_DAYS),
]
},
]
paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object
bucket_objects = get_s3_bucket_objects("foo-bucket", subfolder="bar")
assert len(bucket_objects) == 2
assert set(bucket_objects[0].keys()) == set(
["ETag", "Key", "LastModified"])
