def test_filter_s3_bucket_objects_within_date_range_filters_by_date_range(
notify_api, mocker):
START_DATE = datetime_in_past(days=9)
JUST_BEFORE_START_DATE = START_DATE - timedelta(seconds=1)
JUST_AFTER_START_DATE = START_DATE + timedelta(seconds=1)
END_DATE = datetime_in_past(days=7)
JUST_BEFORE_END_DATE = END_DATE - timedelta(seconds=1)
JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1)
s3_objects_stub = [
single_s3_object_stub("bar/", JUST_BEFORE_START_DATE),
single_s3_object_stub("bar/foo.txt", START_DATE),
single_s3_object_stub("bar/foo2.txt", JUST_AFTER_START_DATE),
single_s3_object_stub("bar/foo3.txt", JUST_BEFORE_END_DATE),
single_s3_object_stub("bar/foo4.txt", END_DATE),
single_s3_object_stub("bar/foo5.txt", JUST_AFTER_END_DATE),
]
filtered_items = filter_s3_bucket_objects_within_date_range(
s3_objects_stub)
assert len(filtered_items) == 2
assert filtered_items[0]["Key"] == "bar/foo2.txt"
assert filtered_items[0]["LastModified"] == JUST_AFTER_START_DATE
assert filtered_items[1]["Key"] == "bar/foo3.txt"
assert filtered_items[1]["LastModified"] == JUST_BEFORE_END_DATE
def test_get_s3_bucket_objects_removes_redundant_root_object(
notify_api, mocker):
AFTER_SEVEN_DAYS = datetime_in_past(days=8)
s3_objects_stub = [
single_s3_object_stub("bar/", AFTER_SEVEN_DAYS),
single_s3_object_stub("bar/foo.txt", AFTER_SEVEN_DAYS),
]
filtered_items = filter_s3_bucket_objects_within_date_range(
s3_objects_stub)
assert len(filtered_items) == 1
assert filtered_items[0]["Key"] == "bar/foo.txt"
assert filtered_items[0]["LastModified"] == datetime_in_past(days=8)
def delete_dvla_response_files_older_than_seven_days():
try:
start = datetime.utcnow()
bucket_objects = s3.get_s3_bucket_objects(
current_app.config['DVLA_RESPONSE_BUCKET_NAME'], 'root/dispatch')
older_than_seven_days = s3.filter_s3_bucket_objects_within_date_range(
bucket_objects)
for f in older_than_seven_days:
s3.remove_s3_object(
current_app.config['DVLA_RESPONSE_BUCKET_NAME'], f['Key'])
current_app.logger.info(
"Delete dvla response files started {} finished {} deleted {} files"
.format(start, datetime.utcnow(), len(older_than_seven_days)))
except SQLAlchemyError:
current_app.logger.exception("Failed to delete dvla response files")
raise
def test_get_s3_bucket_objects_does_not_return_outside_of_date_range(
notify_api, mocker):
START_DATE = datetime_in_past(days=9)
JUST_BEFORE_START_DATE = START_DATE - timedelta(seconds=1)
END_DATE = datetime_in_past(days=7)
JUST_AFTER_END_DATE = END_DATE + timedelta(seconds=1)
s3_objects_stub = [
single_s3_object_stub("bar/", JUST_BEFORE_START_DATE),
single_s3_object_stub("bar/foo1.txt", START_DATE),
single_s3_object_stub("bar/foo2.txt", END_DATE),
single_s3_object_stub("bar/foo3.txt", JUST_AFTER_END_DATE),
]
filtered_items = filter_s3_bucket_objects_within_date_range(
s3_objects_stub)
assert len(filtered_items) == 0
