def test_retention_checks_list_submitted_bucket_items(self):
outside_retention = datetime.datetime.today() - datetime.timedelta(
days=29)
outside_retention_str = outside_retention.strftime(
'%Y-%m-%dT%H:%M:%S.%fZ')
bucket_items = [{
'name': '2018-09-01/person.csv',
'timeCreated': outside_retention_str,
'updated': outside_retention_str
}]
# if the file expires within a day it should not be returned
actual_result = main.list_submitted_bucket_items(bucket_items)
expected_result = []
self.assertListEqual(expected_result, actual_result)
# if the file within retention period it should be returned
within_retention = datetime.datetime.today() - datetime.timedelta(
days=25)
within_retention_str = within_retention.strftime(
'%Y-%m-%dT%H:%M:%S.%fZ')
item_2 = {
'name': '2018-09-01/visit_occurrence.csv',
'timeCreated': within_retention_str,
'updated': within_retention_str
}
bucket_items.append(item_2)
expected_result = [item_2]
actual_result = main.list_submitted_bucket_items(bucket_items)
self.assertListEqual(expected_result, actual_result)
actual_result = main.list_submitted_bucket_items([])
self.assertListEqual([], actual_result)
unknown_item = {
'name': '2018-09-01/nyc_cu_person.csv',
'timeCreated': within_retention_str,
'updated': within_retention_str
}
bucket_items = [unknown_item]
actual_result = main.list_submitted_bucket_items(bucket_items)
self.assertListEqual(actual_result, bucket_items)
ignored_item = dict(name='2018-09-01/' + common.RESULT_CSV,
timeCreated=within_retention_str,
updated=within_retention_str)
bucket_items = [ignored_item]
actual_result = main.list_submitted_bucket_items(bucket_items)
self.assertListEqual([], actual_result)
def test_retention_checks_list_submitted_bucket_items(self):
#Define times to use
within_retention = datetime.datetime.today() - datetime.timedelta(
days=25)
within_retention_str = within_retention.strftime(
'%Y-%m-%dT%H:%M:%S.%fZ')
outside_retention = datetime.datetime.today() - datetime.timedelta(
days=29)
outside_retention_str = outside_retention.strftime(
'%Y-%m-%dT%H:%M:%S.%fZ')
before_lag_time = datetime.datetime.today() - datetime.timedelta(
minutes=3)
before_lag_time_str = before_lag_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
after_lag_time = datetime.datetime.today() - datetime.timedelta(
minutes=7)
after_lag_time_str = after_lag_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
# If any required files are missing, nothing should be returned
bucket_items = self._create_dummy_bucket_items(
within_retention_str,
after_lag_time_str,
file_exclusions=['visit_occurrence.csv'])
actual_result = main.list_submitted_bucket_items(bucket_items)
expected_result = []
self.maxDiff = None
self.assertCountEqual(expected_result, actual_result)
# If all required files are present and files within retention period, files should be returned
bucket_items = self._create_dummy_bucket_items(within_retention_str,
after_lag_time_str)
actual_result = main.list_submitted_bucket_items(bucket_items)
expected_result = bucket_items
self.assertCountEqual(expected_result, actual_result)
# bucket_items = [{
# 'name': '2018-09-01/person.csv',
# 'timeCreated': outside_retention_str,
# 'updated': after_lag_time_str
# }]
# if a file expires within a day, it should not be returned
bucket_items = self._create_dummy_bucket_items(
within_retention_str,
after_lag_time_str,
file_exclusions=['person.csv'])
bucket_items_with_modified_person = bucket_items.copy()
bucket_items_with_modified_person.append({
'name': '2018-09-01/person.csv',
'timeCreated': outside_retention_str,
'updated': after_lag_time_str
})
actual_result = main.list_submitted_bucket_items(
bucket_items_with_modified_person)
expected_result = bucket_items
self.assertCountEqual(expected_result, actual_result)
actual_result = main.list_submitted_bucket_items([])
self.assertCountEqual([], actual_result)
#If unknown item and all other conditions met, return the item
bucket_items = self._create_dummy_bucket_items(within_retention_str,
after_lag_time_str)
unknown_item = {
'name': '2018-09-01/nyc_cu_person.csv',
'timeCreated': within_retention_str,
'updated': after_lag_time_str
}
bucket_items.append(unknown_item)
actual_result = main.list_submitted_bucket_items(bucket_items)
self.assertCountEqual(actual_result, bucket_items)
# If ignored item and all other conditions met, only exclude the ignored item
bucket_items = self._create_dummy_bucket_items(within_retention_str,
after_lag_time_str)
bucket_items_with_ignored_item = bucket_items.copy()
ignored_item = dict(name='2018-09-01/' + common.RESULTS_HTML,
timeCreated=within_retention_str,
updated=within_retention_str)
bucket_items_with_ignored_item.append(ignored_item)
actual_result = main.list_submitted_bucket_items(
bucket_items_with_ignored_item)
expected_result = bucket_items
self.assertCountEqual(expected_result, actual_result)
# If any AOU_REQUIRED file has been updated less than 5 minutes ago, no files should be returned
bucket_items = self._create_dummy_bucket_items(
within_retention_str,
after_lag_time_str,
file_exclusions=['observation.csv'])
bucket_items.append({
'name': '2018-09-01/observation.csv',
'timeCreated': within_retention_str,
'updated': before_lag_time_str
})
actual_result = main.list_submitted_bucket_items(bucket_items)
expected_result = []
self.assertCountEqual(expected_result, actual_result)