def test_should_delete_many_today_duplicates_and_11th_young_version_after_deduplication_and_retain_old_backup(
self, _1, _2, _3):
#given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
young_backups = create_backup_daily_sequence(10,
start_date=datetime(
2017, 8, 1))
newest_duplicated_backup = create_backup(datetime(2017, 8, 19, 10))
today_duplicated_backups = [
newest_duplicated_backup,
create_backup(datetime(2017, 8, 19, 9)),
create_backup(datetime(2017, 8, 19, 8)),
create_backup(datetime(2017, 8, 19, 7))
]
old_backup = create_backup(datetime(2016, 8, 19, 10))
backups = list(young_backups + today_duplicated_backups + [old_backup])
backups_expected_for_deletion = list([young_backups[9]] +
today_duplicated_backups[1:])
#when
eligible_for_deletion = \
self.under_test.get_backups_eligible_for_deletion(
backups=list(backups),
table_reference=reference)
#then
self.sortAndAssertListEqual(backups_expected_for_deletion,
eligible_for_deletion)
def test_should_fill_deleted_field_in_backup_entity_if_table_not_found_error_during_deletion(
self, _):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime.datetime(2017, 2, 1, 16, 30))
table.put()
reference = TableReference.from_table_entity(table)
backup1 = backup_utils.create_backup(datetime.datetime(
2017, 2, 1, 16, 30),
table,
table_id="backup1")
backup2 = backup_utils.create_backup(datetime.datetime(
2017, 2, 2, 16, 30),
table,
table_id="backup2")
ndb.put_multi([backup1, backup2])
self.policy.get_backups_eligible_for_deletion = Mock(
return_value=[backup1, backup2])
# when
self.under_test.perform_retention(reference, table.key.urlsafe())
# then
self.assertTrue(Backup.get_by_key(backup1.key).deleted is not None)
self.assertTrue(Backup.get_by_key(backup2.key).deleted is not None)
def __create_table_with_two_backups():
table = Table(project_id=PROJECT_ID,
dataset_id=DATASET_ID,
table_id=TABLE_ID,
partition_id=PARTITION_ID,
last_checked=NOW)
table.put()
backup_utils.create_backup(NOW, table, BACKUP_TABLE_ID_FROM_NOW).put()
backup_utils.create_backup(OLD_TIME, table,
BACKUP_TABLE_ID_FROM_OLD_TIME).put()
def test_should_sort_backups_by_create_time_desc(self):
# given
b1 = backup_utils.create_backup(datetime(2017, 2, 3, 16, 30))
b2 = backup_utils.create_backup(datetime(2017, 2, 2, 16, 30))
b3 = backup_utils.create_backup(datetime(2017, 2, 1, 16, 30))
expected_list = [b1, b2, b3]
# when
sorted_backups = Backup.sort_backups_by_create_time_desc([b1, b3, b2])
# then
self.assertListEqual(expected_list, sorted_backups)
def test_should_divide_backups_if_any_old_backups(self):
# given
young_backup1 = create_backup(datetime(2017, 1, 20))
young_backup2 = create_backup(datetime(2017, 1, 21))
backups = [young_backup1, young_backup2]
# when
young_backups, old_backups = BackupAgeDivider.divide_backups_by_age(
backups)
# then
self.assertListEqual([young_backup1, young_backup2], young_backups)
self.assertListEqual([], old_backups)
def test_should_not_sort_in_place(self):
# given
b1 = backup_utils.create_backup(datetime(2017, 2, 3, 16, 30))
b2 = backup_utils.create_backup(datetime(2017, 2, 2, 16, 30))
b3 = backup_utils.create_backup(datetime(2017, 2, 1, 16, 30))
expected_list = [b1, b3, b2]
actual_list = [b1, b3, b2]
# when
Backup.sort_backups_by_create_time_desc(actual_list)
# then
self.assertListEqual(expected_list, actual_list)
def test_deduplicate_backups_from_the_2_days(self):
# given
b1 = create_backup(datetime(2017, 2, 1))
b2 = create_backup(datetime(2017, 2, 2))
backups = [b1, b1, b2, b2]
expected_backups = [b2, b1]
# when
actual_backups = self.under_test.filter(list(backups), self.reference)
# then
self.assertListEqual(expected_backups, actual_backups)
def test_should_filter_out_all_but_single_backup_when_all_backups_are_older_than_7_months(
self): # nopep8 pylint: disable=C0301
# given
b1 = create_backup(datetime(2017, 1, 17))
b2 = create_backup(datetime(2017, 1, 18))
youngest_old_backup = create_backup(datetime(2017, 1, 19))
backups = [b1, b2, youngest_old_backup]
expected_backups = [youngest_old_backup]
# when
backups_to_retain = self.under_test.filter(backups=backups,
table_reference=None)
# then
self.sortAndAssertListEqual(expected_backups, backups_to_retain)
def test_should_delete_older_backup_if_two_old_backups(self, _1, _2, _3):
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
older_old_backup = create_backup(datetime(2016, 7, 31))
old_backup = create_backup(datetime(2016, 8, 1))
all_backups = list([older_old_backup, old_backup])
# when
eligible_for_deletion = self.under_test \
.get_backups_eligible_for_deletion(backups=all_backups,
table_reference=reference)
# then
self.sortAndAssertListEqual([older_old_backup], eligible_for_deletion)
def test_should_divide_backups_if_any_young_backups(self):
# given
oldest_backup = create_backup(datetime(2017, 1, 18))
youngest_old_backup = create_backup(datetime(2017, 1, 19))
backups = [oldest_backup, youngest_old_backup]
# when
young_backups, old_backups = BackupAgeDivider.divide_backups_by_age(
backups)
# then
self.assertListEqual([], young_backups)
self.assertListEqual([oldest_backup, youngest_old_backup], old_backups)
def test_return_the_most_recent_backup_from_the_same_day(self):
# given
b1 = create_backup(datetime(2017, 2, 1, hour=15))
b2 = create_backup(datetime(2017, 2, 1, hour=16))
b3 = create_backup(datetime(2017, 2, 1, hour=17))
backups = [b1, b3, b2]
expected_backups = [b3]
# when
actual_backups = self.under_test.filter(list(backups), self.reference)
# then
self.assertListEqual(expected_backups, actual_backups)
def test_should_not_delete_if_single_young_backup_and_single_old_backup(
self, _1, _2, _3):
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
young_backup = create_backup(datetime(2017, 8, 1))
old_backup = create_backup(datetime(2016, 8, 1))
all_backups = [young_backup, old_backup]
# when
eligible_for_deletion = self.under_test \
.get_backups_eligible_for_deletion(backups=list(all_backups),
table_reference=reference)
# then
self.sortAndAssertListEqual([], eligible_for_deletion)
def test_should_retain_all_if_all_backups_are_younger_than_7_months(self): # nopep8 pylint: disable=C0301
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
b1 = create_backup(datetime(2017, 3, 19))
b2 = create_backup(datetime(2017, 3, 20))
b3 = create_backup(datetime(2017, 3, 21))
backups = [b1, b2, b3]
expected_backups = [b1, b2, b3]
# when
backups_to_retain = self.under_test.filter(backups=list(backups),
table_reference=reference)
# then
self.sortAndAssertListEqual(expected_backups, backups_to_retain)
def create_and_insert_table_with_one_backup(project_id,
dataset_id,
table_id,
date,
partition_id=None):
table = Table(project_id=project_id,
dataset_id=dataset_id,
table_id=table_id,
partition_id=partition_id,
last_checked=date)
table.put()
backup_utils.create_backup(date, table,
table_id + date.strftime('%Y%m%d')).put()
return table
def test_should_divide_backups_preserving_order(self):
# given
oldest_backup = create_backup(datetime(2017, 1, 18))
youngest_old_backup = create_backup(datetime(2017, 1, 19))
young_backup1 = create_backup(datetime(2017, 1, 20))
young_backup2 = create_backup(datetime(2017, 1, 21))
backups = [
youngest_old_backup, young_backup1, young_backup2, oldest_backup
]
# when
young_backups, old_backups = BackupAgeDivider.divide_backups_by_age(
backups)
# then
self.assertListEqual([young_backup1, young_backup2], young_backups)
self.assertListEqual([youngest_old_backup, oldest_backup], old_backups)
def test_remove_all_backups_if_source_table_doesnt_exists_for_min_7_months(
self, _, _1, _2): # nopep8 pylint: disable=C0301
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
b1 = create_backup(datetime(2017, 1, 17))
b2 = create_backup(datetime(2017, 1, 18))
backups = [b1, b2]
backups_expected_for_deletion = [b1, b2]
# when
eligible_for_deletion = self.under_test \
.get_backups_eligible_for_deletion(backups=list(backups),
table_reference=reference)
# then
self.sortAndAssertListEqual(backups_expected_for_deletion,
eligible_for_deletion)
def test_should_leave_youngest_backup_from_the_same_day_when_source_data_exists(
self, _, _1, _2): # nopep8 pylint: disable=C0301
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
b1 = create_backup(datetime(2017, 1, 1, hour=13, minute=15))
b2 = create_backup(datetime(2017, 1, 1, hour=16, minute=30))
backups = [b1, b2]
backups_expected_for_deletion = [b1]
# when
eligible_for_deletion = self.under_test \
.get_backups_eligible_for_deletion(backups=list(backups),
table_reference=reference)
# then
self.sortAndAssertListEqual(backups_expected_for_deletion,
eligible_for_deletion)
def test_return_the_same_backups(self):
# given
backups = [create_backup(datetime(2017, 2, 1))]
# when
actual_backups = self.under_test.filter(list(backups), self.reference)
# then
self.assertListEqual(backups, actual_backups)
def test_should_retain_the_youngest_old_backup_and_young_backups(self): # nopep8 pylint: disable=C0301
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
oldest_backup = create_backup(datetime(2017, 1, 18))
youngest_old_backup = create_backup(datetime(2017, 1, 19))
young_backup1 = create_backup(datetime(2017, 1, 20))
young_backup2 = create_backup(datetime(2017, 1, 21))
backups = [
oldest_backup, youngest_old_backup, young_backup1, young_backup2
]
expected_backups = [youngest_old_backup, young_backup1, young_backup2]
# when
backups_to_retain = self.under_test.filter(backups=list(backups),
table_reference=reference)
# then
self.sortAndAssertListEqual(expected_backups, backups_to_retain)
def test_should_not_remove_any_backups_if_source_table_was_deleted_less_than_seven_months_ago(
self, _, _1, _2): # nopep8 pylint: disable=C0301
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
young_backup = create_backup(datetime(2017, 8, 1))
old_backup = create_backup(datetime(2016, 1, 17))
backups = [young_backup, old_backup]
backups_expected_for_deletion = []
# when
eligible_for_deletion = self.under_test \
.get_backups_eligible_for_deletion(backups=list(backups),
table_reference=reference)
# then
self.sortAndAssertListEqual(backups_expected_for_deletion,
eligible_for_deletion)
def test_should_delete_oldest_young_backup_if_11_young_backup_and_single_old_backup(
self, _1, _2, _3):
# given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
young_backups = create_backup_daily_sequence(11,
start_date=datetime(
2017, 8, 1))
old_backup = create_backup(datetime(2016, 8, 1))
all_backups = list(young_backups + [old_backup])
# when
eligible_for_deletion = self.under_test \
.get_backups_eligible_for_deletion(backups=list(all_backups),
table_reference=reference)
# then
self.sortAndAssertListEqual([young_backups[10]], eligible_for_deletion)
def test_should_remove_above_last_10_young_backups(self):
#given
reference = TableReference('example-project-id', 'example-dataset-id',
'example-table-id')
oldest_backup = create_backup(datetime(2017, 6, 1))
_10_last_backups = create_backup_daily_sequence(10,
start_date=datetime(
2017, 6, 2))
backups = list(_10_last_backups)
backups.append(oldest_backup)
backups_expected_for_deletion = [oldest_backup]
#when
eligible_for_deletion = \
self.under_test.get_backups_eligible_for_deletion(
backups=list(backups),
table_reference=reference)
#then
self.sortAndAssertListEqual(backups_expected_for_deletion,
eligible_for_deletion)
