def test_should_delete_many_today_duplicates_and_11th_young_version_after_deduplication_and_retain_old_backup( self, _1, _2, _3): #given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') young_backups = create_backup_daily_sequence(10, start_date=datetime( 2017, 8, 1)) newest_duplicated_backup = create_backup(datetime(2017, 8, 19, 10)) today_duplicated_backups = [ newest_duplicated_backup, create_backup(datetime(2017, 8, 19, 9)), create_backup(datetime(2017, 8, 19, 8)), create_backup(datetime(2017, 8, 19, 7)) ] old_backup = create_backup(datetime(2016, 8, 19, 10)) backups = list(young_backups + today_duplicated_backups + [old_backup]) backups_expected_for_deletion = list([young_backups[9]] + today_duplicated_backups[1:]) #when eligible_for_deletion = \ self.under_test.get_backups_eligible_for_deletion( backups=list(backups), table_reference=reference) #then self.sortAndAssertListEqual(backups_expected_for_deletion, eligible_for_deletion)
def test_should_fill_deleted_field_in_backup_entity_if_table_not_found_error_during_deletion( self, _): # given table = Table(project_id='example-proj-name', dataset_id='example-dataset-name', table_id='example-table-name', last_checked=datetime.datetime(2017, 2, 1, 16, 30)) table.put() reference = TableReference.from_table_entity(table) backup1 = backup_utils.create_backup(datetime.datetime( 2017, 2, 1, 16, 30), table, table_id="backup1") backup2 = backup_utils.create_backup(datetime.datetime( 2017, 2, 2, 16, 30), table, table_id="backup2") ndb.put_multi([backup1, backup2]) self.policy.get_backups_eligible_for_deletion = Mock( return_value=[backup1, backup2]) # when self.under_test.perform_retention(reference, table.key.urlsafe()) # then self.assertTrue(Backup.get_by_key(backup1.key).deleted is not None) self.assertTrue(Backup.get_by_key(backup2.key).deleted is not None)
def __create_table_with_two_backups(): table = Table(project_id=PROJECT_ID, dataset_id=DATASET_ID, table_id=TABLE_ID, partition_id=PARTITION_ID, last_checked=NOW) table.put() backup_utils.create_backup(NOW, table, BACKUP_TABLE_ID_FROM_NOW).put() backup_utils.create_backup(OLD_TIME, table, BACKUP_TABLE_ID_FROM_OLD_TIME).put()
def test_should_sort_backups_by_create_time_desc(self): # given b1 = backup_utils.create_backup(datetime(2017, 2, 3, 16, 30)) b2 = backup_utils.create_backup(datetime(2017, 2, 2, 16, 30)) b3 = backup_utils.create_backup(datetime(2017, 2, 1, 16, 30)) expected_list = [b1, b2, b3] # when sorted_backups = Backup.sort_backups_by_create_time_desc([b1, b3, b2]) # then self.assertListEqual(expected_list, sorted_backups)
def test_should_divide_backups_if_any_old_backups(self): # given young_backup1 = create_backup(datetime(2017, 1, 20)) young_backup2 = create_backup(datetime(2017, 1, 21)) backups = [young_backup1, young_backup2] # when young_backups, old_backups = BackupAgeDivider.divide_backups_by_age( backups) # then self.assertListEqual([young_backup1, young_backup2], young_backups) self.assertListEqual([], old_backups)
def test_should_not_sort_in_place(self): # given b1 = backup_utils.create_backup(datetime(2017, 2, 3, 16, 30)) b2 = backup_utils.create_backup(datetime(2017, 2, 2, 16, 30)) b3 = backup_utils.create_backup(datetime(2017, 2, 1, 16, 30)) expected_list = [b1, b3, b2] actual_list = [b1, b3, b2] # when Backup.sort_backups_by_create_time_desc(actual_list) # then self.assertListEqual(expected_list, actual_list)
def test_deduplicate_backups_from_the_2_days(self): # given b1 = create_backup(datetime(2017, 2, 1)) b2 = create_backup(datetime(2017, 2, 2)) backups = [b1, b1, b2, b2] expected_backups = [b2, b1] # when actual_backups = self.under_test.filter(list(backups), self.reference) # then self.assertListEqual(expected_backups, actual_backups)
def test_should_filter_out_all_but_single_backup_when_all_backups_are_older_than_7_months( self): # nopep8 pylint: disable=C0301 # given b1 = create_backup(datetime(2017, 1, 17)) b2 = create_backup(datetime(2017, 1, 18)) youngest_old_backup = create_backup(datetime(2017, 1, 19)) backups = [b1, b2, youngest_old_backup] expected_backups = [youngest_old_backup] # when backups_to_retain = self.under_test.filter(backups=backups, table_reference=None) # then self.sortAndAssertListEqual(expected_backups, backups_to_retain)
def test_should_delete_older_backup_if_two_old_backups(self, _1, _2, _3): # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') older_old_backup = create_backup(datetime(2016, 7, 31)) old_backup = create_backup(datetime(2016, 8, 1)) all_backups = list([older_old_backup, old_backup]) # when eligible_for_deletion = self.under_test \ .get_backups_eligible_for_deletion(backups=all_backups, table_reference=reference) # then self.sortAndAssertListEqual([older_old_backup], eligible_for_deletion)
def test_should_divide_backups_if_any_young_backups(self): # given oldest_backup = create_backup(datetime(2017, 1, 18)) youngest_old_backup = create_backup(datetime(2017, 1, 19)) backups = [oldest_backup, youngest_old_backup] # when young_backups, old_backups = BackupAgeDivider.divide_backups_by_age( backups) # then self.assertListEqual([], young_backups) self.assertListEqual([oldest_backup, youngest_old_backup], old_backups)
def test_return_the_most_recent_backup_from_the_same_day(self): # given b1 = create_backup(datetime(2017, 2, 1, hour=15)) b2 = create_backup(datetime(2017, 2, 1, hour=16)) b3 = create_backup(datetime(2017, 2, 1, hour=17)) backups = [b1, b3, b2] expected_backups = [b3] # when actual_backups = self.under_test.filter(list(backups), self.reference) # then self.assertListEqual(expected_backups, actual_backups)
def test_should_not_delete_if_single_young_backup_and_single_old_backup( self, _1, _2, _3): # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') young_backup = create_backup(datetime(2017, 8, 1)) old_backup = create_backup(datetime(2016, 8, 1)) all_backups = [young_backup, old_backup] # when eligible_for_deletion = self.under_test \ .get_backups_eligible_for_deletion(backups=list(all_backups), table_reference=reference) # then self.sortAndAssertListEqual([], eligible_for_deletion)
def test_should_retain_all_if_all_backups_are_younger_than_7_months(self): # nopep8 pylint: disable=C0301 # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') b1 = create_backup(datetime(2017, 3, 19)) b2 = create_backup(datetime(2017, 3, 20)) b3 = create_backup(datetime(2017, 3, 21)) backups = [b1, b2, b3] expected_backups = [b1, b2, b3] # when backups_to_retain = self.under_test.filter(backups=list(backups), table_reference=reference) # then self.sortAndAssertListEqual(expected_backups, backups_to_retain)
def create_and_insert_table_with_one_backup(project_id, dataset_id, table_id, date, partition_id=None): table = Table(project_id=project_id, dataset_id=dataset_id, table_id=table_id, partition_id=partition_id, last_checked=date) table.put() backup_utils.create_backup(date, table, table_id + date.strftime('%Y%m%d')).put() return table
def test_should_divide_backups_preserving_order(self): # given oldest_backup = create_backup(datetime(2017, 1, 18)) youngest_old_backup = create_backup(datetime(2017, 1, 19)) young_backup1 = create_backup(datetime(2017, 1, 20)) young_backup2 = create_backup(datetime(2017, 1, 21)) backups = [ youngest_old_backup, young_backup1, young_backup2, oldest_backup ] # when young_backups, old_backups = BackupAgeDivider.divide_backups_by_age( backups) # then self.assertListEqual([young_backup1, young_backup2], young_backups) self.assertListEqual([youngest_old_backup, oldest_backup], old_backups)
def test_remove_all_backups_if_source_table_doesnt_exists_for_min_7_months( self, _, _1, _2): # nopep8 pylint: disable=C0301 # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') b1 = create_backup(datetime(2017, 1, 17)) b2 = create_backup(datetime(2017, 1, 18)) backups = [b1, b2] backups_expected_for_deletion = [b1, b2] # when eligible_for_deletion = self.under_test \ .get_backups_eligible_for_deletion(backups=list(backups), table_reference=reference) # then self.sortAndAssertListEqual(backups_expected_for_deletion, eligible_for_deletion)
def test_should_leave_youngest_backup_from_the_same_day_when_source_data_exists( self, _, _1, _2): # nopep8 pylint: disable=C0301 # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') b1 = create_backup(datetime(2017, 1, 1, hour=13, minute=15)) b2 = create_backup(datetime(2017, 1, 1, hour=16, minute=30)) backups = [b1, b2] backups_expected_for_deletion = [b1] # when eligible_for_deletion = self.under_test \ .get_backups_eligible_for_deletion(backups=list(backups), table_reference=reference) # then self.sortAndAssertListEqual(backups_expected_for_deletion, eligible_for_deletion)
def test_return_the_same_backups(self): # given backups = [create_backup(datetime(2017, 2, 1))] # when actual_backups = self.under_test.filter(list(backups), self.reference) # then self.assertListEqual(backups, actual_backups)
def test_should_retain_the_youngest_old_backup_and_young_backups(self): # nopep8 pylint: disable=C0301 # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') oldest_backup = create_backup(datetime(2017, 1, 18)) youngest_old_backup = create_backup(datetime(2017, 1, 19)) young_backup1 = create_backup(datetime(2017, 1, 20)) young_backup2 = create_backup(datetime(2017, 1, 21)) backups = [ oldest_backup, youngest_old_backup, young_backup1, young_backup2 ] expected_backups = [youngest_old_backup, young_backup1, young_backup2] # when backups_to_retain = self.under_test.filter(backups=list(backups), table_reference=reference) # then self.sortAndAssertListEqual(expected_backups, backups_to_retain)
def test_should_not_remove_any_backups_if_source_table_was_deleted_less_than_seven_months_ago( self, _, _1, _2): # nopep8 pylint: disable=C0301 # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') young_backup = create_backup(datetime(2017, 8, 1)) old_backup = create_backup(datetime(2016, 1, 17)) backups = [young_backup, old_backup] backups_expected_for_deletion = [] # when eligible_for_deletion = self.under_test \ .get_backups_eligible_for_deletion(backups=list(backups), table_reference=reference) # then self.sortAndAssertListEqual(backups_expected_for_deletion, eligible_for_deletion)
def test_should_delete_oldest_young_backup_if_11_young_backup_and_single_old_backup( self, _1, _2, _3): # given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') young_backups = create_backup_daily_sequence(11, start_date=datetime( 2017, 8, 1)) old_backup = create_backup(datetime(2016, 8, 1)) all_backups = list(young_backups + [old_backup]) # when eligible_for_deletion = self.under_test \ .get_backups_eligible_for_deletion(backups=list(all_backups), table_reference=reference) # then self.sortAndAssertListEqual([young_backups[10]], eligible_for_deletion)
def test_should_remove_above_last_10_young_backups(self): #given reference = TableReference('example-project-id', 'example-dataset-id', 'example-table-id') oldest_backup = create_backup(datetime(2017, 6, 1)) _10_last_backups = create_backup_daily_sequence(10, start_date=datetime( 2017, 6, 2)) backups = list(_10_last_backups) backups.append(oldest_backup) backups_expected_for_deletion = [oldest_backup] #when eligible_for_deletion = \ self.under_test.get_backups_eligible_for_deletion( backups=list(backups), table_reference=reference) #then self.sortAndAssertListEqual(backups_expected_for_deletion, eligible_for_deletion)