def __create_backups_with_part_of_referencing_same_table_in_bq():
backup_1 = Backup(table_id='table_id_1', dataset_id='dataset_id_1')
backup_2 = Backup(table_id='table_id_2', dataset_id='dataset_id_1')
backup_3 = Backup(table_id='table_id_3', dataset_id='dataset_id_1')
backup_4 = Backup(table_id='table_id_3', dataset_id='dataset_id_1')
return [backup_4, backup_2, backup_3, backup_1]
def test_that_get_all_backups_sorted_will_return_only_these_with_null_deleted_column(
self): # nopep8 pylint: disable=C0301, W0613
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup1 = Backup(
parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='backup_dataset',
table_id='backup1',
numBytes=1234,
)
backup1.put()
backup2 = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='backup_dataset',
table_id='backup2',
numBytes=1234,
deleted=datetime(2017, 02, 10, 16, 30))
backup2.put()
# when
existing_backups = Backup.get_all_backups_sorted(table.key)
# then
self.assertTrue(backup1 in existing_backups)
self.assertTrue(backup2 not in existing_backups)
def __create_valid_backups():
backup_1 = Backup(table_id='table_id_1', dataset_id='dataset_id_1')
backup_2 = Backup(table_id='table_id_2', dataset_id='dataset_id_1')
backup_3 = Backup(table_id='table_id_3', dataset_id='dataset_id_1')
backup_4 = Backup(table_id='table_id_4', dataset_id='dataset_id_1')
return [backup_1, backup_2, backup_3, backup_4]
def test_should_return_the_same_parent_table_for_child_backups(self):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup_one = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup_two = Backup(parent=table.key,
last_modified=datetime(2018, 03, 2, 00, 00),
created=datetime(2018, 03, 2, 00, 00),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
# when
table1 = Backup.get_table(backup_one)
table2 = Backup.get_table(backup_two)
# then
self.assertEqual(table, table1)
self.assertEqual(table1, table2)
def test_should_not_insert_two_backup_entities_for_the_same_backup_table(
self): # nopep8 pylint: disable=C0301
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup_one = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup_two = Backup(parent=table.key,
last_modified=datetime(2018, 03, 2, 00, 00),
created=datetime(2018, 03, 2, 00, 00),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
# when
Backup.insert_if_absent(backup_one)
Backup.insert_if_absent(backup_two)
backups = list(Backup.get_all())
# then
self.assertEqual(len(backups), 1)
self.assertEqual(backup_one.created, backups[0].created)
def create_backup(created_datetime, table=Table(), table_id=None):
if not table_id:
table_id = 'targetTable' + str(random.randint(1, 1000000))
backup_size = random.randint(1, 1000)
backup = Backup(parent=table.key,
created=created_datetime,
dataset_id='targetDataset',
table_id=table_id,
numBytes=backup_size)
return backup
def test_should_return_true_if_table_was_changed_after_last_backup(self, _):
# given
backup = Backup(
parent=self.table.key,
last_modified=datetime(2016, 11, 13, 15, 00)
)
backup.put()
predicate = OnDemandBackupPredicate()
# when
result = predicate.test(self.big_query_table_metadata, self.table)
# then
self.assertTrue(result, "OnDemandShouldBackupPredicate should return TRUE "
"if table was changed after last backup")
def test_should_return_true_if_table_was_changed_before_last_backup(self, _): # nopep8 pylint: disable=C0301
# given
backup = Backup(
parent=self.table.key,
last_modified=datetime(2016, 11, 13, 15, 00)
)
backup.put()
predicate = OnDemandBackupPredicate()
# when
result = predicate.test(self.big_query_table_metadata, self.table)
# then
self.assertTrue(result, "OnDemandShouldBackupPredicate should return FALSE "
"if table was changed before "
"last backup was made")
def test_should_return_false_if_table_was_changed_at_the_same_time_when_last_backup(
self, _1, _2): # nopep8 pylint: disable=C0301
# given
backup = Backup(parent=self.table.key,
last_modified=datetime(2016, 11, 13, 15, 00),
numBytes=123)
backup.put()
predicate = DefaultBackupPredicate()
# when
result = predicate.test(self.big_query_table_metadata, self.table)
# then
self.assertFalse(
result, "ShouldBackupPredicate should return False "
"if table was change at the same time when "
"last backup was made")
def test_should_return_true_if_changed_table_is_empty_and_last_backup_is_also_empty(
self, _1, _2): # nopep8 pylint: disable=C0301
# given
backup = Backup(parent=self.table.key,
last_modified=datetime(2016, 11, 13, 15, 00),
numBytes=0)
backup.put()
predicate = DefaultBackupPredicate()
# when
result = predicate.test(self.big_query_table_metadata, self.table)
# then
self.assertTrue(
result, "ShouldBackupPredicate should return True "
"if table was changed after last backup was made,"
"but source table is empty and bbq has also empty last backup")
def test_should_return_true_if_table_was_changed_before_last_backup_but_table_size_is_different(
self, _1, _2): # nopep8 pylint: disable=C0301
# given
backup = Backup(parent=self.table.key,
last_modified=datetime(2016, 11, 13, 15, 00),
numBytes=123)
backup.put()
predicate = DefaultBackupPredicate()
# when
result = predicate.test(self.big_query_table_metadata, self.table)
# then
self.assertTrue(
result, "ShouldBackupPredicate should return TRUE "
"if table was changed before "
"last backup was made but "
"backup has different size than source table")
def test_should_retrieve_table_using_backup(self):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup.put()
# then
backup_entity = Backup.get_by_key(backup.key)
table_entity = Table.get_table_from_backup(backup_entity)
self.assertEqual(table_entity, table)
def test_that_not_deleted_backup_doesnt_have_created_field(self):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
backup = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup.put()
# then
backup_to_check = Backup.get_by_key(backup.key)
self.assertEqual(backup_to_check.deleted, None)
self.assertEqual(backup_to_check.created,
datetime(2017, 02, 1, 16, 30))
def test_deleting_backup_is_adding_current_timestamp_in_deleted_field(
self):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
backup = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup.put()
# when
Backup.mark_backup_deleted(backup.key)
# then
deleted_backup = Backup.get_by_key(backup.key)
self.assertEqual(deleted_backup.deleted, datetime(2017, 02, 3, 16, 30))
def __create_backup(backup_table_metadata, copy_job_results):
table_entity = Table.get_table_by_reference(
TableReference.from_bq_table(copy_job_results.source_bq_table))
if table_entity is None:
raise DatastoreTableGetRetriableException()
backup = Backup(parent=table_entity.key,
last_modified=copy_job_results.start_time,
created=copy_job_results.end_time,
dataset_id=copy_job_results.target_dataset_id,
table_id=copy_job_results.target_table_id,
numBytes=backup_table_metadata.table_size_in_bytes())
logging.debug(
"Saving backup to datastore, source:{0}, target:{1}".format(
copy_job_results.source_bq_table,
copy_job_results.target_bq_table))
backup.insert_if_absent(backup)
def __create_backup_entity(source_table_entity, dataset_id, table_id):
return Backup(dataset_id=dataset_id,
table_id=table_id,
parent=source_table_entity.key)