def __create_table_without_backup(project_id, dataset_id):
table_without_backup = Table(project_id=project_id,
dataset_id=dataset_id,
table_id='table_id_without_backup',
partition_id=None,
last_checked=datetime.now())
table_without_backup.put()
def test_should_return_the_same_parent_table_for_child_backups(self):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup_one = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup_two = Backup(parent=table.key,
last_modified=datetime(2018, 03, 2, 00, 00),
created=datetime(2018, 03, 2, 00, 00),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
# when
table1 = Backup.get_table(backup_one)
table2 = Backup.get_table(backup_two)
# then
self.assertEqual(table, table1)
self.assertEqual(table1, table2)
def test_should_disable_partition_expiration_if_backup_table_has_it(
self, disable_partition_expiration, _, _1, _2, _3, _4, _5):
# given
table_entity = Table(project_id="source_project_id",
dataset_id="source_dataset_id",
table_id="source_table_id",
partition_id="123")
table_entity.put()
source_bq_table = TableReference.from_table_entity(
table_entity).create_big_query_table()
destination_bq_table = BigQueryTable("target_project_id",
"target_dataset_id",
"target_table_id")
data = {
"sourceBqTable": source_bq_table,
"targetBqTable": destination_bq_table
}
payload = json.dumps({
"data": data,
"jobJson": JobResultExample.DONE
},
cls=RequestEncoder)
# when
response = self.under_test.post(
'/callback/backup-created/project/dataset/table', params=payload)
# then
self.assertEqual(response.status_int, 200)
disable_partition_expiration.assert_called_once()
def test_that_get_all_backups_sorted_will_return_only_these_with_null_deleted_column(
self): # nopep8 pylint: disable=C0301, W0613
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup1 = Backup(
parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='backup_dataset',
table_id='backup1',
numBytes=1234,
)
backup1.put()
backup2 = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='backup_dataset',
table_id='backup2',
numBytes=1234,
deleted=datetime(2017, 02, 10, 16, 30))
backup2.put()
# when
existing_backups = Backup.get_all_backups_sorted(table.key)
# then
self.assertTrue(backup1 in existing_backups)
self.assertTrue(backup2 not in existing_backups)
def test_that_last_checked_date_is_updated_even_if_table_should_not_be_backed_up( # nopep8 pylint: disable=C0301
self, copy_table, _1, _2):
# given
table = Table(project_id="test-project",
dataset_id="test-dataset",
table_id="test-table",
last_checked=datetime.datetime(2017, 3, 3))
table_reference = TableReference(project_id="test-project",
dataset_id="test-dataset",
table_id="test-table")
# when
table.put()
BackupProcess(table_reference, self.big_query,
self.big_query_table_metadata).start()
table_entity = Table.get_table("test-project", "test-dataset",
"test-table")
# then
self.assertEqual(table_entity.last_checked,
datetime.datetime(2017, 04, 4))
copy_table.assert_not_called()
def test_should_fill_deleted_field_in_backup_entity_if_table_not_found_error_during_deletion(
self, _):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime.datetime(2017, 2, 1, 16, 30))
table.put()
reference = TableReference.from_table_entity(table)
backup1 = backup_utils.create_backup(datetime.datetime(
2017, 2, 1, 16, 30),
table,
table_id="backup1")
backup2 = backup_utils.create_backup(datetime.datetime(
2017, 2, 2, 16, 30),
table,
table_id="backup2")
ndb.put_multi([backup1, backup2])
self.policy.get_backups_eligible_for_deletion = Mock(
return_value=[backup1, backup2])
# when
self.under_test.perform_retention(reference, table.key.urlsafe())
# then
self.assertTrue(Backup.get_by_key(backup1.key).deleted is not None)
self.assertTrue(Backup.get_by_key(backup2.key).deleted is not None)
def test_should_not_insert_two_backup_entities_for_the_same_backup_table(
self): # nopep8 pylint: disable=C0301
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup_one = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup_two = Backup(parent=table.key,
last_modified=datetime(2018, 03, 2, 00, 00),
created=datetime(2018, 03, 2, 00, 00),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
# when
Backup.insert_if_absent(backup_one)
Backup.insert_if_absent(backup_two)
backups = list(Backup.get_all())
# then
self.assertEqual(len(backups), 1)
self.assertEqual(backup_one.created, backups[0].created)
def __create_table_without_backups():
partition_id = "partitionIdWithoutBackup"
table = Table(project_id=PROJECT_ID,
dataset_id=DATASET_ID,
table_id=TABLE_ID,
partition_id=partition_id,
last_checked=NOW)
table.put()
def _create_table_entity(table_id,
partition_id=None,
last_checked=datetime.datetime.now()):
non_partitioned_table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id=table_id,
partition_id=partition_id,
last_checked=last_checked)
non_partitioned_table.put()
def __create_table_with_two_backups():
table = Table(project_id=PROJECT_ID,
dataset_id=DATASET_ID,
table_id=TABLE_ID,
partition_id=PARTITION_ID,
last_checked=NOW)
table.put()
backup_utils.create_backup(NOW, table, BACKUP_TABLE_ID_FROM_NOW).put()
backup_utils.create_backup(OLD_TIME, table,
BACKUP_TABLE_ID_FROM_OLD_TIME).put()
def __create_table_entity(self):
logging.info(
"Creating table entity for %s",
TableReference(self.project_id, self.dataset_id, self.table_id,
self.partition_id))
table_entity = Table(project_id=self.project_id,
dataset_id=self.dataset_id,
table_id=self.table_id,
partition_id=self.partition_id,
last_checked=self.now)
table_entity.put()
return table_entity
def test_should_not_perform_retention_if_no_backups(self, delete_table):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime.datetime(2017, 2, 1, 16, 30))
table.put()
reference = TableReference.from_table_entity(table)
# when
self.under_test.perform_retention(reference, table.key.urlsafe())
# then
delete_table.assert_not_called()
def test_should_create_datastore_backup_entity(self, _create_http, _):
# given
_create_http.return_value = HttpMockSequence([
({
'status': '200'
}, content('tests/json_samples/bigquery_v2_test_schema.json')),
({
'status': '200'
},
content('tests/json_samples/table_get/'
'bigquery_partitioned_table_get.json'))
])
table_entity = Table(project_id="source_project_id",
dataset_id="source_dataset_id",
table_id="source_table_id",
partition_id="123")
table_entity.put()
source_bq_table = TableReference.from_table_entity(
table_entity).create_big_query_table()
destination_bq_table = BigQueryTable("target_project_id",
"target_dataset_id",
"target_table_id")
data = {
"sourceBqTable": source_bq_table,
"targetBqTable": destination_bq_table
}
payload = json.dumps({
"data": data,
"jobJson": JobResultExample.DONE
},
cls=RequestEncoder)
copy_job_result = CopyJobResult(json.loads(payload).get('jobJson'))
# when
response = self.under_test.post(
'/callback/backup-created/project/dataset/table', params=payload)
backup = table_entity.last_backup
# then
self.assertEqual(response.status_int, 200)
self.assertEqual(backup.dataset_id, "target_dataset_id")
self.assertEqual(backup.table_id, "target_table_id")
self.assertTrue(isinstance(backup.created, datetime))
self.assertEqual(backup.created, copy_job_result.end_time)
self.assertTrue(isinstance(backup.last_modified, datetime))
self.assertEqual(backup.last_modified, copy_job_result.start_time)
def create_and_insert_table_with_one_backup(project_id,
dataset_id,
table_id,
date,
partition_id=None):
table = Table(project_id=project_id,
dataset_id=dataset_id,
table_id=table_id,
partition_id=partition_id,
last_checked=date)
table.put()
backup_utils.create_backup(date, table,
table_id + date.strftime('%Y%m%d')).put()
return table
def test_should_not_create_backups_entity_if_backup_table_doesnt_exist(
self, _create_http, error_reporting, _):
# given
_create_http.return_value = HttpMockSequence([
({
'status': '200'
}, content('tests/json_samples/bigquery_v2_test_schema.json')),
(
{
'status': '404'
}, # Table not found
content('tests/json_samples/table_get/'
'bigquery_partitioned_table_get.json'))
])
table_entity = Table(project_id="source_project_id",
dataset_id="source_dataset_id",
table_id="source_table_id",
partition_id="123")
table_entity.put()
source_bq_table = TableReference.from_table_entity(
table_entity).create_big_query_table()
destination_bq_table = BigQueryTable("target_project_id",
"target_dataset_id",
"target_table_id")
data = {
"sourceBqTable": source_bq_table,
"targetBqTable": destination_bq_table
}
payload = json.dumps({
"data": data,
"jobJson": JobResultExample.DONE
},
cls=RequestEncoder)
# when
response = self.under_test.post(
'/callback/backup-created/project/dataset/table', params=payload)
backup = table_entity.last_backup
# then
self.assertEqual(response.status_int, 200)
self.assertIsNone(backup)
error_reporting.assert_called_once()
def test_should_retrieve_table_using_backup(self):
# given
table = Table(project_id='example-proj-name',
dataset_id='example-dataset-name',
table_id='example-table-name',
last_checked=datetime(2017, 02, 1, 16, 30))
table.put()
backup = Backup(parent=table.key,
last_modified=datetime(2017, 02, 1, 16, 30),
created=datetime(2017, 02, 1, 16, 30),
dataset_id='targetDatasetId',
table_id='targetTableId',
numBytes=1234)
backup.put()
# then
backup_entity = Backup.get_by_key(backup.key)
table_entity = Table.get_table_from_backup(backup_entity)
self.assertEqual(table_entity, table)
def test_that_last_checked_date_is_updated_when_backup_is_processed(
self, _, _1, _2):
# given
table = Table(project_id="test-project",
dataset_id="test-dataset",
table_id="test-table",
last_checked=datetime.datetime(2017, 3, 3))
table_reference = TableReference(project_id="test-project",
dataset_id="test-dataset",
table_id="test-table")
# when
table.put()
BackupProcess(table_reference, self.big_query,
self.big_query_table_metadata).start()
table_entity = Table.get_table("test-project", "test-dataset",
"test-table")
# then
self.assertEqual(table_entity.last_checked,
datetime.datetime(2017, 04, 4))