def test_that_last_checked_date_is_updated_even_if_table_should_not_be_backed_up( # nopep8 pylint: disable=C0301 self, copy_table, _1, _2): # given table = Table(project_id="test-project", dataset_id="test-dataset", table_id="test-table", last_checked=datetime.datetime(2017, 3, 3)) table_reference = TableReference(project_id="test-project", dataset_id="test-dataset", table_id="test-table") # when table.put() BackupProcess(table_reference, self.big_query, self.big_query_table_metadata).start() table_entity = Table.get_table("test-project", "test-dataset", "test-table") # then self.assertEqual(table_entity.last_checked, datetime.datetime(2017, 04, 4)) copy_table.assert_not_called()
def start(table_reference): big_query_table_metadata = BigQueryTableMetadata.get_table_by_reference( table_reference) BackupProcess( table_reference=table_reference, big_query=BigQuery(), big_query_table_metadata=big_query_table_metadata, should_backup_predicate=OnDemandBackupPredicate()).start()
def test_that_dataset_will_not_be_unnecessary_created_twice(self, _, _1, _2, _3): # given table_reference_1 = TableReference(project_id="test-project", dataset_id="test-dataset", table_id="test-table-1") table_reference_2 = TableReference(project_id="test-project", dataset_id="test-dataset", table_id="test-table-2") # when self.big_query.create_dataset = MagicMock() BackupProcess(table_reference_1, self.big_query, self.big_query_table_metadata).start() BackupProcess(table_reference_2, self.big_query, self.big_query_table_metadata).start() # then self.big_query.create_dataset.assert_called_once()
def start(table_reference): big_query_table_metadata = BigQueryTableMetadata.get_table_by_reference( table_reference) if big_query_table_metadata.is_daily_partitioned( ) and not big_query_table_metadata.is_partition(): raise ParameterValidationException( "Partition id is required for partitioned table in on-demand mode" ) BackupProcess( table_reference=table_reference, big_query=BigQuery(), big_query_table_metadata=big_query_table_metadata, should_backup_predicate=OnDemandBackupPredicate()).start()
def test_copy_job_and_entity_in_datastore_for_not_partitioned_table( self, _, _1, _2): # given table_reference = TableReference(project_id="test-project", dataset_id="test-dataset", table_id="test-table") # when BackupProcess(table_reference, self.big_query, self.big_query_table_metadata).start() table_entity = Table.get_table("test-project", "test-dataset", "test-table") # then self.assertIsNotNone(table_entity)
def test_that_copy_job_and_entity_in_datastore_is_created_if_empty_partitioned_table( # nopep8 pylint: disable=C0301 self, create_backup, _, _1): # given table_reference = TableReference(project_id="test-project", dataset_id="test-dataset", table_id="test-table", partition_id=None) # when BackupProcess(table_reference, self.big_query, self.big_query_table_metadata).start() table_in_datastore = Table.get_table("test-project", "test-dataset", "test-table") # then create_backup.assert_called_once() self.assertIsNotNone(table_in_datastore)
def start(table_reference): big_query = BigQuery() big_query_table_metadata = BigQueryTableMetadata.get_table_by_reference( table_reference) if big_query_table_metadata.is_daily_partitioned() and \ not big_query_table_metadata.is_partition(): logging.info('Table (%s/%s/%s) is partitioned', table_reference.get_project_id(), table_reference.get_dataset_id(), table_reference.get_table_id()) TablePartitionsBackupScheduler(table_reference, big_query).start() else: BackupProcess( table_reference=table_reference, big_query=big_query, big_query_table_metadata=big_query_table_metadata).start()
def start(table_reference): big_query = BigQuery() big_query_table_metadata = BigQueryTableMetadata.get_table_by_reference( table_reference) if big_query_table_metadata.is_daily_partitioned() and \ not big_query_table_metadata.is_partition(): logging.info(u'Table %s:%s.%s is partitioned', table_reference.get_project_id(), table_reference.get_dataset_id(), table_reference.get_table_id()) TableBackup._schedule_partitioned_table_backup_scheduler_task( table_reference) else: BackupProcess( table_reference=table_reference, big_query=big_query, big_query_table_metadata=big_query_table_metadata).start()
def test_copy_job_and_entity_in_datastore_for_single_partition_of_a_table( self, _, _1, _2): # given table_reference = TableReference(project_id="test-project", dataset_id="test-dataset", table_id="test-table", partition_id="20170330") # when BackupProcess(table_reference, self.big_query, self.big_query_table_metadata).start() ancestor_of_partition = Table.get_table("test-project", "test-dataset", "test-table") partition = Table.get_table("test-project", "test-dataset", "test-table", "20170330") # then self.assertIsNotNone(partition) self.assertIsNone(ancestor_of_partition)
def test_that_last_checked_date_is_updated_when_backup_is_processed( self, _, _1, _2): # given table = Table(project_id="test-project", dataset_id="test-dataset", table_id="test-table", last_checked=datetime.datetime(2017, 3, 3)) table_reference = TableReference(project_id="test-project", dataset_id="test-dataset", table_id="test-table") # when table.put() BackupProcess(table_reference, self.big_query, self.big_query_table_metadata).start() table_entity = Table.get_table("test-project", "test-dataset", "test-table") # then self.assertEqual(table_entity.last_checked, datetime.datetime(2017, 04, 4))