def test_iterating_tables(self, _): # given self._create_http.return_value = self.__create_tables_list_responses() bq = BigQuery() # when tables_ids, next_page_token = bq.list_table_ids( "project1233", "dataset_id") # then self.assertEqual(self.count(tables_ids), 4) self.assertEqual(next_page_token, 'table_id_5') # when tables_ids, next_page_token = bq.list_table_ids( "project1233", "dataset_id", page_token=next_page_token) # then self.assertEqual(self.count(tables_ids), 1) self.assertEqual(next_page_token, None)
class DatasetBackupScheduler(object): def __init__(self): self.big_query = BigQuery() def schedule_backup(self, project_id, dataset_id, page_token=None): table_ids_to_backup, next_page_token = self.big_query.list_table_ids( project_id=project_id, dataset_id=dataset_id, page_token=page_token) self._schedule_tasks_for_tables_backup(project_id, dataset_id, table_ids_to_backup) if next_page_token: logging.info( u'Scheduling Dataset Backup Scheduler task for %s:%s, page_token: %s', project_id, dataset_id, page_token) Tasks.schedule( 'backup-scheduler', TaskCreator.create_dataset_backup_scheduler_task( project_id=project_id, dataset_id=dataset_id, page_token=next_page_token)) @staticmethod def _schedule_tasks_for_tables_backup(project_id, dataset_id, table_ids): logging.info( u'Scheduling Table Backup tasks for %s %s:%s dataset tables: %s', len(table_ids), project_id, dataset_id, table_ids) tasks = [] for table_id in table_ids: tasks.append( TaskCreator.create_table_backup_task(project_id=project_id, dataset_id=dataset_id, table_id=table_id)) Tasks.schedule('backup-worker', tasks)