Esempio n. 1
0
 def test_get_or_create_dataset_fetched(self):
     client = mock.Mock()
     client.datasets.Get.return_value = bigquery.Dataset(
         datasetReference=bigquery.DatasetReference(projectId='project_id',
                                                    datasetId='dataset_id'))
     wrapper = beam.io.bigquery.BigQueryWrapper(client)
     new_dataset = wrapper.get_or_create_dataset('project_id', 'dataset_id')
     self.assertEqual(new_dataset.datasetReference.datasetId, 'dataset_id')
Esempio n. 2
0
 def test_temporary_dataset_is_unique(self, patched_time_sleep):
     client = mock.Mock()
     client.datasets.Get.return_value = bigquery.Dataset(
         datasetReference=bigquery.DatasetReference(projectId='project_id',
                                                    datasetId='dataset_id'))
     wrapper = beam.io.bigquery.BigQueryWrapper(client)
     with self.assertRaises(RuntimeError) as _:
         wrapper.create_temporary_dataset('project_id')
     self.assertTrue(client.datasets.Get.called)
Esempio n. 3
0
 def test_get_or_create_dataset_created(self):
   client = mock.Mock()
   client.datasets.Get.side_effect = HttpError(
       response={'status': '404'}, url='', content='')
   client.datasets.Insert.return_value = bigquery.Dataset(
       datasetReference=bigquery.DatasetReference(
           projectId='project_id', datasetId='dataset_id'))
   wrapper = beam.io.bigquery.BigQueryWrapper(client)
   new_dataset = wrapper.get_or_create_dataset('project_id', 'dataset_id')
   self.assertEqual(new_dataset.datasetReference.datasetId, 'dataset_id')
Esempio n. 4
0
 def get_or_create_dataset(self, project_id, dataset_id):
   # Check if dataset already exists otherwise create it
   try:
     dataset = self.client.datasets.Get(bigquery.BigqueryDatasetsGetRequest(
         projectId=project_id, datasetId=dataset_id))
     return dataset
   except HttpError as exn:
     if exn.status_code == 404:
       dataset = bigquery.Dataset(
           datasetReference=bigquery.DatasetReference(
               projectId=project_id, datasetId=dataset_id))
       request = bigquery.BigqueryDatasetsInsertRequest(
           projectId=project_id, dataset=dataset)
       response = self.client.datasets.Insert(request)
       # The response is a bigquery.Dataset instance.
       return response
     else:
       raise