def __create_load_job_config(
self, ems_load_job_config: EmsLoadJobConfig) -> LoadJobConfig:
config = LoadJobConfig()
config.labels = ems_load_job_config.labels
config.create_disposition = ems_load_job_config.create_disposition.value
config.write_disposition = ems_load_job_config.write_disposition.value
config.schema = _parse_schema_resource(ems_load_job_config.schema)
config.skip_leading_rows = ems_load_job_config.skip_leading_rows
return config
def schema(self):
"""List[google.cloud.bigquery.schema.SchemaField]: Table's schema.
Raises:
TypeError: If 'value' is not a sequence
ValueError:
If any item in the sequence is not a
:class:`~google.cloud.bigquery.schema.SchemaField`
"""
prop = self._properties.get('schema')
if not prop:
return []
else:
return _parse_schema_resource(prop)
def schema(self):
"""List[google.cloud.bigquery.schema.SchemaField]: Table's schema.
Raises:
TypeError: If 'value' is not a sequence
ValueError:
If any item in the sequence is not a
:class:`~google.cloud.bigquery.schema.SchemaField`
"""
prop = self._properties.get('schema')
if not prop:
return []
else:
return _parse_schema_resource(prop)
def test_get_job_list_returnWithEmsLoadJobIterator(
self, bigquery_module_patch: bigquery):
bigquery_module_patch.Client.return_value = self.client_mock
load_job_mock = Mock(LoadJob)
load_job_mock.job_id = "123"
load_job_mock.query = "SELECT 1"
load_job_mock.state = "DONE"
load_job_mock.write_disposition = None
load_job_mock.create_disposition = None
load_job_mock.error_result = None
load_job_mock.source_uris = ["gs://some-bucket-id/some-blob-id"]
destination = Mock(TableReference)
destination.project = "some-other-project-id"
destination.dataset_id = "some-destination-dataset"
destination.table_id = "some-destination-table"
load_job_mock.destination = destination
expected_schema = {
"fields": [{
"description": None,
"mode": "NULLABLE",
"type": "STRING",
"name": "fruit"
}]
}
load_job_mock.schema = _parse_schema_resource(expected_schema)
self.client_mock.list_jobs.return_value = [load_job_mock]
ems_bigquery_client = EmsBigqueryClient("some-project-id")
job_list_iterable = ems_bigquery_client.get_job_list()
result = list(job_list_iterable)
self.assertEqual(1, len(result))
result_job = result[0]
self.assertIsInstance(result_job, EmsLoadJob)
self.assertEqual(EmsJobState("DONE"), result_job.state)
self.assertEqual("123", result_job.job_id)
self.assertFalse(result_job.is_failed)
self.assertIsInstance(result_job.load_config, EmsLoadJobConfig)
self.assertEqual("some-other-project-id",
result_job.load_config.destination_project_id)
self.assertEqual("some-destination-dataset",
result_job.load_config.destination_dataset)
self.assertEqual("some-destination-table",
result_job.load_config.destination_table)
self.assertEqual(expected_schema, result_job.load_config.schema)
self.assertEqual("gs://some-bucket-id/some-blob-id",
result_job.load_config.source_uri_template)
async def insert_rows(
self,
table: str,
rows,
row_ids=None,
skipInvalidRows=False,
ignoreUnknownValues=False,
templateSuffix=None,
):
ep = EndPoint().insert_all.format(**self.parse_table_name(table))
table_json = await self.get_table(table)
schema = _parse_schema_resource(
table_json["schema"]) # 外部ライブラリーの低レベルメソッドなので危険かも
rows_info = []
json_rows = [_record_field_to_json(schema, row) for row in rows]
for index, row in enumerate(json_rows):
info = {"json": row}
if row_ids is not None:
info["insertId"] = row_ids[index]
else:
info["insertId"] = str(uuid.uuid4())
rows_info.append(info)
params = dict(
rows=rows_info,
skipInvalidRows=skipInvalidRows,
ignoreUnknownValues=ignoreUnknownValues,
)
if templateSuffix is not None:
params["templateSuffix"] = templateSuffix
def error(r):
errors = []
for error in r.json().get("insertErrors", ()):
errors.append({
"index": int(error["index"]),
"errors": error["errors"]
})
if len(errors) > 0:
logger.error(errors)
r = await self.post(endpoint=ep, json=params, call_back=error)
return r.json()
def _set_properties(self, api_response):
"""Update properties from resource in body of ``api_response``
:type api_response: dict
:param api_response: response returned from an API call
"""
self._properties.clear()
cleaned = api_response.copy()
schema = cleaned.pop('schema', {'fields': ()})
self.schema = _parse_schema_resource(schema)
ec = cleaned.pop('externalDataConfiguration', None)
if ec:
self.external_data_configuration = ExternalConfig.from_api_repr(ec)
if 'creationTime' in cleaned:
cleaned['creationTime'] = float(cleaned['creationTime'])
if 'lastModifiedTime' in cleaned:
cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime'])
if 'expirationTime' in cleaned:
cleaned['expirationTime'] = float(cleaned['expirationTime'])
if 'labels' not in cleaned:
cleaned['labels'] = {}
self._properties.update(cleaned)
def _set_properties(self, api_response):
"""Update properties from resource in body of ``api_response``
:type api_response: dict
:param api_response: response returned from an API call
"""
self._properties.clear()
cleaned = api_response.copy()
schema = cleaned.pop('schema', {'fields': ()})
self.schema = _parse_schema_resource(schema)
ec = cleaned.pop('externalDataConfiguration', None)
if ec:
self.external_data_configuration = ExternalConfig.from_api_repr(ec)
if 'creationTime' in cleaned:
cleaned['creationTime'] = float(cleaned['creationTime'])
if 'lastModifiedTime' in cleaned:
cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime'])
if 'expirationTime' in cleaned:
cleaned['expirationTime'] = float(cleaned['expirationTime'])
if 'labels' not in cleaned:
cleaned['labels'] = {}
self._properties.update(cleaned)
def _call_fut(self, resource):
from google.cloud.bigquery.schema import _parse_schema_resource
return _parse_schema_resource(resource)
