def __create_load_job_config( self, ems_load_job_config: EmsLoadJobConfig) -> LoadJobConfig: config = LoadJobConfig() config.labels = ems_load_job_config.labels config.create_disposition = ems_load_job_config.create_disposition.value config.write_disposition = ems_load_job_config.write_disposition.value config.schema = _parse_schema_resource(ems_load_job_config.schema) config.skip_leading_rows = ems_load_job_config.skip_leading_rows return config
def schema(self): """List[google.cloud.bigquery.schema.SchemaField]: Table's schema. Raises: TypeError: If 'value' is not a sequence ValueError: If any item in the sequence is not a :class:`~google.cloud.bigquery.schema.SchemaField` """ prop = self._properties.get('schema') if not prop: return [] else: return _parse_schema_resource(prop)
def schema(self): """List[google.cloud.bigquery.schema.SchemaField]: Table's schema. Raises: TypeError: If 'value' is not a sequence ValueError: If any item in the sequence is not a :class:`~google.cloud.bigquery.schema.SchemaField` """ prop = self._properties.get('schema') if not prop: return [] else: return _parse_schema_resource(prop)
def test_get_job_list_returnWithEmsLoadJobIterator( self, bigquery_module_patch: bigquery): bigquery_module_patch.Client.return_value = self.client_mock load_job_mock = Mock(LoadJob) load_job_mock.job_id = "123" load_job_mock.query = "SELECT 1" load_job_mock.state = "DONE" load_job_mock.write_disposition = None load_job_mock.create_disposition = None load_job_mock.error_result = None load_job_mock.source_uris = ["gs://some-bucket-id/some-blob-id"] destination = Mock(TableReference) destination.project = "some-other-project-id" destination.dataset_id = "some-destination-dataset" destination.table_id = "some-destination-table" load_job_mock.destination = destination expected_schema = { "fields": [{ "description": None, "mode": "NULLABLE", "type": "STRING", "name": "fruit" }] } load_job_mock.schema = _parse_schema_resource(expected_schema) self.client_mock.list_jobs.return_value = [load_job_mock] ems_bigquery_client = EmsBigqueryClient("some-project-id") job_list_iterable = ems_bigquery_client.get_job_list() result = list(job_list_iterable) self.assertEqual(1, len(result)) result_job = result[0] self.assertIsInstance(result_job, EmsLoadJob) self.assertEqual(EmsJobState("DONE"), result_job.state) self.assertEqual("123", result_job.job_id) self.assertFalse(result_job.is_failed) self.assertIsInstance(result_job.load_config, EmsLoadJobConfig) self.assertEqual("some-other-project-id", result_job.load_config.destination_project_id) self.assertEqual("some-destination-dataset", result_job.load_config.destination_dataset) self.assertEqual("some-destination-table", result_job.load_config.destination_table) self.assertEqual(expected_schema, result_job.load_config.schema) self.assertEqual("gs://some-bucket-id/some-blob-id", result_job.load_config.source_uri_template)
async def insert_rows( self, table: str, rows, row_ids=None, skipInvalidRows=False, ignoreUnknownValues=False, templateSuffix=None, ): ep = EndPoint().insert_all.format(**self.parse_table_name(table)) table_json = await self.get_table(table) schema = _parse_schema_resource( table_json["schema"]) # 外部ライブラリーの低レベルメソッドなので危険かも rows_info = [] json_rows = [_record_field_to_json(schema, row) for row in rows] for index, row in enumerate(json_rows): info = {"json": row} if row_ids is not None: info["insertId"] = row_ids[index] else: info["insertId"] = str(uuid.uuid4()) rows_info.append(info) params = dict( rows=rows_info, skipInvalidRows=skipInvalidRows, ignoreUnknownValues=ignoreUnknownValues, ) if templateSuffix is not None: params["templateSuffix"] = templateSuffix def error(r): errors = [] for error in r.json().get("insertErrors", ()): errors.append({ "index": int(error["index"]), "errors": error["errors"] }) if len(errors) > 0: logger.error(errors) r = await self.post(endpoint=ep, json=params, call_back=error) return r.json()
def _set_properties(self, api_response): """Update properties from resource in body of ``api_response`` :type api_response: dict :param api_response: response returned from an API call """ self._properties.clear() cleaned = api_response.copy() schema = cleaned.pop('schema', {'fields': ()}) self.schema = _parse_schema_resource(schema) ec = cleaned.pop('externalDataConfiguration', None) if ec: self.external_data_configuration = ExternalConfig.from_api_repr(ec) if 'creationTime' in cleaned: cleaned['creationTime'] = float(cleaned['creationTime']) if 'lastModifiedTime' in cleaned: cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime']) if 'expirationTime' in cleaned: cleaned['expirationTime'] = float(cleaned['expirationTime']) if 'labels' not in cleaned: cleaned['labels'] = {} self._properties.update(cleaned)
def _set_properties(self, api_response): """Update properties from resource in body of ``api_response`` :type api_response: dict :param api_response: response returned from an API call """ self._properties.clear() cleaned = api_response.copy() schema = cleaned.pop('schema', {'fields': ()}) self.schema = _parse_schema_resource(schema) ec = cleaned.pop('externalDataConfiguration', None) if ec: self.external_data_configuration = ExternalConfig.from_api_repr(ec) if 'creationTime' in cleaned: cleaned['creationTime'] = float(cleaned['creationTime']) if 'lastModifiedTime' in cleaned: cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime']) if 'expirationTime' in cleaned: cleaned['expirationTime'] = float(cleaned['expirationTime']) if 'labels' not in cleaned: cleaned['labels'] = {} self._properties.update(cleaned)
def _call_fut(self, resource): from google.cloud.bigquery.schema import _parse_schema_resource return _parse_schema_resource(resource)