def test_schema_hit(self): from google.cloud.bigquery.schema import SchemaField config = self._get_target_class()() all_props_repr = { "mode": "REQUIRED", "name": "foo", "type": "INTEGER", "description": "Foo", } minimal_repr = {"name": "bar", "type": "STRING"} config._properties["load"]["schema"] = { "fields": [all_props_repr, minimal_repr] } all_props, minimal = config.schema self.assertEqual(all_props, SchemaField.from_api_repr(all_props_repr)) self.assertEqual(minimal, SchemaField.from_api_repr(minimal_repr))
def schema(self): """List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema for the data. See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.schema """ prop = self._properties.get("schema", {}) return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])]
def schema(self): """List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema for the data. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).schema https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externalDataConfiguration.schema """ prop = self._properties.get("schema", {}) return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])]
def schema(self): """List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema for the data. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).schema https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externalDataConfiguration.schema """ prop = self._properties.get('schema', {}) return [SchemaField.from_api_repr(field) for field in prop.get('fields', [])]
def schema(self): """Optional[Sequence[Union[ \ :class:`~google.cloud.bigquery.schema.SchemaField`, \ Mapping[str, Any] \ ]]]: Schema of the destination table. See: https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.schema """ schema = _helpers._get_sub_prop(self._properties, ["load", "schema", "fields"]) if schema is None: return return [SchemaField.from_api_repr(field) for field in schema]
def test_from_api_repr_parameterized(api, expect, key2): from google.cloud.bigquery.schema import SchemaField field = SchemaField.from_api_repr(api) assert ( field.name, field.field_type, field.precision, field.scale, field.max_length, ) == expect assert field._key()[:2] == key2
def _get_transitive_schema_fields(fields): """ Recurse into record type and return all the nested field names. As contributed by @sumedhsakdeo on issue #17 """ results = [] for field in fields: results += [field] if field.field_type in STRUCT_FIELD_TYPES: sub_fields = [ SchemaField.from_api_repr( dict(f.to_api_repr(), name=f"{field.name}.{f.name}")) for f in field.fields ] results += _get_transitive_schema_fields(sub_fields) return results
def _get_columns_helper(self, columns, cur_columns): """ Recurse into record type and return all the nested field names. As contributed by @sumedhsakdeo on issue #17 """ results = [] for col in columns: results += [col] if col.field_type == "RECORD": cur_columns.append(col) fields = [ SchemaField.from_api_repr( dict(f.to_api_repr(), name=f"{col.name}.{f.name}")) for f in col.fields ] results += self._get_columns_helper(fields, cur_columns) cur_columns.pop() return results
def to_schema_field_list( from_: Union[BaseResourceLoader, str, List[SchemaField]] ) -> List[SchemaField]: """Transform a schema stored in a file, string or a list of SchemaField into a list of SchemaField Args: from_ (Union[BaseResourceLoader, str, List[SchemaField]]): BigQuery schema Raises: InvalidInstanceException: not a BaseResourceLoader, str or List[SchemaField] Returns: List[SchemaField]: Schema fields used by the BigQuery API """ schema_fields = None if not isinstance(from_, (BaseResourceLoader, str, list)): raise InvalidInstanceException( from_, expected_list_instances=[SchemaField], expected_instances=[BaseResourceLoader, str]) if isinstance(from_, list) and any( [not isinstance(el, SchemaField) for el in from_]): raise InvalidInstanceException( from_, expected_list_instances=[SchemaField], expected_instances=[BaseResourceLoader, str]) if isinstance(from_, (BaseResourceLoader, str)): try: json_schema_str = from_ if isinstance(from_, str) else from_.load() json_schema = json.loads(json_schema_str) schema_fields = [ SchemaField.from_api_repr(field) for field in json_schema ] except Exception: logger.error("Failed to load schema with %s.", from_) raise else: logger.info("Schema loaded successfully with %s.", from_) else: schema_fields = from_ return schema_fields
def get_table_schema(source_schema_file: str) -> List: with open(source_schema_file) as json_file: data = json.load(json_file) schema = [SchemaField.from_api_repr(json_field) for json_field in data] return schema
def get_table_schema(self, schema_file): with open(schema_file) as json_file: data = json.load(json_file) schema = [SchemaField.from_api_repr(json_field) for json_field in data] return schema