def test_schema_hit(self):
from google.cloud.bigquery.schema import SchemaField
config = self._get_target_class()()
all_props_repr = {
"mode": "REQUIRED",
"name": "foo",
"type": "INTEGER",
"description": "Foo",
}
minimal_repr = {"name": "bar", "type": "STRING"}
config._properties["load"]["schema"] = {
"fields": [all_props_repr, minimal_repr]
}
all_props, minimal = config.schema
self.assertEqual(all_props, SchemaField.from_api_repr(all_props_repr))
self.assertEqual(minimal, SchemaField.from_api_repr(minimal_repr))
def schema(self):
"""List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema
for the data.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.schema
"""
prop = self._properties.get("schema", {})
return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])]
def schema(self):
"""List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema
for the data.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).schema
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externalDataConfiguration.schema
"""
prop = self._properties.get("schema", {})
return [SchemaField.from_api_repr(field) for field in prop.get("fields", [])]
def schema(self):
"""List[:class:`~google.cloud.bigquery.schema.SchemaField`]: The schema
for the data.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).schema
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#externalDataConfiguration.schema
"""
prop = self._properties.get('schema', {})
return [SchemaField.from_api_repr(field)
for field in prop.get('fields', [])]
def schema(self):
"""Optional[Sequence[Union[ \
:class:`~google.cloud.bigquery.schema.SchemaField`, \
Mapping[str, Any] \
]]]: Schema of the destination table.
See:
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.schema
"""
schema = _helpers._get_sub_prop(self._properties, ["load", "schema", "fields"])
if schema is None:
return
return [SchemaField.from_api_repr(field) for field in schema]
def test_from_api_repr_parameterized(api, expect, key2):
from google.cloud.bigquery.schema import SchemaField
field = SchemaField.from_api_repr(api)
assert (
field.name,
field.field_type,
field.precision,
field.scale,
field.max_length,
) == expect
assert field._key()[:2] == key2
def _get_transitive_schema_fields(fields):
"""
Recurse into record type and return all the nested field names.
As contributed by @sumedhsakdeo on issue #17
"""
results = []
for field in fields:
results += [field]
if field.field_type in STRUCT_FIELD_TYPES:
sub_fields = [
SchemaField.from_api_repr(
dict(f.to_api_repr(), name=f"{field.name}.{f.name}"))
for f in field.fields
]
results += _get_transitive_schema_fields(sub_fields)
return results
def _get_columns_helper(self, columns, cur_columns):
"""
Recurse into record type and return all the nested field names.
As contributed by @sumedhsakdeo on issue #17
"""
results = []
for col in columns:
results += [col]
if col.field_type == "RECORD":
cur_columns.append(col)
fields = [
SchemaField.from_api_repr(
dict(f.to_api_repr(), name=f"{col.name}.{f.name}"))
for f in col.fields
]
results += self._get_columns_helper(fields, cur_columns)
cur_columns.pop()
return results
def to_schema_field_list(
from_: Union[BaseResourceLoader, str, List[SchemaField]]
) -> List[SchemaField]:
"""Transform a schema stored in a file, string or a list of SchemaField into a list of SchemaField
Args:
from_ (Union[BaseResourceLoader, str, List[SchemaField]]): BigQuery schema
Raises:
InvalidInstanceException: not a BaseResourceLoader, str or List[SchemaField]
Returns:
List[SchemaField]: Schema fields used by the BigQuery API
"""
schema_fields = None
if not isinstance(from_, (BaseResourceLoader, str, list)):
raise InvalidInstanceException(
from_,
expected_list_instances=[SchemaField],
expected_instances=[BaseResourceLoader, str])
if isinstance(from_, list) and any(
[not isinstance(el, SchemaField) for el in from_]):
raise InvalidInstanceException(
from_,
expected_list_instances=[SchemaField],
expected_instances=[BaseResourceLoader, str])
if isinstance(from_, (BaseResourceLoader, str)):
try:
json_schema_str = from_ if isinstance(from_,
str) else from_.load()
json_schema = json.loads(json_schema_str)
schema_fields = [
SchemaField.from_api_repr(field) for field in json_schema
]
except Exception:
logger.error("Failed to load schema with %s.", from_)
raise
else:
logger.info("Schema loaded successfully with %s.", from_)
else:
schema_fields = from_
return schema_fields
def get_table_schema(source_schema_file: str) -> List:
with open(source_schema_file) as json_file:
data = json.load(json_file)
schema = [SchemaField.from_api_repr(json_field) for json_field in data]
return schema
def get_table_schema(self, schema_file):
with open(schema_file) as json_file:
data = json.load(json_file)
schema = [SchemaField.from_api_repr(json_field) for json_field in data]
return schema
