def test_compute_compact_columns():
schema = [
{
"name": "leaf",
"mode": "NULLABLE",
"type": "INT64"
},
{
"name": "repeated",
"mode": "REPEATED",
"type": "INT64"
},
{
"name": "nested",
"mode": "NULLABLE",
"type": "RECORD",
"fields": [{
"name": "leaf",
"type": "INT64",
"mode": "NULLABLE"
}],
},
{
"name": "repeated_nested",
"mode": "REPEATED",
"type": "RECORD",
"fields": [{
"name": "leaf",
"type": "INT64",
"mode": "NULLABLE"
}],
},
]
expected = sorted([
"root.leaf INT64",
"root.nested.leaf INT64",
"root.repeated.[] INT64",
"root.repeated_nested.[].leaf INT64",
])
output = compute_compact_columns(schema)
assert output == expected
def write_compact(path: Path):
for p in path.glob("*.bq"):
out = p.parent / p.name.replace(".bq", ".txt")
bq_schema = json.loads(p.read_text())
out.write_text("\n".join(compute_compact_columns(bq_schema)))
def columns(source):
"""Generate a compact list of columns."""
doc = json.loads(Path(source).read_text())
click.echo("\n".join(compute_compact_columns(doc)))