def _columns_to_metadata(
columns: Mapping[str, Any]) -> Optional[Mapping[str, Any]]:
return ({
"schema":
MetadataValue.table_schema(
TableSchema(columns=[
TableColumn(
name=name,
type=metadata.get("data_type") or "?",
description=metadata.get("description"),
) for name, metadata in columns.items()
]))
} if len(columns) > 0 else None)
def _materialization_for_stream(
name: str,
stream_schema_props: Dict[str, Any],
stream_stats: Dict[str, Any],
asset_key_prefix: List[str],
) -> AssetMaterialization:
return AssetMaterialization(
asset_key=asset_key_prefix + [name],
metadata={
"schema": MetadataValue.table_schema(
TableSchema(
columns=[
TableColumn(name=name, type=str(info.get("type", "unknown")))
for name, info in stream_schema_props.items()
]
)
),
**{k: v for k, v in stream_stats.items() if v is not None},
},
)
def _materialization_for_stream(
name: str,
stream_info: Dict[str, Any],
stream_stats: Dict[str, Any],
asset_key_prefix: List[str],
) -> AssetMaterialization:
return AssetMaterialization(
asset_key=asset_key_prefix + [name],
metadata={
"schema":
MetadataValue.table_schema(
TableSchema(columns=[
TableColumn(name=name, type=str(info["type"]))
for name, info in stream_info["stream"]["jsonSchema"]
["properties"].items()
])),
"columns":
",".join(name for name in stream_info["stream"]["jsonSchema"]
["properties"].keys()),
**{k: v
for k, v in stream_stats.items() if v is not None},
},
)
