def get_schema_fields_for_column(
self,
dataset_name: str,
column: dict,
pk_constraints: dict = None) -> List[SchemaField]:
field = SchemaField(
fieldPath=column["name"],
type=get_column_type(self.report, dataset_name, column["type"]),
nativeDataType=column.get("full_type", repr(column["type"])),
description=column.get("comment", None),
nullable=column["nullable"],
recursive=False,
)
if (pk_constraints is not None and isinstance(
pk_constraints, dict) # some dialects (hive) return list
and column["name"] in pk_constraints.get(
"constrained_columns", [])):
field.isPartOfKey = True
return [field]
def get_schema_metadata(
sql_report: SQLSourceReport,
dataset_name: str,
platform: str,
columns: List[dict],
pk_constraints: dict = None,
foreign_keys: List[ForeignKeyConstraint] = None,
) -> SchemaMetadata:
canonical_schema: List[SchemaField] = []
for column in columns:
field = SchemaField(
fieldPath=column["name"],
type=get_column_type(sql_report, dataset_name, column["type"]),
nativeDataType=column.get("full_type", repr(column["type"])),
description=column.get("comment", None),
nullable=column["nullable"],
recursive=False,
)
if (pk_constraints is not None and isinstance(
pk_constraints, dict) # some dialects (hive) return list
and column["name"] in pk_constraints.get(
"constrained_columns", [])):
field.isPartOfKey = True
canonical_schema.append(field)
schema_metadata = SchemaMetadata(
schemaName=dataset_name,
platform=f"urn:li:dataPlatform:{platform}",
version=0,
hash="",
platformSchema=MySqlDDL(tableSchema=""),
fields=canonical_schema,
)
if foreign_keys is not None and foreign_keys != []:
schema_metadata.foreignKeys = foreign_keys
return schema_metadata
