def _get_custom_properties(
self, looker_view: LookerView) -> DatasetPropertiesClass:
file_path = str(
pathlib.Path(looker_view.absolute_file_path).resolve()).replace(
str(self.source_config.base_folder.resolve()), "")
custom_properties = {
"looker.file.content": looker_view.raw_file_content[
0:512000], # grab a limited slice of characters from the file
"looker.file.path": file_path,
}
dataset_props = DatasetPropertiesClass(
customProperties=custom_properties)
if self.source_config.github_info is not None:
github_file_url = self.source_config.github_info.get_url_for_file_path(
file_path)
dataset_props.externalUrl = github_file_url
return dataset_props
def _to_metadata_events( # noqa: C901
self, config: LookerCommonConfig, reporter: SourceReport,
base_url: str) -> Optional[List[Union[MetadataChangeEvent,
MetadataChangeProposalWrapper]]]:
# We only generate MCE-s for explores that contain from clauses and do NOT contain joins
# All other explores (passthrough explores and joins) end in correct resolution of lineage, and don't need additional nodes in the graph.
dataset_snapshot = DatasetSnapshot(
urn=self.get_explore_urn(config),
aspects=[], # we append to this list later on
)
browse_paths = BrowsePathsClass(
paths=[self.get_explore_browse_path(config)])
dataset_snapshot.aspects.append(browse_paths)
dataset_snapshot.aspects.append(StatusClass(removed=False))
custom_properties = {}
if self.label is not None:
custom_properties["looker.explore.label"] = str(self.label)
if self.source_file is not None:
custom_properties["looker.explore.file"] = str(self.source_file)
dataset_props = DatasetPropertiesClass(
description=self.description,
customProperties=custom_properties,
)
dataset_props.externalUrl = self._get_url(base_url)
dataset_snapshot.aspects.append(dataset_props)
if self.upstream_views is not None:
assert self.project_name is not None
upstreams = [
UpstreamClass(
dataset=LookerViewId(
project_name=self.project_name,
model_name=self.model_name,
view_name=view_name,
).get_urn(config),
type=DatasetLineageTypeClass.VIEW,
) for view_name in sorted(self.upstream_views)
]
upstream_lineage = UpstreamLineage(upstreams=upstreams)
dataset_snapshot.aspects.append(upstream_lineage)
if self.fields is not None:
schema_metadata = LookerUtil._get_schema(
platform_name=config.platform_name,
schema_name=self.name,
view_fields=self.fields,
reporter=reporter,
)
if schema_metadata is not None:
dataset_snapshot.aspects.append(schema_metadata)
mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
mcp = MetadataChangeProposalWrapper(
entityType="dataset",
changeType=ChangeTypeClass.UPSERT,
entityUrn=dataset_snapshot.urn,
aspectName="subTypes",
aspect=SubTypesClass(typeNames=["explore"]),
)
return [mce, mcp]
