def _get_ownership(self, creator_id: int) -> Optional[OwnershipClass]: user_info_url = f"{self.config.connect_uri}/api/user/{creator_id}" try: user_info_response = self.session.get(user_info_url) user_info_response.raise_for_status() user_details = user_info_response.json() except HTTPError as http_error: self.report.report_failure( key=f"metabase-user-{creator_id}", reason=f"Unable to retrieve User info. " f"Reason: {str(http_error)}", ) return None owner_urn = builder.make_user_urn(user_details.get("email", "")) if owner_urn is not None: ownership: OwnershipClass = OwnershipClass(owners=[ OwnerClass( owner=owner_urn, type=OwnershipTypeClass.DATAOWNER, ) ]) return ownership return None
def _get_owners_aspect(self, node: DBTNode) -> OwnershipClass: owners = [ OwnerClass( owner=f"urn:li:corpuser:{node.owner}", type=OwnershipTypeClass.DATAOWNER, ) ] return OwnershipClass(owners=owners, )
def transform_one( self, mce: MetadataChangeEventClass) -> MetadataChangeEventClass: ownership = builder.get_or_add_aspect( mce, OwnershipClass(owners=[], ), ) ownership.owners = [] return mce
def transform_one( self, mce: MetadataChangeEventClass) -> MetadataChangeEventClass: if not isinstance(mce.proposedSnapshot, DatasetSnapshotClass): return mce ownership = builder.get_or_add_aspect( mce, OwnershipClass(owners=[], ), ) ownership.owners = [] return mce
def create_metadata_work_unit(timestamp): dataset_snapshot = DatasetSnapshot( urn="urn:li:dataset:(urn:li:dataPlatform:glue,datalake_grilled.Barbeque,PROD)", aspects=[], ) dataset_snapshot.aspects.append(Status(removed=False)) dataset_snapshot.aspects.append( OwnershipClass( owners=[ OwnerClass( owner="urn:li:corpuser:Susan", type=OwnershipTypeClass.DATAOWNER ) ], lastModified=AuditStampClass( time=timestamp, actor="urn:li:corpuser:datahub" ), ) ) dataset_snapshot.aspects.append( DatasetPropertiesClass( description="Grilled Food", customProperties={}, uri=None, tags=[], ) ) fields = [ SchemaField( fieldPath="Size", nativeDataType="int", type=SchemaFieldDataType(type=NumberTypeClass()), description="Maximum attendees permitted", nullable=True, recursive=False, ) ] schema_metadata = SchemaMetadata( schemaName="datalake_grilled.Barbeque", version=0, fields=fields, platform="urn:li:dataPlatform:glue", created=AuditStamp(time=timestamp, actor="urn:li:corpuser:etl"), lastModified=AuditStamp(time=timestamp, actor="urn:li:corpuser:etl"), hash="", platformSchema=MySqlDDL(tableSchema=""), ) dataset_snapshot.aspects.append(schema_metadata) mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) return MetadataWorkUnit(id="glue-datalake_grilled.Barbeque", mce=mce)
def _generate_base_aspects( self, node: DBTNode, additional_custom_props_filtered: Dict[str, str], mce_platform: str, meta_aspects: Dict[str, Any], ) -> List[Any]: """ There are some common aspects that get generated for both dbt node and platform node depending on whether dbt node creation is enabled or not. """ # create an empty list of aspects and keep adding to it. Initializing with Any to avoid a # large union of aspect types. aspects: List[Any] = [] # add dataset properties aspect dbt_properties = self._create_dataset_properties_aspect( node, additional_custom_props_filtered) aspects.append(dbt_properties) # add status aspect status = StatusClass(removed=False) aspects.append(status) # add owners aspect # we need to aggregate owners added by meta properties and the owners that are coming from server. meta_owner_aspects = meta_aspects.get(Constants.ADD_OWNER_OPERATION) aggregated_owners = self._aggregate_owners(node, meta_owner_aspects) if aggregated_owners: aspects.append(OwnershipClass(owners=aggregated_owners)) # add tags aspects meta_tags_aspect = meta_aspects.get(Constants.ADD_TAG_OPERATION) aggregated_tags = self._aggregate_tags(node, meta_tags_aspect) if aggregated_tags: aspects.append( mce_builder.make_global_tag_aspect_with_tag_list( aggregated_tags)) # add meta term aspects if (meta_aspects.get(Constants.ADD_TERM_OPERATION) and self.config.enable_meta_mapping): aspects.append(meta_aspects.get(Constants.ADD_TERM_OPERATION)) # add schema metadata aspect schema_metadata = get_schema_metadata(self.report, node, mce_platform) # When generating these aspects for a dbt node, we will always include schema information. When generating # these aspects for a platform node (which only happens when disable_dbt_node_creation is set to true) we # honor the flag. if mce_platform == DBT_PLATFORM: aspects.append(schema_metadata) else: if self.config.load_schemas: aspects.append(schema_metadata) return aspects
def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent: assert tag_urn in LookerUtil.tag_definitions ownership = OwnershipClass(owners=[ OwnerClass( owner="urn:li:corpuser:datahub", type=OwnershipTypeClass.DATAOWNER, ) ]) return MetadataChangeEvent(proposedSnapshot=TagSnapshotClass( urn=tag_urn, aspects=[ownership, LookerUtil.tag_definitions[tag_urn]]))
def get_owner() -> Optional[OwnershipClass]: owner = table.get("Owner") if owner: owners = [ OwnerClass( owner=f"urn:li:corpuser:{owner}", type=OwnershipTypeClass.DATAOWNER, ) ] return OwnershipClass(owners=owners, ) return None
def get_owner() -> OwnershipClass: owner = table.get("Owner") if owner: owners = [ OwnerClass( owner=f"urn:li:corpuser:{owner}", type=OwnershipTypeClass.DATAOWNER, ) ] else: owners = [] return OwnershipClass(owners=owners, )
def _get_ownership(self, user: str) -> Optional[OwnershipClass]: if user is not None: owner_urn = builder.make_user_urn(user) ownership: OwnershipClass = OwnershipClass(owners=[ OwnerClass( owner=owner_urn, type=OwnershipTypeClass.DATAOWNER, ) ]) return ownership return None
def transform_aspect( # type: ignore self, entity_urn: str, aspect_name: str, aspect: Optional[OwnershipClass]) -> Optional[OwnershipClass]: owners_to_add = self.owners assert aspect is None or isinstance(aspect, OwnershipClass) if owners_to_add: ownership = (aspect if aspect else OwnershipClass(owners=[], )) ownership.owners.extend(owners_to_add) return ownership
def transform_one( self, mce: MetadataChangeEventClass) -> MetadataChangeEventClass: if not isinstance(mce.proposedSnapshot, DatasetSnapshotClass): return mce owners_to_add = self.config.get_owners_to_add(mce.proposedSnapshot) if owners_to_add: ownership = builder.get_or_add_aspect( mce, OwnershipClass(owners=[], ), ) ownership.owners.extend(owners_to_add) return mce
def get_group_wu( self, group_details: "DescribeModelPackageGroupOutputTypeDef" ) -> MetadataWorkUnit: """ Get a workunit for a model group. """ # params to remove since we extract them redundant_fields = {"ModelPackageGroupName", "CreationTime"} group_arn = group_details["ModelPackageGroupArn"] group_name = group_details["ModelPackageGroupName"] self.group_arn_to_name[group_arn] = group_name owners = [] if group_details.get("CreatedBy", {}).get("UserProfileName") is not None: owners.append( OwnerClass( owner= f"urn:li:corpuser:{group_details['CreatedBy']['UserProfileName']}", type=OwnershipTypeClass.DATAOWNER, )) group_snapshot = MLModelGroupSnapshot( urn=builder.make_ml_model_group_urn("sagemaker", group_name, self.env), aspects=[ MLModelGroupPropertiesClass( createdAt=int( group_details.get("CreationTime", datetime.now()).timestamp() * 1000), description=group_details.get( "ModelPackageGroupDescription"), customProperties={ key: str(value) for key, value in group_details.items() if key not in redundant_fields }, ), OwnershipClass(owners), BrowsePathsClass(paths=[f"/sagemaker/{group_name}"]), ], ) # make the MCE and workunit mce = MetadataChangeEvent(proposedSnapshot=group_snapshot) return MetadataWorkUnit(id=group_name, mce=mce)
def get_ownership( self, looker_dashboard: LookerDashboard) -> Optional[OwnershipClass]: if looker_dashboard.owner is not None: owner_urn = looker_dashboard.owner._get_urn( self.source_config.strip_user_ids_from_email) if owner_urn is not None: ownership: OwnershipClass = OwnershipClass(owners=[ OwnerClass( owner=owner_urn, type=OwnershipTypeClass.DATAOWNER, ) ]) return ownership return None
def add_owner_to_entity_wu(entity_type: str, entity_urn: str, owner_urn: str) -> Iterable[MetadataWorkUnit]: mcp = MetadataChangeProposalWrapper( entityType=entity_type, changeType=ChangeTypeClass.UPSERT, entityUrn=f"{entity_urn}", aspectName="ownership", aspect=OwnershipClass(owners=[ OwnerClass( owner=owner_urn, type=OwnershipTypeClass.DATAOWNER, ) ]), ) wu = MetadataWorkUnit(id=f"{owner_urn}-to-{entity_urn}", mcp=mcp) yield wu
def _make_dashboard_and_chart_mces( self, looker_dashboard: LookerDashboard ) -> List[MetadataChangeEvent]: actor = self.source_config.actor sys_time = get_sys_time() chart_mces = [ self._make_chart_mce(element) for element in looker_dashboard.dashboard_elements ] dashboard_urn = f"urn:li:dashboard:({self.source_config.platform_name},{looker_dashboard.get_urn_dashboard_id()})" dashboard_snapshot = DashboardSnapshot( urn=dashboard_urn, aspects=[], ) last_modified = ChangeAuditStamps( created=AuditStamp(time=sys_time, actor=actor), lastModified=AuditStamp(time=sys_time, actor=actor), ) dashboard_info = DashboardInfoClass( description=looker_dashboard.description if looker_dashboard.description is not None else "", title=looker_dashboard.title, charts=[mce.proposedSnapshot.urn for mce in chart_mces], lastModified=last_modified, dashboardUrl=looker_dashboard.url(self.source_config.base_url), ) dashboard_snapshot.aspects.append(dashboard_info) owners = [OwnerClass(owner=actor, type=OwnershipTypeClass.DATAOWNER)] dashboard_snapshot.aspects.append( OwnershipClass( owners=owners, lastModified=AuditStampClass( time=sys_time, actor=self.source_config.actor ), ) ) dashboard_snapshot.aspects.append(Status(removed=looker_dashboard.is_deleted)) dashboard_mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot) return chart_mces + [dashboard_mce]
def generate_ownership_aspect(self): ownership = OwnershipClass( owners=[ OwnerClass( owner=builder.make_user_urn(owner), type=OwnershipTypeClass.DEVELOPER, source=OwnershipSourceClass( type=OwnershipSourceTypeClass.SERVICE, # url=dag.filepath, ), ) for owner in (self.owners or []) ], lastModified=AuditStampClass(time=0, actor=builder.make_user_urn( self.orchestrator)), ) return [ownership]
def get_owner(time: int) -> OwnershipClass: owner = table.get("Owner") if owner: owners = [ OwnerClass( owner=f"urn:li:corpuser:{owner}", type=OwnershipTypeClass.DATAOWNER, ) ] else: owners = [] return OwnershipClass( owners=owners, lastModified=AuditStampClass( time=time, actor="urn:li:corpuser:datahub", ), )
def create_ownership_aspect_mce( directive: Directive) -> MetadataChangeEventClass: return MetadataChangeEventClass(proposedSnapshot=DatasetSnapshotClass( urn=dataset_name_to_urn(directive.table), aspects=[ OwnershipClass( owners=[ OwnerClass( owner=owner_name_to_urn(clean_owner_name(owner)), type=OwnershipTypeClass.DATAOWNER, ) for owner in directive.owners ], lastModified=AuditStampClass( time=int(time.time() * 1000), actor="urn:li:corpuser:datahub", ), ) ], ))
def make_ownership_aspect_from_urn_list( owner_urns: List[str], source_type: Optional[Union[str, OwnershipSourceTypeClass]]) -> OwnershipClass: for owner_urn in owner_urns: assert owner_urn.startswith( "urn:li:corpuser:"******"urn:li:corpGroup:") ownership_source_type: Union[None, OwnershipSourceClass] = None if source_type: ownership_source_type = OwnershipSourceClass(type=source_type) owners_list = [ OwnerClass( owner=owner_urn, type=OwnershipTypeClass.DATAOWNER, source=ownership_source_type, ) for owner_urn in owner_urns ] return OwnershipClass(owners=owners_list, )
def transform_one( self, mce: MetadataChangeEventClass) -> MetadataChangeEventClass: if not isinstance(mce.proposedSnapshot, DatasetSnapshotClass): return mce owners_to_add = self.config.get_owners_to_add(mce.proposedSnapshot) if owners_to_add: ownership = builder.get_or_add_aspect( mce, OwnershipClass( owners=[], lastModified=AuditStampClass( time=builder.get_sys_time(), actor=self.config.default_actor, ), ), ) ownership.owners.extend(owners_to_add) return mce
def transform_one(self, mce: MetadataChangeEventClass) -> MetadataChangeEventClass: assert isinstance(mce.proposedSnapshot, DatasetSnapshotClass) owners_to_add = self.config.get_owners_to_add(mce.proposedSnapshot) if owners_to_add: ownership = builder.get_or_add_aspect( mce, OwnershipClass( owners=[], ), ) ownership.owners.extend(owners_to_add) if self.config.semantics == Semantics.PATCH: assert self.ctx.graph patch_ownership = AddDatasetOwnership.get_ownership_to_set( self.ctx.graph, mce.proposedSnapshot.urn, ownership ) builder.set_aspect( mce, aspect=patch_ownership, aspect_type=OwnershipClass ) return mce
OwnershipTypeClass, ) log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) # Inputs -> owner, ownership_type, dataset owner_to_add = make_user_urn("jdoe") ownership_type = OwnershipTypeClass.DATAOWNER dataset_urn = make_dataset_urn(platform="hive", name="realestate_db.sales", env="PROD") # Some objects to help with conditional pathways later owner_class_to_add = OwnerClass(owner=owner_to_add, type=ownership_type) ownership_to_add = OwnershipClass(owners=[owner_class_to_add]) # First we get the current owners gms_endpoint = "http://localhost:8080" graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) current_owners: Optional[OwnershipClass] = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="ownership", aspect_type=OwnershipClass, ) need_write = False if current_owners: if (owner_to_add, ownership_type) not in [(x.owner, x.type) for x in current_owners.owners]:
def get_workunits(self) -> Iterable[MetadataWorkUnit]: ( nodes, manifest_schema, manifest_version, catalog_schema, catalog_version, ) = loadManifestAndCatalog( self.config.manifest_path, self.config.catalog_path, self.config.sources_path, self.config.load_schemas, self.config.use_identifiers, self.config.tag_prefix, self.config.target_platform, self.config.env, self.config.node_type_pattern, self.report, ) additional_custom_props = { "manifest_schema": manifest_schema, "manifest_version": manifest_version, "catalog_schema": catalog_schema, "catalog_version": catalog_version, } additional_custom_props_filtered = { key: value for key, value in additional_custom_props.items() if value is not None } for node in nodes: dataset_snapshot = DatasetSnapshot(urn=node.datahub_urn, aspects=[]) description = None if node.comment and node.description and node.comment != node.description: description = f"{self.config.target_platform} comment: {node.comment}\n\ndbt model description: {node.description}" elif node.comment: description = node.comment elif node.description: description = node.description custom_props = { **get_custom_properties(node), **additional_custom_props_filtered, } dbt_properties = DatasetPropertiesClass( description=description, customProperties=custom_props, tags=node.tags) dataset_snapshot.aspects.append(dbt_properties) if node.owner: owners = [ OwnerClass( owner=f"urn:li:corpuser:{node.owner}", type=OwnershipTypeClass.DATAOWNER, ) ] dataset_snapshot.aspects.append(OwnershipClass( owners=owners, )) if node.tags: dataset_snapshot.aspects.append( GlobalTagsClass(tags=[ TagAssociationClass(f"urn:li:tag:{tag}") for tag in node.tags ])) upstreams = get_upstream_lineage(node.upstream_urns) if upstreams is not None: dataset_snapshot.aspects.append(upstreams) if self.config.load_schemas: schema_metadata = get_schema_metadata( self.report, node, self.config.target_platform) dataset_snapshot.aspects.append(schema_metadata) mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) wu = MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce) self.report.report_workunit(wu) yield wu
def __to_datahub_dashboard( self, dashboard: PowerBiAPI.Dashboard, chart_mcps: List[MetadataChangeProposalWrapper], user_mcps: List[MetadataChangeProposalWrapper], ) -> List[MetadataChangeProposalWrapper]: """ Map PowerBi dashboard to Datahub dashboard """ dashboard_urn = builder.make_dashboard_urn(self.__config.platform_name, dashboard.get_urn_part()) chart_urn_list: List[str] = self.to_urn_set(chart_mcps) user_urn_list: List[str] = self.to_urn_set(user_mcps) def chart_custom_properties(dashboard: PowerBiAPI.Dashboard) -> dict: return { "chartCount": str(len(dashboard.tiles)), "workspaceName": dashboard.workspace_name, "workspaceId": dashboard.id, } # DashboardInfo mcp dashboard_info_cls = DashboardInfoClass( description=dashboard.displayName or "", title=dashboard.displayName or "", charts=chart_urn_list, lastModified=ChangeAuditStamps(), dashboardUrl=dashboard.webUrl, customProperties={**chart_custom_properties(dashboard)}, ) info_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.DASHBOARD_INFO, aspect=dashboard_info_cls, ) # removed status mcp removed_status_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.STATUS, aspect=StatusClass(removed=False), ) # dashboardKey mcp dashboard_key_cls = DashboardKeyClass( dashboardTool=self.__config.platform_name, dashboardId=Constant.DASHBOARD_ID.format(dashboard.id), ) # Dashboard key dashboard_key_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.DASHBOARD_KEY, aspect=dashboard_key_cls, ) # Dashboard Ownership owners = [ OwnerClass(owner=user_urn, type=OwnershipTypeClass.CONSUMER) for user_urn in user_urn_list if user_urn is not None ] ownership = OwnershipClass(owners=owners) # Dashboard owner MCP owner_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.OWNERSHIP, aspect=ownership, ) # Dashboard browsePaths browse_path = BrowsePathsClass( paths=["/powerbi/{}".format(self.__config.workspace_id)]) browse_path_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.BROWSERPATH, aspect=browse_path, ) return [ browse_path_mcp, info_mcp, removed_status_mcp, dashboard_key_mcp, owner_mcp, ]
def process_dataflow_node( self, node: Dict[str, Any], flow_urn: str, new_dataset_ids: List[str], new_dataset_mces: List[MetadataChangeEvent], s3_formats: typing.DefaultDict[str, Set[Union[str, None]]], ) -> Dict[str, Any]: node_type = node["NodeType"] # for nodes representing datasets, we construct a dataset URN accordingly if node_type in ["DataSource", "DataSink"]: node_args = { x["Name"]: json.loads(x["Value"]) for x in node["Args"] } # if data object is Glue table if "database" in node_args and "table_name" in node_args: full_table_name = f"{node_args['database']}.{node_args['table_name']}" # we know that the table will already be covered when ingesting Glue tables node_urn = f"urn:li:dataset:(urn:li:dataPlatform:glue,{full_table_name},{self.env})" # if data object is S3 bucket elif node_args.get("connection_type") == "s3": # remove S3 prefix (s3://) s3_name = node_args["connection_options"]["path"][5:] if s3_name.endswith("/"): s3_name = s3_name[:-1] # append S3 format if different ones exist if len(s3_formats[s3_name]) > 1: node_urn = f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name}_{node_args.get('format')},{self.env})" else: node_urn = ( f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name},{self.env})" ) dataset_snapshot = DatasetSnapshot( urn=node_urn, aspects=[], ) dataset_snapshot.aspects.append(Status(removed=False)) dataset_snapshot.aspects.append( OwnershipClass( owners=[], lastModified=AuditStampClass( time=mce_builder.get_sys_time(), actor="urn:li:corpuser:datahub", ), )) dataset_snapshot.aspects.append( DatasetPropertiesClass( customProperties={ k: str(v) for k, v in node_args.items() }, tags=[], )) new_dataset_mces.append( MetadataChangeEvent(proposedSnapshot=dataset_snapshot)) new_dataset_ids.append(f"{node['NodeType']}-{node['Id']}") else: raise ValueError( f"Unrecognized Glue data object type: {node_args}") # otherwise, a node represents a transformation else: node_urn = mce_builder.make_data_job_urn_with_flow( flow_urn, job_id=f'{node["NodeType"]}-{node["Id"]}') return { **node, "urn": node_urn, # to be filled in after traversing edges "inputDatajobs": [], "inputDatasets": [], "outputDatasets": [], }