def _make_dashboard_and_chart_mces( self, looker_dashboard: LookerDashboard ) -> List[MetadataChangeEvent]: actor = self.source_config.actor sys_time = get_sys_time() chart_mces = [ self._make_chart_mce(element) for element in looker_dashboard.dashboard_elements ] dashboard_urn = f"urn:li:dashboard:({self.source_config.platform_name},{looker_dashboard.get_urn_dashboard_id()})" dashboard_snapshot = DashboardSnapshot( urn=dashboard_urn, aspects=[], ) last_modified = ChangeAuditStamps( created=AuditStamp(time=sys_time, actor=actor), lastModified=AuditStamp(time=sys_time, actor=actor), ) dashboard_info = DashboardInfoClass( description=looker_dashboard.description if looker_dashboard.description is not None else "", title=looker_dashboard.title, charts=[mce.proposedSnapshot.urn for mce in chart_mces], lastModified=last_modified, dashboardUrl=looker_dashboard.url(self.source_config.base_url), ) dashboard_snapshot.aspects.append(dashboard_info) owners = [OwnerClass(owner=actor, type=OwnershipTypeClass.DATAOWNER)] dashboard_snapshot.aspects.append( OwnershipClass( owners=owners, lastModified=AuditStampClass( time=sys_time, actor=self.source_config.actor ), ) ) dashboard_snapshot.aspects.append(Status(removed=looker_dashboard.is_deleted)) dashboard_mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot) return chart_mces + [dashboard_mce]
def _make_dashboard_and_chart_mces( self, looker_dashboard: LookerDashboard) -> List[MetadataChangeEvent]: chart_mces = [ self._make_chart_mce(element, looker_dashboard) for element in looker_dashboard.dashboard_elements if element.type == "vis" ] dashboard_urn = builder.make_dashboard_urn( self.source_config.platform_name, looker_dashboard.get_urn_dashboard_id()) dashboard_snapshot = DashboardSnapshot( urn=dashboard_urn, aspects=[], ) dashboard_info = DashboardInfoClass( description=looker_dashboard.description or "", title=looker_dashboard.title, charts=[mce.proposedSnapshot.urn for mce in chart_mces], lastModified=ChangeAuditStamps(), dashboardUrl=looker_dashboard.url( self.source_config.external_base_url), ) dashboard_snapshot.aspects.append(dashboard_info) if looker_dashboard.folder_path is not None: browse_path = BrowsePathsClass(paths=[ f"/looker/{looker_dashboard.folder_path}/{looker_dashboard.id}" ]) dashboard_snapshot.aspects.append(browse_path) ownership = self.get_ownership(looker_dashboard) if ownership is not None: dashboard_snapshot.aspects.append(ownership) dashboard_snapshot.aspects.append( Status(removed=looker_dashboard.is_deleted)) dashboard_mce = MetadataChangeEvent( proposedSnapshot=dashboard_snapshot) return chart_mces + [dashboard_mce]
def construct_dashboard_from_api_data(self, dashboard_data): dashboard_urn = f"urn:li:dashboard:({self.platform},{dashboard_data['id']})" dashboard_snapshot = DashboardSnapshot( urn=dashboard_urn, aspects=[], ) modified_actor = f"urn:li:corpuser:{(dashboard_data.get('changed_by') or {}).get('username', 'unknown')}" modified_ts = int( dp.parse(dashboard_data.get("changed_on_utc", "now")).timestamp() * 1000 ) title = dashboard_data.get("dashboard_title", "") # note: the API does not currently supply created_by usernames due to a bug, but we are required to # provide a created AuditStamp to comply with ChangeAuditStamp model. For now, I sub in the last # modified actor urn last_modified = ChangeAuditStamps( created=AuditStamp(time=modified_ts, actor=modified_actor), lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) dashboard_url = f"{self.config.connect_uri}{dashboard_data.get('url', '')}" chart_urns = [] raw_position_data = dashboard_data.get("position_json", "{}") position_data = ( json.loads(raw_position_data) if raw_position_data is not None else {} ) for key, value in position_data.items(): if not key.startswith("CHART-"): continue chart_urns.append( f"urn:li:chart:({self.platform},{value.get('meta', {}).get('chartId', 'unknown')})" ) dashboard_info = DashboardInfoClass( description="", title=title, charts=chart_urns, lastModified=last_modified, dashboardUrl=dashboard_url, customProperties={}, ) dashboard_snapshot.aspects.append(dashboard_info) return dashboard_snapshot
def _get_dashboard_snapshot(self, dashboard_data): dashboard_id = dashboard_data["id"] dashboard_urn = f"urn:li:dashboard:({self.platform},{dashboard_id})" dashboard_snapshot = DashboardSnapshot( urn=dashboard_urn, aspects=[], ) modified_actor = f"urn:li:corpuser:{dashboard_data.get('changed_by', {}).get('username', 'unknown')}" modified_ts = int( dp.parse(dashboard_data.get("updated_at", "now")).timestamp() * 1000) title = dashboard_data.get("name", "") last_modified = ChangeAuditStamps( created=AuditStamp(time=modified_ts, actor=modified_actor), lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) dashboard_url = ( f"{self.config.connect_uri}/dashboard/{dashboard_data.get('slug', '')}" ) widgets = dashboard_data.get("widgets", []) description = self._get_dashboard_description_from_widgets(widgets) chart_urns = self._get_dashboard_chart_urns_from_widgets(widgets) dashboard_info = DashboardInfoClass( description=description, title=title, charts=chart_urns, lastModified=last_modified, dashboardUrl=dashboard_url, customProperties={}, ) dashboard_snapshot.aspects.append(dashboard_info) return dashboard_snapshot
def __to_datahub_dashboard( self, dashboard: PowerBiAPI.Dashboard, chart_mcps: List[MetadataChangeProposalWrapper], user_mcps: List[MetadataChangeProposalWrapper], ) -> List[MetadataChangeProposalWrapper]: """ Map PowerBi dashboard to Datahub dashboard """ dashboard_urn = builder.make_dashboard_urn(self.__config.platform_name, dashboard.get_urn_part()) chart_urn_list: List[str] = self.to_urn_set(chart_mcps) user_urn_list: List[str] = self.to_urn_set(user_mcps) def chart_custom_properties(dashboard: PowerBiAPI.Dashboard) -> dict: return { "chartCount": str(len(dashboard.tiles)), "workspaceName": dashboard.workspace_name, "workspaceId": dashboard.id, } # DashboardInfo mcp dashboard_info_cls = DashboardInfoClass( description=dashboard.displayName or "", title=dashboard.displayName or "", charts=chart_urn_list, lastModified=ChangeAuditStamps(), dashboardUrl=dashboard.webUrl, customProperties={**chart_custom_properties(dashboard)}, ) info_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.DASHBOARD_INFO, aspect=dashboard_info_cls, ) # removed status mcp removed_status_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.STATUS, aspect=StatusClass(removed=False), ) # dashboardKey mcp dashboard_key_cls = DashboardKeyClass( dashboardTool=self.__config.platform_name, dashboardId=Constant.DASHBOARD_ID.format(dashboard.id), ) # Dashboard key dashboard_key_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.DASHBOARD_KEY, aspect=dashboard_key_cls, ) # Dashboard Ownership owners = [ OwnerClass(owner=user_urn, type=OwnershipTypeClass.CONSUMER) for user_urn in user_urn_list if user_urn is not None ] ownership = OwnershipClass(owners=owners) # Dashboard owner MCP owner_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.OWNERSHIP, aspect=ownership, ) # Dashboard browsePaths browse_path = BrowsePathsClass( paths=["/powerbi/{}".format(self.__config.workspace_id)]) browse_path_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, aspect_name=Constant.BROWSERPATH, aspect=browse_path, ) return [ browse_path_mcp, info_mcp, removed_status_mcp, dashboard_key_mcp, owner_mcp, ]