def _make_dashboard_and_chart_mces(
self, looker_dashboard: LookerDashboard
) -> List[MetadataChangeEvent]:
actor = self.source_config.actor
sys_time = get_sys_time()
chart_mces = [
self._make_chart_mce(element)
for element in looker_dashboard.dashboard_elements
]
dashboard_urn = f"urn:li:dashboard:({self.source_config.platform_name},{looker_dashboard.get_urn_dashboard_id()})"
dashboard_snapshot = DashboardSnapshot(
urn=dashboard_urn,
aspects=[],
)
last_modified = ChangeAuditStamps(
created=AuditStamp(time=sys_time, actor=actor),
lastModified=AuditStamp(time=sys_time, actor=actor),
)
dashboard_info = DashboardInfoClass(
description=looker_dashboard.description
if looker_dashboard.description is not None
else "",
title=looker_dashboard.title,
charts=[mce.proposedSnapshot.urn for mce in chart_mces],
lastModified=last_modified,
dashboardUrl=looker_dashboard.url(self.source_config.base_url),
)
dashboard_snapshot.aspects.append(dashboard_info)
owners = [OwnerClass(owner=actor, type=OwnershipTypeClass.DATAOWNER)]
dashboard_snapshot.aspects.append(
OwnershipClass(
owners=owners,
lastModified=AuditStampClass(
time=sys_time, actor=self.source_config.actor
),
)
)
dashboard_snapshot.aspects.append(Status(removed=looker_dashboard.is_deleted))
dashboard_mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot)
return chart_mces + [dashboard_mce]
def _make_dashboard_and_chart_mces(
self,
looker_dashboard: LookerDashboard) -> List[MetadataChangeEvent]:
chart_mces = [
self._make_chart_mce(element, looker_dashboard)
for element in looker_dashboard.dashboard_elements
if element.type == "vis"
]
dashboard_urn = builder.make_dashboard_urn(
self.source_config.platform_name,
looker_dashboard.get_urn_dashboard_id())
dashboard_snapshot = DashboardSnapshot(
urn=dashboard_urn,
aspects=[],
)
dashboard_info = DashboardInfoClass(
description=looker_dashboard.description or "",
title=looker_dashboard.title,
charts=[mce.proposedSnapshot.urn for mce in chart_mces],
lastModified=ChangeAuditStamps(),
dashboardUrl=looker_dashboard.url(
self.source_config.external_base_url),
)
dashboard_snapshot.aspects.append(dashboard_info)
if looker_dashboard.folder_path is not None:
browse_path = BrowsePathsClass(paths=[
f"/looker/{looker_dashboard.folder_path}/{looker_dashboard.id}"
])
dashboard_snapshot.aspects.append(browse_path)
ownership = self.get_ownership(looker_dashboard)
if ownership is not None:
dashboard_snapshot.aspects.append(ownership)
dashboard_snapshot.aspects.append(
Status(removed=looker_dashboard.is_deleted))
dashboard_mce = MetadataChangeEvent(
proposedSnapshot=dashboard_snapshot)
return chart_mces + [dashboard_mce]
def construct_dashboard_from_api_data(self, dashboard_data):
dashboard_urn = f"urn:li:dashboard:({self.platform},{dashboard_data['id']})"
dashboard_snapshot = DashboardSnapshot(
urn=dashboard_urn,
aspects=[],
)
modified_actor = f"urn:li:corpuser:{(dashboard_data.get('changed_by') or {}).get('username', 'unknown')}"
modified_ts = int(
dp.parse(dashboard_data.get("changed_on_utc", "now")).timestamp() * 1000
)
title = dashboard_data.get("dashboard_title", "")
# note: the API does not currently supply created_by usernames due to a bug, but we are required to
# provide a created AuditStamp to comply with ChangeAuditStamp model. For now, I sub in the last
# modified actor urn
last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor),
lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
)
dashboard_url = f"{self.config.connect_uri}{dashboard_data.get('url', '')}"
chart_urns = []
raw_position_data = dashboard_data.get("position_json", "{}")
position_data = (
json.loads(raw_position_data) if raw_position_data is not None else {}
)
for key, value in position_data.items():
if not key.startswith("CHART-"):
continue
chart_urns.append(
f"urn:li:chart:({self.platform},{value.get('meta', {}).get('chartId', 'unknown')})"
)
dashboard_info = DashboardInfoClass(
description="",
title=title,
charts=chart_urns,
lastModified=last_modified,
dashboardUrl=dashboard_url,
customProperties={},
)
dashboard_snapshot.aspects.append(dashboard_info)
return dashboard_snapshot
def _get_dashboard_snapshot(self, dashboard_data):
dashboard_id = dashboard_data["id"]
dashboard_urn = f"urn:li:dashboard:({self.platform},{dashboard_id})"
dashboard_snapshot = DashboardSnapshot(
urn=dashboard_urn,
aspects=[],
)
modified_actor = f"urn:li:corpuser:{dashboard_data.get('changed_by', {}).get('username', 'unknown')}"
modified_ts = int(
dp.parse(dashboard_data.get("updated_at", "now")).timestamp() *
1000)
title = dashboard_data.get("name", "")
last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor),
lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
)
dashboard_url = (
f"{self.config.connect_uri}/dashboard/{dashboard_data.get('slug', '')}"
)
widgets = dashboard_data.get("widgets", [])
description = self._get_dashboard_description_from_widgets(widgets)
chart_urns = self._get_dashboard_chart_urns_from_widgets(widgets)
dashboard_info = DashboardInfoClass(
description=description,
title=title,
charts=chart_urns,
lastModified=last_modified,
dashboardUrl=dashboard_url,
customProperties={},
)
dashboard_snapshot.aspects.append(dashboard_info)
return dashboard_snapshot
def __to_datahub_dashboard(
self,
dashboard: PowerBiAPI.Dashboard,
chart_mcps: List[MetadataChangeProposalWrapper],
user_mcps: List[MetadataChangeProposalWrapper],
) -> List[MetadataChangeProposalWrapper]:
"""
Map PowerBi dashboard to Datahub dashboard
"""
dashboard_urn = builder.make_dashboard_urn(self.__config.platform_name,
dashboard.get_urn_part())
chart_urn_list: List[str] = self.to_urn_set(chart_mcps)
user_urn_list: List[str] = self.to_urn_set(user_mcps)
def chart_custom_properties(dashboard: PowerBiAPI.Dashboard) -> dict:
return {
"chartCount": str(len(dashboard.tiles)),
"workspaceName": dashboard.workspace_name,
"workspaceId": dashboard.id,
}
# DashboardInfo mcp
dashboard_info_cls = DashboardInfoClass(
description=dashboard.displayName or "",
title=dashboard.displayName or "",
charts=chart_urn_list,
lastModified=ChangeAuditStamps(),
dashboardUrl=dashboard.webUrl,
customProperties={**chart_custom_properties(dashboard)},
)
info_mcp = self.new_mcp(
entity_type=Constant.DASHBOARD,
entity_urn=dashboard_urn,
aspect_name=Constant.DASHBOARD_INFO,
aspect=dashboard_info_cls,
)
# removed status mcp
removed_status_mcp = self.new_mcp(
entity_type=Constant.DASHBOARD,
entity_urn=dashboard_urn,
aspect_name=Constant.STATUS,
aspect=StatusClass(removed=False),
)
# dashboardKey mcp
dashboard_key_cls = DashboardKeyClass(
dashboardTool=self.__config.platform_name,
dashboardId=Constant.DASHBOARD_ID.format(dashboard.id),
)
# Dashboard key
dashboard_key_mcp = self.new_mcp(
entity_type=Constant.DASHBOARD,
entity_urn=dashboard_urn,
aspect_name=Constant.DASHBOARD_KEY,
aspect=dashboard_key_cls,
)
# Dashboard Ownership
owners = [
OwnerClass(owner=user_urn, type=OwnershipTypeClass.CONSUMER)
for user_urn in user_urn_list if user_urn is not None
]
ownership = OwnershipClass(owners=owners)
# Dashboard owner MCP
owner_mcp = self.new_mcp(
entity_type=Constant.DASHBOARD,
entity_urn=dashboard_urn,
aspect_name=Constant.OWNERSHIP,
aspect=ownership,
)
# Dashboard browsePaths
browse_path = BrowsePathsClass(
paths=["/powerbi/{}".format(self.__config.workspace_id)])
browse_path_mcp = self.new_mcp(
entity_type=Constant.DASHBOARD,
entity_urn=dashboard_urn,
aspect_name=Constant.BROWSERPATH,
aspect=browse_path,
)
return [
browse_path_mcp,
info_mcp,
removed_status_mcp,
dashboard_key_mcp,
owner_mcp,
]