def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# Query
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_query_node_key()),
('name', self._query_name), ('id', self._query_id),
('url', self._url), ('queryText', self._query_text)]
query_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list, 'dashboard', AtlasDashboardTypes.metadata,
DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group=self._dashboard_group_id,
dashboard_name=self._dashboard_id))
query_entity = AtlasEntity(
typeName=AtlasDashboardTypes.query,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=query_entity_attrs,
relationships=get_entity_relationships(relationship_list))
yield query_entity
def _create_atlas_column_entity(
self, column_metadata: ColumnMetadata) -> AtlasEntity:
qualified_name = column_metadata.COLUMN_KEY_FORMAT.format(
db=self.database,
cluster=self.cluster,
schema=self.schema,
tbl=self.name,
col=column_metadata.name)
attrs_mapping = [(AtlasCommonParams.qualified_name, qualified_name),
('name', column_metadata.name or ''),
('description', column_metadata.description.text
if column_metadata.description else ''),
('type', column_metadata.type),
('position', column_metadata.sort_order),
('displayName', column_metadata.name or '')]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'table',
AtlasTableTypes.table, self._get_table_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.column,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
attrs_mapping = [
(AtlasCommonParams.qualified_name, self._get_last_execution_node_key()),
('state', self._execution_state),
('timestamp', self._execution_timestamp)
]
query_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list,
'dashboard',
AtlasDashboardTypes.metadata,
DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group=self._dashboard_group_id,
dashboard_name=self._dashboard_id
)
)
execution_entity = AtlasEntity(
typeName=AtlasDashboardTypes.execution,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=query_entity_attrs,
relationships=get_entity_relationships(relationship_list)
)
yield execution_entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# Chart
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_chart_node_key()),
('name', self._chart_name),
('type', self._chart_type), ('url', self._chart_url)]
chart_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list, 'query', AtlasDashboardTypes.query,
DashboardQuery.DASHBOARD_QUERY_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group_id=self._dashboard_group_id,
dashboard_id=self._dashboard_id,
query_id=self._query_id))
chart_entity = AtlasEntity(
typeName=AtlasDashboardTypes.chart,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=chart_entity_attrs,
relationships=get_entity_relationships(relationship_list))
yield chart_entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# dashboard group
group_attrs_mapping = [
(AtlasCommonParams.qualified_name,
self._get_dashboard_group_key()),
('name', self.dashboard_group),
('id', self.dashboard_group_id),
('description', self.dashboard_group_description),
('url', self.dashboard_group_url),
]
dashboard_group_entity_attrs = get_entity_attrs(group_attrs_mapping)
dashboard_group_entity = AtlasEntity(
typeName=AtlasDashboardTypes.group,
operation=AtlasSerializedEntityOperation.CREATE,
relationships=None,
attributes=dashboard_group_entity_attrs,
)
yield dashboard_group_entity
# dashboard
attrs_mapping: List[Tuple[Any, Any]] = [
(AtlasCommonParams.qualified_name, self._get_dashboard_key()),
('name', self.dashboard_name),
('description', self.description),
('url', self.dashboard_url),
('cluster', self.cluster),
('product', self.product),
(AtlasCommonParams.created_timestamp, self.created_timestamp),
]
dashboard_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list,
'group',
AtlasDashboardTypes.group,
self._get_dashboard_group_key(),
)
dashboard_entity = AtlasEntity(
typeName=AtlasDashboardTypes.metadata,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=dashboard_entity_attrs,
relationships=get_entity_relationships(relationship_list),
)
yield dashboard_entity
def _create_atlas_schema_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_schema_key()), ('name', self.schema),
('displayName', self.schema)]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'cluster',
AtlasCommonTypes.cluster,
self._get_cluster_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.schema,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_atlas_partition_entity(self, spec: Tuple[str,
str]) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self.get_watermark_model_key()), ('name', spec[1]),
('displayName', spec[1]), ('key', spec[0]),
('create_time', self.create_time)]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'table',
AtlasTableTypes.table,
self.get_metadata_model_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.watermark,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_atlas_table_entity(self) -> AtlasEntity:
table_type = 'table' if not self.is_view else 'view'
attrs_mapping = [
(AtlasCommonParams.qualified_name, self._get_table_key()),
('name', self.name), ('tableType', table_type),
('description', self.description.text if self.description else ''),
('displayName', self.name)
]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'amundsen_schema',
AtlasTableTypes.schema, self._get_schema_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.table,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity