def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# dashboard group
group_attrs_mapping = [
(AtlasCommonParams.qualified_name,
self._get_dashboard_group_key()),
('name', self.dashboard_group),
('id', self.dashboard_group_id),
('description', self.dashboard_group_description),
('url', self.dashboard_group_url),
]
dashboard_group_entity_attrs = get_entity_attrs(group_attrs_mapping)
dashboard_group_entity = AtlasEntity(
typeName=AtlasDashboardTypes.group,
operation=AtlasSerializedEntityOperation.CREATE,
relationships=None,
attributes=dashboard_group_entity_attrs,
)
yield dashboard_group_entity
# dashboard
attrs_mapping: List[Tuple[Any, Any]] = [
(AtlasCommonParams.qualified_name, self._get_dashboard_key()),
('name', self.dashboard_name),
('description', self.description),
('url', self.dashboard_url),
('cluster', self.cluster),
('product', self.product),
(AtlasCommonParams.created_timestamp, self.created_timestamp),
]
dashboard_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list,
'group',
AtlasDashboardTypes.group,
self._get_dashboard_group_key(),
)
dashboard_entity = AtlasEntity(
typeName=AtlasDashboardTypes.metadata,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=dashboard_entity_attrs,
relationships=get_entity_relationships(relationship_list),
)
yield dashboard_entity
def _create_atlas_column_entity(
self, column_metadata: ColumnMetadata) -> AtlasEntity:
qualified_name = column_metadata.COLUMN_KEY_FORMAT.format(
db=self.database,
cluster=self.cluster,
schema=self.schema,
tbl=self.name,
col=column_metadata.name)
attrs_mapping = [(AtlasCommonParams.qualified_name, qualified_name),
('name', column_metadata.name or ''),
('description', column_metadata.description.text
if column_metadata.description else ''),
('type', column_metadata.type),
('position', column_metadata.sort_order),
('displayName', column_metadata.name or '')]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'table',
AtlasTableTypes.table, self._get_table_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.column,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_atlas_user_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
User.get_user_model_key(email=self.email)),
('email', self.email),
('first_name', self.first_name),
('last_name', self.last_name),
('full_name', self.full_name),
('github_username', self.github_username),
('team_name', self.team_name),
('employee_type', self.employee_type),
('manager_email', self.manager_email),
('slack_id', self.slack_id),
('is_active', self.is_active),
('profile_url', self.profile_url),
('updated_at', self.updated_at),
('role_name', self.role_name),
('displayName', self.email)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.user,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
attrs_mapping = [
(AtlasCommonParams.qualified_name, self._get_last_execution_node_key()),
('state', self._execution_state),
('timestamp', self._execution_timestamp)
]
query_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list,
'dashboard',
AtlasDashboardTypes.metadata,
DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group=self._dashboard_group_id,
dashboard_name=self._dashboard_id
)
)
execution_entity = AtlasEntity(
typeName=AtlasDashboardTypes.execution,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=query_entity_attrs,
relationships=get_entity_relationships(relationship_list)
)
yield execution_entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# Query
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_query_node_key()),
('name', self._query_name), ('id', self._query_id),
('url', self._url), ('queryText', self._query_text)]
query_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list, 'dashboard', AtlasDashboardTypes.metadata,
DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group=self._dashboard_group_id,
dashboard_name=self._dashboard_id))
query_entity = AtlasEntity(
typeName=AtlasDashboardTypes.query,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=query_entity_attrs,
relationships=get_entity_relationships(relationship_list))
yield query_entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# Chart
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_chart_node_key()),
('name', self._chart_name),
('type', self._chart_type), ('url', self._chart_url)]
chart_entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(
relationship_list, 'query', AtlasDashboardTypes.query,
DashboardQuery.DASHBOARD_QUERY_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group_id=self._dashboard_group_id,
dashboard_id=self._dashboard_id,
query_id=self._query_id))
chart_entity = AtlasEntity(
typeName=AtlasDashboardTypes.chart,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=chart_entity_attrs,
relationships=get_entity_relationships(relationship_list))
yield chart_entity
def _create_atlas_glossary_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name, self._name),
('glossary', self._tag_type), ('term', self._name)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.tag,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_classification_entity(self, badge: Badge) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name, badge.name),
('category', badge.category), ('name', badge.name)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.badge,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_owner_entity(self, owner: str) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name, owner),
('email', owner)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.user,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_cluster_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_cluster_key()), ('name', self.cluster),
('displayName', self.cluster)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.cluster,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_atlas_process_key()),
('name', self._get_atlas_process_key())]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasTableTypes.process,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
yield entity
def _create_atlas_source_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self.get_source_model_key()), ('name', self.source),
('source_type', self.source_type),
('displayName', self.source)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasTableTypes.source,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
group_attrs_mapping = [(AtlasCommonParams.qualified_name,
self.resource_report_key),
('name', self.report_name),
('url', self.report_url)]
entity_attrs = get_entity_attrs(group_attrs_mapping)
entity = AtlasEntity(
typeName=AtlasCommonTypes.resource_report,
operation=AtlasSerializedEntityOperation.CREATE,
relationships=None,
attributes=entity_attrs,
)
yield entity
def _create_atlas_schema_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name, self._schema_key),
('name', self._schema_key),
('description',
self._description.text if self._description else '')]
entity_attrs = get_entity_attrs(attrs_mapping)
# Since Schema cannot exist without Cluster (COMPOSITION relationship type), we assume Schema entity was created
# by different process and we only update schema description here using UPDATE operation.
entity = AtlasEntity(typeName=AtlasTableTypes.schema,
operation=AtlasSerializedEntityOperation.UPDATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_reader_entity(self) -> AtlasEntity:
attrs_mapping = [
(AtlasCommonParams.qualified_name, self._get_reader_key()),
('count', self.read_count),
('entityUri', self.start_label)
]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(
typeName=AtlasCommonTypes.reader,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None
)
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
group_attrs_mapping = [
(AtlasCommonParams.qualified_name, self.application_key),
('name', self.application_type), ('id', self.application_id),
('description', self.application_description or ''),
('application_url', self.application_url)
]
entity_attrs = get_entity_attrs(group_attrs_mapping)
entity = AtlasEntity(
typeName=AtlasCommonTypes.application,
operation=AtlasSerializedEntityOperation.CREATE,
relationships=None,
attributes=entity_attrs,
)
yield entity
def _create_atlas_schema_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_schema_key()), ('name', self.schema),
('displayName', self.schema)]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'cluster',
AtlasCommonTypes.cluster,
self._get_cluster_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.schema,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# last modified
attrs_mapping = [
(AtlasCommonParams.qualified_name,
DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group=self._dashboard_group_id,
dashboard_name=self._dashboard_id)),
(AtlasCommonParams.last_modified_timestamp,
self._last_modified_timestamp),
]
dashboard_entity_attrs = get_entity_attrs(attrs_mapping)
last_modified = AtlasEntity(
typeName=AtlasDashboardTypes.metadata,
operation=AtlasSerializedEntityOperation.UPDATE,
relationships=None,
attributes=dashboard_entity_attrs)
yield last_modified
def _create_atlas_partition_entity(self, spec: Tuple[str,
str]) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self.get_watermark_model_key()), ('name', spec[1]),
('displayName', spec[1]), ('key', spec[0]),
('create_time', self.create_time)]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'table',
AtlasTableTypes.table,
self.get_metadata_model_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.watermark,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_atlas_table_entity(self) -> AtlasEntity:
table_type = 'table' if not self.is_view else 'view'
attrs_mapping = [
(AtlasCommonParams.qualified_name, self._get_table_key()),
('name', self.name), ('tableType', table_type),
('description', self.description.text if self.description else ''),
('displayName', self.name)
]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'amundsen_schema',
AtlasTableTypes.schema, self._get_schema_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.table,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity