def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: # dashboard group group_attrs_mapping = [ (AtlasCommonParams.qualified_name, self._get_dashboard_group_key()), ('name', self.dashboard_group), ('id', self.dashboard_group_id), ('description', self.dashboard_group_description), ('url', self.dashboard_group_url), ] dashboard_group_entity_attrs = get_entity_attrs(group_attrs_mapping) dashboard_group_entity = AtlasEntity( typeName=AtlasDashboardTypes.group, operation=AtlasSerializedEntityOperation.CREATE, relationships=None, attributes=dashboard_group_entity_attrs, ) yield dashboard_group_entity # dashboard attrs_mapping: List[Tuple[Any, Any]] = [ (AtlasCommonParams.qualified_name, self._get_dashboard_key()), ('name', self.dashboard_name), ('description', self.description), ('url', self.dashboard_url), ('cluster', self.cluster), ('product', self.product), (AtlasCommonParams.created_timestamp, self.created_timestamp), ] dashboard_entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship( relationship_list, 'group', AtlasDashboardTypes.group, self._get_dashboard_group_key(), ) dashboard_entity = AtlasEntity( typeName=AtlasDashboardTypes.metadata, operation=AtlasSerializedEntityOperation.CREATE, attributes=dashboard_entity_attrs, relationships=get_entity_relationships(relationship_list), ) yield dashboard_entity
def _create_atlas_column_entity( self, column_metadata: ColumnMetadata) -> AtlasEntity: qualified_name = column_metadata.COLUMN_KEY_FORMAT.format( db=self.database, cluster=self.cluster, schema=self.schema, tbl=self.name, col=column_metadata.name) attrs_mapping = [(AtlasCommonParams.qualified_name, qualified_name), ('name', column_metadata.name or ''), ('description', column_metadata.description.text if column_metadata.description else ''), ('type', column_metadata.type), ('position', column_metadata.sort_order), ('displayName', column_metadata.name or '')] entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship(relationship_list, 'table', AtlasTableTypes.table, self._get_table_key()) entity = AtlasEntity( typeName=AtlasTableTypes.column, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=get_entity_relationships(relationship_list)) return entity
def _create_atlas_user_entity(self) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, User.get_user_model_key(email=self.email)), ('email', self.email), ('first_name', self.first_name), ('last_name', self.last_name), ('full_name', self.full_name), ('github_username', self.github_username), ('team_name', self.team_name), ('employee_type', self.employee_type), ('manager_email', self.manager_email), ('slack_id', self.slack_id), ('is_active', self.is_active), ('profile_url', self.profile_url), ('updated_at', self.updated_at), ('role_name', self.role_name), ('displayName', self.email)] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity(typeName=AtlasCommonTypes.user, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None) return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: attrs_mapping = [ (AtlasCommonParams.qualified_name, self._get_last_execution_node_key()), ('state', self._execution_state), ('timestamp', self._execution_timestamp) ] query_entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship( relationship_list, 'dashboard', AtlasDashboardTypes.metadata, DashboardMetadata.DASHBOARD_KEY_FORMAT.format( product=self._product, cluster=self._cluster, dashboard_group=self._dashboard_group_id, dashboard_name=self._dashboard_id ) ) execution_entity = AtlasEntity( typeName=AtlasDashboardTypes.execution, operation=AtlasSerializedEntityOperation.CREATE, attributes=query_entity_attrs, relationships=get_entity_relationships(relationship_list) ) yield execution_entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: # Query attrs_mapping = [(AtlasCommonParams.qualified_name, self._get_query_node_key()), ('name', self._query_name), ('id', self._query_id), ('url', self._url), ('queryText', self._query_text)] query_entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship( relationship_list, 'dashboard', AtlasDashboardTypes.metadata, DashboardMetadata.DASHBOARD_KEY_FORMAT.format( product=self._product, cluster=self._cluster, dashboard_group=self._dashboard_group_id, dashboard_name=self._dashboard_id)) query_entity = AtlasEntity( typeName=AtlasDashboardTypes.query, operation=AtlasSerializedEntityOperation.CREATE, attributes=query_entity_attrs, relationships=get_entity_relationships(relationship_list)) yield query_entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: # Chart attrs_mapping = [(AtlasCommonParams.qualified_name, self._get_chart_node_key()), ('name', self._chart_name), ('type', self._chart_type), ('url', self._chart_url)] chart_entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship( relationship_list, 'query', AtlasDashboardTypes.query, DashboardQuery.DASHBOARD_QUERY_KEY_FORMAT.format( product=self._product, cluster=self._cluster, dashboard_group_id=self._dashboard_group_id, dashboard_id=self._dashboard_id, query_id=self._query_id)) chart_entity = AtlasEntity( typeName=AtlasDashboardTypes.chart, operation=AtlasSerializedEntityOperation.CREATE, attributes=chart_entity_attrs, relationships=get_entity_relationships(relationship_list)) yield chart_entity
def _create_atlas_glossary_entity(self) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, self._name), ('glossary', self._tag_type), ('term', self._name)] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity(typeName=AtlasCommonTypes.tag, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None) return entity
def _create_atlas_classification_entity(self, badge: Badge) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, badge.name), ('category', badge.category), ('name', badge.name)] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity(typeName=AtlasCommonTypes.badge, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None) return entity
def _create_atlas_owner_entity(self, owner: str) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, owner), ('email', owner)] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity(typeName=AtlasCommonTypes.user, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None) return entity
def _create_atlas_cluster_entity(self) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, self._get_cluster_key()), ('name', self.cluster), ('displayName', self.cluster)] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity(typeName=AtlasCommonTypes.cluster, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None) return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: attrs_mapping = [(AtlasCommonParams.qualified_name, self._get_atlas_process_key()), ('name', self._get_atlas_process_key())] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity(typeName=AtlasTableTypes.process, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None) yield entity
def _create_atlas_source_entity(self) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, self.get_source_model_key()), ('name', self.source), ('source_type', self.source_type), ('displayName', self.source)] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity(typeName=AtlasTableTypes.source, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None) return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: group_attrs_mapping = [(AtlasCommonParams.qualified_name, self.resource_report_key), ('name', self.report_name), ('url', self.report_url)] entity_attrs = get_entity_attrs(group_attrs_mapping) entity = AtlasEntity( typeName=AtlasCommonTypes.resource_report, operation=AtlasSerializedEntityOperation.CREATE, relationships=None, attributes=entity_attrs, ) yield entity
def _create_atlas_schema_entity(self) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, self._schema_key), ('name', self._schema_key), ('description', self._description.text if self._description else '')] entity_attrs = get_entity_attrs(attrs_mapping) # Since Schema cannot exist without Cluster (COMPOSITION relationship type), we assume Schema entity was created # by different process and we only update schema description here using UPDATE operation. entity = AtlasEntity(typeName=AtlasTableTypes.schema, operation=AtlasSerializedEntityOperation.UPDATE, attributes=entity_attrs, relationships=None) return entity
def _create_atlas_reader_entity(self) -> AtlasEntity: attrs_mapping = [ (AtlasCommonParams.qualified_name, self._get_reader_key()), ('count', self.read_count), ('entityUri', self.start_label) ] entity_attrs = get_entity_attrs(attrs_mapping) entity = AtlasEntity( typeName=AtlasCommonTypes.reader, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=None ) return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: group_attrs_mapping = [ (AtlasCommonParams.qualified_name, self.application_key), ('name', self.application_type), ('id', self.application_id), ('description', self.application_description or ''), ('application_url', self.application_url) ] entity_attrs = get_entity_attrs(group_attrs_mapping) entity = AtlasEntity( typeName=AtlasCommonTypes.application, operation=AtlasSerializedEntityOperation.CREATE, relationships=None, attributes=entity_attrs, ) yield entity
def _create_atlas_schema_entity(self) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, self._get_schema_key()), ('name', self.schema), ('displayName', self.schema)] entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship(relationship_list, 'cluster', AtlasCommonTypes.cluster, self._get_cluster_key()) entity = AtlasEntity( typeName=AtlasTableTypes.schema, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=get_entity_relationships(relationship_list)) return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]: # last modified attrs_mapping = [ (AtlasCommonParams.qualified_name, DashboardMetadata.DASHBOARD_KEY_FORMAT.format( product=self._product, cluster=self._cluster, dashboard_group=self._dashboard_group_id, dashboard_name=self._dashboard_id)), (AtlasCommonParams.last_modified_timestamp, self._last_modified_timestamp), ] dashboard_entity_attrs = get_entity_attrs(attrs_mapping) last_modified = AtlasEntity( typeName=AtlasDashboardTypes.metadata, operation=AtlasSerializedEntityOperation.UPDATE, relationships=None, attributes=dashboard_entity_attrs) yield last_modified
def _create_atlas_partition_entity(self, spec: Tuple[str, str]) -> AtlasEntity: attrs_mapping = [(AtlasCommonParams.qualified_name, self.get_watermark_model_key()), ('name', spec[1]), ('displayName', spec[1]), ('key', spec[0]), ('create_time', self.create_time)] entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship(relationship_list, 'table', AtlasTableTypes.table, self.get_metadata_model_key()) entity = AtlasEntity( typeName=AtlasTableTypes.watermark, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=get_entity_relationships(relationship_list)) return entity
def _create_atlas_table_entity(self) -> AtlasEntity: table_type = 'table' if not self.is_view else 'view' attrs_mapping = [ (AtlasCommonParams.qualified_name, self._get_table_key()), ('name', self.name), ('tableType', table_type), ('description', self.description.text if self.description else ''), ('displayName', self.name) ] entity_attrs = get_entity_attrs(attrs_mapping) relationship_list = [] # type: ignore add_entity_relationship(relationship_list, 'amundsen_schema', AtlasTableTypes.schema, self._get_schema_key()) entity = AtlasEntity( typeName=AtlasTableTypes.table, operation=AtlasSerializedEntityOperation.CREATE, attributes=entity_attrs, relationships=get_entity_relationships(relationship_list)) return entity