def _create_atlas_classification_entity(self, badge: Badge) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name, badge.name),
('category', badge.category), ('name', badge.name)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.badge,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_glossary_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name, self._name),
('glossary', self._tag_type), ('term', self._name)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.tag,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_cluster_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_cluster_key()), ('name', self.cluster),
('displayName', self.cluster)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasCommonTypes.cluster,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_atlas_process_key()),
('name', self._get_atlas_process_key())]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasTableTypes.process,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
yield entity
def _create_atlas_source_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self.get_source_model_key()), ('name', self.source),
('source_type', self.source_type),
('displayName', self.source)]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(typeName=AtlasTableTypes.source,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_schema_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name, self._schema_key),
('name', self._schema_key),
('description',
self._description.text if self._description else '')]
entity_attrs = get_entity_attrs(attrs_mapping)
# Since Schema cannot exist without Cluster (COMPOSITION relationship type), we assume Schema entity was created
# by different process and we only update schema description here using UPDATE operation.
entity = AtlasEntity(typeName=AtlasTableTypes.schema,
operation=AtlasSerializedEntityOperation.UPDATE,
attributes=entity_attrs,
relationships=None)
return entity
def _create_atlas_reader_entity(self) -> AtlasEntity:
attrs_mapping = [
(AtlasCommonParams.qualified_name, self._get_reader_key()),
('count', self.read_count),
('entityUri', self.start_label)
]
entity_attrs = get_entity_attrs(attrs_mapping)
entity = AtlasEntity(
typeName=AtlasCommonTypes.reader,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=None
)
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
group_attrs_mapping = [(AtlasCommonParams.qualified_name,
self.resource_report_key),
('name', self.report_name),
('url', self.report_url)]
entity_attrs = get_entity_attrs(group_attrs_mapping)
entity = AtlasEntity(
typeName=AtlasCommonTypes.resource_report,
operation=AtlasSerializedEntityOperation.CREATE,
relationships=None,
attributes=entity_attrs,
)
yield entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
group_attrs_mapping = [
(AtlasCommonParams.qualified_name, self.application_key),
('name', self.application_type), ('id', self.application_id),
('description', self.application_description or ''),
('application_url', self.application_url)
]
entity_attrs = get_entity_attrs(group_attrs_mapping)
entity = AtlasEntity(
typeName=AtlasCommonTypes.application,
operation=AtlasSerializedEntityOperation.CREATE,
relationships=None,
attributes=entity_attrs,
)
yield entity
def _create_atlas_schema_entity(self) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self._get_schema_key()), ('name', self.schema),
('displayName', self.schema)]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'cluster',
AtlasCommonTypes.cluster,
self._get_cluster_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.schema,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_atlas_partition_entity(self, spec: Tuple[str,
str]) -> AtlasEntity:
attrs_mapping = [(AtlasCommonParams.qualified_name,
self.get_watermark_model_key()), ('name', spec[1]),
('displayName', spec[1]), ('key', spec[0]),
('create_time', self.create_time)]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'table',
AtlasTableTypes.table,
self.get_metadata_model_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.watermark,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_next_atlas_entity(self) -> Iterator[AtlasEntity]:
# last modified
attrs_mapping = [
(AtlasCommonParams.qualified_name,
DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
product=self._product,
cluster=self._cluster,
dashboard_group=self._dashboard_group_id,
dashboard_name=self._dashboard_id)),
(AtlasCommonParams.last_modified_timestamp,
self._last_modified_timestamp),
]
dashboard_entity_attrs = get_entity_attrs(attrs_mapping)
last_modified = AtlasEntity(
typeName=AtlasDashboardTypes.metadata,
operation=AtlasSerializedEntityOperation.UPDATE,
relationships=None,
attributes=dashboard_entity_attrs)
yield last_modified
def _create_atlas_table_entity(self) -> AtlasEntity:
table_type = 'table' if not self.is_view else 'view'
attrs_mapping = [
(AtlasCommonParams.qualified_name, self._get_table_key()),
('name', self.name), ('tableType', table_type),
('description', self.description.text if self.description else ''),
('displayName', self.name)
]
entity_attrs = get_entity_attrs(attrs_mapping)
relationship_list = [] # type: ignore
add_entity_relationship(relationship_list, 'amundsen_schema',
AtlasTableTypes.schema, self._get_schema_key())
entity = AtlasEntity(
typeName=AtlasTableTypes.table,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=entity_attrs,
relationships=get_entity_relationships(relationship_list))
return entity
def _create_next_atlas_entity(self) -> Iterable[AtlasEntity]:
for actor in self._actors:
attrs_mapping = [
(AtlasCommonParams.qualified_name, actor.KEY_FORMAT.format(actor.name)),
('name', actor.name),
]
actor_entity_attrs = {}
for attr in attrs_mapping:
attr_key, attr_value = attr
actor_entity_attrs[attr_key] = attr_value
actor_entity = AtlasEntity(
typeName=actor.TYPE,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=actor_entity_attrs,
relationships=None,
)
yield actor_entity
for city in self._cities:
attrs_mapping = [
(AtlasCommonParams.qualified_name, city.KEY_FORMAT.format(city.name)),
('name', city.name),
]
city_entity_attrs = {}
for attr in attrs_mapping:
attr_key, attr_value = attr
city_entity_attrs[attr_key] = attr_value
city_entity = AtlasEntity(
typeName=city.TYPE,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=city_entity_attrs,
relationships=None,
)
yield city_entity
attrs_mapping = [
(AtlasCommonParams.qualified_name, self.KEY_FORMAT.format(self._name)),
('name', self._name),
]
movie_entity_attrs = {}
for attr in attrs_mapping:
attr_key, attr_value = attr
movie_entity_attrs[attr_key] = attr_value
relationship_list = []
"""
relationship in form 'relation_attribute#relation_entity_type#qualified_name_of_related_object
"""
for actor in self._actors:
relationship_list.append(
AtlasSerializedEntityFields.relationships_kv_separator
.join((
'actors',
self.MOVIE_ACTOR_RELATION_TYPE,
actor.KEY_FORMAT.format(actor.name),
)),
)
movie_entity = AtlasEntity(
typeName=self.TYPE,
operation=AtlasSerializedEntityOperation.CREATE,
attributes=movie_entity_attrs,
relationships=AtlasSerializedEntityFields.relationships_separator.join(relationship_list),
)
yield movie_entity