def _create_next_node(self) -> Iterator[GraphNode]:
yield self._create_table_node()
if self.description:
node_key = self._get_table_description_key(self.description)
yield self.description.get_node(node_key)
# Create the table tag nodes
if self.tags:
for tag in self.tags:
tag_node = TagMetadata(tag).get_node()
yield tag_node
for col in self.columns:
yield from self._create_column_nodes(col)
# Database, cluster, schema
others = [
GraphNode(key=self._get_database_key(),
label=TableMetadata.DATABASE_NODE_LABEL,
attributes={'name': self.database}),
GraphNode(key=self._get_cluster_key(),
label=TableMetadata.CLUSTER_NODE_LABEL,
attributes={'name': self.cluster}),
GraphNode(key=self._get_schema_key(),
label=TableMetadata.SCHEMA_NODE_LABEL,
attributes={'name': self.schema})
]
for node_tuple in others:
if node_tuple.key not in TableMetadata.serialized_nodes_keys:
TableMetadata.serialized_nodes_keys.add(node_tuple.key)
yield node_tuple
def _create_next_node(self) -> Iterator[GraphNode]:
yield self._create_table_node()
if self.description:
node_key = self._get_table_description_key(self.description)
yield self.description.get_node(node_key)
# Create the table tag nodes
if self.tags:
for tag in self.tags:
tag_node = TagMetadata(tag).get_node()
yield tag_node
for col in self.columns:
column_node = GraphNode(key=self._get_col_key(col),
label=ColumnMetadata.COLUMN_NODE_LABEL,
attributes={
ColumnMetadata.COLUMN_NAME: col.name,
ColumnMetadata.COLUMN_TYPE: col.type,
ColumnMetadata.COLUMN_ORDER:
col.sort_order
})
yield column_node
if col.description:
node_key = self._get_col_description_key(col, col.description)
yield col.description.get_node(node_key)
if col.badges:
col_badge_metadata = BadgeMetadata(
start_label=ColumnMetadata.COLUMN_NODE_LABEL,
start_key=self._get_col_key(col),
badges=col.badges)
badge_nodes = col_badge_metadata.get_badge_nodes()
for node in badge_nodes:
yield node
# Database, cluster, schema
others = [
GraphNode(key=self._get_database_key(),
label=TableMetadata.DATABASE_NODE_LABEL,
attributes={'name': self.database}),
GraphNode(key=self._get_cluster_key(),
label=TableMetadata.CLUSTER_NODE_LABEL,
attributes={'name': self.cluster}),
GraphNode(key=self._get_schema_key(),
label=TableMetadata.SCHEMA_NODE_LABEL,
attributes={'name': self.schema})
]
for node_tuple in others:
if node_tuple.key not in TableMetadata.serialized_nodes_keys:
TableMetadata.serialized_nodes_keys.add(node_tuple.key)
yield node_tuple
def get_user_node(self) -> GraphNode:
node_attributes = {
User.USER_NODE_EMAIL: self.email,
User.USER_NODE_IS_ACTIVE: self.is_active,
User.USER_NODE_PROFILE_URL: self.profile_url or '',
User.USER_NODE_FIRST_NAME: self.first_name or '',
User.USER_NODE_LAST_NAME: self.last_name or '',
User.USER_NODE_FULL_NAME: self.full_name or '',
User.USER_NODE_GITHUB_NAME: self.github_username or '',
User.USER_NODE_TEAM: self.team_name or '',
User.USER_NODE_EMPLOYEE_TYPE: self.employee_type or '',
User.USER_NODE_SLACK_ID: self.slack_id or '',
User.USER_NODE_ROLE_NAME: self.role_name or ''
}
if self.updated_at:
node_attributes[User.USER_NODE_UPDATED_AT] = self.updated_at
elif not self.do_not_update_empty_attribute:
node_attributes[User.USER_NODE_UPDATED_AT] = 0
if self.attrs:
for k, v in self.attrs.items():
if k not in node_attributes:
node_attributes[k] = v
if self.do_not_update_empty_attribute:
for k, v in list(node_attributes.items()):
if not v:
del node_attributes[k]
node = GraphNode(key=User.get_user_model_key(email=self.email),
label=User.USER_NODE_LABEL,
attributes=node_attributes)
return node
def create_node_iterator(self) -> Iterator[GraphNode]:
node_attributes: Dict[str, Union[str, None, int]] = {
TypeMetadata.KIND: self.kind,
TypeMetadata.NAME: self.name,
TypeMetadata.DATA_TYPE: self.type_str
}
if isinstance(self.sort_order, int):
node_attributes[TypeMetadata.SORT_ORDER] = self.sort_order
yield GraphNode(
key=self.key(),
label=TypeMetadata.NODE_LABEL,
attributes=node_attributes
)
if self.description:
description_key = self.description_key()
assert description_key is not None, f"Could not retrieve description key for {self.name}"
yield self.description.get_node(description_key)
if not self.is_terminal_type():
assert self.struct_items, f"Struct items must be set for {self.name}"
for name, data_type in self.struct_items.items():
yield from data_type.create_node_iterator()
def _create_node_iterator(self) -> Iterator[GraphNode]:
yield GraphNode(key=self._get_watermark_key(),
label=FeatureWatermark.NODE_LABEL,
attributes={
TIMESTAMP_PROPERTY: self.timestamp,
FeatureWatermark.TYPE_ATTR: self.wm_type,
})
def create_nodes(self) -> List[GraphNode]:
"""
Create a list of Neo4j node records
:return:
"""
results = []
application_description = '{app_type} with id {id}'.format(
app_type=Application.APPLICATION_TYPE,
id=Application.APPLICATION_ID_FORMAT.format(dag_id=self.dag,
task_id=self.task))
application_id = Application.APPLICATION_ID_FORMAT.format(
dag_id=self.dag, task_id=self.task)
application_node = GraphNode(key=self.get_application_model_key(),
label=Application.APPLICATION_LABEL,
attributes={
Application.APPLICATION_URL_NAME:
self.application_url,
Application.APPLICATION_NAME:
Application.APPLICATION_TYPE,
Application.APPLICATION_DESCRIPTION:
application_description,
Application.APPLICATION_ID:
application_id
})
results.append(application_node)
return results
def _create_column_nodes(self, col: ColumnMetadata) -> Iterator[GraphNode]:
column_node = GraphNode(key=self._get_col_key(col),
label=ColumnMetadata.COLUMN_NODE_LABEL,
attributes={
ColumnMetadata.COLUMN_NAME: col.name,
ColumnMetadata.COLUMN_TYPE: col.type,
ColumnMetadata.COLUMN_ORDER: col.sort_order
})
yield column_node
if col.description:
node_key = self._get_col_description_key(col, col.description)
yield col.description.get_node(node_key)
if col.badges:
col_badge_metadata = BadgeMetadata(
start_label=ColumnMetadata.COLUMN_NODE_LABEL,
start_key=self._get_col_key(col),
badges=col.badges)
badge_nodes = col_badge_metadata.get_badge_nodes()
for node in badge_nodes:
yield node
type_metadata = col.get_type_metadata()
if type_metadata:
yield from type_metadata.create_node_iterator()
def _create_node_iterator(self) -> Iterator[GraphNode]:
node = GraphNode(key=self._get_last_execution_node_key(),
label=DashboardExecution.DASHBOARD_EXECUTION_LABEL,
attributes={
'timestamp': self._execution_timestamp,
'state': self._execution_state
})
yield node
def _create_node_iterator(self) -> Iterator[GraphNode]:
for email in self.owner_emails:
if email:
yield GraphNode(key=User.get_user_model_key(email=email),
label=User.USER_NODE_LABEL,
attributes={
User.USER_NODE_EMAIL: email,
})
def setUp(self) -> None:
super(TestWatermark, self).setUp()
self.watermark = Watermark(create_time='2017-09-18T00:00:00',
database=DATABASE,
schema=SCHEMA,
table_name=TABLE,
cluster=CLUSTER,
part_type=PART_TYPE,
part_name=NESTED_PART)
self.start_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}/{PART_TYPE}/'
self.end_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}'
self.expected_node_result = GraphNode(key=self.start_key,
label='Watermark',
attributes={
'partition_key':
'ds',
'partition_value':
'2017-09-18/feature_id=9',
'create_time':
'2017-09-18T00:00:00'
})
self.expected_serialized_node_results = [{
NODE_KEY:
self.start_key,
NODE_LABEL:
'Watermark',
'partition_key':
'ds',
'partition_value':
'2017-09-18/feature_id=9',
'create_time':
'2017-09-18T00:00:00'
}]
self.expected_relation_result = GraphRelationship(
start_label='Watermark',
end_label='Table',
start_key=self.start_key,
end_key=self.end_key,
type='BELONG_TO_TABLE',
reverse_type='WATERMARK',
attributes={})
self.expected_serialized_relation_results = [{
RELATION_START_KEY:
self.start_key,
RELATION_START_LABEL:
'Watermark',
RELATION_END_KEY:
self.end_key,
RELATION_END_LABEL:
'Table',
RELATION_TYPE:
'BELONG_TO_TABLE',
RELATION_REVERSE_TYPE:
'WATERMARK'
}]
def _create_next_node(self) -> Iterator[GraphNode]:
# Metric node
metric_node = GraphNode(key=self._get_metric_key(),
label=MetricMetadata.METRIC_NODE_LABEL,
attributes={
MetricMetadata.METRIC_NAME:
self.name,
MetricMetadata.METRIC_EXPRESSION_VALUE:
self.expression
})
yield metric_node
# Description node
if self.description:
description_node = GraphNode(
key=self._get_metric_description_key(),
label=MetricMetadata.DESCRIPTION_NODE_LABEL,
attributes={
MetricMetadata.METRIC_DESCRIPTION: self.description
})
yield description_node
# Metric tag node
if self.tags:
for tag in self.tags:
tag_node = GraphNode(
key=TagMetadata.get_tag_key(tag),
label=TagMetadata.TAG_NODE_LABEL,
attributes={TagMetadata.TAG_TYPE: 'metric'})
yield tag_node
# Metric type node
if self.type:
type_node = GraphNode(key=self._get_metric_type_key(),
label=MetricMetadata.METRIC_TYPE_NODE_LABEL,
attributes={'name': self.type})
yield type_node
# FIXME: this logic is wrong and does nothing presently
others: List[Any] = []
for node_tuple in others:
if node_tuple not in MetricMetadata.serialized_nodes:
MetricMetadata.serialized_nodes.add(node_tuple)
yield node_tuple
def create_nodes(self) -> List[GraphNode]:
"""
Create a list of Neo4j node records.
"""
node = GraphNode(
key=Neo4jESLastUpdated.KEY,
label=Neo4jESLastUpdated.LABEL,
attributes={Neo4jESLastUpdated.LATEST_TIMESTAMP: self.timestamp})
return [node]
def _create_node_iterator(self) -> Iterator[GraphNode]:
"""
Create an es_updated_timestamp node
"""
node = GraphNode(
key=ESLastUpdated.KEY,
label=ESLastUpdated.LABEL,
attributes={ESLastUpdated.LATEST_TIMESTAMP: self.timestamp})
yield node
def create_tag_node(name: str, tag_type: str = DEFAULT_TYPE) -> GraphNode:
node = GraphNode(
key=TagMetadata.get_tag_key(name),
label=TagMetadata.TAG_NODE_LABEL,
attributes={
TagMetadata.TAG_TYPE: tag_type
}
)
return node
def get_node(self) -> GraphNode:
node = GraphNode(
key=TagMetadata.get_tag_key(self._name),
label=TagMetadata.TAG_NODE_LABEL,
attributes={
TagMetadata.TAG_TYPE: self._tag_type
}
)
return node
def get_node(self, node_key: str) -> GraphNode:
node = GraphNode(key=node_key,
label=self._label,
attributes={
DescriptionMetadata.DESCRIPTION_SOURCE:
self._source,
DescriptionMetadata.DESCRIPTION_TEXT: self._text
})
return node
def get_badge_nodes(self) -> List[GraphNode]:
nodes = []
for badge in self.badges:
if badge:
node = GraphNode(
key=self.get_badge_key(badge.name),
label=self.BADGE_NODE_LABEL,
attributes={self.BADGE_CATEGORY: badge.category})
nodes.append(node)
return nodes
def create_nodes(self) -> Iterable[GraphNode]:
result = [
GraphNode(key=Movie.KEY_FORMAT.format(self._name),
label=Movie.LABEL,
attributes={'name': self._name})
]
for actor in self._actors:
actor_node = GraphNode(key=Actor.KEY_FORMAT.format(actor.name),
label=Actor.LABEL,
attributes={'name': self._name})
result.append(actor_node)
for city in self._cities:
city_node = GraphNode(key=City.KEY_FORMAT.format(city.name),
label=City.LABEL,
attributes={'name': self._name})
result.append(city_node)
return result
def _create_node_iterator(self) -> Iterator[GraphNode]:
node = GraphNode(key=self._schema_key,
label=SCHEMA_NODE_LABEL,
attributes={
SCHEMA_NAME_ATTR: self._schema,
})
yield node
if self._description:
yield self._description.get_node(self._get_description_node_key())
def _create_node_iterator(self) -> Iterator[GraphNode]:
"""
Create table owner nodes
:return:
"""
for owner in self.owners:
if owner:
node = GraphNode(key=self.get_owner_model_key(owner),
label=User.USER_NODE_LABEL,
attributes={User.USER_NODE_EMAIL: owner})
yield node
def _create_node_iterator(self) -> Iterator[GraphNode]:
node_attributes = {
timestamp_constants.TIMESTAMP_PROPERTY:
self._last_modified_timestamp,
timestamp_constants.TIMESTAMP_NAME_PROPERTY:
timestamp_constants.TimestampName.last_updated_timestamp.name
}
node = GraphNode(key=self._get_last_modified_node_key(),
label=timestamp_constants.NODE_LABEL,
attributes=node_attributes)
yield node
def create_nodes(self) -> List[GraphNode]:
"""
Create a list of Neo4j node records
:return:
"""
node_attributes = {
User.USER_NODE_EMAIL: self.email,
User.USER_NODE_IS_ACTIVE: self.is_active,
}
node_attributes[
User.
USER_NODE_FIRST_NAME] = self.first_name if self.first_name else ''
node_attributes[
User.
USER_NODE_LAST_NAME] = self.last_name if self.last_name else ''
node_attributes[
User.USER_NODE_FULL_NAME] = self.name if self.name else ''
node_attributes[
User.
USER_NODE_GITHUB_NAME] = self.github_username if self.github_username else ''
node_attributes[
User.USER_NODE_TEAM] = self.team_name if self.team_name else ''
node_attributes[
User.
USER_NODE_EMPLOYEE_TYPE] = self.employee_type if self.employee_type else ''
node_attributes[
User.USER_NODE_SLACK_ID] = self.slack_id if self.slack_id else ''
node_attributes[
User.
USER_NODE_ROLE_NAME] = self.role_name if self.role_name else ''
if self.updated_at:
node_attributes[User.USER_NODE_UPDATED_AT] = self.updated_at
elif not self.do_not_update_empty_attribute:
node_attributes[User.USER_NODE_UPDATED_AT] = 0
if self.attrs:
for k, v in self.attrs.items():
if k not in node_attributes:
node_attributes[k] = v
if self.do_not_update_empty_attribute:
for k, v in list(node_attributes.items()):
if not v:
del node_attributes[k]
node = GraphNode(key=User.get_user_model_key(email=self.email),
label=User.USER_NODE_LABEL,
attributes=node_attributes)
return [node]
def _create_node_iterator(self) -> Iterator[GraphNode]:
"""
Create a table source node
:return:
"""
node = GraphNode(key=self.get_source_model_key(),
label=TableSource.LABEL,
attributes={
'source': self.source,
'source_type': self.source_type
})
yield node
def create_nodes(self) -> List[GraphNode]:
"""
Create a list of Neo4j node records
:return:
"""
node = GraphNode(key=self.get_source_model_key(),
label=TableSource.LABEL,
attributes={
'source': self.source,
'source_type': self.source_type
})
results = [node]
return results
def create_nodes(self) -> List[GraphNode]:
"""
Create a list of Neo4j node records
:return:
"""
results = []
for owner in self.owners:
if owner:
node = GraphNode(key=self.get_owner_model_key(owner),
label=User.USER_NODE_LABEL,
attributes={User.USER_NODE_EMAIL: owner})
results.append(node)
return results
def _create_table_node(self) -> GraphNode:
table_attributes = {
TableMetadata.TABLE_NAME: self.name,
TableMetadata.IS_VIEW: self.is_view
}
if self.attrs:
for k, v in self.attrs.items():
if k not in table_attributes:
table_attributes[k] = v
return GraphNode(key=self._get_table_key(),
label=TableMetadata.TABLE_NODE_LABEL,
attributes=table_attributes)
def _create_node_iterator(self) -> Iterator[GraphNode]:
"""
Create a table stat node
:return:
"""
node = GraphNode(key=self.get_table_stat_model_key(),
label=TableColumnStats.LABEL,
attributes={
'stat_val': self.stat_val,
'stat_type': self.stat_type,
'start_epoch': self.start_epoch,
'end_epoch': self.end_epoch,
})
yield node
def _create_node_iterator(self) -> Iterator[GraphNode]:
"""
Create watermark nodes
:return:
"""
for part in self.parts:
part_node = GraphNode(key=self.get_watermark_model_key(),
label=Watermark.LABEL,
attributes={
'partition_key': part[0],
'partition_value': part[1],
'create_time': self.create_time
})
yield part_node
def _create_node_iterator(self) -> Iterator[GraphNode]:
"""
Create an application node
:return:
"""
report_node = GraphNode(key=self.resource_report_key,
label=ResourceReport.RESOURCE_REPORT_LABEL,
attributes={
ResourceReport.RESOURCE_REPORT_NAME:
self.report_name,
ResourceReport.RESOURCE_REPORT_URL:
self.report_url
})
yield report_node
def create_nodes(self) -> List[GraphNode]:
"""
Create a list of Neo4j node records
:return:
"""
node = GraphNode(key=self.get_table_stat_model_key(),
label=TableColumnStats.LABEL,
attributes={
'stat_val': self.stat_val,
'stat_name': self.stat_name,
'start_epoch': self.start_epoch,
'end_epoch': self.end_epoch,
})
results = [node]
return results