Esempio n. 1
0
    def _create_next_node(self) -> Iterator[GraphNode]:
        yield self._create_table_node()

        if self.description:
            node_key = self._get_table_description_key(self.description)
            yield self.description.get_node(node_key)

        # Create the table tag nodes
        if self.tags:
            for tag in self.tags:
                tag_node = TagMetadata(tag).get_node()
                yield tag_node

        for col in self.columns:
            yield from self._create_column_nodes(col)

        # Database, cluster, schema
        others = [
            GraphNode(key=self._get_database_key(),
                      label=TableMetadata.DATABASE_NODE_LABEL,
                      attributes={'name': self.database}),
            GraphNode(key=self._get_cluster_key(),
                      label=TableMetadata.CLUSTER_NODE_LABEL,
                      attributes={'name': self.cluster}),
            GraphNode(key=self._get_schema_key(),
                      label=TableMetadata.SCHEMA_NODE_LABEL,
                      attributes={'name': self.schema})
        ]

        for node_tuple in others:
            if node_tuple.key not in TableMetadata.serialized_nodes_keys:
                TableMetadata.serialized_nodes_keys.add(node_tuple.key)
                yield node_tuple
Esempio n. 2
0
    def _create_next_node(self) -> Iterator[GraphNode]:
        yield self._create_table_node()

        if self.description:
            node_key = self._get_table_description_key(self.description)
            yield self.description.get_node(node_key)

        # Create the table tag nodes
        if self.tags:
            for tag in self.tags:
                tag_node = TagMetadata(tag).get_node()
                yield tag_node

        for col in self.columns:
            column_node = GraphNode(key=self._get_col_key(col),
                                    label=ColumnMetadata.COLUMN_NODE_LABEL,
                                    attributes={
                                        ColumnMetadata.COLUMN_NAME: col.name,
                                        ColumnMetadata.COLUMN_TYPE: col.type,
                                        ColumnMetadata.COLUMN_ORDER:
                                        col.sort_order
                                    })
            yield column_node

            if col.description:
                node_key = self._get_col_description_key(col, col.description)
                yield col.description.get_node(node_key)

            if col.badges:
                col_badge_metadata = BadgeMetadata(
                    start_label=ColumnMetadata.COLUMN_NODE_LABEL,
                    start_key=self._get_col_key(col),
                    badges=col.badges)
                badge_nodes = col_badge_metadata.get_badge_nodes()
                for node in badge_nodes:
                    yield node

        # Database, cluster, schema
        others = [
            GraphNode(key=self._get_database_key(),
                      label=TableMetadata.DATABASE_NODE_LABEL,
                      attributes={'name': self.database}),
            GraphNode(key=self._get_cluster_key(),
                      label=TableMetadata.CLUSTER_NODE_LABEL,
                      attributes={'name': self.cluster}),
            GraphNode(key=self._get_schema_key(),
                      label=TableMetadata.SCHEMA_NODE_LABEL,
                      attributes={'name': self.schema})
        ]

        for node_tuple in others:
            if node_tuple.key not in TableMetadata.serialized_nodes_keys:
                TableMetadata.serialized_nodes_keys.add(node_tuple.key)
                yield node_tuple
Esempio n. 3
0
    def get_user_node(self) -> GraphNode:
        node_attributes = {
            User.USER_NODE_EMAIL: self.email,
            User.USER_NODE_IS_ACTIVE: self.is_active,
            User.USER_NODE_PROFILE_URL: self.profile_url or '',
            User.USER_NODE_FIRST_NAME: self.first_name or '',
            User.USER_NODE_LAST_NAME: self.last_name or '',
            User.USER_NODE_FULL_NAME: self.full_name or '',
            User.USER_NODE_GITHUB_NAME: self.github_username or '',
            User.USER_NODE_TEAM: self.team_name or '',
            User.USER_NODE_EMPLOYEE_TYPE: self.employee_type or '',
            User.USER_NODE_SLACK_ID: self.slack_id or '',
            User.USER_NODE_ROLE_NAME: self.role_name or ''
        }

        if self.updated_at:
            node_attributes[User.USER_NODE_UPDATED_AT] = self.updated_at
        elif not self.do_not_update_empty_attribute:
            node_attributes[User.USER_NODE_UPDATED_AT] = 0

        if self.attrs:
            for k, v in self.attrs.items():
                if k not in node_attributes:
                    node_attributes[k] = v

        if self.do_not_update_empty_attribute:
            for k, v in list(node_attributes.items()):
                if not v:
                    del node_attributes[k]

        node = GraphNode(key=User.get_user_model_key(email=self.email),
                         label=User.USER_NODE_LABEL,
                         attributes=node_attributes)

        return node
Esempio n. 4
0
    def create_node_iterator(self) -> Iterator[GraphNode]:
        node_attributes: Dict[str, Union[str, None, int]] = {
            TypeMetadata.KIND: self.kind,
            TypeMetadata.NAME: self.name,
            TypeMetadata.DATA_TYPE: self.type_str
        }

        if isinstance(self.sort_order, int):
            node_attributes[TypeMetadata.SORT_ORDER] = self.sort_order

        yield GraphNode(
            key=self.key(),
            label=TypeMetadata.NODE_LABEL,
            attributes=node_attributes
        )

        if self.description:
            description_key = self.description_key()
            assert description_key is not None, f"Could not retrieve description key for {self.name}"
            yield self.description.get_node(description_key)

        if not self.is_terminal_type():
            assert self.struct_items, f"Struct items must be set for {self.name}"
            for name, data_type in self.struct_items.items():
                yield from data_type.create_node_iterator()
Esempio n. 5
0
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     yield GraphNode(key=self._get_watermark_key(),
                     label=FeatureWatermark.NODE_LABEL,
                     attributes={
                         TIMESTAMP_PROPERTY: self.timestamp,
                         FeatureWatermark.TYPE_ATTR: self.wm_type,
                     })
Esempio n. 6
0
    def create_nodes(self) -> List[GraphNode]:
        """
        Create a list of Neo4j node records
        :return:
        """
        results = []
        application_description = '{app_type} with id {id}'.format(
            app_type=Application.APPLICATION_TYPE,
            id=Application.APPLICATION_ID_FORMAT.format(dag_id=self.dag,
                                                        task_id=self.task))
        application_id = Application.APPLICATION_ID_FORMAT.format(
            dag_id=self.dag, task_id=self.task)
        application_node = GraphNode(key=self.get_application_model_key(),
                                     label=Application.APPLICATION_LABEL,
                                     attributes={
                                         Application.APPLICATION_URL_NAME:
                                         self.application_url,
                                         Application.APPLICATION_NAME:
                                         Application.APPLICATION_TYPE,
                                         Application.APPLICATION_DESCRIPTION:
                                         application_description,
                                         Application.APPLICATION_ID:
                                         application_id
                                     })
        results.append(application_node)

        return results
Esempio n. 7
0
    def _create_column_nodes(self, col: ColumnMetadata) -> Iterator[GraphNode]:
        column_node = GraphNode(key=self._get_col_key(col),
                                label=ColumnMetadata.COLUMN_NODE_LABEL,
                                attributes={
                                    ColumnMetadata.COLUMN_NAME: col.name,
                                    ColumnMetadata.COLUMN_TYPE: col.type,
                                    ColumnMetadata.COLUMN_ORDER: col.sort_order
                                })
        yield column_node

        if col.description:
            node_key = self._get_col_description_key(col, col.description)
            yield col.description.get_node(node_key)

        if col.badges:
            col_badge_metadata = BadgeMetadata(
                start_label=ColumnMetadata.COLUMN_NODE_LABEL,
                start_key=self._get_col_key(col),
                badges=col.badges)
            badge_nodes = col_badge_metadata.get_badge_nodes()
            for node in badge_nodes:
                yield node

        type_metadata = col.get_type_metadata()
        if type_metadata:
            yield from type_metadata.create_node_iterator()
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     node = GraphNode(key=self._get_last_execution_node_key(),
                      label=DashboardExecution.DASHBOARD_EXECUTION_LABEL,
                      attributes={
                          'timestamp': self._execution_timestamp,
                          'state': self._execution_state
                      })
     yield node
Esempio n. 9
0
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     for email in self.owner_emails:
         if email:
             yield GraphNode(key=User.get_user_model_key(email=email),
                             label=User.USER_NODE_LABEL,
                             attributes={
                                 User.USER_NODE_EMAIL: email,
                             })
    def setUp(self) -> None:
        super(TestWatermark, self).setUp()
        self.watermark = Watermark(create_time='2017-09-18T00:00:00',
                                   database=DATABASE,
                                   schema=SCHEMA,
                                   table_name=TABLE,
                                   cluster=CLUSTER,
                                   part_type=PART_TYPE,
                                   part_name=NESTED_PART)
        self.start_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}/{PART_TYPE}/'
        self.end_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}'
        self.expected_node_result = GraphNode(key=self.start_key,
                                              label='Watermark',
                                              attributes={
                                                  'partition_key':
                                                  'ds',
                                                  'partition_value':
                                                  '2017-09-18/feature_id=9',
                                                  'create_time':
                                                  '2017-09-18T00:00:00'
                                              })

        self.expected_serialized_node_results = [{
            NODE_KEY:
            self.start_key,
            NODE_LABEL:
            'Watermark',
            'partition_key':
            'ds',
            'partition_value':
            '2017-09-18/feature_id=9',
            'create_time':
            '2017-09-18T00:00:00'
        }]

        self.expected_relation_result = GraphRelationship(
            start_label='Watermark',
            end_label='Table',
            start_key=self.start_key,
            end_key=self.end_key,
            type='BELONG_TO_TABLE',
            reverse_type='WATERMARK',
            attributes={})

        self.expected_serialized_relation_results = [{
            RELATION_START_KEY:
            self.start_key,
            RELATION_START_LABEL:
            'Watermark',
            RELATION_END_KEY:
            self.end_key,
            RELATION_END_LABEL:
            'Table',
            RELATION_TYPE:
            'BELONG_TO_TABLE',
            RELATION_REVERSE_TYPE:
            'WATERMARK'
        }]
    def _create_next_node(self) -> Iterator[GraphNode]:
        # Metric node
        metric_node = GraphNode(key=self._get_metric_key(),
                                label=MetricMetadata.METRIC_NODE_LABEL,
                                attributes={
                                    MetricMetadata.METRIC_NAME:
                                    self.name,
                                    MetricMetadata.METRIC_EXPRESSION_VALUE:
                                    self.expression
                                })
        yield metric_node

        # Description node
        if self.description:
            description_node = GraphNode(
                key=self._get_metric_description_key(),
                label=MetricMetadata.DESCRIPTION_NODE_LABEL,
                attributes={
                    MetricMetadata.METRIC_DESCRIPTION: self.description
                })
            yield description_node

        # Metric tag node
        if self.tags:
            for tag in self.tags:
                tag_node = GraphNode(
                    key=TagMetadata.get_tag_key(tag),
                    label=TagMetadata.TAG_NODE_LABEL,
                    attributes={TagMetadata.TAG_TYPE: 'metric'})
                yield tag_node

        # Metric type node
        if self.type:
            type_node = GraphNode(key=self._get_metric_type_key(),
                                  label=MetricMetadata.METRIC_TYPE_NODE_LABEL,
                                  attributes={'name': self.type})
            yield type_node

        # FIXME: this logic is wrong and does nothing presently
        others: List[Any] = []

        for node_tuple in others:
            if node_tuple not in MetricMetadata.serialized_nodes:
                MetricMetadata.serialized_nodes.add(node_tuple)
                yield node_tuple
Esempio n. 12
0
 def create_nodes(self) -> List[GraphNode]:
     """
     Create a list of Neo4j node records.
     """
     node = GraphNode(
         key=Neo4jESLastUpdated.KEY,
         label=Neo4jESLastUpdated.LABEL,
         attributes={Neo4jESLastUpdated.LATEST_TIMESTAMP: self.timestamp})
     return [node]
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     """
     Create an es_updated_timestamp node
     """
     node = GraphNode(
         key=ESLastUpdated.KEY,
         label=ESLastUpdated.LABEL,
         attributes={ESLastUpdated.LATEST_TIMESTAMP: self.timestamp})
     yield node
Esempio n. 14
0
 def create_tag_node(name: str, tag_type: str = DEFAULT_TYPE) -> GraphNode:
     node = GraphNode(
         key=TagMetadata.get_tag_key(name),
         label=TagMetadata.TAG_NODE_LABEL,
         attributes={
             TagMetadata.TAG_TYPE: tag_type
         }
     )
     return node
 def get_node(self) -> GraphNode:
     node = GraphNode(
         key=TagMetadata.get_tag_key(self._name),
         label=TagMetadata.TAG_NODE_LABEL,
         attributes={
             TagMetadata.TAG_TYPE: self._tag_type
         }
     )
     return node
Esempio n. 16
0
 def get_node(self, node_key: str) -> GraphNode:
     node = GraphNode(key=node_key,
                      label=self._label,
                      attributes={
                          DescriptionMetadata.DESCRIPTION_SOURCE:
                          self._source,
                          DescriptionMetadata.DESCRIPTION_TEXT: self._text
                      })
     return node
Esempio n. 17
0
 def get_badge_nodes(self) -> List[GraphNode]:
     nodes = []
     for badge in self.badges:
         if badge:
             node = GraphNode(
                 key=self.get_badge_key(badge.name),
                 label=self.BADGE_NODE_LABEL,
                 attributes={self.BADGE_CATEGORY: badge.category})
             nodes.append(node)
     return nodes
    def create_nodes(self) -> Iterable[GraphNode]:
        result = [
            GraphNode(key=Movie.KEY_FORMAT.format(self._name),
                      label=Movie.LABEL,
                      attributes={'name': self._name})
        ]

        for actor in self._actors:
            actor_node = GraphNode(key=Actor.KEY_FORMAT.format(actor.name),
                                   label=Actor.LABEL,
                                   attributes={'name': self._name})
            result.append(actor_node)

        for city in self._cities:
            city_node = GraphNode(key=City.KEY_FORMAT.format(city.name),
                                  label=City.LABEL,
                                  attributes={'name': self._name})
            result.append(city_node)
        return result
Esempio n. 19
0
    def _create_node_iterator(self) -> Iterator[GraphNode]:
        node = GraphNode(key=self._schema_key,
                         label=SCHEMA_NODE_LABEL,
                         attributes={
                             SCHEMA_NAME_ATTR: self._schema,
                         })
        yield node

        if self._description:
            yield self._description.get_node(self._get_description_node_key())
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     """
     Create table owner nodes
     :return:
     """
     for owner in self.owners:
         if owner:
             node = GraphNode(key=self.get_owner_model_key(owner),
                              label=User.USER_NODE_LABEL,
                              attributes={User.USER_NODE_EMAIL: owner})
             yield node
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     node_attributes = {
         timestamp_constants.TIMESTAMP_PROPERTY:
         self._last_modified_timestamp,
         timestamp_constants.TIMESTAMP_NAME_PROPERTY:
         timestamp_constants.TimestampName.last_updated_timestamp.name
     }
     node = GraphNode(key=self._get_last_modified_node_key(),
                      label=timestamp_constants.NODE_LABEL,
                      attributes=node_attributes)
     yield node
Esempio n. 22
0
    def create_nodes(self) -> List[GraphNode]:
        """
        Create a list of Neo4j node records
        :return:
        """

        node_attributes = {
            User.USER_NODE_EMAIL: self.email,
            User.USER_NODE_IS_ACTIVE: self.is_active,
        }

        node_attributes[
            User.
            USER_NODE_FIRST_NAME] = self.first_name if self.first_name else ''
        node_attributes[
            User.
            USER_NODE_LAST_NAME] = self.last_name if self.last_name else ''
        node_attributes[
            User.USER_NODE_FULL_NAME] = self.name if self.name else ''
        node_attributes[
            User.
            USER_NODE_GITHUB_NAME] = self.github_username if self.github_username else ''
        node_attributes[
            User.USER_NODE_TEAM] = self.team_name if self.team_name else ''
        node_attributes[
            User.
            USER_NODE_EMPLOYEE_TYPE] = self.employee_type if self.employee_type else ''
        node_attributes[
            User.USER_NODE_SLACK_ID] = self.slack_id if self.slack_id else ''
        node_attributes[
            User.
            USER_NODE_ROLE_NAME] = self.role_name if self.role_name else ''

        if self.updated_at:
            node_attributes[User.USER_NODE_UPDATED_AT] = self.updated_at
        elif not self.do_not_update_empty_attribute:
            node_attributes[User.USER_NODE_UPDATED_AT] = 0

        if self.attrs:
            for k, v in self.attrs.items():
                if k not in node_attributes:
                    node_attributes[k] = v

        if self.do_not_update_empty_attribute:
            for k, v in list(node_attributes.items()):
                if not v:
                    del node_attributes[k]

        node = GraphNode(key=User.get_user_model_key(email=self.email),
                         label=User.USER_NODE_LABEL,
                         attributes=node_attributes)

        return [node]
Esempio n. 23
0
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     """
     Create a table source node
     :return:
     """
     node = GraphNode(key=self.get_source_model_key(),
                      label=TableSource.LABEL,
                      attributes={
                          'source': self.source,
                          'source_type': self.source_type
                      })
     yield node
 def create_nodes(self) -> List[GraphNode]:
     """
     Create a list of Neo4j node records
     :return:
     """
     node = GraphNode(key=self.get_source_model_key(),
                      label=TableSource.LABEL,
                      attributes={
                          'source': self.source,
                          'source_type': self.source_type
                      })
     results = [node]
     return results
Esempio n. 25
0
 def create_nodes(self) -> List[GraphNode]:
     """
     Create a list of Neo4j node records
     :return:
     """
     results = []
     for owner in self.owners:
         if owner:
             node = GraphNode(key=self.get_owner_model_key(owner),
                              label=User.USER_NODE_LABEL,
                              attributes={User.USER_NODE_EMAIL: owner})
             results.append(node)
     return results
Esempio n. 26
0
    def _create_table_node(self) -> GraphNode:
        table_attributes = {
            TableMetadata.TABLE_NAME: self.name,
            TableMetadata.IS_VIEW: self.is_view
        }
        if self.attrs:
            for k, v in self.attrs.items():
                if k not in table_attributes:
                    table_attributes[k] = v

        return GraphNode(key=self._get_table_key(),
                         label=TableMetadata.TABLE_NODE_LABEL,
                         attributes=table_attributes)
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     """
     Create a table stat node
     :return:
     """
     node = GraphNode(key=self.get_table_stat_model_key(),
                      label=TableColumnStats.LABEL,
                      attributes={
                          'stat_val': self.stat_val,
                          'stat_type': self.stat_type,
                          'start_epoch': self.start_epoch,
                          'end_epoch': self.end_epoch,
                      })
     yield node
Esempio n. 28
0
 def _create_node_iterator(self) -> Iterator[GraphNode]:
     """
     Create watermark nodes
     :return:
     """
     for part in self.parts:
         part_node = GraphNode(key=self.get_watermark_model_key(),
                               label=Watermark.LABEL,
                               attributes={
                                   'partition_key': part[0],
                                   'partition_value': part[1],
                                   'create_time': self.create_time
                               })
         yield part_node
Esempio n. 29
0
    def _create_node_iterator(self) -> Iterator[GraphNode]:
        """
        Create an application node
        :return:
        """
        report_node = GraphNode(key=self.resource_report_key,
                                label=ResourceReport.RESOURCE_REPORT_LABEL,
                                attributes={
                                    ResourceReport.RESOURCE_REPORT_NAME:
                                    self.report_name,
                                    ResourceReport.RESOURCE_REPORT_URL:
                                    self.report_url
                                })

        yield report_node
 def create_nodes(self) -> List[GraphNode]:
     """
     Create a list of Neo4j node records
     :return:
     """
     node = GraphNode(key=self.get_table_stat_model_key(),
                      label=TableColumnStats.LABEL,
                      attributes={
                          'stat_val': self.stat_val,
                          'stat_name': self.stat_name,
                          'start_epoch': self.start_epoch,
                          'end_epoch': self.end_epoch,
                      })
     results = [node]
     return results