def test_serialize(self) -> None: # First test node_row = self.metric_metadata.next_node() actual = [] while node_row: serialized_node = neo4_serializer.serialize_node(node_row) actual.append(serialized_node) node_row = self.metric_metadata.next_node() self.assertEqual(self.expected_nodes, actual) relation_row = self.metric_metadata.next_relation() actual = [] while relation_row: serialized_relation = neo4_serializer.serialize_relationship( relation_row) actual.append(serialized_relation) relation_row = self.metric_metadata.next_relation() self.assertEqual(self.expected_rels, actual) # Second test node_row = self.metric_metadata2.next_node() actual = [] while node_row: serialized_node = neo4_serializer.serialize_node(node_row) actual.append(serialized_node) node_row = self.metric_metadata2.next_node() self.assertEqual(self.expected_nodes_deduped2, actual) relation_row = self.metric_metadata2.next_relation() actual = [] while relation_row: serialized_relation = neo4_serializer.serialize_relationship( relation_row) actual.append(serialized_relation) relation_row = self.metric_metadata2.next_relation() self.assertEqual(self.expected_rels_deduped2, actual) # Third test node_row = self.metric_metadata3.next_node() actual = [] while node_row: serialized_node = neo4_serializer.serialize_node(node_row) actual.append(serialized_node) node_row = self.metric_metadata3.next_node() self.assertEqual(self.expected_nodes_deduped3, actual) relation_row = self.metric_metadata3.next_relation() actual = [] while relation_row: serialized_relation = neo4_serializer.serialize_relationship( relation_row) actual.append(serialized_relation) relation_row = self.metric_metadata3.next_relation() self.assertEqual(self.expected_rels_deduped3, actual)
def test_serialize(self) -> None: node_row = self.table_col_usage.next_node() actual = [] while node_row: actual.append(neo4_serializer.serialize_node(node_row)) node_row = self.table_col_usage.next_node() expected = [{ 'LABEL': 'User', 'KEY': '*****@*****.**', 'email': '*****@*****.**' }] self.assertEqual(expected, actual) rel_row = self.table_col_usage.next_relation() actual = [] while rel_row: actual.append(neo4_serializer.serialize_relationship(rel_row)) rel_row = self.table_col_usage.next_relation() expected = [{ 'read_count:UNQUOTED': 1, 'END_KEY': '*****@*****.**', 'START_LABEL': 'Table', 'END_LABEL': 'User', 'START_KEY': 'db://gold.scm/foo', 'TYPE': 'READ_BY', 'REVERSE_TYPE': 'READ' }] self.assertEqual(expected, actual)
def test_create_relation(self) -> None: expected_relations = [ { RELATION_START_KEY: self.start_key, RELATION_START_LABEL: 'Table', RELATION_END_KEY: self.end_key1, RELATION_END_LABEL: 'Table', RELATION_TYPE: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, RELATION_REVERSE_TYPE: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE }, { RELATION_START_KEY: self.start_key, RELATION_START_LABEL: 'Table', RELATION_END_KEY: self.end_key2, RELATION_END_LABEL: 'Table', RELATION_TYPE: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, RELATION_REVERSE_TYPE: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE } ] actual = [] relation = self.table_lineage.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship(relation) actual.append(serialized_relation) relation = self.table_lineage.create_next_relation() self.assertEqual(actual, expected_relations)
def test_create_relation(self) -> None: expected_relation1 = { RELATION_START_KEY: TABLE_KEY, RELATION_START_LABEL: 'Table', RELATION_END_KEY: owner1, RELATION_END_LABEL: User.USER_NODE_LABEL, RELATION_TYPE: OWNER_RELATION_TYPE, RELATION_REVERSE_TYPE: OWNER_OF_OBJECT_RELATION_TYPE, } expected_relation2 = { RELATION_START_KEY: TABLE_KEY, RELATION_START_LABEL: 'Table', RELATION_END_KEY: owner2, RELATION_END_LABEL: User.USER_NODE_LABEL, RELATION_TYPE: OWNER_RELATION_TYPE, RELATION_REVERSE_TYPE: OWNER_OF_OBJECT_RELATION_TYPE, } expected = [expected_relation1, expected_relation2] actual = [] relation = self.table_owner.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship(relation) actual.append(serialized_relation) relation = self.table_owner.create_next_relation() self.assertEqual(actual, expected)
def test_create_relation(self) -> None: relations = self.table_lineage.create_relation() self.assertEqual(len(relations), 2) start_key = '{db}://{cluster}.{schema}/{tbl}'.format(db=DB, schema=SCHEMA, tbl=TABLE, cluster=CLUSTER) end_key1 = '{db}://{cluster}.{schema}/{tbl}'.format(db=DB, schema='test_schema', tbl='test_table1', cluster=CLUSTER) relation = { RELATION_START_KEY: start_key, RELATION_START_LABEL: 'Table', RELATION_END_KEY: end_key1, RELATION_END_LABEL: 'Table', RELATION_TYPE: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, RELATION_REVERSE_TYPE: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE } actual_relations = [ neo4_serializer.serialize_relationship(relation) for relation in relations ] self.assertTrue(len(relations), 2) self.assertTrue(relation in actual_relations)
def test_create_relation(self) -> None: expected_relations = [{ RELATION_START_KEY: self.start_key, RELATION_START_LABEL: TableSource.LABEL, RELATION_END_KEY: self.end_key, RELATION_END_LABEL: 'Table', RELATION_TYPE: TableSource.SOURCE_TABLE_RELATION_TYPE, RELATION_REVERSE_TYPE: TableSource.TABLE_SOURCE_RELATION_TYPE }] actual = [] relation = self.table_source.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship( relation) actual.append(serialized_relation) relation = self.table_source.create_next_relation() self.assertEqual(expected_relations, actual)
def test_create_relation(self) -> None: expected_relation1 = { RELATION_START_KEY: owner1, RELATION_START_LABEL: User.USER_NODE_LABEL, RELATION_END_KEY: self.table_owner.get_metadata_model_key(), RELATION_END_LABEL: 'Table', RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE, RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE } expected_relation2 = { RELATION_START_KEY: owner2, RELATION_START_LABEL: User.USER_NODE_LABEL, RELATION_END_KEY: self.table_owner.get_metadata_model_key(), RELATION_END_LABEL: 'Table', RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE, RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE } expected = [expected_relation1, expected_relation2] actual = [] relation = self.table_owner.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship( relation) actual.append(serialized_relation) relation = self.table_owner.create_next_relation() self.assertEqual(actual, expected)
def test_create_relation(self) -> None: actual = [] relation = self.query_join_metadata.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship( relation) actual.append(serialized_relation) relation = self.query_join_metadata.create_next_relation() self.maxDiff = None expected_relations = [{ RELATION_END_KEY: self._expected_key, RELATION_END_LABEL: QueryExecutionsMetadata.NODE_LABEL, RELATION_REVERSE_TYPE: QueryExecutionsMetadata.INVERSE_QUERY_EXECUTION_RELATION_TYPE, RELATION_START_KEY: self.query_metadata.get_key_self(), RELATION_START_LABEL: QueryMetadata.NODE_LABEL, RELATION_TYPE: QueryExecutionsMetadata.QUERY_EXECUTION_RELATION_TYPE }] self.assertEquals(expected_relations, actual)
def test_col_badge_field(self) -> None: self.table_metadata4 = TableMetadata('hive', 'gold', 'test_schema4', 'test_table4', 'test_table4', [ ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0, ['col-badge1', 'col-badge2'])], is_view=False, attr1='uri', attr2='attr2') node_row = self.table_metadata4.next_node() actual = [] while node_row: serialized_node_row = neo4_serializer.serialize_node(node_row) actual.append(serialized_node_row) node_row = self.table_metadata4.next_node() self.assertEqual(actual[4].get('KEY'), 'col-badge1') self.assertEqual(actual[5].get('KEY'), 'col-badge2') relation_row = self.table_metadata4.next_relation() actual = [] while relation_row: serialized_relation_row = neo4_serializer.serialize_relationship(relation_row) actual.append(serialized_relation_row) relation_row = self.table_metadata4.next_relation() expected_col_badge_rel1 = {'END_KEY': 'col-badge1', 'START_LABEL': 'Column', 'END_LABEL': 'Badge', 'START_KEY': 'hive://gold.test_schema4/test_table4/test_id1', 'TYPE': 'HAS_BADGE', 'REVERSE_TYPE': 'BADGE_FOR'} expected_col_badge_rel2 = {'END_KEY': 'col-badge2', 'START_LABEL': 'Column', 'END_LABEL': 'Badge', 'START_KEY': 'hive://gold.test_schema4/test_table4/test_id1', 'TYPE': 'HAS_BADGE', 'REVERSE_TYPE': 'BADGE_FOR'} self.assertEqual(actual[4], expected_col_badge_rel1) self.assertEqual(actual[5], expected_col_badge_rel2)
def test_create_relation(self) -> None: actual = [] relation = self.user.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship( relation) actual.append(serialized_relation) relation = self.user.create_next_relation() start_key = '*****@*****.**' end_key = '*****@*****.**' expected_relations = [{ RELATION_START_KEY: start_key, RELATION_START_LABEL: User.USER_NODE_LABEL, RELATION_END_KEY: end_key, RELATION_END_LABEL: User.USER_NODE_LABEL, RELATION_TYPE: User.USER_MANAGER_RELATION_TYPE, RELATION_REVERSE_TYPE: User.MANAGER_USER_RELATION_TYPE }] self.assertTrue(expected_relations, actual)
def test_create_relation(self) -> None: actual = [] relation = self.badge_metada.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship( relation) actual.append(serialized_relation) relation = self.badge_metada.create_next_relation() relation1 = { RELATION_START_LABEL: self.badge_metada.start_label, RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL, RELATION_START_KEY: self.badge_metada.start_key, RELATION_END_KEY: BadgeMetadata.get_badge_key(badge1.name), RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE, RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, } relation2 = { RELATION_START_LABEL: self.badge_metada.start_label, RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL, RELATION_START_KEY: self.badge_metada.start_key, RELATION_END_KEY: BadgeMetadata.get_badge_key(badge2.name), RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE, RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, } expected = [relation1, relation2] self.assertEqual(expected, actual)
def test_dashboard_owner_relations(self) -> None: dashboard_usage = DashboardUsage( dashboard_group_id='dashboard_group_id', dashboard_id='dashboard_id', email='*****@*****.**', view_count=123, cluster='cluster_id', product='product_id') actual = dashboard_usage.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) expected: Dict[str, Any] = { 'read_count:UNQUOTED': 123, RELATION_END_KEY: '*****@*****.**', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: 'User', RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', RELATION_TYPE: 'READ_BY', RELATION_REVERSE_TYPE: 'READ' } assert actual is not None self.assertDictEqual(expected, actual_serialized) self.assertIsNone(dashboard_usage.create_next_relation())
def test_dashboard_owner_relations(self) -> None: dashboard_last_modified = DashboardLastModifiedTimestamp( last_modified_timestamp=123456789, cluster='cluster_id', product='product_id', dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id') actual = dashboard_last_modified.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) expected: Dict[str, Any] = { RELATION_END_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id' '/_last_modified_timestamp', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: 'Timestamp', RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', RELATION_TYPE: 'LAST_UPDATED_AT', RELATION_REVERSE_TYPE: 'LAST_UPDATED_TIME_OF' } assert actual is not None self.assertDictEqual(actual_serialized, expected) self.assertIsNone(dashboard_last_modified.create_next_relation())
def test_serialize(self) -> None: node_row = self.table_col_usage.next_node() actual = [] while node_row: actual.append(neo4_serializer.serialize_node(node_row)) node_row = self.table_col_usage.next_node() expected = [{'first_name': '', 'last_name': '', 'full_name': '', 'employee_type': '', 'is_active:UNQUOTED': True, 'updated_at:UNQUOTED': 0, 'LABEL': 'User', 'slack_id': '', 'KEY': '*****@*****.**', 'github_username': '', 'team_name': '', 'email': '*****@*****.**', 'role_name': ''}] self.assertEqual(expected, actual) rel_row = self.table_col_usage.next_relation() actual = [] while rel_row: actual.append(neo4_serializer.serialize_relationship(rel_row)) rel_row = self.table_col_usage.next_relation() expected = [{'read_count:UNQUOTED': 1, 'END_KEY': '*****@*****.**', 'START_LABEL': 'Table', 'END_LABEL': 'User', 'START_KEY': 'db://gold.scm/foo', 'TYPE': 'READ_BY', 'REVERSE_TYPE': 'READ'}] self.assertEqual(expected, actual)
def test_tags_populated_from_str(self) -> None: self.table_metadata5 = TableMetadata('hive', 'gold', 'test_schema5', 'test_table5', 'test_table5', [ ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0)], tags="tag3, tag4") # Test table tag field populated from str node_row = self.table_metadata5.next_node() actual = [] while node_row: node_row_serialized = neo4_serializer.serialize_node(node_row) actual.append(node_row_serialized) node_row = self.table_metadata5.next_node() self.assertEqual(actual[2].get('LABEL'), 'Tag') self.assertEqual(actual[2].get('KEY'), 'tag3') self.assertEqual(actual[3].get('KEY'), 'tag4') relation_row = self.table_metadata5.next_relation() actual = [] while relation_row: relation_row_serialized = neo4_serializer.serialize_relationship(relation_row) actual.append(relation_row_serialized) relation_row = self.table_metadata5.next_relation() # Table tag relationship expected_tab_tag_rel3 = {'END_KEY': 'tag3', 'START_LABEL': 'Table', 'END_LABEL': 'Tag', 'START_KEY': 'hive://gold.test_schema5/test_table5', 'TYPE': 'TAGGED_BY', 'REVERSE_TYPE': 'TAG'} expected_tab_tag_rel4 = {'END_KEY': 'tag4', 'START_LABEL': 'Table', 'END_LABEL': 'Tag', 'START_KEY': 'hive://gold.test_schema5/test_table5', 'TYPE': 'TAGGED_BY', 'REVERSE_TYPE': 'TAG'} self.assertEqual(actual[2], expected_tab_tag_rel3) self.assertEqual(actual[3], expected_tab_tag_rel4)
def test_create_relation(self) -> None: actual = [] relation = self.tableLastUpdated.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship(relation) actual.append(serialized_relation) relation = self.tableLastUpdated.create_next_relation()
def test_create_relation(self) -> None: relation = self.table_stats.create_relation() self.assertEquals(len(relation), 1) serialized_relation = neo4_serializer.serialize_relationship( relation[0]) self.assertEquals(serialized_relation, self.expected_relation_result)
def test_create_relation(self) -> None: relations = self.table_owner.create_relation() self.assertEqual(len(relations), 2) expected_relation1 = { RELATION_START_KEY: owner1, RELATION_START_LABEL: User.USER_NODE_LABEL, RELATION_END_KEY: self.table_owner.get_metadata_model_key(), RELATION_END_LABEL: 'Table', RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE, RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE } expected_relation2 = { RELATION_START_KEY: owner2, RELATION_START_LABEL: User.USER_NODE_LABEL, RELATION_END_KEY: self.table_owner.get_metadata_model_key(), RELATION_END_LABEL: 'Table', RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE, RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE } actual_relations = [ neo4_serializer.serialize_relationship(relation) for relation in relations ] self.assertTrue(expected_relation1 in actual_relations) self.assertTrue(expected_relation2 in actual_relations)
def test_create_relation(self) -> None: relations = self.badge_metada.create_relation() serialized_relations = [ neo4_serializer.serialize_relationship(relation) for relation in relations ] self.assertEqual(len(relations), 2) relation1 = { RELATION_START_LABEL: self.badge_metada.start_label, RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL, RELATION_START_KEY: self.badge_metada.start_key, RELATION_END_KEY: BadgeMetadata.get_badge_key(badge1.name), RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE, RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, } relation2 = { RELATION_START_LABEL: self.badge_metada.start_label, RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL, RELATION_START_KEY: self.badge_metada.start_key, RELATION_END_KEY: BadgeMetadata.get_badge_key(badge2.name), RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE, RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, } self.assertTrue(relation1 in serialized_relations) self.assertTrue(relation2 in serialized_relations)
def test_create_relation(self) -> None: actual = self.schema.create_next_relation() serialized_actual = neo4_serializer.serialize_relationship(actual) expected = {'END_KEY': 'db://cluster.schema/_description', 'START_LABEL': 'Schema', 'END_LABEL': 'Description', 'START_KEY': 'db://cluster.schema', 'TYPE': 'DESCRIPTION', 'REVERSE_TYPE': 'DESCRIPTION_OF'} self.assertEqual(expected, serialized_actual) self.assertIsNone(self.schema.create_next_relation())
def test_create_relation(self) -> None: actual = [] relation = self.application.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship(relation) actual.append(serialized_relation) relation = self.application.create_next_relation() self.assertEqual(actual, self.expected_relation_results)
def test_tags_field(self) -> None: self.table_metadata4 = TableMetadata( 'hive', 'gold', 'test_schema4', 'test_table4', 'test_table4', [ ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0) ], is_view=False, tags=['tag1', 'tag2'], attr1='uri', attr2='attr2') node_row = self.table_metadata4.next_node() actual = [] while node_row: node_row_serialized = neo4_serializer.serialize_node(node_row) actual.append(node_row_serialized) node_row = self.table_metadata4.next_node() self.assertEqual(actual[0].get('attr1'), 'uri') self.assertEqual(actual[0].get('attr2'), 'attr2') self.assertEqual(actual[2].get('LABEL'), 'Tag') self.assertEqual(actual[2].get('KEY'), 'tag1') self.assertEqual(actual[3].get('KEY'), 'tag2') relation_row = self.table_metadata4.next_relation() actual = [] while relation_row: relation_row_serialized = neo4_serializer.serialize_relationship( relation_row) actual.append(relation_row_serialized) relation_row = self.table_metadata4.next_relation() # Table tag relationship expected_tab_tag_rel1 = { 'END_KEY': 'tag1', 'START_LABEL': 'Table', 'END_LABEL': 'Tag', 'START_KEY': 'hive://gold.test_schema4/test_table4', 'TYPE': 'TAGGED_BY', 'REVERSE_TYPE': 'TAG' } expected_tab_tag_rel2 = { 'END_KEY': 'tag2', 'START_LABEL': 'Table', 'END_LABEL': 'Tag', 'START_KEY': 'hive://gold.test_schema4/test_table4', 'TYPE': 'TAGGED_BY', 'REVERSE_TYPE': 'TAG' } self.assertEqual(actual[2], expected_tab_tag_rel1) self.assertEqual(actual[3], expected_tab_tag_rel2)
def test_dashboard_table_relations(self) -> None: dashboard_table = DashboardTable(table_ids=['hive://gold.schema/table1'], cluster='cluster_id', product='product_id', dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id') actual = dashboard_table.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) actual_neptune_serialized = neptune_serializer.convert_relationship(actual) expected = {RELATION_END_KEY: 'hive://gold.schema/table1', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: 'Table', RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', RELATION_TYPE: 'DASHBOARD_WITH_TABLE', RELATION_REVERSE_TYPE: 'TABLE_OF_DASHBOARD'} neptune_forward_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', to_vertex_id='Table:hive://gold.schema/table1', label='DASHBOARD_WITH_TABLE' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', to_vertex_id='Table:hive://gold.schema/table1', label='DASHBOARD_WITH_TABLE' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.schema/table1', NEPTUNE_HEADER_LABEL: 'DASHBOARD_WITH_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:hive://gold.schema/table1', to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='TABLE_OF_DASHBOARD' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:hive://gold.schema/table1', to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='TABLE_OF_DASHBOARD' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.schema/table1', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_HEADER_LABEL: 'TABLE_OF_DASHBOARD', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } assert actual is not None self.assertDictEqual(actual_serialized, expected) self.assertDictEqual(actual_neptune_serialized[0], neptune_forward_expected) self.assertDictEqual(actual_neptune_serialized[1], neptune_reversed_expected)
def test_create_relation(self) -> None: dashboard_chart = DashboardChart(dashboard_group_id='dg_id', dashboard_id='d_id', query_id='q_id', chart_id='c_id', chart_name='c_name', chart_type='bar', ) actual = dashboard_chart.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) actual_neptune_serialized = neptune_serializer.convert_relationship(actual) start_key = '_dashboard://gold.dg_id/d_id/query/q_id' end_key = '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id' expected: Dict[str, Any] = { RELATION_END_KEY: end_key, RELATION_START_LABEL: 'Query', RELATION_END_LABEL: 'Chart', RELATION_START_KEY: start_key, RELATION_TYPE: 'HAS_CHART', RELATION_REVERSE_TYPE: 'CHART_OF' } neptune_forward_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=start_key, to_vertex_id=end_key, label='HAS_CHART' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: end_key, NEPTUNE_HEADER_LABEL: 'HAS_CHART', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=end_key, to_vertex_id=start_key, label='CHART_OF' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: 'CHART_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } assert actual is not None self.assertEqual(expected, actual_serialized) self.assertEqual(neptune_forward_expected, actual_neptune_serialized[0]) self.assertEqual(neptune_reversed_expected, actual_neptune_serialized[1]) self.assertIsNone(dashboard_chart.create_next_relation())
def test_serialize(self) -> None: col_readers = [ ColumnReader(database='db', cluster='gold', schema='scm', table='foo', column='*', user_email='*****@*****.**') ] table_col_usage = TableColumnUsage(col_readers=col_readers) node_row = table_col_usage.next_node() actual = [] while node_row: actual.append(neo4_serializer.serialize_node(node_row)) node_row = table_col_usage.next_node() expected = [{ 'first_name': '', 'last_name': '', 'full_name': '', 'employee_type': '', 'is_active:UNQUOTED': True, 'updated_at:UNQUOTED': 0, 'LABEL': 'User', 'slack_id': '', 'KEY': '*****@*****.**', 'github_username': '', 'team_name': '', 'email': '*****@*****.**', 'role_name': '' }] self.assertEqual(expected, actual) rel_row = table_col_usage.next_relation() actual = [] while rel_row: actual.append(neo4_serializer.serialize_relationship(rel_row)) rel_row = table_col_usage.next_relation() expected = [{ 'read_count:UNQUOTED': 1, 'END_KEY': '*****@*****.**', 'START_LABEL': 'Table', 'END_LABEL': 'User', 'START_KEY': 'db://gold.scm/foo', 'TYPE': 'READ_BY', 'REVERSE_TYPE': 'READ' }] self.assertEqual(expected, actual)
def test_dashboard_table_without_dot_as_name(self) -> None: dashboard_table = DashboardTable(table_ids=['bq-name://project-id.schema-name/table-name'], cluster='cluster_id', product='product_id', dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id') actual = dashboard_table.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) expected = {RELATION_END_KEY: 'bq-name://project-id.schema-name/table-name', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: 'Table', RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', RELATION_TYPE: 'DASHBOARD_WITH_TABLE', RELATION_REVERSE_TYPE: 'TABLE_OF_DASHBOARD'} assert actual is not None self.assertDictEqual(actual_serialized, expected)
def test_create_relation(self) -> None: actual = self.dashboard_query.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) expected = { RELATION_END_KEY: '_dashboard://gold.dg_id/d_id/query/q_id', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL, RELATION_START_KEY: '_dashboard://gold.dg_id/d_id', RELATION_TYPE: 'HAS_QUERY', RELATION_REVERSE_TYPE: 'QUERY_OF' } self.assertEqual(expected, actual_serialized)
def test_create_relation(self) -> None: actual = [] relation = self.query_join_metadata.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship( relation) actual.append(serialized_relation) relation = self.query_join_metadata.create_next_relation() expected_relations = [{ RELATION_END_KEY: self._expected_key, RELATION_END_LABEL: QueryJoinMetadata.NODE_LABEL, RELATION_REVERSE_TYPE: QueryJoinMetadata.INVERSE_COLUMN_JOIN_RELATION_TYPE, RELATION_START_KEY: 'hive://gold.test_schema1/test_table1/field', RELATION_START_LABEL: ColumnMetadata.COLUMN_NODE_LABEL, RELATION_TYPE: QueryJoinMetadata.COLUMN_JOIN_RELATION_TYPE }, { RELATION_END_KEY: self._expected_key, RELATION_END_LABEL: QueryJoinMetadata.NODE_LABEL, RELATION_REVERSE_TYPE: QueryJoinMetadata.INVERSE_COLUMN_JOIN_RELATION_TYPE, RELATION_START_KEY: 'hive://gold.test_schema1/test_table2/field', RELATION_START_LABEL: ColumnMetadata.COLUMN_NODE_LABEL, RELATION_TYPE: QueryJoinMetadata.COLUMN_JOIN_RELATION_TYPE }, { RELATION_END_KEY: self._expected_key, RELATION_END_LABEL: QueryJoinMetadata.NODE_LABEL, RELATION_REVERSE_TYPE: QueryJoinMetadata.INVERSE_QUERY_JOIN_RELATION_TYPE, RELATION_START_KEY: '748c28f86de411b1d2b9deb6ae105eba', RELATION_START_LABEL: QueryMetadata.NODE_LABEL, RELATION_TYPE: QueryJoinMetadata.QUERY_JOIN_RELATION_TYPE }] self.assertEquals(expected_relations, actual)
def test_dashboard_owner_relations(self) -> None: actual = self.dashboard_owner.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) expected = { RELATION_END_KEY: '*****@*****.**', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: 'User', RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', RELATION_TYPE: 'OWNER', RELATION_REVERSE_TYPE: 'OWNER_OF' } assert actual is not None self.assertDictEqual(actual_serialized, expected)
def test_create_relation_programmatic_description(self) -> None: schema = SchemaModel(schema_key='db://cluster.schema', schema='schema_name', description='foo', description_source='bar') actual = schema.create_next_relation() serialized_actual = neo4_serializer.serialize_relationship(actual) expected = { 'END_KEY': 'db://cluster.schema/_bar_description', 'START_LABEL': 'Schema', 'END_LABEL': 'Programmatic_Description', 'START_KEY': 'db://cluster.schema', 'TYPE': 'DESCRIPTION', 'REVERSE_TYPE': 'DESCRIPTION_OF' } self.assertEqual(expected, serialized_actual) self.assertIsNone(schema.create_next_relation())