def test_dashboard_owner_relations_neptune(self) -> None: actual = self.dashboard_owner.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', to_vertex_id='*****@*****.**', label='OWNER' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_RELATIONSHIP_HEADER_TO: '*****@*****.**', NEPTUNE_HEADER_LABEL: 'OWNER', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id='*****@*****.**', to_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='OWNER_OF' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: '*****@*****.**', NEPTUNE_RELATIONSHIP_HEADER_TO: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_HEADER_LABEL: 'OWNER_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } assert actual is not None self.assertDictEqual(actual_serialized[0], neptune_forward_expected) self.assertDictEqual(actual_serialized[1], neptune_reversed_expected)
def test_create_relation_neptune(self) -> None: actual = self.schema.create_next_relation() serialized_actual = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id='db://cluster.schema', to_vertex_id='db://cluster.schema/_description', label='DESCRIPTION' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'db://cluster.schema', NEPTUNE_RELATIONSHIP_HEADER_TO: 'db://cluster.schema/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id='db://cluster.schema/_description', to_vertex_id='db://cluster.schema', label='DESCRIPTION_OF' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'db://cluster.schema/_description', NEPTUNE_RELATIONSHIP_HEADER_TO: 'db://cluster.schema', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } self.assertDictEqual(serialized_actual[0], neptune_forward_expected) self.assertDictEqual(serialized_actual[1], neptune_reversed_expected)
def test_create_relation_neptune(self) -> None: relations = self.table_owner.create_relation() expected = [ { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=owner1, to_vertex_id=self.table_owner.get_metadata_model_key(), label=TableOwner.OWNER_TABLE_RELATION_TYPE ), NEPTUNE_RELATIONSHIP_HEADER_FROM: owner1, NEPTUNE_RELATIONSHIP_HEADER_TO: self.table_owner.get_metadata_model_key(), NEPTUNE_HEADER_LABEL: TableOwner.OWNER_TABLE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.table_owner.get_metadata_model_key(), to_vertex_id=owner1, label=TableOwner.TABLE_OWNER_RELATION_TYPE ), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.table_owner.get_metadata_model_key(), NEPTUNE_RELATIONSHIP_HEADER_TO: owner1, NEPTUNE_HEADER_LABEL: TableOwner.TABLE_OWNER_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } ] actual_relations = [ neptune_serializer.convert_relationship(relation) for relation in relations ] self.assertTrue(expected in actual_relations)
def test_create_relation_neptune(self) -> None: actual = [] next_relation = self.tableLastUpdated.create_next_relation() while next_relation: next_relation_serialized = neptune_serializer.convert_relationship( next_relation) actual.append(next_relation_serialized) next_relation = self.tableLastUpdated.create_next_relation() expected = [[{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:hive://gold.default/test_table', to_vertex_id= 'Timestamp:hive://gold.default/test_table/timestamp', label='LAST_UPDATED_AT'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:hive://gold.default/test_table', to_vertex_id= 'Timestamp:hive://gold.default/test_table/timestamp', label='LAST_UPDATED_AT'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.default/test_table', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Timestamp:hive://gold.default/test_table/timestamp', NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_AT', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Timestamp:hive://gold.default/test_table/timestamp', to_vertex_id='Table:hive://gold.default/test_table', label='LAST_UPDATED_TIME_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Timestamp:hive://gold.default/test_table/timestamp', to_vertex_id='Table:hive://gold.default/test_table', label='LAST_UPDATED_TIME_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Timestamp:hive://gold.default/test_table/timestamp', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.default/test_table', NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_TIME_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }]] self.assertEqual(actual, expected)
def test_create_relation_neptune(self) -> None: self.expected_relation_result = { RELATION_START_KEY: 'hive://gold.base/test/col/avg/', RELATION_START_LABEL: 'Stat', RELATION_END_KEY: 'hive://gold.base/test/col', RELATION_END_LABEL: 'Column', RELATION_TYPE: 'STAT_OF', RELATION_REVERSE_TYPE: 'STAT' } expected = [ [ { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Stat:hive://gold.base/test/col/avg/', to_vertex_id='Column:hive://gold.base/test/col', label='STAT_OF' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Stat:hive://gold.base/test/col/avg/', to_vertex_id='Column:hive://gold.base/test/col', label='STAT_OF' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Stat:hive://gold.base/test/col/avg/', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.base/test/col', NEPTUNE_HEADER_LABEL: 'STAT_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Column:hive://gold.base/test/col', to_vertex_id='Stat:hive://gold.base/test/col/avg/', label='STAT' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Column:hive://gold.base/test/col', to_vertex_id='Stat:hive://gold.base/test/col/avg/', label='STAT' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.base/test/col', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Stat:hive://gold.base/test/col/avg/', NEPTUNE_HEADER_LABEL: 'STAT', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } ] ] actual = [] next_relation = self.table_stats.create_next_relation() while next_relation: serialized_relation = neptune_serializer.convert_relationship(next_relation) actual.append(serialized_relation) next_relation = self.table_stats.create_next_relation() self.assertListEqual(actual, expected)
def test_create_relation_neptune(self) -> None: actual = [] relation = self.user.create_next_relation() while relation: serialized = neptune_serializer.convert_relationship(relation) actual.append(serialized) relation = self.user.create_next_relation() start_key = 'User:{email}'.format(email='*****@*****.**') end_key = 'User:{email}'.format(email='*****@*****.**') expected = [[{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=start_key, to_vertex_id=end_key, label=User.USER_MANAGER_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=start_key, to_vertex_id=end_key, label=User.USER_MANAGER_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: end_key, NEPTUNE_HEADER_LABEL: User.USER_MANAGER_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=end_key, to_vertex_id=start_key, label=User.MANAGER_USER_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=end_key, to_vertex_id=start_key, label=User.MANAGER_USER_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: User.MANAGER_USER_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }]] self.assertListEqual(actual, expected)
def test_neptune_serialize(self) -> None: rel_row = self.table_col_usage.next_relation() actual = [] while rel_row: actual.append(neptune_serializer.convert_relationship(rel_row)) rel_row = self.table_col_usage.next_relation() expected = [[{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:db://gold.scm/foo', to_vertex_id='User:[email protected]', label='READ_BY'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:db://gold.scm/foo', to_vertex_id='User:[email protected]', label='READ_BY'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:db://gold.scm/foo', NEPTUNE_RELATIONSHIP_HEADER_TO: 'User:[email protected]', NEPTUNE_HEADER_LABEL: 'READ_BY', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'read_count:Long(single)': 1 }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='User:[email protected]', to_vertex_id='Table:db://gold.scm/foo', label='READ'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='User:[email protected]', to_vertex_id='Table:db://gold.scm/foo', label='READ'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'User:[email protected]', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:db://gold.scm/foo', NEPTUNE_HEADER_LABEL: 'READ', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'read_count:Long(single)': 1 }]] self.maxDiff = None self.assertListEqual(expected, actual)
def test_dashboard_table_relations(self) -> None: dashboard_table = DashboardTable(table_ids=['hive://gold.schema/table1'], cluster='cluster_id', product='product_id', dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id') actual = dashboard_table.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) actual_neptune_serialized = neptune_serializer.convert_relationship(actual) expected = {RELATION_END_KEY: 'hive://gold.schema/table1', RELATION_START_LABEL: 'Dashboard', RELATION_END_LABEL: 'Table', RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', RELATION_TYPE: 'DASHBOARD_WITH_TABLE', RELATION_REVERSE_TYPE: 'TABLE_OF_DASHBOARD'} neptune_forward_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', to_vertex_id='Table:hive://gold.schema/table1', label='DASHBOARD_WITH_TABLE' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', to_vertex_id='Table:hive://gold.schema/table1', label='DASHBOARD_WITH_TABLE' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.schema/table1', NEPTUNE_HEADER_LABEL: 'DASHBOARD_WITH_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:hive://gold.schema/table1', to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='TABLE_OF_DASHBOARD' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:hive://gold.schema/table1', to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='TABLE_OF_DASHBOARD' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.schema/table1', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_HEADER_LABEL: 'TABLE_OF_DASHBOARD', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } assert actual is not None self.assertDictEqual(actual_serialized, expected) self.assertDictEqual(actual_neptune_serialized[0], neptune_forward_expected) self.assertDictEqual(actual_neptune_serialized[1], neptune_reversed_expected)
def test_create_relation(self) -> None: dashboard_chart = DashboardChart(dashboard_group_id='dg_id', dashboard_id='d_id', query_id='q_id', chart_id='c_id', chart_name='c_name', chart_type='bar', ) actual = dashboard_chart.create_next_relation() actual_serialized = neo4_serializer.serialize_relationship(actual) actual_neptune_serialized = neptune_serializer.convert_relationship(actual) start_key = '_dashboard://gold.dg_id/d_id/query/q_id' end_key = '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id' expected: Dict[str, Any] = { RELATION_END_KEY: end_key, RELATION_START_LABEL: 'Query', RELATION_END_LABEL: 'Chart', RELATION_START_KEY: start_key, RELATION_TYPE: 'HAS_CHART', RELATION_REVERSE_TYPE: 'CHART_OF' } neptune_forward_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=start_key, to_vertex_id=end_key, label='HAS_CHART' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: end_key, NEPTUNE_HEADER_LABEL: 'HAS_CHART', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=end_key, to_vertex_id=start_key, label='CHART_OF' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: 'CHART_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } assert actual is not None self.assertEqual(expected, actual_serialized) self.assertEqual(neptune_forward_expected, actual_neptune_serialized[0]) self.assertEqual(neptune_reversed_expected, actual_neptune_serialized[1]) self.assertIsNone(dashboard_chart.create_next_relation())
def test_create_relation_neptune(self) -> None: actual = [] relation = self.watermark.create_next_relation() while relation: serialized_relation = neptune_serializer.convert_relationship( relation) actual.append(serialized_relation) relation = self.watermark.create_next_relation() expected = [[{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Watermark:" + self.start_key, to_vertex_id="Table:" + self.end_key, label='BELONG_TO_TABLE'), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Watermark:" + self.start_key, to_vertex_id="Table:" + self.end_key, label='BELONG_TO_TABLE'), NEPTUNE_RELATIONSHIP_HEADER_FROM: "Watermark:" + self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: "Table:" + self.end_key, NEPTUNE_HEADER_LABEL: 'BELONG_TO_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.end_key, to_vertex_id="Watermark:" + self.start_key, label='WATERMARK'), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.end_key, to_vertex_id="Watermark:" + self.start_key, label='WATERMARK'), NEPTUNE_RELATIONSHIP_HEADER_FROM: "Table:" + self.end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: "Watermark:" + self.start_key, NEPTUNE_HEADER_LABEL: 'WATERMARK', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }]] self.assertListEqual(actual, expected)
def test_create_relation_neptune(self) -> None: actual = [] relation = self.table_source.create_next_relation() while relation: serialized_relation = neptune_serializer.convert_relationship( relation) actual.append(serialized_relation) relation = self.table_source.create_next_relation() expected = [[{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Source:" + self.start_key, to_vertex_id="Table:" + self.end_key, label=TableSource.SOURCE_TABLE_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Source:" + self.start_key, to_vertex_id="Table:" + self.end_key, label=TableSource.SOURCE_TABLE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: "Source:" + self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: "Table:" + self.end_key, NEPTUNE_HEADER_LABEL: TableSource.SOURCE_TABLE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.end_key, to_vertex_id="Source:" + self.start_key, label=TableSource.TABLE_SOURCE_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.end_key, to_vertex_id="Source:" + self.start_key, label=TableSource.TABLE_SOURCE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: "Table:" + self.end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: "Source:" + self.start_key, NEPTUNE_HEADER_LABEL: TableSource.TABLE_SOURCE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }]] self.assertListEqual(expected, actual)
def test_dashboard_owner_relations_neptune(self) -> None: actual = self.dashboard_last_modified.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:' + self.expected_dashboard_key, to_vertex_id='Timestamp:' + self.expected_ts_key, label='LAST_UPDATED_AT'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:' + self.expected_dashboard_key, to_vertex_id='Timestamp:' + self.expected_ts_key, label='LAST_UPDATED_AT'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:' + self.expected_dashboard_key, NEPTUNE_RELATIONSHIP_HEADER_TO: 'Timestamp:' + self.expected_ts_key, NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_AT', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Timestamp:' + self.expected_ts_key, to_vertex_id='Dashboard:' + self.expected_dashboard_key, label='LAST_UPDATED_TIME_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Timestamp:' + self.expected_ts_key, to_vertex_id='Dashboard:' + self.expected_dashboard_key, label='LAST_UPDATED_TIME_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Timestamp:' + self.expected_ts_key, NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:' + self.expected_dashboard_key, NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_TIME_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } self.maxDiff = None assert actual is not None self.assertDictEqual(actual_serialized[0], neptune_forward_expected) self.assertDictEqual(actual_serialized[1], neptune_reversed_expected) self.assertIsNone(self.dashboard_last_modified.create_next_relation())
def test_create_relation_neptune(self) -> None: actual = self.dashboard_query.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', to_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', label='HAS_QUERY'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', to_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', label='HAS_QUERY'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.dg_id/d_id', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Query:_dashboard://gold.dg_id/d_id/query/q_id', NEPTUNE_HEADER_LABEL: 'HAS_QUERY', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', to_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', label='QUERY_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', to_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', label='QUERY_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Query:_dashboard://gold.dg_id/d_id/query/q_id', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.dg_id/d_id', NEPTUNE_HEADER_LABEL: 'QUERY_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } assert actual is not None self.assertDictEqual(actual_serialized[0], neptune_forward_expected) self.assertDictEqual(actual_serialized[1], neptune_reversed_expected)
def test_create_relations_neptune(self) -> None: application_id = 'Application:application://gold.airflow/event_test/hive.default.test_table' table_id = 'Table:hive://gold.default/test_table' neptune_forward_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=table_id, to_vertex_id=application_id, label='DERIVED_FROM' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=table_id, to_vertex_id=application_id, label='DERIVED_FROM' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: table_id, NEPTUNE_RELATIONSHIP_HEADER_TO: application_id, NEPTUNE_HEADER_LABEL: 'DERIVED_FROM', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=application_id, to_vertex_id=table_id, label='GENERATES' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=application_id, to_vertex_id=table_id, label='GENERATES' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: application_id, NEPTUNE_RELATIONSHIP_HEADER_TO: table_id, NEPTUNE_HEADER_LABEL: 'GENERATES', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_expected = [[neptune_forward_expected, neptune_reversed_expected]] actual = [] next_relation = self.application.create_next_relation() while next_relation: serialized_next_relation = neptune_serializer.convert_relationship(next_relation) actual.append(serialized_next_relation) next_relation = self.application.create_next_relation() self.assertEqual(actual, neptune_expected)
def test_create_next_relation_neptune(self) -> None: neptune_forward_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id='hive://gold.default/test_table', to_vertex_id= 'application://gold.airflow/event_test/hive.default.test_table', label='DERIVED_FROM'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.default/test_table', NEPTUNE_RELATIONSHIP_HEADER_TO: 'application://gold.airflow/event_test/hive.default.test_table', NEPTUNE_HEADER_LABEL: 'DERIVED_FROM', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id= 'application://gold.airflow/event_test/hive.default.test_table', to_vertex_id='hive://gold.default/test_table', label='GENERATES'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'application://gold.airflow/event_test/hive.default.test_table', NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.default/test_table', NEPTUNE_HEADER_LABEL: 'GENERATES', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } next_relation = self.application.create_next_relation() serialized_next_relations = neptune_serializer.convert_relationship( next_relation) self.assertDictEqual(serialized_next_relations[0], neptune_forward_expected) self.assertDictEqual(serialized_next_relations[1], neptune_reversed_expected)
def load(self, csv_serializable: GraphSerializable) -> None: """ Writes GraphSerializable into CSV files. There are multiple CSV files that this method writes. This is because there're not only node and relationship, but also it can also have different nodes, and relationships. Common pattern for both nodes and relations: 1. retrieve csv row (a dict where keys represent a header, values represent a row) 2. using this dict to get a appropriate csv writer and write to it. 3. repeat 1 and 2 :param csv_serializable: :return: """ node = csv_serializable.next_node() while node: node.attributes[ PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag node_dict = neptune_serializer.convert_node(node) if node_dict: key = (node.label, len(node_dict)) file_suffix = '{}_{}'.format(*key) node_writer = self._get_writer(node_dict, self._node_file_mapping, key, self._node_dir, file_suffix) node_writer.writerow(node_dict) node = csv_serializable.next_node() relation = csv_serializable.next_relation() while relation: relation.attributes[ PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag relation_dicts = neptune_serializer.convert_relationship(relation) if relation_dicts: key2 = (relation.start_label, relation.end_label, relation.type, len(relation_dicts[0])) file_suffix = '{}_{}_{}'.format(key2[0], key2[1], key2[2]) relation_writer = self._get_writer(relation_dicts[0], self._relation_file_mapping, key2, self._relation_dir, file_suffix) relation_writer.writerows(relation_dicts) relation = csv_serializable.next_relation()
def test_serialize_neptune(self) -> None: node_row = self.table_metadata.next_node() actual = [] while node_row: node_row_serialized = neptune_serializer.convert_node(node_row) actual.append(node_row_serialized) node_row = self.table_metadata.next_node() self.assertEqual(EXPECTED_NEPTUNE_NODES, actual) relation_row = self.table_metadata.next_relation() neptune_actual: List[List[Dict]] = [] while relation_row: relation_row_serialized = neptune_serializer.convert_relationship(relation_row) neptune_actual.append(relation_row_serialized) relation_row = self.table_metadata.next_relation() self.maxDiff = None self.assertEqual(EXPECTED_RELATIONSHIPS_NEPTUNE, neptune_actual)
def test_dashboard_owner_relations_neptune(self) -> None: dashboard_usage = DashboardUsage(dashboard_group_id='dashboard_group_id', dashboard_id='dashboard_id', email='*****@*****.**', view_count=123, cluster='cluster_id', product='product_id') actual = dashboard_usage.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', to_vertex_id='*****@*****.**', label='READ_BY' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_RELATIONSHIP_HEADER_TO: '*****@*****.**', NEPTUNE_HEADER_LABEL: 'READ_BY', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'read_count:Long(single)': 123 } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id='*****@*****.**', to_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='READ' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: '*****@*****.**', NEPTUNE_RELATIONSHIP_HEADER_TO: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_HEADER_LABEL: 'READ', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'read_count:Long(single)': 123, } assert actual is not None self.assertDictEqual(neptune_forward_expected, actual_serialized[0]) self.assertDictEqual(neptune_reversed_expected, actual_serialized[1]) self.assertIsNone(dashboard_usage.create_next_relation())
def test_create_relation_neptune(self) -> None: relations = self.table_source.create_relation() serialized_relations = neptune_serializer.convert_relationship( relations[0]) expected = [{ NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.start_key, to_vertex_id=self.end_key, label=TableSource.SOURCE_TABLE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: self.end_key, NEPTUNE_HEADER_LABEL: TableSource.SOURCE_TABLE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.end_key, to_vertex_id=self.start_key, label=TableSource.TABLE_SOURCE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: self.start_key, NEPTUNE_HEADER_LABEL: TableSource.TABLE_SOURCE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }] self.assertListEqual(expected, serialized_relations)
def test_create_relation_neptune(self) -> None: relation = self.watermark.create_relation() serialized_relation = neptune_serializer.convert_relationship( relation[0]) expected = [{ NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.start_key, to_vertex_id=self.end_key, label='BELONG_TO_TABLE'), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: self.end_key, NEPTUNE_HEADER_LABEL: 'BELONG_TO_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.end_key, to_vertex_id=self.start_key, label='WATERMARK'), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: self.start_key, NEPTUNE_HEADER_LABEL: 'WATERMARK', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }] self.assertListEqual(serialized_relation, expected)
def test_create_relation_neptune(self) -> None: relations = self.user.create_relation() serialized = neptune_serializer.convert_relationship(relations[0]) start_key = '{email}'.format(email='*****@*****.**') end_key = '{email}'.format(email='*****@*****.**') expected = [ { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=start_key, to_vertex_id=end_key, label=User.USER_MANAGER_RELATION_TYPE ), NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: end_key, NEPTUNE_HEADER_LABEL: User.USER_MANAGER_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=end_key, to_vertex_id=start_key, label=User.MANAGER_USER_RELATION_TYPE ), NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: User.MANAGER_USER_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } ] self.assertListEqual(serialized, expected)
def test_create_relation_neptune(self) -> None: relations = self.badge_metada.create_relation() serialized_relations: List[Dict] = sum([ neptune_serializer.convert_relationship(rel) for rel in relations ], []) neptune_forward_expected_1 = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.badge_metada.start_key, to_vertex_id=BadgeMetadata.get_badge_key(badge1.name), label=BadgeMetadata.BADGE_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.badge_metada.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: BadgeMetadata.get_badge_key(badge1.name), NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected_1 = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=BadgeMetadata.get_badge_key(badge1.name), to_vertex_id=self.badge_metada.start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: BadgeMetadata.get_badge_key(badge1.name), NEPTUNE_RELATIONSHIP_HEADER_TO: self.badge_metada.start_key, NEPTUNE_HEADER_LABEL: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_forward_expected_2 = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.badge_metada.start_key, to_vertex_id=BadgeMetadata.get_badge_key(badge2.name), label=BadgeMetadata.BADGE_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.badge_metada.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: BadgeMetadata.get_badge_key(badge2.name), NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected_2 = { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=BadgeMetadata.get_badge_key(badge2.name), to_vertex_id=self.badge_metada.start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: BadgeMetadata.get_badge_key(badge2.name), NEPTUNE_RELATIONSHIP_HEADER_TO: self.badge_metada.start_key, NEPTUNE_HEADER_LABEL: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } self.assertTrue(neptune_forward_expected_1 in serialized_relations) self.assertTrue(neptune_reversed_expected_1 in serialized_relations) self.assertTrue(neptune_forward_expected_2 in serialized_relations) self.assertTrue(neptune_reversed_expected_2 in serialized_relations)
def test_full_dashboard_example_neptune(self) -> None: expected_neptune_rels = [ [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Cluster:_dashboard://gold', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_GROUP'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Cluster:_dashboard://gold', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_GROUP'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Cluster:_dashboard://gold', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DASHBOARD_GROUP', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id='Cluster:_dashboard://gold', label='DASHBOARD_GROUP_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id='Cluster:_dashboard://gold', label='DASHBOARD_GROUP_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Cluster:_dashboard://gold', NEPTUNE_HEADER_LABEL: 'DASHBOARD_GROUP_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', label='DESCRIPTION'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', label='DESCRIPTION'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DESCRIPTION_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DESCRIPTION_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DASHBOARD_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DASHBOARD'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DASHBOARD'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'DASHBOARD', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', label='DESCRIPTION'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', label='DESCRIPTION'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DESCRIPTION_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DESCRIPTION_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:test_tag', label='TAG'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:test_tag', label='TAG'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Tag:test_tag', NEPTUNE_HEADER_LABEL: 'TAG', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:test_tag', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:test_tag', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Tag:test_tag', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'TAG_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:tag2', label='TAG'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:tag2', label='TAG'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Tag:tag2', NEPTUNE_HEADER_LABEL: 'TAG', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:tag2', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:tag2', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Tag:tag2', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'TAG_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], ] expected_neptune_nodes = [ { NEPTUNE_HEADER_ID: 'Cluster:_dashboard://gold', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold', NEPTUNE_HEADER_LABEL: 'Cluster', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'gold' }, { NEPTUNE_HEADER_ID: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'Dashboard', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'Agent', 'dashboard_url:String(single)': 'https://foo.bar/dashboard_group/foo/dashboard/bar', 'created_timestamp:Long(single)': 123456789, }, { NEPTUNE_HEADER_ID: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'Dashboardgroup', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'Product - Jobs.cz', 'dashboard_group_url:String(single)': 'https://foo.bar/dashboard_group/foo' }, { NEPTUNE_HEADER_ID: 'Description:_dashboard://gold.Product - Jobs.cz/_description', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'description:String(single)': 'foo dashboard group description', }, { NEPTUNE_HEADER_ID: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'description:String(single)': 'Agent dashboard description' }, { NEPTUNE_HEADER_ID: 'Tag:test_tag', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'test_tag', NEPTUNE_HEADER_LABEL: 'Tag', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'tag_type:String(single)': 'dashboard' }, { NEPTUNE_HEADER_ID: 'Tag:tag2', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'tag2', NEPTUNE_HEADER_LABEL: 'Tag', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'tag_type:String(single)': 'dashboard' }, ] self.maxDiff = None node_row = self.full_dashboard_metadata.next_node() actual = [] while node_row: node_serialized = neptune_serializer.convert_node(node_row) actual.append(node_serialized) node_row = self.full_dashboard_metadata.next_node() self.assertEqual(expected_neptune_nodes, actual) relation_row = self.full_dashboard_metadata.next_relation() neptune_actual: List[List[Dict]] = [] while relation_row: relation_serialized = neptune_serializer.convert_relationship( relation_row) neptune_actual.append(relation_serialized) relation_row = self.full_dashboard_metadata.next_relation() self.assertEqual(expected_neptune_rels, neptune_actual)
def test_create_relation_neptune(self) -> None: actual = [] relation = self.badge_metada.create_next_relation() while relation: serialized_relations = neptune_serializer.convert_relationship( relation) actual.append(serialized_relations) relation = self.badge_metada.create_next_relation() badge_id_1 = BadgeMetadata.BADGE_NODE_LABEL + ':' + BadgeMetadata.get_badge_key( badge1.name) badge_id_2 = BadgeMetadata.BADGE_NODE_LABEL + ':' + BadgeMetadata.get_badge_key( badge2.name) start_key = self.badge_metada.start_label + ':' + self.badge_metada.start_key neptune_forward_expected_1 = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=start_key, to_vertex_id=badge_id_1, label=BadgeMetadata.BADGE_RELATION_TYPE, ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=start_key, to_vertex_id=badge_id_1, label=BadgeMetadata.BADGE_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: badge_id_1, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected_1 = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=badge_id_1, to_vertex_id=start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=badge_id_1, to_vertex_id=start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: badge_id_1, NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_forward_expected_2 = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=start_key, to_vertex_id=badge_id_2, label=BadgeMetadata.BADGE_RELATION_TYPE, ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=start_key, to_vertex_id=badge_id_2, label=BadgeMetadata.BADGE_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: badge_id_2, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected_2 = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=badge_id_2, to_vertex_id=start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=badge_id_2, to_vertex_id=start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: badge_id_2, NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } expected = [[neptune_forward_expected_1, neptune_reversed_expected_1], [neptune_forward_expected_2, neptune_reversed_expected_2]] self.assertEqual(expected, actual)
def test_create_relation_neptune(self) -> None: expected = [[{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(), label=TableOwner.OWNER_TABLE_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(), label=TableOwner.OWNER_TABLE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: "User:"******"Table:" + self.table_owner.get_metadata_model_key(), NEPTUNE_HEADER_LABEL: TableOwner.OWNER_TABLE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), to_vertex_id="User:"******"{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), to_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(), NEPTUNE_RELATIONSHIP_HEADER_TO: "User:"******"{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(), label=TableOwner.OWNER_TABLE_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(), label=TableOwner.OWNER_TABLE_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: "User:"******"Table:" + self.table_owner.get_metadata_model_key(), NEPTUNE_HEADER_LABEL: TableOwner.OWNER_TABLE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), to_vertex_id="User:"******"{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), to_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(), NEPTUNE_RELATIONSHIP_HEADER_TO: "User:" + owner2, NEPTUNE_HEADER_LABEL: TableOwner.TABLE_OWNER_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }]] actual = [] relation = self.table_owner.create_next_relation() while relation: serialized_relation = neptune_serializer.convert_relationship( relation) actual.append(serialized_relation) relation = self.table_owner.create_next_relation() self.assertEqual(expected, actual)
def test_create_relations_neptune(self) -> None: for tc in self.test_cases: application_id = f'Application:{tc.application.application_key}' table_id = 'Table:hive://gold.default/test_table' neptune_forward_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=table_id, to_vertex_id=application_id, label=tc.expected_relation_results[0][RELATION_TYPE], ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=table_id, to_vertex_id=application_id, label=tc.expected_relation_results[0][RELATION_TYPE], ), NEPTUNE_RELATIONSHIP_HEADER_FROM: table_id, NEPTUNE_RELATIONSHIP_HEADER_TO: application_id, NEPTUNE_HEADER_LABEL: tc.expected_relation_results[0][RELATION_TYPE], NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=application_id, to_vertex_id=table_id, label=tc.expected_relation_results[0] [RELATION_REVERSE_TYPE], ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=application_id, to_vertex_id=table_id, label=tc.expected_relation_results[0] [RELATION_REVERSE_TYPE], ), NEPTUNE_RELATIONSHIP_HEADER_FROM: application_id, NEPTUNE_RELATIONSHIP_HEADER_TO: table_id, NEPTUNE_HEADER_LABEL: tc.expected_relation_results[0][RELATION_REVERSE_TYPE], NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_expected = [[ neptune_forward_expected, neptune_reversed_expected ]] actual = [] next_relation = tc.application.create_next_relation() while next_relation: serialized_next_relation = neptune_serializer.convert_relationship( next_relation) actual.append(serialized_next_relation) next_relation = tc.application.create_next_relation() self.assertEqual(actual, neptune_expected)
def test_create_relation_neptune(self) -> None: expected = [[{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.start_key, to_vertex_id='Table:' + self.end_key1, label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.start_key, to_vertex_id='Table:' + self.end_key1, label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:' + self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:' + self.end_key1, NEPTUNE_HEADER_LABEL: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.end_key1, to_vertex_id='Table:' + self.start_key, label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.end_key1, to_vertex_id='Table:' + self.start_key, label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:' + self.end_key1, NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:' + self.start_key, NEPTUNE_HEADER_LABEL: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.start_key, to_vertex_id='Table:' + self.end_key2, label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.start_key, to_vertex_id='Table:' + self.end_key2, label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:' + self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:' + self.end_key2, NEPTUNE_HEADER_LABEL: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.end_key2, to_vertex_id='Table:' + self.start_key, label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE ), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Table:' + self.end_key2, to_vertex_id='Table:' + self.start_key, label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE ), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:' + self.end_key2, NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:' + self.start_key, NEPTUNE_HEADER_LABEL: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }]] actual = [] relation = self.table_lineage.create_next_relation() while relation: serialized_relation = neptune_serializer.convert_relationship( relation) actual.append(serialized_relation) relation = self.table_lineage.create_next_relation() self.assertEqual(actual, expected)