def test_create_nodes_neptune(self) -> None: schema_node = self.schema.create_next_node() expected_serialized_schema_node = { NEPTUNE_HEADER_ID: 'Schema:db://cluster.schema', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'db://cluster.schema', NEPTUNE_HEADER_LABEL: 'Schema', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'schema_name', } serialized_schema_node = neptune_serializer.convert_node(schema_node) self.assertDictEqual( expected_serialized_schema_node, serialized_schema_node ) schema_desc_node = self.schema.create_next_node() excepted_serialized_schema_desc_node = { NEPTUNE_HEADER_ID: 'Description:db://cluster.schema/_description', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'db://cluster.schema/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'description_source:String(single)': 'description', 'description:String(single)': 'foo', } serialized_schema_desc_node = neptune_serializer.convert_node(schema_desc_node) self.assertDictEqual( excepted_serialized_schema_desc_node, serialized_schema_desc_node )
def test_create_next_node_neptune(self) -> None: actual = [] next_node = self.application.create_next_node() while next_node: serialized_next_node = neptune_serializer.convert_node(next_node) actual.append(serialized_next_node) next_node = self.application.create_next_node() node_id = 'Application:application://gold.airflow/event_test/hive.default.test_table' neptune_expected = [{ NEPTUNE_HEADER_ID: node_id, METADATA_KEY_PROPERTY_NAME: node_id, NEPTUNE_HEADER_LABEL: 'Application', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'application_url:String(single)': 'airflow_host.net/admin/airflow/tree?dag_id=event_test', 'id:String(single)': 'event_test/hive.default.test_table', 'name:String(single)': 'Airflow', 'description:String(single)': 'Airflow with id event_test/hive.default.test_table', }] self.assertEqual(neptune_expected, actual)
def test_create_nodes_neptune(self) -> None: node_id = TableLastUpdated.LAST_UPDATED_NODE_LABEL + ":" + self.tableLastUpdated.get_last_updated_model_key( ) expected_nodes = [{ NEPTUNE_HEADER_ID: node_id, METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: self.tableLastUpdated.get_last_updated_model_key(), NEPTUNE_HEADER_LABEL: TableLastUpdated.LAST_UPDATED_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'last_updated_timestamp', 'last_updated_timestamp:Long(single)': 25195665, timestamp_constants.TIMESTAMP_PROPERTY + ":Long(single)": 25195665, }] actual = [] next_node = self.tableLastUpdated.create_next_node() while next_node: next_node_serialized = neptune_serializer.convert_node(next_node) actual.append(next_node_serialized) next_node = self.tableLastUpdated.create_next_node() self.assertEqual(actual, expected_nodes)
def test_create_nodes_neptune(self) -> None: expected_node1 = { NEPTUNE_HEADER_ID: "User:"******":String(single)": owner1 } expected_node2 = { NEPTUNE_HEADER_ID: "User:"******":String(single)": owner2 } expected = [expected_node1, expected_node2] actual = [] node = self.table_owner.create_next_node() while node: serialized_node = neptune_serializer.convert_node(node) actual.append(serialized_node) node = self.table_owner.create_next_node() self.assertEqual(actual, expected)
def test_create_nodes_neptune(self) -> None: for tc in self.test_cases: actual = [] next_node = tc.application.create_next_node() while next_node: serialized_next_node = neptune_serializer.convert_node( next_node) actual.append(serialized_next_node) next_node = tc.application.create_next_node() node_id = f'Application:{tc.application.application_key}' node_key = tc.application.application_key neptune_expected = [{ NEPTUNE_HEADER_ID: node_id, METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: node_key, NEPTUNE_HEADER_LABEL: 'Application', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'application_url:String(single)': tc.application.application_url, 'id:String(single)': tc.application.application_id, 'name:String(single)': tc.application.application_type, 'description:String(single)': tc.application.application_description, }] self.assertEqual(neptune_expected, actual)
def test_create_nodes_neptune(self) -> None: nodes = self.badge_metada.create_nodes() serialized_nodes = [ neptune_serializer.convert_node(node) for node in nodes ] expected_node1 = { NEPTUNE_HEADER_ID: BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge1.name), NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, BadgeMetadata.BADGE_CATEGORY + ':String(single)': badge1.category } expected_node2 = { NEPTUNE_HEADER_ID: BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge2.name), NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, BadgeMetadata.BADGE_CATEGORY + ':String(single)': badge2.category } self.assertTrue(expected_node1 in serialized_nodes) self.assertTrue(expected_node2 in serialized_nodes)
def test_create_nodes_neptune(self) -> None: expected_serialized_node_results = [{ NEPTUNE_HEADER_ID: 'Watermark:' + self.start_key, METADATA_KEY_PROPERTY_NAME: 'Watermark:' + self.start_key, NEPTUNE_HEADER_LABEL: 'Watermark', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'partition_key:String(single)': 'ds', 'partition_value:String(single)': '2017-09-18/feature_id=9', 'create_time:String(single)': '2017-09-18T00:00:00' }] actual = [] node = self.watermark.create_next_node() while node: serialized_node = neptune_serializer.convert_node(node) actual.append(serialized_node) node = self.watermark.create_next_node() self.assertEqual(expected_serialized_node_results, actual)
def test_create_nodes_neptune(self) -> None: actual = [] next_node = self.table_stats.create_next_node() while next_node: serialized_node = neptune_serializer.convert_node(next_node) actual.append(serialized_node) next_node = self.table_stats.create_next_node() expected_neptune_nodes = [{ NEPTUNE_HEADER_ID: 'Stat:hive://gold.base/test/col/avg/', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'hive://gold.base/test/col/avg/', NEPTUNE_HEADER_LABEL: 'Stat', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'stat_val:String(single)': '1', 'stat_type:String(single)': 'avg', 'start_epoch:String(single)': '1', 'end_epoch:String(single)': '2', }] self.assertEqual(actual, expected_neptune_nodes)
def test_create_nodes_neptune(self) -> None: nodes = self.table_owner.create_nodes() expected_node1 = { NEPTUNE_HEADER_ID: User.USER_NODE_KEY_FORMAT.format(email=owner1), NEPTUNE_HEADER_LABEL: User.USER_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, User.USER_NODE_EMAIL + ":String(single)": owner1 } expected_node2 = { NEPTUNE_HEADER_ID: User.USER_NODE_KEY_FORMAT.format(email=owner2), NEPTUNE_HEADER_LABEL: User.USER_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, User.USER_NODE_EMAIL + ":String(single)": owner2 } actual_nodes = [ neptune_serializer.convert_node(node) for node in nodes ] self.assertTrue(expected_node1 in actual_nodes) self.assertTrue(expected_node2 in actual_nodes)
def test_neptune_dashboard_timestamp_nodes(self) -> None: actual = self.dashboard_last_modified.create_next_node() actual_neptune_serialized = neptune_serializer.convert_node(actual) neptune_expected = { NEPTUNE_HEADER_ID: 'Timestamp:' + self.expected_ts_key, METADATA_KEY_PROPERTY_NAME: 'Timestamp:' + self.expected_ts_key, NEPTUNE_HEADER_LABEL: 'Timestamp', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'last_updated_timestamp', 'timestamp:Long(single)': 123456789, } self.assertDictEqual(actual_neptune_serialized, neptune_expected)
def load(self, csv_serializable: GraphSerializable) -> None: """ Writes GraphSerializable into CSV files. There are multiple CSV files that this method writes. This is because there're not only node and relationship, but also it can also have different nodes, and relationships. Common pattern for both nodes and relations: 1. retrieve csv row (a dict where keys represent a header, values represent a row) 2. using this dict to get a appropriate csv writer and write to it. 3. repeat 1 and 2 :param csv_serializable: :return: """ node = csv_serializable.next_node() while node: node.attributes[ PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag node_dict = neptune_serializer.convert_node(node) if node_dict: key = (node.label, len(node_dict)) file_suffix = '{}_{}'.format(*key) node_writer = self._get_writer(node_dict, self._node_file_mapping, key, self._node_dir, file_suffix) node_writer.writerow(node_dict) node = csv_serializable.next_node() relation = csv_serializable.next_relation() while relation: relation.attributes[ PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag relation_dicts = neptune_serializer.convert_relationship(relation) if relation_dicts: key2 = (relation.start_label, relation.end_label, relation.type, len(relation_dicts[0])) file_suffix = '{}_{}_{}'.format(key2[0], key2[1], key2[2]) relation_writer = self._get_writer(relation_dicts[0], self._relation_file_mapping, key2, self._relation_dir, file_suffix) relation_writer.writerows(relation_dicts) relation = csv_serializable.next_relation()
def test_serialize_neptune(self) -> None: node_row = self.table_metadata.next_node() actual = [] while node_row: node_row_serialized = neptune_serializer.convert_node(node_row) actual.append(node_row_serialized) node_row = self.table_metadata.next_node() self.assertEqual(EXPECTED_NEPTUNE_NODES, actual) relation_row = self.table_metadata.next_relation() neptune_actual: List[List[Dict]] = [] while relation_row: relation_row_serialized = neptune_serializer.convert_relationship(relation_row) neptune_actual.append(relation_row_serialized) relation_row = self.table_metadata.next_relation() self.maxDiff = None self.assertEqual(EXPECTED_RELATIONSHIPS_NEPTUNE, neptune_actual)
def test_create_node_additional_attr_neptune(self) -> None: test_user = User(first_name='test_first', last_name='test_last', name='test_first test_last', email='*****@*****.**', github_username='******', team_name='test_team', employee_type='FTE', manager_email='*****@*****.**', slack_id='slack', is_active=True, updated_at=1, role_name='swe', enable_notify=True) nodes = test_user.create_nodes() serialized_node = neptune_serializer.convert_node(nodes[0]) self.assertEqual(serialized_node['email:String(single)'], '*****@*****.**') self.assertEqual(serialized_node['role_name:String(single)'], 'swe') self.assertTrue(serialized_node['enable_notify:Bool(single)'])
def test_create_nodes_neptune(self) -> None: actual = [] node = self.badge_metada.create_next_node() while node: serialized_node = neptune_serializer.convert_node(node) actual.append(serialized_node) node = self.badge_metada.create_next_node() node_id_1 = BadgeMetadata.BADGE_NODE_LABEL + ":" + BadgeMetadata.BADGE_KEY_FORMAT.format( badge=badge1.name) expected_node1 = { NEPTUNE_HEADER_ID: node_id_1, METADATA_KEY_PROPERTY_NAME: node_id_1, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, BadgeMetadata.BADGE_CATEGORY + ':String(single)': badge1.category } node_id_2 = BadgeMetadata.BADGE_NODE_LABEL + ":" + BadgeMetadata.BADGE_KEY_FORMAT.format( badge=badge2.name) expected_node2 = { NEPTUNE_HEADER_ID: node_id_2, METADATA_KEY_PROPERTY_NAME: node_id_2, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, BadgeMetadata.BADGE_CATEGORY + ':String(single)': badge2.category } expected = [expected_node1, expected_node2] self.assertEqual(expected, actual)
def test_create_nodes_neptune(self) -> None: actual = self.dashboard_query.create_next_node() actual_serialized = neptune_serializer.convert_node(actual) neptune_expected = { NEPTUNE_HEADER_ID: '_dashboard://gold.dg_id/d_id/query/q_id', NEPTUNE_HEADER_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'id:String(single)': 'q_id', 'query_text:String(single)': 'SELECT * FROM foo.bar', 'name:String(single)': 'q_name', 'url:String(single)': 'http://foo.bar/query/baz' } self.assertEqual(neptune_expected, actual_serialized)
def test_create_nodes_neptune(self) -> None: nodes = self.watermark.create_nodes() expected_serialized_node_result = { NEPTUNE_HEADER_ID: self.start_key, NEPTUNE_HEADER_LABEL: 'Watermark', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'partition_key:String(single)': 'ds', 'partition_value:String(single)': '2017-09-18/feature_id=9', 'create_time:String(single)': '2017-09-18T00:00:00' } serialized_node = neptune_serializer.convert_node(nodes[0]) self.assertDictEqual(expected_serialized_node_result, serialized_node)
def test_create_next_node_neptune(self) -> None: next_node = self.table_stats.create_next_node() serialized_node = neptune_serializer.convert_node(next_node) expected_neptune_node = { NEPTUNE_HEADER_ID: 'hive://gold.base/test/col/avg/', NEPTUNE_HEADER_LABEL: 'Stat', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'stat_val:String(single)': '1', 'stat_name:String(single)': 'avg', 'start_epoch:String(single)': '1', 'end_epoch:String(single)': '2', } self.assertDictEqual(serialized_node, expected_neptune_node)
def test_full_dashboard_example_neptune(self) -> None: expected_neptune_rels = [ [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Cluster:_dashboard://gold', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_GROUP'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Cluster:_dashboard://gold', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_GROUP'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Cluster:_dashboard://gold', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DASHBOARD_GROUP', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id='Cluster:_dashboard://gold', label='DASHBOARD_GROUP_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id='Cluster:_dashboard://gold', label='DASHBOARD_GROUP_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Cluster:_dashboard://gold', NEPTUNE_HEADER_LABEL: 'DASHBOARD_GROUP_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', label='DESCRIPTION'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', label='DESCRIPTION'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DESCRIPTION_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/_description', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DESCRIPTION_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DASHBOARD_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DASHBOARD'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DASHBOARD'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'DASHBOARD', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', label='DESCRIPTION'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', label='DESCRIPTION'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DESCRIPTION_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DESCRIPTION_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:test_tag', label='TAG'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:test_tag', label='TAG'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Tag:test_tag', NEPTUNE_HEADER_LABEL: 'TAG', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:test_tag', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:test_tag', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Tag:test_tag', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'TAG_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], [{ NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:tag2', label='TAG'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', to_vertex_id='Tag:tag2', label='TAG'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Tag:tag2', NEPTUNE_HEADER_LABEL: 'TAG', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:tag2', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id='Tag:tag2', to_vertex_id= 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF'), NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Tag:tag2', NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'TAG_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }], ] expected_neptune_nodes = [ { NEPTUNE_HEADER_ID: 'Cluster:_dashboard://gold', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold', NEPTUNE_HEADER_LABEL: 'Cluster', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'gold' }, { NEPTUNE_HEADER_ID: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'Dashboard', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'Agent', 'dashboard_url:String(single)': 'https://foo.bar/dashboard_group/foo/dashboard/bar', 'created_timestamp:Long(single)': 123456789, }, { NEPTUNE_HEADER_ID: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'Dashboardgroup', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'Product - Jobs.cz', 'dashboard_group_url:String(single)': 'https://foo.bar/dashboard_group/foo' }, { NEPTUNE_HEADER_ID: 'Description:_dashboard://gold.Product - Jobs.cz/_description', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'description:String(single)': 'foo dashboard group description', }, { NEPTUNE_HEADER_ID: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'description:String(single)': 'Agent dashboard description' }, { NEPTUNE_HEADER_ID: 'Tag:test_tag', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'test_tag', NEPTUNE_HEADER_LABEL: 'Tag', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'tag_type:String(single)': 'dashboard' }, { NEPTUNE_HEADER_ID: 'Tag:tag2', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'tag2', NEPTUNE_HEADER_LABEL: 'Tag', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'tag_type:String(single)': 'dashboard' }, ] self.maxDiff = None node_row = self.full_dashboard_metadata.next_node() actual = [] while node_row: node_serialized = neptune_serializer.convert_node(node_row) actual.append(node_serialized) node_row = self.full_dashboard_metadata.next_node() self.assertEqual(expected_neptune_nodes, actual) relation_row = self.full_dashboard_metadata.next_relation() neptune_actual: List[List[Dict]] = [] while relation_row: relation_serialized = neptune_serializer.convert_relationship( relation_row) neptune_actual.append(relation_serialized) relation_row = self.full_dashboard_metadata.next_relation() self.assertEqual(expected_neptune_rels, neptune_actual)
def test_create_nodes(self) -> None: dashboard_chart = DashboardChart(dashboard_group_id='dg_id', dashboard_id='d_id', query_id='q_id', chart_id='c_id', chart_name='c_name', chart_type='bar', chart_url='http://gold.foo/chart') actual = dashboard_chart.create_next_node() actual_serialized = neo4_serializer.serialize_node(actual) neptune_serialized = neptune_serializer.convert_node(actual) expected: Dict[str, Any] = { 'name': 'c_name', 'type': 'bar', 'id': 'c_id', 'url': 'http://gold.foo/chart', 'KEY': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', 'LABEL': 'Chart' } neptune_expected = { '~id': 'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', '~label': 'Chart', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'type:String(single)': 'bar', 'name:String(single)': 'c_name', 'id:String(single)': 'c_id', 'url:String(single)': 'http://gold.foo/chart', } assert actual is not None self.assertDictEqual(expected, actual_serialized) self.assertDictEqual(neptune_expected, neptune_serialized) self.assertIsNone(dashboard_chart.create_next_node()) dashboard_chart = DashboardChart(dashboard_group_id='dg_id', dashboard_id='d_id', query_id='q_id', chart_id='c_id', chart_url='http://gold.foo.bar/') actual2 = dashboard_chart.create_next_node() actual2_serialized = neo4_serializer.serialize_node(actual2) actual2_neptune_serialized = neptune_serializer.convert_node(actual2) expected2: Dict[str, Any] = { 'id': 'c_id', 'KEY': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', 'LABEL': 'Chart', 'url': 'http://gold.foo.bar/' } neptune_expected2 = { '~id': 'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', '~label': 'Chart', 'id:String(single)': 'c_id', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'url:String(single)': 'http://gold.foo.bar/', } assert actual2 is not None self.assertDictEqual(expected2, actual2_serialized) self.assertDictEqual(neptune_expected2, actual2_neptune_serialized)