def test_create_nodes_neptune(self) -> None:
schema_node = self.schema.create_next_node()
expected_serialized_schema_node = {
NEPTUNE_HEADER_ID: 'Schema:db://cluster.schema',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'db://cluster.schema',
NEPTUNE_HEADER_LABEL: 'Schema',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)': 'schema_name',
}
serialized_schema_node = neptune_serializer.convert_node(schema_node)
self.assertDictEqual(
expected_serialized_schema_node,
serialized_schema_node
)
schema_desc_node = self.schema.create_next_node()
excepted_serialized_schema_desc_node = {
NEPTUNE_HEADER_ID: 'Description:db://cluster.schema/_description',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT: 'db://cluster.schema/_description',
NEPTUNE_HEADER_LABEL: 'Description',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
'description_source:String(single)': 'description',
'description:String(single)': 'foo',
}
serialized_schema_desc_node = neptune_serializer.convert_node(schema_desc_node)
self.assertDictEqual(
excepted_serialized_schema_desc_node,
serialized_schema_desc_node
)
def test_create_next_node_neptune(self) -> None:
actual = []
next_node = self.application.create_next_node()
while next_node:
serialized_next_node = neptune_serializer.convert_node(next_node)
actual.append(serialized_next_node)
next_node = self.application.create_next_node()
node_id = 'Application:application://gold.airflow/event_test/hive.default.test_table'
neptune_expected = [{
NEPTUNE_HEADER_ID:
node_id,
METADATA_KEY_PROPERTY_NAME:
node_id,
NEPTUNE_HEADER_LABEL:
'Application',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'application_url:String(single)':
'airflow_host.net/admin/airflow/tree?dag_id=event_test',
'id:String(single)':
'event_test/hive.default.test_table',
'name:String(single)':
'Airflow',
'description:String(single)':
'Airflow with id event_test/hive.default.test_table',
}]
self.assertEqual(neptune_expected, actual)
def test_create_nodes_neptune(self) -> None:
node_id = TableLastUpdated.LAST_UPDATED_NODE_LABEL + ":" + self.tableLastUpdated.get_last_updated_model_key(
)
expected_nodes = [{
NEPTUNE_HEADER_ID:
node_id,
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
self.tableLastUpdated.get_last_updated_model_key(),
NEPTUNE_HEADER_LABEL:
TableLastUpdated.LAST_UPDATED_NODE_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)':
'last_updated_timestamp',
'last_updated_timestamp:Long(single)':
25195665,
timestamp_constants.TIMESTAMP_PROPERTY + ":Long(single)":
25195665,
}]
actual = []
next_node = self.tableLastUpdated.create_next_node()
while next_node:
next_node_serialized = neptune_serializer.convert_node(next_node)
actual.append(next_node_serialized)
next_node = self.tableLastUpdated.create_next_node()
self.assertEqual(actual, expected_nodes)
def test_create_nodes_neptune(self) -> None:
expected_node1 = {
NEPTUNE_HEADER_ID: "User:"******":String(single)": owner1
}
expected_node2 = {
NEPTUNE_HEADER_ID: "User:"******":String(single)": owner2
}
expected = [expected_node1, expected_node2]
actual = []
node = self.table_owner.create_next_node()
while node:
serialized_node = neptune_serializer.convert_node(node)
actual.append(serialized_node)
node = self.table_owner.create_next_node()
self.assertEqual(actual, expected)
def test_create_nodes_neptune(self) -> None:
for tc in self.test_cases:
actual = []
next_node = tc.application.create_next_node()
while next_node:
serialized_next_node = neptune_serializer.convert_node(
next_node)
actual.append(serialized_next_node)
next_node = tc.application.create_next_node()
node_id = f'Application:{tc.application.application_key}'
node_key = tc.application.application_key
neptune_expected = [{
NEPTUNE_HEADER_ID:
node_id,
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
node_key,
NEPTUNE_HEADER_LABEL:
'Application',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'application_url:String(single)':
tc.application.application_url,
'id:String(single)':
tc.application.application_id,
'name:String(single)':
tc.application.application_type,
'description:String(single)':
tc.application.application_description,
}]
self.assertEqual(neptune_expected, actual)
def test_create_nodes_neptune(self) -> None:
nodes = self.badge_metada.create_nodes()
serialized_nodes = [
neptune_serializer.convert_node(node) for node in nodes
]
expected_node1 = {
NEPTUNE_HEADER_ID:
BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge1.name),
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_NODE_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
BadgeMetadata.BADGE_CATEGORY + ':String(single)':
badge1.category
}
expected_node2 = {
NEPTUNE_HEADER_ID:
BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge2.name),
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_NODE_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
BadgeMetadata.BADGE_CATEGORY + ':String(single)':
badge2.category
}
self.assertTrue(expected_node1 in serialized_nodes)
self.assertTrue(expected_node2 in serialized_nodes)
def test_create_nodes_neptune(self) -> None:
expected_serialized_node_results = [{
NEPTUNE_HEADER_ID:
'Watermark:' + self.start_key,
METADATA_KEY_PROPERTY_NAME:
'Watermark:' + self.start_key,
NEPTUNE_HEADER_LABEL:
'Watermark',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'partition_key:String(single)':
'ds',
'partition_value:String(single)':
'2017-09-18/feature_id=9',
'create_time:String(single)':
'2017-09-18T00:00:00'
}]
actual = []
node = self.watermark.create_next_node()
while node:
serialized_node = neptune_serializer.convert_node(node)
actual.append(serialized_node)
node = self.watermark.create_next_node()
self.assertEqual(expected_serialized_node_results, actual)
def test_create_nodes_neptune(self) -> None:
actual = []
next_node = self.table_stats.create_next_node()
while next_node:
serialized_node = neptune_serializer.convert_node(next_node)
actual.append(serialized_node)
next_node = self.table_stats.create_next_node()
expected_neptune_nodes = [{
NEPTUNE_HEADER_ID:
'Stat:hive://gold.base/test/col/avg/',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'hive://gold.base/test/col/avg/',
NEPTUNE_HEADER_LABEL:
'Stat',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'stat_val:String(single)':
'1',
'stat_type:String(single)':
'avg',
'start_epoch:String(single)':
'1',
'end_epoch:String(single)':
'2',
}]
self.assertEqual(actual, expected_neptune_nodes)
def test_create_nodes_neptune(self) -> None:
nodes = self.table_owner.create_nodes()
expected_node1 = {
NEPTUNE_HEADER_ID: User.USER_NODE_KEY_FORMAT.format(email=owner1),
NEPTUNE_HEADER_LABEL: User.USER_NODE_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
User.USER_NODE_EMAIL + ":String(single)": owner1
}
expected_node2 = {
NEPTUNE_HEADER_ID: User.USER_NODE_KEY_FORMAT.format(email=owner2),
NEPTUNE_HEADER_LABEL: User.USER_NODE_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
User.USER_NODE_EMAIL + ":String(single)": owner2
}
actual_nodes = [
neptune_serializer.convert_node(node)
for node in nodes
]
self.assertTrue(expected_node1 in actual_nodes)
self.assertTrue(expected_node2 in actual_nodes)
def test_neptune_dashboard_timestamp_nodes(self) -> None:
actual = self.dashboard_last_modified.create_next_node()
actual_neptune_serialized = neptune_serializer.convert_node(actual)
neptune_expected = {
NEPTUNE_HEADER_ID: 'Timestamp:' + self.expected_ts_key,
METADATA_KEY_PROPERTY_NAME: 'Timestamp:' + self.expected_ts_key,
NEPTUNE_HEADER_LABEL: 'Timestamp',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)': 'last_updated_timestamp',
'timestamp:Long(single)': 123456789,
}
self.assertDictEqual(actual_neptune_serialized, neptune_expected)
def load(self, csv_serializable: GraphSerializable) -> None:
"""
Writes GraphSerializable into CSV files.
There are multiple CSV files that this method writes.
This is because there're not only node and relationship, but also it
can also have different nodes, and relationships.
Common pattern for both nodes and relations:
1. retrieve csv row (a dict where keys represent a header,
values represent a row)
2. using this dict to get a appropriate csv writer and write to it.
3. repeat 1 and 2
:param csv_serializable:
:return:
"""
node = csv_serializable.next_node()
while node:
node.attributes[
PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag
node_dict = neptune_serializer.convert_node(node)
if node_dict:
key = (node.label, len(node_dict))
file_suffix = '{}_{}'.format(*key)
node_writer = self._get_writer(node_dict,
self._node_file_mapping, key,
self._node_dir, file_suffix)
node_writer.writerow(node_dict)
node = csv_serializable.next_node()
relation = csv_serializable.next_relation()
while relation:
relation.attributes[
PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag
relation_dicts = neptune_serializer.convert_relationship(relation)
if relation_dicts:
key2 = (relation.start_label, relation.end_label,
relation.type, len(relation_dicts[0]))
file_suffix = '{}_{}_{}'.format(key2[0], key2[1], key2[2])
relation_writer = self._get_writer(relation_dicts[0],
self._relation_file_mapping,
key2, self._relation_dir,
file_suffix)
relation_writer.writerows(relation_dicts)
relation = csv_serializable.next_relation()
def test_serialize_neptune(self) -> None:
node_row = self.table_metadata.next_node()
actual = []
while node_row:
node_row_serialized = neptune_serializer.convert_node(node_row)
actual.append(node_row_serialized)
node_row = self.table_metadata.next_node()
self.assertEqual(EXPECTED_NEPTUNE_NODES, actual)
relation_row = self.table_metadata.next_relation()
neptune_actual: List[List[Dict]] = []
while relation_row:
relation_row_serialized = neptune_serializer.convert_relationship(relation_row)
neptune_actual.append(relation_row_serialized)
relation_row = self.table_metadata.next_relation()
self.maxDiff = None
self.assertEqual(EXPECTED_RELATIONSHIPS_NEPTUNE, neptune_actual)
def test_create_node_additional_attr_neptune(self) -> None:
test_user = User(first_name='test_first',
last_name='test_last',
name='test_first test_last',
email='*****@*****.**',
github_username='******',
team_name='test_team',
employee_type='FTE',
manager_email='*****@*****.**',
slack_id='slack',
is_active=True,
updated_at=1,
role_name='swe',
enable_notify=True)
nodes = test_user.create_nodes()
serialized_node = neptune_serializer.convert_node(nodes[0])
self.assertEqual(serialized_node['email:String(single)'], '*****@*****.**')
self.assertEqual(serialized_node['role_name:String(single)'], 'swe')
self.assertTrue(serialized_node['enable_notify:Bool(single)'])
def test_create_nodes_neptune(self) -> None:
actual = []
node = self.badge_metada.create_next_node()
while node:
serialized_node = neptune_serializer.convert_node(node)
actual.append(serialized_node)
node = self.badge_metada.create_next_node()
node_id_1 = BadgeMetadata.BADGE_NODE_LABEL + ":" + BadgeMetadata.BADGE_KEY_FORMAT.format(
badge=badge1.name)
expected_node1 = {
NEPTUNE_HEADER_ID:
node_id_1,
METADATA_KEY_PROPERTY_NAME:
node_id_1,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_NODE_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
BadgeMetadata.BADGE_CATEGORY + ':String(single)':
badge1.category
}
node_id_2 = BadgeMetadata.BADGE_NODE_LABEL + ":" + BadgeMetadata.BADGE_KEY_FORMAT.format(
badge=badge2.name)
expected_node2 = {
NEPTUNE_HEADER_ID:
node_id_2,
METADATA_KEY_PROPERTY_NAME:
node_id_2,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_NODE_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
BadgeMetadata.BADGE_CATEGORY + ':String(single)':
badge2.category
}
expected = [expected_node1, expected_node2]
self.assertEqual(expected, actual)
def test_create_nodes_neptune(self) -> None:
actual = self.dashboard_query.create_next_node()
actual_serialized = neptune_serializer.convert_node(actual)
neptune_expected = {
NEPTUNE_HEADER_ID:
'_dashboard://gold.dg_id/d_id/query/q_id',
NEPTUNE_HEADER_LABEL:
DashboardQuery.DASHBOARD_QUERY_LABEL,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'id:String(single)':
'q_id',
'query_text:String(single)':
'SELECT * FROM foo.bar',
'name:String(single)':
'q_name',
'url:String(single)':
'http://foo.bar/query/baz'
}
self.assertEqual(neptune_expected, actual_serialized)
def test_create_nodes_neptune(self) -> None:
nodes = self.watermark.create_nodes()
expected_serialized_node_result = {
NEPTUNE_HEADER_ID:
self.start_key,
NEPTUNE_HEADER_LABEL:
'Watermark',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'partition_key:String(single)':
'ds',
'partition_value:String(single)':
'2017-09-18/feature_id=9',
'create_time:String(single)':
'2017-09-18T00:00:00'
}
serialized_node = neptune_serializer.convert_node(nodes[0])
self.assertDictEqual(expected_serialized_node_result, serialized_node)
def test_create_next_node_neptune(self) -> None:
next_node = self.table_stats.create_next_node()
serialized_node = neptune_serializer.convert_node(next_node)
expected_neptune_node = {
NEPTUNE_HEADER_ID:
'hive://gold.base/test/col/avg/',
NEPTUNE_HEADER_LABEL:
'Stat',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'stat_val:String(single)':
'1',
'stat_name:String(single)':
'avg',
'start_epoch:String(single)':
'1',
'end_epoch:String(single)':
'2',
}
self.assertDictEqual(serialized_node, expected_neptune_node)
def test_full_dashboard_example_neptune(self) -> None:
expected_neptune_rels = [
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Cluster:_dashboard://gold',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_GROUP'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Cluster:_dashboard://gold',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_GROUP'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Cluster:_dashboard://gold',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'DASHBOARD_GROUP',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id='Cluster:_dashboard://gold',
label='DASHBOARD_GROUP_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id='Cluster:_dashboard://gold',
label='DASHBOARD_GROUP_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Cluster:_dashboard://gold',
NEPTUNE_HEADER_LABEL:
'DASHBOARD_GROUP_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
label='DESCRIPTION'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
label='DESCRIPTION'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Description:_dashboard://gold.Product - Jobs.cz/_description',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DESCRIPTION_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DESCRIPTION_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Description:_dashboard://gold.Product - Jobs.cz/_description',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'DASHBOARD_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DASHBOARD'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DASHBOARD'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'DASHBOARD',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
label='DESCRIPTION'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
label='DESCRIPTION'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DESCRIPTION_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DESCRIPTION_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:test_tag',
label='TAG'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:test_tag',
label='TAG'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Tag:test_tag',
NEPTUNE_HEADER_LABEL:
'TAG',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:test_tag',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:test_tag',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Tag:test_tag',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'TAG_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:tag2',
label='TAG'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:tag2',
label='TAG'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Tag:tag2',
NEPTUNE_HEADER_LABEL:
'TAG',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:tag2',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:tag2',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Tag:tag2',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'TAG_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
]
expected_neptune_nodes = [
{
NEPTUNE_HEADER_ID:
'Cluster:_dashboard://gold',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold',
NEPTUNE_HEADER_LABEL:
'Cluster',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)':
'gold'
},
{
NEPTUNE_HEADER_ID:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'Dashboard',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)':
'Agent',
'dashboard_url:String(single)':
'https://foo.bar/dashboard_group/foo/dashboard/bar',
'created_timestamp:Long(single)':
123456789,
},
{
NEPTUNE_HEADER_ID:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'Dashboardgroup',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)':
'Product - Jobs.cz',
'dashboard_group_url:String(single)':
'https://foo.bar/dashboard_group/foo'
},
{
NEPTUNE_HEADER_ID:
'Description:_dashboard://gold.Product - Jobs.cz/_description',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz/_description',
NEPTUNE_HEADER_LABEL:
'Description',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'description:String(single)':
'foo dashboard group description',
},
{
NEPTUNE_HEADER_ID:
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz/Agent/_description',
NEPTUNE_HEADER_LABEL:
'Description',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'description:String(single)':
'Agent dashboard description'
},
{
NEPTUNE_HEADER_ID:
'Tag:test_tag',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'test_tag',
NEPTUNE_HEADER_LABEL:
'Tag',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'tag_type:String(single)':
'dashboard'
},
{
NEPTUNE_HEADER_ID:
'Tag:tag2',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'tag2',
NEPTUNE_HEADER_LABEL:
'Tag',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'tag_type:String(single)':
'dashboard'
},
]
self.maxDiff = None
node_row = self.full_dashboard_metadata.next_node()
actual = []
while node_row:
node_serialized = neptune_serializer.convert_node(node_row)
actual.append(node_serialized)
node_row = self.full_dashboard_metadata.next_node()
self.assertEqual(expected_neptune_nodes, actual)
relation_row = self.full_dashboard_metadata.next_relation()
neptune_actual: List[List[Dict]] = []
while relation_row:
relation_serialized = neptune_serializer.convert_relationship(
relation_row)
neptune_actual.append(relation_serialized)
relation_row = self.full_dashboard_metadata.next_relation()
self.assertEqual(expected_neptune_rels, neptune_actual)
def test_create_nodes(self) -> None:
dashboard_chart = DashboardChart(dashboard_group_id='dg_id',
dashboard_id='d_id',
query_id='q_id',
chart_id='c_id',
chart_name='c_name',
chart_type='bar',
chart_url='http://gold.foo/chart')
actual = dashboard_chart.create_next_node()
actual_serialized = neo4_serializer.serialize_node(actual)
neptune_serialized = neptune_serializer.convert_node(actual)
expected: Dict[str, Any] = {
'name': 'c_name',
'type': 'bar',
'id': 'c_id',
'url': 'http://gold.foo/chart',
'KEY': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id',
'LABEL': 'Chart'
}
neptune_expected = {
'~id':
'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id',
'~label':
'Chart',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'type:String(single)':
'bar',
'name:String(single)':
'c_name',
'id:String(single)':
'c_id',
'url:String(single)':
'http://gold.foo/chart',
}
assert actual is not None
self.assertDictEqual(expected, actual_serialized)
self.assertDictEqual(neptune_expected, neptune_serialized)
self.assertIsNone(dashboard_chart.create_next_node())
dashboard_chart = DashboardChart(dashboard_group_id='dg_id',
dashboard_id='d_id',
query_id='q_id',
chart_id='c_id',
chart_url='http://gold.foo.bar/')
actual2 = dashboard_chart.create_next_node()
actual2_serialized = neo4_serializer.serialize_node(actual2)
actual2_neptune_serialized = neptune_serializer.convert_node(actual2)
expected2: Dict[str, Any] = {
'id': 'c_id',
'KEY': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id',
'LABEL': 'Chart',
'url': 'http://gold.foo.bar/'
}
neptune_expected2 = {
'~id':
'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id',
'~label':
'Chart',
'id:String(single)':
'c_id',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'url:String(single)':
'http://gold.foo.bar/',
}
assert actual2 is not None
self.assertDictEqual(expected2, actual2_serialized)
self.assertDictEqual(neptune_expected2, actual2_neptune_serialized)
