def test_dashboard_owner_relations_neptune(self) -> None:
actual = self.dashboard_owner.create_next_relation()
actual_serialized = neptune_serializer.convert_relationship(actual)
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
to_vertex_id='*****@*****.**',
label='OWNER'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_RELATIONSHIP_HEADER_TO: '*****@*****.**',
NEPTUNE_HEADER_LABEL: 'OWNER',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id='*****@*****.**',
to_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
label='OWNER_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: '*****@*****.**',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_HEADER_LABEL: 'OWNER_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
assert actual is not None
self.assertDictEqual(actual_serialized[0], neptune_forward_expected)
self.assertDictEqual(actual_serialized[1], neptune_reversed_expected)
def test_create_relation_neptune(self) -> None:
actual = self.schema.create_next_relation()
serialized_actual = neptune_serializer.convert_relationship(actual)
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id='db://cluster.schema',
to_vertex_id='db://cluster.schema/_description',
label='DESCRIPTION'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'db://cluster.schema',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'db://cluster.schema/_description',
NEPTUNE_HEADER_LABEL: 'DESCRIPTION',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id='db://cluster.schema/_description',
to_vertex_id='db://cluster.schema',
label='DESCRIPTION_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'db://cluster.schema/_description',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'db://cluster.schema',
NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
self.assertDictEqual(serialized_actual[0], neptune_forward_expected)
self.assertDictEqual(serialized_actual[1], neptune_reversed_expected)
def test_create_relation_neptune(self) -> None:
relations = self.table_owner.create_relation()
expected = [
{
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=owner1,
to_vertex_id=self.table_owner.get_metadata_model_key(),
label=TableOwner.OWNER_TABLE_RELATION_TYPE
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: owner1,
NEPTUNE_RELATIONSHIP_HEADER_TO: self.table_owner.get_metadata_model_key(),
NEPTUNE_HEADER_LABEL: TableOwner.OWNER_TABLE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
},
{
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.table_owner.get_metadata_model_key(),
to_vertex_id=owner1,
label=TableOwner.TABLE_OWNER_RELATION_TYPE
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: self.table_owner.get_metadata_model_key(),
NEPTUNE_RELATIONSHIP_HEADER_TO: owner1,
NEPTUNE_HEADER_LABEL: TableOwner.TABLE_OWNER_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
]
actual_relations = [
neptune_serializer.convert_relationship(relation)
for relation in relations
]
self.assertTrue(expected in actual_relations)
def test_create_relation_neptune(self) -> None:
actual = []
next_relation = self.tableLastUpdated.create_next_relation()
while next_relation:
next_relation_serialized = neptune_serializer.convert_relationship(
next_relation)
actual.append(next_relation_serialized)
next_relation = self.tableLastUpdated.create_next_relation()
expected = [[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:hive://gold.default/test_table',
to_vertex_id=
'Timestamp:hive://gold.default/test_table/timestamp',
label='LAST_UPDATED_AT'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:hive://gold.default/test_table',
to_vertex_id=
'Timestamp:hive://gold.default/test_table/timestamp',
label='LAST_UPDATED_AT'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Table:hive://gold.default/test_table',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Timestamp:hive://gold.default/test_table/timestamp',
NEPTUNE_HEADER_LABEL:
'LAST_UPDATED_AT',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Timestamp:hive://gold.default/test_table/timestamp',
to_vertex_id='Table:hive://gold.default/test_table',
label='LAST_UPDATED_TIME_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Timestamp:hive://gold.default/test_table/timestamp',
to_vertex_id='Table:hive://gold.default/test_table',
label='LAST_UPDATED_TIME_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Timestamp:hive://gold.default/test_table/timestamp',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Table:hive://gold.default/test_table',
NEPTUNE_HEADER_LABEL:
'LAST_UPDATED_TIME_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]]
self.assertEqual(actual, expected)
def test_create_relation_neptune(self) -> None:
self.expected_relation_result = {
RELATION_START_KEY: 'hive://gold.base/test/col/avg/',
RELATION_START_LABEL: 'Stat',
RELATION_END_KEY: 'hive://gold.base/test/col',
RELATION_END_LABEL: 'Column',
RELATION_TYPE: 'STAT_OF',
RELATION_REVERSE_TYPE: 'STAT'
}
expected = [
[
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Stat:hive://gold.base/test/col/avg/',
to_vertex_id='Column:hive://gold.base/test/col',
label='STAT_OF'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Stat:hive://gold.base/test/col/avg/',
to_vertex_id='Column:hive://gold.base/test/col',
label='STAT_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Stat:hive://gold.base/test/col/avg/',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.base/test/col',
NEPTUNE_HEADER_LABEL: 'STAT_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
},
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Column:hive://gold.base/test/col',
to_vertex_id='Stat:hive://gold.base/test/col/avg/',
label='STAT'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Column:hive://gold.base/test/col',
to_vertex_id='Stat:hive://gold.base/test/col/avg/',
label='STAT'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.base/test/col',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'Stat:hive://gold.base/test/col/avg/',
NEPTUNE_HEADER_LABEL: 'STAT',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
]
]
actual = []
next_relation = self.table_stats.create_next_relation()
while next_relation:
serialized_relation = neptune_serializer.convert_relationship(next_relation)
actual.append(serialized_relation)
next_relation = self.table_stats.create_next_relation()
self.assertListEqual(actual, expected)
def test_create_relation_neptune(self) -> None:
actual = []
relation = self.user.create_next_relation()
while relation:
serialized = neptune_serializer.convert_relationship(relation)
actual.append(serialized)
relation = self.user.create_next_relation()
start_key = 'User:{email}'.format(email='*****@*****.**')
end_key = 'User:{email}'.format(email='*****@*****.**')
expected = [[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=start_key,
to_vertex_id=end_key,
label=User.USER_MANAGER_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=start_key,
to_vertex_id=end_key,
label=User.USER_MANAGER_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
end_key,
NEPTUNE_HEADER_LABEL:
User.USER_MANAGER_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=end_key,
to_vertex_id=start_key,
label=User.MANAGER_USER_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=end_key,
to_vertex_id=start_key,
label=User.MANAGER_USER_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
start_key,
NEPTUNE_HEADER_LABEL:
User.MANAGER_USER_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]]
self.assertListEqual(actual, expected)
def test_neptune_serialize(self) -> None:
rel_row = self.table_col_usage.next_relation()
actual = []
while rel_row:
actual.append(neptune_serializer.convert_relationship(rel_row))
rel_row = self.table_col_usage.next_relation()
expected = [[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:db://gold.scm/foo',
to_vertex_id='User:[email protected]',
label='READ_BY'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:db://gold.scm/foo',
to_vertex_id='User:[email protected]',
label='READ_BY'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Table:db://gold.scm/foo',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'User:[email protected]',
NEPTUNE_HEADER_LABEL:
'READ_BY',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'read_count:Long(single)':
1
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='User:[email protected]',
to_vertex_id='Table:db://gold.scm/foo',
label='READ'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='User:[email protected]',
to_vertex_id='Table:db://gold.scm/foo',
label='READ'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'User:[email protected]',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Table:db://gold.scm/foo',
NEPTUNE_HEADER_LABEL:
'READ',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'read_count:Long(single)':
1
}]]
self.maxDiff = None
self.assertListEqual(expected, actual)
def test_dashboard_table_relations(self) -> None:
dashboard_table = DashboardTable(table_ids=['hive://gold.schema/table1'],
cluster='cluster_id', product='product_id',
dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id')
actual = dashboard_table.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
actual_neptune_serialized = neptune_serializer.convert_relationship(actual)
expected = {RELATION_END_KEY: 'hive://gold.schema/table1', RELATION_START_LABEL: 'Dashboard',
RELATION_END_LABEL: 'Table',
RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
RELATION_TYPE: 'DASHBOARD_WITH_TABLE',
RELATION_REVERSE_TYPE: 'TABLE_OF_DASHBOARD'}
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
to_vertex_id='Table:hive://gold.schema/table1',
label='DASHBOARD_WITH_TABLE'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
to_vertex_id='Table:hive://gold.schema/table1',
label='DASHBOARD_WITH_TABLE'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.schema/table1',
NEPTUNE_HEADER_LABEL: 'DASHBOARD_WITH_TABLE',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:hive://gold.schema/table1',
to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
label='TABLE_OF_DASHBOARD'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:hive://gold.schema/table1',
to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
label='TABLE_OF_DASHBOARD'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.schema/table1',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_HEADER_LABEL: 'TABLE_OF_DASHBOARD',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
assert actual is not None
self.assertDictEqual(actual_serialized, expected)
self.assertDictEqual(actual_neptune_serialized[0], neptune_forward_expected)
self.assertDictEqual(actual_neptune_serialized[1], neptune_reversed_expected)
def test_create_relation(self) -> None:
dashboard_chart = DashboardChart(dashboard_group_id='dg_id',
dashboard_id='d_id',
query_id='q_id',
chart_id='c_id',
chart_name='c_name',
chart_type='bar',
)
actual = dashboard_chart.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
actual_neptune_serialized = neptune_serializer.convert_relationship(actual)
start_key = '_dashboard://gold.dg_id/d_id/query/q_id'
end_key = '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id'
expected: Dict[str, Any] = {
RELATION_END_KEY: end_key,
RELATION_START_LABEL: 'Query',
RELATION_END_LABEL: 'Chart',
RELATION_START_KEY: start_key,
RELATION_TYPE: 'HAS_CHART',
RELATION_REVERSE_TYPE: 'CHART_OF'
}
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=start_key,
to_vertex_id=end_key,
label='HAS_CHART'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: end_key,
NEPTUNE_HEADER_LABEL: 'HAS_CHART',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=end_key,
to_vertex_id=start_key,
label='CHART_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: start_key,
NEPTUNE_HEADER_LABEL: 'CHART_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
assert actual is not None
self.assertEqual(expected, actual_serialized)
self.assertEqual(neptune_forward_expected, actual_neptune_serialized[0])
self.assertEqual(neptune_reversed_expected, actual_neptune_serialized[1])
self.assertIsNone(dashboard_chart.create_next_relation())
def test_create_relation_neptune(self) -> None:
actual = []
relation = self.watermark.create_next_relation()
while relation:
serialized_relation = neptune_serializer.convert_relationship(
relation)
actual.append(serialized_relation)
relation = self.watermark.create_next_relation()
expected = [[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Watermark:" + self.start_key,
to_vertex_id="Table:" + self.end_key,
label='BELONG_TO_TABLE'),
METADATA_KEY_PROPERTY_NAME:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Watermark:" + self.start_key,
to_vertex_id="Table:" + self.end_key,
label='BELONG_TO_TABLE'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
"Watermark:" + self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
"Table:" + self.end_key,
NEPTUNE_HEADER_LABEL:
'BELONG_TO_TABLE',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" + self.end_key,
to_vertex_id="Watermark:" + self.start_key,
label='WATERMARK'),
METADATA_KEY_PROPERTY_NAME:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" + self.end_key,
to_vertex_id="Watermark:" + self.start_key,
label='WATERMARK'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
"Table:" + self.end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
"Watermark:" + self.start_key,
NEPTUNE_HEADER_LABEL:
'WATERMARK',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]]
self.assertListEqual(actual, expected)
def test_create_relation_neptune(self) -> None:
actual = []
relation = self.table_source.create_next_relation()
while relation:
serialized_relation = neptune_serializer.convert_relationship(
relation)
actual.append(serialized_relation)
relation = self.table_source.create_next_relation()
expected = [[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Source:" + self.start_key,
to_vertex_id="Table:" + self.end_key,
label=TableSource.SOURCE_TABLE_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Source:" + self.start_key,
to_vertex_id="Table:" + self.end_key,
label=TableSource.SOURCE_TABLE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
"Source:" + self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
"Table:" + self.end_key,
NEPTUNE_HEADER_LABEL:
TableSource.SOURCE_TABLE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" + self.end_key,
to_vertex_id="Source:" + self.start_key,
label=TableSource.TABLE_SOURCE_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" + self.end_key,
to_vertex_id="Source:" + self.start_key,
label=TableSource.TABLE_SOURCE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
"Table:" + self.end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
"Source:" + self.start_key,
NEPTUNE_HEADER_LABEL:
TableSource.TABLE_SOURCE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]]
self.assertListEqual(expected, actual)
def test_dashboard_owner_relations_neptune(self) -> None:
actual = self.dashboard_last_modified.create_next_relation()
actual_serialized = neptune_serializer.convert_relationship(actual)
neptune_forward_expected = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:' + self.expected_dashboard_key,
to_vertex_id='Timestamp:' + self.expected_ts_key,
label='LAST_UPDATED_AT'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:' + self.expected_dashboard_key,
to_vertex_id='Timestamp:' + self.expected_ts_key,
label='LAST_UPDATED_AT'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:' + self.expected_dashboard_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Timestamp:' + self.expected_ts_key,
NEPTUNE_HEADER_LABEL:
'LAST_UPDATED_AT',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Timestamp:' + self.expected_ts_key,
to_vertex_id='Dashboard:' + self.expected_dashboard_key,
label='LAST_UPDATED_TIME_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Timestamp:' + self.expected_ts_key,
to_vertex_id='Dashboard:' + self.expected_dashboard_key,
label='LAST_UPDATED_TIME_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Timestamp:' + self.expected_ts_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:' + self.expected_dashboard_key,
NEPTUNE_HEADER_LABEL:
'LAST_UPDATED_TIME_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
self.maxDiff = None
assert actual is not None
self.assertDictEqual(actual_serialized[0], neptune_forward_expected)
self.assertDictEqual(actual_serialized[1], neptune_reversed_expected)
self.assertIsNone(self.dashboard_last_modified.create_next_relation())
def test_create_relation_neptune(self) -> None:
actual = self.dashboard_query.create_next_relation()
actual_serialized = neptune_serializer.convert_relationship(actual)
neptune_forward_expected = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id',
to_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id',
label='HAS_QUERY'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id',
to_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id',
label='HAS_QUERY'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.dg_id/d_id',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Query:_dashboard://gold.dg_id/d_id/query/q_id',
NEPTUNE_HEADER_LABEL:
'HAS_QUERY',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id',
to_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id',
label='QUERY_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id',
to_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id',
label='QUERY_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Query:_dashboard://gold.dg_id/d_id/query/q_id',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.dg_id/d_id',
NEPTUNE_HEADER_LABEL:
'QUERY_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
assert actual is not None
self.assertDictEqual(actual_serialized[0], neptune_forward_expected)
self.assertDictEqual(actual_serialized[1], neptune_reversed_expected)
def test_create_relations_neptune(self) -> None:
application_id = 'Application:application://gold.airflow/event_test/hive.default.test_table'
table_id = 'Table:hive://gold.default/test_table'
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=table_id,
to_vertex_id=application_id,
label='DERIVED_FROM'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=table_id,
to_vertex_id=application_id,
label='DERIVED_FROM'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: table_id,
NEPTUNE_RELATIONSHIP_HEADER_TO: application_id,
NEPTUNE_HEADER_LABEL: 'DERIVED_FROM',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=application_id,
to_vertex_id=table_id,
label='GENERATES'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=application_id,
to_vertex_id=table_id,
label='GENERATES'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: application_id,
NEPTUNE_RELATIONSHIP_HEADER_TO: table_id,
NEPTUNE_HEADER_LABEL: 'GENERATES',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_expected = [[neptune_forward_expected, neptune_reversed_expected]]
actual = []
next_relation = self.application.create_next_relation()
while next_relation:
serialized_next_relation = neptune_serializer.convert_relationship(next_relation)
actual.append(serialized_next_relation)
next_relation = self.application.create_next_relation()
self.assertEqual(actual, neptune_expected)
def test_create_next_relation_neptune(self) -> None:
neptune_forward_expected = {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id='hive://gold.default/test_table',
to_vertex_id=
'application://gold.airflow/event_test/hive.default.test_table',
label='DERIVED_FROM'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'hive://gold.default/test_table',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'application://gold.airflow/event_test/hive.default.test_table',
NEPTUNE_HEADER_LABEL:
'DERIVED_FROM',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=
'application://gold.airflow/event_test/hive.default.test_table',
to_vertex_id='hive://gold.default/test_table',
label='GENERATES'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'application://gold.airflow/event_test/hive.default.test_table',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'hive://gold.default/test_table',
NEPTUNE_HEADER_LABEL:
'GENERATES',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
next_relation = self.application.create_next_relation()
serialized_next_relations = neptune_serializer.convert_relationship(
next_relation)
self.assertDictEqual(serialized_next_relations[0],
neptune_forward_expected)
self.assertDictEqual(serialized_next_relations[1],
neptune_reversed_expected)
def load(self, csv_serializable: GraphSerializable) -> None:
"""
Writes GraphSerializable into CSV files.
There are multiple CSV files that this method writes.
This is because there're not only node and relationship, but also it
can also have different nodes, and relationships.
Common pattern for both nodes and relations:
1. retrieve csv row (a dict where keys represent a header,
values represent a row)
2. using this dict to get a appropriate csv writer and write to it.
3. repeat 1 and 2
:param csv_serializable:
:return:
"""
node = csv_serializable.next_node()
while node:
node.attributes[
PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag
node_dict = neptune_serializer.convert_node(node)
if node_dict:
key = (node.label, len(node_dict))
file_suffix = '{}_{}'.format(*key)
node_writer = self._get_writer(node_dict,
self._node_file_mapping, key,
self._node_dir, file_suffix)
node_writer.writerow(node_dict)
node = csv_serializable.next_node()
relation = csv_serializable.next_relation()
while relation:
relation.attributes[
PUBLISHED_TAG_PROPERTY_NAME] = self.job_publisher_tag
relation_dicts = neptune_serializer.convert_relationship(relation)
if relation_dicts:
key2 = (relation.start_label, relation.end_label,
relation.type, len(relation_dicts[0]))
file_suffix = '{}_{}_{}'.format(key2[0], key2[1], key2[2])
relation_writer = self._get_writer(relation_dicts[0],
self._relation_file_mapping,
key2, self._relation_dir,
file_suffix)
relation_writer.writerows(relation_dicts)
relation = csv_serializable.next_relation()
def test_serialize_neptune(self) -> None:
node_row = self.table_metadata.next_node()
actual = []
while node_row:
node_row_serialized = neptune_serializer.convert_node(node_row)
actual.append(node_row_serialized)
node_row = self.table_metadata.next_node()
self.assertEqual(EXPECTED_NEPTUNE_NODES, actual)
relation_row = self.table_metadata.next_relation()
neptune_actual: List[List[Dict]] = []
while relation_row:
relation_row_serialized = neptune_serializer.convert_relationship(relation_row)
neptune_actual.append(relation_row_serialized)
relation_row = self.table_metadata.next_relation()
self.maxDiff = None
self.assertEqual(EXPECTED_RELATIONSHIPS_NEPTUNE, neptune_actual)
def test_dashboard_owner_relations_neptune(self) -> None:
dashboard_usage = DashboardUsage(dashboard_group_id='dashboard_group_id', dashboard_id='dashboard_id',
email='*****@*****.**', view_count=123, cluster='cluster_id',
product='product_id')
actual = dashboard_usage.create_next_relation()
actual_serialized = neptune_serializer.convert_relationship(actual)
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
to_vertex_id='*****@*****.**',
label='READ_BY'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_RELATIONSHIP_HEADER_TO: '*****@*****.**',
NEPTUNE_HEADER_LABEL: 'READ_BY',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
'read_count:Long(single)': 123
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id='*****@*****.**',
to_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
label='READ'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: '*****@*****.**',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_HEADER_LABEL: 'READ',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
'read_count:Long(single)': 123,
}
assert actual is not None
self.assertDictEqual(neptune_forward_expected, actual_serialized[0])
self.assertDictEqual(neptune_reversed_expected, actual_serialized[1])
self.assertIsNone(dashboard_usage.create_next_relation())
def test_create_relation_neptune(self) -> None:
relations = self.table_source.create_relation()
serialized_relations = neptune_serializer.convert_relationship(
relations[0])
expected = [{
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.start_key,
to_vertex_id=self.end_key,
label=TableSource.SOURCE_TABLE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.end_key,
NEPTUNE_HEADER_LABEL:
TableSource.SOURCE_TABLE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.end_key,
to_vertex_id=self.start_key,
label=TableSource.TABLE_SOURCE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.start_key,
NEPTUNE_HEADER_LABEL:
TableSource.TABLE_SOURCE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]
self.assertListEqual(expected, serialized_relations)
def test_create_relation_neptune(self) -> None:
relation = self.watermark.create_relation()
serialized_relation = neptune_serializer.convert_relationship(
relation[0])
expected = [{
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.start_key,
to_vertex_id=self.end_key,
label='BELONG_TO_TABLE'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.end_key,
NEPTUNE_HEADER_LABEL:
'BELONG_TO_TABLE',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.end_key,
to_vertex_id=self.start_key,
label='WATERMARK'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.start_key,
NEPTUNE_HEADER_LABEL:
'WATERMARK',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]
self.assertListEqual(serialized_relation, expected)
def test_create_relation_neptune(self) -> None:
relations = self.user.create_relation()
serialized = neptune_serializer.convert_relationship(relations[0])
start_key = '{email}'.format(email='*****@*****.**')
end_key = '{email}'.format(email='*****@*****.**')
expected = [
{
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=start_key,
to_vertex_id=end_key,
label=User.USER_MANAGER_RELATION_TYPE
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: end_key,
NEPTUNE_HEADER_LABEL: User.USER_MANAGER_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
},
{
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=end_key,
to_vertex_id=start_key,
label=User.MANAGER_USER_RELATION_TYPE
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: start_key,
NEPTUNE_HEADER_LABEL: User.MANAGER_USER_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
]
self.assertListEqual(serialized, expected)
def test_create_relation_neptune(self) -> None:
relations = self.badge_metada.create_relation()
serialized_relations: List[Dict] = sum([
neptune_serializer.convert_relationship(rel) for rel in relations
], [])
neptune_forward_expected_1 = {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.badge_metada.start_key,
to_vertex_id=BadgeMetadata.get_badge_key(badge1.name),
label=BadgeMetadata.BADGE_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.badge_metada.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
BadgeMetadata.get_badge_key(badge1.name),
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected_1 = {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=BadgeMetadata.get_badge_key(badge1.name),
to_vertex_id=self.badge_metada.start_key,
label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
BadgeMetadata.get_badge_key(badge1.name),
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.badge_metada.start_key,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_forward_expected_2 = {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.badge_metada.start_key,
to_vertex_id=BadgeMetadata.get_badge_key(badge2.name),
label=BadgeMetadata.BADGE_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.badge_metada.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
BadgeMetadata.get_badge_key(badge2.name),
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected_2 = {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=BadgeMetadata.get_badge_key(badge2.name),
to_vertex_id=self.badge_metada.start_key,
label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
BadgeMetadata.get_badge_key(badge2.name),
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.badge_metada.start_key,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
self.assertTrue(neptune_forward_expected_1 in serialized_relations)
self.assertTrue(neptune_reversed_expected_1 in serialized_relations)
self.assertTrue(neptune_forward_expected_2 in serialized_relations)
self.assertTrue(neptune_reversed_expected_2 in serialized_relations)
def test_full_dashboard_example_neptune(self) -> None:
expected_neptune_rels = [
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Cluster:_dashboard://gold',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_GROUP'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Cluster:_dashboard://gold',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_GROUP'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Cluster:_dashboard://gold',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'DASHBOARD_GROUP',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id='Cluster:_dashboard://gold',
label='DASHBOARD_GROUP_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id='Cluster:_dashboard://gold',
label='DASHBOARD_GROUP_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Cluster:_dashboard://gold',
NEPTUNE_HEADER_LABEL:
'DASHBOARD_GROUP_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
label='DESCRIPTION'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
label='DESCRIPTION'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Description:_dashboard://gold.Product - Jobs.cz/_description',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DESCRIPTION_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/_description',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DESCRIPTION_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Description:_dashboard://gold.Product - Jobs.cz/_description',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
label='DASHBOARD_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'DASHBOARD_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DASHBOARD'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DASHBOARD'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'DASHBOARD',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
label='DESCRIPTION'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
label='DESCRIPTION'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DESCRIPTION_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='DESCRIPTION_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'DESCRIPTION_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:test_tag',
label='TAG'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:test_tag',
label='TAG'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Tag:test_tag',
NEPTUNE_HEADER_LABEL:
'TAG',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:test_tag',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:test_tag',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Tag:test_tag',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'TAG_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:tag2',
label='TAG'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
to_vertex_id='Tag:tag2',
label='TAG'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Tag:tag2',
NEPTUNE_HEADER_LABEL:
'TAG',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:tag2',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Tag:tag2',
to_vertex_id=
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
label='TAG_OF'),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Tag:tag2',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'TAG_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
]
expected_neptune_nodes = [
{
NEPTUNE_HEADER_ID:
'Cluster:_dashboard://gold',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold',
NEPTUNE_HEADER_LABEL:
'Cluster',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)':
'gold'
},
{
NEPTUNE_HEADER_ID:
'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz/Agent',
NEPTUNE_HEADER_LABEL:
'Dashboard',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)':
'Agent',
'dashboard_url:String(single)':
'https://foo.bar/dashboard_group/foo/dashboard/bar',
'created_timestamp:Long(single)':
123456789,
},
{
NEPTUNE_HEADER_ID:
'Dashboardgroup:_dashboard://gold.Product - Jobs.cz',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz',
NEPTUNE_HEADER_LABEL:
'Dashboardgroup',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'name:String(single)':
'Product - Jobs.cz',
'dashboard_group_url:String(single)':
'https://foo.bar/dashboard_group/foo'
},
{
NEPTUNE_HEADER_ID:
'Description:_dashboard://gold.Product - Jobs.cz/_description',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz/_description',
NEPTUNE_HEADER_LABEL:
'Description',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'description:String(single)':
'foo dashboard group description',
},
{
NEPTUNE_HEADER_ID:
'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'_dashboard://gold.Product - Jobs.cz/Agent/_description',
NEPTUNE_HEADER_LABEL:
'Description',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'description:String(single)':
'Agent dashboard description'
},
{
NEPTUNE_HEADER_ID:
'Tag:test_tag',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'test_tag',
NEPTUNE_HEADER_LABEL:
'Tag',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'tag_type:String(single)':
'dashboard'
},
{
NEPTUNE_HEADER_ID:
'Tag:tag2',
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
'tag2',
NEPTUNE_HEADER_LABEL:
'Tag',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB,
'tag_type:String(single)':
'dashboard'
},
]
self.maxDiff = None
node_row = self.full_dashboard_metadata.next_node()
actual = []
while node_row:
node_serialized = neptune_serializer.convert_node(node_row)
actual.append(node_serialized)
node_row = self.full_dashboard_metadata.next_node()
self.assertEqual(expected_neptune_nodes, actual)
relation_row = self.full_dashboard_metadata.next_relation()
neptune_actual: List[List[Dict]] = []
while relation_row:
relation_serialized = neptune_serializer.convert_relationship(
relation_row)
neptune_actual.append(relation_serialized)
relation_row = self.full_dashboard_metadata.next_relation()
self.assertEqual(expected_neptune_rels, neptune_actual)
def test_create_relation_neptune(self) -> None:
actual = []
relation = self.badge_metada.create_next_relation()
while relation:
serialized_relations = neptune_serializer.convert_relationship(
relation)
actual.append(serialized_relations)
relation = self.badge_metada.create_next_relation()
badge_id_1 = BadgeMetadata.BADGE_NODE_LABEL + ':' + BadgeMetadata.get_badge_key(
badge1.name)
badge_id_2 = BadgeMetadata.BADGE_NODE_LABEL + ':' + BadgeMetadata.get_badge_key(
badge2.name)
start_key = self.badge_metada.start_label + ':' + self.badge_metada.start_key
neptune_forward_expected_1 = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=start_key,
to_vertex_id=badge_id_1,
label=BadgeMetadata.BADGE_RELATION_TYPE,
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=start_key,
to_vertex_id=badge_id_1,
label=BadgeMetadata.BADGE_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
badge_id_1,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected_1 = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=badge_id_1,
to_vertex_id=start_key,
label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=badge_id_1,
to_vertex_id=start_key,
label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
badge_id_1,
NEPTUNE_RELATIONSHIP_HEADER_TO:
start_key,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_forward_expected_2 = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=start_key,
to_vertex_id=badge_id_2,
label=BadgeMetadata.BADGE_RELATION_TYPE,
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=start_key,
to_vertex_id=badge_id_2,
label=BadgeMetadata.BADGE_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
badge_id_2,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected_2 = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=badge_id_2,
to_vertex_id=start_key,
label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=badge_id_2,
to_vertex_id=start_key,
label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
badge_id_2,
NEPTUNE_RELATIONSHIP_HEADER_TO:
start_key,
NEPTUNE_HEADER_LABEL:
BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
expected = [[neptune_forward_expected_1, neptune_reversed_expected_1],
[neptune_forward_expected_2, neptune_reversed_expected_2]]
self.assertEqual(expected, actual)
def test_create_relation_neptune(self) -> None:
expected = [[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="User:"******"Table:" +
self.table_owner.get_metadata_model_key(),
label=TableOwner.OWNER_TABLE_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="User:"******"Table:" +
self.table_owner.get_metadata_model_key(),
label=TableOwner.OWNER_TABLE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
"User:"******"Table:" + self.table_owner.get_metadata_model_key(),
NEPTUNE_HEADER_LABEL:
TableOwner.OWNER_TABLE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" +
self.table_owner.get_metadata_model_key(),
to_vertex_id="User:"******"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" +
self.table_owner.get_metadata_model_key(),
to_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(),
NEPTUNE_RELATIONSHIP_HEADER_TO:
"User:"******"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="User:"******"Table:" +
self.table_owner.get_metadata_model_key(),
label=TableOwner.OWNER_TABLE_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="User:"******"Table:" +
self.table_owner.get_metadata_model_key(),
label=TableOwner.OWNER_TABLE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
"User:"******"Table:" + self.table_owner.get_metadata_model_key(),
NEPTUNE_HEADER_LABEL:
TableOwner.OWNER_TABLE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" +
self.table_owner.get_metadata_model_key(),
to_vertex_id="User:"******"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" +
self.table_owner.get_metadata_model_key(),
to_vertex_id="User:"******"Table:" + self.table_owner.get_metadata_model_key(),
NEPTUNE_RELATIONSHIP_HEADER_TO:
"User:" + owner2,
NEPTUNE_HEADER_LABEL:
TableOwner.TABLE_OWNER_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]]
actual = []
relation = self.table_owner.create_next_relation()
while relation:
serialized_relation = neptune_serializer.convert_relationship(
relation)
actual.append(serialized_relation)
relation = self.table_owner.create_next_relation()
self.assertEqual(expected, actual)
def test_create_relations_neptune(self) -> None:
for tc in self.test_cases:
application_id = f'Application:{tc.application.application_key}'
table_id = 'Table:hive://gold.default/test_table'
neptune_forward_expected = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=table_id,
to_vertex_id=application_id,
label=tc.expected_relation_results[0][RELATION_TYPE],
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=table_id,
to_vertex_id=application_id,
label=tc.expected_relation_results[0][RELATION_TYPE],
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
table_id,
NEPTUNE_RELATIONSHIP_HEADER_TO:
application_id,
NEPTUNE_HEADER_LABEL:
tc.expected_relation_results[0][RELATION_TYPE],
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=application_id,
to_vertex_id=table_id,
label=tc.expected_relation_results[0]
[RELATION_REVERSE_TYPE],
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id=application_id,
to_vertex_id=table_id,
label=tc.expected_relation_results[0]
[RELATION_REVERSE_TYPE],
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
application_id,
NEPTUNE_RELATIONSHIP_HEADER_TO:
table_id,
NEPTUNE_HEADER_LABEL:
tc.expected_relation_results[0][RELATION_REVERSE_TYPE],
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}
neptune_expected = [[
neptune_forward_expected, neptune_reversed_expected
]]
actual = []
next_relation = tc.application.create_next_relation()
while next_relation:
serialized_next_relation = neptune_serializer.convert_relationship(
next_relation)
actual.append(serialized_next_relation)
next_relation = tc.application.create_next_relation()
self.assertEqual(actual, neptune_expected)
def test_create_relation_neptune(self) -> None:
expected = [[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.start_key,
to_vertex_id='Table:' + self.end_key1,
label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.start_key,
to_vertex_id='Table:' + self.end_key1,
label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Table:' + self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Table:' + self.end_key1,
NEPTUNE_HEADER_LABEL:
TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.end_key1,
to_vertex_id='Table:' + self.start_key,
label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.end_key1,
to_vertex_id='Table:' + self.start_key,
label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Table:' + self.end_key1,
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Table:' + self.start_key,
NEPTUNE_HEADER_LABEL:
TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}],
[{
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.start_key,
to_vertex_id='Table:' + self.end_key2,
label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.start_key,
to_vertex_id='Table:' + self.end_key2,
label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Table:' + self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Table:' + self.end_key2,
NEPTUNE_HEADER_LABEL:
TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.end_key2,
to_vertex_id='Table:' + self.start_key,
label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE
),
METADATA_KEY_PROPERTY_NAME_BULK_LOADER_FORMAT:
"{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:' + self.end_key2,
to_vertex_id='Table:' + self.start_key,
label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Table:' + self.end_key2,
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Table:' + self.start_key,
NEPTUNE_HEADER_LABEL:
TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]]
actual = []
relation = self.table_lineage.create_next_relation()
while relation:
serialized_relation = neptune_serializer.convert_relationship(
relation)
actual.append(serialized_relation)
relation = self.table_lineage.create_next_relation()
self.assertEqual(actual, expected)