def test_serialize(self) -> None:
# First test
node_row = self.metric_metadata.next_node()
actual = []
while node_row:
serialized_node = neo4_serializer.serialize_node(node_row)
actual.append(serialized_node)
node_row = self.metric_metadata.next_node()
self.assertEqual(self.expected_nodes, actual)
relation_row = self.metric_metadata.next_relation()
actual = []
while relation_row:
serialized_relation = neo4_serializer.serialize_relationship(
relation_row)
actual.append(serialized_relation)
relation_row = self.metric_metadata.next_relation()
self.assertEqual(self.expected_rels, actual)
# Second test
node_row = self.metric_metadata2.next_node()
actual = []
while node_row:
serialized_node = neo4_serializer.serialize_node(node_row)
actual.append(serialized_node)
node_row = self.metric_metadata2.next_node()
self.assertEqual(self.expected_nodes_deduped2, actual)
relation_row = self.metric_metadata2.next_relation()
actual = []
while relation_row:
serialized_relation = neo4_serializer.serialize_relationship(
relation_row)
actual.append(serialized_relation)
relation_row = self.metric_metadata2.next_relation()
self.assertEqual(self.expected_rels_deduped2, actual)
# Third test
node_row = self.metric_metadata3.next_node()
actual = []
while node_row:
serialized_node = neo4_serializer.serialize_node(node_row)
actual.append(serialized_node)
node_row = self.metric_metadata3.next_node()
self.assertEqual(self.expected_nodes_deduped3, actual)
relation_row = self.metric_metadata3.next_relation()
actual = []
while relation_row:
serialized_relation = neo4_serializer.serialize_relationship(
relation_row)
actual.append(serialized_relation)
relation_row = self.metric_metadata3.next_relation()
self.assertEqual(self.expected_rels_deduped3, actual)
def test_serialize(self) -> None:
node_row = self.table_col_usage.next_node()
actual = []
while node_row:
actual.append(neo4_serializer.serialize_node(node_row))
node_row = self.table_col_usage.next_node()
expected = [{
'LABEL': 'User',
'KEY': '*****@*****.**',
'email': '*****@*****.**'
}]
self.assertEqual(expected, actual)
rel_row = self.table_col_usage.next_relation()
actual = []
while rel_row:
actual.append(neo4_serializer.serialize_relationship(rel_row))
rel_row = self.table_col_usage.next_relation()
expected = [{
'read_count:UNQUOTED': 1,
'END_KEY': '*****@*****.**',
'START_LABEL': 'Table',
'END_LABEL': 'User',
'START_KEY': 'db://gold.scm/foo',
'TYPE': 'READ_BY',
'REVERSE_TYPE': 'READ'
}]
self.assertEqual(expected, actual)
def test_create_relation(self) -> None:
expected_relations = [
{
RELATION_START_KEY: self.start_key,
RELATION_START_LABEL: 'Table',
RELATION_END_KEY: self.end_key1,
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE
},
{
RELATION_START_KEY: self.start_key,
RELATION_START_LABEL: 'Table',
RELATION_END_KEY: self.end_key2,
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE
}
]
actual = []
relation = self.table_lineage.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(relation)
actual.append(serialized_relation)
relation = self.table_lineage.create_next_relation()
self.assertEqual(actual, expected_relations)
def test_create_relation(self) -> None:
expected_relation1 = {
RELATION_START_KEY: TABLE_KEY,
RELATION_START_LABEL: 'Table',
RELATION_END_KEY: owner1,
RELATION_END_LABEL: User.USER_NODE_LABEL,
RELATION_TYPE: OWNER_RELATION_TYPE,
RELATION_REVERSE_TYPE: OWNER_OF_OBJECT_RELATION_TYPE,
}
expected_relation2 = {
RELATION_START_KEY: TABLE_KEY,
RELATION_START_LABEL: 'Table',
RELATION_END_KEY: owner2,
RELATION_END_LABEL: User.USER_NODE_LABEL,
RELATION_TYPE: OWNER_RELATION_TYPE,
RELATION_REVERSE_TYPE: OWNER_OF_OBJECT_RELATION_TYPE,
}
expected = [expected_relation1, expected_relation2]
actual = []
relation = self.table_owner.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(relation)
actual.append(serialized_relation)
relation = self.table_owner.create_next_relation()
self.assertEqual(actual, expected)
def test_create_relation(self) -> None:
relations = self.table_lineage.create_relation()
self.assertEqual(len(relations), 2)
start_key = '{db}://{cluster}.{schema}/{tbl}'.format(db=DB,
schema=SCHEMA,
tbl=TABLE,
cluster=CLUSTER)
end_key1 = '{db}://{cluster}.{schema}/{tbl}'.format(db=DB,
schema='test_schema',
tbl='test_table1',
cluster=CLUSTER)
relation = {
RELATION_START_KEY: start_key,
RELATION_START_LABEL: 'Table',
RELATION_END_KEY: end_key1,
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE
}
actual_relations = [
neo4_serializer.serialize_relationship(relation)
for relation in relations
]
self.assertTrue(len(relations), 2)
self.assertTrue(relation in actual_relations)
def test_create_relation(self) -> None:
expected_relations = [{
RELATION_START_KEY:
self.start_key,
RELATION_START_LABEL:
TableSource.LABEL,
RELATION_END_KEY:
self.end_key,
RELATION_END_LABEL:
'Table',
RELATION_TYPE:
TableSource.SOURCE_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE:
TableSource.TABLE_SOURCE_RELATION_TYPE
}]
actual = []
relation = self.table_source.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(
relation)
actual.append(serialized_relation)
relation = self.table_source.create_next_relation()
self.assertEqual(expected_relations, actual)
def test_create_relation(self) -> None:
expected_relation1 = {
RELATION_START_KEY: owner1,
RELATION_START_LABEL: User.USER_NODE_LABEL,
RELATION_END_KEY: self.table_owner.get_metadata_model_key(),
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE
}
expected_relation2 = {
RELATION_START_KEY: owner2,
RELATION_START_LABEL: User.USER_NODE_LABEL,
RELATION_END_KEY: self.table_owner.get_metadata_model_key(),
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE
}
expected = [expected_relation1, expected_relation2]
actual = []
relation = self.table_owner.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(
relation)
actual.append(serialized_relation)
relation = self.table_owner.create_next_relation()
self.assertEqual(actual, expected)
def test_create_relation(self) -> None:
actual = []
relation = self.query_join_metadata.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(
relation)
actual.append(serialized_relation)
relation = self.query_join_metadata.create_next_relation()
self.maxDiff = None
expected_relations = [{
RELATION_END_KEY:
self._expected_key,
RELATION_END_LABEL:
QueryExecutionsMetadata.NODE_LABEL,
RELATION_REVERSE_TYPE:
QueryExecutionsMetadata.INVERSE_QUERY_EXECUTION_RELATION_TYPE,
RELATION_START_KEY:
self.query_metadata.get_key_self(),
RELATION_START_LABEL:
QueryMetadata.NODE_LABEL,
RELATION_TYPE:
QueryExecutionsMetadata.QUERY_EXECUTION_RELATION_TYPE
}]
self.assertEquals(expected_relations, actual)
def test_col_badge_field(self) -> None:
self.table_metadata4 = TableMetadata('hive', 'gold', 'test_schema4', 'test_table4', 'test_table4', [
ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0, ['col-badge1', 'col-badge2'])],
is_view=False, attr1='uri', attr2='attr2')
node_row = self.table_metadata4.next_node()
actual = []
while node_row:
serialized_node_row = neo4_serializer.serialize_node(node_row)
actual.append(serialized_node_row)
node_row = self.table_metadata4.next_node()
self.assertEqual(actual[4].get('KEY'), 'col-badge1')
self.assertEqual(actual[5].get('KEY'), 'col-badge2')
relation_row = self.table_metadata4.next_relation()
actual = []
while relation_row:
serialized_relation_row = neo4_serializer.serialize_relationship(relation_row)
actual.append(serialized_relation_row)
relation_row = self.table_metadata4.next_relation()
expected_col_badge_rel1 = {'END_KEY': 'col-badge1', 'START_LABEL': 'Column',
'END_LABEL': 'Badge',
'START_KEY': 'hive://gold.test_schema4/test_table4/test_id1',
'TYPE': 'HAS_BADGE', 'REVERSE_TYPE': 'BADGE_FOR'}
expected_col_badge_rel2 = {'END_KEY': 'col-badge2', 'START_LABEL': 'Column',
'END_LABEL': 'Badge',
'START_KEY': 'hive://gold.test_schema4/test_table4/test_id1',
'TYPE': 'HAS_BADGE', 'REVERSE_TYPE': 'BADGE_FOR'}
self.assertEqual(actual[4], expected_col_badge_rel1)
self.assertEqual(actual[5], expected_col_badge_rel2)
def test_create_relation(self) -> None:
actual = []
relation = self.user.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(
relation)
actual.append(serialized_relation)
relation = self.user.create_next_relation()
start_key = '*****@*****.**'
end_key = '*****@*****.**'
expected_relations = [{
RELATION_START_KEY:
start_key,
RELATION_START_LABEL:
User.USER_NODE_LABEL,
RELATION_END_KEY:
end_key,
RELATION_END_LABEL:
User.USER_NODE_LABEL,
RELATION_TYPE:
User.USER_MANAGER_RELATION_TYPE,
RELATION_REVERSE_TYPE:
User.MANAGER_USER_RELATION_TYPE
}]
self.assertTrue(expected_relations, actual)
def test_create_relation(self) -> None:
actual = []
relation = self.badge_metada.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(
relation)
actual.append(serialized_relation)
relation = self.badge_metada.create_next_relation()
relation1 = {
RELATION_START_LABEL: self.badge_metada.start_label,
RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL,
RELATION_START_KEY: self.badge_metada.start_key,
RELATION_END_KEY: BadgeMetadata.get_badge_key(badge1.name),
RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE,
RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
}
relation2 = {
RELATION_START_LABEL: self.badge_metada.start_label,
RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL,
RELATION_START_KEY: self.badge_metada.start_key,
RELATION_END_KEY: BadgeMetadata.get_badge_key(badge2.name),
RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE,
RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
}
expected = [relation1, relation2]
self.assertEqual(expected, actual)
def test_dashboard_owner_relations(self) -> None:
dashboard_usage = DashboardUsage(
dashboard_group_id='dashboard_group_id',
dashboard_id='dashboard_id',
email='*****@*****.**',
view_count=123,
cluster='cluster_id',
product='product_id')
actual = dashboard_usage.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
expected: Dict[str, Any] = {
'read_count:UNQUOTED': 123,
RELATION_END_KEY: '*****@*****.**',
RELATION_START_LABEL: 'Dashboard',
RELATION_END_LABEL: 'User',
RELATION_START_KEY:
'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
RELATION_TYPE: 'READ_BY',
RELATION_REVERSE_TYPE: 'READ'
}
assert actual is not None
self.assertDictEqual(expected, actual_serialized)
self.assertIsNone(dashboard_usage.create_next_relation())
def test_dashboard_owner_relations(self) -> None:
dashboard_last_modified = DashboardLastModifiedTimestamp(
last_modified_timestamp=123456789,
cluster='cluster_id',
product='product_id',
dashboard_id='dashboard_id',
dashboard_group_id='dashboard_group_id')
actual = dashboard_last_modified.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
expected: Dict[str, Any] = {
RELATION_END_KEY:
'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id'
'/_last_modified_timestamp',
RELATION_START_LABEL:
'Dashboard',
RELATION_END_LABEL:
'Timestamp',
RELATION_START_KEY:
'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
RELATION_TYPE:
'LAST_UPDATED_AT',
RELATION_REVERSE_TYPE:
'LAST_UPDATED_TIME_OF'
}
assert actual is not None
self.assertDictEqual(actual_serialized, expected)
self.assertIsNone(dashboard_last_modified.create_next_relation())
def test_serialize(self) -> None:
node_row = self.table_col_usage.next_node()
actual = []
while node_row:
actual.append(neo4_serializer.serialize_node(node_row))
node_row = self.table_col_usage.next_node()
expected = [{'first_name': '',
'last_name': '',
'full_name': '',
'employee_type': '',
'is_active:UNQUOTED': True,
'updated_at:UNQUOTED': 0,
'LABEL': 'User',
'slack_id': '',
'KEY': '*****@*****.**',
'github_username': '',
'team_name': '',
'email': '*****@*****.**',
'role_name': ''}]
self.assertEqual(expected, actual)
rel_row = self.table_col_usage.next_relation()
actual = []
while rel_row:
actual.append(neo4_serializer.serialize_relationship(rel_row))
rel_row = self.table_col_usage.next_relation()
expected = [{'read_count:UNQUOTED': 1, 'END_KEY': '*****@*****.**', 'START_LABEL': 'Table',
'END_LABEL': 'User', 'START_KEY': 'db://gold.scm/foo', 'TYPE': 'READ_BY', 'REVERSE_TYPE': 'READ'}]
self.assertEqual(expected, actual)
def test_tags_populated_from_str(self) -> None:
self.table_metadata5 = TableMetadata('hive', 'gold', 'test_schema5', 'test_table5', 'test_table5', [
ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0)], tags="tag3, tag4")
# Test table tag field populated from str
node_row = self.table_metadata5.next_node()
actual = []
while node_row:
node_row_serialized = neo4_serializer.serialize_node(node_row)
actual.append(node_row_serialized)
node_row = self.table_metadata5.next_node()
self.assertEqual(actual[2].get('LABEL'), 'Tag')
self.assertEqual(actual[2].get('KEY'), 'tag3')
self.assertEqual(actual[3].get('KEY'), 'tag4')
relation_row = self.table_metadata5.next_relation()
actual = []
while relation_row:
relation_row_serialized = neo4_serializer.serialize_relationship(relation_row)
actual.append(relation_row_serialized)
relation_row = self.table_metadata5.next_relation()
# Table tag relationship
expected_tab_tag_rel3 = {'END_KEY': 'tag3', 'START_LABEL': 'Table', 'END_LABEL':
'Tag', 'START_KEY': 'hive://gold.test_schema5/test_table5',
'TYPE': 'TAGGED_BY', 'REVERSE_TYPE': 'TAG'}
expected_tab_tag_rel4 = {'END_KEY': 'tag4', 'START_LABEL': 'Table',
'END_LABEL': 'Tag', 'START_KEY': 'hive://gold.test_schema5/test_table5',
'TYPE': 'TAGGED_BY', 'REVERSE_TYPE': 'TAG'}
self.assertEqual(actual[2], expected_tab_tag_rel3)
self.assertEqual(actual[3], expected_tab_tag_rel4)
def test_create_relation(self) -> None:
actual = []
relation = self.tableLastUpdated.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(relation)
actual.append(serialized_relation)
relation = self.tableLastUpdated.create_next_relation()
def test_create_relation(self) -> None:
relation = self.table_stats.create_relation()
self.assertEquals(len(relation), 1)
serialized_relation = neo4_serializer.serialize_relationship(
relation[0])
self.assertEquals(serialized_relation, self.expected_relation_result)
def test_create_relation(self) -> None:
relations = self.table_owner.create_relation()
self.assertEqual(len(relations), 2)
expected_relation1 = {
RELATION_START_KEY: owner1,
RELATION_START_LABEL: User.USER_NODE_LABEL,
RELATION_END_KEY: self.table_owner.get_metadata_model_key(),
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE
}
expected_relation2 = {
RELATION_START_KEY: owner2,
RELATION_START_LABEL: User.USER_NODE_LABEL,
RELATION_END_KEY: self.table_owner.get_metadata_model_key(),
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableOwner.OWNER_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableOwner.TABLE_OWNER_RELATION_TYPE
}
actual_relations = [
neo4_serializer.serialize_relationship(relation)
for relation in relations
]
self.assertTrue(expected_relation1 in actual_relations)
self.assertTrue(expected_relation2 in actual_relations)
def test_create_relation(self) -> None:
relations = self.badge_metada.create_relation()
serialized_relations = [
neo4_serializer.serialize_relationship(relation)
for relation in relations
]
self.assertEqual(len(relations), 2)
relation1 = {
RELATION_START_LABEL: self.badge_metada.start_label,
RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL,
RELATION_START_KEY: self.badge_metada.start_key,
RELATION_END_KEY: BadgeMetadata.get_badge_key(badge1.name),
RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE,
RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
}
relation2 = {
RELATION_START_LABEL: self.badge_metada.start_label,
RELATION_END_LABEL: BadgeMetadata.BADGE_NODE_LABEL,
RELATION_START_KEY: self.badge_metada.start_key,
RELATION_END_KEY: BadgeMetadata.get_badge_key(badge2.name),
RELATION_TYPE: BadgeMetadata.BADGE_RELATION_TYPE,
RELATION_REVERSE_TYPE: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE,
}
self.assertTrue(relation1 in serialized_relations)
self.assertTrue(relation2 in serialized_relations)
def test_create_relation(self) -> None:
actual = self.schema.create_next_relation()
serialized_actual = neo4_serializer.serialize_relationship(actual)
expected = {'END_KEY': 'db://cluster.schema/_description', 'START_LABEL': 'Schema', 'END_LABEL': 'Description',
'START_KEY': 'db://cluster.schema', 'TYPE': 'DESCRIPTION', 'REVERSE_TYPE': 'DESCRIPTION_OF'}
self.assertEqual(expected, serialized_actual)
self.assertIsNone(self.schema.create_next_relation())
def test_create_relation(self) -> None:
actual = []
relation = self.application.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(relation)
actual.append(serialized_relation)
relation = self.application.create_next_relation()
self.assertEqual(actual, self.expected_relation_results)
def test_tags_field(self) -> None:
self.table_metadata4 = TableMetadata(
'hive',
'gold',
'test_schema4',
'test_table4',
'test_table4', [
ColumnMetadata('test_id1', 'description of test_table1',
'bigint', 0)
],
is_view=False,
tags=['tag1', 'tag2'],
attr1='uri',
attr2='attr2')
node_row = self.table_metadata4.next_node()
actual = []
while node_row:
node_row_serialized = neo4_serializer.serialize_node(node_row)
actual.append(node_row_serialized)
node_row = self.table_metadata4.next_node()
self.assertEqual(actual[0].get('attr1'), 'uri')
self.assertEqual(actual[0].get('attr2'), 'attr2')
self.assertEqual(actual[2].get('LABEL'), 'Tag')
self.assertEqual(actual[2].get('KEY'), 'tag1')
self.assertEqual(actual[3].get('KEY'), 'tag2')
relation_row = self.table_metadata4.next_relation()
actual = []
while relation_row:
relation_row_serialized = neo4_serializer.serialize_relationship(
relation_row)
actual.append(relation_row_serialized)
relation_row = self.table_metadata4.next_relation()
# Table tag relationship
expected_tab_tag_rel1 = {
'END_KEY': 'tag1',
'START_LABEL': 'Table',
'END_LABEL': 'Tag',
'START_KEY': 'hive://gold.test_schema4/test_table4',
'TYPE': 'TAGGED_BY',
'REVERSE_TYPE': 'TAG'
}
expected_tab_tag_rel2 = {
'END_KEY': 'tag2',
'START_LABEL': 'Table',
'END_LABEL': 'Tag',
'START_KEY': 'hive://gold.test_schema4/test_table4',
'TYPE': 'TAGGED_BY',
'REVERSE_TYPE': 'TAG'
}
self.assertEqual(actual[2], expected_tab_tag_rel1)
self.assertEqual(actual[3], expected_tab_tag_rel2)
def test_dashboard_table_relations(self) -> None:
dashboard_table = DashboardTable(table_ids=['hive://gold.schema/table1'],
cluster='cluster_id', product='product_id',
dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id')
actual = dashboard_table.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
actual_neptune_serialized = neptune_serializer.convert_relationship(actual)
expected = {RELATION_END_KEY: 'hive://gold.schema/table1', RELATION_START_LABEL: 'Dashboard',
RELATION_END_LABEL: 'Table',
RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
RELATION_TYPE: 'DASHBOARD_WITH_TABLE',
RELATION_REVERSE_TYPE: 'TABLE_OF_DASHBOARD'}
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
to_vertex_id='Table:hive://gold.schema/table1',
label='DASHBOARD_WITH_TABLE'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
to_vertex_id='Table:hive://gold.schema/table1',
label='DASHBOARD_WITH_TABLE'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.schema/table1',
NEPTUNE_HEADER_LABEL: 'DASHBOARD_WITH_TABLE',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:hive://gold.schema/table1',
to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
label='TABLE_OF_DASHBOARD'
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id='Table:hive://gold.schema/table1',
to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
label='TABLE_OF_DASHBOARD'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.schema/table1',
NEPTUNE_RELATIONSHIP_HEADER_TO:
'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
NEPTUNE_HEADER_LABEL: 'TABLE_OF_DASHBOARD',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
assert actual is not None
self.assertDictEqual(actual_serialized, expected)
self.assertDictEqual(actual_neptune_serialized[0], neptune_forward_expected)
self.assertDictEqual(actual_neptune_serialized[1], neptune_reversed_expected)
def test_create_relation(self) -> None:
dashboard_chart = DashboardChart(dashboard_group_id='dg_id',
dashboard_id='d_id',
query_id='q_id',
chart_id='c_id',
chart_name='c_name',
chart_type='bar',
)
actual = dashboard_chart.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
actual_neptune_serialized = neptune_serializer.convert_relationship(actual)
start_key = '_dashboard://gold.dg_id/d_id/query/q_id'
end_key = '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id'
expected: Dict[str, Any] = {
RELATION_END_KEY: end_key,
RELATION_START_LABEL: 'Query',
RELATION_END_LABEL: 'Chart',
RELATION_START_KEY: start_key,
RELATION_TYPE: 'HAS_CHART',
RELATION_REVERSE_TYPE: 'CHART_OF'
}
neptune_forward_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=start_key,
to_vertex_id=end_key,
label='HAS_CHART'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: end_key,
NEPTUNE_HEADER_LABEL: 'HAS_CHART',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
neptune_reversed_expected = {
NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=end_key,
to_vertex_id=start_key,
label='CHART_OF'
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: start_key,
NEPTUNE_HEADER_LABEL: 'CHART_OF',
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
assert actual is not None
self.assertEqual(expected, actual_serialized)
self.assertEqual(neptune_forward_expected, actual_neptune_serialized[0])
self.assertEqual(neptune_reversed_expected, actual_neptune_serialized[1])
self.assertIsNone(dashboard_chart.create_next_relation())
def test_serialize(self) -> None:
col_readers = [
ColumnReader(database='db',
cluster='gold',
schema='scm',
table='foo',
column='*',
user_email='*****@*****.**')
]
table_col_usage = TableColumnUsage(col_readers=col_readers)
node_row = table_col_usage.next_node()
actual = []
while node_row:
actual.append(neo4_serializer.serialize_node(node_row))
node_row = table_col_usage.next_node()
expected = [{
'first_name': '',
'last_name': '',
'full_name': '',
'employee_type': '',
'is_active:UNQUOTED': True,
'updated_at:UNQUOTED': 0,
'LABEL': 'User',
'slack_id': '',
'KEY': '*****@*****.**',
'github_username': '',
'team_name': '',
'email': '*****@*****.**',
'role_name': ''
}]
self.assertEqual(expected, actual)
rel_row = table_col_usage.next_relation()
actual = []
while rel_row:
actual.append(neo4_serializer.serialize_relationship(rel_row))
rel_row = table_col_usage.next_relation()
expected = [{
'read_count:UNQUOTED': 1,
'END_KEY': '*****@*****.**',
'START_LABEL': 'Table',
'END_LABEL': 'User',
'START_KEY': 'db://gold.scm/foo',
'TYPE': 'READ_BY',
'REVERSE_TYPE': 'READ'
}]
self.assertEqual(expected, actual)
def test_dashboard_table_without_dot_as_name(self) -> None:
dashboard_table = DashboardTable(table_ids=['bq-name://project-id.schema-name/table-name'],
cluster='cluster_id', product='product_id',
dashboard_id='dashboard_id', dashboard_group_id='dashboard_group_id')
actual = dashboard_table.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
expected = {RELATION_END_KEY: 'bq-name://project-id.schema-name/table-name', RELATION_START_LABEL: 'Dashboard',
RELATION_END_LABEL: 'Table',
RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
RELATION_TYPE: 'DASHBOARD_WITH_TABLE',
RELATION_REVERSE_TYPE: 'TABLE_OF_DASHBOARD'}
assert actual is not None
self.assertDictEqual(actual_serialized, expected)
def test_create_relation(self) -> None:
actual = self.dashboard_query.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
expected = {
RELATION_END_KEY: '_dashboard://gold.dg_id/d_id/query/q_id',
RELATION_START_LABEL: 'Dashboard',
RELATION_END_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL,
RELATION_START_KEY: '_dashboard://gold.dg_id/d_id',
RELATION_TYPE: 'HAS_QUERY',
RELATION_REVERSE_TYPE: 'QUERY_OF'
}
self.assertEqual(expected, actual_serialized)
def test_create_relation(self) -> None:
actual = []
relation = self.query_join_metadata.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(
relation)
actual.append(serialized_relation)
relation = self.query_join_metadata.create_next_relation()
expected_relations = [{
RELATION_END_KEY:
self._expected_key,
RELATION_END_LABEL:
QueryJoinMetadata.NODE_LABEL,
RELATION_REVERSE_TYPE:
QueryJoinMetadata.INVERSE_COLUMN_JOIN_RELATION_TYPE,
RELATION_START_KEY:
'hive://gold.test_schema1/test_table1/field',
RELATION_START_LABEL:
ColumnMetadata.COLUMN_NODE_LABEL,
RELATION_TYPE:
QueryJoinMetadata.COLUMN_JOIN_RELATION_TYPE
}, {
RELATION_END_KEY:
self._expected_key,
RELATION_END_LABEL:
QueryJoinMetadata.NODE_LABEL,
RELATION_REVERSE_TYPE:
QueryJoinMetadata.INVERSE_COLUMN_JOIN_RELATION_TYPE,
RELATION_START_KEY:
'hive://gold.test_schema1/test_table2/field',
RELATION_START_LABEL:
ColumnMetadata.COLUMN_NODE_LABEL,
RELATION_TYPE:
QueryJoinMetadata.COLUMN_JOIN_RELATION_TYPE
}, {
RELATION_END_KEY:
self._expected_key,
RELATION_END_LABEL:
QueryJoinMetadata.NODE_LABEL,
RELATION_REVERSE_TYPE:
QueryJoinMetadata.INVERSE_QUERY_JOIN_RELATION_TYPE,
RELATION_START_KEY:
'748c28f86de411b1d2b9deb6ae105eba',
RELATION_START_LABEL:
QueryMetadata.NODE_LABEL,
RELATION_TYPE:
QueryJoinMetadata.QUERY_JOIN_RELATION_TYPE
}]
self.assertEquals(expected_relations, actual)
def test_dashboard_owner_relations(self) -> None:
actual = self.dashboard_owner.create_next_relation()
actual_serialized = neo4_serializer.serialize_relationship(actual)
expected = {
RELATION_END_KEY: '*****@*****.**',
RELATION_START_LABEL: 'Dashboard',
RELATION_END_LABEL: 'User',
RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
RELATION_TYPE: 'OWNER',
RELATION_REVERSE_TYPE: 'OWNER_OF'
}
assert actual is not None
self.assertDictEqual(actual_serialized, expected)
def test_create_relation_programmatic_description(self) -> None:
schema = SchemaModel(schema_key='db://cluster.schema',
schema='schema_name',
description='foo',
description_source='bar')
actual = schema.create_next_relation()
serialized_actual = neo4_serializer.serialize_relationship(actual)
expected = {
'END_KEY': 'db://cluster.schema/_bar_description', 'START_LABEL': 'Schema',
'END_LABEL': 'Programmatic_Description', 'START_KEY': 'db://cluster.schema', 'TYPE': 'DESCRIPTION',
'REVERSE_TYPE': 'DESCRIPTION_OF'
}
self.assertEqual(expected, serialized_actual)
self.assertIsNone(schema.create_next_relation())
