class TestTableSource(unittest.TestCase):
def setUp(self):
# type: () -> None
super(TestTableSource, self).setUp()
self.table_source = TableSource(db_name='hive',
schema_name=SCHEMA,
table_name=TABLE,
cluster=CLUSTER,
source=SOURCE)
def test_get_source_model_key(self):
# type: () -> None
source = self.table_source.get_source_model_key()
self.assertEquals(source, '{db}://{cluster}.{schema}/{tbl}/_source'.format(db=DB,
schema=SCHEMA,
tbl=TABLE,
cluster=CLUSTER,
))
def test_get_metadata_model_key(self):
# type: () -> None
metadata = self.table_source.get_metadata_model_key()
self.assertEquals(metadata, 'hive://default.base/test')
def test_create_nodes(self):
# type: () -> None
nodes = self.table_source.create_nodes()
self.assertEquals(len(nodes), 1)
def test_create_relation(self):
# type: () -> None
relations = self.table_source.create_relation()
self.assertEquals(len(relations), 1)
start_key = '{db}://{cluster}.{schema}/{tbl}/_source'.format(db=DB,
schema=SCHEMA,
tbl=TABLE,
cluster=CLUSTER)
end_key = '{db}://{cluster}.{schema}/{tbl}'.format(db=DB,
schema=SCHEMA,
tbl=TABLE,
cluster=CLUSTER)
relation = {
RELATION_START_KEY: start_key,
RELATION_START_LABEL: TableSource.LABEL,
RELATION_END_KEY: end_key,
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableSource.SOURCE_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableSource.TABLE_SOURCE_RELATION_TYPE
}
self.assertTrue(relation in relations)
class TestTableSource(unittest.TestCase):
def setUp(self) -> None:
super(TestTableSource, self).setUp()
self.table_source = TableSource(db_name='hive',
schema=SCHEMA,
table_name=TABLE,
cluster=CLUSTER,
source=SOURCE)
def test_get_source_model_key(self) -> None:
source = self.table_source.get_source_model_key()
self.assertEqual(source, f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}/_source')
def test_get_metadata_model_key(self) -> None:
metadata = self.table_source.get_metadata_model_key()
self.assertEqual(metadata, 'hive://default.base/test')
def test_create_nodes(self) -> None:
nodes = self.table_source.create_nodes()
self.assertEqual(len(nodes), 1)
def test_create_relation(self) -> None:
relations = self.table_source.create_relation()
self.assertEquals(len(relations), 1)
serialized_relation = neo4_serializer.serialize_relationship(relations[0])
start_key = f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}/_source'
end_key = f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}'
expected_relation = {
RELATION_START_KEY: start_key,
RELATION_START_LABEL: TableSource.LABEL,
RELATION_END_KEY: end_key,
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableSource.SOURCE_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableSource.TABLE_SOURCE_RELATION_TYPE
}
self.assertDictEqual(expected_relation, serialized_relation)
class TestTableSource(unittest.TestCase):
def setUp(self) -> None:
super(TestTableSource, self).setUp()
self.table_source = TableSource(db_name='hive',
schema=SCHEMA,
table_name=TABLE,
cluster=CLUSTER,
source=SOURCE)
self.start_key = f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}/_source'
self.end_key = f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}'
def test_get_source_model_key(self) -> None:
source = self.table_source.get_source_model_key()
self.assertEqual(source, f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}/_source')
def test_get_metadata_model_key(self) -> None:
metadata = self.table_source.get_metadata_model_key()
self.assertEqual(metadata, 'hive://default.base/test')
def test_create_nodes(self) -> None:
nodes = self.table_source.create_nodes()
self.assertEqual(len(nodes), 1)
def test_create_relation(self) -> None:
relations = self.table_source.create_relation()
self.assertEquals(len(relations), 1)
serialized_relation = neo4_serializer.serialize_relationship(
relations[0])
expected_relation = {
RELATION_START_KEY: self.start_key,
RELATION_START_LABEL: TableSource.LABEL,
RELATION_END_KEY: self.end_key,
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableSource.SOURCE_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableSource.TABLE_SOURCE_RELATION_TYPE
}
self.assertDictEqual(expected_relation, serialized_relation)
def test_create_relation_neptune(self) -> None:
relations = self.table_source.create_relation()
serialized_relations = neptune_serializer.convert_relationship(
relations[0])
expected = [{
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.start_key,
to_vertex_id=self.end_key,
label=TableSource.SOURCE_TABLE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.end_key,
NEPTUNE_HEADER_LABEL:
TableSource.SOURCE_TABLE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}, {
NEPTUNE_HEADER_ID:
"{from_vertex_id}_{to_vertex_id}_{label}".format(
from_vertex_id=self.end_key,
to_vertex_id=self.start_key,
label=TableSource.TABLE_SOURCE_RELATION_TYPE),
NEPTUNE_RELATIONSHIP_HEADER_FROM:
self.end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO:
self.start_key,
NEPTUNE_HEADER_LABEL:
TableSource.TABLE_SOURCE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT:
NEPTUNE_CREATION_TYPE_JOB
}]
self.assertListEqual(expected, serialized_relations)
class TestTableSource(unittest.TestCase):
def setUp(self) -> None:
super(TestTableSource, self).setUp()
self.table_source = TableSource(db_name='hive',
schema=SCHEMA,
table_name=TABLE,
cluster=CLUSTER,
source=SOURCE)
self.start_key = f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}/_source'
self.end_key = f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}'
def test_get_source_model_key(self) -> None:
source = self.table_source.get_source_model_key()
self.assertEqual(source, f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}/_source')
def test_get_metadata_model_key(self) -> None:
metadata = self.table_source.get_metadata_model_key()
self.assertEqual(metadata, 'hive://default.base/test')
def test_create_nodes(self) -> None:
expected_nodes = [{
'LABEL': 'Source',
'KEY': f'{DB}://{CLUSTER}.{SCHEMA}/{TABLE}/_source',
'source': SOURCE,
'source_type': 'github'
}]
actual = []
node = self.table_source.create_next_node()
while node:
serialized_node = neo4_serializer.serialize_node(node)
actual.append(serialized_node)
node = self.table_source.create_next_node()
self.assertEqual(expected_nodes, actual)
def test_create_relation(self) -> None:
expected_relations = [{
RELATION_START_KEY: self.start_key,
RELATION_START_LABEL: TableSource.LABEL,
RELATION_END_KEY: self.end_key,
RELATION_END_LABEL: 'Table',
RELATION_TYPE: TableSource.SOURCE_TABLE_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableSource.TABLE_SOURCE_RELATION_TYPE
}]
actual = []
relation = self.table_source.create_next_relation()
while relation:
serialized_relation = neo4_serializer.serialize_relationship(relation)
actual.append(serialized_relation)
relation = self.table_source.create_next_relation()
self.assertEqual(expected_relations, actual)
def test_create_relation_neptune(self) -> None:
actual = []
relation = self.table_source.create_next_relation()
while relation:
serialized_relation = neptune_serializer.convert_relationship(relation)
actual.append(serialized_relation)
relation = self.table_source.create_next_relation()
expected = [
[
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Source:" + self.start_key,
to_vertex_id="Table:" + self.end_key,
label=TableSource.SOURCE_TABLE_RELATION_TYPE
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Source:" + self.start_key,
to_vertex_id="Table:" + self.end_key,
label=TableSource.SOURCE_TABLE_RELATION_TYPE
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: "Source:" + self.start_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: "Table:" + self.end_key,
NEPTUNE_HEADER_LABEL: TableSource.SOURCE_TABLE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
},
{
NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" + self.end_key,
to_vertex_id="Source:" + self.start_key,
label=TableSource.TABLE_SOURCE_RELATION_TYPE
),
METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
from_vertex_id="Table:" + self.end_key,
to_vertex_id="Source:" + self.start_key,
label=TableSource.TABLE_SOURCE_RELATION_TYPE
),
NEPTUNE_RELATIONSHIP_HEADER_FROM: "Table:" + self.end_key,
NEPTUNE_RELATIONSHIP_HEADER_TO: "Source:" + self.start_key,
NEPTUNE_HEADER_LABEL: TableSource.TABLE_SOURCE_RELATION_TYPE,
NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
}
]
]
self.assertListEqual(expected, actual)
def test_create_records(self) -> None:
expected = [{
'rk': self.table_source.get_source_model_key(),
'source': self.table_source.source,
'source_type': self.table_source.source_type,
'table_rk': self.table_source.get_metadata_model_key()
}]
actual = []
record = self.table_source.create_next_record()
while record:
serialized_record = mysql_serializer.serialize_record(record)
actual.append(serialized_record)
record = self.table_source.create_next_record()
self.assertEqual(expected, actual)