class TestWatermark(unittest.TestCase): def setUp(self) -> None: super(TestWatermark, self).setUp() self.watermark = Watermark(create_time='2017-09-18T00:00:00', database=DATABASE, schema=SCHEMA, table_name=TABLE, cluster=CLUSTER, part_type=PART_TYPE, part_name=NESTED_PART) self.start_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}/{PART_TYPE}/' self.end_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}' self.expected_node_result = GraphNode(key=self.start_key, label='Watermark', attributes={ 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00' }) self.expected_serialized_node_result = { NODE_KEY: self.start_key, NODE_LABEL: 'Watermark', 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00' } self.expected_relation_result = GraphRelationship( start_label='Watermark', end_label='Table', start_key=self.start_key, end_key=self.end_key, type='BELONG_TO_TABLE', reverse_type='WATERMARK', attributes={}) self.expected_serialized_relation_result = { RELATION_START_KEY: self.start_key, RELATION_START_LABEL: 'Watermark', RELATION_END_KEY: self.end_key, RELATION_END_LABEL: 'Table', RELATION_TYPE: 'BELONG_TO_TABLE', RELATION_REVERSE_TYPE: 'WATERMARK' } def test_get_watermark_model_key(self) -> None: watermark = self.watermark.get_watermark_model_key() self.assertEqual( watermark, f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}/{PART_TYPE}/') def test_get_metadata_model_key(self) -> None: metadata = self.watermark.get_metadata_model_key() self.assertEqual(metadata, f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}') def test_create_nodes(self) -> None: nodes = self.watermark.create_nodes() self.assertEquals(len(nodes), 1) self.assertEquals(nodes[0], self.expected_node_result) self.assertEqual(neo4_serializer.serialize_node(nodes[0]), self.expected_serialized_node_result) def test_create_nodes_neptune(self) -> None: nodes = self.watermark.create_nodes() expected_serialized_node_result = { NEPTUNE_HEADER_ID: self.start_key, NEPTUNE_HEADER_LABEL: 'Watermark', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'partition_key:String(single)': 'ds', 'partition_value:String(single)': '2017-09-18/feature_id=9', 'create_time:String(single)': '2017-09-18T00:00:00' } serialized_node = neptune_serializer.convert_node(nodes[0]) self.assertDictEqual(expected_serialized_node_result, serialized_node) def test_create_relation(self) -> None: relation = self.watermark.create_relation() self.assertEquals(len(relation), 1) self.assertEquals(relation[0], self.expected_relation_result) self.assertEqual(neo4_serializer.serialize_relationship(relation[0]), self.expected_serialized_relation_result) def test_create_relation_neptune(self) -> None: relation = self.watermark.create_relation() serialized_relation = neptune_serializer.convert_relationship( relation[0]) expected = [{ NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.start_key, to_vertex_id=self.end_key, label='BELONG_TO_TABLE'), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: self.end_key, NEPTUNE_HEADER_LABEL: 'BELONG_TO_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( from_vertex_id=self.end_key, to_vertex_id=self.start_key, label='WATERMARK'), NEPTUNE_RELATIONSHIP_HEADER_FROM: self.end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: self.start_key, NEPTUNE_HEADER_LABEL: 'WATERMARK', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }] self.assertListEqual(serialized_relation, expected) def test_create_next_node(self) -> None: next_node = self.watermark.create_next_node() self.assertEquals(neo4_serializer.serialize_node(next_node), self.expected_serialized_node_result) def test_create_next_relation(self) -> None: next_relation = self.watermark.create_next_relation() self.assertEquals( neo4_serializer.serialize_relationship(next_relation), self.expected_serialized_relation_result)
class TestWatermark(unittest.TestCase): def setUp(self) -> None: super(TestWatermark, self).setUp() self.watermark = Watermark(create_time='2017-09-18T00:00:00', database=DATABASE, schema=SCHEMA, table_name=TABLE, cluster=CLUSTER, part_type=PART_TYPE, part_name=NESTED_PART) start_key = '{database}://{cluster}.{schema}/{table}/{part_type}/'.format( database=DATABASE, cluster=CLUSTER, schema=SCHEMA, table=TABLE, part_type=PART_TYPE) end_key = '{database}://{cluster}.{schema}/{table}'.format( database=DATABASE, cluster=CLUSTER, schema=SCHEMA, table=TABLE) self.expected_node_result = GraphNode(key=start_key, label='Watermark', attributes={ 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00' }) self.expected_serialized_node_result = { NODE_KEY: start_key, NODE_LABEL: 'Watermark', 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00' } self.expected_relation_result = GraphRelationship( start_label='Watermark', end_label='Table', start_key=start_key, end_key=end_key, type='BELONG_TO_TABLE', reverse_type='WATERMARK', attributes={}) self.expected_serialized_relation_result = { RELATION_START_KEY: start_key, RELATION_START_LABEL: 'Watermark', RELATION_END_KEY: end_key, RELATION_END_LABEL: 'Table', RELATION_TYPE: 'BELONG_TO_TABLE', RELATION_REVERSE_TYPE: 'WATERMARK' } def test_get_watermark_model_key(self) -> None: watermark = self.watermark.get_watermark_model_key() self.assertEqual( watermark, '{database}://{cluster}.{schema}/{table}/{part_type}/'.format( database=DATABASE, cluster=CLUSTER, schema=SCHEMA, table=TABLE, part_type=PART_TYPE)) def test_get_metadata_model_key(self) -> None: metadata = self.watermark.get_metadata_model_key() self.assertEqual( metadata, '{database}://{cluster}.{schema}/{table}'.format(database=DATABASE, cluster=CLUSTER, schema=SCHEMA, table=TABLE)) def test_create_nodes(self) -> None: nodes = self.watermark.create_nodes() self.assertEquals(len(nodes), 1) self.assertEquals(nodes[0], self.expected_node_result) self.assertEqual(neo4_serializer.serialize_node(nodes[0]), self.expected_serialized_node_result) def test_create_relation(self) -> None: relation = self.watermark.create_relation() self.assertEquals(len(relation), 1) self.assertEquals(relation[0], self.expected_relation_result) self.assertEqual(neo4_serializer.serialize_relationship(relation[0]), self.expected_serialized_relation_result) def test_create_next_node(self) -> None: next_node = self.watermark.create_next_node() self.assertEquals(neo4_serializer.serialize_node(next_node), self.expected_serialized_node_result) def test_create_next_relation(self) -> None: next_relation = self.watermark.create_next_relation() self.assertEquals( neo4_serializer.serialize_relationship(next_relation), self.expected_serialized_relation_result)
class TestWatermark(unittest.TestCase): def setUp(self): # type: () -> None super(TestWatermark, self).setUp() self.watermark = Watermark(create_time='2017-09-18T00:00:00', database=DATABASE, schema_name=SCHEMA, table_name=TABLE, cluster=CLUSTER, part_type=PART_TYPE, part_name=NESTED_PART) self.expected_node_result = { NODE_KEY: '{database}://{cluster}.{schema}/{table}/{part_type}/' .format( database=DATABASE.lower(), cluster=CLUSTER.lower(), schema=SCHEMA.lower(), table=TABLE.lower(), part_type=PART_TYPE.lower()), NODE_LABEL: 'Watermark', 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00' } self.expected_relation_result = { RELATION_START_KEY: '{database}://{cluster}.{schema}/{table}/{part_type}/' .format( database=DATABASE.lower(), cluster=CLUSTER.lower(), schema=SCHEMA.lower(), table=TABLE.lower(), part_type=PART_TYPE.lower()), RELATION_START_LABEL: 'Watermark', RELATION_END_KEY: '{database}://{cluster}.{schema}/{table}' .format( database=DATABASE.lower(), cluster=CLUSTER.lower(), schema=SCHEMA.lower(), table=TABLE.lower()), RELATION_END_LABEL: 'Table', RELATION_TYPE: 'BELONG_TO_TABLE', RELATION_REVERSE_TYPE: 'WATERMARK' } def test_get_watermark_model_key(self): # type: () -> None watermark = self.watermark.get_watermark_model_key() self.assertEquals( watermark, '{database}://{cluster}.{schema}/{table}/{part_type}/' .format(database=DATABASE.lower(), cluster=CLUSTER.lower(), schema=SCHEMA.lower(), table=TABLE.lower(), part_type=PART_TYPE.lower())) def test_get_metadata_model_key(self): # type: () -> None metadata = self.watermark.get_metadata_model_key() self.assertEquals(metadata, '{database}://{cluster}.{schema}/{table}' .format(database=DATABASE.lower(), cluster=CLUSTER.lower(), schema=SCHEMA.lower(), table=TABLE.lower())) def test_create_nodes(self): # type: () -> None nodes = self.watermark.create_nodes() self.assertEquals(len(nodes), 1) self.assertEquals(nodes[0], self.expected_node_result) def test_create_relation(self): # type: () -> None relation = self.watermark.create_relation() self.assertEquals(len(relation), 1) self.assertEquals(relation[0], self.expected_relation_result) def test_create_next_node(self): # type: () -> None next_node = self.watermark.create_next_node() self.assertEquals(next_node, self.expected_node_result) def test_create_next_relation(self): # type: () -> None next_relation = self.watermark.create_next_relation() self.assertEquals(next_relation, self.expected_relation_result)
class TestWatermark(unittest.TestCase): def setUp(self) -> None: super(TestWatermark, self).setUp() self.watermark = Watermark( create_time='2017-09-18T00:00:00', database=DATABASE, schema=SCHEMA, table_name=TABLE, cluster=CLUSTER, part_type=PART_TYPE, part_name=NESTED_PART ) self.start_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}/{PART_TYPE}/' self.end_key = f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}' self.expected_node_result = GraphNode( key=self.start_key, label='Watermark', attributes={ 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00' } ) self.expected_serialized_node_results = [{ NODE_KEY: self.start_key, NODE_LABEL: 'Watermark', 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00' }] self.expected_relation_result = GraphRelationship( start_label='Watermark', end_label='Table', start_key=self.start_key, end_key=self.end_key, type='BELONG_TO_TABLE', reverse_type='WATERMARK', attributes={} ) self.expected_serialized_relation_results = [{ RELATION_START_KEY: self.start_key, RELATION_START_LABEL: 'Watermark', RELATION_END_KEY: self.end_key, RELATION_END_LABEL: 'Table', RELATION_TYPE: 'BELONG_TO_TABLE', RELATION_REVERSE_TYPE: 'WATERMARK' }] def test_get_watermark_model_key(self) -> None: watermark = self.watermark.get_watermark_model_key() self.assertEqual(watermark, f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}/{PART_TYPE}/') def test_get_metadata_model_key(self) -> None: metadata = self.watermark.get_metadata_model_key() self.assertEqual(metadata, f'{DATABASE}://{CLUSTER}.{SCHEMA}/{TABLE}') def test_create_nodes(self) -> None: actual = [] node = self.watermark.create_next_node() while node: serialized_node = neo4_serializer.serialize_node(node) actual.append(serialized_node) node = self.watermark.create_next_node() self.assertEqual(actual, self.expected_serialized_node_results) def test_create_nodes_neptune(self) -> None: expected_serialized_node_results = [{ NEPTUNE_HEADER_ID: 'Watermark:' + self.start_key, METADATA_KEY_PROPERTY_NAME: 'Watermark:' + self.start_key, NEPTUNE_HEADER_LABEL: 'Watermark', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'partition_key:String(single)': 'ds', 'partition_value:String(single)': '2017-09-18/feature_id=9', 'create_time:String(single)': '2017-09-18T00:00:00' }] actual = [] node = self.watermark.create_next_node() while node: serialized_node = neptune_serializer.convert_node(node) actual.append(serialized_node) node = self.watermark.create_next_node() self.assertEqual(expected_serialized_node_results, actual) def test_create_relation(self) -> None: actual = [] relation = self.watermark.create_next_relation() while relation: serialized_relation = neo4_serializer.serialize_relationship(relation) actual.append(serialized_relation) relation = self.watermark.create_next_relation() self.assertEqual(actual, self.expected_serialized_relation_results) def test_create_relation_neptune(self) -> None: actual = [] relation = self.watermark.create_next_relation() while relation: serialized_relation = neptune_serializer.convert_relationship(relation) actual.append(serialized_relation) relation = self.watermark.create_next_relation() expected = [ [ { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Watermark:" + self.start_key, to_vertex_id="Table:" + self.end_key, label='BELONG_TO_TABLE' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Watermark:" + self.start_key, to_vertex_id="Table:" + self.end_key, label='BELONG_TO_TABLE' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: "Watermark:" + self.start_key, NEPTUNE_RELATIONSHIP_HEADER_TO: "Table:" + self.end_key, NEPTUNE_HEADER_LABEL: 'BELONG_TO_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.end_key, to_vertex_id="Watermark:" + self.start_key, label='WATERMARK' ), METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id="Table:" + self.end_key, to_vertex_id="Watermark:" + self.start_key, label='WATERMARK' ), NEPTUNE_RELATIONSHIP_HEADER_FROM: "Table:" + self.end_key, NEPTUNE_RELATIONSHIP_HEADER_TO: "Watermark:" + self.start_key, NEPTUNE_HEADER_LABEL: 'WATERMARK', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } ] ] self.assertListEqual(actual, expected) def test_create_records(self) -> None: expected = [{ 'rk': self.start_key, 'partition_key': 'ds', 'partition_value': '2017-09-18/feature_id=9', 'create_time': '2017-09-18T00:00:00', 'table_rk': self.end_key }] actual = [] record = self.watermark.create_next_record() while record: serialized_record = mysql_serializer.serialize_record(record) actual.append(serialized_record) record = self.watermark.create_next_record() self.assertEqual(actual, expected)