コード例 #1
0
class TestTableLastUpdated(unittest.TestCase):
    def setUp(self) -> None:
        super(TestTableLastUpdated, self).setUp()

        self.tableLastUpdated = TableLastUpdated(
            table_name='test_table',
            last_updated_time_epoch=25195665,
            schema='default')

        self.expected_node_result = {
            NODE_KEY: 'hive://gold.default/test_table/timestamp',
            NODE_LABEL: 'Timestamp',
            'last_updated_timestamp:UNQUOTED': 25195665,
            timestamp_constants.TIMESTAMP_PROPERTY + ":UNQUOTED": 25195665,
            'name': 'last_updated_timestamp'
        }

        self.expected_relation_result = {
            RELATION_START_KEY: 'hive://gold.default/test_table',
            RELATION_START_LABEL: 'Table',
            RELATION_END_KEY: 'hive://gold.default/test_table/timestamp',
            RELATION_END_LABEL: 'Timestamp',
            RELATION_TYPE: 'LAST_UPDATED_AT',
            RELATION_REVERSE_TYPE: 'LAST_UPDATED_TIME_OF'
        }

    def test_create_next_node(self) -> None:
        next_node = self.tableLastUpdated.create_next_node()
        next_node_serialized = neo4_serializer.serialize_node(next_node)
        self.assertEqual(next_node_serialized, self.expected_node_result)

    def test_create_next_relation(self) -> None:
        next_relation = self.tableLastUpdated.create_next_relation()
        next_relation_serialized = neo4_serializer.serialize_relationship(
            next_relation)
        self.assertEqual(next_relation_serialized,
                         self.expected_relation_result)

    def test_get_table_model_key(self) -> None:
        table = self.tableLastUpdated.get_table_model_key()
        self.assertEqual(table, 'hive://gold.default/test_table')

    def test_get_last_updated_model_key(self) -> None:
        last_updated = self.tableLastUpdated.get_last_updated_model_key()
        self.assertEqual(last_updated,
                         'hive://gold.default/test_table/timestamp')

    def test_create_nodes(self) -> None:
        nodes = self.tableLastUpdated.create_nodes()
        self.assertEquals(len(nodes), 1)
        serialize_node = neo4_serializer.serialize_node(nodes[0])
        self.assertEquals(serialize_node, self.expected_node_result)

    def test_create_relation(self) -> None:
        relation = self.tableLastUpdated.create_relation()
        self.assertEquals(len(relation), 1)
        serialized_relation = neo4_serializer.serialize_relationship(
            relation[0])
        self.assertEquals(serialized_relation, self.expected_relation_result)
コード例 #2
0
class TestTableLastUpdated(unittest.TestCase):
    def setUp(self):
        # type: () -> None
        super(TestTableLastUpdated, self).setUp()

        self.tableLastUpdated = TableLastUpdated(
            table_name='test_table',
            last_updated_time_epoch=25195665,
            schema='default')

        self.expected_node_result = {
            NODE_KEY: 'hive://gold.default/test_table/timestamp',
            NODE_LABEL: 'Timestamp',
            'last_updated_timestamp': 25195665,
            'name': 'last_updated_timestamp'
        }

        self.expected_relation_result = {
            RELATION_START_KEY: 'hive://gold.default/test_table',
            RELATION_START_LABEL: 'Table',
            RELATION_END_KEY: 'hive://gold.default/test_table/timestamp',
            RELATION_END_LABEL: 'Timestamp',
            RELATION_TYPE: 'LAST_UPDATED_AT',
            RELATION_REVERSE_TYPE: 'LAST_UPDATED_TIME_OF'
        }

    def test_create_next_node(self):
        # type: () -> None
        next_node = self.tableLastUpdated.create_next_node()
        self.assertEquals(next_node, self.expected_node_result)

    def test_create_next_relation(self):
        # type: () -> None
        next_relation = self.tableLastUpdated.create_next_relation()
        self.assertEquals(next_relation, self.expected_relation_result)

    def test_get_table_model_key(self):
        # type: () -> None
        table = self.tableLastUpdated.get_table_model_key()
        self.assertEquals(table, 'hive://gold.default/test_table')

    def test_get_last_updated_model_key(self):
        # type: () -> None
        last_updated = self.tableLastUpdated.get_last_updated_model_key()
        self.assertEquals(last_updated,
                          'hive://gold.default/test_table/timestamp')

    def test_create_nodes(self):
        # type: () -> None
        nodes = self.tableLastUpdated.create_nodes()
        self.assertEquals(len(nodes), 1)
        self.assertEquals(nodes[0], self.expected_node_result)

    def test_create_relation(self):
        # type: () -> None
        relation = self.tableLastUpdated.create_relation()
        self.assertEquals(len(relation), 1)
        self.assertEquals(relation[0], self.expected_relation_result)
コード例 #3
0
class TestTableLastUpdated(unittest.TestCase):

    def setUp(self) -> None:
        super(TestTableLastUpdated, self).setUp()

        self.tableLastUpdated = TableLastUpdated(table_name='test_table',
                                                 last_updated_time_epoch=25195665,
                                                 schema='default')

        self.expected_node_results = [{
            NODE_KEY: 'hive://gold.default/test_table/timestamp',
            NODE_LABEL: 'Timestamp',
            'last_updated_timestamp:UNQUOTED': 25195665,
            timestamp_constants.TIMESTAMP_PROPERTY + ":UNQUOTED": 25195665,
            'name': 'last_updated_timestamp'
        }]

        self.expected_relation_results = [{
            RELATION_START_KEY: 'hive://gold.default/test_table',
            RELATION_START_LABEL: 'Table',
            RELATION_END_KEY: 'hive://gold.default/test_table/timestamp',
            RELATION_END_LABEL: 'Timestamp',
            RELATION_TYPE: 'LAST_UPDATED_AT',
            RELATION_REVERSE_TYPE: 'LAST_UPDATED_TIME_OF'
        }]

    def test_get_table_model_key(self) -> None:
        table = self.tableLastUpdated.get_table_model_key()
        self.assertEqual(table, 'hive://gold.default/test_table')

    def test_get_last_updated_model_key(self) -> None:
        last_updated = self.tableLastUpdated.get_last_updated_model_key()
        self.assertEqual(last_updated, 'hive://gold.default/test_table/timestamp')

    def test_create_nodes(self) -> None:
        actual = []
        node = self.tableLastUpdated.create_next_node()
        while node:
            serialize_node = neo4_serializer.serialize_node(node)
            actual.append(serialize_node)
            node = self.tableLastUpdated.create_next_node()

        self.assertEqual(actual, self.expected_node_results)

    def test_create_nodes_neptune(self) -> None:
        node_id = TableLastUpdated.LAST_UPDATED_NODE_LABEL + ":" + self.tableLastUpdated.get_last_updated_model_key()
        expected_nodes = [{
            NEPTUNE_HEADER_ID: node_id,
            METADATA_KEY_PROPERTY_NAME: node_id,
            NEPTUNE_HEADER_LABEL: TableLastUpdated.LAST_UPDATED_NODE_LABEL,
            NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
            NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB,
            'name:String(single)': 'last_updated_timestamp',
            'last_updated_timestamp:Long(single)': 25195665,
            timestamp_constants.TIMESTAMP_PROPERTY + ":Long(single)": 25195665,
        }]

        actual = []
        next_node = self.tableLastUpdated.create_next_node()
        while next_node:
            next_node_serialized = neptune_serializer.convert_node(next_node)
            actual.append(next_node_serialized)
            next_node = self.tableLastUpdated.create_next_node()

        self.assertEqual(actual, expected_nodes)

    def test_create_relation(self) -> None:
        actual = []
        relation = self.tableLastUpdated.create_next_relation()
        while relation:
            serialized_relation = neo4_serializer.serialize_relationship(relation)
            actual.append(serialized_relation)
            relation = self.tableLastUpdated.create_next_relation()

    def test_create_relation_neptune(self) -> None:
        actual = []
        next_relation = self.tableLastUpdated.create_next_relation()
        while next_relation:
            next_relation_serialized = neptune_serializer.convert_relationship(next_relation)
            actual.append(next_relation_serialized)
            next_relation = self.tableLastUpdated.create_next_relation()

        expected = [
            [
                {
                    NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
                        from_vertex_id='Table:hive://gold.default/test_table',
                        to_vertex_id='Timestamp:hive://gold.default/test_table/timestamp',
                        label='LAST_UPDATED_AT'
                    ),
                    METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
                        from_vertex_id='Table:hive://gold.default/test_table',
                        to_vertex_id='Timestamp:hive://gold.default/test_table/timestamp',
                        label='LAST_UPDATED_AT'
                    ),
                    NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.default/test_table',
                    NEPTUNE_RELATIONSHIP_HEADER_TO: 'Timestamp:hive://gold.default/test_table/timestamp',
                    NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_AT',
                    NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
                    NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
                },
                {
                    NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format(
                        from_vertex_id='Timestamp:hive://gold.default/test_table/timestamp',
                        to_vertex_id='Table:hive://gold.default/test_table',
                        label='LAST_UPDATED_TIME_OF'
                    ),
                    METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format(
                        from_vertex_id='Timestamp:hive://gold.default/test_table/timestamp',
                        to_vertex_id='Table:hive://gold.default/test_table',
                        label='LAST_UPDATED_TIME_OF'
                    ),
                    NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Timestamp:hive://gold.default/test_table/timestamp',
                    NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.default/test_table',
                    NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_TIME_OF',
                    NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY,
                    NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB
                }
            ]
        ]

        self.assertEqual(actual, expected)