class FailureTests(BaseTestCase):
    def setUp(self):
        super(FailureTests, self).setUp()
        self.primary_node = self.input["node"]
        self.util = Util(self.primary_node)

    def tearDown(self):
        super(FailureTests, self).tearDown()

    def test_reads_on_stopped_node(self):
        """
        Negative test to verify that reads fail on a stopped node
        """
        bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
                                                         nodes_to_add=nodes_to_add)
        key = "key"
        value = "1"
        bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
                                                              key=key, value=value)
        self.util.stop_node("8081")
        try:
            _ = self.util.get_value(self.primary_node + ":8081", key=key)
        except Exception as e:
            self.log.info("Read on stopped node failed as expected: {0}".format(e))
        else:
            self.fail("Read on stopped node did not fail")

    def test_reads_on_revived_node(self):
        """
        Start a 3 node cluster. Stop the second node and immediately
        revive the node.
        Issue get and write from the revived node and verify
        """
        bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
                                                         nodes_to_add=nodes_to_add)
        key = "key"
        value = "1"
        bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
                                                              key=key, value=value)
        self.util.stop_node("8081")
        self.util.start_node("8081")

        content = self.util.get_value(self.primary_node + ":8081", key=key)
        self.assertEqual(str(content), value)

        second_key = "key2"
        second_value = "2"
        bool_val, content, response = self.util.add_key_value(self.primary_node + ":8081",
                                                              key=second_key, value=second_value)
        content = self.util.get_value(self.primary_node + ":8081", key=second_key)
        self.assertEqual(str(content), second_value)

    def test_quorum_reads_cruds_before_after_reviving_random_node(self):
        """
        Start a 3 node cluster. Do:
        Add key/value pairs before stopping random node
        Add some more keys and update some values after stopping node
        Verify history(quorum reads) after reviving node, from revived node
        """
        bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
                                                         nodes_to_add=nodes_to_add)
        for i in range(50):
            key = "key" + str(i)
            value = str(i)
            bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
                                                                  key=key, value=value)
        self.util.stop_node("8081")
        for i in range(50):
            key = "key" + str(i)
            value = str(i + 1)
            bool_val, content, response = self.util.update_value(self.primary_node + ":8080",
                                                                 key=key, value=value)
        for i in range(50, 100):
            key = "key" + str(i)
            value = str(i + 1)
            bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
                                                                  key=key, value=value)
        self.util.start_node("8081")
        for i in range(100):
            key = "key" + str(i)
            expected_value = str(i + 1)
            content = self.util.get_value(self.primary_node + ":8081", key=key, consistency_level="quorum")
            self.assertEqual(str(content), expected_value)

    def test_quorum_reads_cruds_before_after_reviving_leader_node(self):
        """
        Start a 3 node cluster. Do:
        Add key/value pairs before stopping leader node
        Add some more keys and update some values after stopping leader node
        Verify history(quorum reads) after reviving node, from revived node
        """
        bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
                                                         nodes_to_add=nodes_to_add)
        for i in range(50):
            key = "key" + str(i)
            value = str(i)
            bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
                                                                  key=key, value=value)
        self.util.stop_node("8080")
        time.sleep(20)  # ToDo: Why does it not work without sleep? Time for Replicated log sync maybe?
        for i in range(50):
            key = "key" + str(i)
            value = str(i + 1)
            bool_val, content, response = self.util.update_value(self.primary_node + ":8081",
                                                                 key=key, value=value)
        for i in range(50, 100):
            key = "key" + str(i)
            value = str(i + 1)
            bool_val, content, response = self.util.add_key_value(self.primary_node + ":8081",
                                                                  key=key, value=value)
        self.util.start_node("8080")
        for i in range(100):
            key = "key" + str(i)
            expected_value = str(i + 1)
            content = self.util.get_value(self.primary_node + ":8080", key=key, consistency_level="quorum")
            self.assertEqual(str(content), expected_value)
Esempio n. 2
0
class SimpleTests(BaseTestCase):
    def setUp(self):
        super(SimpleTests, self).setUp()
        self.primary_node = self.input["node"]
        self.util = Util(self.primary_node)

    def tearDown(self):
        super(SimpleTests, self).tearDown()

    def test_single_node_cluster(self):
        """
        test provision a single node cluster
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        self.log.info("{0} {1} {2}".format(bool_val, content, response))
        assert bool_val

    def test_multi_node_cluster(self):
        """
        Test adding a 5 node cluster
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]", ' \
                       '"[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)
        self.log.info("{0} {1} {2}".format(bool_val, content, response))
        assert bool_val

    def test_remove_node(self):
        """
        Test remove single node
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]", ' \
                       '"[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)

        nodes_to_remove = '"[email protected]"'
        bool_val, content, response = self.util.remove_node(
            self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
        self.log.info("{0} {1} {2}".format(bool_val, content, response))
        assert bool_val

    def test_remove_nodes(self):
        """
        Test remove multiple nodes
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]", ' \
                       '"[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)

        nodes_to_remove = '["[email protected]", ' \
                          '"[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.remove_node(
            self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
        self.log.info("{0} {1} {2}".format(bool_val, content, response))
        assert bool_val

    def test_add_key_value_pair(self):
        """
        Create a single node cluster. Add a key,value pair. Add a node.
        Verify the new node returns the value for the key that was added
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        key = "key"
        value = "1"
        bool_val, content, response = self.util.add_key_value(
            self.primary_node + ":8080", key=key, value=value)
        self.log.info("{0} {1} {2}".format(bool_val, content, response))
        nodes_to_add = '["[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)
        content = self.util.get_value(self.primary_node + ":8081", key=key)
        self.log.info(content)
        self.assertEqual(str(content), value)

    def test_update_value_single_node(self):
        """
        Create a single node. Add a key-value pair. Update the value for the key.
        Verify the value returned is the updated-value
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        key = "key"
        value = "1"
        bool_val, content, response = self.util.add_key_value(
            self.primary_node + ":8080", key=key, value=value)
        updated_value = "3"
        bool_val, content, response = self.util.update_value(
            self.primary_node + ":8080", key=key, value=updated_value)

        content = self.util.get_value(self.primary_node + ":8080", key=key)
        self.log.info(content)
        self.assertEqual(str(content), updated_value)

    def test_update_value_two_node(self):
        """
        Create a single node. Add a key-value pair. Add a new node. Update the value
        for the key. Verify that the new node returns the updated-value
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        key = "key"
        value = "1"
        bool_val, content, response = self.util.add_key_value(
            self.primary_node + ":8080", key=key, value=value)
        nodes_to_add = '["[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)
        updated_value = "3"
        bool_val, content, response = self.util.update_value(
            self.primary_node + ":8080", key=key, value=updated_value)
        self.log.info("{0} {1} {2}".format(bool_val, content, response))

        content = self.util.get_value(self.primary_node + ":8081", key=key)
        self.log.info(content)
        self.assertEqual(str(content), updated_value)

    def test_delete_key_one_node(self):
        """
        Create a single node. Add a key-value pair. Delete the key.
        Verify reading the key throws exception
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        key = "key"
        value = "1"
        bool_val, content, response = self.util.add_key_value(
            self.primary_node + ":8080", key=key, value=value)
        bool_val, content, response = self.util.delete_key(self.primary_node +
                                                           ":8080",
                                                           key=key)
        try:
            content = self.util.get_value(self.primary_node + ":8081", key=key)
        except Exception as e:
            self.log.info(str(e) + "as expected")
        else:
            self.fail("Get value of a deleted key worked")

    def test_delete_key_multiple_nodes(self):
        """
        Create a 5 node cluster
        Delete a key on one node and verify it is deleted on all other nodes
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")

        nodes_to_add = '["[email protected]", "[email protected]", ' \
                       '"[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)
        key = "key"
        value = "1"
        bool_val, content, response = self.util.add_key_value(
            self.primary_node + ":8084", key=key, value=value)
        bool_val, content, response = self.util.delete_key(self.primary_node +
                                                           ":8084",
                                                           key=key)
        for port in [":8080", ":8081", ":8082", ":8083"]:
            try:
                content = self.util.get_value(self.primary_node + port,
                                              key=key,
                                              consistency_level="quorum")
            except Exception as e:
                self.log.info(str(e) + " as expected")
            else:
                self.fail(
                    "Get value of a deleted key worked on node {0}".format(
                        self.primary_node + port))

    def test_local_reads_after_removing_leader(self):
        """
        Create a 3 node cluster. Add a series of key value pairs.
        Remove the primary node. Verify local reads against those keys.
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        nodes_to_add = '["[email protected]", "[email protected]", ' \
                       '"[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)
        base_key = "key"
        base_val = ""
        values_added = list()
        for i in range(5):
            key = base_key + str(i)
            value = base_val + str(i)
            key = str(key)
            value = str(value)
            bool_val, content, response = self.util.add_key_value(
                self.primary_node + ":8080", key=key, value=value)
            values_added.append(value)

        nodes_to_remove = '"[email protected]"'
        bool_val, content, response = self.util.remove_node(
            self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
        values = list()
        for i in range(5):
            key = base_key + str(i)
            content = self.util.get_value(self.primary_node + ":8081", key=key)
            values.append(str(content))
        self.log.info(values)
        self.assertEqual(values_added, values)

    def test_quorum_reads_after_removing_leader(self):
        """
        Create a 3 node cluster. Add a series of key value pairs.
        Remove the primary node. Verify quorum reads against those keys.
        """
        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        nodes_to_add = '["[email protected]", "[email protected]", ' \
                       '"[email protected]", "[email protected]"]'
        bool_val, content, response = self.util.add_node(
            self.primary_node + ":8080", nodes_to_add=nodes_to_add)
        base_key = "key"
        base_val = ""
        values_added = list()
        for i in range(5):
            key = base_key + str(i)
            value = base_val + str(i)
            key = str(key)
            value = str(value)
            bool_val, content, response = self.util.add_key_value(
                self.primary_node + ":8080", key=key, value=value)
            values_added.append(value)

        nodes_to_remove = '"[email protected]"'
        bool_val, content, response = self.util.remove_node(
            self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
        values = list()
        for i in range(5):
            key = base_key + str(i)
            content = self.util.get_value(self.primary_node + ":8081",
                                          key=key,
                                          consistency_level="quorum")
            values.append(str(content))
        self.log.info(values)
        self.assertEqual(values_added, values)
Esempio n. 3
0
class Volume(BaseTestCase):
    def setUp(self):
        super(Volume, self).setUp()
        self.primary_node = self.input["node"]
        self.util = Util(self.primary_node)
        self.data_load_flag = False
        self.number_of_cycles = 300  # ToDO: scale it up later
        self.number_of_kv_pairs = 100000  # ToDO: scale it up later

    def tearDown(self):
        super(Volume, self).tearDown()

    def assert_rebalance(self, bool):
        self.assertTrue(bool, "Rebalance failed")

    def wipe_removed_nodes(self, removed_nodes):
        """
        Wipes the removed nodes
        :param removed_nodes: eg:'["[email protected]", "[email protected]"]'
        """
        removed_nodes_list = removed_nodes.strip('][').split(', ')
        for removed_node in removed_nodes_list:
            number = removed_node[11]
            port = 8080 + int(number)
            bool_val, content, response = self.util.wipe_node(
                self.primary_node + ":" + str(port))
            self.assertTrue(bool_val, "could not wipe node")

    def infinite_data_load(self):
        """
        Add million key value pairs
        Update million key value pairs
        Read million key value pairs
        Delete million key value pairs
        repeat
        """
        self.log.info("Starting infinite data load")
        while self.data_load_flag:
            self.log.info("Adding keys")
            for i in range(1, self.number_of_kv_pairs):
                key = "key" + str(i)
                value = str(i)
                bool_val, content, response = self.util.add_key_value(
                    self.primary_node + ":8080", key=key, value=value)
            self.log.info("Updating keys")
            for i in range(1, self.number_of_kv_pairs):
                key = "key" + str(i)
                value = str(i + 1)
                bool_val, content, response = self.util.update_value(
                    self.primary_node + ":8080", key=key, value=value)
            self.log.info("Reading keys")
            for i in range(1, self.number_of_kv_pairs):
                key = "key" + str(i)
                content = self.util.get_value(self.primary_node + ":8080",
                                              key=key)
                expected_value = str(i + 1)
                self.assertEqual(str(content), expected_value)

            self.log.info("Removing keys")
            for i in range(1, self.number_of_kv_pairs):
                key = "key" + str(i)
                bool_val, content, response = self.util.delete_key(
                    self.primary_node + ":8080", key=key)

    def test_volume(self):
        """
        Step 0: create a 1 node cluster
        Step 0.1: Start infinite data load
        Repeat for 300 cycles below steps:
        Step 1: Add one node
        Step 2: Remove one node
        Step 3: Add multiple nodes
        Step 4: Remove multiple nodes (all except init nodes)
        """

        bool_val, content, response = self.util.provision_node(
            self.primary_node + ":8080")
        self.data_load_flag = True
        data_load_thread = threading.Thread(target=self.infinite_data_load)
        data_load_thread.start()

        for cycle in range(self.number_of_cycles):
            self.log.info("starting cycle: {0}".format(cycle))

            print(
                "#############################################################################"
            )
            self.log.info("Step 1: Adding a single node")
            nodes_to_add = '"[email protected]"'
            bool_val, content, response = self.util.add_node(
                self.primary_node + ":8080", nodes_to_add=nodes_to_add)
            self.assert_rebalance(bool_val)
            bool_val, content, response = self.util.get_config(
                self.primary_node + ":8080")
            self.log.info("config of the cluster: {0} {1} {2}".format(
                bool_val, content, response))

            print(
                "#############################################################################"
            )
            self.log.info("Step 2: Removing a single node")
            nodes_to_remove = '["[email protected]"]'
            bool_val, content, response = self.util.remove_node(
                self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
            self.assert_rebalance(bool_val)
            bool_val, content, response = self.util.get_config(
                self.primary_node + ":8080")
            self.log.info("config of the cluster: {0} {1} {2}".format(
                bool_val, content, response))
            self.wipe_removed_nodes(nodes_to_remove)
            self.log.info("config of the cluster: {0} {1} {2}".format(
                bool_val, content, response))

            print(
                "#############################################################################"
            )
            self.log.info("Step 3: Adding multiple nodes")
            nodes_to_add = '["[email protected]", "[email protected]", "[email protected]", "[email protected]"]'
            bool_val, content, response = self.util.add_node(
                self.primary_node + ":8080", nodes_to_add=nodes_to_add)
            self.assert_rebalance(bool_val)
            bool_val, content, response = self.util.get_config(
                self.primary_node + ":8080")
            self.log.info("config of the cluster: {0} {1} {2}".format(
                bool_val, content, response))

            print(
                "#############################################################################"
            )
            self.log.info("Step 4: Removing multiple nodes")
            nodes_to_remove = '["[email protected]", "[email protected]", "[email protected]", "[email protected]"]'
            bool_val, content, response = self.util.remove_node(
                self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
            self.assert_rebalance(bool_val)
            bool_val, content, response = self.util.get_config(
                self.primary_node + ":8080")
            self.log.info("config of the cluster: {0} {1} {2}".format(
                bool_val, content, response))
            self.wipe_removed_nodes(nodes_to_remove)

        self.data_load_flag = False
        data_load_thread.join()