class FailureTests(BaseTestCase):
def setUp(self):
super(FailureTests, self).setUp()
self.primary_node = self.input["node"]
self.util = Util(self.primary_node)
def tearDown(self):
super(FailureTests, self).tearDown()
def test_reads_on_stopped_node(self):
"""
Negative test to verify that reads fail on a stopped node
"""
bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
nodes_to_add=nodes_to_add)
key = "key"
value = "1"
bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
key=key, value=value)
self.util.stop_node("8081")
try:
_ = self.util.get_value(self.primary_node + ":8081", key=key)
except Exception as e:
self.log.info("Read on stopped node failed as expected: {0}".format(e))
else:
self.fail("Read on stopped node did not fail")
def test_reads_on_revived_node(self):
"""
Start a 3 node cluster. Stop the second node and immediately
revive the node.
Issue get and write from the revived node and verify
"""
bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
nodes_to_add=nodes_to_add)
key = "key"
value = "1"
bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
key=key, value=value)
self.util.stop_node("8081")
self.util.start_node("8081")
content = self.util.get_value(self.primary_node + ":8081", key=key)
self.assertEqual(str(content), value)
second_key = "key2"
second_value = "2"
bool_val, content, response = self.util.add_key_value(self.primary_node + ":8081",
key=second_key, value=second_value)
content = self.util.get_value(self.primary_node + ":8081", key=second_key)
self.assertEqual(str(content), second_value)
def test_quorum_reads_cruds_before_after_reviving_random_node(self):
"""
Start a 3 node cluster. Do:
Add key/value pairs before stopping random node
Add some more keys and update some values after stopping node
Verify history(quorum reads) after reviving node, from revived node
"""
bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
nodes_to_add=nodes_to_add)
for i in range(50):
key = "key" + str(i)
value = str(i)
bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
key=key, value=value)
self.util.stop_node("8081")
for i in range(50):
key = "key" + str(i)
value = str(i + 1)
bool_val, content, response = self.util.update_value(self.primary_node + ":8080",
key=key, value=value)
for i in range(50, 100):
key = "key" + str(i)
value = str(i + 1)
bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
key=key, value=value)
self.util.start_node("8081")
for i in range(100):
key = "key" + str(i)
expected_value = str(i + 1)
content = self.util.get_value(self.primary_node + ":8081", key=key, consistency_level="quorum")
self.assertEqual(str(content), expected_value)
def test_quorum_reads_cruds_before_after_reviving_leader_node(self):
"""
Start a 3 node cluster. Do:
Add key/value pairs before stopping leader node
Add some more keys and update some values after stopping leader node
Verify history(quorum reads) after reviving node, from revived node
"""
bool_val, content, response = self.util.provision_node(self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(self.primary_node + ":8080",
nodes_to_add=nodes_to_add)
for i in range(50):
key = "key" + str(i)
value = str(i)
bool_val, content, response = self.util.add_key_value(self.primary_node + ":8080",
key=key, value=value)
self.util.stop_node("8080")
time.sleep(20) # ToDo: Why does it not work without sleep? Time for Replicated log sync maybe?
for i in range(50):
key = "key" + str(i)
value = str(i + 1)
bool_val, content, response = self.util.update_value(self.primary_node + ":8081",
key=key, value=value)
for i in range(50, 100):
key = "key" + str(i)
value = str(i + 1)
bool_val, content, response = self.util.add_key_value(self.primary_node + ":8081",
key=key, value=value)
self.util.start_node("8080")
for i in range(100):
key = "key" + str(i)
expected_value = str(i + 1)
content = self.util.get_value(self.primary_node + ":8080", key=key, consistency_level="quorum")
self.assertEqual(str(content), expected_value)
class SimpleTests(BaseTestCase):
def setUp(self):
super(SimpleTests, self).setUp()
self.primary_node = self.input["node"]
self.util = Util(self.primary_node)
def tearDown(self):
super(SimpleTests, self).tearDown()
def test_single_node_cluster(self):
"""
test provision a single node cluster
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
self.log.info("{0} {1} {2}".format(bool_val, content, response))
assert bool_val
def test_multi_node_cluster(self):
"""
Test adding a 5 node cluster
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]", ' \
'"[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
self.log.info("{0} {1} {2}".format(bool_val, content, response))
assert bool_val
def test_remove_node(self):
"""
Test remove single node
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]", ' \
'"[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
nodes_to_remove = '"[email protected]"'
bool_val, content, response = self.util.remove_node(
self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
self.log.info("{0} {1} {2}".format(bool_val, content, response))
assert bool_val
def test_remove_nodes(self):
"""
Test remove multiple nodes
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]", ' \
'"[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
nodes_to_remove = '["[email protected]", ' \
'"[email protected]", "[email protected]"]'
bool_val, content, response = self.util.remove_node(
self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
self.log.info("{0} {1} {2}".format(bool_val, content, response))
assert bool_val
def test_add_key_value_pair(self):
"""
Create a single node cluster. Add a key,value pair. Add a node.
Verify the new node returns the value for the key that was added
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
key = "key"
value = "1"
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8080", key=key, value=value)
self.log.info("{0} {1} {2}".format(bool_val, content, response))
nodes_to_add = '["[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
content = self.util.get_value(self.primary_node + ":8081", key=key)
self.log.info(content)
self.assertEqual(str(content), value)
def test_update_value_single_node(self):
"""
Create a single node. Add a key-value pair. Update the value for the key.
Verify the value returned is the updated-value
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
key = "key"
value = "1"
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8080", key=key, value=value)
updated_value = "3"
bool_val, content, response = self.util.update_value(
self.primary_node + ":8080", key=key, value=updated_value)
content = self.util.get_value(self.primary_node + ":8080", key=key)
self.log.info(content)
self.assertEqual(str(content), updated_value)
def test_update_value_two_node(self):
"""
Create a single node. Add a key-value pair. Add a new node. Update the value
for the key. Verify that the new node returns the updated-value
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
key = "key"
value = "1"
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8080", key=key, value=value)
nodes_to_add = '["[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
updated_value = "3"
bool_val, content, response = self.util.update_value(
self.primary_node + ":8080", key=key, value=updated_value)
self.log.info("{0} {1} {2}".format(bool_val, content, response))
content = self.util.get_value(self.primary_node + ":8081", key=key)
self.log.info(content)
self.assertEqual(str(content), updated_value)
def test_delete_key_one_node(self):
"""
Create a single node. Add a key-value pair. Delete the key.
Verify reading the key throws exception
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
key = "key"
value = "1"
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8080", key=key, value=value)
bool_val, content, response = self.util.delete_key(self.primary_node +
":8080",
key=key)
try:
content = self.util.get_value(self.primary_node + ":8081", key=key)
except Exception as e:
self.log.info(str(e) + "as expected")
else:
self.fail("Get value of a deleted key worked")
def test_delete_key_multiple_nodes(self):
"""
Create a 5 node cluster
Delete a key on one node and verify it is deleted on all other nodes
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]", ' \
'"[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
key = "key"
value = "1"
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8084", key=key, value=value)
bool_val, content, response = self.util.delete_key(self.primary_node +
":8084",
key=key)
for port in [":8080", ":8081", ":8082", ":8083"]:
try:
content = self.util.get_value(self.primary_node + port,
key=key,
consistency_level="quorum")
except Exception as e:
self.log.info(str(e) + " as expected")
else:
self.fail(
"Get value of a deleted key worked on node {0}".format(
self.primary_node + port))
def test_local_reads_after_removing_leader(self):
"""
Create a 3 node cluster. Add a series of key value pairs.
Remove the primary node. Verify local reads against those keys.
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]", ' \
'"[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
base_key = "key"
base_val = ""
values_added = list()
for i in range(5):
key = base_key + str(i)
value = base_val + str(i)
key = str(key)
value = str(value)
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8080", key=key, value=value)
values_added.append(value)
nodes_to_remove = '"[email protected]"'
bool_val, content, response = self.util.remove_node(
self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
values = list()
for i in range(5):
key = base_key + str(i)
content = self.util.get_value(self.primary_node + ":8081", key=key)
values.append(str(content))
self.log.info(values)
self.assertEqual(values_added, values)
def test_quorum_reads_after_removing_leader(self):
"""
Create a 3 node cluster. Add a series of key value pairs.
Remove the primary node. Verify quorum reads against those keys.
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
nodes_to_add = '["[email protected]", "[email protected]", ' \
'"[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
base_key = "key"
base_val = ""
values_added = list()
for i in range(5):
key = base_key + str(i)
value = base_val + str(i)
key = str(key)
value = str(value)
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8080", key=key, value=value)
values_added.append(value)
nodes_to_remove = '"[email protected]"'
bool_val, content, response = self.util.remove_node(
self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
values = list()
for i in range(5):
key = base_key + str(i)
content = self.util.get_value(self.primary_node + ":8081",
key=key,
consistency_level="quorum")
values.append(str(content))
self.log.info(values)
self.assertEqual(values_added, values)
class Volume(BaseTestCase):
def setUp(self):
super(Volume, self).setUp()
self.primary_node = self.input["node"]
self.util = Util(self.primary_node)
self.data_load_flag = False
self.number_of_cycles = 300 # ToDO: scale it up later
self.number_of_kv_pairs = 100000 # ToDO: scale it up later
def tearDown(self):
super(Volume, self).tearDown()
def assert_rebalance(self, bool):
self.assertTrue(bool, "Rebalance failed")
def wipe_removed_nodes(self, removed_nodes):
"""
Wipes the removed nodes
:param removed_nodes: eg:'["[email protected]", "[email protected]"]'
"""
removed_nodes_list = removed_nodes.strip('][').split(', ')
for removed_node in removed_nodes_list:
number = removed_node[11]
port = 8080 + int(number)
bool_val, content, response = self.util.wipe_node(
self.primary_node + ":" + str(port))
self.assertTrue(bool_val, "could not wipe node")
def infinite_data_load(self):
"""
Add million key value pairs
Update million key value pairs
Read million key value pairs
Delete million key value pairs
repeat
"""
self.log.info("Starting infinite data load")
while self.data_load_flag:
self.log.info("Adding keys")
for i in range(1, self.number_of_kv_pairs):
key = "key" + str(i)
value = str(i)
bool_val, content, response = self.util.add_key_value(
self.primary_node + ":8080", key=key, value=value)
self.log.info("Updating keys")
for i in range(1, self.number_of_kv_pairs):
key = "key" + str(i)
value = str(i + 1)
bool_val, content, response = self.util.update_value(
self.primary_node + ":8080", key=key, value=value)
self.log.info("Reading keys")
for i in range(1, self.number_of_kv_pairs):
key = "key" + str(i)
content = self.util.get_value(self.primary_node + ":8080",
key=key)
expected_value = str(i + 1)
self.assertEqual(str(content), expected_value)
self.log.info("Removing keys")
for i in range(1, self.number_of_kv_pairs):
key = "key" + str(i)
bool_val, content, response = self.util.delete_key(
self.primary_node + ":8080", key=key)
def test_volume(self):
"""
Step 0: create a 1 node cluster
Step 0.1: Start infinite data load
Repeat for 300 cycles below steps:
Step 1: Add one node
Step 2: Remove one node
Step 3: Add multiple nodes
Step 4: Remove multiple nodes (all except init nodes)
"""
bool_val, content, response = self.util.provision_node(
self.primary_node + ":8080")
self.data_load_flag = True
data_load_thread = threading.Thread(target=self.infinite_data_load)
data_load_thread.start()
for cycle in range(self.number_of_cycles):
self.log.info("starting cycle: {0}".format(cycle))
print(
"#############################################################################"
)
self.log.info("Step 1: Adding a single node")
nodes_to_add = '"[email protected]"'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
self.assert_rebalance(bool_val)
bool_val, content, response = self.util.get_config(
self.primary_node + ":8080")
self.log.info("config of the cluster: {0} {1} {2}".format(
bool_val, content, response))
print(
"#############################################################################"
)
self.log.info("Step 2: Removing a single node")
nodes_to_remove = '["[email protected]"]'
bool_val, content, response = self.util.remove_node(
self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
self.assert_rebalance(bool_val)
bool_val, content, response = self.util.get_config(
self.primary_node + ":8080")
self.log.info("config of the cluster: {0} {1} {2}".format(
bool_val, content, response))
self.wipe_removed_nodes(nodes_to_remove)
self.log.info("config of the cluster: {0} {1} {2}".format(
bool_val, content, response))
print(
"#############################################################################"
)
self.log.info("Step 3: Adding multiple nodes")
nodes_to_add = '["[email protected]", "[email protected]", "[email protected]", "[email protected]"]'
bool_val, content, response = self.util.add_node(
self.primary_node + ":8080", nodes_to_add=nodes_to_add)
self.assert_rebalance(bool_val)
bool_val, content, response = self.util.get_config(
self.primary_node + ":8080")
self.log.info("config of the cluster: {0} {1} {2}".format(
bool_val, content, response))
print(
"#############################################################################"
)
self.log.info("Step 4: Removing multiple nodes")
nodes_to_remove = '["[email protected]", "[email protected]", "[email protected]", "[email protected]"]'
bool_val, content, response = self.util.remove_node(
self.primary_node + ":8080", nodes_to_remove=nodes_to_remove)
self.assert_rebalance(bool_val)
bool_val, content, response = self.util.get_config(
self.primary_node + ":8080")
self.log.info("config of the cluster: {0} {1} {2}".format(
bool_val, content, response))
self.wipe_removed_nodes(nodes_to_remove)
self.data_load_flag = False
data_load_thread.join()