def runTest(self):
self.assertTrue(
self.ts.wait_processes(self.node_count, self.node_count),
"Startup timeout")
# add nodes to the group (seed is first node)
seed_addr = self.ts.get_process_addr(self.nodes[0])
for node in self.nodes:
self.ts.send_local_message(node, Message('JOIN', seed_addr))
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=10)
init_message_count = self.ts._message_count
# node disconnects
offline_node = random.choice(self.nodes)
self.ts.disconnect_process(offline_node)
self.nodes.remove(offline_node)
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=30)
init_message_count = self.ts._message_count
# node reconnects
self.ts.connect_process(offline_node)
self.nodes.append(offline_node)
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=10)
# number of used messages
logging.debug("MESSAGE COUNT: %d" %
(self.ts._message_count - init_message_count))
print(offline_node)
def runTest(self):
self.assertTrue(
self.ts.wait_processes(self.node_count, self.node_count),
"Startup timeout")
# add nodes to the group (seed is first node)
seed_addr = self.ts.get_process_addr(self.nodes[0])
for node in self.nodes:
self.ts.send_local_message(node, Message('JOIN', seed_addr))
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=10)
init_message_count = self.ts._message_count
# network partition (odd and even nodes are isolated from each other)
group1 = self.nodes[::2]
group2 = self.nodes[1::2]
self.ts.partition_network(group1, group2)
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=30, group=group1)
self.step_until_stabilized(steps=10, timeout=30, group=group2)
# network partition is healed
self.ts.reset_network()
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=30)
# number of used messages
logging.debug("MESSAGE COUNT: %d" %
(self.ts._message_count - init_message_count))
def runTest(self):
self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
# send message from Eve
self.ts.send_local_message(self.peers[4], Message('SEND', 'Hello'))
for _ in range(4):
self.ts.step(1)
# crash Alice
self.ts.crash_process(self.peers[0])
for _ in range(4):
self.ts.step(1)
# crash Bob
self.ts.crash_process(self.peers[1])
# deliver the message
self.ts.step_until_no_events(1)
# make sure all alive peers delivered the message
for peer in self.peers[2:]:
msg = self.ts.wait_local_message(peer, 0)
self.assertIsNotNone(msg, "Peer not delivered the message")
self.assertEqual(msg.type, 'DELIVER')
self.assertEqual(msg.body, 'Eve: Hello')
def snapshot(self):
dumped_keys = []
for node in self.nodes:
self.ts.send_local_message(node, Message('DUMP_KEYS'))
msg = self.ts.step_until_local_message(node, 2)
self.assertEqual(msg.type, 'DUMP_KEYS_RESP')
dumped_keys.append(msg.body)
return dumped_keys
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
client_req = Message("CALL", "remove field")
self.ts.send_local_message('client', client_req, 1)
self.ts.step_until_no_events(1)
client_resp = self.ts.step_until_local_message('client', 1)
self.assertIsNotNone(client_resp)
self.assertEqual(client_resp.type, "ERROR")
self.assertEqual(client_resp.body, "Key field not found")
client_req = Message("CALL", "put field 2 False")
self.ts.send_local_message('client', client_req, 1)
self.ts.step_until_no_events(1)
client_resp = self.ts.step_until_local_message('client', 1)
self.assertIsNotNone(client_resp)
self.assertEqual(client_resp.type, "RESULT")
self.assertEqual(client_resp.body, True)
client_req = Message("CALL", "get field")
self.ts.send_local_message('client', client_req, 1)
self.ts.step_until_no_events(1)
client_resp = self.ts.step_until_local_message('client', 1)
self.assertIsNotNone(client_resp)
self.assertEqual(client_resp.type, "RESULT")
self.assertEqual(client_resp.body, "2")
client_req = Message("CALL", "remove field")
self.ts.send_local_message('client', client_req, 1)
self.ts.step_until_no_events(1)
client_resp = self.ts.step_until_local_message('client', 1)
self.assertIsNotNone(client_resp)
self.assertEqual(client_resp.type, "RESULT")
self.assertEqual(client_resp.body, "2")
client_req = Message("CALL", "get field")
self.ts.send_local_message('client', client_req, 1)
self.ts.step_until_no_events(1)
client_resp = self.ts.step_until_local_message('client', 1)
self.assertIsNotNone(client_resp)
self.assertEqual(client_resp.type, "ERROR")
self.assertEqual(client_resp.body, "Key field not found")
client_resp = self.ts.step_until_local_message('client', 1)
self.assertIsNone(client_resp)
def lookup_key(self, node, key, timeout=1):
self.ts.send_local_message(node, Message('LOOKUP', key))
msg = self.ts.step_until_local_message(node, timeout)
self.assertIsNotNone(msg, "LOOKUP response is not received")
self.assertEqual(msg.type, 'LOOKUP_RESP')
self.assertIsNotNone(msg.body)
self.assertTrue(isinstance(msg.body, list),
"LOOKUP response is not list")
return msg.body
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
req = Message('PING', 'Hello!')
self.ts.send_local_message('client', req)
resp = self.ts.step_until_local_message('client', 1)
self.assertIsNotNone(resp, "Client response timeout")
self.assertEqual(resp.type, 'PONG')
self.assertEqual(resp.body, req.body)
def runTest(self):
self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
# send two messages from Carl
self.ts.send_local_message(self.peers[2], Message('SEND', 'Hello'))
self.ts.send_local_message(self.peers[2], Message('SEND', 'How are you?'))
# deliver the message
self.ts.set_message_delay(0, 1)
self.ts.step_until_no_events(1)
# make sure all peers received the first message from Carl first
for peer in self.peers:
msg = self.ts.wait_local_message(peer, 0)
self.assertIsNotNone(msg, "Peer " + peer + " not received the first message")
self.assertEqual(msg.type, 'DELIVER')
self.assertEqual(msg.body, 'Carl: Hello')
def step_until_stabilized(self,
steps=10,
timeout=10,
group=None,
expect_keys=None):
if group is None:
group = self.nodes
synced_nodes = set()
counts = [0] * len(group)
start = time.time()
while time.time() - start < timeout:
step_status = self.ts.steps(steps, 1)
for node in group:
if node not in synced_nodes:
self.ts.send_local_message(node, Message('GET_MEMBERS'))
msg = self.ts.step_until_local_message(node, 1)
self.assertIsNotNone(msg, "Members list is not returned")
self.assertEqual(msg.type, 'MEMBERS')
if len(msg.body) == len(group) and set(
msg.body) == set(group):
synced_nodes.add(node)
if expect_keys is not None:
counts_changed = False
for i, node in enumerate(group):
self.ts.send_local_message(node, Message('COUNT_RECORDS'))
msg = self.ts.step_until_local_message(node, 1)
self.assertEqual(msg.type, 'COUNT_RECORDS_RESP')
count = int(msg.body)
if count != counts[i]:
counts_changed = True
counts[i] = count
if len(synced_nodes) == len(group) and (expect_keys is None or
(sum(counts) == expect_keys
and not counts_changed)):
break
if not step_status:
break
self.assertEqual(synced_nodes, set(group),
"Members lists are not stabilized")
if expect_keys is not None:
self.assertEqual(sum(counts), expect_keys,
"Keys are not stabilized")
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
# Drop with probability 1
self.ts.set_message_drop_rate(1)
sender_req = Message('INFO-1', 'message1')
self.ts.send_local_message('sender', sender_req, 1)
# Wait for receiver to respond
sender_resp = self.ts.step_until_local_message('receiver', 1)
# Nothing to respond
self.assertIsNone(sender_resp)
def runTest(self):
self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout")
self.keys_count = 100
self.init_cluster()
leave_node = random.choice(self.nodes)
self.ts.send_local_message(leave_node, Message('COUNT_RECORDS'))
msg = self.ts.step_until_local_message(leave_node, 1)
self.assertIsNotNone(msg, "COUNT_RECORDS is not responced")
self.assertEqual(msg.type, 'COUNT_RECORDS_RESP')
self.assertNotEqual(msg.body, 0, "Node stores no records, bad distribution")
self.ts.send_local_message(leave_node, Message('LEAVE'))
self.nodes.remove(leave_node)
self.step_until_stabilized()
self.check_distribution()
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
req = Message('PING', 'Hello!')
self.ts.send_local_message('client', req)
self.ts.set_event_reordering(True)
self.ts.steps(100, 1)
self.ts.set_event_reordering(False)
resp = self.ts.step_until_local_message('client', 1)
self.assertIsNotNone(resp, "Client response timeout")
self.assertEqual(resp.type, 'PONG')
self.assertEqual(resp.body, req.body)
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
# Reliable network, should get the result.
sender_req = Message('INFO-1', 'message1')
self.ts.send_local_message('sender', sender_req, 1)
sender_resp = self.ts.step_until_local_message('receiver', 1)
self.assertIsNotNone(sender_resp, "Receiver response timeout")
self.assertEqual(sender_resp.type, 'INFO-1')
self.assertEqual(sender_resp.body, 'message1')
# Should be no other messages
sender_resp = self.ts.step_until_local_message('receiver', 1)
self.assertIsNone(sender_resp)
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
# With probability 1 repeat 5 times
self.ts.set_repeat_rate(1, 5)
sender_req = Message('INFO-1', 'message1')
self.ts.send_local_message('sender', sender_req, 1)
sender_resp = self.ts.step_until_local_message('receiver', 1)
self.assertIsNotNone(sender_resp, "Receiver response timeout")
self.assertEqual(sender_resp.type, 'INFO-1')
self.assertEqual(sender_resp.body, 'message1')
# Should be no other messages
sender_resp = self.ts.step_until_local_message('receiver', 1)
self.assertIsNone(sender_resp)
def runTest(self):
self.ts.set_real_time_mode(False)
self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout")
self.keys_count = 100
group = self.nodes[:5]
self.init_cluster(group=group)
seed_addr = self.ts.get_process_addr(random.choice(self.nodes[:5]))
for node in self.nodes[5:]:
self.ts.send_local_message(node, Message('JOIN', seed_addr))
group.append(node)
self.step_until_stabilized(group=group)
self.check_distribution()
for node in self.nodes[5:]:
self.ts.send_local_message(node, Message('LEAVE'))
group.remove(node)
self.step_until_stabilized(group=group)
self.nodes = self.nodes[:5]
self.check_distribution()
def runTest(self):
self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
# send message from Eve
self.ts.send_local_message(self.peers[4], Message('SEND', 'Hello'))
for _ in range(random.randint(0,10)):
self.ts.step(1)
# crash Alice
self.ts.crash_process(self.peers[0])
for _ in range(random.randint(0,10)):
self.ts.step(1)
# crash Bob
self.ts.crash_process(self.peers[1])
for _ in range(random.randint(0,10)):
self.ts.step(1)
# crash Carl
self.ts.crash_process(self.peers[2])
# deliver the message
self.ts.step_until_no_events(1)
# check agreement property
crashed = ['Alice', 'Bob', 'Carl']
correct = ['Dan', 'Eve']
crashed_delivered = 0
correct_delivered = 0
for peer in self.peers:
msg = self.ts.wait_local_message(peer, 0)
if msg is not None:
if peer in crashed:
crashed_delivered += 1
else:
correct_delivered += 1
passed = False
if crashed_delivered == 0 and correct_delivered == 0:
passed = True
if crashed_delivered == 0 and correct_delivered == len(correct):
passed = True
if crashed_delivered > 0 and correct_delivered == len(correct):
passed = True
self.assertTrue(passed,
"Agreement property is not satisfied: correct - " + str(correct_delivered) + "/2, crashed - " + str(crashed_delivered) + "/3")
def runTest(self):
self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout")
seed_addr = self.ts.get_process_addr(self.nodes[0])
for i in range(self.node_count):
self.ts.send_local_message(self.nodes[i], Message('JOIN', seed_addr))
self.step_until_stabilized(expect_keys=0)
self.keys_count = self.node_count
self.keys = list(''.join(random.choices(string.ascii_lowercase, k=8)) for i in range(self.keys_count))
self.values = {
self.keys[i] : ''.join(random.choices(string.ascii_lowercase, k=8)) for i in range(self.keys_count)
}
random.shuffle(self.keys)
for i in range(self.node_count):
self.ts.send_local_message(self.nodes[i], Message('PUT', f"{str(self.keys[i])}={self.values[self.keys[i]]}"))
msg = self.ts.step_until_local_message(self.nodes[i], 2)
self.assertIsNotNone(msg, "PUT response is not received")
self.assertEqual(msg.type, 'PUT_RESP')
tests = []
for i in range(0, self.node_count):
for k, v in self.values.items():
tests.append([i, k, v])
random.shuffle(tests)
for nodeid, key, value in tests:
self.ts.send_local_message(self.nodes[nodeid], Message('GET', key))
msg = self.ts.step_until_local_message(self.nodes[nodeid], 1)
self.assertIsNotNone(msg, "GET response is not received")
self.assertEqual(msg.type, 'GET_RESP')
self.assertEqual(msg.body, value)
self.check_distribution()
def runTest(self):
self.assertTrue(
self.ts.wait_processes(self.node_count, self.node_count),
"Startup timeout")
# add nodes to the group (seed is fisrt node)
seed_addr = self.ts.get_process_addr(self.nodes[0])
for node in self.nodes:
self.ts.send_local_message(node, Message('JOIN', seed_addr))
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=10)
# number of used messages
logging.debug("MESSAGE COUNT: %d" % self.ts._message_count)
def runTest(self):
self.assertTrue(
self.ts.wait_processes(self.node_count, self.node_count),
"Startup timeout")
# add nodes to the group (seed is first node)
seed_addr = self.ts.get_process_addr(self.nodes[0])
for node in self.nodes:
self.ts.send_local_message(node, Message('JOIN', seed_addr))
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=10)
# node crashes
crashed_node = random.choice(self.nodes)
self.ts.crash_process(crashed_node)
self.nodes.remove(crashed_node)
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=30)
init_message_count = self.ts._message_count
# crashed node recovers
self.restart_node(crashed_node)
time.sleep(1)
seed_addr = seed_addr = self.ts.get_process_addr(
random.choice(tuple(self.nodes)))
self.ts.send_local_message(crashed_node, Message('JOIN', seed_addr))
self.nodes.append(crashed_node)
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=10)
# number of used messages
logging.debug("MESSAGE COUNT: %d" %
(self.ts._message_count - init_message_count))
def send_get(self, node, key, quorum=2, timeout=1):
self.ts.send_local_message(
node, Message('GET', {
'key': key,
'quorum': quorum
}))
msg = self.ts.step_until_local_message(node, timeout)
self.assertIsNotNone(msg, "GET response is not received")
self.assertEqual(msg.type, 'GET_RESP')
self.assertIsNotNone(msg.body)
self.assertTrue(isinstance(msg.body, dict), "GET response is not dict")
self.assertTrue('values' in msg.body)
self.assertTrue('metadata' in msg.body)
self.assertEqual(len(msg.body['values']), len(msg.body['metadata']))
return (msg.body['values'], msg.body['metadata'])
def runTest(self):
self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
# send message from Alice
self.ts.send_local_message(self.peers[0], Message('SEND', 'hello'))
# deliver the message
self.ts.step_until_no_events(1)
# make sure all peers delivered the message
for peer in self.peers[1:]:
msg = self.ts.wait_local_message(peer, 0)
self.assertIsNotNone(msg, "Peer not delivered the message")
self.assertEqual(msg.type, 'DELIVER')
self.assertEqual(msg.body, 'Alice: hello')
def send_put(self,
node,
key,
value,
metadata=None,
quorum=2,
wait=True,
timeout=1):
body = {'key': key, 'value': value, 'quorum': quorum}
if metadata is not None:
body['metadata'] = metadata
self.ts.send_local_message(node, Message('PUT', body))
if wait:
msg = self.ts.step_until_local_message(node, timeout)
self.assertIsNotNone(msg, "PUT response is not received")
self.assertEqual(msg.type, 'PUT_RESP')
self.assertIsNotNone(msg.body)
return msg.body
def runTest(self):
self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
# send message from Bob
self.ts.send_local_message(self.peers[1], Message('SEND', 'Hello'))
# if Bob has not delivered the message yet, make a step
# (so he will receive his own message and possibly deliver it)
delivered = 0
if self.ts.wait_local_message(self.peers[1], 0) is None:
self.ts.step(1)
if self.ts.wait_local_message(self.peers[1], 0) is not None:
delivered = 1
else:
delivered = 1
# crash Bob
self.ts.crash_process(self.peers[1])
# complete execution
self.ts.step_until_no_events(3)
# check agreement property
crashed = ['Bob']
correct = ['Alice', 'Carl', 'Dan', 'Eve']
crashed_delivered = delivered
correct_delivered = 0
for peer in self.peers:
if peer not in crashed:
msg = self.ts.wait_local_message(peer, 0)
if msg is not None:
correct_delivered += 1
passed = False
if crashed_delivered == 0 and correct_delivered == 0:
passed = True
if crashed_delivered == 0 and correct_delivered == len(correct):
passed = True
if crashed_delivered > 0 and correct_delivered == len(correct):
passed = True
self.assertTrue(passed,
"Agreement property is not satisfied: correct - " + str(correct_delivered) + "/4, crashed - " + str(crashed_delivered) + "/1")
def runTest(self):
self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
# send message from Bob
self.ts.send_local_message(self.peers[1], Message('SEND', 'Hello'))
# deliver locally
self.ts.step(1)
# crash Bob
self.ts.crash_process(self.peers[1])
# complete execution
self.ts.step_until_no_events(3)
# make sure no peer delivered the message
for peer in self.peers:
msg = self.ts.wait_local_message(peer, 0)
self.assertIsNone(msg, "Peer delivered some message")
def runTest(self):
self.assertTrue(
self.ts.wait_processes(self.node_count, self.node_count),
"Startup timeout")
# add nodes to the group (seed is random)
members = set()
for node in self.nodes:
if len(members) == 0:
seed_addr = self.ts.get_process_addr(self.nodes[0])
else:
seed_addr = self.ts.get_process_addr(
random.choice(tuple(members)))
self.ts.send_local_message(node, Message('JOIN', seed_addr))
members.add(node)
# step until all nodes know each other
self.step_until_stabilized(steps=10, timeout=10)
# number of used messages
logging.debug("MESSAGE COUNT: %d" % self.ts._message_count)
def runTest(self):
self.ts.set_real_time_mode(False)
self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout")
self.keys_count = 1000
self.init_cluster()
s1 = self.snapshot()
node = random.choice(self.nodes)
s1.pop(self.nodes.index(node))
self.ts.send_local_message(node, Message('LEAVE'))
self.nodes.remove(node)
self.step_until_stabilized()
s2 = self.snapshot()
target = self.keys_count / self.node_count
moved_keys_count = self.keys_count - sum(map(lambda a, b: len(set(a) & set(b)), s1, s2))
deviation = ((moved_keys_count - target) / target) * 100
self.check_distribution()
logging.info(f" - moved keys: {moved_keys_count} | target: {target:.2f} | deviation: {deviation:.2f}%")
self.assertTrue(deviation <= 20, "Deviation from target is more than 20%")
def step_until_stabilized(self, steps=10, timeout=10, group=None):
if group is None:
group = self.nodes
synced_nodes = set()
start = time.time()
while time.time() - start < timeout and len(synced_nodes) < len(group):
step_status = self.ts.steps(steps, 1)
for node in group:
if node not in synced_nodes:
self.ts.send_local_message(node, Message('GET_MEMBERS'))
msg = self.ts.step_until_local_message(node, 1)
self.assertIsNotNone(msg, "Members list is not returned")
self.assertEqual(msg.type, 'MEMBERS')
if len(msg.body) == len(group) and set(
msg.body) == set(group):
synced_nodes.add(node)
if not step_status:
break
# print(len(synced_nodes))
self.assertEqual(synced_nodes, set(group),
"Members lists are not stabilized")
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
num_messages = 3
rate = 3
self.ts.set_repeat_rate(1, rate)
self.ts.set_event_reordering(True)
for i in range(num_messages):
sender_req = Message('INFO-2', str(i))
self.ts.send_local_message('sender', sender_req, 1)
messages = []
while True:
sender_resp = self.ts.step_until_local_message('receiver', 1)
if sender_resp is None:
break
messages.append(sender_resp)
self.assertGreaterEqual(len(messages), num_messages)
all_present = [0 for i in range(num_messages)]
for i in range(len(messages)):
self.assertEqual(messages[i].type, 'INFO-2')
all_present[int(messages[i].body)] += 1
for el in all_present:
self.assertGreaterEqual(el, rate + 1)
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
num_messages = 5
self.ts.set_repeat_rate(1, 5)
self.ts.set_event_reordering(True)
# Send the same messages
for i in range(2 * num_messages):
sender_req = Message('INFO-1', str(i % num_messages))
self.ts.send_local_message('sender', sender_req, 1)
messages = []
while True:
sender_resp = self.ts.step_until_local_message('receiver', 1)
if sender_resp is None:
break
messages.append(sender_resp)
# We should get all them
self.assertEqual(len(messages), num_messages)
all_present = [0 for i in range(num_messages)]
# Only once
for i in range(num_messages):
self.assertEqual(messages[i].type, 'INFO-1')
all_present[int(messages[i].body)] += 1
self.assertListEqual(all_present, [1 for i in range(num_messages)])
def runTest(self):
self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout")
self.ts.set_real_time_mode(False)
client_req = Message('INFO-3', 'some message')
# Send 5 identical messages
self.ts.send_local_message('sender', client_req)
self.ts.send_local_message('sender', client_req)
self.ts.send_local_message('sender', client_req)
self.ts.send_local_message('sender', client_req)
self.ts.send_local_message('sender', client_req)
# Drop them and force the retry
self.ts.set_message_drop_rate(0.5)
# Try to get something from 100 steps, very high probability of success
self.ts.steps(100, 1)
self.ts.set_message_drop_rate(0)
sender_resp = self.ts.step_until_local_message('receiver', 1)
self.assertIsNotNone(sender_resp, "Receiver response timeout")
self.assertEqual(sender_resp.type, 'INFO-3')
self.assertEqual(sender_resp.body, 'some message')
# Should be no other messages
sender_resp = self.ts.step_until_local_message('receiver', 1)
self.assertIsNone(sender_resp)
