def runTest(self): self.assertTrue( self.ts.wait_processes(self.node_count, self.node_count), "Startup timeout") # add nodes to the group (seed is first node) seed_addr = self.ts.get_process_addr(self.nodes[0]) for node in self.nodes: self.ts.send_local_message(node, Message('JOIN', seed_addr)) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=10) init_message_count = self.ts._message_count # node disconnects offline_node = random.choice(self.nodes) self.ts.disconnect_process(offline_node) self.nodes.remove(offline_node) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=30) init_message_count = self.ts._message_count # node reconnects self.ts.connect_process(offline_node) self.nodes.append(offline_node) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=10) # number of used messages logging.debug("MESSAGE COUNT: %d" % (self.ts._message_count - init_message_count)) print(offline_node)
def runTest(self): self.assertTrue( self.ts.wait_processes(self.node_count, self.node_count), "Startup timeout") # add nodes to the group (seed is first node) seed_addr = self.ts.get_process_addr(self.nodes[0]) for node in self.nodes: self.ts.send_local_message(node, Message('JOIN', seed_addr)) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=10) init_message_count = self.ts._message_count # network partition (odd and even nodes are isolated from each other) group1 = self.nodes[::2] group2 = self.nodes[1::2] self.ts.partition_network(group1, group2) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=30, group=group1) self.step_until_stabilized(steps=10, timeout=30, group=group2) # network partition is healed self.ts.reset_network() # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=30) # number of used messages logging.debug("MESSAGE COUNT: %d" % (self.ts._message_count - init_message_count))
def runTest(self): self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout") self.ts.set_real_time_mode(False) # send message from Eve self.ts.send_local_message(self.peers[4], Message('SEND', 'Hello')) for _ in range(4): self.ts.step(1) # crash Alice self.ts.crash_process(self.peers[0]) for _ in range(4): self.ts.step(1) # crash Bob self.ts.crash_process(self.peers[1]) # deliver the message self.ts.step_until_no_events(1) # make sure all alive peers delivered the message for peer in self.peers[2:]: msg = self.ts.wait_local_message(peer, 0) self.assertIsNotNone(msg, "Peer not delivered the message") self.assertEqual(msg.type, 'DELIVER') self.assertEqual(msg.body, 'Eve: Hello')
def snapshot(self): dumped_keys = [] for node in self.nodes: self.ts.send_local_message(node, Message('DUMP_KEYS')) msg = self.ts.step_until_local_message(node, 2) self.assertEqual(msg.type, 'DUMP_KEYS_RESP') dumped_keys.append(msg.body) return dumped_keys
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") client_req = Message("CALL", "remove field") self.ts.send_local_message('client', client_req, 1) self.ts.step_until_no_events(1) client_resp = self.ts.step_until_local_message('client', 1) self.assertIsNotNone(client_resp) self.assertEqual(client_resp.type, "ERROR") self.assertEqual(client_resp.body, "Key field not found") client_req = Message("CALL", "put field 2 False") self.ts.send_local_message('client', client_req, 1) self.ts.step_until_no_events(1) client_resp = self.ts.step_until_local_message('client', 1) self.assertIsNotNone(client_resp) self.assertEqual(client_resp.type, "RESULT") self.assertEqual(client_resp.body, True) client_req = Message("CALL", "get field") self.ts.send_local_message('client', client_req, 1) self.ts.step_until_no_events(1) client_resp = self.ts.step_until_local_message('client', 1) self.assertIsNotNone(client_resp) self.assertEqual(client_resp.type, "RESULT") self.assertEqual(client_resp.body, "2") client_req = Message("CALL", "remove field") self.ts.send_local_message('client', client_req, 1) self.ts.step_until_no_events(1) client_resp = self.ts.step_until_local_message('client', 1) self.assertIsNotNone(client_resp) self.assertEqual(client_resp.type, "RESULT") self.assertEqual(client_resp.body, "2") client_req = Message("CALL", "get field") self.ts.send_local_message('client', client_req, 1) self.ts.step_until_no_events(1) client_resp = self.ts.step_until_local_message('client', 1) self.assertIsNotNone(client_resp) self.assertEqual(client_resp.type, "ERROR") self.assertEqual(client_resp.body, "Key field not found") client_resp = self.ts.step_until_local_message('client', 1) self.assertIsNone(client_resp)
def lookup_key(self, node, key, timeout=1): self.ts.send_local_message(node, Message('LOOKUP', key)) msg = self.ts.step_until_local_message(node, timeout) self.assertIsNotNone(msg, "LOOKUP response is not received") self.assertEqual(msg.type, 'LOOKUP_RESP') self.assertIsNotNone(msg.body) self.assertTrue(isinstance(msg.body, list), "LOOKUP response is not list") return msg.body
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") req = Message('PING', 'Hello!') self.ts.send_local_message('client', req) resp = self.ts.step_until_local_message('client', 1) self.assertIsNotNone(resp, "Client response timeout") self.assertEqual(resp.type, 'PONG') self.assertEqual(resp.body, req.body)
def runTest(self): self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout") self.ts.set_real_time_mode(False) # send two messages from Carl self.ts.send_local_message(self.peers[2], Message('SEND', 'Hello')) self.ts.send_local_message(self.peers[2], Message('SEND', 'How are you?')) # deliver the message self.ts.set_message_delay(0, 1) self.ts.step_until_no_events(1) # make sure all peers received the first message from Carl first for peer in self.peers: msg = self.ts.wait_local_message(peer, 0) self.assertIsNotNone(msg, "Peer " + peer + " not received the first message") self.assertEqual(msg.type, 'DELIVER') self.assertEqual(msg.body, 'Carl: Hello')
def step_until_stabilized(self, steps=10, timeout=10, group=None, expect_keys=None): if group is None: group = self.nodes synced_nodes = set() counts = [0] * len(group) start = time.time() while time.time() - start < timeout: step_status = self.ts.steps(steps, 1) for node in group: if node not in synced_nodes: self.ts.send_local_message(node, Message('GET_MEMBERS')) msg = self.ts.step_until_local_message(node, 1) self.assertIsNotNone(msg, "Members list is not returned") self.assertEqual(msg.type, 'MEMBERS') if len(msg.body) == len(group) and set( msg.body) == set(group): synced_nodes.add(node) if expect_keys is not None: counts_changed = False for i, node in enumerate(group): self.ts.send_local_message(node, Message('COUNT_RECORDS')) msg = self.ts.step_until_local_message(node, 1) self.assertEqual(msg.type, 'COUNT_RECORDS_RESP') count = int(msg.body) if count != counts[i]: counts_changed = True counts[i] = count if len(synced_nodes) == len(group) and (expect_keys is None or (sum(counts) == expect_keys and not counts_changed)): break if not step_status: break self.assertEqual(synced_nodes, set(group), "Members lists are not stabilized") if expect_keys is not None: self.assertEqual(sum(counts), expect_keys, "Keys are not stabilized")
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") # Drop with probability 1 self.ts.set_message_drop_rate(1) sender_req = Message('INFO-1', 'message1') self.ts.send_local_message('sender', sender_req, 1) # Wait for receiver to respond sender_resp = self.ts.step_until_local_message('receiver', 1) # Nothing to respond self.assertIsNone(sender_resp)
def runTest(self): self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout") self.keys_count = 100 self.init_cluster() leave_node = random.choice(self.nodes) self.ts.send_local_message(leave_node, Message('COUNT_RECORDS')) msg = self.ts.step_until_local_message(leave_node, 1) self.assertIsNotNone(msg, "COUNT_RECORDS is not responced") self.assertEqual(msg.type, 'COUNT_RECORDS_RESP') self.assertNotEqual(msg.body, 0, "Node stores no records, bad distribution") self.ts.send_local_message(leave_node, Message('LEAVE')) self.nodes.remove(leave_node) self.step_until_stabilized() self.check_distribution()
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") self.ts.set_real_time_mode(False) req = Message('PING', 'Hello!') self.ts.send_local_message('client', req) self.ts.set_event_reordering(True) self.ts.steps(100, 1) self.ts.set_event_reordering(False) resp = self.ts.step_until_local_message('client', 1) self.assertIsNotNone(resp, "Client response timeout") self.assertEqual(resp.type, 'PONG') self.assertEqual(resp.body, req.body)
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") # Reliable network, should get the result. sender_req = Message('INFO-1', 'message1') self.ts.send_local_message('sender', sender_req, 1) sender_resp = self.ts.step_until_local_message('receiver', 1) self.assertIsNotNone(sender_resp, "Receiver response timeout") self.assertEqual(sender_resp.type, 'INFO-1') self.assertEqual(sender_resp.body, 'message1') # Should be no other messages sender_resp = self.ts.step_until_local_message('receiver', 1) self.assertIsNone(sender_resp)
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") # With probability 1 repeat 5 times self.ts.set_repeat_rate(1, 5) sender_req = Message('INFO-1', 'message1') self.ts.send_local_message('sender', sender_req, 1) sender_resp = self.ts.step_until_local_message('receiver', 1) self.assertIsNotNone(sender_resp, "Receiver response timeout") self.assertEqual(sender_resp.type, 'INFO-1') self.assertEqual(sender_resp.body, 'message1') # Should be no other messages sender_resp = self.ts.step_until_local_message('receiver', 1) self.assertIsNone(sender_resp)
def runTest(self): self.ts.set_real_time_mode(False) self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout") self.keys_count = 100 group = self.nodes[:5] self.init_cluster(group=group) seed_addr = self.ts.get_process_addr(random.choice(self.nodes[:5])) for node in self.nodes[5:]: self.ts.send_local_message(node, Message('JOIN', seed_addr)) group.append(node) self.step_until_stabilized(group=group) self.check_distribution() for node in self.nodes[5:]: self.ts.send_local_message(node, Message('LEAVE')) group.remove(node) self.step_until_stabilized(group=group) self.nodes = self.nodes[:5] self.check_distribution()
def runTest(self): self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout") self.ts.set_real_time_mode(False) # send message from Eve self.ts.send_local_message(self.peers[4], Message('SEND', 'Hello')) for _ in range(random.randint(0,10)): self.ts.step(1) # crash Alice self.ts.crash_process(self.peers[0]) for _ in range(random.randint(0,10)): self.ts.step(1) # crash Bob self.ts.crash_process(self.peers[1]) for _ in range(random.randint(0,10)): self.ts.step(1) # crash Carl self.ts.crash_process(self.peers[2]) # deliver the message self.ts.step_until_no_events(1) # check agreement property crashed = ['Alice', 'Bob', 'Carl'] correct = ['Dan', 'Eve'] crashed_delivered = 0 correct_delivered = 0 for peer in self.peers: msg = self.ts.wait_local_message(peer, 0) if msg is not None: if peer in crashed: crashed_delivered += 1 else: correct_delivered += 1 passed = False if crashed_delivered == 0 and correct_delivered == 0: passed = True if crashed_delivered == 0 and correct_delivered == len(correct): passed = True if crashed_delivered > 0 and correct_delivered == len(correct): passed = True self.assertTrue(passed, "Agreement property is not satisfied: correct - " + str(correct_delivered) + "/2, crashed - " + str(crashed_delivered) + "/3")
def runTest(self): self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout") seed_addr = self.ts.get_process_addr(self.nodes[0]) for i in range(self.node_count): self.ts.send_local_message(self.nodes[i], Message('JOIN', seed_addr)) self.step_until_stabilized(expect_keys=0) self.keys_count = self.node_count self.keys = list(''.join(random.choices(string.ascii_lowercase, k=8)) for i in range(self.keys_count)) self.values = { self.keys[i] : ''.join(random.choices(string.ascii_lowercase, k=8)) for i in range(self.keys_count) } random.shuffle(self.keys) for i in range(self.node_count): self.ts.send_local_message(self.nodes[i], Message('PUT', f"{str(self.keys[i])}={self.values[self.keys[i]]}")) msg = self.ts.step_until_local_message(self.nodes[i], 2) self.assertIsNotNone(msg, "PUT response is not received") self.assertEqual(msg.type, 'PUT_RESP') tests = [] for i in range(0, self.node_count): for k, v in self.values.items(): tests.append([i, k, v]) random.shuffle(tests) for nodeid, key, value in tests: self.ts.send_local_message(self.nodes[nodeid], Message('GET', key)) msg = self.ts.step_until_local_message(self.nodes[nodeid], 1) self.assertIsNotNone(msg, "GET response is not received") self.assertEqual(msg.type, 'GET_RESP') self.assertEqual(msg.body, value) self.check_distribution()
def runTest(self): self.assertTrue( self.ts.wait_processes(self.node_count, self.node_count), "Startup timeout") # add nodes to the group (seed is fisrt node) seed_addr = self.ts.get_process_addr(self.nodes[0]) for node in self.nodes: self.ts.send_local_message(node, Message('JOIN', seed_addr)) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=10) # number of used messages logging.debug("MESSAGE COUNT: %d" % self.ts._message_count)
def runTest(self): self.assertTrue( self.ts.wait_processes(self.node_count, self.node_count), "Startup timeout") # add nodes to the group (seed is first node) seed_addr = self.ts.get_process_addr(self.nodes[0]) for node in self.nodes: self.ts.send_local_message(node, Message('JOIN', seed_addr)) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=10) # node crashes crashed_node = random.choice(self.nodes) self.ts.crash_process(crashed_node) self.nodes.remove(crashed_node) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=30) init_message_count = self.ts._message_count # crashed node recovers self.restart_node(crashed_node) time.sleep(1) seed_addr = seed_addr = self.ts.get_process_addr( random.choice(tuple(self.nodes))) self.ts.send_local_message(crashed_node, Message('JOIN', seed_addr)) self.nodes.append(crashed_node) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=10) # number of used messages logging.debug("MESSAGE COUNT: %d" % (self.ts._message_count - init_message_count))
def send_get(self, node, key, quorum=2, timeout=1): self.ts.send_local_message( node, Message('GET', { 'key': key, 'quorum': quorum })) msg = self.ts.step_until_local_message(node, timeout) self.assertIsNotNone(msg, "GET response is not received") self.assertEqual(msg.type, 'GET_RESP') self.assertIsNotNone(msg.body) self.assertTrue(isinstance(msg.body, dict), "GET response is not dict") self.assertTrue('values' in msg.body) self.assertTrue('metadata' in msg.body) self.assertEqual(len(msg.body['values']), len(msg.body['metadata'])) return (msg.body['values'], msg.body['metadata'])
def runTest(self): self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout") self.ts.set_real_time_mode(False) # send message from Alice self.ts.send_local_message(self.peers[0], Message('SEND', 'hello')) # deliver the message self.ts.step_until_no_events(1) # make sure all peers delivered the message for peer in self.peers[1:]: msg = self.ts.wait_local_message(peer, 0) self.assertIsNotNone(msg, "Peer not delivered the message") self.assertEqual(msg.type, 'DELIVER') self.assertEqual(msg.body, 'Alice: hello')
def send_put(self, node, key, value, metadata=None, quorum=2, wait=True, timeout=1): body = {'key': key, 'value': value, 'quorum': quorum} if metadata is not None: body['metadata'] = metadata self.ts.send_local_message(node, Message('PUT', body)) if wait: msg = self.ts.step_until_local_message(node, timeout) self.assertIsNotNone(msg, "PUT response is not received") self.assertEqual(msg.type, 'PUT_RESP') self.assertIsNotNone(msg.body) return msg.body
def runTest(self): self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout") self.ts.set_real_time_mode(False) # send message from Bob self.ts.send_local_message(self.peers[1], Message('SEND', 'Hello')) # if Bob has not delivered the message yet, make a step # (so he will receive his own message and possibly deliver it) delivered = 0 if self.ts.wait_local_message(self.peers[1], 0) is None: self.ts.step(1) if self.ts.wait_local_message(self.peers[1], 0) is not None: delivered = 1 else: delivered = 1 # crash Bob self.ts.crash_process(self.peers[1]) # complete execution self.ts.step_until_no_events(3) # check agreement property crashed = ['Bob'] correct = ['Alice', 'Carl', 'Dan', 'Eve'] crashed_delivered = delivered correct_delivered = 0 for peer in self.peers: if peer not in crashed: msg = self.ts.wait_local_message(peer, 0) if msg is not None: correct_delivered += 1 passed = False if crashed_delivered == 0 and correct_delivered == 0: passed = True if crashed_delivered == 0 and correct_delivered == len(correct): passed = True if crashed_delivered > 0 and correct_delivered == len(correct): passed = True self.assertTrue(passed, "Agreement property is not satisfied: correct - " + str(correct_delivered) + "/4, crashed - " + str(crashed_delivered) + "/1")
def runTest(self): self.assertTrue(self.ts.wait_processes(5, 1), "Startup timeout") self.ts.set_real_time_mode(False) # send message from Bob self.ts.send_local_message(self.peers[1], Message('SEND', 'Hello')) # deliver locally self.ts.step(1) # crash Bob self.ts.crash_process(self.peers[1]) # complete execution self.ts.step_until_no_events(3) # make sure no peer delivered the message for peer in self.peers: msg = self.ts.wait_local_message(peer, 0) self.assertIsNone(msg, "Peer delivered some message")
def runTest(self): self.assertTrue( self.ts.wait_processes(self.node_count, self.node_count), "Startup timeout") # add nodes to the group (seed is random) members = set() for node in self.nodes: if len(members) == 0: seed_addr = self.ts.get_process_addr(self.nodes[0]) else: seed_addr = self.ts.get_process_addr( random.choice(tuple(members))) self.ts.send_local_message(node, Message('JOIN', seed_addr)) members.add(node) # step until all nodes know each other self.step_until_stabilized(steps=10, timeout=10) # number of used messages logging.debug("MESSAGE COUNT: %d" % self.ts._message_count)
def runTest(self): self.ts.set_real_time_mode(False) self.assertTrue(self.ts.wait_processes(self.node_count, 5), "Startup timeout") self.keys_count = 1000 self.init_cluster() s1 = self.snapshot() node = random.choice(self.nodes) s1.pop(self.nodes.index(node)) self.ts.send_local_message(node, Message('LEAVE')) self.nodes.remove(node) self.step_until_stabilized() s2 = self.snapshot() target = self.keys_count / self.node_count moved_keys_count = self.keys_count - sum(map(lambda a, b: len(set(a) & set(b)), s1, s2)) deviation = ((moved_keys_count - target) / target) * 100 self.check_distribution() logging.info(f" - moved keys: {moved_keys_count} | target: {target:.2f} | deviation: {deviation:.2f}%") self.assertTrue(deviation <= 20, "Deviation from target is more than 20%")
def step_until_stabilized(self, steps=10, timeout=10, group=None): if group is None: group = self.nodes synced_nodes = set() start = time.time() while time.time() - start < timeout and len(synced_nodes) < len(group): step_status = self.ts.steps(steps, 1) for node in group: if node not in synced_nodes: self.ts.send_local_message(node, Message('GET_MEMBERS')) msg = self.ts.step_until_local_message(node, 1) self.assertIsNotNone(msg, "Members list is not returned") self.assertEqual(msg.type, 'MEMBERS') if len(msg.body) == len(group) and set( msg.body) == set(group): synced_nodes.add(node) if not step_status: break # print(len(synced_nodes)) self.assertEqual(synced_nodes, set(group), "Members lists are not stabilized")
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") num_messages = 3 rate = 3 self.ts.set_repeat_rate(1, rate) self.ts.set_event_reordering(True) for i in range(num_messages): sender_req = Message('INFO-2', str(i)) self.ts.send_local_message('sender', sender_req, 1) messages = [] while True: sender_resp = self.ts.step_until_local_message('receiver', 1) if sender_resp is None: break messages.append(sender_resp) self.assertGreaterEqual(len(messages), num_messages) all_present = [0 for i in range(num_messages)] for i in range(len(messages)): self.assertEqual(messages[i].type, 'INFO-2') all_present[int(messages[i].body)] += 1 for el in all_present: self.assertGreaterEqual(el, rate + 1)
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") num_messages = 5 self.ts.set_repeat_rate(1, 5) self.ts.set_event_reordering(True) # Send the same messages for i in range(2 * num_messages): sender_req = Message('INFO-1', str(i % num_messages)) self.ts.send_local_message('sender', sender_req, 1) messages = [] while True: sender_resp = self.ts.step_until_local_message('receiver', 1) if sender_resp is None: break messages.append(sender_resp) # We should get all them self.assertEqual(len(messages), num_messages) all_present = [0 for i in range(num_messages)] # Only once for i in range(num_messages): self.assertEqual(messages[i].type, 'INFO-1') all_present[int(messages[i].body)] += 1 self.assertListEqual(all_present, [1 for i in range(num_messages)])
def runTest(self): self.assertTrue(self.ts.wait_processes(2, 1), "Startup timeout") self.ts.set_real_time_mode(False) client_req = Message('INFO-3', 'some message') # Send 5 identical messages self.ts.send_local_message('sender', client_req) self.ts.send_local_message('sender', client_req) self.ts.send_local_message('sender', client_req) self.ts.send_local_message('sender', client_req) self.ts.send_local_message('sender', client_req) # Drop them and force the retry self.ts.set_message_drop_rate(0.5) # Try to get something from 100 steps, very high probability of success self.ts.steps(100, 1) self.ts.set_message_drop_rate(0) sender_resp = self.ts.step_until_local_message('receiver', 1) self.assertIsNotNone(sender_resp, "Receiver response timeout") self.assertEqual(sender_resp.type, 'INFO-3') self.assertEqual(sender_resp.body, 'some message') # Should be no other messages sender_resp = self.ts.step_until_local_message('receiver', 1) self.assertIsNone(sender_resp)