async def startup(self, another_node: int): trace_log("SuccRouter.startup addr:", self.addr, "Running Startup!") if (another_node == None): self.first_node = True self.predecessor = None self.successor = self.addr else: self.first_node = False # Find our successor request_successor = self.message_sender.create_message( MessageType.FIND_NODE_RESPONSIBLE_FOR_ADDR, another_node, {'addr': self.addr}) node_who_used_to_be_responsible_reply = await self.send_message_and_await( request_successor, lambda msg: msg.has_type( MessageType.FIND_NODE_RESPONSIBLE_FOR_ADDR_REPLY) and msg. src == another_node) assert ('addr' in node_who_used_to_be_responsible_reply.args and 'node' in node_who_used_to_be_responsible_reply.args) node_who_used_to_be_responsible = node_who_used_to_be_responsible_reply.args[ 'node'] self.successor = node_who_used_to_be_responsible trace_log("SuccRouter.startup addr:", self.addr, "Found Successor! ", self.successor) await self.post_startup()
async def find_node_responsible_for_key(self, key: str): hashed_key = hash_key(key) node_responsible = await self.find_node_responsible_for_addr(hashed_key ) trace_log('SuccRouter: Found the node responsible for key ', key, ' : ', node_responsible) return node_responsible
def __init__(self, addr: int, async_scheduler: AsyncScheduler, message_sender: MessageSender, key_transferer: KeyTransferer): trace_log("Starting SuccRouter!") self.addr = addr self.async_scheduler = async_scheduler self.message_sender = message_sender self.key_transferer = key_transferer key_transferer.register_router(self) self.predecessor = None # initialized in startup self.successor = None # defined in startup self.async_scheduler.register_handler( Handler(lambda msg: msg.has_type(MessageType.FIND_SUCCESSOR), self.handle_find_successor)) self.async_scheduler.register_handler( Handler(lambda msg: msg.has_type(MessageType.FIND_PREDECESSOR), self.handle_find_predecessor)) self.async_scheduler.register_handler( Handler( lambda msg: msg.has_type(MessageType. FIND_NODE_RESPONSIBLE_FOR_ADDR), self.handle_find_node_responsible_for_addr)) self.async_scheduler.register_handler( Handler( lambda msg: msg.has_type(MessageType. I_MIGHT_BE_YOUR_PREDECESSOR), self.handle_i_might_be_your_predecessor))
async def copy_key_values_to(self, to_node: int, interval: Interval, dict_to_transfer: Dict[str, str]): """ Copy key_values in the `interval` from persistent_storage to `to_node` """ keys_to_transfer = dict_to_transfer.keys() msg_to_transfer = self.message_sender.create_message( MessageType.TRANSFER_KEYS, to_node, { 'interval': interval.to_string(), 'data_dict': dict_to_transfer, 'copy_node': self.chord_addr, 'receiving_node': to_node }) recieved_msg = await self.send_message_and_await_response( msg_to_transfer, MessageType.TRANSFER_KEYS_CONFIRMED) keys_transfered_successfully, interval_transfered_successfully = recieved_msg.get_args( ['keys_transfered', 'interval']) assert (set(keys_transfered_successfully) == set( dict_to_transfer.keys())) trace_log(colorama.Fore.MAGENTA + "KeyTransferer", self.chord_addr, ": Copy Successfully completed from", self.chord_addr, "to", to_node, "of interval", interval.to_string()) if debug_log_key_transfers: debug_log("KeyTransferer", self.chord_addr, ": Copy Successfully completed from", self.chord_addr, "to", to_node, "of interval", interval.to_string())
def __init__(self, addr: int, another_node: int, persistent_storage: PersistantKeyValueStore, async_scheduler: AsyncScheduler, message_sender: MessageSender, key_transferer: KeyTransferer): self.alive = True trace_log('ChordNode.__init__: Starting Node addr: ', addr, ' with relay-node ', another_node) self.chord_addr = addr self.router = FingerTableRouter(addr, async_scheduler, message_sender, key_transferer, persistent_storage) self.persistent_storage = persistent_storage self.async_scheduler = async_scheduler self.message_sender = message_sender self.another_node = another_node self.key_transferer = key_transferer # Register the handlers self.handlers = [ async_scheduler.register_handler(Handler(lambda msg: msg.message_type == MessageType.GET.value, self.try_twice_get)) ,async_scheduler.register_handler(Handler(lambda msg: msg.message_type == MessageType.SET.value, self.try_twice_set)) ,async_scheduler.register_handler(Handler(lambda msg: msg.message_type == MessageType.GET_VALUE.value, self.handle_get_value)) ,async_scheduler.register_handler(Handler(lambda msg: msg.message_type == MessageType.SET_VALUE.value, self.handle_set_value))] trace_log('ChordNode.__init__: Done Starting Node addr: ', addr, ' with relay-node ', another_node)
async def handle_get(self, msg: Message): """ Handle a client request to get the value of a key among the chord network. """ if not self.alive: return trace_log('ChordNode: Running get handler') assert ('key' in msg.args and 'id' in msg.args) req_id = msg.args['id'] key = msg.args['key'] node_responsible_addr = await self.router.find_node_responsible_for_key(key) debug_test("Chord", self.chord_addr,": found node responsible for", key, node_responsible_addr) if (node_responsible_addr == self.chord_addr): value = await self.persistent_storage.get(key) else: get_value_msg = self.message_sender.create_message(MessageType.GET_VALUE, node_responsible_addr, {'key' : key}) get_value_reply = await self.keep_resending_until_response(get_value_msg, lambda msg: msg.message_type == MessageType.GET_VALUE_REPLY.value and msg.src == node_responsible_addr) assert ('key' in get_value_reply.args and 'value' in get_value_reply.args) assert (get_value_reply.args['key'] == key) value = get_value_reply.args['value'] self.message_sender.create_and_send_message(MessageType.GET_RESPONSE_TO_BROKER, None, {'id' : req_id, 'key': str(key), 'value' : value})
def startup(self): """ Startup the montior that stabalizes the successor list. """ trace_log('SuccessorList.Startup', self.addr, 'starting up.') self.async_scheduler.schedule_monitor_job( self.stabalize_successor_list, config.MONITOR_TIME) self.async_scheduler.schedule_monitor_job(self.stabalize_predecessor, config.MONITOR_TIME)
def __init__(self, addr: str, peers: List[int], async_scheduler: AsyncScheduler, message_sender: MessageSender, on_become_leader, on_step_down): trace_log(colorama.Fore.LIGHTRED_EX + "RaftPersistentKeyValueStore", addr, "starting up") self.addr = addr self.persistent_storage = PersistantKeyValueStore() self.raft_node = RaftNode(addr, peers, self.persistent_storage, async_scheduler, message_sender, on_become_leader, on_step_down)
async def copy_to(self, to_node: int, interval: Interval): """ Copy the keys of an interval from our persistant storage to the `to_node`. """ trace_log("KeyTransferer ", self.chord_addr, " : Starting copy of keys from myself to ", to_node) all_keys = await self.persistant_storage.get_keys() keys_to_transfer = [ key for key in all_keys if interval.in_interval(hash_key(key)) ] dict_to_transfer = {} for key in keys_to_transfer: dict_to_transfer[key] = await self.persistant_storage.get(key) await self.copy_key_values_to(to_node, interval, dict_to_transfer)
async def stabilize(self): trace_log("SuccRouter.stabilize: addr ", self.addr, " Stabilizing...") if (self.successor == self.addr): # If we were the first one to start return # Find our sucessors predecessor # request_predecessor = self.message_sender.create_message(MessageType.FIND_PREDECESSOR, node_id) # predecessor_msg = await self.send_message_and_await(request_predecessor, # lambda msg: msg.has_type(MessageType.FIND_PREDECESSOR_REPLY) and msg.src == self.successor) # assert ('predecessor' in predecessor_msg.args) # N Node Network #1. Find Predecessor of our Successor and Wait for Response successor_predecessor = await self.get_predecessor(self.successor) trace_log("SuccRouter.stabilize: addr ", self.addr, "found predecessor ", successor_predecessor) #UPDATE NODES #1. If Succesor does not have Predecessor, we may be their Predecessor if successor_predecessor == None: self.notify_i_might_be_your_predecessor(self.successor) return #2. If Node < Sucessor's predecessor < Succesor # then our Sucessors Predecessor is Most likely adjacent to us on # Ring so they become our new Sucessor if in_interval(self.addr, self.successor, successor_predecessor): self.successor = successor_predecessor #3. If Sucessor's predecessor ... < Node < Succesor # then our Sucessors Predecessor is Most likely us if (successor_predecessor != self.addr): # notify our sucessor that we might be their predecessor self.notify_i_might_be_your_predecessor(self.successor) debug_log("SuccRouter.stabilize Complete: addr: ", self.addr, "predecessor: ", self.predecessor, " successor: ", self.successor)
async def handle_i_might_be_your_predecessor(self, msg: Message): assert ('addr' in msg.args) candidate_predecessor = msg.args['addr'] # If we are the first node. if (self.successor == self.addr): self.successor = candidate_predecessor trace_log("SuccRouter addr:", self.addr, "Setting new successor ", candidate_predecessor) # transfer the stuff from (self.addr, candidate_predecessor] because we are no longer responsible for that # half of addr space # debug_log(self.addr, "Transfer because self.successor == self.addr, pred", self.predecessor, 'suc', self.successor) # await self.key_transferer.copy_to(candidate_predecessor, Interval(self.addr, False, candidate_predecessor, True)) # We know there are at least two nodes in the system. if (self.predecessor == None): # From our point of view, we used to be responsible for (self.sucessor, self.addr] # Now, candidate_predecessor is coming in and taking (self.successor, candidate_predecessor] from us # So we transfer (self.successor, candidate_predecessor] to them. if (self.successor != candidate_predecessor): debug_log(self.addr, "Transfer because self.prdecessor == None, pred", self.predecessor, 'suc', self.successor) await self.key_transferer.copy_to( candidate_predecessor, Interval(self.successor, False, candidate_predecessor, True)) else: # There is only one other node, so we should transfer him the other side debug_log(self.addr, "Transfer because self.prdecessor == None, pred", self.predecessor, 'suc', self.successor) await self.key_transferer.copy_to( candidate_predecessor, Interval(self.addr, False, candidate_predecessor, True)) # Normally, we don't need this. However, this fixes an edge cases involved with starting up with 1 node and having # 2 concurrent joins. self.predecessor = candidate_predecessor trace_log("SuccRouter addr:", self.addr, "No current predecessor. Setting predecessor ", candidate_predecessor) elif (in_interval(self.predecessor, self.addr, candidate_predecessor)): # From our point of view, we are responsible for (self.predecessor, self.addr] # However, now we are only responisble for (candidate_predecessor, self.addr] # So, we transfer out the difference: (self.predecessor, candidate_predecessor] debug_log( self.addr, "Transfer because candidate_predecessor \in (self.predecessor, self.addr), pred", self.predecessor, 'suc', self.successor) await self.key_transferer.copy_to( candidate_predecessor, Interval(self.predecessor, False, candidate_predecessor, True)) self.predecessor = candidate_predecessor trace_log("SuccRouter addr:", self.addr, "Setting new predecessor ", candidate_predecessor)
async def stabalize_successor_list(self): """ Maintains the successor_list """ # prune the start of the list for dead entries. trace_log('SuccessorList.stabalize', self.addr, ':Starting to stabalize.') await self.prune_start_of_successor_list() old_successors = set(self.successor_list) # If no successor alive, we are our own successor. if self.successor_list[0] == None: self.router.successor = self.addr else: # refresh the list. await self.refresh_successor_list() self.router.successor = self.successor_list[0] # We always need to get to this point. We cannot block before getting here otherwise the successor pointers will not # eventually be right. # Copy data to all new successors if self.replicate_data_to_successors: await self.replicate_our_data_to( set(self.successor_list).difference(old_successors))
async def find_node_responsible_for_addr(self, addr: int): trace_log( 'SuccRouter.find_node_responsible : Finding the node responsible for addr', addr) # This means no other node has joined yet if (self.addr == self.successor): return self.addr previous_node_id = self.addr next_node_id = self.successor iteration_num = 1 # A Key k is assigned to # the first node whose identifier is equal to or follows (the identifier # of) k in the identifier space while addr_later(self.addr, addr, next_node_id): trace_log( 'SuccRouter.find_node_responsible: Running Successor iteration ', iteration_num) msg_to_send = self.message_sender.create_message( MessageType.FIND_SUCCESSOR, next_node_id) find_successor_message: Message = await self.send_message_and_await( msg_to_send, lambda msg: msg.message_type == MessageType. FIND_SUCCESSOR_REPLY.value and msg.src == next_node_id) assert ('successor' in find_successor_message.args) trace_log( 'SuccRouter.find_node_responsible: Finished successor iteration ', iteration_num, '. Found succesor of ', next_node_id, ' which is ', find_successor_message.args['successor']) previous_node_id = next_node_id next_node_id = int(find_successor_message.args['successor']) iteration_num += 1 if (next_node_id == self.addr): # This means we went in a circle and found that we are the correct one break node_responsible = next_node_id trace_log('SuccRouter: Found the node responsible for addr ', addr, ' : ', node_responsible) return node_responsible
async def prune_start_of_successor_list(self): """ Prunes the start of the successor list for nodes that have died. """ trace_log('SuccessorList.prune_start_of_sucessor_list', self.addr, ':Starting to stabalize.') i = 0 while i < config.NUM_SUCCESSORS_IN_SUCCESSOR_LIST: # Run this NUM_SUCCESSORS_IN_SUCCESSOR_LIST times successor = self.successor_list[0] if successor == None: return # If there are no successors in the list, we return None. trace_log('SuccessorList.prune_start_of_sucessor_list', self.addr, ':Checking if', successor, 'is alive') is_alive = await self.is_alive(successor) trace_log('SuccessorList.prune_start_of_sucessor_list', self.addr, ':Checking if', successor, 'is alive', is_alive) # If the node is alive, we have pruned! if is_alive: return # The node is not alive, so we shift the successor list over one self.successor_list = self.successor_list[1:] + [None]
async def set(self, key, value): """ Set the value for a key """ trace_log("PersistantStorage: setting key ", key, " to value ", value) self.dict[key] = value