def FindValue(self, request, context): """Complicated - we might get a value back, or we might get k nodes with distance closest to key requested """ print(f'Serving FindKey({request.idkey}) request for {request.node.id}', flush=True) if self.values.get(request.idkey) is not None: # Found the key return csci4220_hw3_pb2.KV_Node_Wrapper( responding_node = self.DHT.ourNode, mode_kv = True, kv = csci4220_hw3_pb2.KeyValue(node=self.DHT.ourNode, key=request.idkey, value=self.values[request.idkey]), nodes = [] ) # Consider having the nodes be kClosest for Test 6 else: # Exact same as FindNode kClosest = self.DHT.kClosest(request.idkey) # update the k-bucket of the requestor i.e. request.node.id self.DHT.updateNodeLoc(request.node) return csci4220_hw3_pb2.KV_Node_Wrapper( responding_node = self.DHT.ourNode, mode_kv = False, kv = None, nodes = kClosest )
def handle_store(self, key, value): near_point = self.find_nearest_k(key, 1)[0] if near_point.id == self.id: self.library[key] = value print("Storing key %d at node %d" % (key, self.id)) else: with grpc.insecure_channel(near_point.address + ":" + str(near_point.port)) as channel: stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) cur_node = csci4220_hw3_pb2.Node(id=self.id, port=int(self.port), address=self.address) kv = csci4220_hw3_pb2.KeyValue(node=cur_node, key=key, value=value) print("Storing key %d at node %d" % (key, near_point.id)) res = stub.Store(kv)
def Store(key, value): distance = local_node.id ^ key closest = local_node for i in k_buckets: for j in i: if j.id ^ key < distance: distance = j.id ^ key closest = j print("Storing key %d at node %d" % (key, closest.id)) if closest == local_node: hash_table[key] = value else: with grpc.insecure_channel("%s:%d" % (closest.address, closest.port)) as channel: stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) stub.Store( csci4220_hw3_pb2.KeyValue(node=local_node, key=key, value=value))
def FindValue(self, IDKey, context): key = IDKey.idkey print("Serving FindKey(%d) request for %d" % (IDKey.idkey, IDKey.node.id)) if key in hash_table.keys(): value = csci4220_hw3_pb2.KeyValue(key=key, value=hash_table[key]) AddorUpdateNode(IDKey.node) return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=local_node, mode_kv=True, kv=value, nodes=[]) else: closest = [] for i in range(4): closest += k_buckets[i] closest = sorted(closest, key=lambda x: key ^ x.id)[:k] AddorUpdateNode(IDKey.node) return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=local_node, mode_kv=False, nodes=closest)
def store(args): key = int(args.split()[1]) value = args.split()[2] k_closest = find_k_closest(key) closest_node = None if len(k_closest) < 1 else k_closest[0] if closest_node is None or key ^ local_id < key ^ closest_node.id: print("Storing key {} at node {}".format(key, local_id)) hash_table.put(key, value) else: print("Storing key {} at node {}".format(key, closest_node.id)) channel = grpc.insecure_channel("{}:{}".format(closest_node.address, closest_node.port)) kad = csci4220_hw3_pb2_grpc.KadImplStub(channel) kad.Store( csci4220_hw3_pb2.KeyValue(node=csci4220_hw3_pb2.Node( id=local_id, port=int(my_port), address=my_address), key=key, value=value))
def FindValue(self, request, context): print("Serving FindKey({}) request for {}".format( request.idkey, request.node.id)) has_key = hash_table.contains_key(request.idkey) value = "" nodes = [] if has_key: value = hash_table.get(request.idkey) else: nodes = find_k_closest(request.idkey) save_node(request.node) return csci4220_hw3_pb2.KV_Node_Wrapper( responding_node=csci4220_hw3_pb2.Node(id=local_id, port=int(my_port), address=my_address), mode_kv=has_key, kv=csci4220_hw3_pb2.KeyValue(node=request.node, key=request.idkey, value=value), nodes=nodes)
def FindValue(self, request, context): print("Serving FindKey({}) request for {}".format( request.idkey, request.node.id)) # See if the key/value pair has already been stored for pair in self.key_value: if pair[0] == request.idkey: # Update the k_buckets self.update_k_buckets(request.node, True) kv = csci4220_hw3_pb2.KeyValue(node=self.node, key=request.idkey, value=pair[1]) return csci4220_hw3_pb2.KV_Node_Wrapper( responding_node=self.node, mode_kv=True, kv=kv) k_closest_nodes = self.find_k_closest_nodes(request.idkey) # Update the k_buckets self.update_k_buckets(request.node, True) return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=self.node, mode_kv=False, nodes=k_closest_nodes)
def FindValue(self, request, context): print("Serving FindKey(%d) request for %d" % (request.idkey, request.node.id)) key = request.idkey if self.id != request.node.id: self.update_k_bucket(request.node) if key in self.library: cur_node = csci4220_hw3_pb2.Node(id=self.id, port=int(self.port), address=self.address) key_value = csci4220_hw3_pb2.KeyValue(node=cur_node, key=key, value=self.library[key]) kv_wrapper = csci4220_hw3_pb2.KV_Node_Wrapper( responding_node=cur_node, mode_kv=1, kv=key_value, nodes=[]) return kv_wrapper else: cur_node = csci4220_hw3_pb2.Node(id=self.id, port=int(self.port), address=self.address) k_nodes = self.find_nearest_k(request.idkey, self.k) kv_wrapper = csci4220_hw3_pb2.KV_Node_Wrapper( responding_node=cur_node, mode_kv=0, kv=None, nodes=k_nodes) return kv_wrapper
def FindValue(self, request, context): global bucket # delete same nodes for k,v in list(bucket.items()): for i in range(0,len(v)): if v[i].id==local_node: if len(v)==1: del bucket[k] else: v.remove(i) print("Serving FindKey(" + str(request.idkey) + ") request for " + str(request.node.id)) sys.stdout.flush() # if found the key in server node, return the key directly if request.idkey==local_key: r_node = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address) mode = True key_value = csci4220_hw3_pb2.KeyValue(node=r_node,key=local_key,value=local_value) return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=r_node,mode_kv=mode,kv=key_value,nodes=[]) # if not found, return a k-near list of nodes else: mode = False index = locateBucket(local_node,request.node.id) if index!=-1: if index not in bucket: tmp = [] tmp.append(nodeObj(request.node.id,request.node.port,request.node.address)) bucket[index] = tmp else: adding = True n = len(bucket[index]) for i in range(0,n): if bucket[index][i].id==request.node.id: adding = False break if adding==True: if n==bucket_max: tmp = bucket[index] tmp.pop() tmp.insert(0,nodeObj(request.node.id,request.node.port,request.node.address)) bucket[index] = tmp all_nodes = [] for k,v in bucket.items(): for j in range(len(v)): all_nodes.append(v[j]) ordered_nodes = sorted(all_nodes, key=lambda x : XOR(x.id,request.node.id)) node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address) ret = [] for node in ordered_nodes: if node.id==0: tmp = csci4220_hw3_pb2.Node(id=1000,port=node.port,address=node.address) else: tmp = csci4220_hw3_pb2.Node(id=node.id,port=node.port,address=node.address) ret.append(tmp) r_node = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address) key_value = csci4220_hw3_pb2.KeyValue(node=r_node,key=local_key,value=local_value) return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=r_node,mode_kv=mode,kv=key_value,nodes=ret)
min_distrance = XOR(v[i].id,key) target = v[i] tmp_id = target.id if target.id==local_id: local_key = key local_value = value print("Storing key " + str(key) + " at node " + str(local_id)) sys.stdout.flush() else: if target.id==0: tmp_id = 1000 node = csci4220_hw3_pb2.Node(id=tmp_id,port=target.port,address=target.address) ky = csci4220_hw3_pb2.KeyValue(node=node,key=key,value=value) with grpc.insecure_channel(target.address + ':' + str(target.port)) as channel: stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) stub.Store(ky) print("Storing key " + str(key) + " at node " + str(target.id)) sys.stdout.flush() channel.close() elif msg.split(" ")[0]=='QUIT': if already_quit==False: for k,v in list(bucket.items()): for i in range(0,len(v)): if v[i].id!=local_node: re_id = v[i].id
def run(): if len(sys.argv) != 4: print("Error, correct usage is {} [my id] [my port] [k]".format(sys.argv[0])) sys.exit(-1) local_id = int(sys.argv[1]) my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string k = int(sys.argv[3]) my_hostname = socket.gethostname() # Gets my host name my_address = socket.gethostbyname(my_hostname) # Gets my IP address from my hostname ''' Use the following code to convert a hostname to an IP and start a channel Note that every stub needs a channel attached to it When you are done with a channel you should call .close() on the channel. Submitty may kill your program if you have too many file descriptors open at the same time. ''' #remote_addr = socket.gethostbyname(remote_addr_string) #remote_port = int(remote_port_string) #channel = grpc.insecure_channel(remote_addr + ':' + str(remote_port)) ourNode = csci4220_hw3_pb2.Node(id = local_id, port = int(my_port), address = my_address) ourServicer = KadImplServicer(k, ourNode) # Code to start the servicer for our node server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) csci4220_hw3_pb2_grpc.add_KadImplServicer_to_server(ourServicer, server) server.add_insecure_port(f'[::]:{my_port}') server.start() # Let servicer start listening w/t blocking while True: userInput = input() # Perhaps change this to select, but that's a discussion for later :P userInput = userInput.strip().split() if ((userInput[0]).lower().strip() == "bootstrap"): remoteHost = (userInput[1]).strip() remotePort = (userInput[2]).strip() remote_addr = socket.gethostbyname(remoteHost) remote_port = int(remotePort) with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel: stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) result = stub.FindNode(csci4220_hw3_pb2.IDKey(node = ourNode, idkey = ourNode.id)) remoteID = result.responding_node.id] ourServicer.DHT.updateNodeLoc(result.responding_node) for node_ in result.nodes: ourServicer.DHT.updateNodeLoc(node_) print(f'After BOOTSTRAP({remoteID}), k_buckets now look like:') ourServicer.DHT.print() elif ((userInput[0]).lower().strip() == "find_node" or (userInput[0]).lower().strip() == "find_value"): nodeID = int((userInput[1]).strip()) findNode = (userInput[0]).lower().strip() == "find_node" print(f"Before FIND_{'NODE' if findNode else 'VALUE'} command, k-buckets are:") ourServicer.DHT.print() found = nodeID == local_id if findNode else ourServicer.values.get(nodeID) is not None value = (found, ourServicer.values.get(nodeID)) if (not found): # Do the search Visited = set() # Stores id of visited nodes nodeOfInterest = csci4220_hw3_pb2.IDKey(node = ourNode, idkey = nodeID) # Condition for us to use FindNode or FindValue # S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest to idkey S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest nodes to idkey # print(f"Closest nodes to {nodeOfInterest.idkey} are {[n.id for n in S]}") notVisited = [n for n in S if n.id not in Visited] while not found and len(notVisited) > 0: for node in notVisited: with grpc.insecure_channel(node.address + ':' + str(node.port)) as channel: stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) # Condition for us to use FindNode or FindValue R = stub.FindNode(nodeOfInterest) if findNode else stub.FindValue(nodeOfInterest) # Update our k-buckets with node ourServicer.DHT.updateNodeLoc(R.responding_node) if not findNode and R.mode_kv: # If we're for FindValue found = True value = (R.responding_node.id == ourNode.id, R.kv.value) # If a node in R was already in a k-bucket, its position does not change. # If it was not in the bucket yet, then it is added as the most recently used in that bucket. # This _may_ kick out the node from above. for n in R.nodes: if findNode and (n.id ^ nodeOfInterest.idkey) == 0: found = True ourServicer.DHT.updateNodeLoc(n) Visited.add(node.id) if found: break S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest nodes to idkey notVisited = [n for n in S if n.id not in Visited] # Do the printing if (found): if findNode: print(f'Found destination id {nodeID}', flush=True) else: if value[0]: print(f'Found data "{value[1]}" for key {nodeID}') else: print(f'Found value "{value[1]}" for key {nodeID}') else: if findNode: print(f'Could not find destination id {nodeID}', flush=True) else: print(f'Could not find key {nodeID}') print(f"After FIND_{'NODE' if findNode else 'VALUE'} command, k-buckets are:") ourServicer.DHT.print() elif ((userInput[0]).lower().strip() == "store"): key = int(userInput[1].strip()) val = userInput[2].strip() # Find the node it knows that's closest to the key closestNodes = ourServicer.DHT.kClosest(key) if len(closestNodes) == 0 or (ourNode.id ^ key <= closestNodes[0].id ^ key): print(f'Storing key {key} at node {ourNode.id}', flush=True) ourServicer.values[key] = val else: print(f'Storing key {key} at node {closestNodes[0].id}', flush=True) with grpc.insecure_channel(closestNodes[0].address + ':' + str(closestNodes[0].port)) as channel: stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) stub.Store(csci4220_hw3_pb2.KeyValue(node = ourNode, key = key, value = val)) elif ((userInput[0]).lower().strip() == "quit"): for node in ourServicer.DHT.getAllNodes(): with grpc.insecure_channel(node.address + ':' + str(node.port)) as channel: stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) print(f'Letting {node.id} know I\'m quitting.', flush=True) try: stub.Quit(csci4220_hw3_pb2.IDKey(node = ourNode, idkey = ourNode.id)) except: pass # The stub already closed print(f'Shut down node {ourNode.id}', flush=True) server.stop(None) break else: print("Invalid Command", flush=True)
def run(): ''' Check the # of command-line arguments, which should have hw3.py <nodeID> <port> <k> ''' if len(sys.argv) != 4: print("Error, correct usage is {} [my id] [my port] [k]".format( sys.argv[0])) sys.exit(-1) local_id = int(sys.argv[1]) my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string k = int(sys.argv[3]) my_hostname = socket.gethostname() # Gets my host name my_address = socket.gethostbyname( my_hostname) # Gets my IP address from my hostname # print("[myself] Running as {0} @ {1}".format(my_hostname,my_address)) # Setting up the server server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) servicer = KadImplServicer(local_id, int(my_port), my_address, k) csci4220_hw3_pb2_grpc.add_KadImplServicer_to_server(servicer, server) server.add_insecure_port("[::]:" + my_port) server.start() # Run the client while True: inputs = input() arguments = inputs.split() if arguments[0] == "BOOTSTRAP": remote_hostname = arguments[1] my_port = int(arguments[2]) remote_addr = socket.gethostbyname(remote_hostname) # Connect to other peer channel = grpc.insecure_channel( str(remote_addr) + ':' + str(my_port)) stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) # Find peer node using servicer node nodee = stub.FindNode( csci4220_hw3_pb2.IDKey(node=servicer.node, idkey=local_id)) # Add node that was responding res_node = nodee.responding_node bit_len = ((res_node.id) ^ local_id).bit_length() bit_len -= 1 if len(servicer.k_buckets[bit_len]) == k: servicer.k_buckets[bit_len].popleft() servicer.k_buckets[bit_len].append(res_node) # Print bucket contents print('After BOOTSTRAP({}), k_buckets now look like:'.format( str(res_node.id))) servicer.print_k_buckets() channel.close() elif arguments[0] == "FIND_NODE": # Load the inputs argument node_id = int(arguments[1]) print("Before FIND_NODE command, k-buckets are:") servicer.print_k_buckets() S = servicer.find_k_closest_nodes(node_id) S_no = S # Nodes in S that have not been contacted yet S_yes = [] # Nodes that have been contacted found = False while S_no and (not found): for no in S_no: # Start a channel channel = grpc.insecure_channel(no.address + ':' + str(no.port)) stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) # Send a FindNode RPC R = stub.FindNode( csci4220_hw3_pb2.IDKey(node=servicer.node, idkey=node_id)) # Update the k-buckets servicer.update_k_buckets(no, True) for node in R.nodes: servicer.update_k_buckets(node, False) # Update S_yes S_yes.append(no) # Close the channel channel.close() # Check if the node has been found for bucket in servicer.k_buckets: if any(node.id == node_id for node in bucket): found = True print("Found destination id {}".format(node_id)) break if not found: # Update S and S_no S = servicer.find_k_closest_nodes(node_id) S_no.clear() for node in S: if not any(yes.id == node.id for yes in S_yes): S_no.append(node) if not found: print("Could not find destination id {}".format(node_id)) print("After FIND_NODE command, k-buckets are:") servicer.print_k_buckets() elif arguments[0] == "FIND_VALUE": # Load the inputs argument key = int(arguments[1]) print("Before FIND_VALUE command, k-buckets are:") servicer.print_k_buckets() found = False # See if the key/value pair has already been stored for pair in servicer.key_value: if pair[0] == key: found = True print("Found data \"{}\" for key {}".format(pair[1], key)) break if not found: S = servicer.find_k_closest_nodes(key) S_no = S # Nodes in S that have not been contacted yet S_yes = [] # Nodes that have been contacted while S_no and (not found): for no in S_no: # Start a channel channel = grpc.insecure_channel(no.address + ':' + str(no.port)) stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) # Send a FindNode RPC R = stub.FindValue( csci4220_hw3_pb2.IDKey(node=servicer.node, idkey=key)) # Update the k-buckets servicer.update_k_buckets(no, True) # Update S_yes S_yes.append(no) # Close the channel channel.close() # Check if the value is found if R.mode_kv: found = True print("Found value \"{}\" for key {}".format( R.kv.value, key)) break else: # Update the k-buckets for node in R.nodes: servicer.update_k_buckets(node, False) if not found: # Update S and S_no S = servicer.find_k_closest_nodes(key) S_no.clear() for node in S: if not any(yes.id == node.id for yes in S_yes): S_no.append(node) if not found: print("Could not find key {}".format(key)) print("After FIND_VALUE command, k-buckets are:") servicer.print_k_buckets() elif arguments[0] == "STORE": this_key = int(arguments[1]) this_value = arguments[2] closest_node = csci4220_hw3_pb2.Node(id=local_id, port=int(my_port), address=str(my_address)) distance = abs(local_id - this_key) for bucket in servicer.k_buckets: for entry in bucket: if abs(int(entry.id) - this_key) < distance: closest_node = entry distance = abs(int(entry.id) - this_key) closest_port = int(closest_node.port) #Connect to server & stub this_addr = closest_node.address + ':' + str(closest_port) channel = grpc.insecure_channel(this_addr) stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) some_idkey = stub.Store( csci4220_hw3_pb2.KeyValue(node=servicer.node, key=this_key, value=this_value)) print("Storing key {} at node {}".format(this_key, some_idkey.idkey)) channel.close() elif arguments[0] == "QUIT": # Tell everyone we are quitting, goodbye :( for bucket in servicer.k_buckets: for entry in bucket: remote_host = str(entry.id) remote_port = int(entry.port) my_addr = entry.address + ':' + str(remote_port) # Create channel and tell other peer we are quitting channel = grpc.insecure_channel(my_addr) stub = csci4220_hw3_pb2_grpc.KadImplStub(channel) print("Letting " + remote_host + " know I'm quitting.") some_idkey = stub.Quit( csci4220_hw3_pb2.IDKey(node=servicer.node, idkey=local_id)) channel.close() print("Shut down node " + str(local_id)) break else: # Unknown command print("Unknown command. Use BOOTSTRAP , FIND_VALUE , STORE , QUIT")