def FindValue(self, request, context):
"""Complicated - we might get a value back, or we might get k nodes with
distance closest to key requested
"""
print(f'Serving FindKey({request.idkey}) request for {request.node.id}', flush=True)
if self.values.get(request.idkey) is not None:
# Found the key
return csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node = self.DHT.ourNode,
mode_kv = True,
kv = csci4220_hw3_pb2.KeyValue(node=self.DHT.ourNode, key=request.idkey, value=self.values[request.idkey]),
nodes = []
) # Consider having the nodes be kClosest for Test 6
else:
# Exact same as FindNode
kClosest = self.DHT.kClosest(request.idkey)
# update the k-bucket of the requestor i.e. request.node.id
self.DHT.updateNodeLoc(request.node)
return csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node = self.DHT.ourNode,
mode_kv = False,
kv = None,
nodes = kClosest
)
def handle_store(self, key, value):
near_point = self.find_nearest_k(key, 1)[0]
if near_point.id == self.id:
self.library[key] = value
print("Storing key %d at node %d" % (key, self.id))
else:
with grpc.insecure_channel(near_point.address + ":" +
str(near_point.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
kv = csci4220_hw3_pb2.KeyValue(node=cur_node,
key=key,
value=value)
print("Storing key %d at node %d" % (key, near_point.id))
res = stub.Store(kv)
def Store(key, value):
distance = local_node.id ^ key
closest = local_node
for i in k_buckets:
for j in i:
if j.id ^ key < distance:
distance = j.id ^ key
closest = j
print("Storing key %d at node %d" % (key, closest.id))
if closest == local_node:
hash_table[key] = value
else:
with grpc.insecure_channel("%s:%d" %
(closest.address, closest.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
stub.Store(
csci4220_hw3_pb2.KeyValue(node=local_node,
key=key,
value=value))
def FindValue(self, IDKey, context):
key = IDKey.idkey
print("Serving FindKey(%d) request for %d" %
(IDKey.idkey, IDKey.node.id))
if key in hash_table.keys():
value = csci4220_hw3_pb2.KeyValue(key=key, value=hash_table[key])
AddorUpdateNode(IDKey.node)
return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=local_node,
mode_kv=True,
kv=value,
nodes=[])
else:
closest = []
for i in range(4):
closest += k_buckets[i]
closest = sorted(closest, key=lambda x: key ^ x.id)[:k]
AddorUpdateNode(IDKey.node)
return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=local_node,
mode_kv=False,
nodes=closest)
def store(args):
key = int(args.split()[1])
value = args.split()[2]
k_closest = find_k_closest(key)
closest_node = None if len(k_closest) < 1 else k_closest[0]
if closest_node is None or key ^ local_id < key ^ closest_node.id:
print("Storing key {} at node {}".format(key, local_id))
hash_table.put(key, value)
else:
print("Storing key {} at node {}".format(key, closest_node.id))
channel = grpc.insecure_channel("{}:{}".format(closest_node.address,
closest_node.port))
kad = csci4220_hw3_pb2_grpc.KadImplStub(channel)
kad.Store(
csci4220_hw3_pb2.KeyValue(node=csci4220_hw3_pb2.Node(
id=local_id, port=int(my_port), address=my_address),
key=key,
value=value))
def FindValue(self, request, context):
print("Serving FindKey({}) request for {}".format(
request.idkey, request.node.id))
has_key = hash_table.contains_key(request.idkey)
value = ""
nodes = []
if has_key:
value = hash_table.get(request.idkey)
else:
nodes = find_k_closest(request.idkey)
save_node(request.node)
return csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node=csci4220_hw3_pb2.Node(id=local_id,
port=int(my_port),
address=my_address),
mode_kv=has_key,
kv=csci4220_hw3_pb2.KeyValue(node=request.node,
key=request.idkey,
value=value),
nodes=nodes)
def FindValue(self, request, context):
print("Serving FindKey({}) request for {}".format(
request.idkey, request.node.id))
# See if the key/value pair has already been stored
for pair in self.key_value:
if pair[0] == request.idkey:
# Update the k_buckets
self.update_k_buckets(request.node, True)
kv = csci4220_hw3_pb2.KeyValue(node=self.node,
key=request.idkey,
value=pair[1])
return csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node=self.node, mode_kv=True, kv=kv)
k_closest_nodes = self.find_k_closest_nodes(request.idkey)
# Update the k_buckets
self.update_k_buckets(request.node, True)
return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=self.node,
mode_kv=False,
nodes=k_closest_nodes)
def FindValue(self, request, context):
print("Serving FindKey(%d) request for %d" %
(request.idkey, request.node.id))
key = request.idkey
if self.id != request.node.id:
self.update_k_bucket(request.node)
if key in self.library:
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
key_value = csci4220_hw3_pb2.KeyValue(node=cur_node,
key=key,
value=self.library[key])
kv_wrapper = csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node=cur_node, mode_kv=1, kv=key_value, nodes=[])
return kv_wrapper
else:
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
k_nodes = self.find_nearest_k(request.idkey, self.k)
kv_wrapper = csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node=cur_node, mode_kv=0, kv=None, nodes=k_nodes)
return kv_wrapper
def FindValue(self, request, context):
global bucket
# delete same nodes
for k,v in list(bucket.items()):
for i in range(0,len(v)):
if v[i].id==local_node:
if len(v)==1:
del bucket[k]
else:
v.remove(i)
print("Serving FindKey(" + str(request.idkey) + ") request for " + str(request.node.id))
sys.stdout.flush()
# if found the key in server node, return the key directly
if request.idkey==local_key:
r_node = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
mode = True
key_value = csci4220_hw3_pb2.KeyValue(node=r_node,key=local_key,value=local_value)
return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=r_node,mode_kv=mode,kv=key_value,nodes=[])
# if not found, return a k-near list of nodes
else:
mode = False
index = locateBucket(local_node,request.node.id)
if index!=-1:
if index not in bucket:
tmp = []
tmp.append(nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
else:
adding = True
n = len(bucket[index])
for i in range(0,n):
if bucket[index][i].id==request.node.id:
adding = False
break
if adding==True:
if n==bucket_max:
tmp = bucket[index]
tmp.pop()
tmp.insert(0,nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
all_nodes = []
for k,v in bucket.items():
for j in range(len(v)):
all_nodes.append(v[j])
ordered_nodes = sorted(all_nodes, key=lambda x : XOR(x.id,request.node.id))
node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
ret = []
for node in ordered_nodes:
if node.id==0:
tmp = csci4220_hw3_pb2.Node(id=1000,port=node.port,address=node.address)
else:
tmp = csci4220_hw3_pb2.Node(id=node.id,port=node.port,address=node.address)
ret.append(tmp)
r_node = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
key_value = csci4220_hw3_pb2.KeyValue(node=r_node,key=local_key,value=local_value)
return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=r_node,mode_kv=mode,kv=key_value,nodes=ret)
min_distrance = XOR(v[i].id,key)
target = v[i]
tmp_id = target.id
if target.id==local_id:
local_key = key
local_value = value
print("Storing key " + str(key) + " at node " + str(local_id))
sys.stdout.flush()
else:
if target.id==0:
tmp_id = 1000
node = csci4220_hw3_pb2.Node(id=tmp_id,port=target.port,address=target.address)
ky = csci4220_hw3_pb2.KeyValue(node=node,key=key,value=value)
with grpc.insecure_channel(target.address + ':' + str(target.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
stub.Store(ky)
print("Storing key " + str(key) + " at node " + str(target.id))
sys.stdout.flush()
channel.close()
elif msg.split(" ")[0]=='QUIT':
if already_quit==False:
for k,v in list(bucket.items()):
for i in range(0,len(v)):
if v[i].id!=local_node:
re_id = v[i].id
def run():
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(sys.argv[0]))
sys.exit(-1)
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(my_hostname) # Gets my IP address from my hostname
''' Use the following code to convert a hostname to an IP and start a channel
Note that every stub needs a channel attached to it
When you are done with a channel you should call .close() on the channel.
Submitty may kill your program if you have too many file descriptors open
at the same time. '''
#remote_addr = socket.gethostbyname(remote_addr_string)
#remote_port = int(remote_port_string)
#channel = grpc.insecure_channel(remote_addr + ':' + str(remote_port))
ourNode = csci4220_hw3_pb2.Node(id = local_id, port = int(my_port), address = my_address)
ourServicer = KadImplServicer(k, ourNode)
# Code to start the servicer for our node
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
csci4220_hw3_pb2_grpc.add_KadImplServicer_to_server(ourServicer, server)
server.add_insecure_port(f'[::]:{my_port}')
server.start() # Let servicer start listening w/t blocking
while True:
userInput = input() # Perhaps change this to select, but that's a discussion for later :P
userInput = userInput.strip().split()
if ((userInput[0]).lower().strip() == "bootstrap"):
remoteHost = (userInput[1]).strip()
remotePort = (userInput[2]).strip()
remote_addr = socket.gethostbyname(remoteHost)
remote_port = int(remotePort)
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
result = stub.FindNode(csci4220_hw3_pb2.IDKey(node = ourNode, idkey = ourNode.id))
remoteID = result.responding_node.id]
ourServicer.DHT.updateNodeLoc(result.responding_node)
for node_ in result.nodes:
ourServicer.DHT.updateNodeLoc(node_)
print(f'After BOOTSTRAP({remoteID}), k_buckets now look like:')
ourServicer.DHT.print()
elif ((userInput[0]).lower().strip() == "find_node" or (userInput[0]).lower().strip() == "find_value"):
nodeID = int((userInput[1]).strip())
findNode = (userInput[0]).lower().strip() == "find_node"
print(f"Before FIND_{'NODE' if findNode else 'VALUE'} command, k-buckets are:")
ourServicer.DHT.print()
found = nodeID == local_id if findNode else ourServicer.values.get(nodeID) is not None
value = (found, ourServicer.values.get(nodeID))
if (not found):
# Do the search
Visited = set() # Stores id of visited nodes
nodeOfInterest = csci4220_hw3_pb2.IDKey(node = ourNode, idkey = nodeID)
# Condition for us to use FindNode or FindValue
# S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest to idkey
S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest nodes to idkey
# print(f"Closest nodes to {nodeOfInterest.idkey} are {[n.id for n in S]}")
notVisited = [n for n in S if n.id not in Visited]
while not found and len(notVisited) > 0:
for node in notVisited:
with grpc.insecure_channel(node.address + ':' + str(node.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Condition for us to use FindNode or FindValue
R = stub.FindNode(nodeOfInterest) if findNode else stub.FindValue(nodeOfInterest)
# Update our k-buckets with node
ourServicer.DHT.updateNodeLoc(R.responding_node)
if not findNode and R.mode_kv: # If we're for FindValue
found = True
value = (R.responding_node.id == ourNode.id, R.kv.value)
# If a node in R was already in a k-bucket, its position does not change.
# If it was not in the bucket yet, then it is added as the most recently used in that bucket.
# This _may_ kick out the node from above.
for n in R.nodes:
if findNode and (n.id ^ nodeOfInterest.idkey) == 0:
found = True
ourServicer.DHT.updateNodeLoc(n)
Visited.add(node.id)
if found:
break
S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest nodes to idkey
notVisited = [n for n in S if n.id not in Visited]
# Do the printing
if (found):
if findNode:
print(f'Found destination id {nodeID}', flush=True)
else:
if value[0]:
print(f'Found data "{value[1]}" for key {nodeID}')
else:
print(f'Found value "{value[1]}" for key {nodeID}')
else:
if findNode:
print(f'Could not find destination id {nodeID}', flush=True)
else:
print(f'Could not find key {nodeID}')
print(f"After FIND_{'NODE' if findNode else 'VALUE'} command, k-buckets are:")
ourServicer.DHT.print()
elif ((userInput[0]).lower().strip() == "store"):
key = int(userInput[1].strip())
val = userInput[2].strip()
# Find the node it knows that's closest to the key
closestNodes = ourServicer.DHT.kClosest(key)
if len(closestNodes) == 0 or (ourNode.id ^ key <= closestNodes[0].id ^ key):
print(f'Storing key {key} at node {ourNode.id}', flush=True)
ourServicer.values[key] = val
else:
print(f'Storing key {key} at node {closestNodes[0].id}', flush=True)
with grpc.insecure_channel(closestNodes[0].address + ':' + str(closestNodes[0].port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
stub.Store(csci4220_hw3_pb2.KeyValue(node = ourNode,
key = key,
value = val))
elif ((userInput[0]).lower().strip() == "quit"):
for node in ourServicer.DHT.getAllNodes():
with grpc.insecure_channel(node.address + ':' + str(node.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
print(f'Letting {node.id} know I\'m quitting.', flush=True)
try:
stub.Quit(csci4220_hw3_pb2.IDKey(node = ourNode, idkey = ourNode.id))
except:
pass # The stub already closed
print(f'Shut down node {ourNode.id}', flush=True)
server.stop(None)
break
else:
print("Invalid Command", flush=True)
def run():
'''
Check the # of command-line arguments, which should have
hw3.py <nodeID> <port> <k>
'''
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(
sys.argv[0]))
sys.exit(-1)
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(
my_hostname) # Gets my IP address from my hostname
# print("[myself] Running as {0} @ {1}".format(my_hostname,my_address))
# Setting up the server
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
servicer = KadImplServicer(local_id, int(my_port), my_address, k)
csci4220_hw3_pb2_grpc.add_KadImplServicer_to_server(servicer, server)
server.add_insecure_port("[::]:" + my_port)
server.start()
# Run the client
while True:
inputs = input()
arguments = inputs.split()
if arguments[0] == "BOOTSTRAP":
remote_hostname = arguments[1]
my_port = int(arguments[2])
remote_addr = socket.gethostbyname(remote_hostname)
# Connect to other peer
channel = grpc.insecure_channel(
str(remote_addr) + ':' + str(my_port))
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Find peer node using servicer node
nodee = stub.FindNode(
csci4220_hw3_pb2.IDKey(node=servicer.node, idkey=local_id))
# Add node that was responding
res_node = nodee.responding_node
bit_len = ((res_node.id) ^ local_id).bit_length()
bit_len -= 1
if len(servicer.k_buckets[bit_len]) == k:
servicer.k_buckets[bit_len].popleft()
servicer.k_buckets[bit_len].append(res_node)
# Print bucket contents
print('After BOOTSTRAP({}), k_buckets now look like:'.format(
str(res_node.id)))
servicer.print_k_buckets()
channel.close()
elif arguments[0] == "FIND_NODE":
# Load the inputs argument
node_id = int(arguments[1])
print("Before FIND_NODE command, k-buckets are:")
servicer.print_k_buckets()
S = servicer.find_k_closest_nodes(node_id)
S_no = S # Nodes in S that have not been contacted yet
S_yes = [] # Nodes that have been contacted
found = False
while S_no and (not found):
for no in S_no:
# Start a channel
channel = grpc.insecure_channel(no.address + ':' +
str(no.port))
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Send a FindNode RPC
R = stub.FindNode(
csci4220_hw3_pb2.IDKey(node=servicer.node,
idkey=node_id))
# Update the k-buckets
servicer.update_k_buckets(no, True)
for node in R.nodes:
servicer.update_k_buckets(node, False)
# Update S_yes
S_yes.append(no)
# Close the channel
channel.close()
# Check if the node has been found
for bucket in servicer.k_buckets:
if any(node.id == node_id for node in bucket):
found = True
print("Found destination id {}".format(node_id))
break
if not found:
# Update S and S_no
S = servicer.find_k_closest_nodes(node_id)
S_no.clear()
for node in S:
if not any(yes.id == node.id for yes in S_yes):
S_no.append(node)
if not found:
print("Could not find destination id {}".format(node_id))
print("After FIND_NODE command, k-buckets are:")
servicer.print_k_buckets()
elif arguments[0] == "FIND_VALUE":
# Load the inputs argument
key = int(arguments[1])
print("Before FIND_VALUE command, k-buckets are:")
servicer.print_k_buckets()
found = False
# See if the key/value pair has already been stored
for pair in servicer.key_value:
if pair[0] == key:
found = True
print("Found data \"{}\" for key {}".format(pair[1], key))
break
if not found:
S = servicer.find_k_closest_nodes(key)
S_no = S # Nodes in S that have not been contacted yet
S_yes = [] # Nodes that have been contacted
while S_no and (not found):
for no in S_no:
# Start a channel
channel = grpc.insecure_channel(no.address + ':' +
str(no.port))
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Send a FindNode RPC
R = stub.FindValue(
csci4220_hw3_pb2.IDKey(node=servicer.node,
idkey=key))
# Update the k-buckets
servicer.update_k_buckets(no, True)
# Update S_yes
S_yes.append(no)
# Close the channel
channel.close()
# Check if the value is found
if R.mode_kv:
found = True
print("Found value \"{}\" for key {}".format(
R.kv.value, key))
break
else:
# Update the k-buckets
for node in R.nodes:
servicer.update_k_buckets(node, False)
if not found:
# Update S and S_no
S = servicer.find_k_closest_nodes(key)
S_no.clear()
for node in S:
if not any(yes.id == node.id for yes in S_yes):
S_no.append(node)
if not found:
print("Could not find key {}".format(key))
print("After FIND_VALUE command, k-buckets are:")
servicer.print_k_buckets()
elif arguments[0] == "STORE":
this_key = int(arguments[1])
this_value = arguments[2]
closest_node = csci4220_hw3_pb2.Node(id=local_id,
port=int(my_port),
address=str(my_address))
distance = abs(local_id - this_key)
for bucket in servicer.k_buckets:
for entry in bucket:
if abs(int(entry.id) - this_key) < distance:
closest_node = entry
distance = abs(int(entry.id) - this_key)
closest_port = int(closest_node.port)
#Connect to server & stub
this_addr = closest_node.address + ':' + str(closest_port)
channel = grpc.insecure_channel(this_addr)
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
some_idkey = stub.Store(
csci4220_hw3_pb2.KeyValue(node=servicer.node,
key=this_key,
value=this_value))
print("Storing key {} at node {}".format(this_key,
some_idkey.idkey))
channel.close()
elif arguments[0] == "QUIT":
# Tell everyone we are quitting, goodbye :(
for bucket in servicer.k_buckets:
for entry in bucket:
remote_host = str(entry.id)
remote_port = int(entry.port)
my_addr = entry.address + ':' + str(remote_port)
# Create channel and tell other peer we are quitting
channel = grpc.insecure_channel(my_addr)
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
print("Letting " + remote_host + " know I'm quitting.")
some_idkey = stub.Quit(
csci4220_hw3_pb2.IDKey(node=servicer.node,
idkey=local_id))
channel.close()
print("Shut down node " + str(local_id))
break
else:
# Unknown command
print("Unknown command. Use BOOTSTRAP , FIND_VALUE , STORE , QUIT")