def handle_find_node(self, node_id):
# if the finding node is this node
finded_node = None
if self.id == node_id:
return True
else:
flag = False
visited = {self.id}
k_nodes = deque(self.find_nearest_k(node_id, self.k))
while k_nodes:
node = k_nodes.popleft()
if not node.id in visited:
self.update_k_bucket(node)
if node.id == node_id:
flag = True
break
with grpc.insecure_channel(node.address + ":" +
str(node.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
IDKey = csci4220_hw3_pb2.IDKey(node=cur_node,
idkey=node_id)
res = stub.FindNode(IDKey)
visited.add(node.id)
for n in res.nodes:
if not n.id in visited:
k_nodes.append(n)
while k_nodes:
node = k_nodes.popleft()
self.update_k_bucket(node)
return flag
def find_value(args):
print("Before FIND_VALUE command, k-buckets are:\n{}".format(
print_buckets()))
key = int(args.split()[1]) # Getting value from the input
if hash_table.contains_key(
key): # the hash table is where we keep the key value pairs
print("Found data \"{}\" for key {}".format(hash_table.get(key), key))
print("After FIND_VALUE command, k-buckets are:\n" + print_buckets())
return
else:
unvisited = find_k_closest(
key) # if we dont have the value in out hash ...
visited = list()
visited.append(
csci4220_hw3_pb2.
Node( # we need to loop over the k closest nodes, similar to findnode
id=local_id,
port=int(my_port),
address=my_address))
value_found = False
value = None
while len(
unvisited
) > 0 and not value_found: # Now, we loop over all of the unvisited nodes
for node in unvisited:
channel = grpc.insecure_channel("{}:{}".format(
node.address, node.port))
kad = csci4220_hw3_pb2_grpc.KadImplStub(channel)
response = kad.FindValue( # Create a connection to that node
csci4220_hw3_pb2.IDKey(node=csci4220_hw3_pb2.Node(
id=local_id, port=int(my_port), address=my_address),
idkey=key))
save_node(
node) # Save it, as it needs to be updated based on LRU
visited.append(
node) # And we add it to the visited nodes array
if response.mode_kv:
value = response.kv.value
value_found = True
break
for resp_node in response.nodes: # If we havent found the value in our direct k_buckets....
if not node_is_stored(
resp_node
) and resp_node.id != local_id: # We look in each node's k_buckets
save_node(resp_node)
# Look for any unvisited nodes and add them for next iteration
unvisited = []
for node in find_k_closest(key):
if node not in visited:
unvisited.append(node)
if value_found: # Handling the output statements for the found value, if it is found/not
print("Found value \"{}\" for key {}".format(value, key))
else:
print("Could not find key {}".format(key))
print("After FIND_VALUE command, k-buckets are:\n" + print_buckets())
def Store(self, request, context):
global bucket
global local_key
global local_value
# store nodes directly at server node
local_key = request.key
local_value = request.value
print('Storing key {} value "{}"'.format(request.key,request.value))
sys.stdout.flush()
if request.node.id==1000:
request.node.id = 0
index = locateBucket(local_node,request.node.id)
if index!=-1:
if index not in bucket:
tmp = []
tmp.append(nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
else:
n = len(bucket[index])
if n==bucket_max:
tmp = bucket[index]
tmp.pop()
tmp.insert(0,nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
return csci4220_hw3_pb2.IDKey(node=tmp,idkey=request.key)
def handle_find_value(self, key):
#reserve the visited node id
if key in self.library:
print("Found data \"%s\" for key %d" % (self.library[key], key))
return
visited = set()
visited.add(self.id)
k_nodes = deque(self.find_nearest_k(key, self.k))
# change to while with pop
while k_nodes:
node = k_nodes.popleft()
if not node.id in visited:
with grpc.insecure_channel(node.address + ":" +
str(node.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
IDKey = csci4220_hw3_pb2.IDKey(node=cur_node, idkey=key)
res = stub.FindValue(IDKey)
visited.add(node.id)
# contain key value or nodes
if res.mode_kv:
print("Found value \"%s\" for key %d" %
(res.kv.value, key))
if res.responding_node != self.id:
self.update_k_bucket(node)
return
else:
for n in res.nodes:
if not n.id in visited:
k_nodes.append(n)
print("Could not find key %d" % (key))
def Store(self, request, context):
print("Storing key %d value \"%s\"" % (request.key, request.value))
self.library[request.key] = request.value
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
IDKey = csci4220_hw3_pb2.IDKey(node=cur_node, idkey=request.key)
return IDKey
def FindNode(self, request, context):
node = request.node
id_key = request.idkey
print("Serving FindNode({}) request for {}".format(id_key, node.id))
closest_nodes = find_k_closest(id_key)
save_node(node)
return csci4220_hw3_pb2.NodeList(responding_node=csci4220_hw3_pb2.Node(
id=local_id, port=int(my_port), address=my_address),
nodes=closest_nodes)
def find_nearest_k(self, key, k, filter=[]):
res = {}
lst = []
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
lst.append((cur_node, self.compute_dist(self.id, key)))
for i in range(4):
for k_i in self.k_bucket[i]:
v = self.k_bucket[i][k_i]
lst.append((v, self.compute_dist(v.id, key)))
sorted_lst = sorted(lst, key=lambda x: (x[1], x[0].id ^ key))
nodes = [i[0] for i in sorted_lst if not i[0].id in filter]
return nodes[:k]
def Quit(self, request, context):
quit_id = request.idkey
i = self.compute_dist(self.id, quit_id) - 1
flag = True
if quit_id in self.k_bucket[i]:
del self.k_bucket[i][quit_id]
print("Evicting quitting node %d from bucket %i" % (quit_id, i))
else:
print("No record of quitting node %d in k-buckets." % quit_id)
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
IDKey = csci4220_hw3_pb2.IDKey(node=cur_node, idkey=quit_id)
return IDKey
def FindValue(self, request, context):
print("Serving FindKey(%d) request for %d" %
(request.idkey, request.node.id))
key = request.idkey
if self.id != request.node.id:
self.update_k_bucket(request.node)
if key in self.library:
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
key_value = csci4220_hw3_pb2.KeyValue(node=cur_node,
key=key,
value=self.library[key])
kv_wrapper = csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node=cur_node, mode_kv=1, kv=key_value, nodes=[])
return kv_wrapper
else:
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
k_nodes = self.find_nearest_k(request.idkey, self.k)
kv_wrapper = csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node=cur_node, mode_kv=0, kv=None, nodes=k_nodes)
return kv_wrapper
def FindNode(self, request, context):
print("Serving FindNode(%d) request for %d" %
(request.idkey, request.node.id))
#update key buckets:
if self.id != request.node.id:
i = self.compute_dist(self.id, request.node.id) - 1
self.k_bucket[i][request.node.id] = request.node
k_nodes = self.find_nearest_k(request.idkey, self.k,
[request.node.id, self.id])
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
nl = csci4220_hw3_pb2.NodeList(responding_node=cur_node, nodes=k_nodes)
return nl
def run():
global local_node
global k
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(
sys.argv[0]))
sys.exit(-1)
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(
my_hostname) # Gets my IP address from my hostname
local_node = csci4220_hw3_pb2.Node(id=local_id,
port=int(my_port),
address=my_address)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
csci4220_hw3_pb2_grpc.add_KadImplServicer_to_server(
KadImplServicer(), server)
server.add_insecure_port('[::]:' + my_port)
server.start()
while (1):
inp = input().split()
if inp[0] == "QUIT":
Quit()
print("Shut down node %d" % local_id)
sys.exit(0)
elif inp[0] == "BOOTSTRAP" and (len(inp) == 3):
Bootstrap(inp[1], inp[2])
elif inp[0] == "FIND_NODE" and (len(inp) == 2):
print("Before FIND_NODE command, k-buckets are:")
print_k_buckets()
Find_Node(int(inp[1]))
print("After FIND_NODE command, k-buckets are:")
print_k_buckets()
elif inp[0] == "FIND_VALUE" and (len(inp) == 2):
print("Before FIND_VALUE command, k-buckets are:")
print_k_buckets()
Find_Value(int(inp[1]))
print("After FIND_VALUE command, k-buckets are:")
print_k_buckets()
elif inp[0] == "STORE" and (len(inp) == 3):
Store(int(inp[1]), inp[2])
else:
print("Invalid arguement!")
def __init__(self, id, port, addr, k):
self.id = id
self.port = port
self.addr = addr
self.k = k
self.node = csci4220_hw3_pb2.Node(id=self.id,
port=self.port,
address=self.addr)
'''
Least Recently Used (LRU) List
index 0 <-- list --> index -1
least recently used most recently used
'''
# k_buckets have a fixed size of four
self.k_buckets = [[], [], [], []] # Store Node(id, port, address)
self.key_value = [] # Store tuple(key, value)
def handle_store(self, key, value):
near_point = self.find_nearest_k(key, 1)[0]
if near_point.id == self.id:
self.library[key] = value
print("Storing key %d at node %d" % (key, self.id))
else:
with grpc.insecure_channel(near_point.address + ":" +
str(near_point.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
kv = csci4220_hw3_pb2.KeyValue(node=cur_node,
key=key,
value=value)
print("Storing key %d at node %d" % (key, near_point.id))
res = stub.Store(kv)
def broadcast(self, remote_hostname, remote_port):
with grpc.insecure_channel(remote_hostname + ":" +
str(remote_port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
IDKey = csci4220_hw3_pb2.IDKey(node=cur_node, idkey=self.id)
# run the remote FindNode
res = stub.FindNode(IDKey)
#update the k_bucket
self.update_k_bucket(res.responding_node)
for new_node in res.nodes:
self.update_k_bucket(new_node)
#print k_bucket
print("After BOOTSTRAP(%d), k_buckets now look like:" %
res.responding_node.id)
self.print_k_bucket()
def handle_quit(self):
for i in range(4):
for k in self.k_bucket[i]:
v = self.k_bucket[i][k]
print("Letting %d know I'm quitting." % (v.id))
try:
with grpc.insecure_channel(v.address + ":" +
str(v.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
cur_node = csci4220_hw3_pb2.Node(id=self.id,
port=int(self.port),
address=self.address)
IDKey = csci4220_hw3_pb2.IDKey(node=cur_node,
idkey=self.id)
stub.Quit(IDKey)
except Exception as e:
pass
print("Shut down node %d" % self.id)
def store(args):
key = int(args.split()[1])
value = args.split()[2]
k_closest = find_k_closest(key)
closest_node = None if len(k_closest) < 1 else k_closest[0]
if closest_node is None or key ^ local_id < key ^ closest_node.id:
print("Storing key {} at node {}".format(key, local_id))
hash_table.put(key, value)
else:
print("Storing key {} at node {}".format(key, closest_node.id))
channel = grpc.insecure_channel("{}:{}".format(closest_node.address,
closest_node.port))
kad = csci4220_hw3_pb2_grpc.KadImplStub(channel)
kad.Store(
csci4220_hw3_pb2.KeyValue(node=csci4220_hw3_pb2.Node(
id=local_id, port=int(my_port), address=my_address),
key=key,
value=value))
def FindValue(self, request, context):
print("Serving FindKey({}) request for {}".format(
request.idkey, request.node.id))
has_key = hash_table.contains_key(request.idkey)
value = ""
nodes = []
if has_key:
value = hash_table.get(request.idkey)
else:
nodes = find_k_closest(request.idkey)
save_node(request.node)
return csci4220_hw3_pb2.KV_Node_Wrapper(
responding_node=csci4220_hw3_pb2.Node(id=local_id,
port=int(my_port),
address=my_address),
mode_kv=has_key,
kv=csci4220_hw3_pb2.KeyValue(node=request.node,
key=request.idkey,
value=value),
nodes=nodes)
def Quit(self, request, context):
global bucket
Found = False
if request.node.id==1000:
request.node.id = 0
# loop over to find client node and delete it from bucket
for k,v in list(bucket.items()):
mark = -1
for i in range(0,len(v)):
if v[i].id==request.idkey:
mark = i
if len(v)>1:
Found = True
print("Evicting quitting node {} from bucket {}".format(request.idkey,k))
sys.stdout.flush()
if mark!=-1 and Found==True:
v.remove(mark)
for k,v in list(bucket.items()):
for i in range(0,len(v)):
if v[i].id==request.idkey:
if len(v)==1:
Found = True
del bucket[k]
print("Evicting quitting node {} from bucket {}".format(request.idkey,k))
sys.stdout.flush()
if Found==False:
print("No record of quitting node {} in k-buckets.".format(request.node.id))
sys.stdout.flush()
r_node = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
return csci4220_hw3_pb2.IDKey(node=r_node,idkey=local_node)
def bootstrap(args):
hostname = args.split()[1] # Gettting the hostname from input
port = args.split()[2] # Getting the port from input
address = socket.gethostbyname(hostname) # Getting the hostname
channel = grpc.insecure_channel("{}:{}".format(address, port))
kad = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Get NodeList from remote node to update k_buckets
response = kad.FindNode(
csci4220_hw3_pb2.IDKey(node=csci4220_hw3_pb2.Node(id=local_id,
port=int(my_port),
address=my_address),
idkey=local_id))
# Save bootstrap node
save_node(response.responding_node)
# Save nodes learned from bootstrap node
for node in response.nodes:
save_node(node)
print("After BOOTSTRAP({}), k_buckets now look like:\n{}".format(
response.responding_node.id, print_buckets()))
def findingNode(find_node_id):
visited = []
Found = False
print("Before FIND_NODE command, k-buckets are:")
sys.stdout.flush()
for i in range(0,N):
tmp = str(i)+":"
for k,v in bucket.items():
if k==i and len(v)!=0:
for m in v:
tmp += " "
tmp += str(m.id)
tmp += ":"
tmp += str(m.port)
print(tmp)
sys.stdout.flush()
all_nodes = []
for k,v in bucket.items():
for j in range(len(v)):
if v[j].id==find_node_id:
Found = True
index = locateBucket(find_node_id,local_node)
tmp = bucket[index]
x = v[j]
tmp.remove(x)
tmp.insert(0, x)
print("Found destination id " + str(find_node_id))
sys.stdout.flush()
print("After FIND_NODE command, k-buckets are:")
sys.stdout.flush()
for i in range(0,N):
tmp = str(i)+":"
for k,v in bucket.items():
if k==i and len(v)!=0:
for m in v:
tmp += " "
tmp += str(m.id)
tmp += ":"
tmp += str(m.port)
print(tmp)
sys.stdout.flush()
return
else:
all_nodes.append(v[j])
ordered_nodes = sorted(all_nodes, key=lambda x : XOR(x.id,find_node_id))
S = []
S_prime = []
if len(ordered_nodes)<bucket_max:
S = ordered_nodes
else:
for i in range(0,bucket_max):
S.append(ordered_nodes[i])
for x in S:
add = True
for v_node in visited:
if x==v_node:
add = False
if add==True:
S_prime.append(x)
for search_node in S_prime:
if search_node.id==find_node_id:
Found = True
print("Found destination id " + str(find_node_id))
break
visited.append(search_node.id)
remote_host = search_node.id
remote_port = int(search_node.port)
remote_addr = search_node.address
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
node_tmp = csci4220_hw3_pb2.Node(id=find_node_id,port=0,address="")
idKey = csci4220_hw3_pb2.IDKey(node=node_tmp,idkey=local_node)
nodeList = stub.FindNode(idKey)
R = []
for node in nodeList.nodes:
R.append(node)
mr_used = search_node
mr_index = locateBucket(local_node,mr_used.id)
for x in R:
if x.id==1000:
x.id = 0
if x.id==find_node_id:
Found = True
print("Found destination id " + str(find_node_id))
index = locateBucket(local_node,x.id)
update = True
for k,v in bucket.items():
if k==index:
for i in range(0,len(v)):
if v[i].id==mr_used.id or v[i].id==local_node:
update==False
for j in range(0,len(v)):
if v[i].id==v[j].id:
update==False
if update==True:
mr_used = x
if index not in bucket:
tmp = []
tmp.append(x)
bucket[index] = tmp
else:
tmp = bucket[index]
tmp.append(x)
n = len(tmp)
if n>bucket_max:
tmp.pop(0)
bucket[index] = tmp
if mr_index in bucket.values():
bucket[mr_index].remove(mr_used)
bucket[mr_index].insert(0,mr_used)
channel.close()
print("After FIND_NODE command, k-buckets are:")
sys.stdout.flush()
for i in range(0,N):
tmp = str(i)+":"
for k,v in bucket.items():
if k==i and len(v)!=0:
for m in v:
tmp += " "
tmp += str(m.id)
tmp += ":"
tmp += str(m.port)
print(tmp)
sys.stdout.flush()
if Found==False:
print("Could not find destination id " + str(find_node_id))
sys.stdout.flush()
def bootStrap(stub,remote_host,remote_port):
global bucket_max
# locate other side send connection
node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
idKey = csci4220_hw3_pb2.IDKey(node=node_tmp,idkey=local_node)
nodeList = stub.FindNode(idKey)
to_bucket = []
to_bucket.append(nodeList.responding_node)
for node in nodeList.nodes:
to_bucket.append(node)
# update bucket
for x in to_bucket:
if x.id==1000:
x.id = 0
add = True
for item in bucket.items():
for node in item[1]:
if x.id==node.id:
add = False
if add==True:
index = locateBucket(local_node,x.id)
if index!=-1:
if index not in bucket:
tmp = []
obj = nodeObj(x.id,x.port,x.address)
tmp.append(obj)
bucket[index] = tmp
else:
n = len(bucket[index])
tmp = bucket[index]
if n==bucket_max:
l = tmp[-1]
tmp[-1] = tmp[0]
tmp[0] = l
tmp.pop()
obj = nodeObj(x.id,x.port,x.address)
tmp.append(obj)
bucket[index] = tmp
# outprint
print("After BOOTSTRAP(" + str(remote_host) + "), k_buckets now look like:")
sys.stdout.flush()
for i in range(0,N):
tmp = str(i)+":"
for k,v in bucket.items():
if k==i and len(v)!=0:
for m in v:
tmp += " "
tmp += str(m.id)
tmp += ":"
tmp += str(m.port)
print(tmp)
sys.stdout.flush()
def FindValue(self, request, context):
global bucket
# delete same nodes
for k,v in list(bucket.items()):
for i in range(0,len(v)):
if v[i].id==local_node:
if len(v)==1:
del bucket[k]
else:
v.remove(i)
print("Serving FindKey(" + str(request.idkey) + ") request for " + str(request.node.id))
sys.stdout.flush()
# if found the key in server node, return the key directly
if request.idkey==local_key:
r_node = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
mode = True
key_value = csci4220_hw3_pb2.KeyValue(node=r_node,key=local_key,value=local_value)
return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=r_node,mode_kv=mode,kv=key_value,nodes=[])
# if not found, return a k-near list of nodes
else:
mode = False
index = locateBucket(local_node,request.node.id)
if index!=-1:
if index not in bucket:
tmp = []
tmp.append(nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
else:
adding = True
n = len(bucket[index])
for i in range(0,n):
if bucket[index][i].id==request.node.id:
adding = False
break
if adding==True:
if n==bucket_max:
tmp = bucket[index]
tmp.pop()
tmp.insert(0,nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
all_nodes = []
for k,v in bucket.items():
for j in range(len(v)):
all_nodes.append(v[j])
ordered_nodes = sorted(all_nodes, key=lambda x : XOR(x.id,request.node.id))
node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
ret = []
for node in ordered_nodes:
if node.id==0:
tmp = csci4220_hw3_pb2.Node(id=1000,port=node.port,address=node.address)
else:
tmp = csci4220_hw3_pb2.Node(id=node.id,port=node.port,address=node.address)
ret.append(tmp)
r_node = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
key_value = csci4220_hw3_pb2.KeyValue(node=r_node,key=local_key,value=local_value)
return csci4220_hw3_pb2.KV_Node_Wrapper(responding_node=r_node,mode_kv=mode,kv=key_value,nodes=ret)
if XOR(v[i].id,key)<=min_distrance:
min_distrance = XOR(v[i].id,key)
target = v[i]
tmp_id = target.id
if target.id==local_id:
local_key = key
local_value = value
print("Storing key " + str(key) + " at node " + str(local_id))
sys.stdout.flush()
else:
if target.id==0:
tmp_id = 1000
node = csci4220_hw3_pb2.Node(id=tmp_id,port=target.port,address=target.address)
ky = csci4220_hw3_pb2.KeyValue(node=node,key=key,value=value)
with grpc.insecure_channel(target.address + ':' + str(target.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
stub.Store(ky)
print("Storing key " + str(key) + " at node " + str(target.id))
sys.stdout.flush()
channel.close()
elif msg.split(" ")[0]=='QUIT':
if already_quit==False:
for k,v in list(bucket.items()):
for i in range(0,len(v)):
if v[i].id!=local_node:
def run():
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(sys.argv[0]))
sys.exit(-1)
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(my_hostname) # Gets my IP address from my hostname
''' Use the following code to convert a hostname to an IP and start a channel
Note that every stub needs a channel attached to it
When you are done with a channel you should call .close() on the channel.
Submitty may kill your program if you have too many file descriptors open
at the same time. '''
#remote_addr = socket.gethostbyname(remote_addr_string)
#remote_port = int(remote_port_string)
#channel = grpc.insecure_channel(remote_addr + ':' + str(remote_port))
ourNode = csci4220_hw3_pb2.Node(id = local_id, port = int(my_port), address = my_address)
ourServicer = KadImplServicer(k, ourNode)
# Code to start the servicer for our node
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
csci4220_hw3_pb2_grpc.add_KadImplServicer_to_server(ourServicer, server)
server.add_insecure_port(f'[::]:{my_port}')
server.start() # Let servicer start listening w/t blocking
while True:
userInput = input() # Perhaps change this to select, but that's a discussion for later :P
userInput = userInput.strip().split()
if ((userInput[0]).lower().strip() == "bootstrap"):
remoteHost = (userInput[1]).strip()
remotePort = (userInput[2]).strip()
remote_addr = socket.gethostbyname(remoteHost)
remote_port = int(remotePort)
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
result = stub.FindNode(csci4220_hw3_pb2.IDKey(node = ourNode, idkey = ourNode.id))
remoteID = result.responding_node.id]
ourServicer.DHT.updateNodeLoc(result.responding_node)
for node_ in result.nodes:
ourServicer.DHT.updateNodeLoc(node_)
print(f'After BOOTSTRAP({remoteID}), k_buckets now look like:')
ourServicer.DHT.print()
elif ((userInput[0]).lower().strip() == "find_node" or (userInput[0]).lower().strip() == "find_value"):
nodeID = int((userInput[1]).strip())
findNode = (userInput[0]).lower().strip() == "find_node"
print(f"Before FIND_{'NODE' if findNode else 'VALUE'} command, k-buckets are:")
ourServicer.DHT.print()
found = nodeID == local_id if findNode else ourServicer.values.get(nodeID) is not None
value = (found, ourServicer.values.get(nodeID))
if (not found):
# Do the search
Visited = set() # Stores id of visited nodes
nodeOfInterest = csci4220_hw3_pb2.IDKey(node = ourNode, idkey = nodeID)
# Condition for us to use FindNode or FindValue
# S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest to idkey
S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest nodes to idkey
# print(f"Closest nodes to {nodeOfInterest.idkey} are {[n.id for n in S]}")
notVisited = [n for n in S if n.id not in Visited]
while not found and len(notVisited) > 0:
for node in notVisited:
with grpc.insecure_channel(node.address + ':' + str(node.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Condition for us to use FindNode or FindValue
R = stub.FindNode(nodeOfInterest) if findNode else stub.FindValue(nodeOfInterest)
# Update our k-buckets with node
ourServicer.DHT.updateNodeLoc(R.responding_node)
if not findNode and R.mode_kv: # If we're for FindValue
found = True
value = (R.responding_node.id == ourNode.id, R.kv.value)
# If a node in R was already in a k-bucket, its position does not change.
# If it was not in the bucket yet, then it is added as the most recently used in that bucket.
# This _may_ kick out the node from above.
for n in R.nodes:
if findNode and (n.id ^ nodeOfInterest.idkey) == 0:
found = True
ourServicer.DHT.updateNodeLoc(n)
Visited.add(node.id)
if found:
break
S = ourServicer.DHT.kClosest(nodeOfInterest.idkey) # get k closest nodes to idkey
notVisited = [n for n in S if n.id not in Visited]
# Do the printing
if (found):
if findNode:
print(f'Found destination id {nodeID}', flush=True)
else:
if value[0]:
print(f'Found data "{value[1]}" for key {nodeID}')
else:
print(f'Found value "{value[1]}" for key {nodeID}')
else:
if findNode:
print(f'Could not find destination id {nodeID}', flush=True)
else:
print(f'Could not find key {nodeID}')
print(f"After FIND_{'NODE' if findNode else 'VALUE'} command, k-buckets are:")
ourServicer.DHT.print()
elif ((userInput[0]).lower().strip() == "store"):
key = int(userInput[1].strip())
val = userInput[2].strip()
# Find the node it knows that's closest to the key
closestNodes = ourServicer.DHT.kClosest(key)
if len(closestNodes) == 0 or (ourNode.id ^ key <= closestNodes[0].id ^ key):
print(f'Storing key {key} at node {ourNode.id}', flush=True)
ourServicer.values[key] = val
else:
print(f'Storing key {key} at node {closestNodes[0].id}', flush=True)
with grpc.insecure_channel(closestNodes[0].address + ':' + str(closestNodes[0].port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
stub.Store(csci4220_hw3_pb2.KeyValue(node = ourNode,
key = key,
value = val))
elif ((userInput[0]).lower().strip() == "quit"):
for node in ourServicer.DHT.getAllNodes():
with grpc.insecure_channel(node.address + ':' + str(node.port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
print(f'Letting {node.id} know I\'m quitting.', flush=True)
try:
stub.Quit(csci4220_hw3_pb2.IDKey(node = ourNode, idkey = ourNode.id))
except:
pass # The stub already closed
print(f'Shut down node {ourNode.id}', flush=True)
server.stop(None)
break
else:
print("Invalid Command", flush=True)
def findingValue(find_key):
visited = []
Found = False
print("Before FIND_VALUE command, k-buckets are:")
sys.stdout.flush()
for i in range(0,N):
tmp = str(i)+":"
for k,v in bucket.items():
if k==i and len(v)!=0:
for m in v:
tmp += " "
tmp += str(m.id)
tmp += ":"
tmp += str(m.port)
print(tmp)
sys.stdout.flush()
all_nodes = []
for k,v in bucket.items():
for j in range(len(v)):
if v[j].id!=local_node:
all_nodes.append(v[j])
ordered_nodes = sorted(all_nodes, key=lambda x : XOR(x.id,local_node))
S = []
S_prime = []
if len(ordered_nodes)<bucket_max:
S = ordered_nodes
else:
for i in range(0,bucket_max):
S.append(ordered_nodes[i])
for x in S:
add = True
for v_node in visited:
if x==v_node:
add = False
if add==True:
S_prime.append(x)
for search_node in S_prime:
visited.append(search_node.id)
remote_host = search_node.id
remote_port = int(search_node.port)
remote_addr = search_node.address
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
idKey = csci4220_hw3_pb2.IDKey(node=node_tmp,idkey=find_key)
ret = stub.FindValue(idKey)
if ret.mode_kv==True:
Found = True
tmp_p = '"'+ret.kv.value+'"'
print("Found value ""{}"" for key {}".format(tmp_p,find_key))
mr_used = ret.responding_node
mr_index = locateBucket(local_node,mr_used.id)
if mr_index not in bucket:
tmp = []
tmp.append(mr_used)
bucket[mr_index] = tmp
else:
adding = True
for y in bucket[mr_index]:
if y.id==mr_used.id and len(bucket[mr_index])==1:
adding = False
elif y.id==mr_used.id and len(bucket[mr_index])==bucket_max and bucket[mr_index][-1].id==mr_used.id:
adding = False
if adding==True:
tmp = bucket[mr_index]
tmp.pop(0)
tmp.append(mr_used)
bucket[mr_index] = tmp
break
elif ret.mode_kv==False:
R = []
for node in ret.nodes:
R.append(node)
mr_used = search_node
mr_index = locateBucket(local_node,mr_used.id)
for x in R:
if x.id==1000:
x.id = 0
if x.id==find_key:
Found = True
remote_host = x.id
remote_port = int(x.port)
remote_addr = x.address
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
idKey = csci4220_hw3_pb2.IDKey(node=node_tmp,idkey=find_key)
result = stub.FindValue(idKey)
tmp_p = '"'+result.kv.value+'"'
print("Found value ""{}"" for key {}".format(tmp_p,find_key))
index = locateBucket(local_node,x.id)
update = True
for k,v in bucket.items():
if k==index:
for i in range(0,len(v)):
if v[i].id==mr_used.id or v[i].id==local_node:
update==False
if update==True:
if index not in bucket:
tmp = []
tmp.append(x)
bucket[index] = tmp
else:
adding = True
for y in bucket[index]:
if y.id==mr_used.id:
adding = False
if adding==True:
tmp = bucket[index]
n = len(tmp)
if n<bucket_max:
tmp.insert(0,x)
else:
tmp.insert(0,x)
tmp.pop()
# print(tmp)
bucket[index] = tmp
if mr_index in bucket.values():
bucket[mr_index].remove(mr_used)
bucket[mr_index].insert(0,mr_used)
channel.close()
if Found==False:
print("Could not find key " + str(find_key))
sys.stdout.flush()
print("After FIND_VALUE command, k-buckets are:")
sys.stdout.flush()
for i in range(0,N):
tmp = str(i)+":"
for k,v in bucket.items():
if k==i and len(v)!=0:
for m in v:
tmp += " "
tmp += str(m.id)
tmp += ":"
tmp += str(m.port)
print(tmp)
sys.stdout.flush()
def FindNode(self, request, context):
global bucket
# Findnode function for bootstrap
if request.idkey==request.node.id:
index = locateBucket(local_node,request.node.id)
if index!=-1:
if index not in bucket:
tmp = []
tmp.append(nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
else:
adding = True
for k,v in bucket.items():
for i in range(0,len(v)):
if v[i].id==request.node.id:
adding = False
if adding==True:
n = len(bucket[index])
tmp = bucket[index]
if n==bucket_max:
tmp.pop()
tmp.insert(0,nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
# add all nodes to a list and sort
all_nodes = []
for k,v in bucket.items():
for j in range(len(v)):
if v[j].id!=request.node.id:
all_nodes.append(v[j])
ordered_nodes = sorted(all_nodes, key=lambda x : XOR(x.id,request.node.id))
node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
ret = []
# update bucket
i = 0
for node in ordered_nodes:
i += 1
if i>bucket_max:
break
else:
if node.id==0:
tmp = csci4220_hw3_pb2.Node(id=1000,port=node.port,address=node.address)
ret.append(tmp)
else:
tmp = csci4220_hw3_pb2.Node(id=node.id,port=node.port,address=node.address)
ret.append(tmp)
print("Serving FindNode(" + str(request.node.id) + ") request for " + str(request.idkey))
sys.stdout.flush()
return csci4220_hw3_pb2.NodeList(responding_node=node_tmp,nodes=ret)
# Findnode function for find node
else:
index = locateBucket(local_node,request.node.id)
if index!=-1:
if index not in bucket:
tmp = []
tmp.append(nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
else:
adding = True
for k,v in bucket.items():
for i in range(0,len(v)):
if v[i].id==request.node.id:
adding = False
if adding==True:
n = len(bucket[index])
tmp = bucket[index]
if n==bucket_max:
tmp.pop()
tmp.insert(0,nodeObj(request.node.id,request.node.port,request.node.address))
bucket[index] = tmp
all_nodes = []
for k,v in bucket.items():
for j in range(len(v)):
all_nodes.append(v[j])
ordered_nodes = sorted(all_nodes, key=lambda x : XOR(x.id,request.node.id))
node_tmp = csci4220_hw3_pb2.Node(id=local_node,port=local_port,address=local_address)
ret = []
# update bucket
for node in ordered_nodes:
if node.id!=request.idkey:
if node.id==0:
tmp = csci4220_hw3_pb2.Node(id=1000,port=node.port,address=node.address)
ret.append(tmp)
else:
tmp = csci4220_hw3_pb2.Node(id=node.id,port=node.port,address=node.address)
ret.append(tmp)
print("Serving FindNode(" + str(request.node.id) + ") request for " + str(request.idkey))
sys.stdout.flush()
return csci4220_hw3_pb2.NodeList(responding_node=node_tmp,nodes=ret)
def run():
'''
Check the # of command-line arguments, which should have
hw3.py <nodeID> <port> <k>
'''
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(
sys.argv[0]))
sys.exit(-1)
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(
my_hostname) # Gets my IP address from my hostname
# print("[myself] Running as {0} @ {1}".format(my_hostname,my_address))
# Setting up the server
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
servicer = KadImplServicer(local_id, int(my_port), my_address, k)
csci4220_hw3_pb2_grpc.add_KadImplServicer_to_server(servicer, server)
server.add_insecure_port("[::]:" + my_port)
server.start()
# Run the client
while True:
inputs = input()
arguments = inputs.split()
if arguments[0] == "BOOTSTRAP":
remote_hostname = arguments[1]
my_port = int(arguments[2])
remote_addr = socket.gethostbyname(remote_hostname)
# Connect to other peer
channel = grpc.insecure_channel(
str(remote_addr) + ':' + str(my_port))
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Find peer node using servicer node
nodee = stub.FindNode(
csci4220_hw3_pb2.IDKey(node=servicer.node, idkey=local_id))
# Add node that was responding
res_node = nodee.responding_node
bit_len = ((res_node.id) ^ local_id).bit_length()
bit_len -= 1
if len(servicer.k_buckets[bit_len]) == k:
servicer.k_buckets[bit_len].popleft()
servicer.k_buckets[bit_len].append(res_node)
# Print bucket contents
print('After BOOTSTRAP({}), k_buckets now look like:'.format(
str(res_node.id)))
servicer.print_k_buckets()
channel.close()
elif arguments[0] == "FIND_NODE":
# Load the inputs argument
node_id = int(arguments[1])
print("Before FIND_NODE command, k-buckets are:")
servicer.print_k_buckets()
S = servicer.find_k_closest_nodes(node_id)
S_no = S # Nodes in S that have not been contacted yet
S_yes = [] # Nodes that have been contacted
found = False
while S_no and (not found):
for no in S_no:
# Start a channel
channel = grpc.insecure_channel(no.address + ':' +
str(no.port))
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Send a FindNode RPC
R = stub.FindNode(
csci4220_hw3_pb2.IDKey(node=servicer.node,
idkey=node_id))
# Update the k-buckets
servicer.update_k_buckets(no, True)
for node in R.nodes:
servicer.update_k_buckets(node, False)
# Update S_yes
S_yes.append(no)
# Close the channel
channel.close()
# Check if the node has been found
for bucket in servicer.k_buckets:
if any(node.id == node_id for node in bucket):
found = True
print("Found destination id {}".format(node_id))
break
if not found:
# Update S and S_no
S = servicer.find_k_closest_nodes(node_id)
S_no.clear()
for node in S:
if not any(yes.id == node.id for yes in S_yes):
S_no.append(node)
if not found:
print("Could not find destination id {}".format(node_id))
print("After FIND_NODE command, k-buckets are:")
servicer.print_k_buckets()
elif arguments[0] == "FIND_VALUE":
# Load the inputs argument
key = int(arguments[1])
print("Before FIND_VALUE command, k-buckets are:")
servicer.print_k_buckets()
found = False
# See if the key/value pair has already been stored
for pair in servicer.key_value:
if pair[0] == key:
found = True
print("Found data \"{}\" for key {}".format(pair[1], key))
break
if not found:
S = servicer.find_k_closest_nodes(key)
S_no = S # Nodes in S that have not been contacted yet
S_yes = [] # Nodes that have been contacted
while S_no and (not found):
for no in S_no:
# Start a channel
channel = grpc.insecure_channel(no.address + ':' +
str(no.port))
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
# Send a FindNode RPC
R = stub.FindValue(
csci4220_hw3_pb2.IDKey(node=servicer.node,
idkey=key))
# Update the k-buckets
servicer.update_k_buckets(no, True)
# Update S_yes
S_yes.append(no)
# Close the channel
channel.close()
# Check if the value is found
if R.mode_kv:
found = True
print("Found value \"{}\" for key {}".format(
R.kv.value, key))
break
else:
# Update the k-buckets
for node in R.nodes:
servicer.update_k_buckets(node, False)
if not found:
# Update S and S_no
S = servicer.find_k_closest_nodes(key)
S_no.clear()
for node in S:
if not any(yes.id == node.id for yes in S_yes):
S_no.append(node)
if not found:
print("Could not find key {}".format(key))
print("After FIND_VALUE command, k-buckets are:")
servicer.print_k_buckets()
elif arguments[0] == "STORE":
this_key = int(arguments[1])
this_value = arguments[2]
closest_node = csci4220_hw3_pb2.Node(id=local_id,
port=int(my_port),
address=str(my_address))
distance = abs(local_id - this_key)
for bucket in servicer.k_buckets:
for entry in bucket:
if abs(int(entry.id) - this_key) < distance:
closest_node = entry
distance = abs(int(entry.id) - this_key)
closest_port = int(closest_node.port)
#Connect to server & stub
this_addr = closest_node.address + ':' + str(closest_port)
channel = grpc.insecure_channel(this_addr)
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
some_idkey = stub.Store(
csci4220_hw3_pb2.KeyValue(node=servicer.node,
key=this_key,
value=this_value))
print("Storing key {} at node {}".format(this_key,
some_idkey.idkey))
channel.close()
elif arguments[0] == "QUIT":
# Tell everyone we are quitting, goodbye :(
for bucket in servicer.k_buckets:
for entry in bucket:
remote_host = str(entry.id)
remote_port = int(entry.port)
my_addr = entry.address + ':' + str(remote_port)
# Create channel and tell other peer we are quitting
channel = grpc.insecure_channel(my_addr)
stub = csci4220_hw3_pb2_grpc.KadImplStub(channel)
print("Letting " + remote_host + " know I'm quitting.")
some_idkey = stub.Quit(
csci4220_hw3_pb2.IDKey(node=servicer.node,
idkey=local_id))
channel.close()
print("Shut down node " + str(local_id))
break
else:
# Unknown command
print("Unknown command. Use BOOTSTRAP , FIND_VALUE , STORE , QUIT")
