def SendFindValue(self, target_key):
if target_key in self.data.keys():
print("Found data \""+ self.data[target_key]+ "\" for key " + str(target_key))
return
obj = csci4220_hw4_pb2.IDKey(node=csci4220_hw4_pb2.Node(id=self.my_id, port=int(self.my_port), address=self.my_address), idkey=target_key)
visited = []
visited.append(self.my_id)
# for all peers
to_sort = []
for item in self.k_buckets.items():
for peer in item[1]:
to_sort.append(peer)
# sort the peers based on distance to target_key
to_sort.sort(key=lambda node: (target_key ^ node.node_id).bit_length() -1)
# for all peers in order
for peer in to_sort:
with grpc.insecure_channel(peer.address + ":" + str(peer.port)) as channel:
# send find value command
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
R = stub.FindValue(obj)
visited.append(peer.node_id)
b = peer.node_id ^ self.my_id
b = b.bit_length() - 1
# update location of the peer
self.UpdateBucket(b, peer)
# if found
if (R.mode_kv == True):
print("Found value \"" + R.kv.value + "\" for key " + str(R.kv.key))
return
else:
for i in R.nodes:
if i.id not in visited:
b = self.my_id ^ i.id
b = b.bit_length() - 1
self.UpdateBucket(b, Node(i.address, i.port, i.id))
# search through the returned k nearest peers
with grpc.insecure_channel(i.address + ":" + str(i.port)) as to_ask:
stub = csci4220_hw4_pb2_grpc.KadImplStub(to_ask)
response = stub.FindValue(obj)
visited.append(i.id)
# if found in peer list
if (response.mode_kv == True):
print("Found value \"" + response.kv.value + "\" for key " + str(response.kv.key))
return
# if haven't found at this point, print that it couldn't be found
print("Could not find key " + str(target_key))
def SendBootstrap(self, peer_host, peer_port):
# temporarily connect with them to get
with grpc.insecure_channel(peer_host + ":" + peer_port) as channel:
# access the remote server
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
obj = csci4220_hw4_pb2.IDKey(node=csci4220_hw4_pb2.Node(id=self.my_id, port=int(self.my_port), address=self.my_address), idkey=self.my_id)
# node contains the return from FindNode
node = stub.FindNode(obj)
# get the bucket it should be stored in
bucket = self.my_id ^ node.responding_node.id
bucket = bucket.bit_length() - 1
# create a new node from the responding node and append it
n = Node(node.responding_node.address, node.responding_node.port, node.responding_node.id)
self.UpdateBucket(bucket,n)
# add all the nodes that were neighbors
for i in node.nodes:
b = self.my_id ^ i.id
b = b.bit_length() - 1
if (b >= 0):
self.UpdateBucket(b, Node(i.address, i.port, i.id))
print("After BOOTSTRAP(" + str(node.responding_node.id) + "), k_buckets now look like:")
self.PrintBuckets()
def rpcFindNode(recipient, requestedId, meNode):
remote_addr = recipient.address
remote_port = recipient.port
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
idKey = csci4220_hw4_pb2.IDKey(node = meNode, idkey = requestedId)
try:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
response = stub.FindNode(idKey)
except:
print("Try Failed in rpcFindNode")
return
return response.nodes
def quit(myId, meNode, k_buckets):
for lst in k_buckets:
for node in lst:
print("Letting {} know I'm quitting.".format(node.id))
with grpc.insecure_channel(node.address + ':' + str(node.port)) as channel:
try:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
request = csci4220_hw4_pb2.IDKey(node=meNode, idkey=myId)
response = stub.Quit(request)
except:
#print("Try Failed in quit")
pass
print("Shut down node {}".format(myId))
def bootstrap(remote_addr_string, remote_port_string, myId, meNode, k_buckets, k):
remote_addr = socket.gethostbyname(remote_addr_string)
remote_port = int(remote_port_string)
#set up RPC channel
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
#Set up stub, send message, get response
try:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
request = csci4220_hw4_pb2.IDKey(node=meNode, idkey=myId)
response = stub.FindNode(request)
except:
print("Try Failed in bootstrap")
return
#add responding node if not in kbuckets
responding_node = response.responding_node
found = False
for lst in k_buckets:
if responding_node in lst:
found = True
break
if not found:
addNode(k_buckets, responding_node, myId, k)
#add new nodes to kbuckets
nodeList = response.nodes
for node in nodeList:
if node.id == myId:
continue
found = False
for lst in k_buckets:
if node in lst:
found = True
break
if found:
continue
addNode(k_buckets, node, myId, k)
#make the responding node the most recently used
makeMostRecent(responding_node, k_buckets)
print("After BOOTSTRAP({}), k_buckets now look like:".format(
response.responding_node.id))
printBuckets(k_buckets)
def SendQuit(self):
obj = csci4220_hw4_pb2.IDKey(node=csci4220_hw4_pb2.Node(id=self.my_id, port=int(self.my_port), address=self.my_address), idkey=self.my_id)
# for all peers
for item in self.k_buckets.items():
for peer in item[1]:
#send a quit to all pears
with grpc.insecure_channel(peer.address + ":" + str(peer.port)) as channel:
print("Letting "+ str(peer.node_id)+" know I'm quitting.")
# send quit to them
try:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
ret = stub.Quit(obj)
# if quit does not connect, do nothing
# hacky way as of submitty 195
except:
pass
def SendFindNode(self, target_id):
obj = csci4220_hw4_pb2.IDKey(node=csci4220_hw4_pb2.Node(id=self.my_id, port=int(self.my_port), address=self.my_address),
idkey=target_id)
# keep track of visited
visited = []
for item in self.k_buckets.items():
bucket = item[1]
for index in range(len(bucket)):
# if already visited
if (bucket[index].node_id in visited):
print("alerady visited")
continue
# send the peer the find node command
with grpc.insecure_channel(bucket[index].address + ":" + str(bucket[index].port)) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
# make the request
R = stub.FindNode(obj)
# add this to visited
visited.append(bucket[index].node_id)
# update the position of this first node
self.UpdateBucket(item[0],bucket[index])
# update k buckets with node (move it to the front)
for i in R.nodes:
b = self.my_id ^ i.id
b = b.bit_length() - 1
# make sure not self
if (b >= 0):
found = False
for match in bucket:
if match.node_id == i.id:
found = True
# found it
if (target_id == i.id):
print("Found destination id " + str(target_id))
self.UpdateBucket(b, Node(i.address, i.port, i.id))
def store(key, value, k_buckets, k, meNode, storedDict):
minDist = meNode.id ^ key
minNode = meNode
# iterate over all known nodes to find node closest to the key
for bucket in k_buckets:
for node in bucket:
dist = node.id ^ key
if(dist < minDist):
minDist = dist
minNode = node
# if this node is the closest node, store the value locally
if(minNode.id == meNode.id):
print("Storing key %d at node %d" % (key, meNode.id))
storedDict[key] = value
return
print("Storing key %d at node %d" % (key, minNode.id))
#Set up RPC and send Store RPC to proper node
remote_addr = minNode.address
remote_port = minNode.port
with grpc.insecure_channel(remote_addr + ':' + str(remote_port)) as channel:
keyVal = csci4220_hw4_pb2.KeyValue(node = meNode, key = key, value = value)
try:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
response = stub.Store(keyVal)
makeMostRecent(minNode, k_buckets)
except:
print("Try Failed in store")
return
return
def run():
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(sys.argv[0]))
sys.exit(-1)
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(my_hostname) # Gets my IP address from my hostname
''' Use the following code to convert a hostname to an IP and start a channel
Note that every stub needs a channel attached to it
When you are done with a channel you should call .close() on the channel.
Submitty may kill your program if you have too many file descriptors open
at the same time.'''
# start the server in the background
hash_table = HashTable()
hash_table.my_port = int(my_port)
hash_table.my_id = local_id
hash_table.my_address = my_address
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
csci4220_hw4_pb2_grpc.add_KadImplServicer_to_server(hash_table, server)
server.add_insecure_port('[::]:'+my_port)
server.start()
# initializing the buckets 2^k
buckets = pow(2, k)
for i in range(buckets):
hash_table.k_buckets[i] = []
# main listening loop
while (True):
stdin = str(input())
arguments = stdin.split(" ")
# connect to another peer directly
if (arguments[0] == "BOOTSTRAP"):
peer_host = arguments[1]
peer_port = arguments[2]
hash_table.SendBootstrap(peer_host, peer_port)
# if the command is to store a value at a key
if (arguments[0] == "STORE"):
key = int(arguments[1])
value = arguments[2]
min_dist = key ^ hash_table.my_id
min_key = Node(hash_table.my_address, hash_table.my_port, hash_table.my_id)
# get the minimum distance peer to the key
for item in hash_table.k_buckets.items():
for peer in item[1]:
dist = key ^ peer.node_id
if dist < min_dist:
min_dist = dist
min_key = peer
print("Storing key " + str(key) + " at node " + str(min_key.node_id))
if (min_key.node_id == hash_table.my_id):
hash_table.data[key] = value
else:
# send the store command to that remote
with grpc.insecure_channel(min_key.address + ":" + str(min_key.port)) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
obj = csci4220_hw4_pb2.KeyValue(node=csci4220_hw4_pb2.Node(id=hash_table.my_id, port=int(hash_table.my_port), address=hash_table.my_address), key=key, value=value)
id_key = stub.Store(obj)
# command to find a node
if (arguments[0] == "FIND_NODE"):
target_id = int(arguments[1])
print("Before FIND_NODE command, k-buckets are:")
hash_table.PrintBuckets()
# perform searching and updating here
hash_table.SendFindNode(target_id)
print("After FIND_NODE command, k-buckets are:")
hash_table.PrintBuckets()
# command to find a value
if (arguments[0] == "FIND_VALUE"):
find_target = int(arguments[1])
print("Before FIND_VALUE command, k-buckets are:")
hash_table.PrintBuckets()
# perform search and updating here
hash_table.SendFindValue(find_target)
print("After FIND_VALUE command, k-buckets are:")
hash_table.PrintBuckets()
# command to quit and unregister self from pers
if (arguments[0] == "QUIT"):
# send quit to all peers
hash_table.SendQuit()
print("Shut down node " + str(local_id))
break
def run():
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(sys.argv[0]))
sys.exit(-1)
global val
global val_key
global buckets
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
#4 buckets needed
i = 4
while i > 0:
#append 4 empty deques into the bucket, the deques should contain Nodes
buckets.append(deque([]))
i -= 1
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(my_hostname) # Gets my IP address from my hostname
#don't ask, gRPC tutorial said so
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
csci4220_hw4_pb2_grpc.add_KadImplServicer_to_server(KadImplServicer(), server)
#listen from the port
server.add_insecure_port("127.0.0.1" + ':' + my_port)
server.start()
# Use the following code to convert a hostname to an IP and start a channel Note that every stub needs a channel attached to it When you are done with a channel you should call .close() on the channel. Submitty may kill your program if you have too many file descriptors open at the same time.
#remote_addr = socket.gethostbyname(my_hostname)
#remote_port = int(my_port)
#channel = grpc.insecure_channel(remote_addr + ':' + str(remote_port))
while True:
input_str = str(raw_input())
input_args = input_str.split()
if input_args[0] == "BOOTSTRAP":
#print("bootstrap")
remote_hostname = str(input_args[1])
remote_port = int(input_args[2])
remote_addr = socket.gethostbyname(remote_hostname)
#connect to server & create stub
channel = grpc.insecure_channel("127.0.0.1" + ':' + str(remote_port))
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
#create the node object
this_node = csci4220_hw4_pb2.Node(id = local_id, port = int(my_port), address = str(my_address))
#call FindNode
node_list = stub.FindNode(csci4220_hw4_pb2.IDKey(node = this_node, idkey = local_id))
#add nodes from the list in node_list
for node in node_list.nodes:
bit_len = ((node.id)^local_id).bit_length()
bit_len -= 1
#pop an element if the bucket is full
if len(buckets[bit_len]) == k:
buckets[bit_len].popleft()
buckets[bit_len].append(node)
#add the node that it just sent RPC to
r_node = node_list.responding_node
bit_len = ((r_node.id)^local_id).bit_length()
bit_len -= 1
if len(buckets[bit_len]) == k:
buckets[bit_len].popleft()
buckets[bit_len].append(r_node)
#done (hopefully)
print('After BOOTSTRAP({}), k_buckets now look like:'.format(str(r_node.id)))
count = 0
for bucket in buckets:
sys.stdout.write('{}:'.format(str(count)))
for entry in bucket:
sys.stdout.write(' {}:{}'.format(str(entry.id), str(entry.port)))
sys.stdout.write('\n')
count += 1
channel.close()
if input_args[0] == "STORE":
print("store")
this_key = int(input_args[1])
this_value = input_args[2]
closest_node = csci4220_hw4_pb2.Node(id = local_id, port = int(my_port), address = str(my_address))
distance = abs(local_id - this_key)
for bucket in buckets:
for entry in bucket:
if abs(int(entry.id) - this_key) < distance:
closest_node = entry
distance = abs(int(entry.id) - this_key)
remote_hostname = str(closest_node.id)
remote_port = int(closest_node.port)
remote_addr = socket.gethostbyname(remote_hostname)
#connect to server & create stub
this_addr = "127.0.0.1" + ':' + str(remote_port)
channel = grpc.insecure_channel(this_addr)
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
print(this_addr)
some_idkey = stub.Store(csci4220_hw4_pb2.KeyValue(node = None, key = this_key, value = this_value))
channel.close()
if input_args[0] == "QUIT":
for bucket in buckets:
for entry in bucket:
remote_hostname = str(entry.id)
remote_port = int(entry.port)
remote_addr = socket.gethostbyname(remote_hostname)
this_addr = "127.0.0.1" + ':' + str(remote_port)
channel = grpc.insecure_channel(this_addr)
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
print("Letting " + remote_hostname + " know I'm quitting.")
some_idkey = stub.Quit(csci4220_hw4_pb2.IDKey(node = None, idkey = local_id))
channel.close()
print("Shut down node " + str(local_id))
break
def run():
if len(sys.argv) != 4:
print("Error, correct usage is {} [my id] [my port] [k]".format(
sys.argv[0]))
sys.exit(-1)
local_id = int(sys.argv[1])
my_port = str(int(sys.argv[2])) # add_insecure_port() will want a string
k = int(sys.argv[3])
my_hostname = socket.gethostname() # Gets my host name
my_address = socket.gethostbyname(
my_hostname) # Gets my IP address from my hostname
# Create server
server = grpc.server(futures.ThreadPoolExecutor(max_workers=16))
servicer = KadImplServicer()
csci4220_hw4_pb2_grpc.add_KadImplServicer_to_server(servicer, server)
server.add_insecure_port('[::]:' + my_port)
server.start()
servicer.k = k
for i in range(n):
servicer.k_buckets.append([])
servicer.this_node = csci4220_hw4_pb2.Node(id=local_id,
port=int(my_port),
address=my_address)
# Listen for commands from standard input
while 1:
for line in sys.stdin:
command = line.split(" ")
for i in range(len(command)):
command[i] = command[i].strip()
# BOOTSTRAP command
if (command[0] == "BOOTSTRAP"):
remote_addr = socket.gethostbyname(command[1])
remote_port = int(command[2])
with grpc.insecure_channel(remote_addr + ':' +
str(remote_port)) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
nodeList = stub.FindNode(
csci4220_hw4_pb2.IDKey(node=servicer.this_node,
idkey=servicer.this_node.id))
k_buckets_add(servicer, nodeList.responding_node)
# Add nodes in nodeList to k_buckets
for node in nodeList.nodes:
k_buckets_add(servicer, node)
print("After BOOTSTRAP(" +
str(nodeList.responding_node.id) +
"), k_buckets now look like:")
print_k_buckets(servicer.k_buckets)
# FIND_NODE command
if (command[0] == "FIND_NODE"):
print("Before FIND_NODE command, k-buckets are:")
print_k_buckets(servicer.k_buckets)
key = int(command[1])
found = 0
if local_id == key:
print("Found destination id " + str(key))
else:
for i in range(n):
for j in range(len(servicer.k_buckets[i])):
if found == 0:
with grpc.insecure_channel(
servicer.k_buckets[i][j].address +
':' +
str(servicer.k_buckets[i][j].port)
) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(
channel)
nodelist = stub.FindNode(
csci4220_hw4_pb2.IDKey(
node=servicer.this_node,
idkey=key))
for node in nodelist.nodes:
k_buckets_add(servicer, node)
if node.id == key:
print("Found destination id " +
str(key))
found = 1
if found == 0:
print("Could not find destination id " + str(key))
print("After FIND_NODE command, k-buckets are:")
print_k_buckets(servicer.k_buckets)
# FIND_VALUE command
if (command[0] == "FIND_VALUE"):
print("Before FIND_VALUE command, k-buckets are:")
print_k_buckets(servicer.k_buckets)
key = int(command[1])
found = 0
# First check if key is stored locally
if key in servicer.values:
print('Found data "' + servicer.values[key] +
'" for key ' + str(key))
found = 1
else:
# Find node with id closest to key
closest = None
for i in range(n):
for j in range(len(servicer.k_buckets[i])):
if closest == None:
closest = servicer.k_buckets[i][j]
if (servicer.k_buckets[i][j].id
^ key) < (closest.id ^ key):
closest = servicer.k_buckets[i][j]
# Check if bucket is empty
if closest != None:
# Ask closest node for value
with grpc.insecure_channel(
closest.address + ':' +
str(closest.port)) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
kv_wrapper = stub.FindValue(
csci4220_hw4_pb2.IDKey(node=servicer.this_node,
idkey=key))
k_buckets_add(servicer, kv_wrapper.responding_node)
# If value was found for key
if kv_wrapper.mode_kv:
print('Found value "' + kv_wrapper.kv.value +
'" for key ' + str(key))
found = 1
else:
for node in kv_wrapper.nodes:
k_buckets_add(servicer, node)
# Correct node found, ask for value
if node.id == key:
with grpc.insecure_channel(
node.address + ':' +
str(node.port)) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(
channel)
kv_wrapper1 = stub.FindValue(
csci4220_hw4_pb2.IDKey(
node=servicer.this_node,
idkey=key))
if kv_wrapper1.mode_kv:
print('Found value "' +
kv_wrapper1.kv.value +
'" for key ' + str(key))
found = 1
if found == 0:
print("Could not find key " + str(key))
print("After FIND_VALUE command, k-buckets are:")
print_k_buckets(servicer.k_buckets)
# STORE command
if (command[0] == "STORE"):
key = int(command[1])
value = command[2]
# Find node with id closest to key
closest = servicer.this_node
for i in range(n):
for j in range(len(servicer.k_buckets[i])):
if (servicer.k_buckets[i][j].id ^ key) < (closest.id
^ key):
closest = servicer.k_buckets[i][j]
# Check if this is the closest node -> store locally
if closest.id == servicer.this_node.id:
servicer.values[key] = value
else:
# Send value to closest node
with grpc.insecure_channel(closest.address + ':' +
str(closest.port)) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
stub.Store(
csci4220_hw4_pb2.KeyValue(node=servicer.this_node,
key=key,
value=value))
print("Storing key " + str(key) + " at node " +
str(closest.id))
# QUIT command
if (command[0] == "QUIT"):
for i in reversed(range(n)):
for node in reversed(servicer.k_buckets[i]):
with grpc.insecure_channel(node.address + ':' +
str(node.port)) as channel:
stub = csci4220_hw4_pb2_grpc.KadImplStub(channel)
print("Letting " + str(node.id) +
" know I'm quitting.")
stub.Quit(
csci4220_hw4_pb2.IDKey(
node=servicer.this_node,
idkey=servicer.this_node.id))
print("Shut down node " + str(local_id))
sys.exit()