def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
# DELETE all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
global clients
clients = udp_clients
hash_codes = set()
# PUT all users.
print("begin to put users")
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put(key, u, data_bytes)
hash_codes.add(response)
print(f"PUT response: {response}")
# print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# TODO: PART I
# GET all users.
print("begin to get users")
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes)
# debug code, check response
# resStr = deserialize(response)
print(f"GET response: {response}")
# DELETE all users.
print("begin to delete users")
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = delete(key, data_bytes)
# debug code, check response
print(f"DELETE response: {response}")
# DELETE all users.
print("begin to delete users2")
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = delete(key, data_bytes)
# debug code, check response
# resStr = deserialize(response)
print(f"DELETE response2: {response}")
# GET all users again.
print("begin to get users2")
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes)
# debug code, check response
# resStr = deserialize(response)
print(f"GET response2: {response}")
def get(hc, udp_clients):
if BLOOM_FILTER.is_member(hc):
print(hc)
data_bytes, key = serialize_GET(hc)
fix_me_server_id = NODE_RING.get_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
return response
def process(udp_clients):
udp_client = udp_clients
hash_codes = set()
client_ring = NodeRing(udp_client)
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put(key, data_bytes, client_ring)
hash_codes.add(response)
print(hash_codes)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
for hc in hash_codes:
# print(hc)
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes, client_ring)
# print(response)
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
print("keys from DELETE OPERATAIONS:", data_bytes, key)
response = delete(key, data_bytes, client_ring)
print(response)
def process(udp_clients):
rhw_ring = RhwNodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
key = key.decode("utf-8")
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
key = key.decode("utf-8")
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
print(response)
def process(udp_clients):
client_ring = NodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
put(key, u)
server = client_ring.get_node(key)
response = udp_clients.send(data_bytes, server)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = get(key)
if not (response):
server = client_ring.get_node(key)
response = udp_clients.send(data_bytes, server)
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put([key, data_bytes, client_ring])
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users
for u in USERS:
data_bytes, key = serialize_GET(u)
response = get([key, data_bytes, client_ring])
print(response)
# DELETE all users.
for u in USERS:
data_bytes, key = serialize_DELETE(u)
response = delete([key, data_bytes, client_ring])
print(response)
def process(udp_clients):
ring = NodeRing(nodes=NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
# TODO: PART II - Instead of going to server 0, use Naive hashing to split data into multiple servers
#fix_me_server_id = 0
fix_me_server_id = NODES.index(ring.get_node(key))
print(f"Server id: {fix_me_server_id}")
response = udp_clients[fix_me_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# TODO: PART I
# GET all users.
for hc in hash_codes:
print(f"GET:{hc}")
data_bytes, key = serialize_GET(hc)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
# print(hc)
data_bytes, key = serialize_DELETE(hc)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
def process(udp_clients):
# client_ring = NodeRing(udp_clients)
# Consistent hash
client_ring = ConsistentHashNodeRing(udp_clients, 4)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
# RHW hash
# response = client_ring.get_node_RHW_hash(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
# RHW hash
# response = client_ring.get_node_RHW_hash(key).send(data_bytes)
print(response)
def process(udp_clients):
RH_ring = RHRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response.decode())
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].get_request(hc, data_bytes)
print(response)
# Delete all Users
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].delete(key, data_bytes)
print(response.decode())
def process():
node_ring = NodeRing()
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server, replication_server = node_ring.get_node(key)
response = UDPClient().send(data_bytes, server)
replication_response = UDPClient().send(data_bytes, replication_server)
print(response)
print(replication_response)
hash_codes.add(str(response.decode()))
hash_codes.add(str(replication_response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
server, replication_server = node_ring.get_node(key)
response = UDPClient().send(data_bytes, server)
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
get_ch_node = client_ring.get_ch_node(key)
response = get_ch_node.send(data_bytes)
replica = replica_map[get_ch_node]
print("Replicating in next node")
replica.send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
get_ch_node = client_ring.get_ch_node(key)
response = get_ch_node.send(data_bytes)
replica = replica_map[get_ch_node]
print("Response from corresponding node")
print(response)
print("Getting response from replica node")
rep_response = replica.send(data_bytes)
print(rep_response)
def process(udp_clients):
hash_codes = set()
for u in USERS:
data_bytes, key = serialize_PUT(u)
put(hash_codes, data_bytes, key)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
for hc in hash_codes:
data_bytes, key = serialize_GET(u)
print('ley: ' + str(key))
hash_code = hashlib.md5(str(key).encode())
hash_val = hash_code.hexdigest()
print("hash val: " + str(hash_val))
get(hash_val, data_bytes, hash_codes)
for hc in hash_codes:
data_bytes, key = serialize_DELETE(u)
delete(hc, data_bytes, key)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
print("Done!")
def process(udp_clients):
ch_ring = ChNodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(
server)) #ast.literal to convert string output to dict element
response = udp_clients[ch_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
print(response)
def wrapper(*args2):
three_params = True
keys = args2[0]
try:
data_bytes = args2[1]
hash_code = args2[2]
except:
print("one params passed")
three_params = False
func = args[0]
data_bytes, key = serialize_GET(keys)
data = self.get_lru_cache(key)
if (data != False):
print("Local LRU Cache Response: " + str(data))
self.add_to_lru_cache(key)
return data
else:
res = None
if (three_params):
print("key: " + str(key))
print("self.data_bytes: " + str(data_bytes))
print("self.hashcode: " + str(hash_code))
res = func(keys, data_bytes, hash_code)
else:
res = func(keys)
self.add_to_lru_cache(key)
return res
return func(key)
def process(udp_clients):
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].get_request(hc, data_bytes)
print(response)
# Delete all Users
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].delete(key, data_bytes)
print(response)
def get(client_ring, hash_codes):
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
# create client ring of the udp client, actually the server sockets they connect to
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
# serialize the user data
data_bytes, key = serialize_PUT(u)
# first find out the nodes by the key, then send the data by that client
response = client_ring.get_node(key).send(data_bytes)
print(response)
# add the response,
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
# delete
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
#print("User key after serialization : " + key)
dest_nodes = client_ring.get_node(key)
for node in dest_nodes:
response = node.send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
dest_nodes = client_ring.get_node(key)
for node in dest_nodes:
response = node.send(data_bytes)
if response:
print("Retrieved data from : ", node.port)
break
print(response)
def process(udp_clients):
hash_codes = set()
# PUT all users.
print('====PUT====')
for u in USERS:
data_bytes, key = serialize_PUT(u)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
hash_codes.add(response.decode())
print(response)
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
print('====GET====')
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
# DELETE all users
print("====DELETE====")
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
def process(udp_clients):
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
nodes = get_node(key)
for i in range(len(nodes)):
node = nodes[i]
print(
f"Sending data to= Physical Node : {node.physical_node}, Virtual Node : {node.virtual_node}"
)
response = node.send(data_bytes)
print(response)
print("\n")
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}\n"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
node = get_node(key)[0]
print(
f"Getting data from= Physical Node : {node.physical_node}, Virtual Node : {node.virtual_node}"
)
response = node.send(data_bytes)
print(response)
print("\n")
def process(udp_clients):
hash_ring = HashRing(2, 64, udp_clients)
hash_ring.assign_nodes()
hash_codes = set()
# PUT all users.
i = 0
for u in USERS:
data_bytes, key = serialize_PUT(u)
nodes_to_send = hash_ring.get_node(key)
for n in nodes_to_send:
resp = n.send(data_bytes)
if i != 0:
print(f' REPLICA PUT Response from server {n.port} is {resp}')
else:
print(f' ORIGINAL PUT Response from server {n.port} is {resp}')
hash_codes.add(str(resp.decode()))
i += 1
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
nodes_to_send = hash_ring.get_node(key)
i = 0
for n in nodes_to_send:
resp = n.send(data_bytes)
if i != 0:
print(f'REPLICA GET Response from server {n.port} is {resp}')
else:
print(f'ORIGINAL GET Response from server {n.port} is {resp}')
i += 1
def get(userHashcode, client_ring):
# only send GET if user is in bloom filter
if(bloomfilter.is_member(userHashcode)):
data_bytes, key = serialize_GET(userHashcode)
response = client_ring.get_node(key).send(data_bytes)
return response
else:
return None
def get(udp_clients,id):
client_ring = NodeRing(udp_clients)
data_bytes, key= serialize_GET(id)
global bloomfilter
if not bloomfilter.is_member(key):
print("Not found in bloom filter")
return None
response = client_ring.get_node(key).send(data_bytes)
return response
def getAll(self, udp_clients):
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
ring = NodeRing(NODES)
fix_me_server_id = NODES.index(ring.get_node(key))
print(data_bytes)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(f"Server GETALL--->{response}")
return response
def get(self, key):
client_ring = NodeRing(NODES)
data_bytes, key = serialize_GET(key)
server = client_ring.get_node(key)
udp_client.host = server['host']
udp_client.port = server['port']
response = udp_client.send(data_bytes)
return response
def get(key):
if (bloomfilter.is_member(key)):
print("Found in bloom filter, fetching from server")
data_bytes, key1 = serialize_GET(key)
client_ring = NodeRing(clients)
response = client_ring.get_node(key1).send(data_bytes)
#print(response)
return response
else:
return None
def get_users(hc):
myOb = NodeRing(NODES)
data_bytes, key = serialize_GET(hc)
print(f'In GET {data_bytes},{key}')
if bf.is_member(key):
node = myOb.get_node(key)
response = UDPClient(node['host'], node['port']).send(data_bytes)
else:
print("Data not in bloom filter and probably not in server")
return response
def get(self, udp_clients, key):
k = key.encode()
for hc in hash_codes:
if (hc == k):
data_bytes, key = serialize_GET(hc)
ring = NodeRing(NODES)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
print(f"Server GET--->{response}")
return response
return None
def get(key):
if (bloom_f.is_member(key)):
data_bytes, key_ser = serialize_GET(key.encode())
print(f"*** GET from SERVER: ***")
fix_me_server_id = NODES.index(ring.get_node(key_ser))
response = clients[fix_me_server_id].send(data_bytes)
# print(deserialize(response))
#print(response)
return response
else:
print("### KEY FOR GET NOT FOUND IN BLOOM FILTER ###")
return None