def process(udp_clients):
global clients
clients = udp_clients
hash_codes = set()
# PUT all users.
print("begin to put users")
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put(key, u, data_bytes)
hash_codes.add(response)
print(f"PUT response: {response}")
# print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# TODO: PART I
# GET all users.
print("begin to get users")
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes)
# debug code, check response
# resStr = deserialize(response)
print(f"GET response: {response}")
# DELETE all users.
print("begin to delete users")
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = delete(key, data_bytes)
# debug code, check response
print(f"DELETE response: {response}")
# DELETE all users.
print("begin to delete users2")
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = delete(key, data_bytes)
# debug code, check response
# resStr = deserialize(response)
print(f"DELETE response2: {response}")
# GET all users again.
print("begin to get users2")
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes)
# debug code, check response
# resStr = deserialize(response)
print(f"GET response2: {response}")
def delete(hc, udp_clients):
if BLOOM_FILTER.is_member(hc):
print(hc)
data_bytes, key = serialize_DELETE(hc)
fix_me_server_id = NODE_RING.get_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
def process(udp_clients):
udp_client = udp_clients
hash_codes = set()
client_ring = NodeRing(udp_client)
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put(key, data_bytes, client_ring)
hash_codes.add(response)
print(hash_codes)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
for hc in hash_codes:
# print(hc)
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes, client_ring)
# print(response)
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
print("keys from DELETE OPERATAIONS:", data_bytes, key)
response = delete(key, data_bytes, client_ring)
print(response)
def process(udp_clients):
rhw_ring = RhwNodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
key = key.decode("utf-8")
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
key = key.decode("utf-8")
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
print(response)
def process(udp_clients):
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].get_request(hc, data_bytes)
print(response)
# Delete all Users
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].delete(key, data_bytes)
print(response)
def process(udp_clients):
ring = NodeRing(nodes=NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
# TODO: PART II - Instead of going to server 0, use Naive hashing to split data into multiple servers
#fix_me_server_id = 0
fix_me_server_id = NODES.index(ring.get_node(key))
print(f"Server id: {fix_me_server_id}")
response = udp_clients[fix_me_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# TODO: PART I
# GET all users.
for hc in hash_codes:
print(f"GET:{hc}")
data_bytes, key = serialize_GET(hc)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
# print(hc)
data_bytes, key = serialize_DELETE(hc)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
def process(udp_clients):
RH_ring = RHRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response.decode())
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].get_request(hc, data_bytes)
print(response)
# Delete all Users
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].delete(key, data_bytes)
print(response.decode())
def process(udp_clients):
ch_ring = ChNodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(
server)) #ast.literal to convert string output to dict element
response = udp_clients[ch_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
print(response)
def process(udp_clients):
client_ring = RendezvousHash(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
# DELETE
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
hash_codes = set()
for u in USERS:
data_bytes, key = serialize_PUT(u)
put(hash_codes, data_bytes, key)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
for hc in hash_codes:
data_bytes, key = serialize_GET(u)
print('ley: ' + str(key))
hash_code = hashlib.md5(str(key).encode())
hash_val = hash_code.hexdigest()
print("hash val: " + str(hash_val))
get(hash_val, data_bytes, hash_codes)
for hc in hash_codes:
data_bytes, key = serialize_DELETE(u)
delete(hc, data_bytes, key)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
print("Done!")
def process(udp_clients):
# create client ring of the udp client, actually the server sockets they connect to
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
# serialize the user data
data_bytes, key = serialize_PUT(u)
# first find out the nodes by the key, then send the data by that client
response = client_ring.get_node(key).send(data_bytes)
print(response)
# add the response,
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
# delete
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
bloomfilter.add(key)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
if bloomfilter.is_member(key):
response = client_ring.get_node(key).send(data_bytes)
else:
return None
print(response)
# DELETE all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
if bloomfilter.is_member(key):
response = client_ring.get_node(key).send(data_bytes)
else:
return None
print(response)
def process(udp_clients):
hash_codes = set()
# PUT all users.
print('====PUT====')
for u in USERS:
data_bytes, key = serialize_PUT(u)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
hash_codes.add(response.decode())
print(response)
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
print('====GET====')
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
# DELETE all users
print("====DELETE====")
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put([key, data_bytes, client_ring])
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users
for u in USERS:
data_bytes, key = serialize_GET(u)
response = get([key, data_bytes, client_ring])
print(response)
# DELETE all users.
for u in USERS:
data_bytes, key = serialize_DELETE(u)
response = delete([key, data_bytes, client_ring])
print(response)
def delete(userHashcode, client_ring):
if (bloomfilter.is_member(userHashcode)):
data_bytes, key = serialize_DELETE(userHashcode)
response = client_ring.get_node(key).send(data_bytes)
return response
else:
return None
def delete(self, key):
client_ring = NodeRing(NODES)
data_bytes, key = serialize_DELETE(key)
server = client_ring.get_node(key)
udp_client.host = server['host']
udp_client.port = server['port']
response = udp_client.send(data_bytes)
return response
def delete(udp_clients,id):
client_ring = NodeRing(udp_clients)
data_bytes, key = serialize_DELETE(id)
global bloomfilter
if not bloomfilter.is_member(key):
print("Not found in bloom filter")
return None
response = client_ring.get_node(key).send(data_bytes)
return response
def delete(key):
if (bloomfilter.is_member(key)):
data_bytes, key1 = serialize_DELETE(key)
client_ring = NodeRing(clients)
server_details = client_ring.get_node(key1)
response = server_details.send(data_bytes)
print(response)
return response
else:
return None
def delete(key):
if (bloom_f.is_member(key)):
print(f"DELETE on SERVER:")
data_bytes, key_ser = serialize_DELETE(key.encode())
fix_me_server_id = NODES.index(ring.get_node(key_ser))
response = clients[fix_me_server_id].send(data_bytes)
#print(response)
return response
else:
print("### KEY FOR DELETE NOT FOUND IN BLOOM FILTER ###")
return None
def delete_users(hc):
myOb = NodeRing(NODES)
data_bytes, key = serialize_DELETE(hc)
node = myOb.get_node(key)
if bf.is_member(key):
response = UDPClient(node['host'], node['port']).send(data_bytes)
if (response.decode() == 'Success'):
print('Deleted Succesfully')
# bf.delete(key)
else:
print("Data not in bloom filter and probably not in server")
def delete():
global hash_codes
for hc in hash_codes.copy():
print(hc)
data_bytes, key = serialize_DELETE(hc)
if bloom.is_member(key):
response = client_ring.get_node(key).send(data_bytes)
print(response)
else:
print(None)
hash_codes.remove(hc)
def delete(hc):
global globalclient
if bloomfilter.is_member(hc):
data_bytes, key = serialize_DELETE(hc)
ring = Rendezvous_node(nodes=NODES)
node = ring.get_node(key)
fix_me_server_id = node['port'] - 4000
#fix_me_server_id=0
response = globalclient[fix_me_server_id].send(data_bytes)
return response
else:
return None
def delete(userHashcode, client_ring):
if (bloomfilter.is_member(userHashcode)):
data_bytes, key = serialize_DELETE(userHashcode)
# sends DELET request to all nodes that have a copy of this user
nodesToSendTo = client_ring.get_nodes(key)
responses = []
for node in nodesToSendTo:
response = node.send(data_bytes)
print(response)
responses.append(response)
return responses
else:
return None
def delete(self, udp_clients, key):
k = key.encode()
for hc in hash_codes:
print(hash_codes)
if (hc == k):
data_bytes, key = serialize_DELETE(hc)
ring = NodeRing(NODES)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
hash_codes.remove(hc)
#print(hash_codes)
break
print(f"Server DELETE--->Number of Users Cached={len(hash_codes)}")
def delete(hash_code, data_bytes, key):
# Delete all users
print(hash_code)
data_bytes, key = serialize_DELETE(hash_code)
print('delete key: ' + str(key))
nr = node_ring.NodeRing(NODES)
node = nr.get_node(key)
if (bloomFilter.is_member(key)):
client = UDPClient(node['host'], node['port'])
response = client.send(data_bytes)
else:
response = "Key {} not found in Bloom Filter for delete request".format(
key)
print(str(response))
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
@lru_cache(5)
def get(key, data_bytes):
if bloomfilter.is_member(key):
response = client_ring.get_node(key).send(data_bytes)
return response
def put(key):
bloomfilter.add(key)
return True
def delete(key):
if bloomfilter.is_member(key):
return True
for u in USERS:
data_bytes, key = serialize_PUT(u)
put(key)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
get(key, data_bytes)
#DELETE the user
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
del_val = delete(key)
if del_val == True:
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
ch_ring = ChNodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server1, server2, server3 = ch_ring.get_node_with_replication(key)
# 3 server ids for data replication on next 2 nodes of the main node at hash node ring
ch_server_id_1 = NODES.index(ast.literal_eval(
server1)) #ast.literal to convert string output to dict element
ch_server_id_2 = NODES.index(ast.literal_eval(server2))
ch_server_id_3 = NODES.index(ast.literal_eval(server3))
response_1 = udp_clients[ch_server_id_1].send(data_bytes)
response_2 = udp_clients[ch_server_id_2].send(data_bytes)
response_3 = udp_clients[ch_server_id_3].send(data_bytes)
hash_codes.add(response_1)
hash_codes.add(response_2)
hash_codes.add(response_3)
print(response_1, response_2, response_3)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
print(response)
def process(udp_clients):
print("parameter: ", udp_clients)
client_ring = NodeRing(udp_clients) # ???????
#hrw = rendezvous_hashing(client_ring, [100,200,300,400])
#hrw= RHW_hashing.Ring()
print("client_ring", client_ring)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
# response = hrw.get_responsible_node(key).send(data_bytes)
#lcu.cash add(response)
hash_codes.add(str(response.decode()))
# print("users",u)
# print("hashcode:",hash_codes)
print(
"user numbers:"
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print("hashcode:", hc)
data_bytes, key = serialize_GET(hc)
print("data_bytes: ", data_bytes)
print("key", key)
#cash
#bloomingfilter
response = client_ring.get_node(key).send(data_bytes)
print("response:", response)
#delete
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = client_ring.get_node(key).send(data_bytes)
print("success")
def process(udp_clients):
udp_client = udp_clients
hash_codes = set()
client_ring = NodeRing(udp_client)
# these couple lines are for the rendezvous hash
# weights = [5, 5, 5, 5]
# seeds = ["node1", "serve2", "salt1", "clasuter"]
# instance_rendezvous = RendezvousHash(weights, seeds)
# uncomment/comment this to apply consistent hash
instance_consistent_hash = ConsistentHash(NODES)
client_ring.apply_new_hash(instance_consistent_hash)
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put(key, data_bytes, client_ring)
hash_codes.add(response)
print(hash_codes)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
for hc in hash_codes:
# print(hc)
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes, client_ring)
# print(response)
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
print("keys from DELETE OPERATAIONS:", data_bytes, key)
response = delete(key, data_bytes, client_ring)
print(response)
def process(udp_clients):
ch_ring = CHRing(NODES, VirtualNodeFactor=4)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server_index = ch_ring.get_node(key)
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
server_index = ch_ring.get_node(key)
response = udp_clients[server_index].get_request(hc, data_bytes)
print(response)
# Delete all Users
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
server_index = ch_ring.get_node(key)
response = udp_clients[server_index].delete(key, data_bytes)
print(response)
# PUT all users with Replication
for u in USERS:
data_bytes, key = serialize_PUT(u)
for server_index in ch_ring.get_node_with_replications(
key, ReplicationFactor=2):
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response)
