def process(udp_clients):
client_ring = consistent_hashing(udp_clients)
client_ring.generate_node_ring()
# PUT all users.
for u in USERS[0:40]:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
print(response)
for node in udp_clients:
print('server at {}:{} cached {} data'.format(node.host, node.port,
node.cached_length))
print(
"*********************************DELETE NODE 2 FOR REPLICATION TESTING****************************************"
)
print(
"*********************************DELETE NODE 2 FOR REPLICATION TESTING****************************************"
)
print(
"*********************************DELETE NODE 2 FOR REPLICATION TESTING****************************************"
)
# testing replication
client_ring.delete_node(udp_clients[2])
for u in USERS[41:]:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
print(response)
for node in udp_clients:
print('server at {}:{} cached {} data'.format(node.host, node.port,
node.cached_length))
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
put(key, data_bytes)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = get(hc)
if key is None:
if data_bytes is None:
response = None
else:
response = data_bytes
else:
response = client_ring.get_node(key).send(data_bytes)
lru_cache_put(key, data_bytes)
print(response)
for hc in hash_codes:
data_bytes, key = delete(hc)
if data_bytes is not None:
response = client_ring.get_node(key).send(data_bytes)
else:
response = "Key not Present"
print(response)
def process(udp_clients):
client_ring = RendezvousHash(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
# DELETE
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
bloomfilter.add(key)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
if bloomfilter.is_member(key):
response = client_ring.get_node(key).send(data_bytes)
else:
return None
print(response)
# DELETE all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
if bloomfilter.is_member(key):
response = client_ring.get_node(key).send(data_bytes)
else:
return None
print(response)
def process(udp_clients):
# create client ring of the udp client, actually the server sockets they connect to
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
# serialize the user data
data_bytes, key = serialize_PUT(u)
# first find out the nodes by the key, then send the data by that client
response = client_ring.get_node(key).send(data_bytes)
print(response)
# add the response,
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
# delete
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
nodes = get_node(key)
for i in range(len(nodes)):
node = nodes[i]
print(
f"Sending data to= Physical Node : {node.physical_node}, Virtual Node : {node.virtual_node}"
)
response = node.send(data_bytes)
print(response)
print("\n")
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}\n"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
node = get_node(key)[0]
print(
f"Getting data from= Physical Node : {node.physical_node}, Virtual Node : {node.virtual_node}"
)
response = node.send(data_bytes)
print(response)
print("\n")
def process(udp_clients):
hash_codes = set()
# PUT all users.
print('====PUT====')
for u in USERS:
data_bytes, key = serialize_PUT(u)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
hash_codes.add(response.decode())
print(response)
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
print('====GET====')
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
# DELETE all users
print("====DELETE====")
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
fix_me_server_id = NODE_RING.rendezvous_hash_node(key)
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
def put(client_ring, hash_codes):
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
def process(udp_clients):
# client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
#call RHW
node = get_hrw_node(key, udp_clients)
# response = client_ring.get_node(key).send(data_bytes)
response = node.send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
# call RHW
node = get_hrw_node(key, udp_clients)
response = node.send(data_bytes)
# response = client_ring.get_node(key).send(data_bytes)
print(response)
def process(udp_clients):
rhw_ring = RhwNodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
key = key.decode("utf-8")
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
key = key.decode("utf-8")
server = rhw_ring.get_node(key)
rhw_server_id = NODES.index(server)
response = udp_clients[rhw_server_id].send(data_bytes)
print(response)
def process(udp_clients):
udp_client = udp_clients
hash_codes = set()
client_ring = NodeRing(udp_client)
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put(key, data_bytes, client_ring)
hash_codes.add(response)
print(hash_codes)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
for hc in hash_codes:
# print(hc)
data_bytes, key = serialize_GET(hc)
response = get(key, data_bytes, client_ring)
# print(response)
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
print("keys from DELETE OPERATAIONS:", data_bytes, key)
response = delete(key, data_bytes, client_ring)
print(response)
def process(udp_clients):
client_ring = NodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
put(key, u)
server = client_ring.get_node(key)
response = udp_clients.send(data_bytes, server)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = get(key)
if not (response):
server = client_ring.get_node(key)
response = udp_clients.send(data_bytes, server)
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put([key, data_bytes, client_ring])
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users
for u in USERS:
data_bytes, key = serialize_GET(u)
response = get([key, data_bytes, client_ring])
print(response)
# DELETE all users.
for u in USERS:
data_bytes, key = serialize_DELETE(u)
response = delete([key, data_bytes, client_ring])
print(response)
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
#print("User key after serialization : " + key)
dest_nodes = client_ring.get_node(key)
for node in dest_nodes:
response = node.send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
dest_nodes = client_ring.get_node(key)
for node in dest_nodes:
response = node.send(data_bytes)
if response:
print("Retrieved data from : ", node.port)
break
print(response)
def process(udp_clients):
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].get_request(hc, data_bytes)
print(response)
# Delete all Users
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
ring = NodeRing(NODES)
server_index = NODES.index(ring.get_node(key))
response = udp_clients[server_index].delete(key, data_bytes)
print(response)
def process(udp_clients):
hash_codes = set()
global globalclient
globalclient = udp_clients
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = put(key, data_bytes)
print(response)
hash_codes.add(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# TODO: PART I
# GET all users.
for hc in hash_codes:
print(hc)
response = get(hc)
print(response)
#delete all users
htest = hash_codes.copy()
for hc in htest:
print(hc)
response = delete(hc)
print(response)
hash_codes.remove(hc)
def process(udp_clients):
ring = NodeRing(nodes=NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
# TODO: PART II - Instead of going to server 0, use Naive hashing to split data into multiple servers
#fix_me_server_id = 0
fix_me_server_id = NODES.index(ring.get_node(key))
print(f"Server id: {fix_me_server_id}")
response = udp_clients[fix_me_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# TODO: PART I
# GET all users.
for hc in hash_codes:
print(f"GET:{hc}")
data_bytes, key = serialize_GET(hc)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
# print(hc)
data_bytes, key = serialize_DELETE(hc)
fix_me_server_id = NODES.index(ring.get_node(key))
response = udp_clients[fix_me_server_id].send(data_bytes)
print(response)
def process(udp_clients):
hash_codes = set()
for u in USERS:
data_bytes, key = serialize_PUT(u)
put(hash_codes, data_bytes, key)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
for hc in hash_codes:
data_bytes, key = serialize_GET(u)
print('ley: ' + str(key))
hash_code = hashlib.md5(str(key).encode())
hash_val = hash_code.hexdigest()
print("hash val: " + str(hash_val))
get(hash_val, data_bytes, hash_codes)
for hc in hash_codes:
data_bytes, key = serialize_DELETE(u)
delete(hc, data_bytes, key)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
print("Done!")
def process(udp_clients):
# client_ring = NodeRing(udp_clients)
# Consistent hash
client_ring = ConsistentHashNodeRing(udp_clients, 4)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
# RHW hash
# response = client_ring.get_node_RHW_hash(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}")
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
response = client_ring.get_node(key).send(data_bytes)
# RHW hash
# response = client_ring.get_node_RHW_hash(key).send(data_bytes)
print(response)
def process(udp_clients, seed, weight):
client_ring = NodeRing(udp_clients, seed, weight)
client_ring = ConsistentHashing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# # GET all users
# for u in USERS:
# data_bytes, key = serialize_GET(u)
# response = client_ring.get_node(key).send(data_bytes)
# print(response)
# # DELETE all users.
# for u in USERS:
# data_bytes, key = serialize_DELETE(u)
# response = client_ring.get_node(key).send(data_bytes)
# print(response)
LB = client_ring.load_balanced()
for index in LB:
print(LB[index])
def process(udp_clients):
RH_ring = RHRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].put(key, data_bytes)
hash_codes.add(response.decode())
print(response.decode())
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].get_request(hc, data_bytes)
print(response)
# Delete all Users
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_DELETE(hc)
server_index = RH_ring.get_node(key)
response = udp_clients[server_index].delete(key, data_bytes)
print(response.decode())
def process():
node_ring = NodeRing()
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server, replication_server = node_ring.get_node(key)
response = UDPClient().send(data_bytes, server)
replication_response = UDPClient().send(data_bytes, replication_server)
print(response)
print(replication_response)
hash_codes.add(str(response.decode()))
hash_codes.add(str(replication_response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
server, replication_server = node_ring.get_node(key)
response = UDPClient().send(data_bytes, server)
print(response)
def put(udp_clients,obj):
client_ring = NodeRing(udp_clients)
global bloomfilter
data_bytes, key = serialize_PUT(obj)
bloomfilter.add(key)
response = client_ring.get_node(key).send(data_bytes)
return str(response.decode())
def process(udp_clients):
client_ring = NodeRing(udp_clients)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
get_ch_node = client_ring.get_ch_node(key)
response = get_ch_node.send(data_bytes)
replica = replica_map[get_ch_node]
print("Replicating in next node")
replica.send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
get_ch_node = client_ring.get_ch_node(key)
response = get_ch_node.send(data_bytes)
replica = replica_map[get_ch_node]
print("Response from corresponding node")
print(response)
print("Getting response from replica node")
rep_response = replica.send(data_bytes)
print(rep_response)
def process(udp_clients):
hash_ring = HashRing(2, 64, udp_clients)
hash_ring.assign_nodes()
hash_codes = set()
# PUT all users.
i = 0
for u in USERS:
data_bytes, key = serialize_PUT(u)
nodes_to_send = hash_ring.get_node(key)
for n in nodes_to_send:
resp = n.send(data_bytes)
if i != 0:
print(f' REPLICA PUT Response from server {n.port} is {resp}')
else:
print(f' ORIGINAL PUT Response from server {n.port} is {resp}')
hash_codes.add(str(resp.decode()))
i += 1
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
print(hc)
data_bytes, key = serialize_GET(hc)
nodes_to_send = hash_ring.get_node(key)
i = 0
for n in nodes_to_send:
resp = n.send(data_bytes)
if i != 0:
print(f'REPLICA GET Response from server {n.port} is {resp}')
else:
print(f'ORIGINAL GET Response from server {n.port} is {resp}')
i += 1
def process(udp_clients):
ch_ring = ChNodeRing(NODES)
hash_codes = set()
# PUT all users.
for u in USERS:
data_bytes, key = serialize_PUT(u)
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(
server)) #ast.literal to convert string output to dict element
response = udp_clients[ch_server_id].send(data_bytes)
hash_codes.add(response)
print(response)
print(
f"Number of Users={len(USERS)}\nNumber of Users Cached={len(hash_codes)}"
)
# GET all users.
for hc in hash_codes:
data_bytes, key = serialize_GET(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
#print(deserialize(response))
print(response)
# DELETE all users
for hc in hash_codes:
data_bytes, key = serialize_DELETE(hc)
key = key.decode("utf-8")
server = ch_ring.get_node(key)
ch_server_id = NODES.index(ast.literal_eval(server))
response = udp_clients[ch_server_id].send(data_bytes)
print(response)
def get_key_value(ALL_USERS):
data_byte_list_put = []
keys_list_put = []
for u in ALL_USERS:
data_bytes, key = serialize_PUT(u)
data_byte_list_put.append(data_bytes)
keys_list_put.append(key)
return keys_list_put, data_byte_list_put
def put(user, client_ring):
userBytes = serialize(user)
userHashcode = hash_code_hex(userBytes)
bloomfilter.add(userHashcode)
data_bytes, key = serialize_PUT(user)
response = client_ring.get_node(key).send(data_bytes)
return response
def put(key, value):
bloomfilter.add(key)
client_ring = NodeRing(clients)
data_bytes, key1 = serialize_PUT(value)
response = client_ring.get_node(key1).send(data_bytes)
print(response)
hash_codes.add(str(response.decode()))
return response
def put():
global hash_codes
for u in USERS:
data_bytes, key = serialize_PUT(u)
response = client_ring.get_node(key).send(data_bytes)
hash_codes.add(str(response.decode()))
bloom.add(key)
print(response)