def client():
global context
form = ClientForm()
response = ""
duration = ""
durations = []
stats = ""
url = ""
if form.is_submitted():
start = time.time()
if form.get_button.data:
# print("GET:", form.get_button.data)
if form.num_requests.data == 1:
response = client_get(get_start_port(), CLIENT_ID, key=form.key.data)
context = response.items[0].context
duration = str((time.time()-start) * 1000)
else:
requests = [client_get]*form.num_requests.data
requests_params = [{'port': get_start_port(), 'client_id': CLIENT_ID, 'key': form.key.data} for _ in range(form.num_requests.data)]
durations, responses = run_parallel(requests, requests_params, start_port=START_PORT)
duration = sum(durations)*1000
stats = str(get_stats(durations))
url = generate_plot(durations, 'GET', form.clear.data)
else:
# print("PUT:", form.put_button.data)
if form.num_requests.data == 1:
response = client_put(get_start_port(), CLIENT_ID, key=form.key.data, val=form.val.data, context=context)
duration = str((time.time()-start) * 1000)
else:
requests = [client_put]*form.num_requests.data
requests_params = [{'port': get_start_port(), 'client_id': CLIENT_ID, 'key': form.key.data, 'val': form.val.data} for _ in range(form.num_requests.data)]
durations, responses = run_parallel(requests, requests_params, start_port=START_PORT)
duration = sum(durations)*1000
stats = str(get_stats(durations))
url = generate_plot(durations, 'PUT', form.clear.data)
return render_template('client.html', server_name=session['server_name'], form=form, response=str(response), duration=duration, stats=stats, url=url)
def test_failure():
logging.basicConfig(filename='failure.log', level=logging.DEBUG)
logger = logging.getLogger('failure.log')
num_tasks = 2
executor = futures.ThreadPoolExecutor(max_workers=num_tasks)
# start server
params = {
'num_proc' : 4,
'hash_size': 3, # 2^3 = 8
'Q' : 2, #
'N' : 3,
'w_timeout': 2,
'r_timeout': 2,
'R': 3,
'W': 3,
'gossip': False,
'update_failure_on_rpcs': True
}
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
network_params = {
'latency': 0,
'randomize_latency': False,
'drop_prob': 0
}
params = Params(params)
network_params = NetworkParams(network_params)
server = start_db_background(params, membership_information, network_params, num_tasks=2, logger=logger)
time.sleep(1)
# server = executor.submit(start_db, params, membership_information)
# fire client request
s = time.time()
ports = [2333,2334,2335,2336]
start_node = 3 # let's hit node 3 with this put request
key_val = 2 # this should go into node 0
port = ports[start_node]
client_fail(2334)
client_put(port, 0, key_val)
mem0, repmem0 = client_get_memory(ports[0])
mem1, repmem1 = client_get_memory(ports[1])
mem2, repmem2 = client_get_memory(ports[2])
mem3, repmem3 = client_get_memory(ports[3])
# check that mem 3 stores info that should have been in 1
assert 2 in repmem3[0].mem
# now fire a get request on node 0 for key 2 and make sure infomraiotn from all replicas is collated.
response = client_get(port, 0, key_val)
print(f"Get response {response}")
e = time.time()
print(f"Time taken {e - s} secs")
print('\n------Test failure passed------\n')
def store_keys(params):
for key in tqdm(range(2**params.hash_size)):
client_put(get_start_port(), CLIENT_ID, key=key, val=str(key))
requests = [client_get] * num_requests
requests_params = [{
'port': get_start_port(failed_port=failed_port),
'client_id': CLIENT_ID,
'key': randint(0, 2**params.hash_size - 1)
} for _ in range(num_requests)]
_durations, _responses = run_parallel(requests,
requests_params,
start_port=START_PORT,
as_np=False)
print(len(_durations))
durations.extend(_durations)
responses.extend(_responses)
return durations, responses
from client_dynamo import client_fail
client_fail(START_PORT)
response = client_put(START_PORT + 1, CLIENT_ID, 0, "0")
print(response)
response = client_get(START_PORT + 1, CLIENT_ID, 0)
print(response)
get_fail_durations, responses = run_multiple_get(100, 10, None, START_PORT)
print(len(get_fail_durations))
print(get_fail_durations)
def test_gossip():
logging.basicConfig(filename='gossip.log', level=logging.DEBUG)
logger = logging.getLogger('gossip.log')
num_tasks = 2
executor = futures.ThreadPoolExecutor(max_workers=num_tasks)
# start server
params = {
'num_proc' : 4,
'hash_size': 3, # 2^3 = 8
'Q' : 2, #
'N' : 3,
'w_timeout': 2,
'r_timeout': 2,
'R': 3,
'W': 3,
'gossip' : False,
'update_failure_on_rpcs': False
}
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
network_params = {
'latency': 0,
'randomize_latency': False,
'drop_prob': 0
}
params = Params(params)
network_params = NetworkParams(network_params)
server = start_db_background(params, membership_information, network_params, num_tasks=2, wait=True, logger=logger)
time.sleep(1)
# # server = executor.submit(start_db, params, membership_information)
# # fire client request
# s = time.time()
ports = [2333,2334,2335,2336]
start_node = 3 # let's hit node 3 with this put request
key_val = 2 # this should go into node 0
port = ports[start_node]
client_fail(2334)
client_put(port, 0, key_val)
# print memory of node 1 and 3
mem1, repmem1 = client_get_memory(ports[1])
mem3, repmem3 = client_get_memory(ports[3])
# replication of key 2 at node 0, at node 2 and 3. Node 1's hinted handoff in 3
# turn gossip on
client_fail(2334, fail=False) # unfail
client_gossip(2336)
# wait for some time and check memory of 3 and 1 again
time.sleep(2)
mem1_update, repmem1_update = client_get_memory(ports[1])
mem3_update, repmem3_update = client_get_memory(ports[3])
assert(key_val not in repmem1[0].mem and key_val in repmem1_update[0].mem) # key is now there
assert(key_val in repmem3[0].mem and key_val not in repmem3_update[0].mem)
print(f"Test gossip has passed")
def main():
"""
This tests that the get and put operations are working properly.
"""
logging.basicConfig(filename='get_put.log', level=logging.DEBUG)
logger = logging.getLogger('get_put.log')
# start server
params = {
'num_proc' : 8,
'hash_size': 8, # 2^3 = 8
'Q' : 2, #
'N' : 2, # actually N-1, excluding coordinator node
'w_timeout': 2,
'r_timeout': 2,
'R': 2,
'W': 2,
'gossip': False,
'update_failure_on_rpcs': True
}
network_params = {
'latency': 0,
'randomize_latency': False,
'drop_prob': 0
}
print("constructing...")
params = Params(params)
membership_information = init_membership_list(params)
network_params = NetworkParams(network_params)
server = start_db_background(params, membership_information, network_params, num_tasks=2, logger=logger)
time.sleep(1)
# fire client request
ports = [2333,2334,2335,2336]
start_node = 3 # let's hit node 3 with this put request
key = 2 # this should go into node 0
val = "2"
key2 = 7 # this should go into node 3
val2 = "7"
port = ports[start_node]
client_put(port, 0, key, val)
client_put(port, 0, key2, val2)
mem0, repmem0 = client_get_memory(ports[0])
mem1, repmem1 = client_get_memory(ports[1])
mem2, repmem2 = client_get_memory(ports[2])
mem3, repmem3 = client_get_memory(ports[3])
s = time.time()
response = client_get(port, 0, key)
e = time.time()
print(f"GET {e - s} secs")
print(f"Get response {response}")
assert response.items[0].val == val
context = response.items[0].context
response = client_get(port, 0, key2)
assert response.items[0].val == val2
# Check clock count updation on passing valid context obtained from GET
s = time.time()
client_put(port, 0, key, val2, context=context)
e = time.time()
print(f"PUT {e - s} secs")
response = client_get(port, 0, key)
assert response.items[0].val == val2
assert response.items[0].context.clock[0].count == 2
print("\n-------Test get_put passed--------\n")
def test_replication():
"""
This tests that the server id replicating put requests correctly.
"""
logging.basicConfig(filename='replication.log', level=logging.DEBUG)
logger = logging.getLogger('replication.log')
num_tasks = 2
executor = futures.ThreadPoolExecutor(max_workers=num_tasks)
# start server
params = {
'num_proc': 4,
'hash_size': 3, # 2^3 = 8
'Q': 2, #
'N': 3,
'w_timeout': 2,
'R': 1,
'W': 2,
'gossip': False,
'update_failure_on_rpcs': False
}
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
params = Params(params)
# start server
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
network_params = {'latency': 0, 'randomize_latency': False, 'drop_prob': 0}
network_params = NetworkParams(network_params)
server = start_db_background(params,
membership_information,
network_params,
num_tasks=2,
wait=False,
logger=logger)
# server = executor.submit(start_db, params, membership_information)
# fire client request
ports = [2333, 2334, 2335, 2336]
start_node = 3 # let's hit node 3 with this put request
key_val = 2 # this should go into node 0
port = ports[start_node]
time.sleep(1)
client_put(port, 0, key_val)
# now check that replication has happened correctly
mem0, repmem0 = client_get_memory(ports[0])
mem1, repmem1 = client_get_memory(ports[1])
mem2, repmem2 = client_get_memory(ports[2])
mem3, repmem3 = client_get_memory(ports[3])
# node 0 should have key 2 in it's mem and nothing in it's replicated mem
assert (key_val in mem0)
assert (key_val not in mem1 and key_val in repmem1[0].mem)
assert (key_val not in mem2 and key_val in repmem2[0].mem)
assert (key_val not in mem3 and 0 not in repmem3)
print("replication test successful")