def index():
global processes_future
logging.basicConfig(filename='simulator.log', level=logging.INFO)
logger = logging.getLogger('simulator.log')
form = CombinedForm()
message = ""
if form.validate_on_submit():
message = "Started Dynamo"
params = Params({
'num_proc': form.dynamo_form.num_proc.data,
'hash_size': form.dynamo_form.hash_size.data,
'Q': form.dynamo_form.Q.data,
'N': form.dynamo_form.N.data,
'R': form.dynamo_form.R.data,
'W': form.dynamo_form.W.data,
'w_timeout': form.dynamo_form.w_timeout.data,
'r_timeout': form.dynamo_form.r_timeout.data,
'gossip': form.dynamo_form.gossip.data,
'update_failure_on_rpcs': False
})
print(params)
membership_information = init_membership_list(params)
network_params = NetworkParams({
'latency': form.network_form.latency.data,
'randomize_latency': form.network_form.randomize_latency.data
})
processes_future = start_db_background(params, membership_information, network_params, wait=True, start_port=START_PORT, logger=logger)
session['server_name'] = form.dynamo_form.server_name.data
session['params'] = params.__dict__
session['network_params'] = network_params.__dict__
session['membership_information'] = membership_information
return redirect(url_for('client'))
return render_template('index.html', form=form, message=message)
def test_failure():
logging.basicConfig(filename='failure.log', level=logging.DEBUG)
logger = logging.getLogger('failure.log')
num_tasks = 2
executor = futures.ThreadPoolExecutor(max_workers=num_tasks)
# start server
params = {
'num_proc' : 4,
'hash_size': 3, # 2^3 = 8
'Q' : 2, #
'N' : 3,
'w_timeout': 2,
'r_timeout': 2,
'R': 3,
'W': 3,
'gossip': False,
'update_failure_on_rpcs': True
}
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
network_params = {
'latency': 0,
'randomize_latency': False,
'drop_prob': 0
}
params = Params(params)
network_params = NetworkParams(network_params)
server = start_db_background(params, membership_information, network_params, num_tasks=2, logger=logger)
time.sleep(1)
# server = executor.submit(start_db, params, membership_information)
# fire client request
s = time.time()
ports = [2333,2334,2335,2336]
start_node = 3 # let's hit node 3 with this put request
key_val = 2 # this should go into node 0
port = ports[start_node]
client_fail(2334)
client_put(port, 0, key_val)
mem0, repmem0 = client_get_memory(ports[0])
mem1, repmem1 = client_get_memory(ports[1])
mem2, repmem2 = client_get_memory(ports[2])
mem3, repmem3 = client_get_memory(ports[3])
# check that mem 3 stores info that should have been in 1
assert 2 in repmem3[0].mem
# now fire a get request on node 0 for key 2 and make sure infomraiotn from all replicas is collated.
response = client_get(port, 0, key_val)
print(f"Get response {response}")
e = time.time()
print(f"Time taken {e - s} secs")
print('\n------Test failure passed------\n')
'N': 4,
'w_timeout': 2,
'r_timeout': 2,
'R': 1,
'W': 3,
'gossip': False
}
network_params = {'latency': 10, 'randomize_latency': False, 'drop_prob': 0}
params = Params(params)
network_params = NetworkParams(network_params)
membership_information = init_membership_list(params)
processes_future = start_db_background(params,
membership_information,
network_params,
wait=True,
start_port=START_PORT)
def run_multiple_get(total, num_requests, get_durations):
for i in tqdm(range(total // num_requests)):
requests = [client_get] * num_requests
requests_params = [{
'port': get_start_port(),
'client_id': CLIENT_ID,
'key': randint(0, 2**params.hash_size - 1)
} for _ in range(num_requests)]
get_durations = np.concatenate((get_durations,
run_parallel(requests,
requests_params,
def test_gossip():
logging.basicConfig(filename='gossip.log', level=logging.DEBUG)
logger = logging.getLogger('gossip.log')
num_tasks = 2
executor = futures.ThreadPoolExecutor(max_workers=num_tasks)
# start server
params = {
'num_proc' : 4,
'hash_size': 3, # 2^3 = 8
'Q' : 2, #
'N' : 3,
'w_timeout': 2,
'r_timeout': 2,
'R': 3,
'W': 3,
'gossip' : False,
'update_failure_on_rpcs': False
}
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
network_params = {
'latency': 0,
'randomize_latency': False,
'drop_prob': 0
}
params = Params(params)
network_params = NetworkParams(network_params)
server = start_db_background(params, membership_information, network_params, num_tasks=2, wait=True, logger=logger)
time.sleep(1)
# # server = executor.submit(start_db, params, membership_information)
# # fire client request
# s = time.time()
ports = [2333,2334,2335,2336]
start_node = 3 # let's hit node 3 with this put request
key_val = 2 # this should go into node 0
port = ports[start_node]
client_fail(2334)
client_put(port, 0, key_val)
# print memory of node 1 and 3
mem1, repmem1 = client_get_memory(ports[1])
mem3, repmem3 = client_get_memory(ports[3])
# replication of key 2 at node 0, at node 2 and 3. Node 1's hinted handoff in 3
# turn gossip on
client_fail(2334, fail=False) # unfail
client_gossip(2336)
# wait for some time and check memory of 3 and 1 again
time.sleep(2)
mem1_update, repmem1_update = client_get_memory(ports[1])
mem3_update, repmem3_update = client_get_memory(ports[3])
assert(key_val not in repmem1[0].mem and key_val in repmem1_update[0].mem) # key is now there
assert(key_val in repmem3[0].mem and key_val not in repmem3_update[0].mem)
print(f"Test gossip has passed")
def main():
"""
This tests that the get and put operations are working properly.
"""
logging.basicConfig(filename='get_put.log', level=logging.DEBUG)
logger = logging.getLogger('get_put.log')
# start server
params = {
'num_proc' : 8,
'hash_size': 8, # 2^3 = 8
'Q' : 2, #
'N' : 2, # actually N-1, excluding coordinator node
'w_timeout': 2,
'r_timeout': 2,
'R': 2,
'W': 2,
'gossip': False,
'update_failure_on_rpcs': True
}
network_params = {
'latency': 0,
'randomize_latency': False,
'drop_prob': 0
}
print("constructing...")
params = Params(params)
membership_information = init_membership_list(params)
network_params = NetworkParams(network_params)
server = start_db_background(params, membership_information, network_params, num_tasks=2, logger=logger)
time.sleep(1)
# fire client request
ports = [2333,2334,2335,2336]
start_node = 3 # let's hit node 3 with this put request
key = 2 # this should go into node 0
val = "2"
key2 = 7 # this should go into node 3
val2 = "7"
port = ports[start_node]
client_put(port, 0, key, val)
client_put(port, 0, key2, val2)
mem0, repmem0 = client_get_memory(ports[0])
mem1, repmem1 = client_get_memory(ports[1])
mem2, repmem2 = client_get_memory(ports[2])
mem3, repmem3 = client_get_memory(ports[3])
s = time.time()
response = client_get(port, 0, key)
e = time.time()
print(f"GET {e - s} secs")
print(f"Get response {response}")
assert response.items[0].val == val
context = response.items[0].context
response = client_get(port, 0, key2)
assert response.items[0].val == val2
# Check clock count updation on passing valid context obtained from GET
s = time.time()
client_put(port, 0, key, val2, context=context)
e = time.time()
print(f"PUT {e - s} secs")
response = client_get(port, 0, key)
assert response.items[0].val == val2
assert response.items[0].context.clock[0].count == 2
print("\n-------Test get_put passed--------\n")
def test_replication():
"""
This tests that the server id replicating put requests correctly.
"""
logging.basicConfig(filename='replication.log', level=logging.DEBUG)
logger = logging.getLogger('replication.log')
num_tasks = 2
executor = futures.ThreadPoolExecutor(max_workers=num_tasks)
# start server
params = {
'num_proc': 4,
'hash_size': 3, # 2^3 = 8
'Q': 2, #
'N': 3,
'w_timeout': 2,
'R': 1,
'W': 2,
'gossip': False,
'update_failure_on_rpcs': False
}
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
params = Params(params)
# start server
membership_information = {
0: [1], # key space -> (2,4]
1: [2], # key space -> (4,6]
2: [3], # key space -> (6,8]
3: [0] # key space -> (0,2]
}
network_params = {'latency': 0, 'randomize_latency': False, 'drop_prob': 0}
network_params = NetworkParams(network_params)
server = start_db_background(params,
membership_information,
network_params,
num_tasks=2,
wait=False,
logger=logger)
# server = executor.submit(start_db, params, membership_information)
# fire client request
ports = [2333, 2334, 2335, 2336]
start_node = 3 # let's hit node 3 with this put request
key_val = 2 # this should go into node 0
port = ports[start_node]
time.sleep(1)
client_put(port, 0, key_val)
# now check that replication has happened correctly
mem0, repmem0 = client_get_memory(ports[0])
mem1, repmem1 = client_get_memory(ports[1])
mem2, repmem2 = client_get_memory(ports[2])
mem3, repmem3 = client_get_memory(ports[3])
# node 0 should have key 2 in it's mem and nothing in it's replicated mem
assert (key_val in mem0)
assert (key_val not in mem1 and key_val in repmem1[0].mem)
assert (key_val not in mem2 and key_val in repmem2[0].mem)
assert (key_val not in mem3 and 0 not in repmem3)
print("replication test successful")