def create_new_shuttle(order_proof, result_proof, myreplica_id, is_result,
result, my_priv_key, operation):
slot_id = order_proof['slot_id']
order_stat = {
'order_tag': "order",
'slot_id': slot_id,
'replica_id': myreplica_id,
'op': operation
}
signed_order_stat = generate_signed_msg(order_stat, my_priv_key)
result_msg_hash = create_hash_and_encode_msg(result)
result_stat = {
'result_tag': "result",
'slot_id': slot_id,
'op': operation,
'result': result,
'digest': result_msg_hash.digest
}
signed_result_stat = generate_signed_msg(result_stat, my_priv_key)
order_proof['order_statements'].append(signed_order_stat)
result_proof['result_statements'].append(signed_result_stat)
result_proof['result'] = result
#>>>>head has to set other fields like slot_id and op
#>>>>perhaps assume they are already set
order_proof['replica_id'] = myreplica_id
result_proof['replica_id'] = myreplica_id
#now create the shuttle
send_shuttle = {
'is_result': is_result,
'order_proof': order_proof,
'result_proof': result_proof
}
return send_shuttle
def send_to_all(config, is_replica):
for idx in config:
wedge_msg = {}
wedge_msg['msg'] = 'wedge_req'
signed_wedge_msg = generate_signed_msg(wedge_msg,
olymp_keys['priv_key'])
send(('wedge_req', signed_wedge_msg), to=config[idx])
def receive(msg=('rep_fwd_req', operation, sender_replica_id, client_id,
client_req_id, signed_client_msg)):
output("\n\n")
if (mode == 2):
output("Replica", myreplica_id,
" :Client req in immutable mode, ignored")
return
output("Replica", myreplica_id, " :Forwarded request from replica",
sender_replica_id, "for client", client_id, "and req_id",
client_req_id)
key = (client_id, client_req_id)
key = str(key)
client_msg[client_id]['num_fwd_req'] += 1
failure = super().check_for_failures(
client_id, 'fwd_req', client_msg[client_id]['num_fwd_req'],
rep_fail_triggers)
if failure and any(
x in failure for x in
["sleep", "crash", "extra_op", "drop", "increment_slot"]):
pending_fail = apply_immediate_failure(mydict, failure, False)
if (pending_fail == "drop"):
return
elif (failure):
output("Replica", myreplica_id, " :Found failure", failure,
"for client", client_id, "at msg_num ",
client_msg[client_id]['num_fwd_req'])
super().update_relevant_failure(
failure, "forwarded_request", client_id,
client_msg[client_id]['num_fwd_req'], pending_failures)
if (curr_handling_req == (key)):
output("Replica", myreplica_id, " :Forwarded req from replica",
sender_replica_id, "already handling", key)
return
else:
output("Replica", myreplica_id, " :Forwarded req from replica",
sender_replica_id, "Will handle ", key)
curr_handling_req = key
if (key in res_proof_cache):
output("Replica", myreplica_id, " :Found cached result for",
client_id, "with req id", client_req_id)
result_proof = res_proof_cache[key]['cached_result_proof']
result = res_proof_cache[key]['cached_result']
order_proof = {}
if (len(pending_failures['res_shuttle']) != 0):
next_failure = pending_failures['res_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof, next_failure)
send_shuttle = {'result_proof': result_proof}
signed_send_shuttle = generate_signed_msg(send_shuttle,
my_priv_key)
send(('client_result_pkg', result, myreplica_id,
signed_send_shuttle, client_req_id,
longest_slot_len + req_slot_map[key]),
to=client_config[client_id])
else:
handle_retransmitted_req(key, operation, sender_replica_id,
client_id, client_req_id,
signed_client_msg)
def inject_failure(order_proof, result_proof, next_failure):
output("Checking pending faiures for replica", myreplica_id)
nreplicas = len(rep_config)
trigger = next_failure[1]
if "change_result" in next_failure[0]:
output(
"Replica", myreplica_id, " : { Trigger :", trigger,
" : failure <change_result> } injected in result_shuttle_pkg")
result_msg_hash = create_hash_and_encode_msg("OK")
sender_public_key_hex = rep_pub_keys[myreplica_id]
myindex = myreplica_id
if (len(result_proof['result_statements']) < nreplicas):
output(
"Replica", myreplica_id,
" : Changing result and btw, someone already dropped Head's result statement"
)
myindex = myreplica_id - 1
orig_rstat = verify_signed_msg(
sender_public_key_hex,
result_proof['result_statements'][myindex])
if (orig_rstat):
orig_rstat['digest'] = result_msg_hash.digest
signed_result_stat = generate_signed_msg(
orig_rstat, my_priv_key)
result_proof['result_statements'][myindex] = signed_result_stat
else:
output("FAILED TO INJECT FAILURE : COuld not verify sign of ",
myindex)
elif "drop_result_stmt" in next_failure[0]:
#is nullifying better than omitting
output(
"Replica", myreplica_id, " : { Trigger", trigger,
" : failure <drop_result_stmt> } injected in result_shuttle_pkg"
)
if (len(result_proof['result_statements']) < nreplicas):
output(
"Replica", myreplica_id,
" : Someone already dropped Head's result statement, ignoring"
)
else:
output("Replica", myreplica_id,
" : Dropping head res statement")
del (result_proof['result_statements'][0])
elif "change_operation" in next_failure[0]:
output(
"Replica", myreplica_id, " : { Trigger", trigger,
" : failure <change_operation> } injected in fwd_shuttle_pkg")
sender_public_key_hex = rep_pub_keys[myreplica_id]
orig_ostat = verify_signed_msg(
sender_public_key_hex,
order_proof['order_statements'][myreplica_id])
orig_rstat = verify_signed_msg(
sender_public_key_hex,
result_proof['result_statements'][myreplica_id])
if (orig_ostat and orig_rstat):
orig_ostat['op'] = "get(x)"
orig_rstat['op'] = "get(x)"
signed_order_stat = generate_signed_msg(
orig_ostat, my_priv_key)
signed_result_stat = generate_signed_msg(
orig_rstat, my_priv_key)
order_proof['order_statements'][
myreplica_id] = signed_order_stat
result_proof['result_statements'][
myreplica_id] = signed_result_stat
else:
output("Replica", myreplica_id,
" : Invalid sign on order or rstat; order stat",
orig_ostat, "result_stat", orig_rstat)
elif "invalid_order_sig" in next_failure[0]:
orig_ostat = order_proof['order_statements'][myreplica_id]
invalid_ostat = gen_invalid_sign_msg(orig_ostat)
order_proof['order_statements'][myreplica_id] = invalid_ostat
output(
"Replica", myreplica_id, " : { Trigger", trigger,
" : failure <invalid_order_sig> } injected in fwd_shuttle_pkg")
elif "invalid_result_sig" in next_failure[0]:
myindex = myreplica_id
if ((len(result_proof['result_statements']) < nreplicas)
and am_I_tail):
output(
"Replica", myreplica_id,
" : Invalidating result stat and btw, someone already dropped Head's result statement"
)
myindex = myreplica_id - 1
orig_rstat = result_proof['result_statements'][myindex]
invalid_rstat = gen_invalid_sign_msg(orig_rstat)
result_proof['result_statements'][myindex] = invalid_rstat
output(
"Replica", myreplica_id, " : { Trigger", trigger,
" : failure <invalid_result_sig> } injected in res_shuttle_pkg"
)
def receive(msg=('client_op_req', signed_client_msg, client_id,
client_req_id)):
output("\n\n")
if (mode == 2):
output("Replica", myreplica_id,
" :Client req in immutable mode, ignored")
return
TIMEOUT = mytimeout
output("Replica", myreplica_id, " :Client_op request from client",
client_id, "with req_id", client_req_id, "potential slot_id",
next_slot_id)
client_msg[client_id]['num_client_req'] += 1
failure = super().check_for_failures(
client_id, 'client_req', client_msg[client_id]['num_client_req'],
rep_fail_triggers)
if failure and any(
x in failure for x in
["sleep", "crash", "extra_op", "drop", "increment_slot"]):
pending_fail = apply_immediate_failure(mydict, failure, False)
if (pending_fail == "drop"):
return
elif (failure):
output("Replica", myreplica_id, " :Found failure", failure,
"for client", client_id, "at msg_num ",
client_msg[client_id]['num_client_req'])
super().update_relevant_failure(
failure, 'client_request', client_id,
client_msg[client_id]['num_client_req'], pending_failures)
sender_public_key_hex = client_pub_keys[client_id] #0 is the client
orig_client_msg = verify_signed_msg(sender_public_key_hex,
signed_client_msg)
if not (orig_client_msg):
output("Replica", myreplica_id,
" :Could NOT verify the identity of msg from client")
return
else:
output("Replica", myreplica_id, " :Verified sign of client",
client_id)
is_retransmit = orig_client_msg['is_retransmit']
operation = orig_client_msg['operation']
if (is_retransmit):
output("Replica", myreplica_id,
" :This is a retransmitted request client-id",
client_id, "with req", client_req_id)
#do I have this result cached?
key = (client_id, client_req_id)
key = str(key)
if (key in res_proof_cache):
output("Replica", myreplica_id,
" :Found cached result for", client_id,
"with req id", client_req_id)
result_proof = res_proof_cache[key]['cached_result_proof']
result = res_proof_cache[key]['cached_result']
order_proof = {}
if (len(pending_failures['res_shuttle']) != 0):
next_failure = pending_failures['res_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof,
next_failure)
send_shuttle = {'result_proof': result_proof}
signed_send_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
send(('client_result_pkg', result, myreplica_id,
signed_send_shuttle, client_req_id,
longest_slot_len + req_slot_map[key]),
to=client_config[client_id])
elif (not am_I_head):
output("Replica", myreplica_id, " :Cache lookup for req",
client_req_id, "from", client_id,
"failed, Sending fwd req to head")
send(('rep_fwd_req', operation, myreplica_id, client_id,
client_req_id, signed_client_msg),
to=rep_config[0])
exp_val = client_msg[client_id]['num_valid_res_shutl'] + 1
#wait for timer to expire or a res_shuttle to arrive, is the check correct
#would it be good to check for specific req id
output("Replica", myreplica_id,
" :Starting timer waiting for (", client_req_id,
client_id, ") to arrive")
if (await (client_msg[client_id]['num_valid_res_shutl'] ==
exp_val)):
result_proof = res_proof_cache[key][
'cached_result_proof']
result = res_proof_cache[key]['cached_result']
order_proof = {}
if (len(pending_failures['res_shuttle']) != 0):
next_failure = pending_failures['res_shuttle'].pop(
0)
super().inject_failure(order_proof, result_proof,
next_failure)
send_shuttle = {'result_proof': result_proof}
signed_send_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
output("Replica", myreplica_id,
" :Res proof arrived and sending to client ",
client_id)
if (not am_I_tail): #tail must have sent anyway
send(('client_result_pkg', result, myreplica_id,
signed_send_shuttle, client_req_id,
longest_slot_len + req_slot_map[key]),
to=client_config[client_id])
elif timeout(TIMEOUT):
output(
"Replica", myreplica_id,
" :Timed out and Complaining to Olympus for req-id",
client_req_id, "from client", client_id)
send(('reconfig_req', myreplica_id, True),
to=olymp_config['config'])
elif (am_I_head):
#head has ordered this operation
output("Replica", myreplica_id,
" or HEAD could not find (", client_id,
client_req_id, ") in cache")
handle_retransmitted_req(key, operation, myreplica_id,
client_id, client_req_id,
signed_client_msg)
else:
output("Replica", myreplica_id, " :Client_req got operation",
operation)
if (pending_slot_failure and am_I_head):
output("Replica", myreplica_id, " :slot failure injected")
next_slot_id += 1
pending_slot_failure = False
is_result = 0
order_proof = {}
key = (client_id, client_req_id)
slot_key = str(key)
#have an if check here?
order_proof['slot_id'] = next_slot_id
req_slot_map[slot_key] = next_slot_id
next_slot_id += 1
order_proof['op'] = operation
result_proof = {}
result = apply_operation(mydict, operation)
order_proof['order_statements'] = []
result_proof['result_statements'] = []
#result_proof['slot_id'] = []
send_shuttle = super().create_new_shuttle(
order_proof, result_proof, myreplica_id, is_result, result,
my_priv_key, operation)
history.append(copy.deepcopy(order_proof))
recent_client_req_map.appendleft(client_id)
signed_res_stat_map[client_id] = (copy.deepcopy(
result_proof['result_statements'][-1]),
copy.deepcopy(
order_proof['slot_id']),
myreplica_id)
'''
if (am_I_head):
if (dummy_msg_num == 4):
return
else:
dummy_msg_num +=1
'''
if (len(pending_failures['fwd_shuttle']) != 0):
next_failure = pending_failures['fwd_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof,
next_failure)
signed_new_fwd_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
send_to_correct_replica(rep_config, myreplica_id,
signed_new_fwd_shuttle, result,
is_result, client_id, client_req_id,
send_shuttle, signed_client_msg)
def handle_retransmitted_req(key, operation, sender_replica_id, client_id,
client_req_id, signed_client_msg):
#head has ordered this operation
TIMEOUT = mytimeout
if (curr_handling_req == (key)):
if (not (sender_replica_id == 0)):
output("Replica", myreplica_id, " :Forwarded req from replica",
sender_replica_id, "already handling", key)
return
else:
if (not (sender_replica_id == 0)):
output("Replica", myreplica_id, " :Forwarded req from replica",
sender_replica_id, "Will handle ", key)
curr_handling_req = key
if key in req_slot_map:
exp_val = client_msg[client_id]['num_valid_res_shutl'] + 1
#wait for timer to expire or a res_shuttle to arrive
output("Replica", myreplica_id,
" :Have seen this req and Starting timer waiting for (",
client_req_id, client_id, ") to arrive")
if (await (
client_msg[client_id]['num_valid_res_shutl'] == exp_val)):
result_proof = res_proof_cache[key]['cached_result_proof']
result = res_proof_cache[key]['cached_result']
order_proof = {}
if (len(pending_failures['res_shuttle']) != 0):
next_failure = pending_failures['res_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof,
next_failure)
send_shuttle = {'result_proof': result_proof}
signed_send_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
output("Replica", myreplica_id,
" :Res proof arrived and sending to client ",
client_id)
send(('client_result_pkg', result, myreplica_id,
signed_send_shuttle, client_req_id,
longest_slot_len + req_slot_map[key]),
to=client_config[client_id])
elif timeout(TIMEOUT):
output("Replica", myreplica_id,
" :Timed out and Complaining to Olympus for req-id",
client_req_id, "from client", client_id)
send(('reconfig_req', myreplica_id, True),
to=olymp_config['config'])
else:
#start from scratch
output("Replica", myreplica_id,
" :Have not seen this retransmitted request <",
client_req_id, client_id,
"> before , will start from scratch")
if (pending_slot_failure and am_I_head):
output("Replica", myreplica_id, " :slot failure injected")
next_slot_id += 1
pending_slot_failure = False
is_result = 0
order_proof = {}
key = (client_id, client_req_id)
slot_key = str(key)
#have an if check here?
order_proof['slot_id'] = next_slot_id
req_slot_map[slot_key] = next_slot_id
next_slot_id += 1
order_proof['op'] = operation
result_proof = {}
result = apply_operation(mydict, operation)
order_proof['order_statements'] = []
result_proof['result_statements'] = []
#result_proof['slot_id'] = []
send_shuttle = super().create_new_shuttle(order_proof,
result_proof,
myreplica_id, is_result,
result, my_priv_key,
operation)
history.append(copy.deepcopy(order_proof))
recent_client_req_map.appendleft(client_id)
signed_res_stat_map[client_id] = (copy.deepcopy(
result_proof['result_statements'][-1]),
copy.deepcopy(
order_proof['slot_id']),
myreplica_id)
if (len(pending_failures['fwd_shuttle']) != 0):
next_failure = pending_failures['fwd_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof, next_failure)
signed_new_fwd_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
send_to_correct_replica(rep_config, myreplica_id,
signed_new_fwd_shuttle, result, is_result,
client_id, client_req_id, send_shuttle,
signed_client_msg)
exp_val = client_msg[client_id]['num_valid_res_shutl'] + 1
#wait for timer to expire or a res_shuttle to arrive
output("Replica", myreplica_id,
" :Scratch start, Starting timer waiting for (",
client_req_id, client_id, ") to arrive")
if (await (
client_msg[client_id]['num_valid_res_shutl'] == exp_val)):
result_proof = res_proof_cache[key]['cached_result_proof']
result = res_proof_cache[key]['cached_result']
order_proof = {}
if (len(pending_failures['res_shuttle']) != 0):
next_failure = pending_failures['res_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof,
next_failure)
send_shuttle = {'result_proof': result_proof}
signed_send_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
output("Replica", myreplica_id,
" :Res proof arrived and sending to client ",
client_id)
send(('client_result_pkg', result, myreplica_id,
signed_send_shuttle, client_req_id,
longest_slot_len + req_slot_map[slot_key]),
to=client_config[client_id])
elif timeout(TIMEOUT):
output("Replica", myreplica_id,
" :Timed out and Complaining to Olympus for req-id",
client_req_id, "from client", client_id)
send(('reconfig_req', myreplica_id, True),
to=olymp_config['config'])
def receive(msg=('signed_result_shuttle_pkg', replica_id,
signed_res_shuttle, client_id, client_req_id)):
output("\n\n")
if (mode == 2):
output("Replica", myreplica_id,
" :Client req in immutable mode, ignored")
return
output("Replica", myreplica_id, " :Res_shuttle_pkg from replica",
replica_id, "for client", client_id, "and req", client_req_id)
sender_public_key_hex = rep_pub_keys[replica_id]
client_msg[client_id]['num_res_shutl'] += 1
failure = super().check_for_failures(
client_id, 'res_shutl', client_msg[client_id]['num_res_shutl'],
rep_fail_triggers)
if failure and any(
x in failure for x in
["sleep", "crash", "extra_op", "drop", "increment_slot"]):
pending_fail = apply_immediate_failure(mydict, failure, False)
if (pending_fail == "drop"):
return
elif (failure):
output("Replica", myreplica_id, " :Found failure", failure,
"for client", client_id, "at msg_num ",
client_msg[client_id]['num_res_shutl'])
super().update_relevant_failure(
failure, "result_shuttle", client_id,
client_msg[client_id]['num_res_shutl'], pending_failures)
orig_res_shuttle = verify_signed_msg(sender_public_key_hex,
signed_res_shuttle)
if not (orig_res_shuttle):
output("Replica", myreplica_id,
" :Could NOT verify the identity of ", replica_id)
else:
output("Replica", myreplica_id, " :Validated identity of ",
replica_id)
result_proof = orig_res_shuttle['result_proof']
order_proof = orig_res_shuttle['order_proof']
is_result = orig_res_shuttle['is_result']
if not (super().validate_result_proof(result_proof)):
output("Replica", myreplica_id,
" :Could NOT validate result proof")
output("Replica", myreplica_id, " :Complaining to olympus")
#what to do here
else:
client_msg[client_id]['num_valid_res_shutl'] += 1
cache_result_shuttle(result_proof, client_id, client_req_id)
output("Replica", myreplica_id, " :Validated result proof")
if (not am_I_head
and (len(pending_failures['res_shuttle']) != 0)):
next_failure = pending_failures['res_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof,
next_failure)
send_shuttle = {
'is_result': is_result,
'order_proof': order_proof,
'result_proof': result_proof
}
signed_new_res_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
result = result_proof['result']
if (signed_new_res_shuttle):
send_to_correct_replica(rep_config, myreplica_id,
signed_new_res_shuttle, result,
is_result, client_id,
client_req_id, send_shuttle, "")
else:
output("Replica", myreplica_id, " :Error in res shuttle")
def receive(msg=('signed_fwd_shuttle_pkg', replica_id, signed_fwd_shuttle,
client_id, client_req_id, signed_client_msg)):
if (mode == 2):
output("Replica", myreplica_id,
" :Client req in immutable mode, ignored")
return
output("\n\n")
output("Replica", myreplica_id, " :Fwd_shuttle_pkg from replica",
replica_id, "for client", client_id, "and req-id",
client_req_id, "potential slot", next_slot_id)
sender_public_key_hex = rep_pub_keys[replica_id]
orig_fwd_shuttle = verify_signed_msg(sender_public_key_hex,
signed_fwd_shuttle)
client_public_key_hex = client_pub_keys[client_id]
orig_client_msg = verify_signed_msg(client_public_key_hex,
signed_client_msg)
client_msg[client_id]['num_fwd_shutl'] += 1
failure = super().check_for_failures(
client_id, 'fwd_shutl', client_msg[client_id]['num_fwd_shutl'],
rep_fail_triggers)
if failure and any(
x in failure for x in
["sleep", "crash", "extra_op", "drop", "increment_slot"]):
pending_fail = apply_immediate_failure(mydict, failure, False)
if (pending_fail == "drop"):
return
elif (failure):
output("Replica", myreplica_id, " :Found fwd_shutl failure",
failure, "for client", client_id, "at msg num ",
client_msg[client_id]['num_fwd_shutl'])
super().update_relevant_failure(
failure, "shuttle", client_id,
client_msg[client_id]['num_fwd_shutl'], pending_failures)
if (not orig_client_msg):
output("Replica", myreplica_id,
" :Could not verify signed client msg")
return
else:
operation = orig_client_msg['operation']
if not (orig_fwd_shuttle):
output("Replica", myreplica_id,
" :fwd_pkg Could NOT verify the identity of msg from ",
replica_id)
return
else:
#orig_fwd_shuttle is of type Shuttle
output("Replica", myreplica_id, " :verified sign of replica",
replica_id)
is_result = orig_fwd_shuttle['is_result']
order_proof = orig_fwd_shuttle['order_proof']
result_proof = orig_fwd_shuttle['result_proof']
#maybe use slot_id and replica_id too?
(is_op_valid, next_slot_id) = super().validate_order_proof(
order_proof, next_slot_id, operation)
if not (is_op_valid):
output("Replica", myreplica_id,
" :Could NOT validate order proof")
output("Replica", myreplica_id, " :Complaining to olympus")
#what to do now?
else:
output("Replica", myreplica_id, " :validated order proof")
output("Replica", myreplica_id, " : Slot_id",
order_proof['slot_id'], "operation", operation)
key = (client_id, client_req_id)
key = str(key)
req_slot_map[key] = order_proof['slot_id']
if (operation == "get(x)"):
output("something went very wrong")
#fail_var=-1
result = apply_operation(mydict, operation)
if (am_I_tail):
is_result = 1
else:
is_result = 0
send_shuttle = super().create_new_shuttle(
order_proof, result_proof, myreplica_id, is_result, result,
my_priv_key, operation)
if (am_I_tail):
client_msg[client_id]['num_valid_res_shutl'] += 1
cache_result_shuttle(result_proof, client_id,
client_req_id)
history.append(copy.deepcopy(order_proof))
recent_client_req_map.appendleft(client_id)
signed_res_stat_map[client_id] = (
copy.deepcopy(result_proof['result_statements'][-1]),
copy.deepcopy(order_proof['slot_id']),
myreplica_id,
)
if (not am_I_tail
and (len(pending_failures['fwd_shuttle']) != 0)):
next_failure = pending_failures['fwd_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof,
next_failure)
if (am_I_tail and (len(pending_failures['res_shuttle']) != 0)):
next_failure = pending_failures['res_shuttle'].pop(0)
super().inject_failure(order_proof, result_proof,
next_failure)
signed_new_fwd_shuttle = generate_signed_msg(
send_shuttle, my_priv_key)
if (signed_new_fwd_shuttle):
send_to_correct_replica(rep_config, myreplica_id,
signed_new_fwd_shuttle, result,
is_result, client_id,
client_req_id, send_shuttle,
signed_client_msg)
else:
output("Fwd: replica", myreplica_id,
" :Error in fwd shuttle")
def receive(msg=('catchup', catch_seq, client_req_map)):
output("\n\n")
output("Replica",
myreplica_id, " :Got catchup req from Olympus catchup len",
len(catch_seq), "catch_seq", catch_seq)
output("Replica", myreplica_id, " :client_req map is", client_req_map)
catchup_msg_num += 1
pseudo_dict = {}
pseudo_dict = copy.deepcopy(mydict)
pseudo_next_slot_id = copy.deepcopy(next_slot_id)
pseudo_signed_res_stat_map = copy.deepcopy(signed_res_stat_map)
failure = super().check_for_failures(-1, 'catch_up', catchup_msg_num,
rep_fail_triggers)
if failure and any(
x in failure for x in
["sleep", "crash", "extra_op", "drop", "increment_slot"]):
pending_fail = apply_immediate_failure(mydict, failure, False)
pending_fail = apply_immediate_failure(pseudo_dict, failure, True)
if ("drop" in pending_fail):
return
elif (failure):
output("Replica", myreplica_id, " :Found failure", failure,
"for catchup at msg num ", catchup_msg_num)
super().update_relevant_failure(failure, "catch_up", -1,
catchup_msg_num, pending_failures)
#(client_id, order_proof['slot_id'], order_proof['operation'])
if catch_seq:
client_queue = next(iter(client_req_map.values()))
lenq = len(client_req_map)
#hope is that reqs get appended in slot order in this queue and only num_client reqs are outstanding at a time
[client_queue.pop() for i in range(lenq - len(catch_seq))]
output("Replica", myreplica_id,
" :client_req map after popping is ", client_queue)
for op in catch_seq:
result = apply_operation(pseudo_dict, op)
#what about?
#history.append(copy.deepcopy(order_proof))
#recent_client_req_map
cli_id = client_queue.pop()
output("Client_id ", cli_id)
result_msg_hash = create_hash_and_encode_msg(result)
result_stat = {
'result_tag': "result",
'slot_id': pseudo_next_slot_id,
'op': op,
'result': result,
'digest': result_msg_hash.digest
}
signed_result_stat = generate_signed_msg(
result_stat, my_priv_key)
pseudo_signed_res_stat_map[cli_id] = (signed_result_stat,
pseudo_next_slot_id,
myreplica_id)
pseudo_next_slot_id += 1
mydict_sorted = OrderedDict(sorted(pseudo_dict.items()))
run_hash = create_hash_and_encode_msg(mydict_sorted)
output("Replica", myreplica_id,
" :Sending caughtup msg to Olumpus with hash of \n",
mydict_sorted, " is \n", run_hash.digest)
send(('caughtup_req', run_hash.digest, pseudo_signed_res_stat_map,
myreplica_id),
to=olymp_config['config'])
def run():
#########
output("\n\nClinent Started")
#return
########
config_file = sys.argv[1]
print("Config file", config_file)
config(clock='Lamport')
output("myclient_id", myclient_id)
config = parse_file(config_file)
myclient_load = get_client_load(config, myclient_id)
if (not myclient_load):
output("Client ", myclient_id, "No workload found")
return
print("myclient_load[0]", myclient_load[0])
if ("pseudorandom" in myclient_load[0]):
output("generating pseudo load")
myclient_load = generate_pseudo_rand_reqs(myclient_load[0])
output("\nGenerated load: ", myclient_load)
client_timeout = get_client_timeout(config)
client_timeout = client_timeout / 1000
output("client-id ", myclient_id, " mytimeout in s", client_timeout,
"\n\n")
#signed_client_msg: signed<op, is_retransmit>
op_msg = {}
#client_req_id = -1
TIMEOUT = client_timeout
for i in myclient_load:
is_retransmit = 0
client_req_id += 1
exp_res[client_req_id] = {}
exp_res[client_req_id]['slot'] = -1
exp_res[client_req_id]['op'] = i
valid_result_proofs[client_req_id] = False
op_msg['operation'] = i
op_msg['is_retransmit'] = 0
signed_op_msg = generate_signed_msg(op_msg, my_priv_key)
num_valid_replicas = 0
is_op_invalid = False
pending_op = i
output("\n\n\n")
output("client ", myclient_id, "sending operation", i,
"with req_id", client_req_id, " to head at ", rep_config[0],
"\n\n")
send(('client_op_req', signed_op_msg, myclient_id, client_req_id),
to=rep_config[0])
if (await (valid_result_proofs[client_req_id])):
if (is_op_invalid or (num_valid_replicas < nreplicas)):
output(
"client ", myclient_id,
"Invalid operation or less than suffcient replicas in result proof, sending reconfig req to Olympus , num_valid_replicas: ",
num_valid_replicas, "is_op_invalid ", is_op_invalid)
send(('reconfig_req', myclient_id, False),
to=olymp['config'])
await ((is_reconfig == True)
and valid_result_proofs[client_req_id])
is_reconfig = False
elif timeout(TIMEOUT):
output("Client", myclient_id,
"timed out waiting for operation", i, "with req-id",
client_req_id)
is_retransmit = 1
op_msg['is_retransmit'] = 1
signed_op_msg = generate_signed_msg(op_msg, my_priv_key)
for idx in rep_config:
output("client ", myclient_id, "RESEND op_req", i,
"with req_id", client_req_id, " to rep ", idx,
"at ", rep_config[idx])
send(('client_op_req', signed_op_msg, myclient_id,
client_req_id),
to=rep_config[idx])
if (await (valid_result_proofs[client_req_id])):
output("canceling timer")
elif timeout(TIMEOUT + 5):
output(
"Client", myclient_id,
"timed out again and seding reconfig reqs to Olympus")
#ask for reconfig from olympus
send(('reconfig_req', myclient_id, False),
to=olymp['config'])
if (await ((is_reconfig == True)
and valid_result_proofs[client_req_id])):
is_reconfig = False
elif timeout(TIMEOUT + 10):
if (is_reconfig):
is_reconfig = False
output(
"Client", myclient_id,
"Timedout waiting for new config and did not recevive a valid result proof for ",
client_req_id, "Retry with new config")
sleep(5)
else:
output(
"Client", myclient_id,
"Not even recnfigured, Don't know what to do now"
)
await (0)
return
#sys.exit()
sleep(1)
output(
"client ", myclient_id,
"Sending to replicas done, now sending results to central config")
send(('client_verify_res', rep_res, exp_res, myclient_id),
to=central_client)
await (0)