def setr_deny(svrs, key):
"""setr not successful."""
deny = {'cmd': 'setr_deny', 'key': key}
for svr in svrs:
svr_addr, svr_port = svr.split(':')
common.send_receive(svr_addr, svr_port, deny)
return {'status': 'setr not successful'}
def sync(replica_list, log, msg):
quorom = (len(replica_list) / 2) + 1
replicas_that_responded = {}
for replica in replica_list:
if replica != config["endpoint"]:
host = replica.partition(":")[0]
port = replica.partition(":")[2]
res = common.send_receive(host, port, {
"cmd": "prepare",
"new_log_entry": msg,
"proposal_num": len(log)})
if "error" in res:
print("Couldn't connect to %s:%s for a prepare msg" % (host, port))
continue
else:
replicas_that_responded[replica] = res
if res["missing_logs_to_vl"] != None:
replay(res["missing_logs_to_vl"], None)
else:
# if you are contacting yourself, just vote yes.
replicas_that_responded[config["endpoint"]] = {
"vote": "yes",
"current_log_length": len(log),
"missing_logs_to_vl": None,
"missing_logs_to_replica": None}
# this counts the "yes" votes and puts them in the list
if len([1 for rep in replicas_that_responded if replicas_that_responded[rep]["vote"] == "yes"]) >= quorom:
log[len(log)] = msg
missing_logs = {}
for replica in replicas_that_responded:
if replica != config["endpoint"]:
host = replica.partition(":")[0]
port = replica.partition(":")[2]
# the LAST value represents the replicas last log entry
# i have to compare this value with the length of my log
# it will be less most likely
if replicas_that_responded[replica]["missing_logs_to_replica"] != None:
last = replicas_that_responded[replica]["missing_logs_to_replica"]
while last <= len(log):
missing_logs[last] = log[last]
last = last + 1
#now missing_logs should hold all of the commands that the replica did not have yet.
res = common.send_receive(host, port, {
"cmd": "accept",
"new_log_entry": msg,
"proposal_num": len(log),
"missing_logs_to_replica": missing_logs})
print res
if "error" in res:
print("couldn't connect to %s:%s for an accept msg" % (host, port))
continue
else:
print("successfully sent new_entry to %s:%s" % (host, port))
else:
return {"error": "failed to sync due to lack of quorom"}
return {"ok": "something went wrong"}
def prepare_accept_rpc(msg, addr):
global log
the_time = time.time()
prep_msg = {
"cmd": "prepare",
"prop_no": len(log),
"to_commit": {
"msg": msg,
"addr": addr,
"time": the_time
}
}
# collect responses to prep_msg from replicas
responses = []
for host_port in reversed(config["view_addr"]):
host, port = host_port.split(':')
if (host, port) != (config["my_addr"], config["my_port"]):
resp = common.send_receive(host, port, prep_msg, timeout=1)
if "status" in resp and resp["status"] == "ok":
# remember the host:port of the response
resp["host"] = host
resp["port"] = port
responses.append(resp)
# use longest log of all replicas to update global log
if responses != []:
log_missing = (max(responses,
key=lambda resp: resp["log_len"]))["log_missing"]
for log_entry in log_missing:
result = commit_log_entry(log_entry)
has_quarum = len(responses) + 1 > (len(config["view_addr"]) // 2)
if has_quarum:
# accept message is essentially the same as prep_msg
# just change "cmd", "prop_no", and append missing log entries
# in particular, the_time (line 159) remains the same
accept_msg = prep_msg
accept_msg["cmd"] = "accept"
accept_msg["prop_no"] = len(log)
accept_responses = []
for resp in responses:
host, port = resp["host"], resp["port"]
accept_msg["log_missing"] = log[resp["log_len"]:]
resp = common.send_receive(host, port, accept_msg)
accept_responses.append(resp)
for resp in accept_responses:
if "error" in resp: print resp
return commit_log_entry(accept_msg["to_commit"])
else:
return {"status": "denied", "info": "too few view replicas for quarum"}
def send_cancel(k,v,aloc, serverdata):
responses=[]
i=0
for sid in aloc:
host=serverdata[sid]["host"]
port=serverdata[sid]["port"]
responses.insert( i,
common.send_receive(host,port,
{"key":k,"value": v, "cmd":"cancel"}))
def rebalancing_lost(lost_lease_index_hash):
"""Rebalance the keys and values when a server is lost."""
print 'Rebalancing...'
lost_lease_index, lost_lease_hash = lost_lease_index_hash
count = collections.defaultdict(list)
# index1 = (lost_lease_index - 3) % len_
# index2 = (lost_lease_index + 2) % len_
# if index1 < index2:
# adjacent_leases = leases[index1:index2+1]
# else:
# adjacent_leases = leases[index1:] + leases[:index2+1]
adjacent_leases = arc(lost_lease_index, 3, 2, leases)
print adjacent_leases
for lease in adjacent_leases:
host, port = lease['lockid'].split(':')
res = common.send_receive(host, port, {'cmd': 'query_all_keys'})
print 'res'
print res
for key in res['result']:
count[key].append(lease['hash'])
print count
# We only need to update a server if there are 2 of a key in
# adjacent_leases
to_add = {
key: value
for (key, value) in count.iteritems() if len(value) == 2
}
print to_add
# These are the keys we need to add to the next server, the one after
# that, and the last one, respectively. Could probably be own function.
to_add_1 = []
to_add_2 = []
to_add_3 = []
for (key, hashes) in to_add.iteritems():
if hashes[0] < lost_lease_hash:
if hashes[1] < lost_lease_hash:
to_add_1.append(key)
else:
to_add_2.append(key)
else:
to_add_3.append(key)
print 'ready to rebalance'
svrs_for_1 = arc(lost_lease_index, 2, -1, leases)
print svrs_for_1
assert len(svrs_for_1) == 2
rebalance_request(to_add_1, svrs_for_1, adjacent_leases[4])
svrs_for_2 = arc(lost_lease_index, 1, 1, leases)
print svrs_for_2
rebalance_request(to_add_2, svrs_for_2, adjacent_leases[5])
svrs_for_3 = arc(lost_lease_index, -1, 2, leases)
print svrs_for_3
rebalance_request(to_add_3, svrs_for_3, adjacent_leases[6])
print 'Rebalancing complete'
def rebalance_request(keys, origin, dests):
"""Request to send keys from servers to a server."""
success = False
for svr in origin:
host, port = svr.split(':')
res = common.send_receive(host, port, {
'cmd': rebalance_request,
'keys': keys,
'dest': dests
})
if res['reply'] == 'ok':
success = True
break
# # Try to send the key to both servers from the last server.
# elif res['reply'] == 'KeyError':
# dests.append(svr)
return {'reply': 'ok'} if success else {'reply': 'fail'}
def heartbeat_rpc():
for host_port in reversed(common.lexi_sort(config["view_addr"])):
host, port = host_port.split(':')
msg = {
"cmd": "heartbeat",
"server_id": config["server_id"],
"port": config["port"]
}
response = common.send_receive(host, port, msg)
if "status" in response and response["status"] == "ok":
break
if "status" in response and response["status"] == "denied":
print response
break
if "error" in response: print response
config["hb_time"] = time.time()
def main():
"""Client entry point."""
parser = argparse.ArgumentParser()
parser.add_argument('--server', default='localhost')
parser.add_argument('--viewleader', default='localhost')
subparsers = parser.add_subparsers(dest='cmd')
parser_set = subparsers.add_parser('set')
parser_set.add_argument('key', type=str)
parser_set.add_argument('val', type=str)
parser_get = subparsers.add_parser('get')
parser_get.add_argument('key', type=str)
parser_print = subparsers.add_parser('print')
parser_print.add_argument('text', nargs="*")
parser_query = subparsers.add_parser('query_all_keys')
parser_server_query = subparsers.add_parser('query_servers')
parser_lock_get = subparsers.add_parser('lock_get')
parser_lock_get.add_argument('lockid', type=str)
parser_lock_get.add_argument('requestor', type=str)
parser_lock_get = subparsers.add_parser('lock_release')
parser_lock_get.add_argument('lockid', type=str)
parser_lock_get.add_argument('requestor', type=str)
parser_setr = subparsers.add_parser('setr')
parser_setr.add_argument('key', type=str)
parser_setr.add_argument('val', type=str)
parser_getr = subparsers.add_parser('getr')
parser_getr.add_argument('key', type=str)
args = parser.parse_args()
if args.cmd in ['query_servers', 'lock_get', 'lock_release']:
while True:
response = common.send_receive_range(args.viewleader,
common2.VIEWLEADER_LOW,
common2.VIEWLEADER_HIGH,
vars(args))
if response.get("status") == "retry":
print "Waiting on lock %s..." % args.lockid
time.sleep(5)
continue
else:
break
print response
elif args.cmd == 'setr':
key, val = args.key, args.val
print 'Trying to setr %s to %s' % (key, val)
msg = common.send_receive_range(args.viewleader,
common2.VIEWLEADER_LOW,
common2.VIEWLEADER_HIGH,
{'cmd': 'query_servers'})
# results are lockid, hash
hashes = [(entry[1], entry[0]) for entry in msg['result']]
# list just the lockids
svrs = [svr[1] for svr in find_svrs(key, hashes)]
ress = [setr_request(svr, key) for svr in svrs]
# we iterate through ress twice, which is a bit inefficient, but ress
# should only be around 3 items, so it should be ok.
if 'no' in [res['reply'] for res in ress]:
res = setr_deny(svrs, key)
print 'key already being set'
elif any([res['epoch'] != msg['epoch'] for res in ress]):
res = setr_deny(svrs, key)
print 'server with wrong epoch'
else:
res = setr_accept(svrs, key, val)
return res
elif args.cmd == 'getr':
key = args.key
msg = common.send_receive_range(args.viewleader,
common2.VIEWLEADER_LOW,
common2.VIEWLEADER_HIGH,
{'cmd': 'query_servers'})
# results are lockid, hash
hashes = [(entry[1], entry[0]) for entry in msg['result']]
# list the lockids
svrs = [svr[1] for svr in find_svrs(key, hashes)]
for svr in svrs:
host, port = svr.split(':')
new_msg = {'cmd': 'get', 'key': key}
try:
print common.send_receive(host, port, new_msg)
break
except socket.Timeouterror:
pass
print {'status': 'unable to retrieve %s from any server' % key}
else:
response = common.send_receive_range(args.server, common2.SERVER_LOW,
common2.SERVER_HIGH, vars(args))
print response
def setr_request(lockid, key):
"""Request votes from servers."""
svr_addr, svr_port = map(str, lockid.split(':'))
msg = {'cmd': 'setr_request', 'key': key}
return common.send_receive(svr_addr, svr_port, msg)
def set_val(lockid, key, val):
"""set value on a server."""
svr_addr, svr_port = lockid.split(':')
msg = {'cmd': 'setr', 'key': key, 'val': val}
print 'set %s to %s' % (key, val)
return common.send_receive(svr_addr, svr_port, msg)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--server', default='localhost')
subparsers = parser.add_subparsers(dest='cmd')
parser_set = subparsers.add_parser('set')
parser_set.add_argument('key', type=str)
parser_set.add_argument('val', type=str)
parser_get = subparsers.add_parser('get')
parser_get.add_argument('key', type=str)
parser_print = subparsers.add_parser('print')
parser_print.add_argument('text', nargs="*")
parser_query = subparsers.add_parser('query_all_keys')
# --------- new for hw2 (below) ---------- #
parser.add_argument('--viewleader', default=common.default_vl())
parser_query_servers = subparsers.add_parser('query_servers')
parser_lock_get = subparsers.add_parser('lock_get')
parser_lock_get.add_argument('name', type=str)
parser_lock_get.add_argument('req_id', type=str)
parser_lock_release = subparsers.add_parser('lock_release')
parser_lock_release.add_argument('name', type=str)
parser_lock_release.add_argument('req_id', type=str)
# --------- new for hw2 (above) ---------- #
args = parser.parse_args()
msg = vars(args)
if msg['cmd'] in {'set', 'get', 'print', 'query_all_keys'}:
for port in range(common2.SERVER_LOW, common2.SERVER_HIGH):
print "Trying to connect to %s:%s..." % (args.server, port)
response = common.send_receive(args.server, port, msg)
if "error" in response:
continue
print response
break
else:
print "Can't connect on any port, giving up"
else:
for host_port in reversed(common.lexi_sort(args.viewleader)):
host, port = host_port.split(':')
print "Trying to connect to %s:%s..." % (host, port)
if msg['cmd'] == "lock_get":
success = False
next_port = False
while (not success) and (not next_port):
response = common.send_receive(host, port, msg)
if "error" in response:
next_port = True
elif response["status"] == 'granted':
success = True
print response
else:
success = False
next_port = False
print response
time.sleep(5)
if success: break
else:
response = common.send_receive(host, port, msg)
if "error" in response:
continue
print response
break
else:
print "Can't connect on any port, giving up"
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--server', default='localhost')
parser.add_argument('--viewleader', default='localhost')
subparsers = parser.add_subparsers(dest='cmd')
parser_set = subparsers.add_parser('set')
parser_set.add_argument('key', type=str)
parser_set.add_argument('val', type=str)
parser_get = subparsers.add_parser('get')
parser_get.add_argument('key', type=str)
parser_print = subparsers.add_parser('print')
parser_print.add_argument('text', nargs="*")
parser_query = subparsers.add_parser('query_all_keys')
parser_server_query = subparsers.add_parser('query_servers')
parser_lock_get = subparsers.add_parser('lock_get')
parser_lock_get.add_argument('lockid', type=str)
parser_lock_get.add_argument('requestor', type=str)
parser_lock_get = subparsers.add_parser('lock_release')
parser_lock_get.add_argument('lockid', type=str)
parser_lock_get.add_argument('requestor', type=str)
parser_setr = subparsers.add_parser('setr')
parser_setr.add_argument('key', type=str)
parser_setr.add_argument('val', type=str)
parser_getr = subparsers.add_parser('getr')
parser_getr.add_argument('key', type=str)
args = parser.parse_args()
if args.cmd in ['query_servers', 'lock_get', 'lock_release']:
while True:
response = common.send_receive_range(args.viewleader, common2.VIEWLEADER_LOW, common2.VIEWLEADER_HIGH, vars(args))
if response.get("status") == "retry":
print "Waiting on lock %s..." % args.lockid
time.sleep(5)
continue
else:
break
print response
elif args.cmd in ['setr', 'getr']:
query={'viewleader' : args.viewleader, 'cmd' : 'query_servers', 'server' : args.server}
view=common.send_receive_range(args.viewleader, common2.VIEWLEADER_LOW, common2.VIEWLEADER_HIGH, query)
if "error" in view:
print "Viewleader failure:", view
return()
else:
servers=view['result']
if servers==[]:
print "No servers available"
else:
server_ids=[]
serverdata={}
for server in servers:
(h,p)=common.formatHP(server["location"])
server["host"]=h
server["port"]=p
del server["location"]
n=int(server["name"])
server_ids.append(n)
serverdata[n]=server
h=common.hash_key(args.key)
aloc=common.bucket_allocator(args.key, server_ids)
#list of destination server ids
if args.cmd=="getr":
responses=[]
i=0
for sid in aloc:
responses.insert( i,
common.send_receive(serverdata[sid]["host"],serverdata[sid]["port"], vars(args)))
if "status" in responses[i] and responses[i]["status"]=="ok":
print responses[i]
break
i+=1
else: print "No such key found in our system"
elif args.cmd=="setr":
responses=common.broadcast_receive(aloc,serverdata,vars(args))
for r in responses:
if "epoch" in r:
epoch=r["epoch"]
for r in responses:
if "error" in r:
print "Set failed: server connection error"
send_cancel(args.key,args.val,aloc,serverdata)
break
elif r["vote"]=="no":
print "Set failed: server voted no"
send_cancel(args.key,args.val,aloc,serverdata)
break
elif r["epoch"]!=epoch:
print "Set failed: epoch inconsistency"
send_cancel(args.key,args.val,aloc,serverdata)
break
else:
send_commit(args.key,args.val,aloc,serverdata)
return
else:
response = common.send_receive_range(args.server, common2.SERVER_LOW, common2.SERVER_HIGH, vars(args))
print response