def pl_kd(config, rg, kvs):
f = gen_hash(config['mbits'])
plens = []
for key, val in tqdm(kvs.items()):
with grpc.insecure_channel(np.random.choice(
config['servers'][:rg])) as channel:
stub = cpg.chordStub(channel)
resp = stub.Put(cp.PutRequest(key=key, val=val, id=str(f(key))))
for key in tqdm(kvs.keys()):
with grpc.insecure_channel(np.random.choice(
config['servers'][:rg])) as channel:
stub = cpg.chordStub(channel)
resp = stub.GetSuccessor(cp.GetSuccessorRequest(id=str(f(key))))
plens.append(resp.path_length)
plens = np.asarray(plens)
kd = {}
for srv in config['servers'][:rg]:
with grpc.insecure_channel(srv) as channel:
stub = cpg.chordStub(channel)
resp = stub.GetStatus(cp.Null())
kd[srv] = list(resp.keys)
for srv in config['servers'][:rg]:
with grpc.insecure_channel(srv) as channel:
stub = cpg.chordStub(channel)
resp = stub.Reset(cp.Null())
return plens, kd
def check_stable(config, rg):
fgs = {}
for srv in config['servers'][:rg]:
with grpc.insecure_channel(srv) as channel:
stub = cpg.chordStub(channel)
resp = stub.GetFingers(cp.Null())
fgs[srv] = resp.server_addrs
time.sleep(12 * rg)
for srv in config['servers'][:rg]:
with grpc.insecure_channel(srv) as channel:
stub = cpg.chordStub(channel)
resp = stub.GetFingers(cp.Null())
if fgs[srv] != resp.server_addrs:
return False
return True
def Join(self, server_addr):
self.initialize()
mprint('join start')
self.predecessor = None
self.hashed_predecessor = None
with grpc.insecure_channel(server_addr) as channel:
stub = cpg.chordStub(channel)
res = stub.GetSuccessor(
cp.GetSuccessorRequest(id=str(self.hashed_addr)),
timeout=self.config['rpc_timeout'])
successor = res.server_addr
successor_list = [successor]+[''] * \
(self.config['successor_list_len']-1)
with grpc.insecure_channel(successor) as channel:
stub = cpg.chordStub(channel)
# get my own data
res = stub.GetPredecessor(cp.Null())
hashed_predecessor = (self.hash(res.server_addr) + 1) & (
(1 << self.config['mbits']) - 1)
res = stub.GetRange(
cp.GetRangeRequest(from_id=str(hashed_predecessor),
to_id=str(self.hashed_addr)))
for k, v in zip(res.keys, res.vals):
self.store[k] = {'val': v, 'id': self.hash(k)}
self.successor = successor
self.successor_list = successor_list
# delete successor's my own data
res = stub.DeleteRange(
cp.DeleteRangeRequest(from_id=str(hashed_predecessor),
to_id=str(self.hashed_addr)))
# res = stub.GetReplicas(cp.GetReplicasRequest(
# remaining=self.config['max_failures']))
# for k, v in zip(res.keys, res.vals):
# self.store[k] = {'val': v, 'id': self.hash(k), 'own': False}
mprint('join', self.successor)
def Stabilize(self):
mprint('stable')
try:
x = None
while self.successor_list[0]:
self.successor = self.successor_list[0]
self.hashed_successor = self.hash(self.successor)
mprint('stable', self.successor)
try:
if self.addr != self.successor:
with grpc.insecure_channel(self.successor) as channel:
stub = cpg.chordStub(channel)
res = stub.GetPredecessor(
cp.Null(), timeout=self.config['rpc_timeout'])
mprint('pred', self.successor, res)
x = res.server_addr
else:
x = self.predecessor
break
except Exception as e:
self.successor_list = self.successor_list[1:] + [None]
mprint('s1', e, self.successor_list)
if x:
hashed_x = self.hash(x)
if in_range(self.hashed_addr, hashed_x, self.hashed_successor):
self.successor = x
self.hashed_successor = self.hash(self.successor)
self.successor_list = [self.successor
] + self.successor_list[:-1]
print('update sl', self.successor_list)
self.successor = self.successor_list[0]
if self.addr != self.successor:
with grpc.insecure_channel(self.successor) as channel:
stub = cpg.chordStub(channel)
res = stub.Notify(cp.NotifyRequest(server_addr=self.addr),
timeout=self.config['rpc_timeout'])
res = stub.GetSuccessorList(
cp.Null(), timeout=self.config['rpc_timeout'])
self.successor_list = [self.successor
] + res.server_addrs[:-1]
else:
self._Notify(self.addr)
except Exception as e:
import traceback as tb
mprint('s2', self.successor_list, e, tb.format_stack())
pass
def wait_until_alive(self, addr):
while True:
try:
with grpc.insecure_channel(addr) as channel:
stub = cpg.chordStub(channel)
stub.Noop(cp.Null())
break
except:
time.sleep(1)
def CheckPredecessor(self):
try:
mprint('checkpred', self.predecessor)
if self.predecessor != None:
mprint(self.predecessor)
with grpc.insecure_channel(self.predecessor) as channel:
stub = cpg.chordStub(channel)
stub.Noop(cp.Null(), timeout=self.config['rpc_timeout'])
except Exception as e:
mprint('predE', e)
self.predecessor = None
self.hashed_predecessor = None
def GetFingers(self, addr):
with grpc.insecure_channel(addr) as channel:
stub = cpg.chordStub(channel)
res = stub.GetFingers(cp.Null())
return res
def GetPredecessor(self, addr):
with grpc.insecure_channel(addr) as channel:
stub = cpg.chordStub(channel)
res = stub.GetPredecessor(cp.Null())
return res
def Leave(self, addr):
with grpc.insecure_channel(addr) as channel:
stub = cpg.chordStub(channel)
stub.Leave(cp.Null())
import chord_pb2 as cp
import chord_pb2_grpc as cpg
import grpc
from utils import gen_hash
h = gen_hash(10)
if __name__ == '__main__':
with grpc.insecure_channel('127.0.0.1:40000') as channel:
# h9 = (h('127.0.0.1:50001')+(1 << 9)) & ((1 << 10)-1)
stub = cpg.chordStub(channel)
# for i in range(10):
# stub.Put(cp.PutRequest(key=str(i), val=str(i), id=str(h(str(i)))))
print(stub.GetStatus(cp.Null()).successor_list)
# print(stub.Get(cp.GetRequest(key="0", id=str(h("0")))))
# print(stub.GetSuccessor(cp.GetSuccessorRequest(id=str(h9))))
exit()
# stub.Leave(cp.Null())
# stub.Resume(cp.ResumeRequest(server_addr='127.0.0.1:50003'))
key = 'key1'
val = 'val2'
hk = str(h(key))
print(key, val, hk)
res = stub.Put(cp.PutRequest(key=key, val=val, id=str(h(key))))
res = stub.Get(cp.GetRequest(key=key, id=hk))
print(res)
# # res = stub.Noop(cp.Null())
# # res = stub.GetSuccessor(
# # cp.GetSuccessorRequest(id=str(h('127.0.0.1:50001'))))
# print(res)
def Reset(self, req, ctx):
if self.leave:
raise Exception('Leave')
return
self.store = {}
return cp.Null()
def Noop(self, req, ctx):
if self.leave:
raise Exception('Leave')
return
return cp.Null()
def Leave(self, req, ctx):
if self.leave:
raise Exception('Leave')
return
self.leave = True
return cp.Null()
fgs[srv] = resp.server_addrs
time.sleep(12 * rg)
for srv in config['servers'][:rg]:
with grpc.insecure_channel(srv) as channel:
stub = cpg.chordStub(channel)
resp = stub.GetFingers(cp.Null())
if fgs[srv] != resp.server_addrs:
return False
return True
if __name__ == "__main__":
config = json.load(open('config.json', 'r'))
args = get_args()
kvs = {}
for _ in tqdm(range(args.n)):
rstr = uuid.uuid4().hex
kvs[rstr[:16]] = rstr[16:]
while not check_stable(config, 50):
time.sleep(1)
for i in tqdm(reversed(range(10))):
plens, kd = pl_kd(config, 5 * (i + 1), kvs)
pickle.dump((plens, kd), open(f'stats_5_{i+1}_{args.n}.pkl', 'wb'))
if i != 0:
for srv in config['servers'][5 * i:5 * (i + 1)]:
with grpc.insecure_channel(srv) as channel:
stub = cpg.chordStub(channel)
stub.Leave(cp.Null())
while not check_stable(config, 5 * i):
time.sleep(1)
