def event(self, x):
if isinstance(x, Send):
self.send(x)
yield Reply()
elif isinstance(x, Tick):
if x.id % 330 == 0 and hasattr(self, 'net_stats'):
rcv = sorted([(k, v) for k, v in self.net_stats.recv.items()])
snd = sorted([(k, v) for k, v in self.net_stats.send.items()])
logger.error(
f'{self.quorum.replica_id} {self.net_stats.traffic_recv} {self.net_stats.traffic_send} {rcv} {snd}'
)
yield Reply()
else:
assert False, x
def event(self, x):
if isinstance(x, Tick):
def lenx(iter_obj):
i = 0
for x in iter_obj:
i += 1
return i
if x.id % 330 == 0:
instc = sorted((x.name, lenx(y)) for x, y in groupby(
sorted(x.state.stage for x in self.store.inst.values())))
logger.error(f'{self.quorum.replica_id} {instc}')
yield Reply()
elif isinstance(x, LoadCommandSlot):
yield Reply(self.store.load_cmd_slot(x.id))
elif isinstance(x, Load):
yield Reply(self.store.load(x.slot).inst)
elif isinstance(x, Store):
# todo: all stores modify timeouts
old, new = self.store.update(x.slot, x.inst)
self.log(lambda: f'{self.quorum.replica_id}\t{x.slot}\t{new}\n')
deps_comm = []
for d in new.state.deps:
r = self.store.load(d)
if not r.exists:
r_old, r_new = self.store.update(d, r.inst)
yield InstanceState(d, r_new)
deps_comm.append(r.inst.state.stage == Stage.Committed)
if new.ballot.b > 10:
logger.error(f'{self.quorum.replica_id} {x.slot} {new} HW')
yield InstanceState(x.slot, new)
yield Reply(new)
elif isinstance(x, CheckpointEvent):
self.store.set_cp(x.at)
yield Reply(None)
else:
assert False, x
def event(self, x):
if isinstance(x, Packet):
assert isinstance(x.payload, packet.ClientRequest)
# we may use the clientrequest as a way of keeping the knowledge of whom to reply.
# client dests are then
loaded = yield LoadCommandSlot(x.payload.command.id)
# todo: this. here we assume that slots are not correlated with commands.
if loaded is None:
self.st_starts += 1
slot = yield LeaderStart(x.payload.command)
else:
self.st_restarts += 1
slot, inst = loaded
inst: Optional[InstanceStoreState]
# logger.error(f'{self.quorum.replica_id} Learned about a new client {x.origin} of slot {slot} {inst.state.command}')
if inst.state.stage >= Stage.Committed:
yield Send(x.origin,
packet.ClientResponse(inst.state.command))
slot: Slot
self.clients[slot] = x.origin
if x.origin not in self.peers:
self.peers[x.origin] = []
self.peers[x.origin].append(slot)
elif isinstance(x, Tick):
if x.id % 330 == 0:
logger.error(
f'{self.quorum.replica_id} St={self.st_starts}/{self.st_restarts}'
)
elif isinstance(x, InstanceState):
if x.slot in self.clients and x.inst.state.stage == Stage.Committed:
# print('REPLY')
yield Send(self.clients[x.slot],
packet.ClientResponse(x.inst.state.command))
peer = self.peers[self.clients[x.slot]]
peer.remove(x.slot)
del self.clients[x.slot]
else:
assert False, x
yield Reply()
def event(self, x):
if isinstance(x, packet.Packet) and isinstance(x.payload,
PACKET_LEADER):
slot = x.payload.slot # type: Slot
if slot in self.waiting_for:
yield from self.run_sub(
slot, (x.origin,
Receive.from_waiting(self.waiting_for.pop(slot),
x.payload)))
yield Reply()
elif isinstance(x, InstanceState):
if x.inst.state.stage >= Stage.Committed:
if x.slot in self.waiting_for:
del self.waiting_for[x.slot]
if x.slot in self.subs:
del self.subs[x.slot]
yield Reply()
elif isinstance(x, LeaderStart):
slot = Slot(self.quorum.replica_id, self.next_instance_id)
self.next_instance_id += 1
self.subs[slot] = leader_client_request(self.quorum, slot,
x.command)
yield from self.run_sub(slot)
yield Reply(slot)
elif isinstance(x, LeaderStop):
if x.slot in self.waiting_for:
del self.waiting_for[x.slot]
if x.slot in self.subs:
del self.subs[x.slot]
yield Reply()
elif isinstance(x, LeaderExplicitPrepare):
prev = self.subs.get(x.slot) is not None
self.subs[x.slot] = leader_explicit_prepare(
self.quorum, x.slot, x.reason)
yield from self.run_sub(x.slot)
yield Reply(prev)
elif isinstance(x, CheckpointEvent):
ctr = 0
for slot in between_checkpoints(*self.cp.cycle(x.at)):
if slot in self.subs:
ctr += 1
del self.subs[slot]
if slot in self.waiting_for:
ctr += 1
del self.waiting_for[slot]
logger.error(
f'{self.quorum.replica_id} cleaned old things between {ctr}: {self.cp}'
)
yield Reply()
else:
assert False, x
def main():
q = Quorum([1, 2, 3, 4], 0, 0)
c = Configuration(1, 3, 33, 10 * 33)
state = StateActor().run()
clients = ClientsActor().run()
leader = LeaderCoroutine(q).run()
acceptor = AcceptorCoroutine(q, c).run()
net = NetActor().run()
next(state)
next(clients)
next(leader)
next(acceptor)
next(net)
m = MainCoroutine(
state,
clients,
leader,
acceptor,
net,
).run()
# print('S', next(m))
x = next(m)
while True:
assert isinstance(x, Wait), x
x = m.send(
Packet(0, 0, 'ClientRequest',
ClientRequest(Command(uuid4(), Checkpoint(1)))))
print(x)
# while isinstance(x, ClientNetComm):
# print(x)
# x = m.send(Reply(True))
print('>>>>>>>>', x)
x = m.send(Reply(True))
print(next(m))
def event(self, x):
if isinstance(x, Packet):
if isinstance(x.payload, packet.PingRequest):
yield Send(x.origin, packet.PongResponse(x.payload.id))
elif isinstance(x.payload, packet.PongResponse):
if x.payload.id == self.last_ping_id.get(x.origin, -1):
time = datetime.now() - self.last_ping[x.origin]
self.pings_rcvd[x.origin] = self.pings_rcvd.get(
x.origin, 0) + 1
self.pings_times[x.origin] = (
self.pings_times.get(x.origin, []) +
[time.total_seconds()])[:self.keep_times]
else:
# todo: reordered pings
pass
else:
assert False, ''
elif isinstance(x, Tick):
if x.id % self.ping_every_tick == 0:
now = datetime.now()
for peer in self.quorum.peers:
self.last_ping[peer] = now
self.last_ping_id[peer] = self.last_ping_id.get(peer, 0)
self.pings_sent[peer] = self.pings_sent.get(peer, 0) + 1
yield Send(peer,
packet.PingRequest(self.last_ping_id[peer]))
if x.id % 300 == 0:
pings_repl = sorted((k, f'{sum(v)/len(v)*1000:0.2f}ms')
for k, v in self.pings_times.items())
pings_recv = sorted((k, f'{v}/{self.pings_sent[k]}')
for k, v in self.pings_rcvd.items())
logger.error(
f'{self.quorum.replica_id} {pings_repl} {pings_recv}')
else:
assert False, ''
yield Reply()
def route(self, req, d=0):
if isinstance(req, STATE_MSGS):
return self.run_sub(self.state, req, d)
elif isinstance(req, LEADER_MSGS):
return self.run_sub(self.leader, req, d)
elif isinstance(req, STATE_MSGS):
return self.run_sub(self.state, req, d)
elif isinstance(req, NET_MSGS):
return self.run_sub(self.net, req, d)
elif isinstance(req, STATE_EVENTS):
self.run_sub(self.clients, req, d)
self.run_sub(self.acceptor, req, d)
self.run_sub(self.leader, req, d)
self.run_sub(self.executor, req, d)
return Reply(None)
elif isinstance(req, CHECKPOINT_EVENTS):
self.run_sub(self.acceptor, req, d)
self.run_sub(self.state, req, d)
self.run_sub(self.leader, req, d)
elif isinstance(req, Reply):
return req
else:
self._trace(f'Unroutable {req}')
raise Unroutable(req)
def event(self, x):
if isinstance(x, InstanceState):
if x.inst.state.stage >= Stage.Committed:
# self.log(lambda: f'{self.quorum.replica_id}\tSTAT\t{x.slot}\t{x.inst}\n')
if not self.is_executed(x.slot) and not self.is_executing(x.slot):
self.executing[x.slot] = True
self.ctr += 1
self.log(lambda: f'{self.quorum.replica_id}\tDPH0\t{self.dph.ccs}\n')
unlocked_list = self.dph.ready(x.slot, [x for x in x.inst.state.deps if not self.is_executed(x)])
self.log(lambda: f'{self.quorum.replica_id}\tDPH1\t{self.dph.ccs}\n')
self.log(lambda: f'{self.quorum.replica_id}\tDPH2\t{unlocked_list}\n')
# if self.ctr % 100:
# self.log(lambda: f'{self.quorum.replica_id}\tDPHX\t{self.dph.ccs}\n')
try:
for checkpoint in self.build_execute_pending(unlocked_list):
xx = self.store.load(checkpoint).inst
yield CheckpointEvent(checkpoint, {x.replica_id: x for x in xx.state.deps})
except:
self.log(lambda: f'{self.quorum.replica_id}\tDPHz\t{self.dph.ccs}\n')
raise
elif isinstance(x, Tick):
if x.id % 330 == 0:
totd = sum(x.depth() for _, x in self.dph.ccs.items())
logger.error(
f'{self.quorum.replica_id} Exec={self.st_exec} Deps={len(self.dph.ccs)} Depth={totd}')
if totd > 10:
for cc in self.dph.ccs.values():
logger.error(
f'{self.quorum.replica_id} {cc}')
else:
assert False, x
yield Reply()
def event(self, x):
if isinstance(x, packet.Packet) and isinstance(x.payload,
PACKET_ACCEPTOR):
slot = x.payload.slot
if slot not in self.subs:
self.subs[slot] = acceptor_single_ep(self.quorum, slot)
yield from self.run_sub(slot)
if slot in self.waiting_for:
rcv = Receive.from_waiting(self.waiting_for.pop(slot),
x.payload)
yield from self.run_sub(slot, (x.origin, rcv))
elif isinstance(x, InstanceState):
if x.slot in self.slots_timeouts:
slot_tick = self.slots_timeouts[x.slot]
del self.timeouts_slots[slot_tick][x.slot]
if x.inst.state.stage == Stage.Committed:
del self.slots_timeouts[x.slot]
if x.inst.state.stage < Stage.Committed:
tick = self.tick + self.config.timeout + random.randint(
0, self.config.timeout_range)
# print(self.quorum.replica_id, 'SET TIMEOUT ', self.tick, tick)
self.slots_timeouts[x.slot] = tick
if tick not in self.timeouts_slots:
self.timeouts_slots[tick] = {}
self.timeouts_slots[tick][x.slot] = True
if x.inst.state.stage >= Stage.Committed:
if x.slot in self.waiting_for:
del self.waiting_for[x.slot]
if x.slot in self.subs:
del self.subs[x.slot]
elif isinstance(x, Tick):
self.tick = x.id
# print(self.quorum.replica_id, self.tick)
if x.id in self.timeouts_slots:
to_start = []
for slot, truth in self.timeouts_slots[x.id].items():
to_start.append(slot)
del self.slots_timeouts[slot]
del self.timeouts_slots[x.id]
for slot in to_start:
yield LeaderExplicitPrepare(slot, 'TIMEOUT')
if x.id % self.config.checkpoint_each == 0:
checkpoint_id = x.id // self.config.checkpoint_each
r_idx = sorted(self.quorum.peers +
[self.quorum.replica_id]).index(
self.quorum.replica_id)
q_length = self.quorum.full_size
if checkpoint_id % q_length == r_idx:
import sys
cp_cmd = Command(
CommandID.create(),
Checkpoint(checkpoint_id * q_length + r_idx))
print('SIZEOF', getsize(cp_cmd))
yield LeaderStart(cp_cmd)
last_tick = x.id
# fmtd = '\n'.join(f'\t\t{x.name}: {y}' for x, y in sorted((y, len(list(x))) for y, x in
# groupby(sorted([v.state.stage for k, v in
# self.replica.store.inst.items()]))))
#
# fmtd3 = '\n'.join(
# f'\t\t{x}: {y}' for x, y in sorted([(k, v) for k, v in self.state.packet_counts.items()]))
# fmtd = ''
# fmtd3 = ''
# logger.debug(
# f'\n{self.quorum.replica_id}\t{x.id}\n\tInstances:\n{fmtd}\n\tPackets:\n{fmtd3}')
elif isinstance(x, CheckpointEvent):
ctr = 0
for slot in between_checkpoints(*self.cp.cycle(x.at)):
if slot in self.subs:
ctr += 1
del self.subs[slot]
if slot in self.waiting_for:
ctr += 1
del self.waiting_for[slot]
logger.error(
f'{self.quorum.replica_id} cleaned old things between {ctr}: {self.cp}'
)
else:
assert False, x
yield Reply()