def acceptor_pre_accept(q: Quorum, slot: Slot, peer: int,
pre_accept: packet.PreAcceptRequest):
try:
inst = yield Store(
slot,
InstanceStoreState(
pre_accept.ballot,
State(Stage.PreAccepted, pre_accept.command, pre_accept.seq,
pre_accept.deps))) # type: InstanceStoreState
deps_comm_mask = []
for dep_slot in inst.state.deps:
xx = yield Load(dep_slot) # type: InstanceStoreState
deps_comm_mask.append(xx.state.stage >= Stage.Committed)
yield LeaderStop(slot, 'acceptor')
yield Send(
peer,
packet.PreAcceptResponseAck(slot, inst.ballot, inst.state.seq,
inst.state.deps, deps_comm_mask))
except IncorrectStage:
return
except IncorrectBallot as e:
yield Send(peer,
packet.PreAcceptResponseNack(slot, e.inst.ballot, 'BALLOT'))
return
except SlotTooOld:
yield Send(peer, packet.DivergedResponse(slot))
def event(self, x):
if isinstance(x, Packet):
assert isinstance(x.payload, packet.ClientRequest)
# we may use the clientrequest as a way of keeping the knowledge of whom to reply.
# client dests are then
loaded = yield LoadCommandSlot(x.payload.command.id)
# todo: this. here we assume that slots are not correlated with commands.
if loaded is None:
self.st_starts += 1
slot = yield LeaderStart(x.payload.command)
else:
self.st_restarts += 1
slot, inst = loaded
inst: Optional[InstanceStoreState]
# logger.error(f'{self.quorum.replica_id} Learned about a new client {x.origin} of slot {slot} {inst.state.command}')
if inst.state.stage >= Stage.Committed:
yield Send(x.origin,
packet.ClientResponse(inst.state.command))
slot: Slot
self.clients[slot] = x.origin
if x.origin not in self.peers:
self.peers[x.origin] = []
self.peers[x.origin].append(slot)
elif isinstance(x, Tick):
if x.id % 330 == 0:
logger.error(
f'{self.quorum.replica_id} St={self.st_starts}/{self.st_restarts}'
)
elif isinstance(x, InstanceState):
if x.slot in self.clients and x.inst.state.stage == Stage.Committed:
# print('REPLY')
yield Send(self.clients[x.slot],
packet.ClientResponse(x.inst.state.command))
peer = self.peers[self.clients[x.slot]]
peer.remove(x.slot)
del self.clients[x.slot]
else:
assert False, x
yield Reply()
def acceptor_prepare(q: Quorum, slot: Slot, peer: int,
prepare: packet.PrepareRequest):
try:
inst = yield Load(slot) # type: InstanceStoreState
except SlotTooOld:
yield Send(peer, packet.DivergedResponse(slot))
else:
if prepare.ballot < inst.ballot:
yield Send(peer, packet.PrepareResponseNack(slot, inst.ballot))
return
yield Send(
peer,
packet.PrepareResponseAck(slot, prepare.ballot, inst.state.command,
inst.state.seq, inst.state.deps,
inst.state.stage))
def acceptor_commit(q: Quorum, slot: Slot, peer: int,
commit: packet.CommitRequest):
try:
inst = yield Store(slot,
InstanceStoreState(
commit.ballot,
State(Stage.Committed, commit.command,
commit.seq,
commit.deps))) # type: InstanceStoreState
yield LeaderStop(slot, 'acceptor')
yield Send(peer, packet.AcceptResponseAck(slot, inst.ballot))
except IncorrectBallot as e:
return
except IncorrectStage as e:
return
except SlotTooOld:
yield Send(peer, packet.DivergedResponse(slot))
def event(self, x):
if isinstance(x, Packet):
if isinstance(x.payload, packet.PingRequest):
yield Send(x.origin, packet.PongResponse(x.payload.id))
elif isinstance(x.payload, packet.PongResponse):
if x.payload.id == self.last_ping_id.get(x.origin, -1):
time = datetime.now() - self.last_ping[x.origin]
self.pings_rcvd[x.origin] = self.pings_rcvd.get(
x.origin, 0) + 1
self.pings_times[x.origin] = (
self.pings_times.get(x.origin, []) +
[time.total_seconds()])[:self.keep_times]
else:
# todo: reordered pings
pass
else:
assert False, ''
elif isinstance(x, Tick):
if x.id % self.ping_every_tick == 0:
now = datetime.now()
for peer in self.quorum.peers:
self.last_ping[peer] = now
self.last_ping_id[peer] = self.last_ping_id.get(peer, 0)
self.pings_sent[peer] = self.pings_sent.get(peer, 0) + 1
yield Send(peer,
packet.PingRequest(self.last_ping_id[peer]))
if x.id % 300 == 0:
pings_repl = sorted((k, f'{sum(v)/len(v)*1000:0.2f}ms')
for k, v in self.pings_times.items())
pings_recv = sorted((k, f'{v}/{self.pings_sent[k]}')
for k, v in self.pings_rcvd.items())
logger.error(
f'{self.quorum.replica_id} {pings_repl} {pings_recv}')
else:
assert False, ''
yield Reply()
def leader_accept(q: Quorum, slot: Slot, inst: InstanceStoreState):
for peer in q.peers:
yield Send(peer, packet.AcceptRequest(slot, inst.ballot, inst.state.command, inst.state.seq, inst.state.deps))
replies = []
while len(replies) + 1 < q.slow_size:
_, (ack, nack) = yield Receive.any(
packet.AcceptResponseAck,
packet.AcceptResponseNack,
)
ack: Optional[packet.AcceptResponseAck]
nack: Optional[packet.AcceptResponseNack]
if ack:
if ack.ballot != inst.ballot:
# logger.debug(f'{q.replica_id} accept Raising do to > {ack} {nack} {ack} {inst}')
# raise ExplicitPrepare('accept:BALLOT')
pass
else:
replies.append(ack)
if nack:
# logger.debug(f'{q.replica_id} accept Raising do to nack {ack} {nack} {ack} {inst}')
pass
# raise ExplicitPrepare('accept:NACK')
inst = yield Store(
slot,
InstanceStoreState(
inst.ballot,
State(
Stage.Committed,
inst.state.command,
inst.state.seq,
inst.state.deps
)
)
)
yield from leader_commit(q, slot, inst)
def leader_explicit_prepare(q: Quorum, slot: Slot, reason=None):
inst = yield Load(slot) # type: InstanceStoreState
try:
ballot = inst.ballot.next(q.replica_id)
inst = yield Store(
slot,
InstanceStoreState(
ballot,
inst.state
)
) # type: InstanceStoreState
except (IncorrectBallot, IncorrectStage) as e:
return
for peer in q.peers:
yield Send(peer, packet.PrepareRequest(slot, inst.ballot))
replies = list() # type: List[RecoveryReply]
replies.append(
RecoveryReply(
q.replica_id,
packet.PrepareResponseAck(
slot,
ballot,
inst.state.command,
inst.state.seq,
inst.state.deps,
inst.state.stage
)
)
)
while len(replies) < q.slow_size:
peer, (ack, nack, diverged) = yield Receive.any(
packet.PrepareResponseAck,
packet.PreAcceptResponseNack,
packet.DivergedResponse
)
peer: int
ack: Optional[packet.PrepareResponseAck]
nack: Optional[packet.PrepareResponseNack]
diverged: Optional[packet.DivergedResponse]
if ack:
if ack.ballot != ballot:
continue
# todo: should replies from non-current ballots be ignored?
replies.append(RecoveryReply(peer, ack))
if nack:
if nack.ballot != ballot:
continue
# logger.debug(f'{q.replica_id} explicit prepare NACK {inst} {ballot}')
raise ExplicitPrepare('explicit:NACK')
if diverged:
raise Exception("I have diverged")
len_rep = len(replies)
max_ballot = max(x.r.ballot for x in replies)
replies = [x for x in replies if x.r.ballot == max_ballot]
max_state = max(x.r.state for x in replies)
replies = [x for x in replies if x.r.state == max_state]
len_rep_fil = len(replies)
loglog = lambda ni, tag=None: logger.info(f'{q.replica_id} Storing {ni} {len_rep} {len_rep_fil} {ballot} {tag}')
loglog = lambda ni, tag=None: None
if max_state == Stage.Committed:
reply = [x for x in replies if x.r.state == max_state][0].r
inst = yield Store(
slot,
InstanceStoreState(
ballot,
State(
Stage.Committed,
reply.command,
reply.seq,
reply.deps
)
)
)
yield from leader_commit(q, slot, inst)
return
elif max_state == Stage.Accepted:
reply = [x for x in replies if x.r.state == max_state][0].r
inst = yield Store(
slot,
InstanceStoreState(
ballot,
State(
Stage.Accepted,
reply.command,
reply.seq,
reply.deps
)
)
)
yield from leader_accept(q, slot, inst)
return
identic_keys = defaultdict(list)
for r in replies:
identic_keys[(r.r.state, r.r.command.id if r.r.command else None, r.r.seq, tuple(sorted(r.r.deps)))].append(r)
identic_groups = [
(x, list(y))
for x, y in identic_keys.items()
] # type: List[Tuple[Tuple[InstanceState, Command, int, List[int]], List[RecoveryReply]]]
identic_groups = [
y
for x, y in identic_groups
if len(y) >= q.slow_size - 1 and
all(z.p != slot.replica_id for z in y) and x[0] == Stage.PreAccepted
]
if len(identic_groups):
reply = identic_groups[0][0].r
new_inst = yield Store(
slot,
InstanceStoreState(
ballot,
State(
Stage.Accepted,
reply.command,
reply.seq,
reply.deps
)
)
)
yield from leader_accept(q, slot, new_inst)
elif max_state == Stage.PreAccepted:
reply = [x for x in replies if x.r.state == max_state][0].r
new_inst = yield Store(
slot,
InstanceStoreState(
ballot,
State(
Stage.PreAccepted,
reply.command,
reply.seq,
reply.deps
)
)
)
yield from leader_pre_accept(q, slot, new_inst, False)
else:
new_inst = yield Store(
slot,
InstanceStoreState(
ballot,
State(
Stage.PreAccepted,
None,
0,
[]
)
)
)
yield from leader_pre_accept(q, slot, new_inst, False)
def leader_commit(q: Quorum, slot: Slot, inst: InstanceStoreState):
for peer in q.peers:
yield Send(peer, packet.CommitRequest(slot, inst.ballot, inst.state.command, inst.state.seq, inst.state.deps))
def leader_pre_accept(q: Quorum, slot: Slot, inst: InstanceStoreState, allow_fast: True):
for peer in q.peers:
yield Send(
peer,
packet.PreAcceptRequest(slot, inst.ballot, inst.state.command, inst.state.seq, inst.state.deps)
)
replies = []
replies: List[packet.PreAcceptResponseAck]
while len(replies) + 1 < q.fast_size:
_, (ack, nack) = yield Receive.any(
packet.PreAcceptResponseAck,
packet.PreAcceptResponseNack,
)
ack: Optional[packet.PreAcceptResponseAck]
nack: Optional[packet.PreAcceptResponseNack]
if ack:
if ack.ballot != inst.ballot:
# logger.debug(f'{q.replica_id} pre_accept Raising do to > {ack} {nack} {ack} {inst}')
# raise ExplicitPrepare('pre_accept:BALLOT')
pass
else:
replies.append(ack)
if nack:
# logger.debug(f'{q.replica_id} pre_accept Raising do to nack {ack} {nack} {ack} {inst}')
pass
# raise ExplicitPrepare('pre_accept:NACK')
if allow_fast and (
all(x.deps == inst.state.deps for x in replies) and
all(x.seq == inst.state.seq for x in replies)
):
inst = yield Store(
slot,
InstanceStoreState(
inst.ballot,
State(
Stage.Committed,
inst.state.command,
inst.state.seq,
inst.state.deps
)
)
)
yield from leader_commit(q, slot, inst)
else:
seq = max(inst.state.seq, max(x.seq for x in replies))
deps = []
for rep in replies:
deps.extend(rep.deps)
deps = sorted(set(deps))
inst = yield Store(
slot,
InstanceStoreState(
inst.ballot,
State(
Stage.Accepted,
inst.state.command,
seq,
deps
)
)
)
yield from leader_accept(q, slot, inst)