def _start_catchup_if_needed(self, key: CheckpointKey):
if self._have_own_checkpoint(key):
return
unknown_stabilized = self._unknown_stabilized_checkpoints()
lag_in_checkpoints = len(unknown_stabilized)
if lag_in_checkpoints <= self.STASHED_CHECKPOINTS_BEFORE_CATCHUP:
return
last_key = sorted(unknown_stabilized,
key=lambda v: (v.view_no, v.pp_seq_no))[-1]
if self.is_master:
if not self._data.is_primary:
logger.display(
'{} has lagged for {} checkpoints so the catchup procedure starts'
.format(self, lag_in_checkpoints))
self._bus.send(NeedMasterCatchup())
else:
logger.info(
'{} has lagged for {} checkpoints so adjust last_ordered_3pc to {}, '
'shift watermarks and clean collections'.format(
self, lag_in_checkpoints, last_key.pp_seq_no))
# Adjust last_ordered_3pc, shift watermarks, clean operational
# collections and process stashed messages which now fit between
# watermarks
# TODO: Actually we might need to process view_no from last_key as well, however
# it wasn't processed before, and it will go away when INDY-1336 gets implemented
key_3pc = (self.view_no, last_key.pp_seq_no)
self._bus.send(
NeedBackupCatchup(inst_id=self._data.inst_id,
caught_up_till_3pc=key_3pc))
self.caught_up_till_3pc(key_3pc)
def check_catchup_started(till_seq_no: int):
if is_master:
master_catchup_handler.assert_called_once_with(NeedMasterCatchup())
backup_catchup_handler.assert_not_called()
else:
master_catchup_handler.assert_not_called()
backup_catchup_handler.assert_called_once_with(
NeedBackupCatchup(inst_id=checkpoint_service._data.inst_id,
caught_up_till_3pc=(checkpoint_service.view_no,
till_seq_no)))
def test_start_catchup_on_quorum_of_stashed_checkpoints(
checkpoint_service, checkpoint, pre_prepare, tconf, ordered,
validators, is_master):
master_catchup_handler = Mock()
backup_catchup_handler = Mock()
checkpoint_service._bus.subscribe(NeedMasterCatchup,
master_catchup_handler)
checkpoint_service._bus.subscribe(NeedBackupCatchup,
backup_catchup_handler)
quorum = checkpoint_service._data.quorums.checkpoint.value
print(quorum)
n = len(validators)
assert quorum == n - getMaxFailures(n) - 1
senders = ["sender{}".format(i) for i in range(quorum + 1)]
old_key = (1, tconf.CHK_FREQ)
key = (old_key[1] + 1, old_key[1] + tconf.CHK_FREQ)
new_checkpoint = Checkpoint(instId=ordered.instId,
viewNo=ordered.viewNo,
seqNoStart=key[0],
seqNoEnd=key[1],
digest=cp_digest(1, 1))
for sender in senders[:quorum]:
assert not checkpoint_service._do_process_checkpoint(
checkpoint, sender)
assert checkpoint_service._stashed_recvd_checkpoints[
checkpoint.viewNo][old_key][sender] == checkpoint
for sender in senders[:quorum - 1]:
assert not checkpoint_service._do_process_checkpoint(
new_checkpoint, sender)
assert checkpoint_service._stashed_recvd_checkpoints[
checkpoint.viewNo][key][sender] == new_checkpoint
assert not checkpoint_service._do_process_checkpoint(
new_checkpoint, senders[quorum - 1])
if is_master:
assert checkpoint_service._data.low_watermark == key[1]
master_catchup_handler.assert_called_once_with(NeedMasterCatchup())
else:
backup_catchup_handler.assert_called_once_with(
NeedBackupCatchup(inst_id=checkpoint_service._data.inst_id,
caught_up_till_3pc=(checkpoint_service.view_no,
key[1])))
def _start_catchup_if_needed(self):
stashed_checkpoint_ends = self._stashed_checkpoints_with_quorum()
lag_in_checkpoints = len(stashed_checkpoint_ends)
if self._checkpoint_state:
(s, e) = firstKey(self._checkpoint_state)
# If the first stored own checkpoint has a not aligned lower bound
# (this means that it was started after a catch-up), is complete
# and there is a quorumed stashed checkpoint from other replicas
# with the same end then don't include this stashed checkpoint
# into the lag
if s % self._config.CHK_FREQ != 0 \
and self._checkpoint_state[(s, e)].seqNo == e \
and e in stashed_checkpoint_ends:
lag_in_checkpoints -= 1
is_stashed_enough = \
lag_in_checkpoints > self.STASHED_CHECKPOINTS_BEFORE_CATCHUP
if not is_stashed_enough:
return
if self.is_master:
self._logger.display(
'{} has lagged for {} checkpoints so updating watermarks to {}'
.format(self, lag_in_checkpoints, stashed_checkpoint_ends[-1]))
self.set_watermarks(low_watermark=stashed_checkpoint_ends[-1])
if not self._data.is_primary:
self._logger.display(
'{} has lagged for {} checkpoints so the catchup procedure starts'
.format(self, lag_in_checkpoints))
self._bus.send(NeedMasterCatchup())
else:
self._logger.info(
'{} has lagged for {} checkpoints so adjust last_ordered_3pc to {}, '
'shift watermarks and clean collections'.format(
self, lag_in_checkpoints, stashed_checkpoint_ends[-1]))
# Adjust last_ordered_3pc, shift watermarks, clean operational
# collections and process stashed messages which now fit between
# watermarks
key_3pc = (self.view_no, stashed_checkpoint_ends[-1])
self._bus.send(
NeedBackupCatchup(inst_id=self._data.inst_id,
caught_up_till_3pc=key_3pc))
self.caught_up_till_3pc(key_3pc)