def gc(self, key_3PC):
keys_to_remove = []
for key in self._signatures.keys():
if compare_3PC_keys(key, key_3PC) >= 0:
keys_to_remove.append(key)
for key in keys_to_remove:
self._signatures.pop(key, None)
def compare_last_ordered_3pc(node):
last_ordered_by_master = node.replicas._master_replica.last_ordered_3pc
comparison_results = {
compare_3PC_keys(replica.last_ordered_3pc, last_ordered_by_master)
for replica in node.replicas.values() if not replica.isMaster
}
assert len(comparison_results) == 1
return comparison_results.pop()
def on_catch_up_finished(self,
last_caught_up_3PC=None,
master_last_ordered_3PC=None):
if master_last_ordered_3PC and last_caught_up_3PC and \
compare_3PC_keys(master_last_ordered_3PC,
last_caught_up_3PC) > 0:
if self.isMaster:
self._caught_up_till_3pc(last_caught_up_3PC)
else:
self._ordering_service.first_batch_after_catchup = True
self._catchup_clear_for_backup()
self.stasher.process_all_stashed(STASH_CATCH_UP)
def test_prepare_in_queue_before_vc(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle):
"""
Test steps:
1. Sent N random requests.
2. Patching processNodeInBox method for node Delta.
This method will process only not Prepare messages and store in nodeInBox queue Prepare messages
3. Sent one request and check, that all Prepares are stored in nodeInBox queue and there is quorum of it
4. Compare last_ordered_3pc_key and last_prepared_certificate. Last_prepared_certificate must be greater then last ordered
5. ppSeqNo in last_prepared_certificate must be at least as ppSeqNo for queued Prepares msgs in nodeInBox queue
"""
def chk_quorumed_prepares_count(prepares, count):
pp_qourum = slow_node.quorums.prepare.value
assert len([
pp for key, pp in prepares.items()
if prepares.hasQuorum(pp.msg, pp_qourum)
]) == count
def patched_startViewChange(self, *args, **kwargs):
self.node.processNodeInBox = functools.partial(
TestNode.processNodeInBox, self.node)
ViewChanger.startViewChange(self, *args, **kwargs)
while stashed_msgs:
self.node.nodestack.rxMsgs.append(stashed_msgs.popleft())
"""Send REQ_COUNT txns"""
slow_node = txnPoolNodeSet[-1]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, REQ_COUNT)
"""Check that there is REQ_COUNT prepares with quorum in queue"""
chk_quorumed_prepares_count(slow_node.master_replica.prepares, REQ_COUNT)
"""Patch processNodeInBox method for saving Prepares in nodeInBox queue"""
not_processing_prepare(slow_node)
"""Send 1 txn"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, REQ_COUNT_AFTER_SLOW)
chk_quorumed_prepares_count(slow_node.master_replica.prepares, REQ_COUNT)
"""Get last ordered 3pc key (should be (0, REQ_COUNT))"""
ordered_lpc = slow_node.master_replica.last_ordered_3pc
"""Delay view_change_done messages"""
slow_node.nodeIbStasher.delay(vcd_delay(100))
"""Patch on_view_change_start method for reverting processNodeInBox method"""
slow_node.view_changer.startViewChange = functools.partial(
patched_startViewChange, slow_node.view_changer)
"""Initiate view change"""
ensure_view_change(looper, txnPoolNodeSet)
"""Last prepared certificate should take into account Prepares in nodeInBox queue too"""
expected_lpc = slow_node.master_replica.last_prepared_before_view_change
assert expected_lpc == (0, 11)
"""Last ordered key should be less than last_prepared_before_view_change"""
assert compare_3PC_keys(ordered_lpc, expected_lpc) > 0
def validate(self, msg):
view_no = getattr(msg, f.VIEW_NO.nm, None)
pp_seq_no = getattr(msg, f.PP_SEQ_NO.nm, None)
# ToDO: this checks should be performed in previous level (ReplicaService)
# 1. Check INSTANCE_ID
# if inst_id is None or inst_id != self.replica.instId:
# return DISCARD, INCORRECT_INSTANCE
# 2. Check pp_seq_no
if pp_seq_no == 0:
# should start with 1
return DISCARD, INCORRECT_PP_SEQ_NO
# 3. Check already ordered
if self.has_already_ordered(view_no, pp_seq_no):
return DISCARD, ALREADY_ORDERED
# 4. Check viewNo
if view_no > self._data.view_no:
return STASH_VIEW, FUTURE_VIEW
if view_no < self._data.view_no - 1:
return DISCARD, OLD_VIEW
if view_no == self._data.view_no - 1:
if not isinstance(msg, Commit):
return DISCARD, OLD_VIEW
if not self._data.legacy_vc_in_progress:
return DISCARD, OLD_VIEW
if self._data.legacy_last_prepared_before_view_change is None:
return DISCARD, OLD_VIEW
if compare_3PC_keys(
(view_no, pp_seq_no),
self._data.legacy_last_prepared_before_view_change) < 0:
return DISCARD, GREATER_PREP_CERT
if view_no == self._data.view_no and self._data.legacy_vc_in_progress:
return STASH_VIEW, FUTURE_VIEW
# ToDo: we assume, that only is_participating needs checking orderability
# If Catchup in View Change finished then process Commit messages
if self._data.is_synced and self._data.legacy_vc_in_progress:
return PROCESS, None
# 5. Check if Participating
if not self._data.is_participating:
return STASH_CATCH_UP, CATCHING_UP
# 6. Check watermarks
if not (self._data.low_watermark < pp_seq_no <=
self._data.high_watermark):
return STASH_WATERMARKS, OUTSIDE_WATERMARKS
return PROCESS, None
def catchupCompleted(self, ledgerId: int, last_3PC: Tuple = (0, 0)):
if ledgerId not in self.ledgerRegistry:
logger.error("{}{} called catchup completed for ledger {}".format(
CATCH_UP_PREFIX, self, ledgerId))
return
# Since multiple ledger will be caught up and catchups might happen
# multiple times for a single ledger, the largest seen
# ppSeqNo needs to be known.
if compare_3PC_keys(self.last_caught_up_3PC, last_3PC) > 0:
self.last_caught_up_3PC = last_3PC
self.mark_ledger_synced(ledgerId)
def process_catchup_finished(self, msg: CatchupFinished):
if compare_3PC_keys(msg.master_last_ordered,
msg.last_caught_up_3PC) > 0:
if self._data.is_master:
self.caught_up_till_3pc(msg.last_caught_up_3PC)
else:
if not self._data.is_primary:
self.catchup_clear_for_backup()
self._bus.send(
CatchupCheckpointsApplied(
last_caught_up_3PC=msg.last_caught_up_3PC,
master_last_ordered=msg.master_last_ordered))
def validate_3pc_msg(self, msg):
node = self.replica.node
inst_id = getattr(msg, f.INST_ID.nm, None)
view_no = getattr(msg, f.VIEW_NO.nm, None)
pp_seq_no = getattr(msg, f.PP_SEQ_NO.nm, None)
# 1. Check INSTANCE_ID
if inst_id is None or inst_id != self.replica.instId:
return DISCARD, INCORRECT_INSTANCE
# 2. Check pp_seq_no
if pp_seq_no == 0:
# should start with 1
return DISCARD, INCORRECT_PP_SEQ_NO
# 3. Check already ordered
if self.replica.has_already_ordered(view_no, pp_seq_no):
return DISCARD, ALREADY_ORDERED
# 4. Check viewNo
if view_no > self.replica.viewNo:
return STASH_VIEW, FUTURE_VIEW
if view_no < self.replica.viewNo - 1:
return DISCARD, OLD_VIEW
if view_no == self.replica.viewNo - 1:
if not isinstance(msg, Commit):
return DISCARD, OLD_VIEW
if not node.view_change_in_progress:
return DISCARD, OLD_VIEW
if self.replica.last_prepared_before_view_change is None:
return DISCARD, OLD_VIEW
if compare_3PC_keys(
(view_no, pp_seq_no),
self.replica.last_prepared_before_view_change) < 0:
return DISCARD, GREATER_PREP_CERT
if view_no == self.replica.viewNo and node.view_change_in_progress:
return STASH_VIEW, FUTURE_VIEW
# If Catchup in View Change finished then process Commit messages
if node.is_synced and node.view_change_in_progress:
return PROCESS, None
# 5. Check if Participating
if not node.isParticipating:
return STASH_CATCH_UP, CATCHING_UP
# 6. Check watermarks
if not (self.replica.h < pp_seq_no <= self.replica.H):
return STASH_WATERMARKS, OUTSIDE_WATERMARKS
return PROCESS, None
def test_3PC_key_comaparison():
assert compare_3PC_keys((1,2), (1,2)) == 0
assert compare_3PC_keys((1,3), (1,2)) < 0
assert compare_3PC_keys((1,2), (1,3)) > 0
assert compare_3PC_keys((1,2), (1,10)) > 0
assert compare_3PC_keys((1, 100), (2, 3)) > 0
assert compare_3PC_keys((1, 100), (4, 3)) > 0
assert compare_3PC_keys((2, 100), (1, 300)) < 0
assert min_3PC_key([(2, 100), (1, 300), (5, 600)]) == (1, 300)
assert min_3PC_key([(2, 100), (2, 300), (2, 600)]) == (2, 100)
assert min_3PC_key([(2, 100), (2, 300), (1, 600)]) == (1, 600)
assert max_3PC_key([(2, 100), (1, 300), (5, 6)]) == (5, 6)
assert max_3PC_key([(2, 100), (3, 20), (4, 1)]) == (4, 1)
assert max_3PC_key([(2, 100), (2, 300), (2, 400)]) == (2, 400)
def test_3PC_key_comaparison():
assert compare_3PC_keys((1, 2), (1, 2)) == 0
assert compare_3PC_keys((1, 3), (1, 2)) < 0
assert compare_3PC_keys((1, 2), (1, 3)) > 0
assert compare_3PC_keys((1, 2), (1, 10)) > 0
assert compare_3PC_keys((1, 100), (2, 3)) > 0
assert compare_3PC_keys((1, 100), (4, 3)) > 0
assert compare_3PC_keys((2, 100), (1, 300)) < 0
assert min_3PC_key([(2, 100), (1, 300), (5, 600)]) == (1, 300)
assert min_3PC_key([(2, 100), (2, 300), (2, 600)]) == (2, 100)
assert min_3PC_key([(2, 100), (2, 300), (1, 600)]) == (1, 600)
assert max_3PC_key([(2, 100), (1, 300), (5, 6)]) == (5, 6)
assert max_3PC_key([(2, 100), (3, 20), (4, 1)]) == (4, 1)
assert max_3PC_key([(2, 100), (2, 300), (2, 400)]) == (2, 400)
def clear_requests_and_fix_last_ordered(self):
if self.isMaster:
return
reqs_for_remove = []
for req in self.requests.values():
ledger_id, seq_no = self.node.seqNoDB.get_by_payload_digest(
req.request.payload_digest)
if seq_no is not None:
reqs_for_remove.append((req.request.digest, ledger_id, seq_no))
for key, ledger_id, seq_no in reqs_for_remove:
self.requests.ordered_by_replica(key)
self.requests.free(key)
self._ordering_service.requestQueues[int(ledger_id)].discard(key)
master_last_ordered_3pc = self.node.master_replica.last_ordered_3pc
if compare_3PC_keys(master_last_ordered_3pc, self.last_ordered_3pc) < 0 \
and self.isPrimary is False:
self.last_ordered_3pc = master_last_ordered_3pc
def test_3PC_key_comaparison():
# Because of INDY-1336 compare_3PC_keys, min_3PC_key, max_3PC_key
# taking into account only ppSeqNo part
assert compare_3PC_keys((1, 2), (1, 2)) == 0
assert compare_3PC_keys((1, 3), (1, 2)) < 0
assert compare_3PC_keys((1, 2), (1, 3)) > 0
assert compare_3PC_keys((1, 2), (1, 10)) > 0
assert compare_3PC_keys((1, 100), (2, 3)) < 0
assert compare_3PC_keys((1, 100), (4, 3)) < 0
assert compare_3PC_keys((2, 100), (1, 300)) > 0
assert min_3PC_key([(2, 100), (1, 300), (5, 600)]) == (2, 100)
assert min_3PC_key([(2, 100), (2, 300), (2, 600)]) == (2, 100)
assert min_3PC_key([(2, 100), (2, 300), (1, 600)]) == (2, 100)
assert max_3PC_key([(2, 100), (1, 300), (5, 6)]) == (1, 300)
assert max_3PC_key([(2, 100), (3, 20), (4, 1)]) == (2, 100)
assert max_3PC_key([(2, 100), (2, 300), (2, 400)]) == (2, 400)
def catchupCompleted(self,
ledgerId: int,
last_3PC: Optional[Tuple] = None):
# Since multiple ledger will be caught up and catchups might happen
# multiple times for a single ledger, the largest seen
# ppSeqNo needs to be known.
if last_3PC is not None \
and compare_3PC_keys(self.last_caught_up_3PC, last_3PC) > 0:
self.last_caught_up_3PC = last_3PC
ledgerInfo = self.getLedgerInfoByType(ledgerId)
ledgerInfo.canSync = False
ledgerInfo.state = LedgerState.synced
ledgerInfo.catchUpTill = None
if self.postAllLedgersCaughtUp:
if all(l.state == LedgerState.synced
for l in self.ledgerRegistry.values()):
self.postAllLedgersCaughtUp()
self.catchup_next_ledger(ledgerId)
def _remove_ordered_from_queue(self, last_caught_up_3PC=None):
"""
Remove any Ordered that the replica might be sending to node which is
less than or equal to `last_caught_up_3PC` if `last_caught_up_3PC` is
passed else remove all ordered, needed in catchup
"""
to_remove = []
for i, msg in enumerate(self.outBox):
if isinstance(msg, Ordered) and \
(not last_caught_up_3PC or
compare_3PC_keys((msg.viewNo, msg.ppSeqNo), last_caught_up_3PC) >= 0):
to_remove.append(i)
self.logger.trace('{} going to remove {} Ordered messages from outbox'.format(self, len(to_remove)))
# Removing Ordered from queue but returning `Ordered` in order that
# they should be processed.
removed = []
for i in reversed(to_remove):
removed.insert(0, self.outBox[i])
del self.outBox[i]
return removed
def catchupCompleted(self, ledgerId: int, last_3PC: Tuple = (0, 0)):
# Since multiple ledger will be caught up and catchups might happen
# multiple times for a single ledger, the largest seen
# ppSeqNo needs to be known.
if compare_3PC_keys(self.last_caught_up_3PC, last_3PC) > 0:
self.last_caught_up_3PC = last_3PC
if ledgerId not in self.ledgerRegistry:
logger.error("{} called catchup completed for ledger {}".format(
self, ledgerId))
return
ledgerInfo = self.getLedgerInfoByType(ledgerId)
ledgerInfo.done_syncing()
logger.info(
"{} completed catching up ledger {}, caught up {} in total".format(
self, ledgerId, ledgerInfo.num_txns_caught_up),
extra={'cli': True})
if self.postAllLedgersCaughtUp:
if all(l.state == LedgerState.synced
for l in self.ledgerRegistry.values()):
self.postAllLedgersCaughtUp()
def chk2():
# last_prepared of slow_node is less than fast_nodes
for rep in fast_master_replicas:
assert compare_3PC_keys(prepareds[slow_master_replica.node.name],
prepareds[rep.node.name]) > 0
def chk1():
for rep in fast_master_replicas:
assert compare_3PC_keys(
slow_master_replica.last_prepared_certificate_in_view(),
rep.last_prepared_certificate_in_view()) > 0
assert slow_master_replica.last_ordered_3pc == rep.last_ordered_3pc
def _has_already_ordered(self, view_no, pp_seq_no):
return compare_3PC_keys(
(view_no, pp_seq_no), self._data.last_ordered_3pc) >= 0
def _is_below_3pc_key(cp: CheckpointKey, key: Tuple[int, int]) -> bool:
return compare_3PC_keys((cp.view_no, cp.pp_seq_no), key) >= 0
def check_nodes_ordered_till(nodes: Iterable, view_no: int, pp_seq_no: int):
for node in nodes:
assert compare_3PC_keys((view_no, pp_seq_no), node.master_replica.last_ordered_3pc) >= 0
def _on_ledger_sync_complete(self, msg: LedgerCatchupComplete):
if msg.last_3pc is not None and compare_3PC_keys(self.last_caught_up_3PC, msg.last_3pc) > 0:
self.last_caught_up_3PC = msg.last_3pc
