def find_primary(self, timeout=3, request_timeout=3):
"""
Find the identity of the primary in the network and return its identity
and the current term.
"""
primary_id = None
term = None
for _ in range(timeout):
for node in self.get_joined_nodes():
with node.node_client(request_timeout=request_timeout) as c:
try:
res = c.do("getPrimaryInfo", {})
if res.error is None:
primary_id = res.result["primary_id"]
term = res.term
break
else:
assert (
res.error["code"]
== infra.jsonrpc.ErrorCode.TX_PRIMARY_UNKNOWN
), "RPC error code is not TX_NOT_PRIMARY"
except TimeoutError:
pass
if primary_id is not None:
break
time.sleep(1)
if primary_id is None:
raise PrimaryNotFound
return (self._get_node_by_id(primary_id), term)
def wait_for_all_nodes_to_catch_up(self, primary, timeout=3):
"""
Wait for all nodes to have joined the network and globally replicated
all transactions executed on the primary (including the transactions
which added the nodes).
"""
with primary.node_client() as c:
res = c.do("getCommit", {})
local_commit_leader = res.commit
term_leader = res.term
for _ in range(timeout):
caught_up_nodes = []
for node in self.get_joined_nodes():
with node.node_client() as c:
resp = c.request("getCommit", {})
if resp.error is not None:
# Node may not have joined the network yet, try again
break
if (
resp.global_commit >= local_commit_leader
and resp.result["term"] == term_leader
):
caught_up_nodes.append(node)
if len(caught_up_nodes) == len(self.get_joined_nodes()):
break
time.sleep(1)
assert len(caught_up_nodes) == len(
self.get_joined_nodes()
), f"Only {len(caught_up_nodes)} (out of {len(self.get_joined_nodes())}) nodes have joined the network"
def find_primary(self, timeout=3, request_timeout=3):
"""
Find the identity of the primary in the network and return its identity
and the current term.
"""
primary_id = None
term = None
for _ in range(timeout):
for node in self.get_joined_nodes():
with node.node_client(request_timeout=request_timeout) as c:
try:
res = c.do("getPrimaryInfo", {})
if res.error is None:
primary_id = res.result["primary_id"]
term = res.term
break
else:
assert "Primary unknown" in res.error, res.error
except TimeoutError:
pass
if primary_id is not None:
break
time.sleep(1)
if primary_id is None:
raise PrimaryNotFound
return (self._get_node_by_id(primary_id), term)
def find_primary(self, timeout=3, request_timeout=3):
"""
Find the identity of the primary in the network and return its identity
and the current view.
"""
primary_id = None
view = None
end_time = time.time() + timeout
while time.time() < end_time:
for node in self.get_joined_nodes():
with node.node_client(request_timeout=request_timeout) as c:
try:
res = c.get("getPrimaryInfo")
if res.error is None:
primary_id = res.result["primary_id"]
view = res.result["current_term"]
break
else:
assert "Primary unknown" in res.error, res.error
except infra.clients.CCFConnectionException:
pass
if primary_id is not None:
break
time.sleep(0.1)
if primary_id is None:
raise PrimaryNotFound
return (self._get_node_by_id(primary_id), view)
def wait_for_all_nodes_to_catch_up(self, primary, timeout=3):
"""
Wait for all nodes to have joined the network and globally replicated
all transactions globally executed on the primary (including transactions
which added the nodes).
"""
end_time = time.time() + timeout
while time.time() < end_time:
with primary.node_client() as c:
resp = c.get("getCommit")
commit_leader = resp.result["commit"]
term_leader = resp.result["term"]
if commit_leader != 0:
break
time.sleep(0.1)
assert (
commit_leader != 0
), f"Primary {primary.node_id} has not made any progress yet (term: {term_leader}, commit: {commit_leader})"
while time.time() < end_time:
caught_up_nodes = []
for node in self.get_joined_nodes():
with node.node_client() as c:
resp = c.get(
"tx",
{
"view": term_leader,
"seqno": commit_leader
},
)
if resp.error is not None:
# Node may not have joined the network yet, try again
break
status = TxStatus(resp.result["status"])
if status == TxStatus.Committed:
caught_up_nodes.append(node)
elif status == TxStatus.Invalid:
raise RuntimeError(
f"Node {node.node_id} reports transaction ID {term_leader}.{commit_leader} is invalid and will never be committed"
)
else:
pass
if len(caught_up_nodes) == len(self.get_joined_nodes()):
break
time.sleep(0.1)
assert len(caught_up_nodes) == len(
self.get_joined_nodes()
), f"Only {len(caught_up_nodes)} (out of {len(self.get_joined_nodes())}) nodes have joined the network"
def wait_for_node_commit_sync(self, timeout=3):
"""
Wait for commit level to get in sync on all nodes. This is expected to
happen once CFTR has been established, in the absence of new transactions.
"""
for _ in range(timeout):
commits = []
for node in self.get_joined_nodes():
with node.node_client() as c:
r = c.request("getCommit", {})
commits.append(r.commit)
if [commits[0]] * len(commits) == commits:
break
time.sleep(1)
assert [commits[0]] * len(commits) == commits, "All nodes at the same commit"
def wait_for_state(self, node, state, timeout=3):
for _ in range(timeout):
try:
with node.node_client() as c:
r = c.request("getSignedIndex", {})
if r.result["state"] == state:
break
except ConnectionRefusedError:
pass
time.sleep(1)
else:
raise TimeoutError(
f"Timed out waiting for public ledger to be read on node {node.node_id}"
)
if state == "partOfNetwork":
self.status = ServiceStatus.OPEN
def wait_for_state(self, node, state, timeout=3):
end_time = time.time() + timeout
while time.time() < end_time:
try:
with node.node_client() as c:
r = c.get("getSignedIndex")
if r.result["state"] == state:
break
except ConnectionRefusedError:
pass
time.sleep(0.1)
else:
raise TimeoutError(
f"Timed out waiting for state {state} on node {node.node_id}")
if state == "partOfNetwork":
self.status = ServiceStatus.OPEN
def wait_for_node_commit_sync(self, consensus, timeout=3):
"""
Wait for commit level to get in sync on all nodes. This is expected to
happen once CFTR has been established, in the absence of new transactions.
"""
end_time = time.time() + timeout
while time.time() < end_time:
commits = []
for node in self.get_joined_nodes():
with node.node_client() as c:
r = c.get("commit")
commits.append(f"{r.view}.{r.seqno}")
if [commits[0]] * len(commits) == commits:
break
time.sleep(0.1)
expected = [commits[0]] * len(commits)
assert expected == commits, f"{commits} != {expected}"
def wait_for_node_commit_sync(self, consensus, timeout=3):
"""
Wait for commit level to get in sync on all nodes. This is expected to
happen once CFTR has been established, in the absence of new transactions.
"""
for _ in range(timeout):
commits = []
for node in self.get_joined_nodes():
with node.node_client() as c:
r = c.request("getCommit", {})
commits.append(r.commit)
if [commits[0]] * len(commits) == commits:
break
time.sleep(1)
# in pbft getCommit increments the commit version, so commits will not be the same
# but they should be in ascending order
assert ([commits[0]] * len(commits) == commits if consensus == "raft"
else sorted(commits) == commits), "All nodes in sync"
