def test_equivocation(intercepted_two_node_network):
"""
Generate an equivocating block and check the system can detect it.
"""
nodes = intercepted_two_node_network.docker_nodes
for node in nodes:
node.proxy_server.set_interceptor(
GenerateEquivocatingBlocksGossipInterceptor)
node = nodes[0]
account = node.genesis_account
block_hash = node.deploy_and_get_block_hash(account,
Contract.HELLO_NAME_DEFINE)
logging.info(f" =============> REAL BLOCK: {block_hash}")
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
block_hash = None
if nodes[0].proxy_server.interceptor.equivocating_block:
block_hash = nodes[
0].proxy_server.interceptor.equivocating_block.block_hash
receiver_node = nodes[1]
logging.info(
f" =============> EQUIVOCATING BLOCK: {block_hash.hex()}")
wait_for_block_hash_propagated_to_all_nodes([receiver_node],
block_hash.hex())
assert "Found equivocation:" in receiver_node.logs()
def test_gossip_proxy(intercepted_two_node_network):
nodes = intercepted_two_node_network.docker_nodes
node = nodes[0]
tls_certificate_path = node.config.tls_certificate_local_path()
tls_parameters = {
"--certificate-file": tls_certificate_path,
"--node-id": extract_common_name(tls_certificate_path),
}
cli = CLI(nodes[0], tls_parameters=tls_parameters)
account = cli.node.genesis_account
cli.set_default_deploy_args(
"--from",
account.public_key_hex,
"--private-key",
cli.private_key_path(account),
"--public-key",
cli.public_key_path(account),
"--payment",
cli.resource(Contract.STANDARD_PAYMENT),
"--payment-args",
cli.payment_json,
)
cli("deploy", "--session", cli.resource(Contract.HELLO_NAME_DEFINE))
block_hash = cli("propose")
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
def test_two_network_repeated_deploy(two_node_network):
"""
Test that a repeated deploy is rejected on a different node
"""
nodes = two_node_network.docker_nodes
clis = [CLI(node) for node in nodes]
accounts = (Account("genesis"), Account(1))
# Generate and save signed_deploy
with signed_deploy_file_path(clis[0], accounts[0]) as signed_deploy_path:
# First deployment of signed_deploy from node-0 should succeed
deploy_hash = clis[0]("send-deploy", "-i", signed_deploy_path)
block_hash = nodes[0].wait_for_deploy_processed_and_get_block_hash(deploy_hash)
deploy_info = clis[0]("show-deploy", deploy_hash)
assert not deploy_info.processing_results[0].is_error
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
# Second deployment of signed_deploy to node-1 should fail.
deploy_hash = clis[1]("send-deploy", "-i", signed_deploy_path)
deploy_info = clis[1]("show-deploy", deploy_hash)
assert not deploy_info.processing_results[0].is_error
result = nodes[1].p_client.client.wait_for_deploy_processed(deploy_hash)
assert "DISCARDED" in str(result)
assert "Duplicate or expired" in str(result)
def test_multiple_bootstraps(three_node_network_with_two_bootstraps):
# Successful setup of fixture three_node_network_with_two_bootstraps
# means that node-2, configured for multiple bootstrap nodes,
# could start when node-0 and node-1 were up.
net = three_node_network_with_two_bootstraps
nodes = net.docker_nodes
block_hash = nodes[0].deploy_and_get_block_hash(nodes[0].genesis_account,
Contract.HELLO_NAME_DEFINE)
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
# Stop node-2 and node-0, leave node-1 running.
# Clear state of node-2 before stopping it so when we restart it
# it has to download the genesis block again.
nodes[2].clear_state()
net.stop_cl_node(2)
net.stop_cl_node(0)
# Start node-2 and check it can bootstrap from node-1.
net.start_cl_node(2)
block_hash = nodes[1].deploy_and_get_block_hash(nodes[1].genesis_account,
Contract.HELLO_NAME_DEFINE)
wait_for_block_hash_propagated_to_all_nodes([nodes[1], nodes[2]],
block_hash)
net.start_cl_node(0)
def test_account(self,
node,
amount=TEST_ACCOUNT_INITIAL_BALANCE) -> Account:
name = test_name()
if not name:
# This happens when a thread tries to deploy.
# Name of the test that spawned the thread does not appear on the inspect.stack.
# Threads that don't want to use genesis account
# should pass from_address, public_key and private_key to deploy explicitly.
return self.genesis_account
elif name not in self.test_accounts:
with self._accounts_lock:
self.test_accounts[name] = Account(self.next_key)
logging.info(
f"=== Creating test account #{self.next_key} {self.test_accounts[name].public_key_hex} for {name} "
)
block_hash = node.transfer_to_account(self.next_key, amount)
# Waiting for the block with transaction that created new account to propagate to all nodes.
# Expensive, but some tests may rely on it.
wait_for_block_hash_propagated_to_all_nodes(
node.cl_network.docker_nodes, block_hash)
for deploy in node.client.show_deploys(block_hash):
assert (deploy.is_error is
False), f"Account creation failed: {deploy}"
self.next_key += 1
return self.test_accounts[name]
def test_check_deploy_signatures(intercepted_two_node_network):
"""
Test node reject block with deploys that have no signatures.
"""
nodes = intercepted_two_node_network.docker_nodes
for node in nodes:
node.proxy_server.set_interceptor(
GenerateBlockWithNoSignaturesGossipInterceptor)
node = nodes[0]
account = node.genesis_account
block_hash = node.deploy_and_get_block_hash(account,
Contract.HELLO_NAME_DEFINE)
logging.info(f" =============> VALID BLOCK: {block_hash}")
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
block_hash = None
if nodes[0].proxy_server.interceptor.new_block:
block_hash = nodes[0].proxy_server.interceptor.new_block.block_hash
receiver_node = nodes[1]
logging.info(
f" =============> BLOCK WITH NO JUSTIFICATIONS: {block_hash.hex()}"
)
with pytest.raises(Exception):
wait_for_block_hash_propagated_to_all_nodes([receiver_node],
block_hash.hex())
assert "InvalidDeploySignature" in receiver_node.logs()
def test_graph_ql(one_node_network):
node = one_node_network.docker_nodes[0]
client = node.p_client.client
deploy_hash = client.deploy(
session=node.resources_folder / Contract.HELLO_NAME_DEFINE,
from_addr=GENESIS_ACCOUNT.public_key_hex,
public_key=GENESIS_ACCOUNT.public_key_path,
private_key=GENESIS_ACCOUNT.private_key_path,
payment_amount=10**8,
)
block_hash = node.wait_for_deploy_processed_and_get_block_hash(deploy_hash)
wait_for_block_hash_propagated_to_all_nodes(one_node_network.docker_nodes,
block_hash)
block_dict = node.graphql.query_block(block_hash)
block = block_dict["data"]["block"]
assert block["blockHash"] == block_hash
assert block["deployCount"] == 1
assert block["deployErrorCount"] == 0
deploy_dict = node.graphql.query_deploy(deploy_hash)
deploy = deploy_dict["data"]["deploy"]["deploy"]
processing_results = deploy_dict["data"]["deploy"]["processingResults"]
assert deploy["deployHash"] == deploy_hash
assert processing_results[0]["block"]["blockHash"] == block_hash
def test_call_stored_contract(
three_node_network_with_combined_contract,
docker_client,
contract,
function_counter,
path,
expected,
):
"""
Feature file: consensus.feature
Scenario: Call contracts deployed on a node from another node.
"""
nodes = three_node_network_with_combined_contract.docker_nodes
from_address = nodes[0].genesis_account.public_key_hex
def state(node, path, block_hash):
return node.p_client.query_state(block_hash=block_hash,
key=from_address,
key_type="address",
path=path)
for node in nodes:
block_hash = deploy_and_propose(node, contract)
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
cur_state = state(node, path, block_hash)
assert expected(cur_state), f"{cur_state!r}"
def test_check_retrieved_block_has_expected_block_hash(
intercepted_two_node_network):
"""
This test uses an interceptor that modifies block retrieved
by node-1 from node-0 with GetBlockChunked method of the gossip service
and removes approvals from deploys in the block.
The test originally checked that the block was rejected by the node
due to no having the required signatures.
Now, however, it is rejected because the modification changes the block's hash.
Currently node rejects the block because it has a different hash than requested.
"""
nodes = intercepted_two_node_network.docker_nodes
for node in nodes:
node.proxy_server.set_interceptor(RemoveSignatureGossipInterceptor)
node = nodes[0]
account = node.genesis_account
block_hash = node.deploy_and_get_block_hash(account,
Contract.HELLO_NAME_DEFINE)
with raises(Exception):
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
assert "Retrieved block has unexpected" in nodes[1].logs()
def test_star_network(star_network):
# Deploy on one of the star edge nodes.
node1 = star_network.docker_nodes[1]
block_hash = node1.deploy_and_get_block_hash(GENESIS_ACCOUNT,
Contract.HELLO_NAME_DEFINE)
# Validate all nodes get block.
wait_for_block_hash_propagated_to_all_nodes(star_network.docker_nodes,
block_hash)
def test_blocks_infect_network(not_all_connected_directly_nodes):
"""
Feature file: block_gossiping.feature
Scenario: Blocks 'infect' the network and nodes 'closest' to the propose see the blocks first.
"""
first, last = (
not_all_connected_directly_nodes[0],
not_all_connected_directly_nodes[-1],
)
block_hash = deploy_and_propose(first, Contract.HELLO_NAME_DEFINE)
wait_for_block_hash_propagated_to_all_nodes([last], block_hash)
def test_python_bonding_and_unbonding_with_deploys(one_node_network_fn):
"""
Feature file: consensus.feature
Scenario: Bonding a validator node to an existing network.
"""
# Bond
logging.info(f"Funding Account {BONDING_ACCT} and bonding to network.")
bonding_amount = 1
assert_pre_state_of_network(one_node_network_fn)
block_hash, account = add_account_and_bond_to_network(
one_node_network_fn, bonding_amount, BONDING_ACCT)
node0, node1 = one_node_network_fn.docker_nodes
check_no_errors_in_deploys(node0, block_hash)
bonds = bonds_by_account_and_stake(node1, block_hash, bonding_amount,
account.public_key_hex)
assert len(bonds) == 1
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
# Test deploy/propose bonded
logging.info(f"Deploy and propose on bonded node.")
block_hash = node1.d_client.deploy_and_propose(
from_address=account.public_key_hex,
session_contract=Contract.HELLO_NAME_DEFINE,
private_key=account.private_key_docker_path,
public_key=account.public_key_docker_path,
)
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
check_no_errors_in_deploys(node1, block_hash)
# Unbond
logging.info(f"Unbonding Account {BONDING_ACCT} from network.")
block_hash, account = unbond_from_network(one_node_network_fn,
bonding_amount, BONDING_ACCT)
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
check_no_errors_in_deploys(node0, block_hash)
bonds = bonds_by_account_and_stake(node0, block_hash, bonding_amount,
account.public_key_hex)
assert len(bonds) == 0
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
# Test deploy/propose unbonded
logging.info(f"Testing unbonded deploy and propose.")
node1.d_client.deploy(
from_address=account.public_key_hex,
session_contract=Contract.HELLO_NAME_DEFINE,
private_key=account.private_key_docker_path,
public_key=account.public_key_docker_path,
)
with pytest.raises(InternalError) as excinfo:
node1.p_client.propose().block_hash.hex()
assert "InvalidUnslashableBlock" in str(excinfo.value)
def three_node_network_with_combined_contract(three_node_network):
"""
Fixture of a network with deployed combined definitions contract,
use later by tests in this module.
"""
tnn = three_node_network
node = tnn.docker_nodes[0]
for contract in [
Contract.HELLO_NAME_DEFINE,
Contract.COUNTER_DEFINE,
Contract.MAILING_LIST_DEFINE,
]:
block_hash = node.deploy_and_get_block_hash(node.genesis_account, contract)
wait_for_block_hash_propagated_to_all_nodes(tnn.docker_nodes, block_hash)
return tnn
def three_node_network_with_combined_contract(three_node_network):
"""
Fixture of a network with deployed combined definitions contract,
use later by tests in this module.
"""
tnn = three_node_network
bootstrap, node1, node2 = tnn.docker_nodes
node = bootstrap
block_hash = bootstrap.p_client.deploy_and_propose(
session_contract=Contract.COMBINED_CONTRACTS_DEFINE,
from_address=node.genesis_account.public_key_hex,
public_key=node.genesis_account.public_key_path,
private_key=node.genesis_account.private_key_path,
)
wait_for_block_hash_propagated_to_all_nodes(tnn.docker_nodes, block_hash)
return tnn
def test_bonding_and_unbonding_with_deploys(one_node_network_fn, acct_num,
cli_method):
"""
Feature file: consensus.feature
Scenario: Bonding a validator node to an existing network.
"""
# Bond
logging.info(f"Funding Account {acct_num} and bonding to network.")
bonding_amount = 1
assert_pre_state_of_network(one_node_network_fn)
block_hash, account = add_account_and_bond_to_network(
one_node_network_fn, bonding_amount, acct_num, cli_method)
node0, node1 = one_node_network_fn.docker_nodes
check_no_errors_in_deploys(node0, block_hash)
bonds = bonds_by_account_and_stake(node1, block_hash, bonding_amount,
account.public_key_hex)
assert len(bonds) == 1
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
# Test deploy on bonded node
logging.info(f"Deploy on bonded node.")
block_hash = node1.deploy_and_get_block_hash(account,
Contract.HELLO_NAME_DEFINE)
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
check_no_errors_in_deploys(node1, block_hash)
# Unbond
logging.info(f"Unbonding Account {acct_num} from network.")
block_hash, account = unbond_from_network(one_node_network_fn,
bonding_amount, acct_num,
cli_method)
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
check_no_errors_in_deploys(node0, block_hash)
bonds = bonds_by_account_and_stake(node0, block_hash, bonding_amount,
account.public_key_hex)
assert len(bonds) == 0
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
# Test deploy on unbonded node
logging.info(f"Testing unbonded deploy.")
node1.d_client.deploy(
from_address=account.public_key_hex,
session_contract=Contract.HELLO_NAME_DEFINE,
private_key=account.private_key_docker_path,
public_key=account.public_key_docker_path,
)
wait_for_invalid_unslashable_block(node1)
def test_equivocation(intercepted_two_node_network):
"""
Generate an equivocating block and check the system can detect it.
"""
nodes = intercepted_two_node_network.docker_nodes
for node in nodes:
node.proxy_server.set_interceptor(
GenerateEquivocatingBlocksGossipInterceptor)
node = nodes[0]
account = node.genesis_account
tls_certificate_path = node.config.tls_certificate_local_path()
tls_parameters = {"--node-id": extract_common_name(tls_certificate_path)}
cli = DockerCLI(nodes[0], tls_parameters=tls_parameters)
cli.set_default_deploy_args(
"--from",
account.public_key_hex,
"--private-key",
cli.private_key_path(account),
"--public-key",
cli.public_key_path(account),
"--payment",
cli.resource(Contract.STANDARD_PAYMENT),
"--payment-args",
cli.payment_json,
)
cli("deploy", "--session", cli.resource(Contract.HELLO_NAME_DEFINE))
block_hash = cli("propose")
logging.info(f" =============> REAL BLOCK: {block_hash}")
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
block_hash = None
if nodes[0].proxy_server.interceptor.equivocating_block:
block_hash = nodes[
0].proxy_server.interceptor.equivocating_block.block_hash
receiver_node = nodes[1]
logging.info(
f" =============> EQUIVOCATING BLOCK: {block_hash.hex()}")
wait_for_block_hash_propagated_to_all_nodes([receiver_node],
block_hash.hex())
assert "Found equivocation:" in receiver_node.logs()
def test(two_node_with_different_accounts_csv_network):
"""
Check that two nodes started with different accounts.csv
will create different genesis blocks
and will not exchange blocks, effectively not becoming part of one network.
"""
nodes = two_node_with_different_accounts_csv_network.docker_nodes
account = nodes[0].genesis_account
nodes[0].client.deploy(
session_contract=Contract.HELLO_NAME_DEFINE,
from_address=account.public_key_hex,
public_key=account.public_key_path,
private_key=account.private_key_path,
)
block_hash = nodes[0].client.propose()
with raises(Exception) as excinfo:
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
assert "Failed to satisfy" in str(excinfo.value)
def test_standalone_nodes_bootstrap_from_each_other(
three_node_network_with_two_bootstraps):
net = three_node_network_with_two_bootstraps
nodes = net.docker_nodes
block_hash = nodes[0].deploy_and_get_block_hash(nodes[0].genesis_account,
Contract.HELLO_NAME_DEFINE)
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
logging.info(f"======= Clearing state of {nodes[0].address}")
nodes[0].clear_state()
logging.info(f"======= Stopping {nodes[0].address}")
net.stop_cl_node(0)
logging.info(f"======= Starting {nodes[0].address}")
net.start_cl_node(0)
wait_for_node_started(nodes[0], 60, 1)
def test_check_deploy_signatures(intercepted_two_node_network):
"""
This test uses an interceptor that modifies block retrieved
by node-1 from node-0 with GetBlockChunked method of the gossip service
and removes approvals from deploys in the block.
node-1 should not accept this block.
"""
nodes = intercepted_two_node_network.docker_nodes
for node in nodes:
node.proxy_server.set_interceptor(RemoveSignatureGossipInterceptor)
node = nodes[0]
account = node.genesis_account
tls_certificate_path = node.config.tls_certificate_local_path()
tls_parameters = {
# Currently only Python client requires --certificate-file
# It may not need it in the future.
# "--certificate-file": tls_certificate_path,
"--node-id": extract_common_name(tls_certificate_path)
}
cli = DockerCLI(nodes[0], tls_parameters=tls_parameters)
cli.set_default_deploy_args(
"--from",
account.public_key_hex,
"--private-key",
cli.private_key_path(account),
"--public-key",
cli.public_key_path(account),
"--payment",
cli.resource(Contract.STANDARD_PAYMENT),
"--payment-args",
cli.payment_json,
)
cli("deploy", "--session", cli.resource(Contract.HELLO_NAME_DEFINE))
block_hash = cli("propose")
with raises(Exception):
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
assert "InvalidDeploySignature" in nodes[1].logs()
def test_python_double_bonding(one_node_network_fn):
"""
Feature file: consensus.feature
Scenario: Bonding a validator node twice to an existing network.
"""
assert_pre_state_of_network(one_node_network_fn)
block_hash, account = add_account_and_bond_to_network(
one_node_network_fn, 1, BONDING_ACCT)
node0, node1 = one_node_network_fn.docker_nodes
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
check_no_errors_in_deploys(node0, block_hash)
block_hash, account = bond_to_the_network(one_node_network_fn, 2,
BONDING_ACCT)
wait_for_block_hash_propagated_to_all_nodes(
one_node_network_fn.docker_nodes, block_hash)
check_no_errors_in_deploys(node1, block_hash)
bonds = bonds_by_account_and_stake(node1, block_hash, 1 + 2,
account.public_key_hex)
assert len(bonds) == 1
def test_unbonding_then_creating_block(payment_node_network):
network = payment_node_network
assert len(network.docker_nodes) == 1
def info(s):
logging.info(f"{'='*10} | test_unbonding_then_creating_block: {s}")
network.add_new_node_to_network()
assert len(network.docker_nodes) == 2
nodes = network.docker_nodes
bonding_account = nodes[1].config.node_account
info(f"CREATE ACCOUNT MATCHING NEW VALIDATOR'S PUBLIC KEY")
block_hash = nodes[0].transfer_to_account(
to_account_id=bonding_account.file_id,
amount=500000000,
from_account_id="genesis",
)
info(f"TRANSFER block_hash={block_hash}")
check_no_errors_in_deploys(nodes[0], block_hash)
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
# Newly added node is already a validator listed in bonds.txt.
# Deploy and propose should succeed.
nodes[1].p_client.deploy(
from_address=bonding_account.public_key_hex,
public_key=bonding_account.public_key_path,
private_key=bonding_account.private_key_path,
session_contract=Contract.HELLO_NAME_DEFINE,
)
first_block_hash_after_bonding = nodes[1].p_client.propose(
).block_hash.hex()
check_no_errors_in_deploys(nodes[1], first_block_hash_after_bonding)
wait_for_block_hash_propagated_to_all_nodes(
nodes, first_block_hash_after_bonding)
# Unbond amount larger than validator's current stake (small 2 digit number).
info(f"UNBONDING: {bonding_account.public_key_hex}")
unbonding_block_hash = unbond(
nodes[0],
bonding_account.public_key_hex,
None,
bonding_account.public_key_path,
bonding_account.private_key_path,
)
check_no_errors_in_deploys(nodes[0], unbonding_block_hash)
wait_for_block_hash_propagated_to_all_nodes(nodes, unbonding_block_hash)
# Now deploy and propose is expected to fail, as validator has 0 stakes.
# Use a different account because otherwise the deploy will get stuck
# in the deploy buffer and we will not be able to make a new deploy
# later from the same account on this node.
genesis_account = nodes[1].genesis_account
nodes[1].p_client.deploy(
from_address=genesis_account.public_key_hex,
public_key=genesis_account.public_key_path,
private_key=genesis_account.private_key_path,
session_contract=Contract.HELLO_NAME_DEFINE,
)
with pytest.raises(InternalError) as excinfo:
nodes[1].p_client.propose().block_hash.hex()
assert "InvalidUnslashableBlock" in str(excinfo.value)
info(f"BONDING AGAIN: {bonding_account.public_key_hex}")
bonding_block_hash = bond(
nodes[0],
bonding_account.public_key_hex,
100,
bonding_account.public_key_path,
bonding_account.private_key_path,
)
info(f"BONDING block_hash={bonding_block_hash}")
check_no_errors_in_deploys(nodes[0], bonding_block_hash)
wait_for_block_hash_propagated_to_all_nodes(nodes, bonding_block_hash)
# After bonding again deploy & propose will succeed.
nodes[1].p_client.deploy(
from_address=bonding_account.public_key_hex,
public_key=bonding_account.public_key_path,
private_key=bonding_account.private_key_path,
session_contract=Contract.HELLO_NAME_DEFINE,
)
block_hash = nodes[1].p_client.propose().block_hash.hex()
check_no_errors_in_deploys(nodes[1], block_hash)
def test_equivocating_node_shutdown_and_other_nodes_are_working_ok(
three_node_network):
network = three_node_network
nodes = network.docker_nodes
account = GENESIS_ACCOUNT
block_hash = nodes[0].deploy_and_get_block_hash(account,
Contract.HELLO_NAME_DEFINE)
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
# Clear state of node-0 so when we restart it
# it doesn't remember the block it created before.
nodes[0].clear_state()
# Stop node-0. Also, stop all other nodes so when node-0 restarts it cannot sync old
# blocks from them.
for i in range(len(nodes)):
network.stop_cl_node(i)
network.start_cl_node(0)
# After node-0 restarted it should only have the genesis block.
assert len(list(nodes[0].p_client.show_blocks(1000))) == 1
# Create the equivocating block.
block_hash = nodes[0].deploy_and_get_block_hash(account,
Contract.HELLO_NAME_DEFINE)
for i in range(1, len(nodes)):
network.start_cl_node(i)
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
assert len(list(nodes[1].p_client.show_blocks(1000))) == 3
block_hash = nodes[1].deploy_and_get_block_hash(account,
Contract.HELLO_NAME_DEFINE)
with pytest.raises(Exception):
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
# Check node-0 is offline. If it terminated the docker container it was running in stopped as well.
with pytest.raises(Exception) as excinfo:
nodes[0].d_client.show_blocks(1)
assert "Unable to resolve host node-0" in str(excinfo.value.output)
assert "Node has detected it's own equivocation with block" in nodes[
0].logs()
# node-0 is out of action. The rest of the nodes have equivocated blocks in their DAGs.
# Regardless, the remaining nodes should carry on working correctly.
# They all should be able to deploy contracts and propose new blocks with them.
working_nodes = nodes[1:]
block_hash = working_nodes[0].deploy_and_get_block_hash(
account, Contract.COUNTER_DEFINE)
wait_for_block_hash_propagated_to_all_nodes(working_nodes, block_hash)
# Deploy counter_call.wasm on each node a couple of times,
# check the value of the counter is correct.
expected_counter_result = count(1)
for i in range(2):
for node in working_nodes:
block_hash = node.deploy_and_get_block_hash(
account, Contract.COUNTER_CALL)
wait_for_block_hash_propagated_to_all_nodes(
working_nodes, block_hash)
state = node.p_client.query_state(
block_hash=block_hash,
key=account.public_key_hex,
key_type="address",
path="counter/count",
)
int_value = state.cl_value.value.i32
assert int_value == next(expected_counter_result)
def test_network_partition_and_rejoin_RS(four_nodes_network):
"""
Feature file: block_gossiping.feature
Scenario: Network partition occurs and rejoin occurs
"""
nodes = four_nodes_network.docker_nodes
n = len(nodes)
# Create a block in order to have test account created before partitioning,
# because account creation involves creating a block with a transfer
# from genesis account and waiting for the block to propagate
# to all nodes in the whole network, it would fail with nodes disconnected.
block_hash = nodes[0].deploy_and_get_block_hash(GENESIS_ACCOUNT, C[0])
wait_for_block_hash_propagated_to_all_nodes(nodes, block_hash)
# Partition the network so node0 connected to node1 and node2 connected to node3 only.
connections_between_partitions = [(i, j) for i in (0, 1) for j in (2, 3)]
logging.info("PARTITIONS: {}".format(connections_between_partitions))
logging.info("DISCONNECT PARTITIONS")
for connection in connections_between_partitions:
logging.info("DISCONNECTING PARTITION: {}".format(connection))
four_nodes_network.disconnect(connection)
partitions = nodes[:int(n / 2)], nodes[int(n / 2):]
logging.info("PARTITIONS: {}".format(partitions))
# Node updates its list of alive peers in background with a certain period
# So we need to wait here for nodes to re-connect partitioned peers
for partition in partitions:
for node in partition:
wait_for_peers_count_exactly(node, len(partition) - 1, 60)
# Propose separately in each partition. They should not see each others' blocks,
# so everyone has the genesis block, extra block created above,
# and the 1 block proposed in its partition.
# Using the same nonce in both partitions because otherwise one of them will
# sit there unable to propose; should use separate accounts really.
# TODO Change to multiple accounts
block_hashes = (
partitions[0][0].deploy_and_get_block_hash(GENESIS_ACCOUNT, C[0]),
partitions[1][0].deploy_and_get_block_hash(GENESIS_ACCOUNT, C[1]),
)
for partition, block_hash in zip(partitions, block_hashes):
wait_for_block_hash_propagated_to_all_nodes(partition, block_hash)
logging.info("CONNECT PARTITIONS")
for connection in connections_between_partitions:
logging.info("CONNECTING PARTITIONS: {}".format(connection))
four_nodes_network.connect(connection)
logging.info("PARTITIONS CONNECTED")
# Node updates its list of alive peers in background with a certain period
# So we need to wait here for nodes to re-connect partitioned peers
for node in nodes:
wait_for_peers_count_exactly(node, n - 1, 60)
# When we propose a node in partition[0] it should propagate to partition[1],
# however, nodes in partition[0] will still not see blocks from partition[1]
# until they also propose a new one on top of the block the created during
# the network outage.
block_hash = nodes[0].deploy_and_get_block_hash(GENESIS_ACCOUNT, C[2])
for partition, old_hash in zip(partitions, block_hashes):
logging.info(
f"CHECK {partition} HAS ALL BLOCKS CREATED IN BOTH PARTITIONS")
wait_for_block_hashes_propagated_to_all_nodes(partition,
[old_hash, block_hash])
