def wait_until(self, test_function_in, *, timeout=60, check_connected=True):
def test_function():
if check_connected:
assert self.is_connected
return test_function_in()
wait_until_helper(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor)
def close(self, timeout=10):
"""Close the connections and network event loop."""
self.network_event_loop.call_soon_threadsafe(self.network_event_loop.stop)
wait_until_helper(lambda: not self.network_event_loop.is_running(), timeout=timeout)
self.network_event_loop.close()
self.join(timeout)
# Safe to remove event loop.
NetworkThread.network_event_loop = None
def wait_for_connect(self, timeout=60):
test_function = lambda: self.is_connected
wait_until_helper(test_function, timeout=timeout, lock=p2p_lock)
def test_spork(self, spork_name, final_value):
# check test spork default state
for node in self.nodes:
assert (self.get_test_spork_value(node, spork_name) == 4070908800)
self.bump_mocktime(1)
# first and second signers set spork value
self.nodes[0].spork(spork_name, 1)
self.nodes[1].spork(spork_name, 1)
self.bump_mocktime(5)
# spork change requires at least 3 signers
time.sleep(10)
for node in self.nodes:
assert (self.get_test_spork_value(node, spork_name) != 1)
# restart with no extra args to trigger CheckAndRemove
self.restart_node(0)
assert (self.get_test_spork_value(self.nodes[0], spork_name) != 1)
# restart again with correct_params, should resync spork parts from other nodes
self.restart_node(0, self.node0_extra_args)
for i in range(1, 5):
self.connect_nodes(0, i)
# third signer set spork value
self.nodes[2].spork(spork_name, 1)
self.bump_mocktime(6)
time.sleep(5)
# now spork state is changed
for node in self.nodes:
wait_until_helper(
lambda: self.get_test_spork_value(node, spork_name) == 1,
sleep=0.1,
timeout=10)
# restart with no extra args to trigger CheckAndRemove, should reset the spork back to its default
self.restart_node(0)
assert (self.get_test_spork_value(self.nodes[0],
spork_name) == 4070908800)
# restart again with correct_params, should resync sporks from other nodes
self.restart_node(0, self.node0_extra_args)
time.sleep(5)
self.bump_mocktime(6)
for i in range(1, 5):
self.connect_nodes(0, i)
wait_until_helper(
lambda: self.get_test_spork_value(self.nodes[0], spork_name) == 1,
sleep=0.1,
timeout=10)
self.bump_mocktime(1)
# now set the spork again with other signers to test
# old and new spork messages interaction
self.nodes[2].spork(spork_name, final_value)
self.nodes[3].spork(spork_name, final_value)
self.nodes[4].spork(spork_name, final_value)
for node in self.nodes:
wait_until_helper(lambda: self.get_test_spork_value(
node, spork_name) == final_value,
sleep=0.1,
timeout=10)
def run_test(self):
self.sync_blocks(self.nodes, timeout=60 * 5)
self.confirm_mns()
for i in range(len(self.nodes)):
force_finish_mnsync(self.nodes[i])
self.nodes[0].spork("SPORK_17_QUORUM_DKG_ENABLED", 0)
if self.options.spork21:
self.nodes[0].spork("SPORK_21_QUORUM_ALL_CONNECTED", 0)
self.wait_for_sporks_same()
self.mine_quorum()
if self.options.spork21:
assert self.mninfo[0].node.getconnectioncount() == 5
id = "0000000000000000000000000000000000000000000000000000000000000001"
msgHash = "0000000000000000000000000000000000000000000000000000000000000002"
msgHashConflict = "0000000000000000000000000000000000000000000000000000000000000003"
def check_sigs(hasrecsigs, isconflicting1, isconflicting2):
for mn in self.mninfo:
if mn.node.quorum_hasrecsig(100, id, msgHash) != hasrecsigs:
return False
if mn.node.quorum_isconflicting(100, id,
msgHash) != isconflicting1:
return False
if mn.node.quorum_isconflicting(
100, id, msgHashConflict) != isconflicting2:
return False
return True
def wait_for_sigs(hasrecsigs, isconflicting1, isconflicting2, timeout):
t = time.time()
while time.time() - t < timeout:
if check_sigs(hasrecsigs, isconflicting1, isconflicting2):
return
self.bump_mocktime(2)
time.sleep(1)
raise AssertionError("wait_for_sigs timed out")
def assert_sigs_nochange(hasrecsigs, isconflicting1, isconflicting2,
timeout):
t = time.time()
while time.time() - t < timeout:
assert (check_sigs(hasrecsigs, isconflicting1, isconflicting2))
time.sleep(0.1)
# Initial state
wait_for_sigs(False, False, False, 1)
# Sign first share without any optional parameter, should not result in recovered sig
self.mninfo[0].node.quorum_sign(100, id, msgHash)
assert_sigs_nochange(False, False, False, 3)
# Sign second share and test optional quorumHash parameter, should not result in recovered sig
# 1. Providing an invalid quorum hash should fail and cause no changes for sigs
assert (not self.mninfo[1].node.quorum_sign(100, id, msgHash, msgHash))
assert_sigs_nochange(False, False, False, 3)
# 2. Providing a valid quorum hash should succeed and cause no changes for sigss
quorumHash = self.mninfo[1].node.quorum_selectquorum(100,
id)["quorumHash"]
assert (self.mninfo[1].node.quorum_sign(100, id, msgHash, quorumHash))
assert_sigs_nochange(False, False, False, 3)
# Sign third share and test optional submit parameter if spork21 is enabled, should result in recovered sig
# and conflict for msgHashConflict
if self.options.spork21:
# 1. Providing an invalid quorum hash and set submit=false, should throw an error
assert_raises_rpc_error(-8, 'quorum not found',
self.mninfo[2].node.quorum_sign, 100, id,
msgHash, id, False)
# 2. Providing a valid quorum hash and set submit=false, should return a valid sigShare object
sig_share_rpc_1 = self.mninfo[2].node.quorum_sign(
100, id, msgHash, quorumHash, False)
sig_share_rpc_2 = self.mninfo[2].node.quorum_sign(
100, id, msgHash, "", False)
assert_equal(sig_share_rpc_1, sig_share_rpc_2)
assert_sigs_nochange(False, False, False, 3)
# 3. Sending the sig share received from RPC to the recovery member through P2P interface, should result
# in a recovered sig
sig_share = CSigShare()
sig_share.llmqType = int(sig_share_rpc_1["llmqType"])
sig_share.quorumHash = int(sig_share_rpc_1["quorumHash"], 16)
sig_share.quorumMember = int(sig_share_rpc_1["quorumMember"])
sig_share.id = int(sig_share_rpc_1["id"], 16)
sig_share.msgHash = int(sig_share_rpc_1["msgHash"], 16)
sig_share.sigShare = hex_str_to_bytes(sig_share_rpc_1["signature"])
for i in range(len(self.mninfo)):
assert self.mninfo[i].node.getconnectioncount() == 5
# Get the current recovery member of the quorum
q = self.nodes[0].quorum_selectquorum(100, id)
mn = self.get_mninfo(q['recoveryMembers'][0])
# Open a P2P connection to it
p2p_interface = mn.node.add_p2p_connection(P2PInterface())
# Send the last required QSIGSHARE message to the recovery member
p2p_interface.send_message(msg_qsigshare([sig_share]))
else:
# If spork21 is not enabled just sign regularly
self.mninfo[2].node.quorum_sign(100, id, msgHash)
wait_for_sigs(True, False, True, 15)
self.bump_mocktime(5)
wait_for_sigs(True, False, True, 15)
if self.options.spork21:
mn.node.disconnect_p2ps()
# Test `quorum verify` rpc
node = self.mninfo[0].node
recsig = node.quorum_getrecsig(100, id, msgHash)
# Find quorum automatically
height = node.getblockcount()
height_bad = node.getblockheader(recsig["quorumHash"])["height"]
hash_bad = node.getblockhash(0)
assert (node.quorum_verify(100, id, msgHash, recsig["sig"]))
assert (node.quorum_verify(100, id, msgHash, recsig["sig"], "",
height))
assert (not node.quorum_verify(100, id, msgHashConflict,
recsig["sig"]))
assert not node.quorum_verify(100, id, msgHash, recsig["sig"], "",
height_bad)
# Use specific quorum
assert (node.quorum_verify(100, id, msgHash, recsig["sig"],
recsig["quorumHash"]))
assert (not node.quorum_verify(100, id, msgHashConflict, recsig["sig"],
recsig["quorumHash"]))
assert_raises_rpc_error(-8, "quorum not found", node.quorum_verify,
100, id, msgHash, recsig["sig"], hash_bad)
# Mine one more quorum, so that we have 2 active ones, nothing should change
self.mine_quorum()
assert_sigs_nochange(True, False, True, 3)
# Create a recovered sig for the oldest quorum i.e. the active quorum which will be moved
# out of the active set when a new quorum appears
request_id = 2
oldest_quorum_hash = node.quorum_list()["llmq_test"][-1]
# Search for a request id which selects the last active quorum
while True:
selected_hash = node.quorum_selectquorum(100, "%064x" %
request_id)["quorumHash"]
if selected_hash == oldest_quorum_hash:
break
else:
request_id += 1
# Produce the recovered signature
id = "%064x" % request_id
for mn in self.mninfo:
mn.node.quorum_sign(100, id, msgHash)
# And mine a quorum to move the quorum which signed out of the active set
self.mine_quorum()
# Verify the recovered sig. This triggers the "signHeight + dkgInterval" verification
recsig = node.quorum_getrecsig(100, id, msgHash)
assert node.quorum_verify(100, id, msgHash, recsig["sig"], "",
node.getblockcount())
recsig_time = self.mocktime
# Mine 4 more quorums, so that the one used for the the recovered sig should become inactive, nothing should change
self.mine_quorum()
self.mine_quorum()
self.mine_quorum()
self.mine_quorum()
assert_sigs_nochange(True, False, True, 3)
# fast forward until 0.5 days before cleanup is expected, recovered sig should still be valid
self.bump_mocktime(recsig_time + int(60 * 60 * 24 * 6.5) -
self.mocktime)
for i in range(len(self.nodes)):
force_finish_mnsync(self.nodes[i])
self.nodes[0].generate(1)
self.sync_blocks()
self.bump_mocktime(5)
# Cleanup starts every 5 seconds
wait_for_sigs(True, False, True, 15)
# fast forward 1 day, recovered sig should not be valid anymore
self.bump_mocktime(int(60 * 60 * 24 * 1))
for i in range(len(self.nodes)):
force_finish_mnsync(self.nodes[i])
self.nodes[0].generate(1)
self.sync_blocks()
self.bump_mocktime(5)
# Cleanup starts every 5 seconds
wait_for_sigs(False, False, False, 15)
for i in range(len(self.nodes)):
force_finish_mnsync(self.nodes[i])
for i in range(2):
self.mninfo[i].node.quorum_sign(100, id, msgHashConflict)
for i in range(2, 5):
self.mninfo[i].node.quorum_sign(100, id, msgHash)
self.nodes[0].generate(1)
self.sync_blocks()
self.bump_mocktime(5)
wait_for_sigs(True, False, True, 15)
if self.options.spork21:
id = "%064x" % (request_id + 1)
# Isolate the node that is responsible for the recovery of a signature and assert that recovery fails
q = self.nodes[0].quorum_selectquorum(100, id)
mn = self.get_mninfo(q['recoveryMembers'][0])
mn.node.setnetworkactive(False)
self.wait_until(lambda: mn.node.getconnectioncount() == 0)
for i in range(4):
self.mninfo[i].node.quorum_sign(100, id, msgHash)
assert_sigs_nochange(False, False, False, 3)
# Need to re-connect so that it later gets the recovered sig
mn.node.setnetworkactive(True)
self.connect_nodes(mn.node.index, 0)
force_finish_mnsync(mn.node)
# Make sure intra-quorum connections were also restored
self.bump_mocktime(
1) # need this to bypass quorum connection retry timeout
wait_until_helper(lambda: mn.node.getconnectioncount() == 5,
timeout=10,
sleep=2)
mn.node.ping()
self.wait_until(lambda: all('pingwait' not in peer
for peer in mn.node.getpeerinfo()))
self.nodes[0].generate(1)
self.sync_blocks()
self.bump_mocktime(5)
wait_for_sigs(True, False, True, 15)
def getBlock(self, blkHash):
self.block = None
inv = CInv(t=2, h=int(blkHash, 16))
self.send_message(msg_getdata(inv=[inv]))
wait_until_helper(lambda: self.block is not None)
return self.block
def run_test(self):
# Connect all nodes to node1 so that we always have the whole network connected
# Otherwise only masternode connections will be established between nodes, which won't propagate TXs/blocks
# Usually node0 is the one that does this, but in this test we isolate it multiple times
for i in range(len(self.nodes)):
if i != 1:
self.connect_nodes(i, 1)
self.nodes[0].generate(10)
self.sync_blocks(self.nodes, timeout=60 * 5)
self.nodes[0].spork("SPORK_17_QUORUM_DKG_ENABLED", 0)
self.nodes[0].spork("SPORK_19_CHAINLOCKS_ENABLED", 4070908800)
self.wait_for_sporks_same()
self.log.info("Mining 4 quorums")
for i in range(4):
self.mine_quorum()
self.nodes[0].spork("SPORK_19_CHAINLOCKS_ENABLED", 0)
self.wait_for_sporks_same()
self.log.info("Mine single block, wait for chainlock")
self.nodes[0].generate(1)
self.wait_for_chainlocked_block_all_nodes(
self.nodes[0].getbestblockhash())
self.log.info("Mine many blocks, wait for chainlock")
self.nodes[0].generate(20)
# We need more time here due to 20 blocks being generated at once
self.wait_for_chainlocked_block_all_nodes(
self.nodes[0].getbestblockhash(), timeout=30)
self.log.info(
"Assert that all blocks up until the tip are chainlocked")
for h in range(1, self.nodes[0].getblockcount()):
block = self.nodes[0].getblock(self.nodes[0].getblockhash(h))
assert (block['chainlock'])
self.log.info("Isolate node, mine on another, and reconnect")
self.isolate_node(self.nodes[0])
node0_mining_addr = self.nodes[0].getnewaddress()
node0_tip = self.nodes[0].getbestblockhash()
self.nodes[1].generatetoaddress(5, node0_mining_addr)
self.wait_for_chainlocked_block(self.nodes[1],
self.nodes[1].getbestblockhash())
assert (self.nodes[0].getbestblockhash() == node0_tip)
self.nodes[0].setnetworkactive(True)
for i in range(self.num_nodes - 1):
self.connect_nodes(0, i + 1)
self.nodes[1].generatetoaddress(1, node0_mining_addr)
self.wait_for_chainlocked_block(self.nodes[0],
self.nodes[1].getbestblockhash())
self.log.info(
"Isolate node, mine on both parts of the network, and reconnect")
self.isolate_node(self.nodes[0])
bad_tip = self.nodes[0].generate(5)[-1]
self.nodes[1].generatetoaddress(1, node0_mining_addr)
good_tip = self.nodes[1].getbestblockhash()
self.wait_for_chainlocked_block(self.nodes[1], good_tip)
assert (not self.nodes[0].getblock(
self.nodes[0].getbestblockhash())["chainlock"])
self.nodes[0].setnetworkactive(True)
for i in range(self.num_nodes - 1):
self.connect_nodes(0, i + 1)
self.nodes[1].generatetoaddress(1, node0_mining_addr)
self.wait_for_chainlocked_block(self.nodes[0],
self.nodes[1].getbestblockhash())
assert (self.nodes[0].getblock(
self.nodes[0].getbestblockhash())["previousblockhash"] == good_tip)
assert (self.nodes[1].getblock(
self.nodes[1].getbestblockhash())["previousblockhash"] == good_tip)
self.log.info(
"The tip mined while this node was isolated should be marked conflicting now"
)
found = False
for tip in self.nodes[0].getchaintips():
if tip["hash"] == bad_tip:
assert (tip["status"] == "conflicting")
found = True
break
assert (found)
def wait_for_headers():
if self.nodes[1].getconnectioncount() == node1_num_peers_before:
return False
if self.nodes[1].getpeerinfo(
)[-1]["synced_headers"] < self.nodes[0].getblockcount():
return False
return True
self.log.info("Keep node connected and let it try to reorg the chain")
good_tip = self.nodes[0].getbestblockhash()
self.log.info(
"Restart it so that it forgets all the chainlock messages from the past"
)
self.stop_node(0)
self.start_node(0)
node1_num_peers_before = self.nodes[1].getconnectioncount()
self.connect_nodes(0, 1)
# We need to wait for nodes to exchange headers first because
# node1 won't recognize node0 as a good peer to send new blocks to otherwise.
wait_until_helper(wait_for_headers, sleep=1, timeout=5)
assert (self.nodes[0].getbestblockhash() == good_tip)
self.nodes[0].invalidateblock(good_tip)
self.log.info("Now try to reorg the chain")
self.nodes[0].generate(2)
time.sleep(6)
assert (self.nodes[1].getbestblockhash() == good_tip)
bad_tip = self.nodes[0].generate(2)[-1]
time.sleep(6)
assert (self.nodes[0].getbestblockhash() == bad_tip)
assert (self.nodes[1].getbestblockhash() == good_tip)
self.log.info(
"Now let the node which is on the wrong chain reorg back to the locked chain"
)
self.nodes[0].reconsiderblock(good_tip)
assert (self.nodes[0].getbestblockhash() != good_tip)
good_fork = good_tip
good_tip = self.nodes[1].generatetoaddress(
1,
node0_mining_addr)[-1] # this should mark bad_tip as conflicting
self.wait_for_chainlocked_block(self.nodes[0], good_tip)
assert (self.nodes[0].getbestblockhash() == good_tip)
found = False
for tip in self.nodes[0].getchaintips():
if tip["hash"] == bad_tip:
assert (tip["status"] == "conflicting")
found = True
break
assert (found)
self.log.info(
"Should switch to the best non-conflicting tip (not to the most work chain) on restart"
)
assert (int(self.nodes[0].getblock(bad_tip)["chainwork"], 16) > int(
self.nodes[1].getblock(good_tip)["chainwork"], 16))
self.stop_node(0)
self.start_node(0)
self.nodes[0].invalidateblock(good_fork)
self.stop_node(0)
self.start_node(0)
time.sleep(1)
assert (self.nodes[0].getbestblockhash() == good_tip)
txs = []
for i in range(3):
txs.append(self.nodes[0].sendtoaddress(
self.nodes[0].getnewaddress(), 1))
txs += self.create_chained_txs(self.nodes[0], 1)
self.log.info(
"Assert that after block generation these TXs are included")
node0_tip = self.nodes[0].generate(1)[-1]
for txid in txs:
tx = self.nodes[0].getrawtransaction(txid, True)
assert ("confirmations" in tx)
time.sleep(1)
node0_tip_block = self.nodes[0].getblock(node0_tip)
assert (not node0_tip_block["chainlock"])
assert (node0_tip_block["previousblockhash"] == good_tip)
self.nodes[0].generate(1)
self.log.info("Assert that TXs got included now")
for txid in txs:
tx = self.nodes[0].getrawtransaction(txid, True)
assert ("confirmations" in tx and tx["confirmations"] > 0)
# Enable network on first node again, which will cause the blocks to propagate
# for the mined TXs, which will then allow the network to create a CLSIG
self.log.info("Re-enable network on first node and wait for chainlock")
self.nodes[0].setnetworkactive(True)
for i in range(self.num_nodes - 1):
self.connect_nodes(0, i + 1)
self.log.info(
"Send fake future clsigs and see if this breaks ChainLocks")
for i in range(len(self.nodes)):
if i != 0:
self.connect_nodes(i, 0)
SIGN_HEIGHT_OFFSET = 20
p2p_node = self.nodes[0].add_p2p_connection(TestP2PConn())
p2p_node.wait_for_verack()
self.wait_for_chainlocked_block_all_nodes(
self.nodes[0].getbestblockhash(), timeout=30)
self.log.info(
"Should accept fake clsig but other quorums should sign the actual block on the same height and override the malicious one"
)
fake_clsig1, fake_block_hash1 = self.create_fake_clsig(1)
p2p_node.send_clsig(fake_clsig1)
self.bump_mocktime(5, nodes=self.nodes)
time.sleep(5)
for node in self.nodes:
self.wait_for_most_recent_chainlock(node,
fake_block_hash1,
timeout=15)
tip = self.nodes[0].generate(1)[-1]
self.wait_for_chainlocked_block_all_nodes(tip, timeout=5)
self.log.info(
"Shouldn't accept fake clsig for 'tip + SIGN_HEIGHT_OFFSET + 1' block height"
)
fake_clsig2, fake_block_hash2 = self.create_fake_clsig(
SIGN_HEIGHT_OFFSET + 1)
p2p_node.send_clsig(fake_clsig2)
self.bump_mocktime(7, nodes=self.nodes)
time.sleep(5)
for node in self.nodes:
assert (self.nodes[0].getchainlocks()["recent_chainlock"]
["blockhash"] == tip)
assert (self.nodes[0].getchainlocks()["active_chainlock"]
["blockhash"] == tip)
self.log.info(
"Should accept fake clsig for 'tip + SIGN_HEIGHT_OFFSET' but new clsigs should still be formed"
)
fake_clsig3, fake_block_hash3 = self.create_fake_clsig(
SIGN_HEIGHT_OFFSET)
p2p_node.send_clsig(fake_clsig3)
self.bump_mocktime(7, nodes=self.nodes)
time.sleep(5)
for node in self.nodes:
self.wait_for_most_recent_chainlock(node,
fake_block_hash3,
timeout=15)
tip = self.nodes[0].generate(1)[-1]
self.bump_mocktime(7, nodes=self.nodes)
self.wait_for_chainlocked_block_all_nodes(tip, timeout=15)
self.nodes[0].disconnect_p2ps()
