def run_node_with_connections(self, title, node_index, args, number_of_connections, ip='127.0.0.1', strSubVer=None, wait_for_verack=True):
logger.debug("setup %s", title)
self.start_node(node_index, args)
connectionCbs = []
for i in range(number_of_connections):
connectionCbs.append(NodeConnCB())
connections = []
for connCb in connectionCbs:
connection = NodeConn(ip, p2p_port(0), self.nodes[node_index], connCb, strSubVer=strSubVer)
connections.append(connection)
connCb.add_connection(connection)
thr = NetworkThread()
thr.start()
if wait_for_verack:
for connCb in connectionCbs:
connCb.wait_for_verack()
logger.debug("before %s", title)
yield tuple(connections)
logger.debug("after %s", title)
for connection in connections:
connection.close()
del connections
# once all connection.close() are complete, NetworkThread run loop completes and thr.join() returns success
thr.join()
self.stop_node(node_index)
logger.debug("finished %s", title)
def run_node_with_associations(self, title, node_index, args, stream_policies, cb_class=AssociationCB, ip='127.0.0.1', strSubVer=None):
logger.debug("setup %s", title)
self.start_node(node_index, args)
# Create associations and their connections to the node
associations = []
for stream_policy in stream_policies:
association = Association(stream_policy, cb_class)
association.create_streams(self.nodes[node_index], ip, strSubVer)
associations.append(association)
# Start network handling thread
thr = NetworkThread()
thr.start()
# Allow associations to exchange their setup messages and fully initialise
for association in associations:
association.setup()
wait_until(lambda: len(self.nodes[node_index].getpeerinfo()) == len(associations))
# Test can now proceed
logger.debug("before %s", title)
yield tuple(associations)
logger.debug("after %s", title)
# Shutdown associations and their connections
for association in associations:
association.close_streams()
del associations
# Once all connections are closed, NetworkThread run loop completes and thr.join() returns success
thr.join()
self.stop_node(node_index)
logger.debug("finished %s", title)
def chksumTest(chksum_zero_recv, chksum_zero_recv_advertise,
chksum_zero_send):
conn = self.restart_node()
self.hndlr.allow0Checksum = chksum_zero_recv
self.hndlr.produce0Checksum = chksum_zero_send
nt = NetworkThread()
nt.start()
conn.wait_for_verack()
conn.send_message(msg_verack())
# now it is time for xversion
conn.send_message(
msg_xversion({0x00020002: int(chksum_zero_recv_advertise)}))
# send extra buversion and buverack, shouldn't harm
conn.send_message(msg_buversion(addrFromPort=12345))
conn.send_message(msg_buverack())
conn.wait_for(lambda: conn.remote_xversion)
if len(self.hndlr.exceptions):
return
conn.send_message(msg_ping())
conn.wait_for(lambda: conn.pong_counter)
# check that we are getting 0-value checksums from the BU node
if chksum_zero_recv:
assert (self.hndlr.num0Checksums > 0)
else:
assert (self.hndlr.num0Checksums == 0)
conn.connection.disconnect_node()
return nt
def run_test(self):
logging.info("Testing xversion handling")
# test regular set up including xversion
conn = self.restart_node()
conn.allow0Checksum = True
nt = NetworkThread()
nt.start()
conn.wait_for_verack()
conn.send_message(msg_verack())
# now it is time for xversion
conn.send_message(msg_xversion({0x00020002: 1}))
# send extra buversion and buverack, shouldn't harm
conn.send_message(msg_buversion(addrFromPort=12345))
conn.send_message(msg_buverack())
conn.wait_for(lambda: conn.remote_xversion)
conn.send_message(msg_ping())
conn.wait_for(lambda: conn.pong_counter)
# check that we are getting 0-value checksums from the BU node
assert (self.hndlr.num0Checksums > 0)
conn.connection.disconnect_node()
nt.join()
def run_test(self):
logging.info("Testing early ping replies")
conn = self.restart_node(send_initial_version=False)
conn.send_message(msg_ping(), pushbuf=True)
nt = NetworkThread()
nt.start()
conn.wait_for(lambda: conn.pong_counter)
conn.connection.disconnect_node()
nt.join()
def run_test(self):
logging.info("Testing xversion handling")
ex1_xver = { 1 : b"2", 3 : b"4"}
def test_too_early(msg):
""" Test that the given message if it comes right after start up will
lead to rejection / banning as it comes too early. """
print("Testing that an an early %s fails." % msg)
conn = self.restart_node(send_initial_version = False)
conn.send_message(msg, pushbuf = True)
self.network_and_finish()
assert conn.disconnected
# test failure due to early receipt
if 0:
for msg in [msg_buversion(1000), msg_buverack(), msg_xversion(), msg_xverack(),
msg_xversion({1:b"2",3:b"45"})]:
test_too_early(msg)
# test regular set up including xversion
conn = self.restart_node()
nt = NetworkThread()
nt.start()
conn.wait_for_verack()
conn.send_message(msg_verack())
# now it is time for xversion
conn.send_message(msg_xversion({1000 : b"test string"}))
# send extra buversion and buverack, shouldn't harm
conn.send_message(msg_buversion(addrFromPort = 12345))
conn.send_message(msg_buverack())
conn.wait_for(lambda : conn.remote_xversion)
# make sure xversion has actually been received properly
# test that it contains the BU_LISTEN_PORT (replacement for buversion message)
# FIXME: use proper constant
assert 1<<17 in conn.remote_xversion.xver.keys()
# Likewise, check that the remote end got our message
node = self.nodes[0]
peer_info = node.getpeerinfo()
assert len(peer_info) == 1
assert "xversion_map" in peer_info[0]
xv_map = peer_info[0]["xversion_map"]
assert len(xv_map) == 1
assert unhexlify(list(xv_map.values())[0]) == b"test string"
conn.connection.disconnect_node()
nt.join()
def check_seeds(network, dnsseeds, print_out=False):
seeds_count = 0
response_count = 0
for dnsseed in dnsseeds:
if print_out:
print(f"Name: {dnsseed[0]} ### Host: {dnsseed[1]}")
ip_addresses = set()
for addinfo in socket.getaddrinfo(dnsseed[1], 80):
ip_addresses.add(addinfo[4][0])
ip_addresses = sorted(ip_addresses)
for ip_address in ip_addresses:
if print_out:
print('%s %-15s' % (dnsseed[1], ip_address),
end=" version_string: ")
node0 = SeedNode()
if print_out:
node0.enable_verion_log()
connection = SeedNodeConn(ip_address,
NETWORK_PORTS[network],
rpc=None,
callback=node0,
net=network)
node0.add_connection(connection)
network_thread = NetworkThread()
network_thread.start()
try:
seeds_count += 1
node0.wait_for_verack(timeout=3)
node0.connection.close()
network_thread.join()
response_count += 1
except:
if print_out:
print("no response")
if node0.connection is not None:
node0.connection.close()
network_thread.join()
if print_out:
print()
if print_out:
print("Results:\n" +
f"Seeds: {seeds_count} Responded: {response_count}")
# we return true if all seed nodes responded
if seeds_count == response_count:
return True
else:
return False
def run_test(self):
logging.info("Testing extversion handling")
# test regular set up including extversion
testStringKey = 18446744073709551600
# a large number that requires uint64_t
testStringvalue = 1844674407370955161
# a large number that requires uint64_t
conn = self.restart_node()
nt = NetworkThread()
nt.start()
logging.info("sent version")
conn.wait_for(lambda: conn.remote_extversion)
logging.info("sent extversion")
conn.send_message(
msg_extversion({
1234: testStringvalue,
testStringKey: b"test string"
}))
conn.wait_for_verack()
logging.info("sent verack")
conn.send_message(msg_verack())
# make sure extversion has actually been received properly
# test that it contains the BU_LISTEN_PORT (replacement for buversion message)
# FIXME: use proper constant
assert 1 << 33 in conn.remote_extversion.xver.keys()
# Likewise, check that the remote end got our message
node = self.nodes[0]
peer_info = node.getpeerinfo()
assert len(peer_info) == 1
assert "extversion_map" in peer_info[0]
xv_map = peer_info[0]["extversion_map"]
assert len(xv_map) == 2
assert unhexlify(list(xv_map.values())[1]) == b"test string"
# send xupdate to test what would happen if someone tries to update non-chaneable key
conn.send_message(msg_xupdate({testStringKey: b"test string changed"}))
# some arbitrary sleep time
time.sleep(3)
# nothing should have changed, 1000 is not listed as a changeable key
node = self.nodes[0]
peer_info = node.getpeerinfo()
assert len(peer_info) == 1
assert "extversion_map" in peer_info[0]
xv_map = peer_info[0]["extversion_map"]
assert len(xv_map) == 2
assert unhexlify(list(xv_map.values())[1]) == b"test string"
# send xupdate to test what would happen if someone tries to update a non-existant key
conn.send_message(msg_xupdate({1111: b"bad string"}))
# some arbitrary sleep time
time.sleep(3)
# nothing should have changed, 1111 is not listed as a known key
node = self.nodes[0]
peer_info = node.getpeerinfo()
assert len(peer_info) == 1
assert "extversion_map" in peer_info[0]
xv_map = peer_info[0]["extversion_map"]
assert len(xv_map) == 2
# TODO appent to this test to test a changeable key once one has been implemented in the node
conn.connection.disconnect_node()
nt.join()
# Test versionbit mismatch
logging.info("Testing extversion service bit mismatch")
# test regular set up including extversion
conn = self.restart_node()
nt = NetworkThread()
nt.start()
logging.info("sent version")
conn.wait_for(lambda: conn.remote_extversion)
# if we send verack instead of extversion we should get a verack response
logging.info("sent verack")
conn.send_message(msg_verack())
conn.wait_for_verack()
conn.connection.disconnect_node()
nt.join()
def network_and_finish(self):
nt = NetworkThread()
nt.start()
nt.join()
class ComparisonTestFramework(BitcoinTestFramework):
"""Test framework for doing p2p comparison testing
Sets up some bitcoind binaries:
- 1 binary: test binary
- 2 binaries: 1 test binary, 1 ref binary
- n>2 binaries: 1 test binary, n-1 ref binaries"""
def __init__(self, destAddress = '127.0.0.1'):
super(ComparisonTestFramework,self).__init__()
self.chain = ChainManager()
self.destAddr = destAddress
self._network_thread = None
if not hasattr(self, "testbinary"):
self.testbinary = [os.getenv("BITCOIND", "bitcoind")]
if not hasattr(self, "refbinary"):
self.refbinary = [os.getenv("BITCOIND", "bitcoind")]
def set_test_params(self):
self.num_nodes = 2
self.setup_clean_chain = True
def add_options(self, parser):
parser.add_option("--testbinary", dest="testbinary", help="bitcoind binary to test")
parser.add_option("--refbinary", dest="refbinary", help="bitcoind binary to use for reference nodes (if any)")
def setup_network(self):
extra_args = [['-whitelist=127.0.0.1']] * self.num_nodes
if hasattr(self, "extra_args"):
extra_args = self.extra_args
if self.options.testbinary:
self.testbinary = [self.options.testbinary]
if self.options.refbinary:
self.refbinary = [self.options.refbinary]
binaries = [self.testbinary] + [self.refbinary] * (self.num_nodes - 1)
self.add_nodes(self.num_nodes, extra_args, binaries=binaries)
self.start_nodes()
self.init_network()
def restart_network(self, timeout=None):
self.test.clear_all_connections()
# If we had a network thread from eariler, make sure it's finished before reconnecting
if self._network_thread is not None:
self._network_thread.join(timeout)
# Reconnect
self.test.add_all_connections(self.nodes)
self._network_thread = NetworkThread()
self._network_thread.start()
def init_network(self):
# Start creating test manager which help to manage test cases
self.test = TestManager(self, self.options.tmpdir)
self.test.destAddr = self.destAddr
# (Re)start network
self.restart_network()
# returns a test case that asserts that the current tip was accepted
def accepted(self, sync_timeout=300):
return TestInstance([[self.chain.tip, True]], sync_timeout=sync_timeout)
# returns a test case that asserts that the current tip was rejected
def rejected(self, reject=None):
if reject is None:
return TestInstance([[self.chain.tip, False]])
else:
return TestInstance([[self.chain.tip, reject]])
def check_mempool(self, rpc, should_be_in_mempool, timeout=20):
wait_until(lambda: {t.hash for t in should_be_in_mempool}.issubset(set(rpc.getrawmempool())), timeout=timeout)
def run_test(self):
block_count = 0
# Create a P2P connection
node0 = NodeConnCB()
connection = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)
node0.add_connection(connection)
network_thread = NetworkThread()
network_thread.start()
# wait_for_verack ensures that the P2P connection is fully up.
node0.wait_for_verack()
self.chain.set_genesis_hash(int(self.nodes[0].getbestblockhash(), 16))
block = self.chain.next_block(block_count)
block_count += 1
self.chain.save_spendable_output()
node0.send_message(msg_block(block))
for i in range(100):
block = self.next_block(block_count)
block_count += 1
self.chain.save_spendable_output()
node0.send_message(msg_block(block))
out = []
for i in range(100):
out.append(self.chain.get_spendable_output())
self.log.info("waiting for block height 101 via rpc")
self.nodes[0].waitforblockheight(101)
block1_num = block_count - 1
# num of sig operations in one transaction
num_of_sig_checks = 70
expensive_scriptPubKey = [
OP_DUP, OP_HASH160,
hash160(self.coinbase_pubkey), OP_EQUALVERIFY, OP_CHECKSIG, OP_DROP
] * num_of_sig_checks + [
OP_DUP, OP_HASH160,
hash160(self.coinbase_pubkey), OP_EQUALVERIFY, OP_CHECKSIG
]
money_to_spend = 5000000000
spend = out[0]
block2_hard = self.next_block(block_count)
# creates 4000 hard transaction and 4000 transaction to spend them. It will be 8k transactions in total
add_txns = self.get_hard_transactions(
spend,
money_to_spend=money_to_spend,
num_of_transactions=4000,
num_of_sig_checks=num_of_sig_checks,
expensive_script=expensive_scriptPubKey)
self.chain.update_block(block_count, add_txns)
block_count += 1
self.log.info(f"block2_hard hash: {block2_hard.hash}")
self.chain.set_tip(block1_num)
block3_easier = self.next_block(block_count)
add_txns = self.get_hard_transactions(
spend,
money_to_spend=money_to_spend,
num_of_transactions=1000,
num_of_sig_checks=num_of_sig_checks,
expensive_script=expensive_scriptPubKey)
self.chain.update_block(block_count, add_txns)
self.log.info(f"block3_easier hash: {block3_easier.hash}")
node0.send_message(msg_block(block2_hard))
node0.send_message(msg_block(block3_easier))
def wait_for_log():
text_activation = f"Block {block2_hard.hash} was not activated as best"
text_block2 = "Verify 8000 txins"
text_block3 = "Verify 2000 txins"
results = 0
for line in open(
glob.glob(self.options.tmpdir + "/node0" +
"/regtest/bitcoind.log")[0]):
if text_activation in line:
results += 1
elif text_block2 in line:
results += 1
elif text_block3 in line:
results += 1
return True if results == 3 else False
# wait that everything is written to the log
# try accounting for slower machines by having a large timeout
wait_until(wait_for_log, timeout=120)
text_activation = f"Block {block2_hard.hash} was not activated as best"
text_block2 = "Verify 8000 txins"
text_block3 = "Verify 2000 txins"
for line in open(
glob.glob(self.options.tmpdir + "/node0" +
"/regtest/bitcoind.log")[0]):
if text_activation in line:
self.log.info(
f"block2_hard was not activated as block3_easy won the validation race"
)
elif text_block2 in line:
line = line.split()
self.log.info(
f"block2_hard took {line[len(line) - 1]} to verify")
elif text_block3 in line:
line = line.split()
self.log.info(
f"block3_easy took {line[len(line)-1]} to verify")
assert_equal(block3_easier.hash, self.nodes[0].getbestblockhash())
node0.connection.close()
class BlockStoringInFile(BitcoinTestFramework):
def set_test_params(self):
self.num_nodes = 3
self.runner_nodes = []
self.setup_clean_chain = True
self.mining_block_max_size = 20 * ONE_MEGABYTE
self.excessive_block_size = 22 * ONE_MEGABYTE
extra_header_space = 8 # one block header takes up 8 bytes
self.preferred_blockfile_size = 2 * (ONE_MEGABYTE + extra_header_space)
# +1 is a magic number for extra block file space as check whether the
# file is already full is written with >= instead of >
self.preferred_blockfile_size += 1
self.extra_args = [
'-whitelist=127.0.0.1',
"-excessiveblocksize=%d" % self.excessive_block_size,
"-preferredblockfilesize=%d" % self.preferred_blockfile_size,
"-blockmaxsize=%d" % self.mining_block_max_size
]
def setup_network(self):
self.add_nodes(self.num_nodes)
for i in range(self.num_nodes):
self.start_node(i, self.extra_args)
self.runner_nodes.append(RunnerNode(self.nodes[i], i))
# Start up network handling in another thread
self._network_thread = NetworkThread()
self._network_thread.start()
for i in range(self.num_nodes):
self.runner_nodes[i].finish_setup_after_network_is_started(
self.options.tmpdir)
def __count_blk_files(self, block_number, expected_number_of_files,
node_number):
blockfile_count = len(
glob.glob(self.options.tmpdir + "/node" + str(node_number) +
"/regtest/blocks/blk0000*.dat"))
assert blockfile_count == expected_number_of_files, (
"unexpected blockfile count for block: " + str(block_number) +
"; node: " + str(node_number) + "; expected: " +
str(expected_number_of_files) + "; got: " + str(blockfile_count))
def __compare_local_and_remote_block_size(self, local_block, remote_node):
remote_best_block_hash = remote_node.getbestblockhash()
assert_equal(remote_best_block_hash, local_block.hash)
# check that we can successfully read block from file
remote_block = remote_node.getblock(remote_best_block_hash)
assert_equal(remote_block['size'], len(local_block.serialize()))
def __send_and_test_block(self, runner_node, block_size,
expected_number_of_files):
local_block, block_number = runner_node.create_and_send_block(
block_size)
self.__compare_local_and_remote_block_size(local_block,
runner_node.remote_node)
self.__count_blk_files(block_number, expected_number_of_files,
runner_node.node_number)
def __test_two_blocks_less_than_preferred_file_size_in_single_file(
self, runner_node):
print(
"- test two blocks size is less than preferred file size, put in single file"
)
self.__send_and_test_block(runner_node, ONE_MEGABYTE, 2)
self.__send_and_test_block(runner_node, ONE_MEGABYTE, 2)
def __test_one_block_in_file_second_block_exceeds_preferred_file_size(
self, runner_node):
print(
"- test one block in file, second block exceeds preferred file size"
)
self.__send_and_test_block(runner_node, ONE_MEGABYTE, 2)
self.__send_and_test_block(runner_node, ONE_MEGABYTE * 2, 3)
self.__send_and_test_block(runner_node, ONE_MEGABYTE, 4)
self.__send_and_test_block(runner_node, ONE_MEGABYTE, 4)
def __test_block_larger_than_preferred_file_size(self, runner_node):
print("- test block larger than preferred filesize")
self.__send_and_test_block(runner_node, ONE_MEGABYTE * 5, 2)
self.__send_and_test_block(runner_node, ONE_MEGABYTE, 3)
self.__send_and_test_block(runner_node, ONE_MEGABYTE, 3)
def run_test(self):
# each test should run with different runner/test node combination as we want to start
# with a fresh blockfile structure
self.__test_two_blocks_less_than_preferred_file_size_in_single_file(
self.runner_nodes[0])
self.__test_one_block_in_file_second_block_exceeds_preferred_file_size(
self.runner_nodes[1])
self.__test_block_larger_than_preferred_file_size(self.runner_nodes[2])