def set_test_params(self):
self.setup_clean_chain = True
self.chain = ChainManager()
self.num_nodes = 1
self.next_block = 0
self.headerSize = 24
self.num_peers = 15
self.excessiveblocksize = 3 * ONE_MEGABYTE
def __init__(self, remote_node, node_number):
super(RunnerNode, self).__init__()
self.chain = ChainManager()
self.next_block = 0
self.remote_node = remote_node
self.node_number = node_number
connections = []
connections.append(
NodeConn('127.0.0.1', p2p_port(self.node_number), self.remote_node,
self))
self.add_connection(connections[0])
class RunnerNode(NodeConnCB):
def __init__(self, remote_node, node_number):
super(RunnerNode, self).__init__()
self.chain = ChainManager()
self.next_block = 0
self.remote_node = remote_node
self.node_number = node_number
connections = []
connections.append(
NodeConn('127.0.0.1', p2p_port(self.node_number), self.remote_node,
self))
self.add_connection(connections[0])
def finish_setup_after_network_is_started(self, tmp_dir):
self.wait_for_verack()
self.chain.set_genesis_hash(
int(self.remote_node.getbestblockhash(), 16))
# Build the blockchain
self.tip = int(self.remote_node.getbestblockhash(), 16)
self.block_time = self.remote_node.getblock(
self.remote_node.getbestblockhash())['time'] + 1
_, _, self.next_block = prepare_init_chain(self.chain,
100,
0,
block_0=False,
start_block=0,
node=self)
self.sync_with_ping()
# Create a new block - half of max block file size. Together with above blocks
# this leaves less than 1MB of free space in the first block file
self.create_and_send_block(ONE_MEGABYTE)
assert (len(
glob.glob(tmp_dir + "/node" + str(self.node_number) +
"/regtest/blocks/blk0000*.dat")) == 1
) # sanity check that there is still only one file
def create_and_send_block(self, block_size):
out = self.chain.get_spendable_output()
block = self.chain.next_block(self.next_block,
spend=out,
block_size=block_size)
self.next_block += 1
self.chain.save_spendable_output()
self.send_and_ping(msg_block(block))
return block, self.next_block - 1
class MaxSendQueuesBytesTest(BitcoinTestFramework):
def set_test_params(self):
self.setup_clean_chain = True
self.chain = ChainManager()
self.num_nodes = 1
self.next_block = 0
self.headerSize = 24
self.num_peers = 15
self.excessiveblocksize = 3 * ONE_MEGABYTE
# Request block "block" from all nodes.
def requestBlocks(self, test_nodes, block):
REJECT_TOOBUSY = int('0x44', 16)
numberOfRejectedMsgs = 0
numberOfReceivedBlocks = 0
def on_block(conn, message):
nonlocal numberOfReceivedBlocks
numberOfReceivedBlocks += 1
def on_reject(conn, message):
assert_equal(message.code, REJECT_TOOBUSY)
nonlocal numberOfRejectedMsgs
numberOfRejectedMsgs += 1
getdata_request = msg_getdata([CInv(2, block)])
for test_node in test_nodes:
test_node.on_block = on_block
test_node.on_reject = on_reject
test_node.send_message(getdata_request)
# Let bitcoind process and send all the messages.
for test_node in test_nodes:
test_node.sync_with_ping()
return numberOfReceivedBlocks, numberOfRejectedMsgs
def prepareChain(self):
node = NodeConnCB()
connections = []
connections.append(
NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node))
node.add_connection(connections[0])
NetworkThread().start()
node.wait_for_verack()
# Generate some old blocks
self.chain.set_genesis_hash(int(self.nodes[0].getbestblockhash(), 16))
# Create the first block with a coinbase output to our key
block = self.chain.next_block(self.next_block)
self.next_block += 1
self.chain.save_spendable_output()
node.send_message(msg_block(block))
# Bury the block 100 deep so the coinbase output is spendable
for i in range(1, 100):
block = self.chain.next_block(self.next_block)
self.next_block += 1
self.chain.save_spendable_output()
node.send_message(msg_block(block))
return node
def mineBigBlock(self, node):
block = self.chain.next_block(self.next_block,
spend=self.chain.get_spendable_output(),
block_size=self.excessiveblocksize)
self.next_block += 1
self.chain.save_spendable_output()
node.send_message(msg_block(block))
node.sync_with_ping()
createdBlock = self.nodes[0].getbestblockhash()
logger.info("Big block %s created (%d B)", createdBlock,
self.excessiveblocksize)
createdBlock = int(createdBlock, 16)
return createdBlock
def run_test(self):
@contextlib.contextmanager
def run_connection(factorMaxSendingBlocksSize, blockSize):
title = "Send GetData and receive block messages while factorMaxSendingBlockSize is {}.".format(
factorMaxSendingBlocksSize)
logger.debug("setup %s", title)
args = [
"-excessiveblocksize={}".format(self.excessiveblocksize +
self.headerSize),
"-blockmaxsize={}".format(self.excessiveblocksize +
self.headerSize),
"-factorMaxSendQueuesBytes={}".format(
factorMaxSendingBlocksSize)
]
self.start_node(0, args)
test_nodes = []
for i in range(self.num_peers):
test_nodes.append(NodeConnCB())
connections = []
for test_node in test_nodes:
connection = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0],
test_node)
connections.append(connection)
test_node.add_connection(connection)
thr = NetworkThread()
thr.start()
for test_node in test_nodes:
test_node.wait_for_verack()
logger.debug("before %s", title)
yield test_nodes
logger.debug("after %s", title)
for connection in connections:
connection.close()
del connections
thr.join()
disconnect_nodes(self.nodes[0], 1)
self.stop_node(0)
logger.debug("finished %s", title)
node = self.prepareChain()
# Mine a big block.
oldBlock = self.mineBigBlock(node)
# Mine another big block so that the previous block is not the tip of chain.
newBlock = self.mineBigBlock(node)
self.stop_node(0)
# Scenario 1: Blocks from bitcoind should be sent in parallel as factorMaxSendQueuesBytes=num_peers.
with run_connection(self.num_peers,
self.excessiveblocksize) as test_nodes:
numberOfReceivedBlocksParallel, numberOfRejectedMsgs = self.requestBlocks(
test_nodes, oldBlock)
assert_equal(self.num_peers, numberOfReceivedBlocksParallel)
assert_equal(0, numberOfRejectedMsgs)
# Scenario 2: Blocks from bitcoind should not be sent in parallel because factorMaxSendQueuesBytes=1
# only allows one 3MB to be downloaded at once.
with run_connection(1, self.excessiveblocksize) as test_nodes:
numberOfReceivedBlocksSeries, numberOfRejectedMsgs = self.requestBlocks(
test_nodes, oldBlock)
# numReceivedBlocksSeries may vary between test runs (based on processing power).
# But still we expect the processing to be slow enough that with 15 messages at least one will be rejected.
assert_greater_than(numberOfReceivedBlocksParallel,
numberOfReceivedBlocksSeries)
assert_greater_than(numberOfRejectedMsgs, 0)
assert_equal(numberOfReceivedBlocksSeries + numberOfRejectedMsgs, 15)
logger.info(
"%d blocks received when running with factorMaxSendQueuesBytes=%d.",
numberOfReceivedBlocksSeries, 1)
# Scenario 3: Blocks from bitcoind should be sent in parallel, because we are requesting the most recent block.
with run_connection(1, self.excessiveblocksize) as test_nodes:
numberOfReceivedBlocksNewBlock, numberOfRejectedMsgs = self.requestBlocks(
test_nodes, newBlock)
assert_equal(self.num_peers, numberOfReceivedBlocksNewBlock)
assert_equal(0, numberOfRejectedMsgs)
def test_blocktree(self):
"""
Test soft rejection block status in non-trivial tree of blocks
"""
# Create a P2P connection to node0 that will be used to send blocks
self.stop_node(0)
with self.run_node_with_connections(
title="test_blocktree",
node_index=0,
args=[
"-whitelist=127.0.0.1"
], # Need to whilelist localhost, so that node accepts any block
number_of_connections=1) as connections:
conn0 = connections[0]
# Create the following tree of blocks:
# genesis
# |
# 1001...1200
# |
# 1 height=201
# / \
# 2 8
# /|\ \
# 3 4 6 9
# | |
# 5 7
chain = ChainManager()
genesis_hash = self.nodes[0].getbestblockhash()
chain.set_genesis_hash(int(genesis_hash, 16))
_, out, _ = prepare_init_chain(chain,
200,
12,
block_0=False,
start_block=1001,
node=conn0.cb)
conn0.cb.sync_with_ping()
# Check that we have created a chain that we wanted
assert_equal(self.nodes[0].getblockcount(), 200)
assert_equal(
self.nodes[0].getblock(chain.blocks[1001].hash)["height"], 1)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[1200].hash)
assert_equal(
self.nodes[0].getblock(chain.blocks[1200].hash)["height"], 200)
def new_blk(idx, prev_idx):
chain.set_tip(prev_idx)
b = chain.next_block(
idx, spend=out[idx]
) # spend output with the same index as block
# NOTE: We don't really care about spending outputs in this test.
# Spending different outputs is only used as a convenient way
# to make two blocks different if they have the same parent.
conn0.cb.send_message(msg_block(b)) # send block to node
conn0.cb.sync_with_ping(
) # wait until node has processed the block
self.log.debug("Created block: idx=%i prev=%i hash=%s" %
(idx, prev_idx, b.hash))
new_blk(1, 1200)
new_blk(2, 1)
new_blk(3, 2)
new_blk(4, 2)
new_blk(5, 4)
new_blk(6, 2)
new_blk(7, 6)
new_blk(8, 1)
new_blk(9, 8)
# Block 5 should be tip of the active chain (it its highest and was received before 7, which is at the same height)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[5].hash)
# If 5 is soft rejected, 7 should become best
self.nodes[0].softrejectblock(chain.blocks[5].hash, 0)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[7].hash)
# If 7 is also soft rejected, 3 should become best
self.nodes[0].softrejectblock(chain.blocks[7].hash, 0)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[3].hash)
# Reset state
self.nodes[0].acceptblock(chain.blocks[5].hash)
self.nodes[0].acceptblock(chain.blocks[7].hash)
assert_equal(self.soft_rej_blocks_hashes(self.nodes[0]), set())
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[5].hash)
# If 2 is soft rejected for next two blocks, 9 should become best
self.nodes[0].softrejectblock(chain.blocks[2].hash, 2)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[9].hash)
# If we reconsider 2 to be soft rejected only for next one block, 5 should again become best
self.nodes[0].acceptblock(chain.blocks[2].hash, 1)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[5].hash)
# Reset state
self.nodes[0].acceptblock(chain.blocks[2].hash)
assert_equal(self.soft_rej_blocks_hashes(self.nodes[0]), set())
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[5].hash)
# If 1 is soft rejected for next 3 blocks, 1200 should become best
self.nodes[0].softrejectblock(chain.blocks[1].hash, 3)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[1200].hash)
# If 1 is soft rejected only until next block, 2 is soft rejected for 2 blocks and 9 is soft rejected until next block, 8 should become best
self.nodes[0].acceptblock(chain.blocks[1].hash, 0)
self.nodes[0].softrejectblock(chain.blocks[9].hash, 0)
self.nodes[0].softrejectblock(chain.blocks[2].hash, 2)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[8].hash)
# If we now receive a new block after 9, it should become best
new_blk(10, 9)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[10].hash)
# Reset state, block 5 should still be best
self.nodes[0].acceptblock(chain.blocks[1].hash)
self.nodes[0].acceptblock(chain.blocks[9].hash)
self.nodes[0].acceptblock(chain.blocks[2].hash)
assert_equal(self.soft_rej_blocks_hashes(self.nodes[0]), set())
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[5].hash)
# Soft rejecting block 1001 for next 202 blocks should have no effect
self.nodes[0].softrejectblock(chain.blocks[1001].hash, 202)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[5].hash)
# If block 1001 is soft rejected for next 203 blocks, genesis should become best
self.nodes[0].softrejectblock(chain.blocks[1001].hash, 203)
assert_equal(self.nodes[0].getbestblockhash(), genesis_hash)
# If we now receive a new block after 5, it should become best
new_blk(11, 5)
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[11].hash)
# Reset state, block 11 should still be best
self.nodes[0].acceptblock(chain.blocks[1001].hash)
assert_equal(self.soft_rej_blocks_hashes(self.nodes[0]), set())
assert_equal(self.nodes[0].getbestblockhash(),
chain.blocks[11].hash)
class MaxSendQueuesBytesTest(BitcoinTestFramework):
def set_test_params(self):
self.setup_clean_chain = True
self.chain = ChainManager()
self.num_nodes = 1
self.next_block = 0
self.headerSize = 24
self.num_peers = 15
self.excessiveblocksize = 5 * ONE_MEGABYTE
# Request block "block" from all nodes.
def requestBlocks(self, test_nodes, block):
REJECT_TOOBUSY = int('0x44', 16)
numberOfRejectedMsgs = 0
numberOfReceivedBlocks = 0
def on_block(conn, message):
nonlocal numberOfReceivedBlocks
numberOfReceivedBlocks += 1
def on_reject(conn, message):
assert_equal(message.code, REJECT_TOOBUSY)
nonlocal numberOfRejectedMsgs
numberOfRejectedMsgs += 1
getdata_request = msg_getdata([CInv(2, block)])
for test_node in test_nodes:
test_node.cb.on_block = on_block
test_node.cb.on_reject = on_reject
test_node.cb.send_message(getdata_request)
# Let bitcoind process and send all the messages.
for test_node in test_nodes:
test_node.cb.sync_with_ping()
return numberOfReceivedBlocks, numberOfRejectedMsgs
def prepareChain(self):
node = NodeConnCB()
connections = []
connections.append(
NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node))
node.add_connection(connections[0])
NetworkThread().start()
node.wait_for_verack()
# Generate some old blocks
self.chain.set_genesis_hash(int(self.nodes[0].getbestblockhash(), 16))
# Create the first block with a coinbase output to our key
block = self.chain.next_block(self.next_block)
self.next_block += 1
self.chain.save_spendable_output()
node.send_message(msg_block(block))
# Bury the block 100 deep so the coinbase output is spendable
for i in range(1, 100):
block = self.chain.next_block(self.next_block)
self.next_block += 1
self.chain.save_spendable_output()
node.send_message(msg_block(block))
return node
def mineBigBlock(self, node):
block = self.chain.next_block(self.next_block,
spend=self.chain.get_spendable_output(),
block_size=self.excessiveblocksize)
self.next_block += 1
self.chain.save_spendable_output()
node.send_message(msg_block(block))
node.sync_with_ping()
createdBlock = self.nodes[0].getbestblockhash()
logger.info("Big block %s created (%d B)", createdBlock,
self.excessiveblocksize)
createdBlock = int(createdBlock, 16)
return createdBlock
def run_test(self):
node = self.prepareChain()
# Mine a big block.
oldBlock = self.mineBigBlock(node)
# Mine another big block so that the previous block is not the tip of chain.
newBlock = self.mineBigBlock(node)
self.stop_node(0)
# Scenario 1: Blocks from bitcoind should be sent in parallel as factormaxsendqueuesbytes=num_peers.
args = [
"-excessiveblocksize={}".format(self.excessiveblocksize +
self.headerSize),
"-blockmaxsize={}".format(self.excessiveblocksize +
self.headerSize), '-rpcservertimeout=500'
]
with self.run_node_with_connections(
"should be sent in parallel as factormaxsendqueuesbytes=num_peers",
0,
args + ["-factormaxsendqueuesbytes={}".format(self.num_peers)],
self.num_peers) as connections:
start = time.time()
numberOfReceivedBlocksParallel, numberOfRejectedMsgs = self.requestBlocks(
connections, oldBlock)
logger.info("finished requestBlock duration %s s",
time.time() - start)
assert_equal(self.num_peers, numberOfReceivedBlocksParallel)
assert_equal(0, numberOfRejectedMsgs)
# Scenario 2: Blocks from bitcoind should not be sent in parallel because factormaxsendqueuesbytes=1
# only allows one 5MB to be downloaded at once.
with self.run_node_with_connections(
"should not be sent in parallel because factormaxsendqueuesbytes=1",
0, args + ["-factormaxsendqueuesbytes=1"],
self.num_peers) as connections:
start = time.time()
numberOfReceivedBlocksSeries, numberOfRejectedMsgs = self.requestBlocks(
connections, oldBlock)
logger.info("finished requestBlock duration %s s",
time.time() - start)
# numReceivedBlocksSeries may vary between test runs (based on processing power).
# But still we expect the processing to be slow enough that with 15 messages at least one will be rejected.
logger.info(
"%d blocks received when running with factormaxsendqueuesbytes=%d.",
numberOfReceivedBlocksSeries, 1)
assert_greater_than(numberOfReceivedBlocksParallel,
numberOfReceivedBlocksSeries)
assert_greater_than(numberOfRejectedMsgs, 0)
assert_equal(numberOfReceivedBlocksSeries + numberOfRejectedMsgs,
15)
# Scenario 3: Blocks from bitcoind should not be sent in parallel if we reached rate limit,
# because we are requesting the most recent block from non whitelisted peer.
with self.run_node_with_connections(
"some blocks should be rejected because non whitelisted peer are requesting most recent block",
0, args + ["-factormaxsendqueuesbytes=1"],
self.num_peers) as connections:
start = time.time()
numberOfReceivedBlocksNewBlock, numberOfRejectedMsgs = self.requestBlocks(
connections, newBlock)
logger.info("finished requestBlock duration %s s",
time.time() - start)
logger.info(
"%d blocks received when running with factormaxsendqueuesbytes=%d.",
numberOfReceivedBlocksNewBlock, 1)
assert_greater_than(self.num_peers, numberOfReceivedBlocksNewBlock)
assert_greater_than(numberOfRejectedMsgs, 0)
# Scenario 4: Blocks from bitcoind should be sent in parallel, because there is no limit on whitelisted peers
# requesting most recent block even if queue is full.
with self.run_node_with_connections(
"should be sent in parallel, because we are requesting the most recent block",
0,
args + ["-factormaxsendqueuesbytes=1", "-whitelist=127.0.0.1"],
self.num_peers) as connections:
start = time.time()
numberOfReceivedBlocksNewBlock, numberOfRejectedMsgs = self.requestBlocks(
connections, newBlock)
logger.info("finished requestBlock duration %s s",
time.time() - start)
assert_equal(self.num_peers, numberOfReceivedBlocksNewBlock)
assert_equal(0, numberOfRejectedMsgs)