def setUp(self):
self.node = MockGatewayNode(gateway_helpers.get_gateway_opts(
8000,
include_default_btc_args=True,
compact_block_min_tx_count=5
))
self.node.block_processing_service = MagicMock()
self.connection = BtcNodeConnection(
MockSocketConnection(node=self.node, ip_address=LOCALHOST, port=123), self.node
)
self.connection.node = self.node
self.connection.peer_ip = LOCALHOST
self.connection.peer_port = 8001
self.connection.network_num = 2
self.sut = BtcNodeConnectionProtocol(self.connection)
full_block_msg = BlockBtcMessage(
buf=bytearray(convert.hex_to_bytes(self.FULL_BLOCK_BYTES_HEX))
)
if self.node.opts.use_extensions:
transaction_service = ExtensionTransactionService(self.node, 0)
else:
transaction_service = TransactionService(self.node, 0)
short_id = 1
for tx in full_block_msg.txns():
tx_hash = btc_common_utils.get_txid(tx)
transaction_service.set_transaction_contents(tx_hash, tx)
transaction_service.assign_short_id(tx_hash, short_id)
short_id += 1
self.sut.connection.node._tx_service = transaction_service
def setUp(self):
super().setUp()
self.node1.alarm_queue = AlarmQueue()
self.node2.alarm_queue = AlarmQueue()
self.network_num = 1
self.magic = 12345
self.version = 23456
self.prev_block_hash = bytearray(crypto.double_sha256(b"123"))
self.prev_block = BtcObjectHash(self.prev_block_hash,
length=crypto.SHA256_HASH_LEN)
self.merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
self.merkle_root = BtcObjectHash(self.merkle_root_hash,
length=crypto.SHA256_HASH_LEN)
self.bits = 2
self.nonce = 3
opts = self.gateway_1_opts()
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.btc_message_converter = btc_message_converter_factory.create_btc_message_converter(
self.magic, opts)
self.btc_transactions = [
TxBtcMessage(self.magic, self.version, [], [], i)
for i in range(self.TRANSACTIONS_COUNT)
]
self.btc_transactions_for_block = [
tx_btc_message.rawbytes()[btc_constants.BTC_HDR_COMMON_OFF:]
for tx_btc_message in self.btc_transactions
]
self.transactions = [
self.btc_message_converter.tx_to_bx_txs(tx_btc_message,
self.network_num)[0][0]
for tx_btc_message in self.btc_transactions
]
self.transactions_with_short_ids = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1, tx_message.tx_val())
for i, tx_message in enumerate(self.transactions)
]
self.transactions_with_no_content = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1) for i, tx_message in enumerate(self.transactions)
]
self.transactions_by_short_id = {
tx_message.short_id(): tx_message
for tx_message in self.transactions_with_short_ids
}
self.block = BlockBtcMessage(self.magic, self.version,
self.prev_block, self.merkle_root,
int(time.time()), self.bits, self.nonce,
self.btc_transactions_for_block)
def clean_block_transactions(
self, block_msg: BlockBtcMessage,
transaction_service: TransactionService) -> None:
block_short_ids = []
block_unknown_tx_hashes = []
start_time = time.time()
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
for tx in block_msg.txns():
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=BTC_SHA_HASH_LEN)
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
block_hash = block_msg.block_hash()
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, block_msg.txn_count(),
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=block_short_ids,
unknown_tx_hashes=block_unknown_tx_hashes)
def _get_sample_block(self, file_path=__file__):
root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(file_path))))
with open(os.path.join(root_dir, "btc_sample_block.txt")) as sample_file:
btc_block = sample_file.read().strip("\n")
buf = bytearray(convert.hex_to_bytes(btc_block))
parsed_block = BlockBtcMessage(buf=buf)
return parsed_block
def test_send_receive_block_decrypted(self):
self.node1.opts.encrypt_blocks = False
send_block = btc_block(int(time.time()))
self._populate_transaction_services(send_block)
# propagate block
helpers.receive_node_message(self.node1, self.blockchain_fileno, send_block.rawbytes())
block_hold_msg = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertIn(BlockHoldingMessage.MESSAGE_TYPE, block_hold_msg.tobytes())
self.node1.on_bytes_sent(self.relay_fileno, len(block_hold_msg))
relayed_block = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertIn(BroadcastMessage.MESSAGE_TYPE, relayed_block.tobytes())
self.node1.on_bytes_sent(self.relay_fileno, len(relayed_block))
# block directly propagated
helpers.receive_node_message(self.node2, self.relay_fileno, relayed_block)
bytes_to_blockchain = self.node2.get_bytes_to_send(self.blockchain_fileno)
self.assertEqual(len(send_block.rawbytes()), len(bytes_to_blockchain))
received_block = BlockBtcMessage(buf=bytearray(bytes_to_blockchain))
self.assertEqual(send_block.magic(), received_block.magic())
self.assertEqual(send_block.version(), received_block.version())
self.assertEqual(send_block.timestamp(), received_block.timestamp())
self.assertEqual(send_block.bits(), received_block.bits())
self.assertEqual(send_block.nonce(), received_block.nonce())
def test_parse_message_success_all_types(self):
# TODO: pull these numbers into constants, along with all the BTC messages
self.create_message_successfully(self.VERSION_BTC_MESSAGE,
VersionBtcMessage)
self.create_message_successfully(VerAckBtcMessage(self.MAGIC),
VerAckBtcMessage)
self.create_message_successfully(PingBtcMessage(self.MAGIC),
PingBtcMessage)
self.create_message_successfully(PongBtcMessage(self.MAGIC, 123),
PongBtcMessage)
self.create_message_successfully(GetAddrBtcMessage(self.MAGIC),
GetAddrBtcMessage)
self.create_message_successfully(
AddrBtcMessage(self.MAGIC,
[(int(time.time()), "127.0.0.1", 8000)]),
AddrBtcMessage)
inv_vector = [(1, self.HASH), (2, self.HASH)]
self.create_message_successfully(InvBtcMessage(self.MAGIC, inv_vector),
InvBtcMessage)
self.create_message_successfully(
GetDataBtcMessage(self.MAGIC, inv_vector), GetDataBtcMessage)
self.create_message_successfully(
NotFoundBtcMessage(self.MAGIC, inv_vector), NotFoundBtcMessage)
hashes = [self.HASH, self.HASH]
self.create_message_successfully(
GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetHeadersBtcMessage)
self.create_message_successfully(
GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetBlocksBtcMessage)
self.create_message_successfully(
TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0), TxBtcMessage)
txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5
self.create_message_successfully(
BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0,
0, 0, txs), BlockBtcMessage)
self.create_message_successfully(
HeadersBtcMessage(self.MAGIC,
[helpers.generate_bytearray(81)] * 2),
HeadersBtcMessage)
self.create_message_successfully(
RejectBtcMessage(self.MAGIC, b"a message",
RejectBtcMessage.REJECT_MALFORMED, b"test break",
helpers.generate_bytearray(10)), RejectBtcMessage)
self.create_message_successfully(SendHeadersBtcMessage(self.MAGIC),
SendHeadersBtcMessage)
self.create_message_successfully(
FeeFilterBtcMessage(self.MAGIC, fee_rate=100), FeeFilterBtcMessage)
self.create_message_successfully(
BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)),
XversionBtcMessage)
def build_btc_block(block_pieces: Deque[Union[bytearray, memoryview]],
size: int) -> Tuple[BlockBtcMessage, int]:
btc_block = bytearray(size)
offset = 0
for piece in block_pieces:
next_offset = offset + len(piece)
btc_block[offset:next_offset] = piece
offset = next_offset
return BlockBtcMessage(buf=btc_block), offset
def msg_block(self, msg: BlockBtcMessage) -> None:
block_stats.add_block_event_by_block_hash(
msg.block_hash(),
BlockStatEventType.REMOTE_BLOCK_RECEIVED_BY_GATEWAY,
network_num=self.connection.network_num,
more_info="Protocol: {}, Network: {}".format(
self.connection.node.opts.blockchain_protocol,
self.connection.node.opts.blockchain_network))
return self.msg_proxy_response(msg)
def get_segwit_block():
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
with open(os.path.join(root_dir, "samples/btc_segwit_sample_block.txt")) as sample_file:
btc_block = sample_file.read().strip("\n")
block = convert.hex_to_bytes(btc_block)
buf = bytearray(BTC_HDR_COMMON_OFF + len(block))
buf[BTC_HDR_COMMON_OFF:] = block
msg = BtcMessage(magic="main", command=BlockBtcMessage.MESSAGE_TYPE, payload_len=len(block), buf=buf)
parsed_block = BlockBtcMessage(buf=msg.buf)
return parsed_block
def on_block_sent(self, block_hash: Sha256Hash,
block_message: BlockBtcMessage):
# After sending block message to Bitcoin node sending INV message for the same block to the node
# This is needed to update Synced Headers value of the gateway peer on the Bitcoin node
# If Synced Headers is not up-to-date than Bitcoin node does not push compact blocks to the gateway
inv_msg = InvBtcMessage(
magic=block_message.magic(),
inv_vects=[(InventoryType.MSG_BLOCK, block_hash)],
)
self.node.send_msg_to_node(inv_msg)
def clean_block_transactions(
self, block_msg: BlockBtcMessage, transaction_service: TransactionService
) -> None:
start_datetime = datetime.utcnow()
start_time = time.time()
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len()
cleanup_task = self.block_cleanup_tasks.borrow_task()
tx_service = typing.cast(ExtensionTransactionService, transaction_service)
cleanup_task.init(tpe.InputBytes(block_msg.buf), tx_service.proxy)
init_time = time.time()
task_pool_proxy.run_task(cleanup_task)
task_run_time = time.time()
unknown_tx_hashes_count = len(cleanup_task.unknown_tx_hashes())
tx_property_fetch_time = time.time()
short_ids = cleanup_task.short_ids()
short_ids_fetch_time = time.time()
short_ids_count = len(short_ids)
tx_service.update_removed_transactions(cleanup_task.total_content_removed(), short_ids)
remove_from_tx_service_time = time.time()
# TODO : clean the short ids/transactions from the alarm queue after refactoring the transaction service
block_hash = block_msg.block_hash()
tx_service.on_block_cleaned_up(block_hash)
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len()
end_datetime = datetime.utcnow()
end_time = time.time()
logger.statistics(
{
"type": "BlockTransactionsCleanup",
"block_hash": repr(block_hash),
"unknown_tx_hashes_count": unknown_tx_hashes_count,
"short_ids_count": short_ids_count,
"block_transactions_count": cleanup_task.txn_count(),
"start_datetime": start_datetime,
"end_datetime": end_datetime,
"task_init_time": init_time - start_time,
"task_run_time": task_run_time - init_time,
"tx_property_fetch_time": tx_property_fetch_time - task_run_time,
"short_ids_fetch_time": short_ids_fetch_time - tx_property_fetch_time,
"remove_from_tx_service_time": remove_from_tx_service_time - short_ids_fetch_time,
"duration": end_time - start_time,
"tx_hash_to_contents_len_before_cleanup": tx_hash_to_contents_len_before_cleanup,
"tx_hash_to_contents_len_after_cleanup": tx_hash_to_contents_len_after_cleanup,
}
)
self.block_cleanup_tasks.return_task(cleanup_task)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=short_ids,
unknown_tx_hashes=(
Sha256Hash(convert.hex_to_bytes(tx_hash.hex_string()))
for tx_hash in cleanup_task.unknown_tx_hashes()
)
)
def send_received_block_and_key(self, block=None):
if block is None:
block = btc_block()
self._populate_transaction_services(block)
# propagate block
helpers.receive_node_message(self.node1, self.blockchain_fileno,
block.rawbytes())
block_hold_request_relay = self.node1.get_bytes_to_send(
self.relay_fileno)
self.assertIn(BlockHoldingMessage.MESSAGE_TYPE,
block_hold_request_relay.tobytes())
self.node1.on_bytes_sent(self.relay_fileno,
len(block_hold_request_relay))
relayed_block = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertIn(BroadcastMessage.MESSAGE_TYPE, relayed_block.tobytes())
self.node1.on_bytes_sent(self.relay_fileno, len(relayed_block))
# block hold sent out
block_hold_request_gateway = self.node1.get_bytes_to_send(
self.gateway_fileno)
self.assertIsNotNone(block_hold_request_gateway)
self.assertIn(BlockHoldingMessage.MESSAGE_TYPE,
block_hold_request_gateway.tobytes())
self.clear_all_buffers()
# key not available until receipt
key_not_yet_available = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(OutputBuffer.EMPTY, key_not_yet_available)
# send receipt
helpers.receive_node_message(self.node2, self.relay_fileno,
relayed_block)
block_receipt = self.node2.get_bytes_to_send(self.gateway_fileno)
self.assertIn(BlockReceivedMessage.MESSAGE_TYPE,
block_receipt.tobytes())
# send key
helpers.receive_node_message(self.node1, self.gateway_fileno,
block_receipt)
key_message = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertIn(KeyMessage.MESSAGE_TYPE, key_message.tobytes())
helpers.receive_node_message(self.node2, self.relay_fileno,
key_message)
bytes_to_blockchain = self.node2.get_bytes_to_send(
self.blockchain_fileno)
self.assertEqual(len(block.rawbytes()), len(bytes_to_blockchain))
return BlockBtcMessage(buf=bytearray(bytes_to_blockchain))
def setUp(self):
self.local_node_fileno = 1
self.remote_node_fileno = 2
self.gateway_node = spies.make_spy_node(BtcGatewayNode,
8000,
include_default_btc_args=True)
self.btc_node_connection = spies.make_spy_connection(
BtcNodeConnection, self.local_node_fileno, 8001, self.gateway_node)
self.btc_remote_node_connection = spies.make_spy_connection(
BtcRemoteConnection, self.remote_node_fileno, 8002,
self.gateway_node)
self.gateway_node.sdn_connection = MagicMock()
self.gateway_node.node_conn = self.btc_node_connection
self.gateway_node.remote_node_conn = self.btc_remote_node_connection
self.gateway_node.connection_pool.add(self.local_node_fileno,
LOCALHOST, 8001,
self.btc_node_connection)
self.gateway_node.connection_pool.add(self.remote_node_fileno,
LOCALHOST, 8002,
self.btc_remote_node_connection)
self.gateway_node.account_id = "1234"
self.handshake = \
bytearray(VersionBtcMessage(100, 1000, "127.0.0.1", 8333, "0.0.0.0", 8333,
13, 0, bytearray("hello", "utf-8"), 0).rawbytes()) + \
bytearray(VerAckBtcMessage(100).rawbytes())
inps = [
TxIn(prev_outpoint_hash=bytearray("0" * 32, "utf-8"),
prev_out_index=10,
sig_script=bytearray(1000),
sequence=32)
]
outs = [TxOut(value=10, pk_script=bytearray(1000))]
self.txmsg = bytearray(
TxBtcMessage(magic=100,
version=0,
tx_in=inps,
tx_out=outs,
lock_time=12345).rawbytes())
self.blockmsg = bytearray(
BlockBtcMessage(magic=100,
version=0,
prev_block=get_object_hash(bytearray(32)),
merkle_root=get_object_hash(bytearray(32)),
timestamp=1023,
bits=1,
txns=[bytearray(64)] * 32000,
nonce=34).rawbytes())
self.transform = MessageTransformations()
def btc_block(timestamp=None, real_block=None):
if real_block is not None:
return BlockBtcMessage(buf=bytearray(convert.hex_to_bytes(real_block)))
if timestamp is None:
timestamp = int(time.time())
magic = 12345
version = 23456
prev_block_hash = bytearray(crypto.double_sha256(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
bits = 2
nonce = 3
txns = [
TxBtcMessage(magic, version, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
return BlockBtcMessage(magic, version, prev_block, merkle_root, timestamp,
bits, nonce, txns)
def test_msg_block_too_old(self):
block_timestamp = int(
time.time()
) - 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval
txns = [
TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
message = BlockBtcMessage(0, 0, self.HASH, self.HASH, 0,
block_timestamp, 0, txns)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_not_called(
)
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.decompression_tasks.return_task(tsk)
header_info = btc_normal_message_converter.parse_bx_block_header(bx_block_msg, deque())
raise message_conversion_error.btc_block_decompression_error(header_info.block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = BtcObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string())
)
if tsk.success():
btc_block_msg = BlockBtcMessage(buf=memoryview(tsk.block_message()))
logger.debug(
"Successfully parsed block broadcast message. {} transactions "
"in block {}",
total_tx_count,
block_hash
)
else:
btc_block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}",
block_hash,
len(unknown_tx_sids),
len(unknown_tx_hashes),
total_tx_count
)
block_info = get_block_info(
bx_block_msg,
block_hash,
tsk.short_ids(),
decompress_start_datetime,
decompress_start_timestamp,
total_tx_count,
btc_block_msg
)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(btc_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult:
start_datetime = datetime.datetime.utcnow()
start_time = time.time()
block_msg = BlockBtcMessage(buf=bx_block_msg)
block_info = BlockInfo(
block_msg.block_hash(),
[],
start_datetime,
datetime.datetime.utcnow(),
(time.time() - start_time) * 1000,
block_msg.txn_count(),
# pyre-fixme[6]: Expected `Optional[str]` for 7th param but got
# `BtcObjectHash`.
block_msg.block_hash(),
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()),
len(block_msg.rawbytes()),
0
)
return BlockDecompressionResult(block_msg, block_info, [], [])
def test_btc_block_to_bloxroute_block_and_back_sids_found(self):
prev_block_hash = bytearray(crypto.bitcoin_hash(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
timestamp = 1
bits = 2
nonce = 3
btc_block = BlockBtcMessage(
self.magic, self.version, prev_block, merkle_root, timestamp, bits, nonce, self.txns
)
block_hash = btc_block.block_hash()
bloxroute_block, block_info = self.btc_message_converter.block_to_bx_block(
btc_block, self.tx_service, True, 0
)
self.assertEqual(10, block_info.txn_count)
self.assertEqual("5a77d1e9612d350b3734f6282259b7ff0a3f87d62cfef5f35e91a5604c0490a3",
block_info.prev_block_hash)
self.assertEqual(self.short_ids, list(block_info.short_ids))
self.assertEqual(btc_block.block_hash(), block_info.block_hash)
# TODO: if we convert bloxroute block to a class, add some tests here
parsed_btc_block, block_info, _, _ = self.btc_message_converter.bx_block_to_block(
bloxroute_block,
self.tx_service)
self.assertIsNotNone(block_info)
self.assertEqual(parsed_btc_block.rawbytes().tobytes(), btc_block.rawbytes().tobytes())
self.assertEqual(self.version, parsed_btc_block.version())
self.assertEqual(self.magic, parsed_btc_block.magic())
self.assertEqual(prev_block_hash, parsed_btc_block.prev_block_hash().get_little_endian())
self.assertEqual(merkle_root_hash, parsed_btc_block.merkle_root().get_little_endian())
self.assertEqual(timestamp, parsed_btc_block.timestamp())
self.assertEqual(bits, parsed_btc_block.bits())
self.assertEqual(nonce, parsed_btc_block.nonce())
self.assertEqual(len(self.txns), parsed_btc_block.txn_count())
self.assertEqual(btc_block.checksum(), parsed_btc_block.checksum())
self.assertEqual(block_hash, parsed_btc_block.block_hash())
self.assertEqual(block_hash.binary, block_info.block_hash.binary)
self.assertEqual(list(block_info.short_ids), self.short_ids)
def _recovered_compact_block_to_bx_block(
self, compression_result: CompactBlockCompressionResult,
recovery_item: CompactBlockRecoveryData
) -> CompactBlockCompressionResult:
"""
Handle recovery of Bitcoin compact block message.
"""
missing_indices = compression_result.missing_indices
recovered_transactions = compression_result.recovered_transactions
block_transactions = recovery_item.block_transactions
if len(missing_indices) != len(recovered_transactions):
logger.debug(
"Number of transactions missing in compact block does not match number of recovered transactions."
"Missing transactions - {}. Recovered transactions - {}",
len(missing_indices), len(recovered_transactions))
return CompactBlockCompressionResult(False, None, None, None,
missing_indices,
recovered_transactions)
for i in range(len(missing_indices)):
missing_index = missing_indices[i]
block_transactions[missing_index] = recovered_transactions[i]
size = 0
total_txs_count = len(block_transactions)
block_msg_parts = deque()
block_header = recovery_item.block_header
block_msg_parts.append(block_header)
size += len(block_header)
tx_count_size = btc_messages_util.get_sizeof_btc_varint(
total_txs_count)
tx_count_buf = bytearray(tx_count_size)
btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf,
0)
block_msg_parts.append(tx_count_buf)
size += tx_count_size
for transaction in block_transactions:
block_msg_parts.append(transaction)
size += len(transaction) # pyre-ignore
msg_header = bytearray(btc_constants.BTC_HDR_COMMON_OFF)
struct.pack_into("<L12sL", msg_header, 0, recovery_item.magic,
BtcMessageType.BLOCK, size)
block_msg_parts.appendleft(msg_header)
size += btc_constants.BTC_HDR_COMMON_OFF
block_msg_bytes = bytearray(size)
off = 0
for blob in block_msg_parts:
next_off = off + len(blob)
block_msg_bytes[off:next_off] = blob
off = next_off
checksum = crypto.bitcoin_hash(
block_msg_bytes[btc_constants.BTC_HDR_COMMON_OFF:size])
block_msg_bytes[btc_constants.BTC_HEADER_MINUS_CHECKSUM:btc_constants.
BTC_HDR_COMMON_OFF] = checksum[0:4]
bx_block, compression_block_info = self.block_to_bx_block(
BlockBtcMessage(buf=block_msg_bytes), recovery_item.tx_service,
True, 0) # TODO need to think about a better algorithm
compress_start_datetime = compression_block_info.start_datetime
compress_end_datetime = datetime.utcnow()
block_info = BlockInfo(
compression_block_info.block_hash,
compression_block_info.short_ids, compress_start_datetime,
compress_end_datetime,
(compress_end_datetime - compress_start_datetime).total_seconds() *
1000, compression_block_info.txn_count,
compression_block_info.compressed_block_hash,
compression_block_info.prev_block_hash,
compression_block_info.original_size,
compression_block_info.compressed_size,
compression_block_info.compression_rate,
compression_block_info.ignored_short_ids)
return CompactBlockCompressionResult(True, block_info, bx_block, None,
[], [])
def build_block_header_message(
self, block_hash: Sha256Hash,
block_message: BlockBtcMessage) -> InvBtcMessage:
return InvBtcMessage(block_message.magic(),
[(InventoryType.MSG_BLOCK, block_hash)])
def btc_block(self, txns=None):
if txns is None:
txns = self.btc_transactions_bytes()
return BlockBtcMessage(self.MAGIC, self.VERSION, self.BTC_HASH, self.BTC_HASH, 0, 0, 0, txns)
def get_previous_block_hash_from_message(
self, block_message: BlockBtcMessage) -> Sha256Hash:
return block_message.prev_block_hash()
class BlockRecoveryTest(AbstractBtcGatewayIntegrationTest):
TRANSACTIONS_COUNT = 10
def setUp(self):
super().setUp()
self.node1.alarm_queue = AlarmQueue()
self.node2.alarm_queue = AlarmQueue()
self.network_num = 1
self.magic = 12345
self.version = 23456
self.prev_block_hash = bytearray(crypto.double_sha256(b"123"))
self.prev_block = BtcObjectHash(self.prev_block_hash,
length=crypto.SHA256_HASH_LEN)
self.merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
self.merkle_root = BtcObjectHash(self.merkle_root_hash,
length=crypto.SHA256_HASH_LEN)
self.bits = 2
self.nonce = 3
opts = self.gateway_1_opts()
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.btc_message_converter = btc_message_converter_factory.create_btc_message_converter(
self.magic, opts)
self.btc_transactions = [
TxBtcMessage(self.magic, self.version, [], [], i)
for i in range(self.TRANSACTIONS_COUNT)
]
self.btc_transactions_for_block = [
tx_btc_message.rawbytes()[btc_constants.BTC_HDR_COMMON_OFF:]
for tx_btc_message in self.btc_transactions
]
self.transactions = [
self.btc_message_converter.tx_to_bx_txs(tx_btc_message,
self.network_num)[0][0]
for tx_btc_message in self.btc_transactions
]
self.transactions_with_short_ids = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1, tx_message.tx_val())
for i, tx_message in enumerate(self.transactions)
]
self.transactions_with_no_content = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1) for i, tx_message in enumerate(self.transactions)
]
self.transactions_by_short_id = {
tx_message.short_id(): tx_message
for tx_message in self.transactions_with_short_ids
}
self.block = BlockBtcMessage(self.magic, self.version,
self.prev_block, self.merkle_root,
int(time.time()), self.bits, self.nonce,
self.btc_transactions_for_block)
def gateway_1_opts(self):
return gateway_helpers.get_gateway_opts(
9000,
peer_gateways=[OutboundPeerModel(LOCALHOST, 7002)],
include_default_btc_args=True,
encrypt_blocks=False,
sync_tx_service=False)
def gateway_2_opts(self):
return gateway_helpers.get_gateway_opts(
9001,
peer_gateways=[OutboundPeerModel(LOCALHOST, 7002)],
include_default_btc_args=True,
encrypt_blocks=False,
sync_tx_service=False)
def _send_compressed_block_to_node_1(self):
for tx_message_with_short_id in self.transactions_with_short_ids:
helpers.receive_node_message(self.node2, self.relay_fileno,
tx_message_with_short_id.rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
helpers.receive_node_message(self.node2, self.blockchain_fileno,
self.block.rawbytes())
broadcast_bytes = helpers.get_queued_node_bytes(
self.node2, self.relay_fileno, BroadcastMessage.MESSAGE_TYPE)
helpers.receive_node_message(self.node1, self.relay_fileno,
broadcast_bytes)
get_txs_bytes = helpers.get_queued_node_bytes(
self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE)
get_txs_message = GetTxsMessage(buf=get_txs_bytes.tobytes())
self._assert_no_block_node_1()
return get_txs_message
def _assert_no_block_node_1(self):
empty_bytes = self.node1.get_bytes_to_send(self.blockchain_fileno)
self.assertEqual(0, len(empty_bytes))
def _assert_sent_block_node_1(self):
found_block = False
bytes_to_send = self.node1.get_bytes_to_send(self.blockchain_fileno)
while bytes_to_send:
found_block |= BlockBtcMessage.MESSAGE_TYPE in bytes_to_send.tobytes(
)
if found_block:
break
self.node1.on_bytes_sent(self.blockchain_fileno,
len(bytes_to_send))
bytes_to_send = self.node1.get_bytes_to_send(
self.blockchain_fileno)
self.assertTrue(found_block)
self.assertEqual(self.block.rawbytes(), bytes_to_send)
def _build_txs_message(self, short_ids: List[int]):
txs: List[TransactionInfo] = []
for short_id in short_ids:
transaction = self.transactions_by_short_id[short_id]
txs.append(
TransactionInfo(transaction.tx_hash(), transaction.tx_val(),
transaction.short_id()))
return TxsMessage(txs)
def test_recover_all_short_ids(self):
for tx_message in self.transactions:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
get_txs_message = self._send_compressed_block_to_node_1()
self.assertEqual(10, len(get_txs_message.get_short_ids()))
txs_message = self._build_txs_message(get_txs_message.get_short_ids())
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
self._assert_sent_block_node_1()
def test_recover_missing_tx_contents(self):
for tx_message in self.transactions_with_no_content[:3]:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
for tx_message in self.transactions[3:]:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
get_txs_message = self._send_compressed_block_to_node_1()
self.assertEqual(10, len(get_txs_message.get_short_ids()))
txs_message = self._build_txs_message(get_txs_message.get_short_ids())
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
self._assert_sent_block_node_1()
def test_recover_multiple_iterations(self):
for tx_message in self.transactions:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
get_txs_message = self._send_compressed_block_to_node_1()
txs_message = self._build_txs_message(
get_txs_message.get_short_ids()[:5])
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
time.time = MagicMock(
return_value=time.time() +
gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[0])
self.node1.alarm_queue.fire_alarms()
get_txs_bytes_2 = helpers.get_queued_node_bytes(
self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE)
get_txs_message_2 = GetTxsMessage(buf=get_txs_bytes_2.tobytes())
self.assertEqual(5, len(get_txs_message_2.get_short_ids()))
txs_message = self._build_txs_message(
get_txs_message_2.get_short_ids()[:-1])
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
# retry again in a longer interval
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
time.time = MagicMock(
return_value=time.time() +
gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[0])
self.node1.alarm_queue.fire_alarms()
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
time.time = MagicMock(
return_value=time.time() +
gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[1])
self.node1.alarm_queue.fire_alarms()
get_txs_bytes_3 = helpers.get_queued_node_bytes(
self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE)
get_txs_message_3 = GetTxsMessage(buf=get_txs_bytes_3.tobytes())
self.assertEqual(1, len(get_txs_message_3.get_short_ids()))
txs_message = self._build_txs_message(
get_txs_message_3.get_short_ids())
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
self._assert_sent_block_node_1()
def test_recover_give_up(self):
gateway_constants.BLOCK_RECOVERY_MAX_RETRY_ATTEMPTS = 3
for tx_message in self.transactions:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
self._send_compressed_block_to_node_1()
txs_message = self._build_txs_message([])
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
# retry, first attempt
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
time.time = MagicMock(
return_value=time.time() +
gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[0])
self.node1.alarm_queue.fire_alarms()
get_txs_bytes_2 = helpers.get_queued_node_bytes(
self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE)
get_txs_message_2 = GetTxsMessage(buf=get_txs_bytes_2.tobytes())
self.assertEqual(10, len(get_txs_message_2.get_short_ids()))
txs_message = self._build_txs_message([])
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
# retry, attempt 2
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
time.time = MagicMock(
return_value=time.time() +
gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[1])
self.node1.alarm_queue.fire_alarms()
get_txs_bytes_3 = helpers.get_queued_node_bytes(
self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE)
get_txs_message_3 = GetTxsMessage(buf=get_txs_bytes_3.tobytes())
self.assertEqual(10, len(get_txs_message_3.get_short_ids()))
txs_message = self._build_txs_message([])
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
# retry, attempt 3
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
time.time = MagicMock(
return_value=time.time() +
gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[2])
self.node1.alarm_queue.fire_alarms()
get_txs_bytes_4 = helpers.get_queued_node_bytes(
self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE)
get_txs_message_4 = GetTxsMessage(buf=get_txs_bytes_4.tobytes())
self.assertEqual(10, len(get_txs_message_4.get_short_ids()))
txs_message = self._build_txs_message([])
helpers.receive_node_message(self.node1, self.relay_fileno,
txs_message.rawbytes())
# retry given up
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
time.time = MagicMock(
return_value=time.time() +
gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[3] * 20)
self.node1.alarm_queue.fire_alarms()
# no bytes, even after timeout
bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno)
self.assertEqual(0, len(bytes_to_send))
self._assert_no_block_node_1()
def test_recover_from_single_tx_short_id(self):
for tx_message in self.transactions_with_short_ids[:-1]:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
_get_txs_message = self._send_compressed_block_to_node_1()
helpers.receive_node_message(
self.node1, self.relay_fileno,
self.transactions_with_short_ids[-1].rawbytes())
self._assert_sent_block_node_1()
def test_recover_from_single_tx_short_id_no_content(self):
for tx_message in self.transactions_with_short_ids[:-1]:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
helpers.receive_node_message(self.node1, self.relay_fileno,
self.transactions[-1].rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
_get_txs_message = self._send_compressed_block_to_node_1()
helpers.receive_node_message(
self.node1, self.relay_fileno,
self.transactions_with_short_ids[-1].rawbytes())
self._assert_sent_block_node_1()
def test_recover_from_single_tx_val(self):
for tx_message in self.transactions_with_short_ids[:-1]:
helpers.receive_node_message(self.node1, self.relay_fileno,
tx_message.rawbytes())
helpers.receive_node_message(
self.node1, self.relay_fileno,
self.transactions_with_no_content[-1].rawbytes())
helpers.clear_node_buffer(self.node1, self.blockchain_fileno)
_get_txs_message = self._send_compressed_block_to_node_1()
helpers.receive_node_message(
self.node1, self.relay_fileno,
self.transactions_with_short_ids[-1].rawbytes())
self._assert_sent_block_node_1()
def get_recovered_compact_block():
buf = bytearray(convert.hex_to_bytes(FULL_BLOCK_BYTES_HEX))
parsed_block = BlockBtcMessage(buf=buf)
return parsed_block
