def msg_inv(self, msg: InvBtcMessage) -> None:
"""
Handle an inventory message.
Requests all transactions and blocks that haven't been previously seen.
:param msg: INV message
"""
contains_block = False
inventory_requests = []
block_hashes = []
for inventory_type, item_hash in msg:
if InventoryType.is_block(inventory_type):
if not self.node.should_process_block_hash(item_hash):
continue
block_hashes.append(item_hash)
if item_hash not in self.node.blocks_seen.contents:
contains_block = True
inventory_requests.append((inventory_type, item_hash))
else:
inventory_requests.append((inventory_type, item_hash))
self.node.block_cleanup_service.mark_blocks_and_request_cleanup(
block_hashes)
if inventory_requests:
get_data = GetDataBtcMessage(
magic=msg.magic(),
inv_vects=inventory_requests,
request_witness_data=self.request_witness_data)
self.connection.enqueue_msg(get_data, prepend=contains_block)
self.node.block_queuing_service.mark_blocks_seen_by_blockchain_node(
block_hashes)
def _request_block(self, block_hash: Sha256Hash):
block_request_message = GetDataBtcMessage(
magic=self.node.opts.blockchain_net_magic,
inv_vects=[(InventoryType.MSG_BLOCK, block_hash)],
request_witness_data=False)
self.node.send_msg_to_node(block_request_message)
logger.trace("Received block cleanup request: {}", block_hash)
def test_parse_message_success_all_types(self):
# TODO: pull these numbers into constants, along with all the BTC messages
self.create_message_successfully(self.VERSION_BTC_MESSAGE,
VersionBtcMessage)
self.create_message_successfully(VerAckBtcMessage(self.MAGIC),
VerAckBtcMessage)
self.create_message_successfully(PingBtcMessage(self.MAGIC),
PingBtcMessage)
self.create_message_successfully(PongBtcMessage(self.MAGIC, 123),
PongBtcMessage)
self.create_message_successfully(GetAddrBtcMessage(self.MAGIC),
GetAddrBtcMessage)
self.create_message_successfully(
AddrBtcMessage(self.MAGIC,
[(int(time.time()), "127.0.0.1", 8000)]),
AddrBtcMessage)
inv_vector = [(1, self.HASH), (2, self.HASH)]
self.create_message_successfully(InvBtcMessage(self.MAGIC, inv_vector),
InvBtcMessage)
self.create_message_successfully(
GetDataBtcMessage(self.MAGIC, inv_vector), GetDataBtcMessage)
self.create_message_successfully(
NotFoundBtcMessage(self.MAGIC, inv_vector), NotFoundBtcMessage)
hashes = [self.HASH, self.HASH]
self.create_message_successfully(
GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetHeadersBtcMessage)
self.create_message_successfully(
GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetBlocksBtcMessage)
self.create_message_successfully(
TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0), TxBtcMessage)
txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5
self.create_message_successfully(
BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0,
0, 0, txs), BlockBtcMessage)
self.create_message_successfully(
HeadersBtcMessage(self.MAGIC,
[helpers.generate_bytearray(81)] * 2),
HeadersBtcMessage)
self.create_message_successfully(
RejectBtcMessage(self.MAGIC, b"a message",
RejectBtcMessage.REJECT_MALFORMED, b"test break",
helpers.generate_bytearray(10)), RejectBtcMessage)
self.create_message_successfully(SendHeadersBtcMessage(self.MAGIC),
SendHeadersBtcMessage)
self.create_message_successfully(
FeeFilterBtcMessage(self.MAGIC, fee_rate=100), FeeFilterBtcMessage)
self.create_message_successfully(
BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)),
XversionBtcMessage)
def _request_block(self, block_hash: Sha256Hash):
block_request_message = GetDataBtcMessage(
magic=self.node.opts.blockchain_net_magic,
inv_vects=[(InventoryType.MSG_BLOCK, block_hash)],
request_witness_data=False
)
logger.trace("Received block cleanup request: {}", block_hash)
node_conn = self.node.get_any_active_blockchain_connection()
if node_conn:
node_conn.enqueue_msg(block_request_message)
else:
logger.debug("Request for block '{}' failed. No connection to node.", repr(block_hash))
def test_get_data_only_new_data(self):
seen_block_hash = BtcObjectHash(
buf=helpers.generate_bytearray(BTC_SHA_HASH_LEN),
length=BTC_SHA_HASH_LEN)
not_seen_block_hash = BtcObjectHash(
buf=helpers.generate_bytearray(BTC_SHA_HASH_LEN),
length=BTC_SHA_HASH_LEN)
self.node.blocks_seen.add(seen_block_hash)
inv_message = InvBtcMessage(
magic=123,
inv_vects=[(InventoryType.MSG_TX, seen_block_hash),
(InventoryType.MSG_BLOCK, not_seen_block_hash),
(InventoryType.MSG_BLOCK, seen_block_hash)])
self.sut.msg_inv(inv_message)
get_data_msg_bytes = self.sut.connection.get_bytes_to_send()
get_data_msg = GetDataBtcMessage(buf=get_data_msg_bytes)
self.assertEqual(2, get_data_msg.count())
self.assertIn((InventoryType.MSG_TX, seen_block_hash), get_data_msg)
self.assertIn((InventoryType.MSG_BLOCK, not_seen_block_hash),
get_data_msg)
def _test_get_data(self, segwit):
self.sut.request_witness_data = segwit
inv_msg = self._create_inv_msg()
self.sut.msg_inv(inv_msg)
get_data_msg_bytes = self.sut.connection.get_bytes_to_send()
get_data_msg = GetDataBtcMessage(buf=get_data_msg_bytes)
self.assertEqual(2, get_data_msg.count())
item_index = 0
for inv_item in get_data_msg:
if (item_index == 0):
self.assertEqual(
InventoryType.MSG_WITNESS_TX
if segwit else InventoryType.MSG_TX, inv_item[0])
self.assertEqual(self.tx_hash, inv_item[1])
else:
self.assertEqual(
InventoryType.MSG_WITNESS_BLOCK
if segwit else InventoryType.MSG_BLOCK, inv_item[0])
self.assertEqual(self.block_hash, inv_item[1])
item_index += 1
def process_compact_block_recovery(
self,
msg: BlockTransactionsBtcMessage,
failure_result: CompactBlockCompressionResult,
connection: BtcNodeConnection
) -> None:
"""
Process compact block recovery .
If no hold exists, compress and broadcast block immediately.
"""
block_hash = msg.block_hash()
for txn in msg.transactions():
failure_result.recovered_transactions.append(txn)
try:
recovery_result = self._node.message_converter.recovered_compact_block_to_bx_block( # pyre-ignore
failure_result
)
except MessageConversionError as e:
block_stats.add_block_event_by_block_hash(
e.msg_hash,
BlockStatEventType.BLOCK_CONVERSION_FAILED,
network_num=connection.network_num,
conversion_type=e.conversion_type.value
)
logger.warning(log_messages.COMPACT_BLOCK_PROCESSING_FAIL,
e.msg_hash, e)
get_data_msg = GetDataBtcMessage(
magic=msg.magic(),
inv_vects=[(InventoryType.MSG_BLOCK, msg.block_hash())]
)
connection.enqueue_msg(get_data_msg)
return
block_info = recovery_result.block_info
if recovery_result.success:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_RECOVERY_SUCCESS,
network_num=connection.network_num,
start_date_time=block_info.start_datetime,
end_date_time=block_info.end_datetime,
duration=block_info.duration_ms / 1000,
success=recovery_result.success,
recoverd_txs_count=len(msg.transactions()),
txs_count=recovery_result.block_info.txn_count,
prev_block_hash=recovery_result.block_info.prev_block_hash,
more_info="{:.2f}ms, {:f}".format(
block_info.duration_ms,
len(msg.transactions())
)
)
prev_block = Sha256Hash(convert.hex_to_bytes(block_info.prev_block_hash))
self._node.block_cleanup_service.on_new_block_received(block_hash, prev_block)
self._process_and_broadcast_compressed_block(
recovery_result.bx_block,
connection,
recovery_result.block_info,
msg.block_hash()
)
else:
start_datetime = block_info.start_datetime
end_datetime = datetime.utcnow()
duration = (end_datetime - start_datetime).total_seconds()
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_RECOVERY_FAILED,
network_num=connection.network_num,
start_date_time=start_datetime,
end_date_time=end_datetime,
duration=duration,
success=recovery_result.success,
recoverd_txs_count=len(msg.transactions()),
more_info="{:.2f}ms, {:f}".format(
duration * 1000,
len(msg.transactions())
)
)
logger.warning(log_messages.COMPACT_BLOCK_RECOVERY_FAIL, msg.block_hash())
get_data_msg = GetDataBtcMessage(
magic=msg.magic(),
inv_vects=[(InventoryType.MSG_BLOCK, msg.block_hash())]
)
connection.enqueue_msg(get_data_msg)
def msg_compact_block(self, msg: CompactBlockBtcMessage) -> None:
"""
Handle COMPACT BLOCK message from Bitcoin node
:param msg: COMPACT BLOCK message
"""
block_hash = msg.block_hash()
if not self.node.should_process_block_hash(block_hash):
return
short_ids_count = len(msg.short_ids())
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE,
network_num=self.connection.network_num,
peer=self.connection.peer_desc,
more_info="{} short ids".format(short_ids_count))
if block_hash in self.node.blocks_seen.contents:
self.node.on_block_seen_by_blockchain_node(block_hash)
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.
COMPACT_BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN,
network_num=self.connection.network_num,
peer=self.connection.peer_desc)
self.connection.log_info(
"Discarding duplicate block {} from local blockchain node.",
block_hash)
return
max_time_offset = self.node.opts.blockchain_block_interval * self.node.opts.blockchain_ignore_block_interval_count
if time.time() - msg.timestamp() >= max_time_offset:
self.connection.log_trace(
"Received block {} more than {} seconds after it was created ({}). Ignoring.",
block_hash, max_time_offset, msg.timestamp())
return
self.node.track_block_from_node_handling_started(block_hash)
if short_ids_count < self.node.opts.compact_block_min_tx_count:
self.connection.log_debug(
"Compact block {} contains {} short transactions, less than limit {}. Requesting full block.",
convert.bytes_to_hex(msg.block_hash().binary), short_ids_count,
btc_constants.BTC_COMPACT_BLOCK_DECOMPRESS_MIN_TX_COUNT)
get_data_msg = GetDataBtcMessage(magic=self.magic,
inv_vects=[
(InventoryType.MSG_BLOCK,
msg.block_hash())
])
self.node.send_msg_to_node(get_data_msg)
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.COMPACT_BLOCK_REQUEST_FULL,
network_num=self.connection.network_num)
return
self.node.block_cleanup_service.on_new_block_received(
msg.block_hash(), msg.prev_block_hash())
self.node.on_block_seen_by_blockchain_node(block_hash)
self.connection.log_info(
"Processing compact block {} from local Bitcoin node.", block_hash)
try:
parse_result = self.node.block_processing_service.process_compact_block(
msg, self.connection)
except MessageConversionError as e:
block_stats.add_block_event_by_block_hash(
e.msg_hash,
BlockStatEventType.BLOCK_CONVERSION_FAILED,
network_num=self.connection.network_num,
conversion_type=e.conversion_type.value)
self.connection.log_warning(log_messages.PROCESS_BLOCK_FAILURE,
e.msg_hash, e)
get_data_msg = GetDataBtcMessage(magic=self.magic,
inv_vects=[
(InventoryType.MSG_BLOCK,
msg.block_hash())
])
self.node.send_msg_to_node(get_data_msg)
return
if not parse_result.success:
self._recovery_compact_blocks.add(block_hash, parse_result)
get_block_txs_msg = GetBlockTransactionsBtcMessage(
magic=self.magic,
block_hash=block_hash,
indices=parse_result.missing_indices)
self.node.send_msg_to_node(get_block_txs_msg)
