def setUp(self): self.node = MockGatewayNode(gateway_helpers.get_gateway_opts( 8000, include_default_btc_args=True, compact_block_min_tx_count=5 )) self.node.block_processing_service = MagicMock() self.connection = BtcNodeConnection( MockSocketConnection(node=self.node, ip_address=LOCALHOST, port=123), self.node ) self.connection.node = self.node self.connection.peer_ip = LOCALHOST self.connection.peer_port = 8001 self.connection.network_num = 2 self.sut = BtcNodeConnectionProtocol(self.connection) full_block_msg = BlockBtcMessage( buf=bytearray(convert.hex_to_bytes(self.FULL_BLOCK_BYTES_HEX)) ) if self.node.opts.use_extensions: transaction_service = ExtensionTransactionService(self.node, 0) else: transaction_service = TransactionService(self.node, 0) short_id = 1 for tx in full_block_msg.txns(): tx_hash = btc_common_utils.get_txid(tx) transaction_service.set_transaction_contents(tx_hash, tx) transaction_service.assign_short_id(tx_hash, short_id) short_id += 1 self.sut.connection.node._tx_service = transaction_service
def setUp(self): super().setUp() self.node1.alarm_queue = AlarmQueue() self.node2.alarm_queue = AlarmQueue() self.network_num = 1 self.magic = 12345 self.version = 23456 self.prev_block_hash = bytearray(crypto.double_sha256(b"123")) self.prev_block = BtcObjectHash(self.prev_block_hash, length=crypto.SHA256_HASH_LEN) self.merkle_root_hash = bytearray(crypto.double_sha256(b"234")) self.merkle_root = BtcObjectHash(self.merkle_root_hash, length=crypto.SHA256_HASH_LEN) self.bits = 2 self.nonce = 3 opts = self.gateway_1_opts() if opts.use_extensions: helpers.set_extensions_parallelism() self.btc_message_converter = btc_message_converter_factory.create_btc_message_converter( self.magic, opts) self.btc_transactions = [ TxBtcMessage(self.magic, self.version, [], [], i) for i in range(self.TRANSACTIONS_COUNT) ] self.btc_transactions_for_block = [ tx_btc_message.rawbytes()[btc_constants.BTC_HDR_COMMON_OFF:] for tx_btc_message in self.btc_transactions ] self.transactions = [ self.btc_message_converter.tx_to_bx_txs(tx_btc_message, self.network_num)[0][0] for tx_btc_message in self.btc_transactions ] self.transactions_with_short_ids = [ TxMessage(tx_message.tx_hash(), tx_message.network_num(), "", i + 1, tx_message.tx_val()) for i, tx_message in enumerate(self.transactions) ] self.transactions_with_no_content = [ TxMessage(tx_message.tx_hash(), tx_message.network_num(), "", i + 1) for i, tx_message in enumerate(self.transactions) ] self.transactions_by_short_id = { tx_message.short_id(): tx_message for tx_message in self.transactions_with_short_ids } self.block = BlockBtcMessage(self.magic, self.version, self.prev_block, self.merkle_root, int(time.time()), self.bits, self.nonce, self.btc_transactions_for_block)
def clean_block_transactions( self, block_msg: BlockBtcMessage, transaction_service: TransactionService) -> None: block_short_ids = [] block_unknown_tx_hashes = [] start_time = time.time() short_ids_count = 0 unknown_tx_hashes_count = 0 transactions_processed = 0 tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_before_cleanup = transaction_service.get_short_id_count( ) for tx in block_msg.txns(): tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx), length=BTC_SHA_HASH_LEN) short_ids = transaction_service.remove_transaction_by_tx_hash( tx_hash, force=True) if short_ids is None: unknown_tx_hashes_count += 1 block_unknown_tx_hashes.append(tx_hash) else: short_ids_count += len(short_ids) block_short_ids.extend(short_ids) transactions_processed += 1 block_hash = block_msg.block_hash() transaction_service.on_block_cleaned_up(block_hash) end_time = time.time() duration = end_time - start_time tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len( ) short_id_count_after_cleanup = transaction_service.get_short_id_count() logger.debug( "Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} " "short ids. Took {:.3f}s.", block_hash, transactions_processed, unknown_tx_hashes_count, short_ids_count, duration) transaction_service.log_block_transaction_cleanup_stats( block_hash, block_msg.txn_count(), tx_hash_to_contents_len_before_cleanup, tx_hash_to_contents_len_after_cleanup, short_id_count_before_cleanup, short_id_count_after_cleanup) self._block_hash_marked_for_cleanup.discard(block_hash) self.node.post_block_cleanup_tasks( block_hash=block_hash, short_ids=block_short_ids, unknown_tx_hashes=block_unknown_tx_hashes)
def _get_sample_block(self, file_path=__file__): root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(file_path)))) with open(os.path.join(root_dir, "btc_sample_block.txt")) as sample_file: btc_block = sample_file.read().strip("\n") buf = bytearray(convert.hex_to_bytes(btc_block)) parsed_block = BlockBtcMessage(buf=buf) return parsed_block
def test_send_receive_block_decrypted(self): self.node1.opts.encrypt_blocks = False send_block = btc_block(int(time.time())) self._populate_transaction_services(send_block) # propagate block helpers.receive_node_message(self.node1, self.blockchain_fileno, send_block.rawbytes()) block_hold_msg = self.node1.get_bytes_to_send(self.relay_fileno) self.assertIn(BlockHoldingMessage.MESSAGE_TYPE, block_hold_msg.tobytes()) self.node1.on_bytes_sent(self.relay_fileno, len(block_hold_msg)) relayed_block = self.node1.get_bytes_to_send(self.relay_fileno) self.assertIn(BroadcastMessage.MESSAGE_TYPE, relayed_block.tobytes()) self.node1.on_bytes_sent(self.relay_fileno, len(relayed_block)) # block directly propagated helpers.receive_node_message(self.node2, self.relay_fileno, relayed_block) bytes_to_blockchain = self.node2.get_bytes_to_send(self.blockchain_fileno) self.assertEqual(len(send_block.rawbytes()), len(bytes_to_blockchain)) received_block = BlockBtcMessage(buf=bytearray(bytes_to_blockchain)) self.assertEqual(send_block.magic(), received_block.magic()) self.assertEqual(send_block.version(), received_block.version()) self.assertEqual(send_block.timestamp(), received_block.timestamp()) self.assertEqual(send_block.bits(), received_block.bits()) self.assertEqual(send_block.nonce(), received_block.nonce())
def test_parse_message_success_all_types(self): # TODO: pull these numbers into constants, along with all the BTC messages self.create_message_successfully(self.VERSION_BTC_MESSAGE, VersionBtcMessage) self.create_message_successfully(VerAckBtcMessage(self.MAGIC), VerAckBtcMessage) self.create_message_successfully(PingBtcMessage(self.MAGIC), PingBtcMessage) self.create_message_successfully(PongBtcMessage(self.MAGIC, 123), PongBtcMessage) self.create_message_successfully(GetAddrBtcMessage(self.MAGIC), GetAddrBtcMessage) self.create_message_successfully( AddrBtcMessage(self.MAGIC, [(int(time.time()), "127.0.0.1", 8000)]), AddrBtcMessage) inv_vector = [(1, self.HASH), (2, self.HASH)] self.create_message_successfully(InvBtcMessage(self.MAGIC, inv_vector), InvBtcMessage) self.create_message_successfully( GetDataBtcMessage(self.MAGIC, inv_vector), GetDataBtcMessage) self.create_message_successfully( NotFoundBtcMessage(self.MAGIC, inv_vector), NotFoundBtcMessage) hashes = [self.HASH, self.HASH] self.create_message_successfully( GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH), GetHeadersBtcMessage) self.create_message_successfully( GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH), GetBlocksBtcMessage) self.create_message_successfully( TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0), TxBtcMessage) txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5 self.create_message_successfully( BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0, 0, 0, txs), BlockBtcMessage) self.create_message_successfully( HeadersBtcMessage(self.MAGIC, [helpers.generate_bytearray(81)] * 2), HeadersBtcMessage) self.create_message_successfully( RejectBtcMessage(self.MAGIC, b"a message", RejectBtcMessage.REJECT_MALFORMED, b"test break", helpers.generate_bytearray(10)), RejectBtcMessage) self.create_message_successfully(SendHeadersBtcMessage(self.MAGIC), SendHeadersBtcMessage) self.create_message_successfully( FeeFilterBtcMessage(self.MAGIC, fee_rate=100), FeeFilterBtcMessage) self.create_message_successfully( BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)), XversionBtcMessage)
def build_btc_block(block_pieces: Deque[Union[bytearray, memoryview]], size: int) -> Tuple[BlockBtcMessage, int]: btc_block = bytearray(size) offset = 0 for piece in block_pieces: next_offset = offset + len(piece) btc_block[offset:next_offset] = piece offset = next_offset return BlockBtcMessage(buf=btc_block), offset
def msg_block(self, msg: BlockBtcMessage) -> None: block_stats.add_block_event_by_block_hash( msg.block_hash(), BlockStatEventType.REMOTE_BLOCK_RECEIVED_BY_GATEWAY, network_num=self.connection.network_num, more_info="Protocol: {}, Network: {}".format( self.connection.node.opts.blockchain_protocol, self.connection.node.opts.blockchain_network)) return self.msg_proxy_response(msg)
def get_segwit_block(): root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) with open(os.path.join(root_dir, "samples/btc_segwit_sample_block.txt")) as sample_file: btc_block = sample_file.read().strip("\n") block = convert.hex_to_bytes(btc_block) buf = bytearray(BTC_HDR_COMMON_OFF + len(block)) buf[BTC_HDR_COMMON_OFF:] = block msg = BtcMessage(magic="main", command=BlockBtcMessage.MESSAGE_TYPE, payload_len=len(block), buf=buf) parsed_block = BlockBtcMessage(buf=msg.buf) return parsed_block
def on_block_sent(self, block_hash: Sha256Hash, block_message: BlockBtcMessage): # After sending block message to Bitcoin node sending INV message for the same block to the node # This is needed to update Synced Headers value of the gateway peer on the Bitcoin node # If Synced Headers is not up-to-date than Bitcoin node does not push compact blocks to the gateway inv_msg = InvBtcMessage( magic=block_message.magic(), inv_vects=[(InventoryType.MSG_BLOCK, block_hash)], ) self.node.send_msg_to_node(inv_msg)
def clean_block_transactions( self, block_msg: BlockBtcMessage, transaction_service: TransactionService ) -> None: start_datetime = datetime.utcnow() start_time = time.time() tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len() cleanup_task = self.block_cleanup_tasks.borrow_task() tx_service = typing.cast(ExtensionTransactionService, transaction_service) cleanup_task.init(tpe.InputBytes(block_msg.buf), tx_service.proxy) init_time = time.time() task_pool_proxy.run_task(cleanup_task) task_run_time = time.time() unknown_tx_hashes_count = len(cleanup_task.unknown_tx_hashes()) tx_property_fetch_time = time.time() short_ids = cleanup_task.short_ids() short_ids_fetch_time = time.time() short_ids_count = len(short_ids) tx_service.update_removed_transactions(cleanup_task.total_content_removed(), short_ids) remove_from_tx_service_time = time.time() # TODO : clean the short ids/transactions from the alarm queue after refactoring the transaction service block_hash = block_msg.block_hash() tx_service.on_block_cleaned_up(block_hash) tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len() end_datetime = datetime.utcnow() end_time = time.time() logger.statistics( { "type": "BlockTransactionsCleanup", "block_hash": repr(block_hash), "unknown_tx_hashes_count": unknown_tx_hashes_count, "short_ids_count": short_ids_count, "block_transactions_count": cleanup_task.txn_count(), "start_datetime": start_datetime, "end_datetime": end_datetime, "task_init_time": init_time - start_time, "task_run_time": task_run_time - init_time, "tx_property_fetch_time": tx_property_fetch_time - task_run_time, "short_ids_fetch_time": short_ids_fetch_time - tx_property_fetch_time, "remove_from_tx_service_time": remove_from_tx_service_time - short_ids_fetch_time, "duration": end_time - start_time, "tx_hash_to_contents_len_before_cleanup": tx_hash_to_contents_len_before_cleanup, "tx_hash_to_contents_len_after_cleanup": tx_hash_to_contents_len_after_cleanup, } ) self.block_cleanup_tasks.return_task(cleanup_task) self._block_hash_marked_for_cleanup.discard(block_hash) self.node.post_block_cleanup_tasks( block_hash=block_hash, short_ids=short_ids, unknown_tx_hashes=( Sha256Hash(convert.hex_to_bytes(tx_hash.hex_string())) for tx_hash in cleanup_task.unknown_tx_hashes() ) )
def send_received_block_and_key(self, block=None): if block is None: block = btc_block() self._populate_transaction_services(block) # propagate block helpers.receive_node_message(self.node1, self.blockchain_fileno, block.rawbytes()) block_hold_request_relay = self.node1.get_bytes_to_send( self.relay_fileno) self.assertIn(BlockHoldingMessage.MESSAGE_TYPE, block_hold_request_relay.tobytes()) self.node1.on_bytes_sent(self.relay_fileno, len(block_hold_request_relay)) relayed_block = self.node1.get_bytes_to_send(self.relay_fileno) self.assertIn(BroadcastMessage.MESSAGE_TYPE, relayed_block.tobytes()) self.node1.on_bytes_sent(self.relay_fileno, len(relayed_block)) # block hold sent out block_hold_request_gateway = self.node1.get_bytes_to_send( self.gateway_fileno) self.assertIsNotNone(block_hold_request_gateway) self.assertIn(BlockHoldingMessage.MESSAGE_TYPE, block_hold_request_gateway.tobytes()) self.clear_all_buffers() # key not available until receipt key_not_yet_available = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(OutputBuffer.EMPTY, key_not_yet_available) # send receipt helpers.receive_node_message(self.node2, self.relay_fileno, relayed_block) block_receipt = self.node2.get_bytes_to_send(self.gateway_fileno) self.assertIn(BlockReceivedMessage.MESSAGE_TYPE, block_receipt.tobytes()) # send key helpers.receive_node_message(self.node1, self.gateway_fileno, block_receipt) key_message = self.node1.get_bytes_to_send(self.relay_fileno) self.assertIn(KeyMessage.MESSAGE_TYPE, key_message.tobytes()) helpers.receive_node_message(self.node2, self.relay_fileno, key_message) bytes_to_blockchain = self.node2.get_bytes_to_send( self.blockchain_fileno) self.assertEqual(len(block.rawbytes()), len(bytes_to_blockchain)) return BlockBtcMessage(buf=bytearray(bytes_to_blockchain))
def setUp(self): self.local_node_fileno = 1 self.remote_node_fileno = 2 self.gateway_node = spies.make_spy_node(BtcGatewayNode, 8000, include_default_btc_args=True) self.btc_node_connection = spies.make_spy_connection( BtcNodeConnection, self.local_node_fileno, 8001, self.gateway_node) self.btc_remote_node_connection = spies.make_spy_connection( BtcRemoteConnection, self.remote_node_fileno, 8002, self.gateway_node) self.gateway_node.sdn_connection = MagicMock() self.gateway_node.node_conn = self.btc_node_connection self.gateway_node.remote_node_conn = self.btc_remote_node_connection self.gateway_node.connection_pool.add(self.local_node_fileno, LOCALHOST, 8001, self.btc_node_connection) self.gateway_node.connection_pool.add(self.remote_node_fileno, LOCALHOST, 8002, self.btc_remote_node_connection) self.gateway_node.account_id = "1234" self.handshake = \ bytearray(VersionBtcMessage(100, 1000, "127.0.0.1", 8333, "0.0.0.0", 8333, 13, 0, bytearray("hello", "utf-8"), 0).rawbytes()) + \ bytearray(VerAckBtcMessage(100).rawbytes()) inps = [ TxIn(prev_outpoint_hash=bytearray("0" * 32, "utf-8"), prev_out_index=10, sig_script=bytearray(1000), sequence=32) ] outs = [TxOut(value=10, pk_script=bytearray(1000))] self.txmsg = bytearray( TxBtcMessage(magic=100, version=0, tx_in=inps, tx_out=outs, lock_time=12345).rawbytes()) self.blockmsg = bytearray( BlockBtcMessage(magic=100, version=0, prev_block=get_object_hash(bytearray(32)), merkle_root=get_object_hash(bytearray(32)), timestamp=1023, bits=1, txns=[bytearray(64)] * 32000, nonce=34).rawbytes()) self.transform = MessageTransformations()
def btc_block(timestamp=None, real_block=None): if real_block is not None: return BlockBtcMessage(buf=bytearray(convert.hex_to_bytes(real_block))) if timestamp is None: timestamp = int(time.time()) magic = 12345 version = 23456 prev_block_hash = bytearray(crypto.double_sha256(b"123")) prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN) merkle_root_hash = bytearray(crypto.double_sha256(b"234")) merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN) bits = 2 nonce = 3 txns = [ TxBtcMessage(magic, version, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:] for i in range(10) ] return BlockBtcMessage(magic, version, prev_block, merkle_root, timestamp, bits, nonce, txns)
def test_msg_block_too_old(self): block_timestamp = int( time.time() ) - 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval txns = [ TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:] for i in range(10) ] message = BlockBtcMessage(0, 0, self.HASH, self.HASH, 0, block_timestamp, 0, txns) self.sut.msg_block(message) self.node.block_processing_service.queue_block_for_processing.assert_not_called( )
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult: decompress_start_datetime = datetime.utcnow() decompress_start_timestamp = time.time() tsk = self.decompression_tasks.borrow_task() tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.decompression_tasks.return_task(tsk) header_info = btc_normal_message_converter.parse_bx_block_header(bx_block_msg, deque()) raise message_conversion_error.btc_block_decompression_error(header_info.block_hash, e) total_tx_count = tsk.tx_count() unknown_tx_hashes = [Sha256Hash(bytearray(unknown_tx_hash.binary())) for unknown_tx_hash in tsk.unknown_tx_hashes()] unknown_tx_sids = tsk.unknown_tx_sids() block_hash = BtcObjectHash( binary=convert.hex_to_bytes(tsk.block_hash().hex_string()) ) if tsk.success(): btc_block_msg = BlockBtcMessage(buf=memoryview(tsk.block_message())) logger.debug( "Successfully parsed block broadcast message. {} transactions " "in block {}", total_tx_count, block_hash ) else: btc_block_msg = None logger.debug( "Block recovery needed for {}. Missing {} sids, {} tx hashes. " "Total txs in block: {}", block_hash, len(unknown_tx_sids), len(unknown_tx_hashes), total_tx_count ) block_info = get_block_info( bx_block_msg, block_hash, tsk.short_ids(), decompress_start_datetime, decompress_start_timestamp, total_tx_count, btc_block_msg ) self.decompression_tasks.return_task(tsk) return BlockDecompressionResult(btc_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult: start_datetime = datetime.datetime.utcnow() start_time = time.time() block_msg = BlockBtcMessage(buf=bx_block_msg) block_info = BlockInfo( block_msg.block_hash(), [], start_datetime, datetime.datetime.utcnow(), (time.time() - start_time) * 1000, block_msg.txn_count(), # pyre-fixme[6]: Expected `Optional[str]` for 7th param but got # `BtcObjectHash`. block_msg.block_hash(), convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), len(block_msg.rawbytes()), 0 ) return BlockDecompressionResult(block_msg, block_info, [], [])
def test_btc_block_to_bloxroute_block_and_back_sids_found(self): prev_block_hash = bytearray(crypto.bitcoin_hash(b"123")) prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN) merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234")) merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN) timestamp = 1 bits = 2 nonce = 3 btc_block = BlockBtcMessage( self.magic, self.version, prev_block, merkle_root, timestamp, bits, nonce, self.txns ) block_hash = btc_block.block_hash() bloxroute_block, block_info = self.btc_message_converter.block_to_bx_block( btc_block, self.tx_service, True, 0 ) self.assertEqual(10, block_info.txn_count) self.assertEqual("5a77d1e9612d350b3734f6282259b7ff0a3f87d62cfef5f35e91a5604c0490a3", block_info.prev_block_hash) self.assertEqual(self.short_ids, list(block_info.short_ids)) self.assertEqual(btc_block.block_hash(), block_info.block_hash) # TODO: if we convert bloxroute block to a class, add some tests here parsed_btc_block, block_info, _, _ = self.btc_message_converter.bx_block_to_block( bloxroute_block, self.tx_service) self.assertIsNotNone(block_info) self.assertEqual(parsed_btc_block.rawbytes().tobytes(), btc_block.rawbytes().tobytes()) self.assertEqual(self.version, parsed_btc_block.version()) self.assertEqual(self.magic, parsed_btc_block.magic()) self.assertEqual(prev_block_hash, parsed_btc_block.prev_block_hash().get_little_endian()) self.assertEqual(merkle_root_hash, parsed_btc_block.merkle_root().get_little_endian()) self.assertEqual(timestamp, parsed_btc_block.timestamp()) self.assertEqual(bits, parsed_btc_block.bits()) self.assertEqual(nonce, parsed_btc_block.nonce()) self.assertEqual(len(self.txns), parsed_btc_block.txn_count()) self.assertEqual(btc_block.checksum(), parsed_btc_block.checksum()) self.assertEqual(block_hash, parsed_btc_block.block_hash()) self.assertEqual(block_hash.binary, block_info.block_hash.binary) self.assertEqual(list(block_info.short_ids), self.short_ids)
def _recovered_compact_block_to_bx_block( self, compression_result: CompactBlockCompressionResult, recovery_item: CompactBlockRecoveryData ) -> CompactBlockCompressionResult: """ Handle recovery of Bitcoin compact block message. """ missing_indices = compression_result.missing_indices recovered_transactions = compression_result.recovered_transactions block_transactions = recovery_item.block_transactions if len(missing_indices) != len(recovered_transactions): logger.debug( "Number of transactions missing in compact block does not match number of recovered transactions." "Missing transactions - {}. Recovered transactions - {}", len(missing_indices), len(recovered_transactions)) return CompactBlockCompressionResult(False, None, None, None, missing_indices, recovered_transactions) for i in range(len(missing_indices)): missing_index = missing_indices[i] block_transactions[missing_index] = recovered_transactions[i] size = 0 total_txs_count = len(block_transactions) block_msg_parts = deque() block_header = recovery_item.block_header block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size for transaction in block_transactions: block_msg_parts.append(transaction) size += len(transaction) # pyre-ignore msg_header = bytearray(btc_constants.BTC_HDR_COMMON_OFF) struct.pack_into("<L12sL", msg_header, 0, recovery_item.magic, BtcMessageType.BLOCK, size) block_msg_parts.appendleft(msg_header) size += btc_constants.BTC_HDR_COMMON_OFF block_msg_bytes = bytearray(size) off = 0 for blob in block_msg_parts: next_off = off + len(blob) block_msg_bytes[off:next_off] = blob off = next_off checksum = crypto.bitcoin_hash( block_msg_bytes[btc_constants.BTC_HDR_COMMON_OFF:size]) block_msg_bytes[btc_constants.BTC_HEADER_MINUS_CHECKSUM:btc_constants. BTC_HDR_COMMON_OFF] = checksum[0:4] bx_block, compression_block_info = self.block_to_bx_block( BlockBtcMessage(buf=block_msg_bytes), recovery_item.tx_service, True, 0) # TODO need to think about a better algorithm compress_start_datetime = compression_block_info.start_datetime compress_end_datetime = datetime.utcnow() block_info = BlockInfo( compression_block_info.block_hash, compression_block_info.short_ids, compress_start_datetime, compress_end_datetime, (compress_end_datetime - compress_start_datetime).total_seconds() * 1000, compression_block_info.txn_count, compression_block_info.compressed_block_hash, compression_block_info.prev_block_hash, compression_block_info.original_size, compression_block_info.compressed_size, compression_block_info.compression_rate, compression_block_info.ignored_short_ids) return CompactBlockCompressionResult(True, block_info, bx_block, None, [], [])
def build_block_header_message( self, block_hash: Sha256Hash, block_message: BlockBtcMessage) -> InvBtcMessage: return InvBtcMessage(block_message.magic(), [(InventoryType.MSG_BLOCK, block_hash)])
def btc_block(self, txns=None): if txns is None: txns = self.btc_transactions_bytes() return BlockBtcMessage(self.MAGIC, self.VERSION, self.BTC_HASH, self.BTC_HASH, 0, 0, 0, txns)
def get_previous_block_hash_from_message( self, block_message: BlockBtcMessage) -> Sha256Hash: return block_message.prev_block_hash()
class BlockRecoveryTest(AbstractBtcGatewayIntegrationTest): TRANSACTIONS_COUNT = 10 def setUp(self): super().setUp() self.node1.alarm_queue = AlarmQueue() self.node2.alarm_queue = AlarmQueue() self.network_num = 1 self.magic = 12345 self.version = 23456 self.prev_block_hash = bytearray(crypto.double_sha256(b"123")) self.prev_block = BtcObjectHash(self.prev_block_hash, length=crypto.SHA256_HASH_LEN) self.merkle_root_hash = bytearray(crypto.double_sha256(b"234")) self.merkle_root = BtcObjectHash(self.merkle_root_hash, length=crypto.SHA256_HASH_LEN) self.bits = 2 self.nonce = 3 opts = self.gateway_1_opts() if opts.use_extensions: helpers.set_extensions_parallelism() self.btc_message_converter = btc_message_converter_factory.create_btc_message_converter( self.magic, opts) self.btc_transactions = [ TxBtcMessage(self.magic, self.version, [], [], i) for i in range(self.TRANSACTIONS_COUNT) ] self.btc_transactions_for_block = [ tx_btc_message.rawbytes()[btc_constants.BTC_HDR_COMMON_OFF:] for tx_btc_message in self.btc_transactions ] self.transactions = [ self.btc_message_converter.tx_to_bx_txs(tx_btc_message, self.network_num)[0][0] for tx_btc_message in self.btc_transactions ] self.transactions_with_short_ids = [ TxMessage(tx_message.tx_hash(), tx_message.network_num(), "", i + 1, tx_message.tx_val()) for i, tx_message in enumerate(self.transactions) ] self.transactions_with_no_content = [ TxMessage(tx_message.tx_hash(), tx_message.network_num(), "", i + 1) for i, tx_message in enumerate(self.transactions) ] self.transactions_by_short_id = { tx_message.short_id(): tx_message for tx_message in self.transactions_with_short_ids } self.block = BlockBtcMessage(self.magic, self.version, self.prev_block, self.merkle_root, int(time.time()), self.bits, self.nonce, self.btc_transactions_for_block) def gateway_1_opts(self): return gateway_helpers.get_gateway_opts( 9000, peer_gateways=[OutboundPeerModel(LOCALHOST, 7002)], include_default_btc_args=True, encrypt_blocks=False, sync_tx_service=False) def gateway_2_opts(self): return gateway_helpers.get_gateway_opts( 9001, peer_gateways=[OutboundPeerModel(LOCALHOST, 7002)], include_default_btc_args=True, encrypt_blocks=False, sync_tx_service=False) def _send_compressed_block_to_node_1(self): for tx_message_with_short_id in self.transactions_with_short_ids: helpers.receive_node_message(self.node2, self.relay_fileno, tx_message_with_short_id.rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) helpers.receive_node_message(self.node2, self.blockchain_fileno, self.block.rawbytes()) broadcast_bytes = helpers.get_queued_node_bytes( self.node2, self.relay_fileno, BroadcastMessage.MESSAGE_TYPE) helpers.receive_node_message(self.node1, self.relay_fileno, broadcast_bytes) get_txs_bytes = helpers.get_queued_node_bytes( self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE) get_txs_message = GetTxsMessage(buf=get_txs_bytes.tobytes()) self._assert_no_block_node_1() return get_txs_message def _assert_no_block_node_1(self): empty_bytes = self.node1.get_bytes_to_send(self.blockchain_fileno) self.assertEqual(0, len(empty_bytes)) def _assert_sent_block_node_1(self): found_block = False bytes_to_send = self.node1.get_bytes_to_send(self.blockchain_fileno) while bytes_to_send: found_block |= BlockBtcMessage.MESSAGE_TYPE in bytes_to_send.tobytes( ) if found_block: break self.node1.on_bytes_sent(self.blockchain_fileno, len(bytes_to_send)) bytes_to_send = self.node1.get_bytes_to_send( self.blockchain_fileno) self.assertTrue(found_block) self.assertEqual(self.block.rawbytes(), bytes_to_send) def _build_txs_message(self, short_ids: List[int]): txs: List[TransactionInfo] = [] for short_id in short_ids: transaction = self.transactions_by_short_id[short_id] txs.append( TransactionInfo(transaction.tx_hash(), transaction.tx_val(), transaction.short_id())) return TxsMessage(txs) def test_recover_all_short_ids(self): for tx_message in self.transactions: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) get_txs_message = self._send_compressed_block_to_node_1() self.assertEqual(10, len(get_txs_message.get_short_ids())) txs_message = self._build_txs_message(get_txs_message.get_short_ids()) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) self._assert_sent_block_node_1() def test_recover_missing_tx_contents(self): for tx_message in self.transactions_with_no_content[:3]: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) for tx_message in self.transactions[3:]: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) get_txs_message = self._send_compressed_block_to_node_1() self.assertEqual(10, len(get_txs_message.get_short_ids())) txs_message = self._build_txs_message(get_txs_message.get_short_ids()) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) self._assert_sent_block_node_1() def test_recover_multiple_iterations(self): for tx_message in self.transactions: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) get_txs_message = self._send_compressed_block_to_node_1() txs_message = self._build_txs_message( get_txs_message.get_short_ids()[:5]) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() time.time = MagicMock( return_value=time.time() + gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[0]) self.node1.alarm_queue.fire_alarms() get_txs_bytes_2 = helpers.get_queued_node_bytes( self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE) get_txs_message_2 = GetTxsMessage(buf=get_txs_bytes_2.tobytes()) self.assertEqual(5, len(get_txs_message_2.get_short_ids())) txs_message = self._build_txs_message( get_txs_message_2.get_short_ids()[:-1]) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) # retry again in a longer interval bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() time.time = MagicMock( return_value=time.time() + gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[0]) self.node1.alarm_queue.fire_alarms() bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() time.time = MagicMock( return_value=time.time() + gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[1]) self.node1.alarm_queue.fire_alarms() get_txs_bytes_3 = helpers.get_queued_node_bytes( self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE) get_txs_message_3 = GetTxsMessage(buf=get_txs_bytes_3.tobytes()) self.assertEqual(1, len(get_txs_message_3.get_short_ids())) txs_message = self._build_txs_message( get_txs_message_3.get_short_ids()) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) self._assert_sent_block_node_1() def test_recover_give_up(self): gateway_constants.BLOCK_RECOVERY_MAX_RETRY_ATTEMPTS = 3 for tx_message in self.transactions: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) self._send_compressed_block_to_node_1() txs_message = self._build_txs_message([]) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) # retry, first attempt bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() time.time = MagicMock( return_value=time.time() + gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[0]) self.node1.alarm_queue.fire_alarms() get_txs_bytes_2 = helpers.get_queued_node_bytes( self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE) get_txs_message_2 = GetTxsMessage(buf=get_txs_bytes_2.tobytes()) self.assertEqual(10, len(get_txs_message_2.get_short_ids())) txs_message = self._build_txs_message([]) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) # retry, attempt 2 bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() time.time = MagicMock( return_value=time.time() + gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[1]) self.node1.alarm_queue.fire_alarms() get_txs_bytes_3 = helpers.get_queued_node_bytes( self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE) get_txs_message_3 = GetTxsMessage(buf=get_txs_bytes_3.tobytes()) self.assertEqual(10, len(get_txs_message_3.get_short_ids())) txs_message = self._build_txs_message([]) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) # retry, attempt 3 bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() time.time = MagicMock( return_value=time.time() + gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[2]) self.node1.alarm_queue.fire_alarms() get_txs_bytes_4 = helpers.get_queued_node_bytes( self.node1, self.relay_fileno, GetTxsMessage.MESSAGE_TYPE) get_txs_message_4 = GetTxsMessage(buf=get_txs_bytes_4.tobytes()) self.assertEqual(10, len(get_txs_message_4.get_short_ids())) txs_message = self._build_txs_message([]) helpers.receive_node_message(self.node1, self.relay_fileno, txs_message.rawbytes()) # retry given up bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() time.time = MagicMock( return_value=time.time() + gateway_constants.BLOCK_RECOVERY_RECOVERY_INTERVAL_S[3] * 20) self.node1.alarm_queue.fire_alarms() # no bytes, even after timeout bytes_to_send = self.node1.get_bytes_to_send(self.relay_fileno) self.assertEqual(0, len(bytes_to_send)) self._assert_no_block_node_1() def test_recover_from_single_tx_short_id(self): for tx_message in self.transactions_with_short_ids[:-1]: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) _get_txs_message = self._send_compressed_block_to_node_1() helpers.receive_node_message( self.node1, self.relay_fileno, self.transactions_with_short_ids[-1].rawbytes()) self._assert_sent_block_node_1() def test_recover_from_single_tx_short_id_no_content(self): for tx_message in self.transactions_with_short_ids[:-1]: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) helpers.receive_node_message(self.node1, self.relay_fileno, self.transactions[-1].rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) _get_txs_message = self._send_compressed_block_to_node_1() helpers.receive_node_message( self.node1, self.relay_fileno, self.transactions_with_short_ids[-1].rawbytes()) self._assert_sent_block_node_1() def test_recover_from_single_tx_val(self): for tx_message in self.transactions_with_short_ids[:-1]: helpers.receive_node_message(self.node1, self.relay_fileno, tx_message.rawbytes()) helpers.receive_node_message( self.node1, self.relay_fileno, self.transactions_with_no_content[-1].rawbytes()) helpers.clear_node_buffer(self.node1, self.blockchain_fileno) _get_txs_message = self._send_compressed_block_to_node_1() helpers.receive_node_message( self.node1, self.relay_fileno, self.transactions_with_short_ids[-1].rawbytes()) self._assert_sent_block_node_1()
def get_recovered_compact_block(): buf = bytearray(convert.hex_to_bytes(FULL_BLOCK_BYTES_HEX)) parsed_block = BlockBtcMessage(buf=buf) return parsed_block