def setUp(self): self.node = MockGatewayNode(gateway_helpers.get_gateway_opts( 8000, include_default_btc_args=True, compact_block_min_tx_count=5 )) self.node.block_processing_service = MagicMock() self.connection = BtcNodeConnection( MockSocketConnection(node=self.node, ip_address=LOCALHOST, port=123), self.node ) self.connection.node = self.node self.connection.peer_ip = LOCALHOST self.connection.peer_port = 8001 self.connection.network_num = 2 self.sut = BtcNodeConnectionProtocol(self.connection) full_block_msg = BlockBtcMessage( buf=bytearray(convert.hex_to_bytes(self.FULL_BLOCK_BYTES_HEX)) ) if self.node.opts.use_extensions: transaction_service = ExtensionTransactionService(self.node, 0) else: transaction_service = TransactionService(self.node, 0) short_id = 1 for tx in full_block_msg.txns(): tx_hash = btc_common_utils.get_txid(tx) transaction_service.set_transaction_contents(tx_hash, tx) transaction_service.assign_short_id(tx_hash, short_id) short_id += 1 self.sut.connection.node._tx_service = transaction_service
def test_compact_block_partial_compression(self): compact_block = get_sample_compact_block() recovered_block = get_recovered_compact_block() recovered_transactions = [] index = 0 for idx, tx in enumerate(recovered_block.txns()): if index % 2 == 0: tx_hash = btc_common_utils.get_txid(tx) self.tx_service.assign_short_id(tx_hash, idx + 1) self.tx_service.set_transaction_contents(tx_hash, tx) else: recovered_transactions.append(tx) index += 1 result = self.btc_message_converter.compact_block_to_bx_block( compact_block, self.tx_service ) self.assertFalse(result.success) self.assertIsNone(result.bx_block) self.assertEqual(len(recovered_transactions), len(result.missing_indices)) for missing_index in result.missing_indices: self.assertNotEqual(0, missing_index % 2) for recovered_transaction in recovered_transactions: result.recovered_transactions.append(recovered_transaction) result = self.btc_message_converter.recovered_compact_block_to_bx_block(result) self.assertTrue(result.success) ref_block, _, _, _ = self.btc_message_converter.bx_block_to_block( result.bx_block, self.tx_service ) self.assertEqual(recovered_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes())
def block_to_bx_block( self, block_msg, tx_service ) -> Tuple[memoryview, BlockInfo]: """ Compresses a Bitcoin block's transactions and packs it into a bloXroute block. """ compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] header = block_msg.header() size += len(header) buf.append(header) for tx in block_msg.txns(): tx_hash = btc_common_utils.get_txid(tx) short_id = tx_service.get_short_id(tx_hash) if short_id == constants.NULL_TX_SID: buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off prev_block_hash = convert.bytes_to_hex(block_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) original_size = len(block_msg.rawbytes()) block_info = BlockInfo( block_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, block_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100 ) return memoryview(block), block_info
def test_non_segwit_tx_hash(self): non_seg_tx = "0100000002d8c8df6a6fdd2addaf589a83d860f18b44872d13ee6ec3526b2b470d42a96d4d000000008b483045022100b31557e47191936cb14e013fb421b1860b5e4fd5d2bc5ec1938f4ffb1651dc8902202661c2920771fd29dd91cd4100cefb971269836da4914d970d333861819265ba014104c54f8ea9507f31a05ae325616e3024bd9878cb0a5dff780444002d731577be4e2e69c663ff2da922902a4454841aa1754c1b6292ad7d317150308d8cce0ad7abffffffff2ab3fa4f68a512266134085d3260b94d3b6cfd351450cff021c045a69ba120b2000000008b4830450220230110bc99ef311f1f8bda9d0d968bfe5dfa4af171adbef9ef71678d658823bf022100f956d4fcfa0995a578d84e7e913f9bb1cf5b5be1440bcede07bce9cd5b38115d014104c6ec27cffce0823c3fecb162dbd576c88dd7cda0b7b32b0961188a392b488c94ca174d833ee6a9b71c0996620ae71e799fc7c77901db147fa7d97732e49c8226ffffffff02c0175302000000001976a914a3d89c53bb956f08917b44d113c6b2bcbe0c29b788acc01c3d09000000001976a91408338e1d5e26db3fce21b011795b1c3c8a5a5d0788ac00000000" non_seg_tx_bytes = convert.hex_to_bytes(non_seg_tx) self.assertFalse(btc_common_utils.is_segwit(non_seg_tx_bytes)) self.assertEqual( convert.bytes_to_hex( btc_common_utils.get_txid(non_seg_tx_bytes).binary), "9021b49d445c719106c95d561b9c3fac7bcb3650db67684a9226cd7fa1e1c1a0")
def test_segwit_tx_hash(self): seg_tx = "010000000001024668fcfeba861f7f1bf4d386f15cc6923bd8425e0214686671775359d17a51d50100000000ffffffffdc5530f864de2fac86246426094d7b8586a452d6bd8c209bb891646afb3548770000000000ffffffff0220040000000000001600147771a1cab96e36344b1693d3d9f29180ca900482f5c40100000000001976a91483121cc1ea476c25d91191ff735a5e90518c732788ac02473044022006db5e6aa36dafb5d89a8522675d304a228d39ede1450aaab04f84b1fb57db2902203efb537cca9738c599d95d5c0ddcec6ebd11c6001541a89a468246318e0bd6fe012102d6b8b2ba44eb621ac9537ed7e11553bb02060abca88a9e6faf7697df5ac6d30c02483045022100b04869e06930db5d4e8e4d453d9aed1097a8dae57eef0274ebdc99a106796335022037a6b744900b9b6392448c961e8d793367a0caf675b9ca80349c593e505d8e9d0121034ef9635ae7cd714b2cf8af7e72f23b8b07c7f75d75df95da8d682ae17459091b00000000" seg_tx_bytes = convert.hex_to_bytes(seg_tx) self.assertTrue(btc_common_utils.is_segwit(seg_tx_bytes)) self.assertEqual( convert.bytes_to_hex( btc_common_utils.get_txid(seg_tx_bytes).binary), "d9a057f11a21cf8afd32278e23fd2290660f05a3ffb582466eb5a1a5ece4ce85")
def _populate_transaction_services(self, block): if len(block.txns()) > 0: first_transaction = block.txns()[0] first_transaction_hash = btc_common_utils.get_txid(first_transaction) self.node1.get_tx_service().assign_short_id(first_transaction_hash, 1) self.node1.get_tx_service().set_transaction_contents(first_transaction_hash, first_transaction) self.node2.get_tx_service().assign_short_id(first_transaction_hash, 1) self.node2.get_tx_service().set_transaction_contents(first_transaction_hash, first_transaction)
def tx_hash(self) -> BtcObjectHash: """ Actually gets the txid, which is the same as the hash for non segwit transactions :return: BtcObjectHash """ if self._tx_hash is None: self._tx_hash = btc_common_utils.get_txid(self.payload()) # pyre-fixme[7]: Expected `BtcObjectHash` but got `None`. return self._tx_hash
def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: """ Compresses a Bitcoin block's transactions and packs it into a bloXroute block. """ compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] original_size = len(block_msg.rawbytes()) ignored_sids = [] header = block_msg.header() size += len(header) buf.append(header) max_timestamp_for_compression = time.time() - min_tx_age_seconds for tx in block_msg.txns(): tx_hash = btc_common_utils.get_txid(tx) transaction_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(transaction_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time( short_id) if short_id == constants.NULL_TX_SID or \ not enable_block_compression or \ short_id_assign_time > max_timestamp_for_compression: if short_id != constants.NULL_TX_SIDS: ignored_sids.append(short_id) buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 block = finalize_block_bytes(buf, size, short_ids) prev_block_hash = convert.bytes_to_hex( block_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo(block_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, block_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100, ignored_sids) return memoryview(block), block_info
def msg_tx(self, msg): if msg.tx_val() != TxMessage.EMPTY_TX_VAL: hash_val = btc_common_utils.get_txid(msg.tx_val()) if hash_val != msg.tx_hash(): self.log_error(log_messages.MALFORMED_TX_FROM_RELAY, hash_val, msg.tx_hash()) return super(BtcRelayConnection, self).msg_tx(msg)
def btc_bdn_tx_to_bx_tx( raw_tx: Union[bytes, bytearray, memoryview], network_num: int, transaction_flag: Optional[TransactionFlag] = None) -> TxMessage: if isinstance(raw_tx, bytes): raw_tx = bytearray(raw_tx) try: tx_hash = btc_common_utils.get_txid(raw_tx) except IndexError: raise ValueError("Invalid raw transaction provided.") return TxMessage(message_hash=tx_hash, network_num=network_num, tx_val=raw_tx, transaction_flag=transaction_flag)
def test_full_compression(self): parsed_block = get_sample_block() transactions = parsed_block.txns()[:] random.shuffle(transactions) for short_id, txn in enumerate(transactions): bx_tx_hash = btc_common_utils.get_txid(txn) self.tx_service.assign_short_id(bx_tx_hash, short_id + 1) self.tx_service.set_transaction_contents(bx_tx_hash, txn) bx_block, block_info = self.btc_message_converter.block_to_bx_block(parsed_block, self.tx_service, True, 0) ref_block, ref_lock_info, unknown_tx_sids, unknown_tx_hashes = self.btc_message_converter.bx_block_to_block( bx_block, self.tx_service ) self.assertEqual(len(block_info.short_ids), block_info.txn_count, "all txs were compressed") self.assertEqual(len(unknown_tx_hashes), 0) self.assertEqual(len(unknown_tx_sids), 0) self.assertEqual( parsed_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes() )
def test_partial_compression(self): parsed_block = get_sample_block() transactions_short = parsed_block.txns()[:] random.shuffle(transactions_short) transactions_short = transactions_short[:int(len(transactions_short) * 0.9)] for short_id, txn in enumerate(transactions_short): bx_tx_hash = btc_common_utils.get_txid(txn) self.tx_service.assign_short_id(bx_tx_hash, short_id + 1) self.tx_service.set_transaction_contents(bx_tx_hash, txn) bx_block, block_info = self.btc_message_converter.block_to_bx_block(parsed_block, self.tx_service, True, 0) ref_block, _, unknown_tx_sids, unknown_tx_hashes = self.btc_message_converter.bx_block_to_block( bx_block, self.tx_service ) self.assertEqual(len(unknown_tx_hashes), 0) self.assertEqual(len(unknown_tx_sids), 0) self.assertEqual( parsed_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes() )
def init(self, use_extensions: bool): opts = Namespace() opts.use_extensions = use_extensions opts.import_extensions = use_extensions opts.tx_mem_pool_bucket_size = DEFAULT_TX_MEM_POOL_BUCKET_SIZE btc_message_converter = converter_factory.create_btc_message_converter(self.MAGIC, opts=opts) if use_extensions: helpers.set_extensions_parallelism() tx_service = ExtensionTransactionService(MockNode( gateway_helpers.get_gateway_opts(8999)), 0) else: tx_service = TransactionService(MockNode( gateway_helpers.get_gateway_opts(8999)), 0) if self.txns: for idx, txn in enumerate(self.txns): sha = btc_common_utils.get_txid(txn) if idx % 2 == 0: tx_service.assign_short_id(sha, self.short_ids[int(idx/2)]) tx_service.set_transaction_contents(sha, txn) return tx_service, btc_message_converter
def test_segwit_partial_compression(self): parsed_block = get_segwit_block() transactions_short = parsed_block.txns()[:] transactions_short = transactions_short[:int(len(transactions_short) * 0.9)] random.shuffle(transactions_short) for short_id, txn in enumerate(transactions_short): bx_tx_hash = btc_common_utils.get_txid(txn) self.tx_service.assign_short_id(bx_tx_hash, short_id + 1) self.tx_service.set_transaction_contents(bx_tx_hash, txn) bx_block, block_info = self.btc_message_converter.block_to_bx_block(parsed_block, self.tx_service, True, 0) self.assertEqual(len(transactions_short), len(block_info.short_ids), "not all txs were compressed") self.assertEqual(int(block_info.txn_count * 0.9), len(block_info.short_ids), "not all txs were compressed") ref_block, _, _, _ = self.btc_message_converter.bx_block_to_block( bx_block, self.tx_service ) seg_bytes = parsed_block.rawbytes().tobytes() ref_bytes = ref_block.rawbytes().tobytes() self.assertEqual( seg_bytes, ref_bytes )