def setUp(self):
self.node = MockGatewayNode(gateway_helpers.get_gateway_opts(
8000,
include_default_btc_args=True,
compact_block_min_tx_count=5
))
self.node.block_processing_service = MagicMock()
self.connection = BtcNodeConnection(
MockSocketConnection(node=self.node, ip_address=LOCALHOST, port=123), self.node
)
self.connection.node = self.node
self.connection.peer_ip = LOCALHOST
self.connection.peer_port = 8001
self.connection.network_num = 2
self.sut = BtcNodeConnectionProtocol(self.connection)
full_block_msg = BlockBtcMessage(
buf=bytearray(convert.hex_to_bytes(self.FULL_BLOCK_BYTES_HEX))
)
if self.node.opts.use_extensions:
transaction_service = ExtensionTransactionService(self.node, 0)
else:
transaction_service = TransactionService(self.node, 0)
short_id = 1
for tx in full_block_msg.txns():
tx_hash = btc_common_utils.get_txid(tx)
transaction_service.set_transaction_contents(tx_hash, tx)
transaction_service.assign_short_id(tx_hash, short_id)
short_id += 1
self.sut.connection.node._tx_service = transaction_service
def test_compact_block_partial_compression(self):
compact_block = get_sample_compact_block()
recovered_block = get_recovered_compact_block()
recovered_transactions = []
index = 0
for idx, tx in enumerate(recovered_block.txns()):
if index % 2 == 0:
tx_hash = btc_common_utils.get_txid(tx)
self.tx_service.assign_short_id(tx_hash, idx + 1)
self.tx_service.set_transaction_contents(tx_hash, tx)
else:
recovered_transactions.append(tx)
index += 1
result = self.btc_message_converter.compact_block_to_bx_block(
compact_block, self.tx_service
)
self.assertFalse(result.success)
self.assertIsNone(result.bx_block)
self.assertEqual(len(recovered_transactions), len(result.missing_indices))
for missing_index in result.missing_indices:
self.assertNotEqual(0, missing_index % 2)
for recovered_transaction in recovered_transactions:
result.recovered_transactions.append(recovered_transaction)
result = self.btc_message_converter.recovered_compact_block_to_bx_block(result)
self.assertTrue(result.success)
ref_block, _, _, _ = self.btc_message_converter.bx_block_to_block(
result.bx_block, self.tx_service
)
self.assertEqual(recovered_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes())
def block_to_bx_block(
self, block_msg, tx_service
) -> Tuple[memoryview, BlockInfo]:
"""
Compresses a Bitcoin block's transactions and packs it into a bloXroute block.
"""
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
size = 0
buf = deque()
short_ids = []
header = block_msg.header()
size += len(header)
buf.append(header)
for tx in block_msg.txns():
tx_hash = btc_common_utils.get_txid(tx)
short_id = tx_service.get_short_id(tx_hash)
if short_id == constants.NULL_TX_SID:
buf.append(tx)
size += len(tx)
else:
short_ids.append(short_id)
buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY)
size += 1
serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids)
buf.append(serialized_short_ids)
size += constants.UL_ULL_SIZE_IN_BYTES
offset_buf = struct.pack("<Q", size)
buf.appendleft(offset_buf)
size += len(serialized_short_ids)
block = bytearray(size)
off = 0
for blob in buf:
next_off = off + len(blob)
block[off:next_off] = blob
off = next_off
prev_block_hash = convert.bytes_to_hex(block_msg.prev_block_hash().binary)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
original_size = len(block_msg.rawbytes())
block_info = BlockInfo(
block_msg.block_hash(),
short_ids,
compress_start_datetime,
datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
block_msg.txn_count(),
bx_block_hash,
prev_block_hash,
original_size,
size,
100 - float(size) / original_size * 100
)
return memoryview(block), block_info
def test_non_segwit_tx_hash(self):
non_seg_tx = "0100000002d8c8df6a6fdd2addaf589a83d860f18b44872d13ee6ec3526b2b470d42a96d4d000000008b483045022100b31557e47191936cb14e013fb421b1860b5e4fd5d2bc5ec1938f4ffb1651dc8902202661c2920771fd29dd91cd4100cefb971269836da4914d970d333861819265ba014104c54f8ea9507f31a05ae325616e3024bd9878cb0a5dff780444002d731577be4e2e69c663ff2da922902a4454841aa1754c1b6292ad7d317150308d8cce0ad7abffffffff2ab3fa4f68a512266134085d3260b94d3b6cfd351450cff021c045a69ba120b2000000008b4830450220230110bc99ef311f1f8bda9d0d968bfe5dfa4af171adbef9ef71678d658823bf022100f956d4fcfa0995a578d84e7e913f9bb1cf5b5be1440bcede07bce9cd5b38115d014104c6ec27cffce0823c3fecb162dbd576c88dd7cda0b7b32b0961188a392b488c94ca174d833ee6a9b71c0996620ae71e799fc7c77901db147fa7d97732e49c8226ffffffff02c0175302000000001976a914a3d89c53bb956f08917b44d113c6b2bcbe0c29b788acc01c3d09000000001976a91408338e1d5e26db3fce21b011795b1c3c8a5a5d0788ac00000000"
non_seg_tx_bytes = convert.hex_to_bytes(non_seg_tx)
self.assertFalse(btc_common_utils.is_segwit(non_seg_tx_bytes))
self.assertEqual(
convert.bytes_to_hex(
btc_common_utils.get_txid(non_seg_tx_bytes).binary),
"9021b49d445c719106c95d561b9c3fac7bcb3650db67684a9226cd7fa1e1c1a0")
def test_segwit_tx_hash(self):
seg_tx = "010000000001024668fcfeba861f7f1bf4d386f15cc6923bd8425e0214686671775359d17a51d50100000000ffffffffdc5530f864de2fac86246426094d7b8586a452d6bd8c209bb891646afb3548770000000000ffffffff0220040000000000001600147771a1cab96e36344b1693d3d9f29180ca900482f5c40100000000001976a91483121cc1ea476c25d91191ff735a5e90518c732788ac02473044022006db5e6aa36dafb5d89a8522675d304a228d39ede1450aaab04f84b1fb57db2902203efb537cca9738c599d95d5c0ddcec6ebd11c6001541a89a468246318e0bd6fe012102d6b8b2ba44eb621ac9537ed7e11553bb02060abca88a9e6faf7697df5ac6d30c02483045022100b04869e06930db5d4e8e4d453d9aed1097a8dae57eef0274ebdc99a106796335022037a6b744900b9b6392448c961e8d793367a0caf675b9ca80349c593e505d8e9d0121034ef9635ae7cd714b2cf8af7e72f23b8b07c7f75d75df95da8d682ae17459091b00000000"
seg_tx_bytes = convert.hex_to_bytes(seg_tx)
self.assertTrue(btc_common_utils.is_segwit(seg_tx_bytes))
self.assertEqual(
convert.bytes_to_hex(
btc_common_utils.get_txid(seg_tx_bytes).binary),
"d9a057f11a21cf8afd32278e23fd2290660f05a3ffb582466eb5a1a5ece4ce85")
def _populate_transaction_services(self, block):
if len(block.txns()) > 0:
first_transaction = block.txns()[0]
first_transaction_hash = btc_common_utils.get_txid(first_transaction)
self.node1.get_tx_service().assign_short_id(first_transaction_hash, 1)
self.node1.get_tx_service().set_transaction_contents(first_transaction_hash, first_transaction)
self.node2.get_tx_service().assign_short_id(first_transaction_hash, 1)
self.node2.get_tx_service().set_transaction_contents(first_transaction_hash, first_transaction)
def tx_hash(self) -> BtcObjectHash:
"""
Actually gets the txid, which is the same as the hash for non segwit transactions
:return: BtcObjectHash
"""
if self._tx_hash is None:
self._tx_hash = btc_common_utils.get_txid(self.payload())
# pyre-fixme[7]: Expected `BtcObjectHash` but got `None`.
return self._tx_hash
def block_to_bx_block(
self, block_msg, tx_service, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
"""
Compresses a Bitcoin block's transactions and packs it into a bloXroute block.
"""
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
size = 0
buf = deque()
short_ids = []
original_size = len(block_msg.rawbytes())
ignored_sids = []
header = block_msg.header()
size += len(header)
buf.append(header)
max_timestamp_for_compression = time.time() - min_tx_age_seconds
for tx in block_msg.txns():
tx_hash = btc_common_utils.get_txid(tx)
transaction_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(transaction_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(
short_id)
if short_id == constants.NULL_TX_SID or \
not enable_block_compression or \
short_id_assign_time > max_timestamp_for_compression:
if short_id != constants.NULL_TX_SIDS:
ignored_sids.append(short_id)
buf.append(tx)
size += len(tx)
else:
short_ids.append(short_id)
buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY)
size += 1
block = finalize_block_bytes(buf, size, short_ids)
prev_block_hash = convert.bytes_to_hex(
block_msg.prev_block_hash().binary)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(block_msg.block_hash(), short_ids,
compress_start_datetime, datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
block_msg.txn_count(), bx_block_hash,
prev_block_hash, original_size, size,
100 - float(size) / original_size * 100,
ignored_sids)
return memoryview(block), block_info
def msg_tx(self, msg):
if msg.tx_val() != TxMessage.EMPTY_TX_VAL:
hash_val = btc_common_utils.get_txid(msg.tx_val())
if hash_val != msg.tx_hash():
self.log_error(log_messages.MALFORMED_TX_FROM_RELAY, hash_val,
msg.tx_hash())
return
super(BtcRelayConnection, self).msg_tx(msg)
def btc_bdn_tx_to_bx_tx(
raw_tx: Union[bytes, bytearray, memoryview],
network_num: int,
transaction_flag: Optional[TransactionFlag] = None) -> TxMessage:
if isinstance(raw_tx, bytes):
raw_tx = bytearray(raw_tx)
try:
tx_hash = btc_common_utils.get_txid(raw_tx)
except IndexError:
raise ValueError("Invalid raw transaction provided.")
return TxMessage(message_hash=tx_hash,
network_num=network_num,
tx_val=raw_tx,
transaction_flag=transaction_flag)
def test_full_compression(self):
parsed_block = get_sample_block()
transactions = parsed_block.txns()[:]
random.shuffle(transactions)
for short_id, txn in enumerate(transactions):
bx_tx_hash = btc_common_utils.get_txid(txn)
self.tx_service.assign_short_id(bx_tx_hash, short_id + 1)
self.tx_service.set_transaction_contents(bx_tx_hash, txn)
bx_block, block_info = self.btc_message_converter.block_to_bx_block(parsed_block, self.tx_service, True, 0)
ref_block, ref_lock_info, unknown_tx_sids, unknown_tx_hashes = self.btc_message_converter.bx_block_to_block(
bx_block, self.tx_service
)
self.assertEqual(len(block_info.short_ids), block_info.txn_count, "all txs were compressed")
self.assertEqual(len(unknown_tx_hashes), 0)
self.assertEqual(len(unknown_tx_sids), 0)
self.assertEqual(
parsed_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes()
)
def test_partial_compression(self):
parsed_block = get_sample_block()
transactions_short = parsed_block.txns()[:]
random.shuffle(transactions_short)
transactions_short = transactions_short[:int(len(transactions_short) * 0.9)]
for short_id, txn in enumerate(transactions_short):
bx_tx_hash = btc_common_utils.get_txid(txn)
self.tx_service.assign_short_id(bx_tx_hash, short_id + 1)
self.tx_service.set_transaction_contents(bx_tx_hash, txn)
bx_block, block_info = self.btc_message_converter.block_to_bx_block(parsed_block, self.tx_service, True, 0)
ref_block, _, unknown_tx_sids, unknown_tx_hashes = self.btc_message_converter.bx_block_to_block(
bx_block, self.tx_service
)
self.assertEqual(len(unknown_tx_hashes), 0)
self.assertEqual(len(unknown_tx_sids), 0)
self.assertEqual(
parsed_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes()
)
def init(self, use_extensions: bool):
opts = Namespace()
opts.use_extensions = use_extensions
opts.import_extensions = use_extensions
opts.tx_mem_pool_bucket_size = DEFAULT_TX_MEM_POOL_BUCKET_SIZE
btc_message_converter = converter_factory.create_btc_message_converter(self.MAGIC, opts=opts)
if use_extensions:
helpers.set_extensions_parallelism()
tx_service = ExtensionTransactionService(MockNode(
gateway_helpers.get_gateway_opts(8999)), 0)
else:
tx_service = TransactionService(MockNode(
gateway_helpers.get_gateway_opts(8999)), 0)
if self.txns:
for idx, txn in enumerate(self.txns):
sha = btc_common_utils.get_txid(txn)
if idx % 2 == 0:
tx_service.assign_short_id(sha, self.short_ids[int(idx/2)])
tx_service.set_transaction_contents(sha, txn)
return tx_service, btc_message_converter
def test_segwit_partial_compression(self):
parsed_block = get_segwit_block()
transactions_short = parsed_block.txns()[:]
transactions_short = transactions_short[:int(len(transactions_short) * 0.9)]
random.shuffle(transactions_short)
for short_id, txn in enumerate(transactions_short):
bx_tx_hash = btc_common_utils.get_txid(txn)
self.tx_service.assign_short_id(bx_tx_hash, short_id + 1)
self.tx_service.set_transaction_contents(bx_tx_hash, txn)
bx_block, block_info = self.btc_message_converter.block_to_bx_block(parsed_block, self.tx_service, True, 0)
self.assertEqual(len(transactions_short), len(block_info.short_ids), "not all txs were compressed")
self.assertEqual(int(block_info.txn_count * 0.9), len(block_info.short_ids), "not all txs were compressed")
ref_block, _, _, _ = self.btc_message_converter.bx_block_to_block(
bx_block, self.tx_service
)
seg_bytes = parsed_block.rawbytes().tobytes()
ref_bytes = ref_block.rawbytes().tobytes()
self.assertEqual(
seg_bytes, ref_bytes
)
