def _test_block_cleanup(self):
block_msg = self._get_sample_block(self._get_file_path())
transactions = block_msg.txns()[:]
block_hash = typing.cast(BtcObjectHash, block_msg.block_hash())
random.shuffle(transactions)
short_len = int(len(transactions) * 0.9)
transactions_short = transactions[:short_len]
unknown_transactions = transactions[short_len:]
transaction_hashes = []
for idx, tx in enumerate(transactions_short):
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=btc_constants.BTC_SHA_HASH_LEN)
transaction_hashes.append(tx_hash)
self.transaction_service.set_transaction_contents(tx_hash, tx)
self.transaction_service.assign_short_id(tx_hash, idx + 1)
for idx, tx in enumerate(unknown_transactions):
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=btc_constants.BTC_SHA_HASH_LEN)
transaction_hashes.append(tx_hash)
if idx % 2 == 0:
self.transaction_service.set_transaction_contents(tx_hash, tx)
self.cleanup_service._block_hash_marked_for_cleanup.add(block_hash)
self.cleanup_service.clean_block_transactions(block_msg,
self.transaction_service)
self.assertEqual(0, self.transaction_service._total_tx_contents_size)
for tx_hash in transaction_hashes:
self.assertFalse(
self.transaction_service.has_transaction_contents(tx_hash))
def setUp(self):
opts = gateway_helpers.get_gateway_opts(8000,
include_default_btc_args=True)
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.node = MockGatewayNode(opts)
self.node.neutrality_service = MagicMock()
self.connection = BtcNodeConnection(
MockSocketConnection(1,
node=self.node,
ip_address=LOCALHOST,
port=123), self.node)
gateway_helpers.add_blockchain_peer(self.node, self.connection)
self.tx_hash = BtcObjectHash(buf=helpers.generate_bytearray(32),
length=BTC_SHA_HASH_LEN)
self.block_hash = BtcObjectHash(buf=helpers.generate_bytearray(32),
length=BTC_SHA_HASH_LEN)
self.sut = BtcNodeConnectionProtocol(self.connection)
while self.connection.outputbuf.length > 0:
initial_bytes = self.connection.get_bytes_to_send()
self.connection.advance_sent_bytes(len(initial_bytes))
def pack_block_header(buffer: bytearray, version: int,
prev_block: BtcObjectHash, merkle_root: BtcObjectHash,
timestamp: int, bits: int, block_nonce: int) -> int:
"""
Packs Bitcoin block header into beginning of buffer
:param buffer: buffer
:param version: version
:param prev_block: previous block hash
:param merkle_root: merkle tree root hash
:param timestamp: timestamp
:param bits: bits
:param block_nonce: block nonce
:return: length of header
"""
off = BTC_HDR_COMMON_OFF
struct.pack_into("<I", buffer, off, version)
off += UL_INT_SIZE_IN_BYTES
buffer[off:off + BTC_SHA_HASH_LEN] = prev_block.get_little_endian()
off += BTC_SHA_HASH_LEN
buffer[off:off + BTC_SHA_HASH_LEN] = merkle_root.get_little_endian()
off += BTC_SHA_HASH_LEN
struct.pack_into("<III", buffer, off, timestamp, bits, block_nonce)
off += 3 * UL_INT_SIZE_IN_BYTES
return off
def _test_mark_blocks_and_request_cleanup(self):
marked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
prev_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
tracked_blocks = []
self.cleanup_service.on_new_block_received(marked_block, prev_block)
self.transaction_service.track_seen_short_ids(marked_block, [])
for _ in range(self.block_confirmations_count - 1):
tracked_block = BtcObjectHash(binary=helpers.generate_bytearray(
btc_constants.BTC_SHA_HASH_LEN))
self.transaction_service.track_seen_short_ids(tracked_block, [])
tracked_blocks.append(tracked_block)
unmarked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
self.assertIsNone(self.cleanup_service.last_confirmed_block)
self.cleanup_service.mark_blocks_and_request_cleanup(
[marked_block, *tracked_blocks])
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
self.assertTrue(
self.cleanup_service.is_marked_for_cleanup(marked_block))
self.assertFalse(
self.cleanup_service.is_marked_for_cleanup(unmarked_block))
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
msg = self.node.send_to_node_messages.pop(-1)
self.assertEqual(1, msg.count())
self.assertEqual((InventoryType.MSG_BLOCK, marked_block),
next(iter(msg)))
def setUp(self):
super().setUp()
self.node1.alarm_queue = AlarmQueue()
self.node2.alarm_queue = AlarmQueue()
self.network_num = 1
self.magic = 12345
self.version = 23456
self.prev_block_hash = bytearray(crypto.double_sha256(b"123"))
self.prev_block = BtcObjectHash(self.prev_block_hash,
length=crypto.SHA256_HASH_LEN)
self.merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
self.merkle_root = BtcObjectHash(self.merkle_root_hash,
length=crypto.SHA256_HASH_LEN)
self.bits = 2
self.nonce = 3
opts = self.gateway_1_opts()
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.btc_message_converter = btc_message_converter_factory.create_btc_message_converter(
self.magic, opts)
self.btc_transactions = [
TxBtcMessage(self.magic, self.version, [], [], i)
for i in range(self.TRANSACTIONS_COUNT)
]
self.btc_transactions_for_block = [
tx_btc_message.rawbytes()[btc_constants.BTC_HDR_COMMON_OFF:]
for tx_btc_message in self.btc_transactions
]
self.transactions = [
self.btc_message_converter.tx_to_bx_txs(tx_btc_message,
self.network_num)[0][0]
for tx_btc_message in self.btc_transactions
]
self.transactions_with_short_ids = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1, tx_message.tx_val())
for i, tx_message in enumerate(self.transactions)
]
self.transactions_with_no_content = [
TxMessage(tx_message.tx_hash(), tx_message.network_num(), "",
i + 1) for i, tx_message in enumerate(self.transactions)
]
self.transactions_by_short_id = {
tx_message.short_id(): tx_message
for tx_message in self.transactions_with_short_ids
}
self.block = BlockBtcMessage(self.magic, self.version,
self.prev_block, self.merkle_root,
int(time.time()), self.bits, self.nonce,
self.btc_transactions_for_block)
def __init__(self, magic: int = None, block_hash: BtcObjectHash = None, transactions: List[memoryview] = None,
# pyre-fixme[9]: buf has type `memoryview`; used as `None`.
buf: memoryview = None):
if buf is None:
total_tx_size = sum(len(tx) for tx in transactions)
# pyre-fixme[9]: buf has type `memoryview`; used as `bytearray`.
buf = bytearray(BTC_HDR_COMMON_OFF + BTC_SHA_HASH_LEN + BTC_VARINT_MIN_SIZE + total_tx_size)
off = BTC_HDR_COMMON_OFF
buf[off:off + BTC_SHA_HASH_LEN] = block_hash.get_big_endian()
off += BTC_SHA_HASH_LEN
off += pack_int_to_btc_varint(len(transactions), buf, off)
for tx in transactions:
buf[off:off + len(tx)] = tx
off += len(tx)
self.buf = buf
super(BlockTransactionsBtcMessage, self).__init__(magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._block_hash = None
# pyre-fixme[8]: Attribute has type `List[int]`; used as `None`.
self._transactions: List[int] = None
def block_to_bx_block(self, block_msg, tx_service) -> Tuple[memoryview, BlockInfo]:
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
self._default_block_size = max(self._default_block_size, len(block_msg.buf))
tsk = self.compression_tasks.borrow_task()
tsk.init(tpe.InputBytes(block_msg.buf), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.compression_tasks.return_task(tsk)
raise message_conversion_error.btc_block_compression_error(block_msg.block_hash(), e)
bx_block = tsk.bx_block()
block = memoryview(bx_block)
compressed_size = len(block)
original_size = len(block_msg.rawbytes())
block_hash = BtcObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string())
)
block_info = BlockInfo(
block_hash,
tsk.short_ids(),
compress_start_datetime,
datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tsk.txn_count(),
tsk.compressed_block_hash().hex_string(),
tsk.prev_block_hash().hex_string(),
original_size,
compressed_size,
100 - float(compressed_size) / original_size * 100
)
self.compression_tasks.return_task(tsk)
return block, block_info
def parse_bx_block_header(
bx_block: memoryview,
block_pieces: Deque[Union[bytearray, memoryview]]) -> BlockHeaderInfo:
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block)
short_ids, short_ids_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block, block_offsets.short_id_offset)
# Compute block header hash
block_header_size = \
block_offsets.block_begin_offset + \
btc_constants.BTC_HDR_COMMON_OFF + \
btc_constants.BTC_BLOCK_HDR_SIZE
block_hash = BtcObjectHash(buf=crypto.bitcoin_hash(
bx_block[block_offsets.block_begin_offset +
btc_constants.BTC_HDR_COMMON_OFF:block_header_size]),
length=btc_constants.BTC_SHA_HASH_LEN)
offset = block_header_size
# Add header piece
txn_count, txn_count_size = btc_common_utils.btc_varint_to_int(
bx_block, block_header_size)
offset += txn_count_size
block_pieces.append(bx_block[block_offsets.block_begin_offset:offset])
return BlockHeaderInfo(block_offsets, short_ids, short_ids_len, block_hash,
offset, txn_count)
def block_hash(self) -> BtcObjectHash:
if self._block_hash is None:
self._block_hash = BtcObjectHash(buf=self.buf,
offset=BTC_HDR_COMMON_OFF,
length=BTC_SHA_HASH_LEN)
# pyre-fixme[7]: Expected `BtcObjectHash` but got `None`.
return self._block_hash
def block_hash(self):
if self._block_hash is None:
header = self._memoryview[:
BTC_BLOCK_HDR_SIZE] # remove the tx count at the end
raw_hash = crypto.bitcoin_hash(header)
self._hash_val = BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN)
return self._hash_val
def block_hash(self) -> BtcObjectHash:
if self._hash_val is None:
header = self._memoryview[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE]
raw_hash = crypto.bitcoin_hash(header)
self._hash_val = BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN)
# pyre-fixme[7]: Expected `BtcObjectHash` but got `None`.
return self._hash_val
def __iter__(self):
off = BTC_HDR_COMMON_OFF + 4 # For the version field.
b_count, size = btc_varint_to_int(self.buf, off)
off += size
for i in range(b_count):
yield BtcObjectHash(buf=self.buf,
offset=off,
length=BTC_SHA_HASH_LEN)
off += 32
def version(self):
if self._version is None:
off = BTC_HDR_COMMON_OFF
self._version = struct.unpack_from('<I', self.buf, off)[0]
off += 4
self._prev_block = BtcObjectHash(self.buf, off, 32)
off += 32
self._merkle_root = BtcObjectHash(self.buf, off, 32)
off += 32
self._timestamp, self._bits, self._nonce = struct.unpack_from(
'<III', self.buf, off)
off += 12
self._txn_count, size = btc_varint_to_int(self.buf, off)
off += size
self._tx_offset = off
self._header = self._memoryview[0:self._tx_offset]
return self._version
class BtcConnectionProtocolTest(AbstractTestCase):
HASH = BtcObjectHash(binary=crypto.double_sha256(b"123"))
def setUp(self):
opts = gateway_helpers.get_gateway_opts(8000,
include_default_btc_args=True)
if opts.use_extensions:
helpers.set_extensions_parallelism()
self.node = MockGatewayNode(opts)
self.node.block_processing_service = MagicMock()
self.connection = MagicMock()
gateway_helpers.add_blockchain_peer(self.node, self.connection)
self.connection.node = self.node
self.connection.peer_ip = LOCALHOST
self.connection.peer_port = 8001
self.connection.network_num = 2
self.connection.endpoint = IpEndpoint(self.connection.peer_ip,
self.connection.peer_port)
self.node.blockchain_peers.add(
BlockchainPeerInfo(self.connection.peer_ip,
self.connection.peer_port))
gateway_bdn_performance_stats_service.set_node(self.node)
self.sut = BtcBaseConnectionProtocol(self.connection)
def test_msg_block_success(self):
block_timestamp = int(
time.time()
) + 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval
txns = [
TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
message = BlockBtcMessage(0, 0, self.HASH, self.HASH, block_timestamp,
0, 0, txns)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_called_once(
)
def test_msg_block_too_old(self):
block_timestamp = int(
time.time()
) - 1 - self.node.opts.blockchain_ignore_block_interval_count * self.node.opts.blockchain_block_interval
txns = [
TxBtcMessage(0, 0, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
message = BlockBtcMessage(0, 0, self.HASH, self.HASH, 0,
block_timestamp, 0, txns)
self.sut.msg_block(message)
self.node.block_processing_service.queue_block_for_processing.assert_not_called(
)
def test_btc_block_to_bloxroute_block_and_back_sids_found(self):
prev_block_hash = bytearray(crypto.bitcoin_hash(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
timestamp = 1
bits = 2
nonce = 3
btc_block = BlockBtcMessage(
self.magic, self.version, prev_block, merkle_root, timestamp, bits, nonce, self.txns
)
block_hash = btc_block.block_hash()
bloxroute_block, block_info = self.btc_message_converter.block_to_bx_block(
btc_block, self.tx_service, True, 0
)
self.assertEqual(10, block_info.txn_count)
self.assertEqual("5a77d1e9612d350b3734f6282259b7ff0a3f87d62cfef5f35e91a5604c0490a3",
block_info.prev_block_hash)
self.assertEqual(self.short_ids, list(block_info.short_ids))
self.assertEqual(btc_block.block_hash(), block_info.block_hash)
# TODO: if we convert bloxroute block to a class, add some tests here
parsed_btc_block, block_info, _, _ = self.btc_message_converter.bx_block_to_block(
bloxroute_block,
self.tx_service)
self.assertIsNotNone(block_info)
self.assertEqual(parsed_btc_block.rawbytes().tobytes(), btc_block.rawbytes().tobytes())
self.assertEqual(self.version, parsed_btc_block.version())
self.assertEqual(self.magic, parsed_btc_block.magic())
self.assertEqual(prev_block_hash, parsed_btc_block.prev_block_hash().get_little_endian())
self.assertEqual(merkle_root_hash, parsed_btc_block.merkle_root().get_little_endian())
self.assertEqual(timestamp, parsed_btc_block.timestamp())
self.assertEqual(bits, parsed_btc_block.bits())
self.assertEqual(nonce, parsed_btc_block.nonce())
self.assertEqual(len(self.txns), parsed_btc_block.txn_count())
self.assertEqual(btc_block.checksum(), parsed_btc_block.checksum())
self.assertEqual(block_hash, parsed_btc_block.block_hash())
self.assertEqual(block_hash.binary, block_info.block_hash.binary)
self.assertEqual(list(block_info.short_ids), self.short_ids)
def clean_block_transactions(
self, block_msg: BlockBtcMessage,
transaction_service: TransactionService) -> None:
block_short_ids = []
block_unknown_tx_hashes = []
start_time = time.time()
short_ids_count = 0
unknown_tx_hashes_count = 0
transactions_processed = 0
tx_hash_to_contents_len_before_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_before_cleanup = transaction_service.get_short_id_count(
)
for tx in block_msg.txns():
tx_hash = BtcObjectHash(buf=crypto.double_sha256(tx),
length=BTC_SHA_HASH_LEN)
short_ids = transaction_service.remove_transaction_by_tx_hash(
tx_hash, force=True)
if short_ids is None:
unknown_tx_hashes_count += 1
block_unknown_tx_hashes.append(tx_hash)
else:
short_ids_count += len(short_ids)
block_short_ids.extend(short_ids)
transactions_processed += 1
block_hash = block_msg.block_hash()
transaction_service.on_block_cleaned_up(block_hash)
end_time = time.time()
duration = end_time - start_time
tx_hash_to_contents_len_after_cleanup = transaction_service.get_tx_hash_to_contents_len(
)
short_id_count_after_cleanup = transaction_service.get_short_id_count()
logger.debug(
"Finished cleaning up block {}. Processed {} hashes, {} of which were unknown, and cleaned up {} "
"short ids. Took {:.3f}s.", block_hash, transactions_processed,
unknown_tx_hashes_count, short_ids_count, duration)
transaction_service.log_block_transaction_cleanup_stats(
block_hash, block_msg.txn_count(),
tx_hash_to_contents_len_before_cleanup,
tx_hash_to_contents_len_after_cleanup,
short_id_count_before_cleanup, short_id_count_after_cleanup)
self._block_hash_marked_for_cleanup.discard(block_hash)
self.node.post_block_cleanup_tasks(
block_hash=block_hash,
short_ids=block_short_ids,
unknown_tx_hashes=block_unknown_tx_hashes)
def test_get_data_only_new_data(self):
seen_block_hash = BtcObjectHash(
buf=helpers.generate_bytearray(BTC_SHA_HASH_LEN),
length=BTC_SHA_HASH_LEN)
not_seen_block_hash = BtcObjectHash(
buf=helpers.generate_bytearray(BTC_SHA_HASH_LEN),
length=BTC_SHA_HASH_LEN)
self.node.blocks_seen.add(seen_block_hash)
inv_message = InvBtcMessage(
magic=123,
inv_vects=[(InventoryType.MSG_TX, seen_block_hash),
(InventoryType.MSG_BLOCK, not_seen_block_hash),
(InventoryType.MSG_BLOCK, seen_block_hash)])
self.sut.msg_inv(inv_message)
get_data_msg_bytes = self.sut.connection.get_bytes_to_send()
get_data_msg = GetDataBtcMessage(buf=get_data_msg_bytes)
self.assertEqual(2, get_data_msg.count())
self.assertIn((InventoryType.MSG_TX, seen_block_hash), get_data_msg)
self.assertIn((InventoryType.MSG_BLOCK, not_seen_block_hash),
get_data_msg)
def version(self):
if self._version is None:
off = 0
self._version = struct.unpack_from('<I', self.buf, off)[0]
off += 4
self._prev_block = BtcObjectHash(self.buf, off, 32)
off += 32
self._merkle_root = self._memoryview[off:off + 32]
off += 32
self._timestamp, self._bits, self._nonce = struct.unpack_from(
'<III', self.buf, off)
self._txn_count, size = btc_varint_to_int(self.buf, off)
return self._version
def btc_block(timestamp=None, real_block=None):
if real_block is not None:
return BlockBtcMessage(buf=bytearray(convert.hex_to_bytes(real_block)))
if timestamp is None:
timestamp = int(time.time())
magic = 12345
version = 23456
prev_block_hash = bytearray(crypto.double_sha256(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.double_sha256(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
bits = 2
nonce = 3
txns = [
TxBtcMessage(magic, version, [], [], i).rawbytes()[BTC_HDR_COMMON_OFF:]
for i in range(10)
]
return BlockBtcMessage(magic, version, prev_block, merkle_root, timestamp,
bits, nonce, txns)
def peek_block(input_buffer):
buf = input_buffer.peek_message(BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE)
is_here = False
if input_buffer.length < BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE:
return is_here, None, None
header = buf[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE]
raw_hash = crypto.bitcoin_hash(header)
payload_len = struct.unpack_from('<L', buf, 16)[0]
is_here = True
return is_here, BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN), payload_len
def get_txid(buffer: Union[memoryview, bytearray]) -> BtcObjectHash:
"""
Actually gets the txid, which is the same as the hash for non segwit transactions
:param buffer: the bytes of the transaction contents
:return: hash object
"""
flag_len = btc_common_constants.TX_SEGWIT_FLAG_LEN if is_segwit(buffer) else 0
txid = sha256(buffer[:btc_common_constants.TX_VERSION_LEN])
end = btc_common_constants.TX_VERSION_LEN + flag_len
io_size, _, _ = get_tx_io_count_and_size(buffer, end, tail=-1)
txid.update(buffer[end:end + io_size])
txid.update(buffer[-btc_common_constants.TX_LOCK_TIME_LEN:])
return BtcObjectHash(buf=sha256(txid.digest()).digest(), length=btc_common_constants.BTC_SHA_HASH_LEN)
def _test_mark_blocks_and_request_cleanup(self):
node_conn = MockConnection(
MockSocketConnection(0, self.node, ip_address=LOCALHOST,
port=9000), self.node)
self.node.get_any_active_blockchain_connection = MagicMock(
return_value=node_conn)
marked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
prev_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
tracked_blocks = []
self.cleanup_service.on_new_block_received(marked_block, prev_block)
self.transaction_service.track_seen_short_ids(marked_block, [])
for _ in range(self.block_confirmations_count - 1):
tracked_block = BtcObjectHash(binary=helpers.generate_bytearray(
btc_constants.BTC_SHA_HASH_LEN))
self.transaction_service.track_seen_short_ids(tracked_block, [])
tracked_blocks.append(tracked_block)
unmarked_block = BtcObjectHash(
binary=helpers.generate_bytearray(btc_constants.BTC_SHA_HASH_LEN))
self.assertIsNone(self.cleanup_service.last_confirmed_block)
self.cleanup_service.mark_blocks_and_request_cleanup(
[marked_block, *tracked_blocks])
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
self.assertTrue(
self.cleanup_service.is_marked_for_cleanup(marked_block))
self.assertFalse(
self.cleanup_service.is_marked_for_cleanup(unmarked_block))
self.assertEqual(marked_block,
self.cleanup_service.last_confirmed_block)
msg = node_conn.enqueued_messages[0]
self.assertEqual(1, msg.count())
self.assertEqual((InventoryType.MSG_BLOCK, marked_block),
next(iter(msg)))
def test_message_preview_success_all_gateway_types(self):
self.get_message_preview_successfully(
GatewayHelloMessage(123, 1, "127.0.0.1", 40000, 1),
GatewayMessageType.HELLO,
GatewayHelloMessage.PAYLOAD_LENGTH,
)
self.get_message_preview_successfully(
BlockReceivedMessage(self.HASH),
GatewayMessageType.BLOCK_RECEIVED,
BlockReceivedMessage.PAYLOAD_LENGTH,
)
self.get_message_preview_successfully(
BlockPropagationRequestMessage(self.BLOCK),
GatewayMessageType.BLOCK_PROPAGATION_REQUEST,
len(self.BLOCK) + constants.CONTROL_FLAGS_LEN,
)
self.get_message_preview_successfully(
BlockHoldingMessage(self.HASH, network_num=123),
BloxrouteMessageType.BLOCK_HOLDING,
BlockHoldingMessage.PAYLOAD_LENGTH,
)
self.get_message_preview_successfully(
ConfirmedTxMessage(self.HASH, self.TX_VAL),
GatewayMessageType.CONFIRMED_TX,
ConfirmedTxMessage.PAYLOAD_LENGTH + len(self.TX_VAL),
)
self.get_message_preview_successfully(
RequestTxStreamMessage(),
GatewayMessageType.REQUEST_TX_STREAM,
constants.CONTROL_FLAGS_LEN,
)
hash_val = BtcObjectHash(buf=crypto.double_sha256(b"123"),
length=crypto.SHA256_HASH_LEN)
blockchain_message = GetBlocksBtcMessage(12345, 23456, [hash_val],
hash_val).rawbytes()
self.get_message_preview_successfully(
BlockchainSyncRequestMessage(GetBlocksBtcMessage.MESSAGE_TYPE,
blockchain_message),
BlockchainSyncRequestMessage.MESSAGE_TYPE,
MSG_TYPE_LEN + len(blockchain_message),
)
self.get_message_preview_successfully(
BlockchainSyncResponseMessage(GetBlocksBtcMessage.MESSAGE_TYPE,
blockchain_message),
BlockchainSyncResponseMessage.MESSAGE_TYPE,
MSG_TYPE_LEN + len(blockchain_message),
)
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult:
decompress_start_datetime = datetime.utcnow()
decompress_start_timestamp = time.time()
tsk = self.decompression_tasks.borrow_task()
tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy)
try:
task_pool_proxy.run_task(tsk)
except tpe.AggregatedException as e:
self.decompression_tasks.return_task(tsk)
header_info = btc_normal_message_converter.parse_bx_block_header(bx_block_msg, deque())
raise message_conversion_error.btc_block_decompression_error(header_info.block_hash, e)
total_tx_count = tsk.tx_count()
unknown_tx_hashes = [Sha256Hash(bytearray(unknown_tx_hash.binary()))
for unknown_tx_hash in tsk.unknown_tx_hashes()]
unknown_tx_sids = tsk.unknown_tx_sids()
block_hash = BtcObjectHash(
binary=convert.hex_to_bytes(tsk.block_hash().hex_string())
)
if tsk.success():
btc_block_msg = BlockBtcMessage(buf=memoryview(tsk.block_message()))
logger.debug(
"Successfully parsed block broadcast message. {} transactions "
"in block {}",
total_tx_count,
block_hash
)
else:
btc_block_msg = None
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}",
block_hash,
len(unknown_tx_sids),
len(unknown_tx_hashes),
total_tx_count
)
block_info = get_block_info(
bx_block_msg,
block_hash,
tsk.short_ids(),
decompress_start_datetime,
decompress_start_timestamp,
total_tx_count,
btc_block_msg
)
self.decompression_tasks.return_task(tsk)
return BlockDecompressionResult(btc_block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)
def __iter__(self) -> Iterator[Tuple[int, BtcObjectHash]]:
off = btc_constants.BTC_HDR_COMMON_OFF
num_items, size = btc_common_utils.btc_varint_to_int(self.buf, off)
off += size
# pyre-fixme[16]: `InventoryBtcMessage` has no attribute `_num_items`.
self._num_items = num_items
# pyre-fixme[16]: `InventoryBtcMessage` has no attribute `_items`.
self._items = list()
for _ in range(num_items):
inv_type, = struct.unpack_from("<L", self.buf, off)
off += 4
yield (inv_type,
BtcObjectHash(buf=self.buf,
offset=off,
length=btc_constants.BTC_SHA_HASH_LEN))
off += 32
def test_compact_block_full_compression(self):
compact_block = get_sample_compact_block()
recovered_block = get_recovered_compact_block()
for short_id, txn in enumerate(recovered_block.txns()):
bx_tx_hash = BtcObjectHash(buf=crypto.double_sha256(txn),
length=BTC_SHA_HASH_LEN)
self.tx_service.set_transaction_contents(bx_tx_hash, txn)
self.tx_service.assign_short_id(bx_tx_hash, short_id + 1)
result = self.btc_message_converter.compact_block_to_bx_block(
compact_block, self.tx_service
)
self.assertTrue(result.success)
ref_block, _, _, _ = self.btc_message_converter.bx_block_to_block(
result.bx_block, self.tx_service
)
self.assertEqual(recovered_block.rawbytes().tobytes(), ref_block.rawbytes().tobytes())
def message(self):
if self._message is None:
off = BTC_HDR_COMMON_OFF
self._message = struct.unpack_from('%dp' % (len(self.buf) - off, ),
self.buf, off)[0]
off += len(self._message) + 1
self._ccode = struct.unpack_from('B', self.buf, off)[0]
off += 1
self._reason = struct.unpack_from('%dp' % (len(self.buf) - off, ),
self.buf, off)[0]
off += len(self._reason) + 1
self._data = self.buf[off:]
self._obj_hash = BtcObjectHash(
buf=self.buf,
offset=len(self.buf) - btc_constants.BTC_SHA_HASH_LEN,
length=btc_constants.BTC_SHA_HASH_LEN)
return self._message
class BlockchainSyncBtcTest(AbstractTestCase):
HASH = BtcObjectHash(binary=crypto.bitcoin_hash(b"hi"))
def setUp(self):
self.local_node_fileno = 1
self.remote_node_fileno = 2
self.gateway_node = spies.make_spy_node(BtcGatewayNode,
8000,
include_default_btc_args=True)
self.btc_node_connection = spies.make_spy_connection(
BtcNodeConnection, self.local_node_fileno, 8001, self.gateway_node)
self.btc_remote_node_connection = spies.make_spy_connection(
BtcRemoteConnection, self.remote_node_fileno, 8002,
self.gateway_node)
self.gateway_node.node_conn = self.btc_node_connection
self.gateway_node.remote_node_conn = self.btc_remote_node_connection
self.gateway_node.connection_pool.add(self.local_node_fileno,
LOCALHOST, 8001,
self.btc_node_connection)
self.gateway_node.connection_pool.add(self.remote_node_fileno,
LOCALHOST, 8002,
self.btc_remote_node_connection)
def test_block_headers_request(self):
sent_get_headers = GetHeadersBtcMessage(12345, 23456, [self.HASH],
self.HASH)
helpers.receive_node_message(self.gateway_node, self.local_node_fileno,
sent_get_headers.rawbytes())
self.btc_remote_node_connection.enqueue_msg.assert_called_once_with(
sent_get_headers)
response_headers = HeadersBtcMessage(12345, [])
helpers.receive_node_message(self.gateway_node,
self.remote_node_fileno,
response_headers.rawbytes())
self.btc_node_connection.enqueue_msg.assert_called_once_with(
response_headers)
def __init__(
self,
magic: int = None,
block_hash: BtcObjectHash = None,
indices: List[int] = None,
# pyre-fixme[9]: buf has type `memoryview`; used as `None`.
buf: memoryview = None):
if buf is None:
# pyre-fixme[9]: buf has type `memoryview`; used as `bytearray`.
buf = bytearray(BTC_HDR_COMMON_OFF + BTC_SHA_HASH_LEN +
BTC_VARINT_MIN_SIZE +
BTC_VARINT_MIN_SIZE * len(indices))
off = BTC_HDR_COMMON_OFF
buf[off:off + BTC_SHA_HASH_LEN] = block_hash.get_big_endian()
off += BTC_SHA_HASH_LEN
off += pack_int_to_btc_varint(len(indices), buf, off)
last_real_index = -1
for real_index in indices:
diff = real_index - last_real_index - 1
last_real_index = real_index
off += pack_int_to_btc_varint(diff, buf, off)
self.buf = buf
super(GetBlockTransactionsBtcMessage,
self).__init__(magic, self.MESSAGE_TYPE,
off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._block_hash = None
# pyre-fixme[8]: Attribute has type `List[int]`; used as `None`.
self._indices: List[int] = None
def get_object_hash(bytearr):
return BtcObjectHash(bytearr, 0, 32)