def parse_bx_block_header(
bx_block: memoryview,
block_pieces: Deque[Union[bytearray, memoryview]]) -> BlockHeaderInfo:
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block)
short_ids, short_ids_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block, block_offsets.short_id_offset)
# Compute block header hash
block_header_size = \
block_offsets.block_begin_offset + \
btc_constants.BTC_HDR_COMMON_OFF + \
btc_constants.BTC_BLOCK_HDR_SIZE
block_hash = BtcObjectHash(buf=crypto.bitcoin_hash(
bx_block[block_offsets.block_begin_offset +
btc_constants.BTC_HDR_COMMON_OFF:block_header_size]),
length=btc_constants.BTC_SHA_HASH_LEN)
offset = block_header_size
# Add header piece
txn_count, txn_count_size = btc_common_utils.btc_varint_to_int(
bx_block, block_header_size)
offset += txn_count_size
block_pieces.append(bx_block[block_offsets.block_begin_offset:offset])
return BlockHeaderInfo(block_offsets, short_ids, short_ids_len, block_hash,
offset, txn_count)
def block_hash(self):
if self._block_hash is None:
header = self._memoryview[:
BTC_BLOCK_HDR_SIZE] # remove the tx count at the end
raw_hash = crypto.bitcoin_hash(header)
self._hash_val = BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN)
return self._hash_val
def block_hash(self) -> BtcObjectHash:
if self._hash_val is None:
header = self._memoryview[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE]
raw_hash = crypto.bitcoin_hash(header)
self._hash_val = BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN)
# pyre-fixme[7]: Expected `BtcObjectHash` but got `None`.
return self._hash_val
def test_ont_block_to_bloxroute_block_and_back_sids_found(self):
prev_block_hash = bytearray(crypto.bitcoin_hash(b"123"))
prev_block = OntObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234"))
merkle_root = OntObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
txns_root_hash = bytearray(crypto.bitcoin_hash(b"345"))
txns_root = OntObjectHash(txns_root_hash, length=SHA256_HASH_LEN)
block_root_hash = bytearray(crypto.bitcoin_hash(b"456"))
block_root = OntObjectHash(block_root_hash, length=SHA256_HASH_LEN)
consensus_payload = bytes(b'111')
next_bookkeeper = bytes(b'222')
bookkeepers = [bytes(33)] * 5
sig_data = [bytes(2)] * 3
txns = []
timestamp = 1
height = 2
consensus_data = 3
ont_block = BlockOntMessage(self.magic, self.version, prev_block,
txns_root, block_root, timestamp, height,
consensus_data, consensus_payload,
next_bookkeeper, bookkeepers, sig_data,
txns, merkle_root)
block_hash = ont_block.block_hash()
bloxroute_block, block_info = self.ont_message_converter.block_to_bx_block(
ont_block, self.tx_service, True, 0)
self.assertEqual(0, block_info.txn_count)
self.assertEqual(self.short_ids, list(block_info.short_ids))
self.assertEqual(ont_block.block_hash(), block_info.block_hash)
parsed_ont_block, block_info, _, _ = self.ont_message_converter.bx_block_to_block(
bloxroute_block, self.tx_service)
self.assertIsNotNone(block_info)
self.assertEqual(ont_block.rawbytes().tobytes(),
parsed_ont_block.rawbytes().tobytes())
self.assertEqual(self.magic, parsed_ont_block.magic())
self.assertEqual(
prev_block_hash,
parsed_ont_block.prev_block_hash().get_little_endian())
self.assertEqual(ont_block.checksum(), parsed_ont_block.checksum())
self.assertEqual(block_hash, parsed_ont_block.block_hash())
self.assertEqual(block_hash.binary, block_info.block_hash.binary)
self.assertEqual(timestamp, parsed_ont_block.timestamp())
def test_btc_block_to_bloxroute_block_and_back_sids_found(self):
prev_block_hash = bytearray(crypto.bitcoin_hash(b"123"))
prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN)
merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234"))
merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN)
timestamp = 1
bits = 2
nonce = 3
btc_block = BlockBtcMessage(
self.magic, self.version, prev_block, merkle_root, timestamp, bits, nonce, self.txns
)
block_hash = btc_block.block_hash()
bloxroute_block, block_info = self.btc_message_converter.block_to_bx_block(
btc_block, self.tx_service, True, 0
)
self.assertEqual(10, block_info.txn_count)
self.assertEqual("5a77d1e9612d350b3734f6282259b7ff0a3f87d62cfef5f35e91a5604c0490a3",
block_info.prev_block_hash)
self.assertEqual(self.short_ids, list(block_info.short_ids))
self.assertEqual(btc_block.block_hash(), block_info.block_hash)
# TODO: if we convert bloxroute block to a class, add some tests here
parsed_btc_block, block_info, _, _ = self.btc_message_converter.bx_block_to_block(
bloxroute_block,
self.tx_service)
self.assertIsNotNone(block_info)
self.assertEqual(parsed_btc_block.rawbytes().tobytes(), btc_block.rawbytes().tobytes())
self.assertEqual(self.version, parsed_btc_block.version())
self.assertEqual(self.magic, parsed_btc_block.magic())
self.assertEqual(prev_block_hash, parsed_btc_block.prev_block_hash().get_little_endian())
self.assertEqual(merkle_root_hash, parsed_btc_block.merkle_root().get_little_endian())
self.assertEqual(timestamp, parsed_btc_block.timestamp())
self.assertEqual(bits, parsed_btc_block.bits())
self.assertEqual(nonce, parsed_btc_block.nonce())
self.assertEqual(len(self.txns), parsed_btc_block.txn_count())
self.assertEqual(btc_block.checksum(), parsed_btc_block.checksum())
self.assertEqual(block_hash, parsed_btc_block.block_hash())
self.assertEqual(block_hash.binary, block_info.block_hash.binary)
self.assertEqual(list(block_info.short_ids), self.short_ids)
def validate_payload(cls, buf, unpacked_args):
command, _magic, checksum, payload_length = unpacked_args
if payload_length != len(buf) - cls.HEADER_LENGTH:
error_message = log_messages.PAYLOAD_LENGTH_MISMATCH.text.format(
payload_length, len(buf))
logger.error(error_message)
raise PayloadLenError(error_message)
ref_checksum = crypto.bitcoin_hash(
buf[cls.HEADER_LENGTH:cls.HEADER_LENGTH + payload_length])[0:4]
if checksum != ref_checksum:
error_message = log_messages.PACKET_CHECKSUM_MISMATCH.text.format(
checksum, ref_checksum, repr(buf))
logger.error(error_message)
raise ChecksumError(error_message, buf)
def peek_block(input_buffer):
buf = input_buffer.peek_message(BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE)
is_here = False
if input_buffer.length < BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE:
return is_here, None, None
header = buf[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF +
BTC_BLOCK_HDR_SIZE]
raw_hash = crypto.bitcoin_hash(header)
payload_len = struct.unpack_from('<L', buf, 16)[0]
is_here = True
return is_here, BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN), payload_len
def __init__(self, magic=None, command=None, payload_len=None, buf=None):
self.buf = buf
self._memoryview = memoryview(buf)
magic_num = magic if magic not in BTC_MAGIC_NUMBERS else BTC_MAGIC_NUMBERS[
magic]
checksum = crypto.bitcoin_hash(
self._memoryview[BTC_HDR_COMMON_OFF:payload_len +
BTC_HDR_COMMON_OFF])
off = 0
struct.pack_into("<L12sL", buf, off, magic_num, command, payload_len)
off += 20
buf[off:off + 4] = checksum[0:4]
self._magic = magic_num
self._command = command
self._payload_len = payload_len
self._payload = None
self._checksum = None
class BlockchainSyncBtcTest(AbstractTestCase):
HASH = BtcObjectHash(binary=crypto.bitcoin_hash(b"hi"))
def setUp(self):
self.local_node_fileno = 1
self.remote_node_fileno = 2
self.gateway_node = spies.make_spy_node(BtcGatewayNode,
8000,
include_default_btc_args=True)
self.btc_node_connection = spies.make_spy_connection(
BtcNodeConnection, self.local_node_fileno, 8001, self.gateway_node)
self.btc_remote_node_connection = spies.make_spy_connection(
BtcRemoteConnection, self.remote_node_fileno, 8002,
self.gateway_node)
self.gateway_node.node_conn = self.btc_node_connection
self.gateway_node.remote_node_conn = self.btc_remote_node_connection
self.gateway_node.connection_pool.add(self.local_node_fileno,
LOCALHOST, 8001,
self.btc_node_connection)
self.gateway_node.connection_pool.add(self.remote_node_fileno,
LOCALHOST, 8002,
self.btc_remote_node_connection)
def test_block_headers_request(self):
sent_get_headers = GetHeadersBtcMessage(12345, 23456, [self.HASH],
self.HASH)
helpers.receive_node_message(self.gateway_node, self.local_node_fileno,
sent_get_headers.rawbytes())
self.btc_remote_node_connection.enqueue_msg.assert_called_once_with(
sent_get_headers)
response_headers = HeadersBtcMessage(12345, [])
helpers.receive_node_message(self.gateway_node,
self.remote_node_fileno,
response_headers.rawbytes())
self.btc_node_connection.enqueue_msg.assert_called_once_with(
response_headers)
def _recovered_compact_block_to_bx_block(
self, compression_result: CompactBlockCompressionResult,
recovery_item: CompactBlockRecoveryData
) -> CompactBlockCompressionResult:
"""
Handle recovery of Bitcoin compact block message.
"""
missing_indices = compression_result.missing_indices
recovered_transactions = compression_result.recovered_transactions
block_transactions = recovery_item.block_transactions
if len(missing_indices) != len(recovered_transactions):
logger.debug(
"Number of transactions missing in compact block does not match number of recovered transactions."
"Missing transactions - {}. Recovered transactions - {}",
len(missing_indices), len(recovered_transactions))
return CompactBlockCompressionResult(False, None, None, None,
missing_indices,
recovered_transactions)
for i in range(len(missing_indices)):
missing_index = missing_indices[i]
block_transactions[missing_index] = recovered_transactions[i]
size = 0
total_txs_count = len(block_transactions)
block_msg_parts = deque()
block_header = recovery_item.block_header
block_msg_parts.append(block_header)
size += len(block_header)
tx_count_size = btc_messages_util.get_sizeof_btc_varint(
total_txs_count)
tx_count_buf = bytearray(tx_count_size)
btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf,
0)
block_msg_parts.append(tx_count_buf)
size += tx_count_size
for transaction in block_transactions:
block_msg_parts.append(transaction)
size += len(transaction) # pyre-ignore
msg_header = bytearray(btc_constants.BTC_HDR_COMMON_OFF)
struct.pack_into("<L12sL", msg_header, 0, recovery_item.magic,
BtcMessageType.BLOCK, size)
block_msg_parts.appendleft(msg_header)
size += btc_constants.BTC_HDR_COMMON_OFF
block_msg_bytes = bytearray(size)
off = 0
for blob in block_msg_parts:
next_off = off + len(blob)
block_msg_bytes[off:next_off] = blob
off = next_off
checksum = crypto.bitcoin_hash(
block_msg_bytes[btc_constants.BTC_HDR_COMMON_OFF:size])
block_msg_bytes[btc_constants.BTC_HEADER_MINUS_CHECKSUM:btc_constants.
BTC_HDR_COMMON_OFF] = checksum[0:4]
bx_block, compression_block_info = self.block_to_bx_block(
BlockBtcMessage(buf=block_msg_bytes), recovery_item.tx_service,
True, 0) # TODO need to think about a better algorithm
compress_start_datetime = compression_block_info.start_datetime
compress_end_datetime = datetime.utcnow()
block_info = BlockInfo(
compression_block_info.block_hash,
compression_block_info.short_ids, compress_start_datetime,
compress_end_datetime,
(compress_end_datetime - compress_start_datetime).total_seconds() *
1000, compression_block_info.txn_count,
compression_block_info.compressed_block_hash,
compression_block_info.prev_block_hash,
compression_block_info.original_size,
compression_block_info.compressed_size,
compression_block_info.compression_rate,
compression_block_info.ignored_short_ids)
return CompactBlockCompressionResult(True, block_info, bx_block, None,
[], [])
class BtcMessageFactoryTest(MessageFactoryTestCase):
MAGIC = 12345
VERSION = 11111
HASH = BtcObjectHash(binary=crypto.bitcoin_hash(b"123"))
VERSION_BTC_MESSAGE = VersionBtcMessage(MAGIC, VERSION, "127.0.0.1", 8000,
"127.0.0.1", 8001, 123, 0,
"hello".encode("utf-8"))
def get_message_factory(self):
return btc_message_factory
def test_peek_message_success_all_types(self):
# TODO: pull these numbers into constants, along with all the BTC messages
self.get_message_preview_successfully(self.VERSION_BTC_MESSAGE,
VersionBtcMessage.MESSAGE_TYPE,
90)
self.get_message_preview_successfully(VerAckBtcMessage(self.MAGIC),
VerAckBtcMessage.MESSAGE_TYPE, 0)
self.get_message_preview_successfully(PingBtcMessage(self.MAGIC),
PingBtcMessage.MESSAGE_TYPE, 8)
self.get_message_preview_successfully(PongBtcMessage(self.MAGIC, 123),
PongBtcMessage.MESSAGE_TYPE, 8)
self.get_message_preview_successfully(GetAddrBtcMessage(self.MAGIC),
GetAddrBtcMessage.MESSAGE_TYPE,
0)
self.get_message_preview_successfully(
AddrBtcMessage(self.MAGIC,
[(int(time.time()), "127.0.0.1", 8000)]),
AddrBtcMessage.MESSAGE_TYPE, 23)
inv_vector = [(1, self.HASH), (2, self.HASH)]
self.get_message_preview_successfully(
InvBtcMessage(self.MAGIC, inv_vector), InvBtcMessage.MESSAGE_TYPE,
73)
self.get_message_preview_successfully(
GetDataBtcMessage(self.MAGIC, inv_vector),
GetDataBtcMessage.MESSAGE_TYPE, 73)
self.get_message_preview_successfully(
NotFoundBtcMessage(self.MAGIC, inv_vector),
NotFoundBtcMessage.MESSAGE_TYPE, 73)
hashes = [self.HASH, self.HASH]
self.get_message_preview_successfully(
GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetHeadersBtcMessage.MESSAGE_TYPE, 101)
self.get_message_preview_successfully(
GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetBlocksBtcMessage.MESSAGE_TYPE, 101)
self.get_message_preview_successfully(
TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0),
TxBtcMessage.MESSAGE_TYPE, 10)
txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5
self.get_message_preview_successfully(
BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0,
0, 0, txs), BlockBtcMessage.MESSAGE_TYPE, 131)
self.get_message_preview_successfully(
HeadersBtcMessage(self.MAGIC,
[helpers.generate_bytearray(81)] * 2),
HeadersBtcMessage.MESSAGE_TYPE, 163)
self.get_message_preview_successfully(
RejectBtcMessage(self.MAGIC, b"a message",
RejectBtcMessage.REJECT_MALFORMED, b"test break",
helpers.generate_bytearray(10)),
RejectBtcMessage.MESSAGE_TYPE, 32)
self.get_message_preview_successfully(
SendHeadersBtcMessage(self.MAGIC),
SendHeadersBtcMessage.MESSAGE_TYPE, 0)
self.get_message_preview_successfully(
FeeFilterBtcMessage(self.MAGIC, fee_rate=100),
FeeFilterBtcMessage.MESSAGE_TYPE, 8)
self.get_message_preview_successfully(
BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)),
XversionBtcMessage.MESSAGE_TYPE, 0)
def test_peek_message_incomplete(self):
is_full_message, command, payload_length = btc_message_factory.get_message_header_preview_from_input_buffer(
create_input_buffer_with_bytes(
self.VERSION_BTC_MESSAGE.rawbytes()[:-10]))
self.assertFalse(is_full_message)
self.assertEqual(b"version", command)
self.assertEqual(90, payload_length)
is_full_message, command, payload_length = btc_message_factory.get_message_header_preview_from_input_buffer(
create_input_buffer_with_bytes(
self.VERSION_BTC_MESSAGE.rawbytes()[:1]))
self.assertFalse(is_full_message)
self.assertIsNone(command)
self.assertIsNone(payload_length)
def test_parse_message_success_all_types(self):
# TODO: pull these numbers into constants, along with all the BTC messages
self.create_message_successfully(self.VERSION_BTC_MESSAGE,
VersionBtcMessage)
self.create_message_successfully(VerAckBtcMessage(self.MAGIC),
VerAckBtcMessage)
self.create_message_successfully(PingBtcMessage(self.MAGIC),
PingBtcMessage)
self.create_message_successfully(PongBtcMessage(self.MAGIC, 123),
PongBtcMessage)
self.create_message_successfully(GetAddrBtcMessage(self.MAGIC),
GetAddrBtcMessage)
self.create_message_successfully(
AddrBtcMessage(self.MAGIC,
[(int(time.time()), "127.0.0.1", 8000)]),
AddrBtcMessage)
inv_vector = [(1, self.HASH), (2, self.HASH)]
self.create_message_successfully(InvBtcMessage(self.MAGIC, inv_vector),
InvBtcMessage)
self.create_message_successfully(
GetDataBtcMessage(self.MAGIC, inv_vector), GetDataBtcMessage)
self.create_message_successfully(
NotFoundBtcMessage(self.MAGIC, inv_vector), NotFoundBtcMessage)
hashes = [self.HASH, self.HASH]
self.create_message_successfully(
GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetHeadersBtcMessage)
self.create_message_successfully(
GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH),
GetBlocksBtcMessage)
self.create_message_successfully(
TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0), TxBtcMessage)
txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5
self.create_message_successfully(
BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0,
0, 0, txs), BlockBtcMessage)
self.create_message_successfully(
HeadersBtcMessage(self.MAGIC,
[helpers.generate_bytearray(81)] * 2),
HeadersBtcMessage)
self.create_message_successfully(
RejectBtcMessage(self.MAGIC, b"a message",
RejectBtcMessage.REJECT_MALFORMED, b"test break",
helpers.generate_bytearray(10)), RejectBtcMessage)
self.create_message_successfully(SendHeadersBtcMessage(self.MAGIC),
SendHeadersBtcMessage)
self.create_message_successfully(
FeeFilterBtcMessage(self.MAGIC, fee_rate=100), FeeFilterBtcMessage)
self.create_message_successfully(
BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)),
XversionBtcMessage)
def test_parse_message_incomplete(self):
with self.assertRaises(PayloadLenError):
btc_message_factory.create_message_from_buffer(
PingBtcMessage(self.MAGIC).rawbytes()[:-1])
ping_message = PingBtcMessage(self.MAGIC)
for i in range(BTC_HEADER_MINUS_CHECKSUM, BTC_HDR_COMMON_OFF):
ping_message.buf[i] = 0
with self.assertRaises(ChecksumError):
btc_message_factory.create_message_from_buffer(
ping_message.rawbytes())
def test_segwit_tx_hash(self):
seg_tx = "010000000001024668fcfeba861f7f1bf4d386f15cc6923bd8425e0214686671775359d17a51d50100000000ffffffffdc5530f864de2fac86246426094d7b8586a452d6bd8c209bb891646afb3548770000000000ffffffff0220040000000000001600147771a1cab96e36344b1693d3d9f29180ca900482f5c40100000000001976a91483121cc1ea476c25d91191ff735a5e90518c732788ac02473044022006db5e6aa36dafb5d89a8522675d304a228d39ede1450aaab04f84b1fb57db2902203efb537cca9738c599d95d5c0ddcec6ebd11c6001541a89a468246318e0bd6fe012102d6b8b2ba44eb621ac9537ed7e11553bb02060abca88a9e6faf7697df5ac6d30c02483045022100b04869e06930db5d4e8e4d453d9aed1097a8dae57eef0274ebdc99a106796335022037a6b744900b9b6392448c961e8d793367a0caf675b9ca80349c593e505d8e9d0121034ef9635ae7cd714b2cf8af7e72f23b8b07c7f75d75df95da8d682ae17459091b00000000"
seg_tx_bytes = convert.hex_to_bytes(seg_tx)
self.assertTrue(btc_common_utils.is_segwit(seg_tx_bytes))
self.assertEqual(
convert.bytes_to_hex(
btc_common_utils.get_txid(seg_tx_bytes).binary),
"d9a057f11a21cf8afd32278e23fd2290660f05a3ffb582466eb5a1a5ece4ce85")
def test_non_segwit_tx_hash(self):
non_seg_tx = "0100000002d8c8df6a6fdd2addaf589a83d860f18b44872d13ee6ec3526b2b470d42a96d4d000000008b483045022100b31557e47191936cb14e013fb421b1860b5e4fd5d2bc5ec1938f4ffb1651dc8902202661c2920771fd29dd91cd4100cefb971269836da4914d970d333861819265ba014104c54f8ea9507f31a05ae325616e3024bd9878cb0a5dff780444002d731577be4e2e69c663ff2da922902a4454841aa1754c1b6292ad7d317150308d8cce0ad7abffffffff2ab3fa4f68a512266134085d3260b94d3b6cfd351450cff021c045a69ba120b2000000008b4830450220230110bc99ef311f1f8bda9d0d968bfe5dfa4af171adbef9ef71678d658823bf022100f956d4fcfa0995a578d84e7e913f9bb1cf5b5be1440bcede07bce9cd5b38115d014104c6ec27cffce0823c3fecb162dbd576c88dd7cda0b7b32b0961188a392b488c94ca174d833ee6a9b71c0996620ae71e799fc7c77901db147fa7d97732e49c8226ffffffff02c0175302000000001976a914a3d89c53bb956f08917b44d113c6b2bcbe0c29b788acc01c3d09000000001976a91408338e1d5e26db3fce21b011795b1c3c8a5a5d0788ac00000000"
non_seg_tx_bytes = convert.hex_to_bytes(non_seg_tx)
self.assertFalse(btc_common_utils.is_segwit(non_seg_tx_bytes))
self.assertEqual(
convert.bytes_to_hex(
btc_common_utils.get_txid(non_seg_tx_bytes).binary),
"9021b49d445c719106c95d561b9c3fac7bcb3650db67684a9226cd7fa1e1c1a0")
def block_hash(self) -> BtcObjectHash:
if self._hash_val is None:
raw_hash = crypto.bitcoin_hash(self.block_header())
self._hash_val = BtcObjectHash(buf=raw_hash,
length=BTC_SHA_HASH_LEN)
return self._hash_val