def parse_bx_block_header( bx_block: memoryview, block_pieces: Deque[Union[bytearray, memoryview]]) -> BlockHeaderInfo: block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets( bx_block) short_ids, short_ids_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer( bx_block, block_offsets.short_id_offset) # Compute block header hash block_header_size = \ block_offsets.block_begin_offset + \ btc_constants.BTC_HDR_COMMON_OFF + \ btc_constants.BTC_BLOCK_HDR_SIZE block_hash = BtcObjectHash(buf=crypto.bitcoin_hash( bx_block[block_offsets.block_begin_offset + btc_constants.BTC_HDR_COMMON_OFF:block_header_size]), length=btc_constants.BTC_SHA_HASH_LEN) offset = block_header_size # Add header piece txn_count, txn_count_size = btc_common_utils.btc_varint_to_int( bx_block, block_header_size) offset += txn_count_size block_pieces.append(bx_block[block_offsets.block_begin_offset:offset]) return BlockHeaderInfo(block_offsets, short_ids, short_ids_len, block_hash, offset, txn_count)
def block_hash(self): if self._block_hash is None: header = self._memoryview[: BTC_BLOCK_HDR_SIZE] # remove the tx count at the end raw_hash = crypto.bitcoin_hash(header) self._hash_val = BtcObjectHash(buf=raw_hash, length=BTC_SHA_HASH_LEN) return self._hash_val
def block_hash(self) -> BtcObjectHash: if self._hash_val is None: header = self._memoryview[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE] raw_hash = crypto.bitcoin_hash(header) self._hash_val = BtcObjectHash(buf=raw_hash, length=BTC_SHA_HASH_LEN) # pyre-fixme[7]: Expected `BtcObjectHash` but got `None`. return self._hash_val
def test_ont_block_to_bloxroute_block_and_back_sids_found(self): prev_block_hash = bytearray(crypto.bitcoin_hash(b"123")) prev_block = OntObjectHash(prev_block_hash, length=SHA256_HASH_LEN) merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234")) merkle_root = OntObjectHash(merkle_root_hash, length=SHA256_HASH_LEN) txns_root_hash = bytearray(crypto.bitcoin_hash(b"345")) txns_root = OntObjectHash(txns_root_hash, length=SHA256_HASH_LEN) block_root_hash = bytearray(crypto.bitcoin_hash(b"456")) block_root = OntObjectHash(block_root_hash, length=SHA256_HASH_LEN) consensus_payload = bytes(b'111') next_bookkeeper = bytes(b'222') bookkeepers = [bytes(33)] * 5 sig_data = [bytes(2)] * 3 txns = [] timestamp = 1 height = 2 consensus_data = 3 ont_block = BlockOntMessage(self.magic, self.version, prev_block, txns_root, block_root, timestamp, height, consensus_data, consensus_payload, next_bookkeeper, bookkeepers, sig_data, txns, merkle_root) block_hash = ont_block.block_hash() bloxroute_block, block_info = self.ont_message_converter.block_to_bx_block( ont_block, self.tx_service, True, 0) self.assertEqual(0, block_info.txn_count) self.assertEqual(self.short_ids, list(block_info.short_ids)) self.assertEqual(ont_block.block_hash(), block_info.block_hash) parsed_ont_block, block_info, _, _ = self.ont_message_converter.bx_block_to_block( bloxroute_block, self.tx_service) self.assertIsNotNone(block_info) self.assertEqual(ont_block.rawbytes().tobytes(), parsed_ont_block.rawbytes().tobytes()) self.assertEqual(self.magic, parsed_ont_block.magic()) self.assertEqual( prev_block_hash, parsed_ont_block.prev_block_hash().get_little_endian()) self.assertEqual(ont_block.checksum(), parsed_ont_block.checksum()) self.assertEqual(block_hash, parsed_ont_block.block_hash()) self.assertEqual(block_hash.binary, block_info.block_hash.binary) self.assertEqual(timestamp, parsed_ont_block.timestamp())
def test_btc_block_to_bloxroute_block_and_back_sids_found(self): prev_block_hash = bytearray(crypto.bitcoin_hash(b"123")) prev_block = BtcObjectHash(prev_block_hash, length=SHA256_HASH_LEN) merkle_root_hash = bytearray(crypto.bitcoin_hash(b"234")) merkle_root = BtcObjectHash(merkle_root_hash, length=SHA256_HASH_LEN) timestamp = 1 bits = 2 nonce = 3 btc_block = BlockBtcMessage( self.magic, self.version, prev_block, merkle_root, timestamp, bits, nonce, self.txns ) block_hash = btc_block.block_hash() bloxroute_block, block_info = self.btc_message_converter.block_to_bx_block( btc_block, self.tx_service, True, 0 ) self.assertEqual(10, block_info.txn_count) self.assertEqual("5a77d1e9612d350b3734f6282259b7ff0a3f87d62cfef5f35e91a5604c0490a3", block_info.prev_block_hash) self.assertEqual(self.short_ids, list(block_info.short_ids)) self.assertEqual(btc_block.block_hash(), block_info.block_hash) # TODO: if we convert bloxroute block to a class, add some tests here parsed_btc_block, block_info, _, _ = self.btc_message_converter.bx_block_to_block( bloxroute_block, self.tx_service) self.assertIsNotNone(block_info) self.assertEqual(parsed_btc_block.rawbytes().tobytes(), btc_block.rawbytes().tobytes()) self.assertEqual(self.version, parsed_btc_block.version()) self.assertEqual(self.magic, parsed_btc_block.magic()) self.assertEqual(prev_block_hash, parsed_btc_block.prev_block_hash().get_little_endian()) self.assertEqual(merkle_root_hash, parsed_btc_block.merkle_root().get_little_endian()) self.assertEqual(timestamp, parsed_btc_block.timestamp()) self.assertEqual(bits, parsed_btc_block.bits()) self.assertEqual(nonce, parsed_btc_block.nonce()) self.assertEqual(len(self.txns), parsed_btc_block.txn_count()) self.assertEqual(btc_block.checksum(), parsed_btc_block.checksum()) self.assertEqual(block_hash, parsed_btc_block.block_hash()) self.assertEqual(block_hash.binary, block_info.block_hash.binary) self.assertEqual(list(block_info.short_ids), self.short_ids)
def validate_payload(cls, buf, unpacked_args): command, _magic, checksum, payload_length = unpacked_args if payload_length != len(buf) - cls.HEADER_LENGTH: error_message = log_messages.PAYLOAD_LENGTH_MISMATCH.text.format( payload_length, len(buf)) logger.error(error_message) raise PayloadLenError(error_message) ref_checksum = crypto.bitcoin_hash( buf[cls.HEADER_LENGTH:cls.HEADER_LENGTH + payload_length])[0:4] if checksum != ref_checksum: error_message = log_messages.PACKET_CHECKSUM_MISMATCH.text.format( checksum, ref_checksum, repr(buf)) logger.error(error_message) raise ChecksumError(error_message, buf)
def peek_block(input_buffer): buf = input_buffer.peek_message(BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE) is_here = False if input_buffer.length < BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE: return is_here, None, None header = buf[BTC_HDR_COMMON_OFF:BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE] raw_hash = crypto.bitcoin_hash(header) payload_len = struct.unpack_from('<L', buf, 16)[0] is_here = True return is_here, BtcObjectHash(buf=raw_hash, length=BTC_SHA_HASH_LEN), payload_len
def __init__(self, magic=None, command=None, payload_len=None, buf=None): self.buf = buf self._memoryview = memoryview(buf) magic_num = magic if magic not in BTC_MAGIC_NUMBERS else BTC_MAGIC_NUMBERS[ magic] checksum = crypto.bitcoin_hash( self._memoryview[BTC_HDR_COMMON_OFF:payload_len + BTC_HDR_COMMON_OFF]) off = 0 struct.pack_into("<L12sL", buf, off, magic_num, command, payload_len) off += 20 buf[off:off + 4] = checksum[0:4] self._magic = magic_num self._command = command self._payload_len = payload_len self._payload = None self._checksum = None
class BlockchainSyncBtcTest(AbstractTestCase): HASH = BtcObjectHash(binary=crypto.bitcoin_hash(b"hi")) def setUp(self): self.local_node_fileno = 1 self.remote_node_fileno = 2 self.gateway_node = spies.make_spy_node(BtcGatewayNode, 8000, include_default_btc_args=True) self.btc_node_connection = spies.make_spy_connection( BtcNodeConnection, self.local_node_fileno, 8001, self.gateway_node) self.btc_remote_node_connection = spies.make_spy_connection( BtcRemoteConnection, self.remote_node_fileno, 8002, self.gateway_node) self.gateway_node.node_conn = self.btc_node_connection self.gateway_node.remote_node_conn = self.btc_remote_node_connection self.gateway_node.connection_pool.add(self.local_node_fileno, LOCALHOST, 8001, self.btc_node_connection) self.gateway_node.connection_pool.add(self.remote_node_fileno, LOCALHOST, 8002, self.btc_remote_node_connection) def test_block_headers_request(self): sent_get_headers = GetHeadersBtcMessage(12345, 23456, [self.HASH], self.HASH) helpers.receive_node_message(self.gateway_node, self.local_node_fileno, sent_get_headers.rawbytes()) self.btc_remote_node_connection.enqueue_msg.assert_called_once_with( sent_get_headers) response_headers = HeadersBtcMessage(12345, []) helpers.receive_node_message(self.gateway_node, self.remote_node_fileno, response_headers.rawbytes()) self.btc_node_connection.enqueue_msg.assert_called_once_with( response_headers)
def _recovered_compact_block_to_bx_block( self, compression_result: CompactBlockCompressionResult, recovery_item: CompactBlockRecoveryData ) -> CompactBlockCompressionResult: """ Handle recovery of Bitcoin compact block message. """ missing_indices = compression_result.missing_indices recovered_transactions = compression_result.recovered_transactions block_transactions = recovery_item.block_transactions if len(missing_indices) != len(recovered_transactions): logger.debug( "Number of transactions missing in compact block does not match number of recovered transactions." "Missing transactions - {}. Recovered transactions - {}", len(missing_indices), len(recovered_transactions)) return CompactBlockCompressionResult(False, None, None, None, missing_indices, recovered_transactions) for i in range(len(missing_indices)): missing_index = missing_indices[i] block_transactions[missing_index] = recovered_transactions[i] size = 0 total_txs_count = len(block_transactions) block_msg_parts = deque() block_header = recovery_item.block_header block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size for transaction in block_transactions: block_msg_parts.append(transaction) size += len(transaction) # pyre-ignore msg_header = bytearray(btc_constants.BTC_HDR_COMMON_OFF) struct.pack_into("<L12sL", msg_header, 0, recovery_item.magic, BtcMessageType.BLOCK, size) block_msg_parts.appendleft(msg_header) size += btc_constants.BTC_HDR_COMMON_OFF block_msg_bytes = bytearray(size) off = 0 for blob in block_msg_parts: next_off = off + len(blob) block_msg_bytes[off:next_off] = blob off = next_off checksum = crypto.bitcoin_hash( block_msg_bytes[btc_constants.BTC_HDR_COMMON_OFF:size]) block_msg_bytes[btc_constants.BTC_HEADER_MINUS_CHECKSUM:btc_constants. BTC_HDR_COMMON_OFF] = checksum[0:4] bx_block, compression_block_info = self.block_to_bx_block( BlockBtcMessage(buf=block_msg_bytes), recovery_item.tx_service, True, 0) # TODO need to think about a better algorithm compress_start_datetime = compression_block_info.start_datetime compress_end_datetime = datetime.utcnow() block_info = BlockInfo( compression_block_info.block_hash, compression_block_info.short_ids, compress_start_datetime, compress_end_datetime, (compress_end_datetime - compress_start_datetime).total_seconds() * 1000, compression_block_info.txn_count, compression_block_info.compressed_block_hash, compression_block_info.prev_block_hash, compression_block_info.original_size, compression_block_info.compressed_size, compression_block_info.compression_rate, compression_block_info.ignored_short_ids) return CompactBlockCompressionResult(True, block_info, bx_block, None, [], [])
class BtcMessageFactoryTest(MessageFactoryTestCase): MAGIC = 12345 VERSION = 11111 HASH = BtcObjectHash(binary=crypto.bitcoin_hash(b"123")) VERSION_BTC_MESSAGE = VersionBtcMessage(MAGIC, VERSION, "127.0.0.1", 8000, "127.0.0.1", 8001, 123, 0, "hello".encode("utf-8")) def get_message_factory(self): return btc_message_factory def test_peek_message_success_all_types(self): # TODO: pull these numbers into constants, along with all the BTC messages self.get_message_preview_successfully(self.VERSION_BTC_MESSAGE, VersionBtcMessage.MESSAGE_TYPE, 90) self.get_message_preview_successfully(VerAckBtcMessage(self.MAGIC), VerAckBtcMessage.MESSAGE_TYPE, 0) self.get_message_preview_successfully(PingBtcMessage(self.MAGIC), PingBtcMessage.MESSAGE_TYPE, 8) self.get_message_preview_successfully(PongBtcMessage(self.MAGIC, 123), PongBtcMessage.MESSAGE_TYPE, 8) self.get_message_preview_successfully(GetAddrBtcMessage(self.MAGIC), GetAddrBtcMessage.MESSAGE_TYPE, 0) self.get_message_preview_successfully( AddrBtcMessage(self.MAGIC, [(int(time.time()), "127.0.0.1", 8000)]), AddrBtcMessage.MESSAGE_TYPE, 23) inv_vector = [(1, self.HASH), (2, self.HASH)] self.get_message_preview_successfully( InvBtcMessage(self.MAGIC, inv_vector), InvBtcMessage.MESSAGE_TYPE, 73) self.get_message_preview_successfully( GetDataBtcMessage(self.MAGIC, inv_vector), GetDataBtcMessage.MESSAGE_TYPE, 73) self.get_message_preview_successfully( NotFoundBtcMessage(self.MAGIC, inv_vector), NotFoundBtcMessage.MESSAGE_TYPE, 73) hashes = [self.HASH, self.HASH] self.get_message_preview_successfully( GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH), GetHeadersBtcMessage.MESSAGE_TYPE, 101) self.get_message_preview_successfully( GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH), GetBlocksBtcMessage.MESSAGE_TYPE, 101) self.get_message_preview_successfully( TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0), TxBtcMessage.MESSAGE_TYPE, 10) txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5 self.get_message_preview_successfully( BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0, 0, 0, txs), BlockBtcMessage.MESSAGE_TYPE, 131) self.get_message_preview_successfully( HeadersBtcMessage(self.MAGIC, [helpers.generate_bytearray(81)] * 2), HeadersBtcMessage.MESSAGE_TYPE, 163) self.get_message_preview_successfully( RejectBtcMessage(self.MAGIC, b"a message", RejectBtcMessage.REJECT_MALFORMED, b"test break", helpers.generate_bytearray(10)), RejectBtcMessage.MESSAGE_TYPE, 32) self.get_message_preview_successfully( SendHeadersBtcMessage(self.MAGIC), SendHeadersBtcMessage.MESSAGE_TYPE, 0) self.get_message_preview_successfully( FeeFilterBtcMessage(self.MAGIC, fee_rate=100), FeeFilterBtcMessage.MESSAGE_TYPE, 8) self.get_message_preview_successfully( BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)), XversionBtcMessage.MESSAGE_TYPE, 0) def test_peek_message_incomplete(self): is_full_message, command, payload_length = btc_message_factory.get_message_header_preview_from_input_buffer( create_input_buffer_with_bytes( self.VERSION_BTC_MESSAGE.rawbytes()[:-10])) self.assertFalse(is_full_message) self.assertEqual(b"version", command) self.assertEqual(90, payload_length) is_full_message, command, payload_length = btc_message_factory.get_message_header_preview_from_input_buffer( create_input_buffer_with_bytes( self.VERSION_BTC_MESSAGE.rawbytes()[:1])) self.assertFalse(is_full_message) self.assertIsNone(command) self.assertIsNone(payload_length) def test_parse_message_success_all_types(self): # TODO: pull these numbers into constants, along with all the BTC messages self.create_message_successfully(self.VERSION_BTC_MESSAGE, VersionBtcMessage) self.create_message_successfully(VerAckBtcMessage(self.MAGIC), VerAckBtcMessage) self.create_message_successfully(PingBtcMessage(self.MAGIC), PingBtcMessage) self.create_message_successfully(PongBtcMessage(self.MAGIC, 123), PongBtcMessage) self.create_message_successfully(GetAddrBtcMessage(self.MAGIC), GetAddrBtcMessage) self.create_message_successfully( AddrBtcMessage(self.MAGIC, [(int(time.time()), "127.0.0.1", 8000)]), AddrBtcMessage) inv_vector = [(1, self.HASH), (2, self.HASH)] self.create_message_successfully(InvBtcMessage(self.MAGIC, inv_vector), InvBtcMessage) self.create_message_successfully( GetDataBtcMessage(self.MAGIC, inv_vector), GetDataBtcMessage) self.create_message_successfully( NotFoundBtcMessage(self.MAGIC, inv_vector), NotFoundBtcMessage) hashes = [self.HASH, self.HASH] self.create_message_successfully( GetHeadersBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH), GetHeadersBtcMessage) self.create_message_successfully( GetBlocksBtcMessage(self.MAGIC, self.VERSION, hashes, self.HASH), GetBlocksBtcMessage) self.create_message_successfully( TxBtcMessage(self.MAGIC, self.VERSION, [], [], 0), TxBtcMessage) txs = [TxIn(buf=bytearray(10), length=10, off=0).rawbytes()] * 5 self.create_message_successfully( BlockBtcMessage(self.MAGIC, self.VERSION, self.HASH, self.HASH, 0, 0, 0, txs), BlockBtcMessage) self.create_message_successfully( HeadersBtcMessage(self.MAGIC, [helpers.generate_bytearray(81)] * 2), HeadersBtcMessage) self.create_message_successfully( RejectBtcMessage(self.MAGIC, b"a message", RejectBtcMessage.REJECT_MALFORMED, b"test break", helpers.generate_bytearray(10)), RejectBtcMessage) self.create_message_successfully(SendHeadersBtcMessage(self.MAGIC), SendHeadersBtcMessage) self.create_message_successfully( FeeFilterBtcMessage(self.MAGIC, fee_rate=100), FeeFilterBtcMessage) self.create_message_successfully( BtcMessage(self.MAGIC, b'xversion', 0, bytearray(30)), XversionBtcMessage) def test_parse_message_incomplete(self): with self.assertRaises(PayloadLenError): btc_message_factory.create_message_from_buffer( PingBtcMessage(self.MAGIC).rawbytes()[:-1]) ping_message = PingBtcMessage(self.MAGIC) for i in range(BTC_HEADER_MINUS_CHECKSUM, BTC_HDR_COMMON_OFF): ping_message.buf[i] = 0 with self.assertRaises(ChecksumError): btc_message_factory.create_message_from_buffer( ping_message.rawbytes()) def test_segwit_tx_hash(self): seg_tx = "010000000001024668fcfeba861f7f1bf4d386f15cc6923bd8425e0214686671775359d17a51d50100000000ffffffffdc5530f864de2fac86246426094d7b8586a452d6bd8c209bb891646afb3548770000000000ffffffff0220040000000000001600147771a1cab96e36344b1693d3d9f29180ca900482f5c40100000000001976a91483121cc1ea476c25d91191ff735a5e90518c732788ac02473044022006db5e6aa36dafb5d89a8522675d304a228d39ede1450aaab04f84b1fb57db2902203efb537cca9738c599d95d5c0ddcec6ebd11c6001541a89a468246318e0bd6fe012102d6b8b2ba44eb621ac9537ed7e11553bb02060abca88a9e6faf7697df5ac6d30c02483045022100b04869e06930db5d4e8e4d453d9aed1097a8dae57eef0274ebdc99a106796335022037a6b744900b9b6392448c961e8d793367a0caf675b9ca80349c593e505d8e9d0121034ef9635ae7cd714b2cf8af7e72f23b8b07c7f75d75df95da8d682ae17459091b00000000" seg_tx_bytes = convert.hex_to_bytes(seg_tx) self.assertTrue(btc_common_utils.is_segwit(seg_tx_bytes)) self.assertEqual( convert.bytes_to_hex( btc_common_utils.get_txid(seg_tx_bytes).binary), "d9a057f11a21cf8afd32278e23fd2290660f05a3ffb582466eb5a1a5ece4ce85") def test_non_segwit_tx_hash(self): non_seg_tx = "0100000002d8c8df6a6fdd2addaf589a83d860f18b44872d13ee6ec3526b2b470d42a96d4d000000008b483045022100b31557e47191936cb14e013fb421b1860b5e4fd5d2bc5ec1938f4ffb1651dc8902202661c2920771fd29dd91cd4100cefb971269836da4914d970d333861819265ba014104c54f8ea9507f31a05ae325616e3024bd9878cb0a5dff780444002d731577be4e2e69c663ff2da922902a4454841aa1754c1b6292ad7d317150308d8cce0ad7abffffffff2ab3fa4f68a512266134085d3260b94d3b6cfd351450cff021c045a69ba120b2000000008b4830450220230110bc99ef311f1f8bda9d0d968bfe5dfa4af171adbef9ef71678d658823bf022100f956d4fcfa0995a578d84e7e913f9bb1cf5b5be1440bcede07bce9cd5b38115d014104c6ec27cffce0823c3fecb162dbd576c88dd7cda0b7b32b0961188a392b488c94ca174d833ee6a9b71c0996620ae71e799fc7c77901db147fa7d97732e49c8226ffffffff02c0175302000000001976a914a3d89c53bb956f08917b44d113c6b2bcbe0c29b788acc01c3d09000000001976a91408338e1d5e26db3fce21b011795b1c3c8a5a5d0788ac00000000" non_seg_tx_bytes = convert.hex_to_bytes(non_seg_tx) self.assertFalse(btc_common_utils.is_segwit(non_seg_tx_bytes)) self.assertEqual( convert.bytes_to_hex( btc_common_utils.get_txid(non_seg_tx_bytes).binary), "9021b49d445c719106c95d561b9c3fac7bcb3650db67684a9226cd7fa1e1c1a0")
def block_hash(self) -> BtcObjectHash: if self._hash_val is None: raw_hash = crypto.bitcoin_hash(self.block_header()) self._hash_val = BtcObjectHash(buf=raw_hash, length=BTC_SHA_HASH_LEN) return self._hash_val