def __init__(self,
magic=None,
version=None,
tx_in=None,
tx_out=None,
lock_time=None,
is_segwit=None,
buf=None):
if buf is None:
tot_size = sum([len(x.rawbytes()) for x in tx_in]) + sum(
[len(x.rawbytes()) for x in tx_out])
buf = bytearray(BTC_HDR_COMMON_OFF + 2 * 9 + 8 + tot_size)
self.buf = buf
off = BTC_HDR_COMMON_OFF
struct.pack_into("<I", buf, off, version)
off += 4
off += btc_messages_util.pack_int_to_btc_varint(
len(tx_in), buf, off)
for inp in tx_in:
rawbytes = inp.rawbytes()
size = len(rawbytes)
buf[off:off + size] = rawbytes
off += size
off += btc_messages_util.pack_int_to_btc_varint(
len(tx_out), buf, off)
for out in tx_out:
rawbytes = out.rawbytes()
size = len(rawbytes)
buf[off:off + size] = rawbytes
off += size
struct.pack_into("<I", buf, off, lock_time)
off += 4
super(TxBtcMessage, self).__init__(magic, self.MESSAGE_TYPE,
off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._version = None
self._tx_in = None
self._tx_out = None
self._lock_time = None
self._tx_hash = None
self._tx_id = None
self._tx_out_count = None
self._tx_in_count = None
self._is_segwit_tx = is_segwit
def __init__(self, magic: int = None, block_hash: BtcObjectHash = None, transactions: List[memoryview] = None,
# pyre-fixme[9]: buf has type `memoryview`; used as `None`.
buf: memoryview = None):
if buf is None:
total_tx_size = sum(len(tx) for tx in transactions)
# pyre-fixme[9]: buf has type `memoryview`; used as `bytearray`.
buf = bytearray(BTC_HDR_COMMON_OFF + BTC_SHA_HASH_LEN + BTC_VARINT_MIN_SIZE + total_tx_size)
off = BTC_HDR_COMMON_OFF
buf[off:off + BTC_SHA_HASH_LEN] = block_hash.get_big_endian()
off += BTC_SHA_HASH_LEN
off += pack_int_to_btc_varint(len(transactions), buf, off)
for tx in transactions:
buf[off:off + len(tx)] = tx
off += len(tx)
self.buf = buf
super(BlockTransactionsBtcMessage, self).__init__(magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._block_hash = None
# pyre-fixme[8]: Attribute has type `List[int]`; used as `None`.
self._transactions: List[int] = None
def __init__(self,
value=None,
pk_script=None,
buf=None,
off=None,
length=None):
if buf is None:
pk_script_len = len(pk_script)
buf = bytearray(8 + 9 + pk_script_len)
self.buf = buf
off = 0
struct.pack_into("<Q", buf, off, value)
off += 8
off += btc_messages_util.pack_int_to_btc_varint(
pk_script_len, buf, off)
buf[off:off + pk_script_len] = pk_script
self.size = off + pk_script_len
self.off = 0
else:
self.buf = buf
self.size = length
self.off = off
self._memoryview = memoryview(buf)
def __init__(self,
prev_outpoint_hash=None,
prev_out_index=None,
sig_script=None,
sequence=None,
buf=None,
off=None,
length=None):
if buf is None:
buf = bytearray(36 + 9 + len(sig_script) + 4)
self.buf = buf
off = 0
pack_outpoint(prev_outpoint_hash, prev_out_index, buf, off)
off += 36
off += btc_messages_util.pack_int_to_btc_varint(
len(sig_script), buf, off)
buf[off:off + len(sig_script)] = sig_script
off += len(sig_script)
struct.pack_into("<I", buf, off, sequence)
off += 4
self.size = off
self.off = 0
else:
self.buf = buf
self.size = length
self.off = off
self._memoryview = memoryview(buf)
def __init__(self,
magic=None,
inv_vect=None,
command=None,
request_witness_data=False,
buf=None):
if buf is None:
buf = bytearray(
btc_constants.BTC_HDR_COMMON_OFF + 9 + 36 * len(inv_vect)
) # we conservatively allocate the max space for varint
self.buf = buf
off = btc_constants.BTC_HDR_COMMON_OFF
off += btc_messages_util.pack_int_to_btc_varint(
len(inv_vect), buf, off)
for inv_item in inv_vect:
inv_type = inv_item[0]
if request_witness_data:
inv_type = _inv_type_to_witness_inv_type(inv_item[0])
struct.pack_into("<L", buf, off, inv_type)
off += 4
buf[off:off + 32] = inv_item[1].get_big_endian()
off += 32
super(InventoryBtcMessage,
self).__init__(magic, command,
off - btc_constants.BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
def __init__(self, magic=None, addrs=None, buf=None):
if addrs is None:
addrs = []
if buf is None:
buf = bytearray(BTC_HDR_COMMON_OFF + 9 + len(addrs) * (4 + 18))
self.buf = buf
off = BTC_HDR_COMMON_OFF
off += pack_int_to_btc_varint(len(addrs), buf, off)
for triplet in addrs:
# pack the timestamp
struct.pack_into('<L', buf, off, triplet[0])
off += 4
# pack the host ip and port pair
buf[off:off + 18] = ipaddrport_to_btcbytearray(
triplet[1], triplet[2])
off += 18
BtcMessage.__init__(self, magic, self.MESSAGE_TYPE,
off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
def __init__(self,
magic=None,
version=None,
hashes=None,
hash_stop=None,
command=None,
buf=None):
if hashes is None:
hashes = []
if buf is None:
buf = bytearray(BTC_HDR_COMMON_OFF + 9 + (len(hashes) + 1) * 32)
self.buf = buf
off = BTC_HDR_COMMON_OFF
struct.pack_into("<I", buf, off, version)
off += 4
off += pack_int_to_btc_varint(len(hashes), buf, off)
for hash_val in hashes:
buf[off:off + 32] = hash_val.get_big_endian()
off += 32
buf[off:off + 32] = hash_stop.get_big_endian()
off += 32
BtcMessage.__init__(self, magic, command, off - BTC_HDR_COMMON_OFF,
buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._version = self._hash_count = self._hashes = self._hash_stop = None
def __init__(
self,
magic: int = None,
block_hash: BtcObjectHash = None,
indices: List[int] = None,
# pyre-fixme[9]: buf has type `memoryview`; used as `None`.
buf: memoryview = None):
if buf is None:
# pyre-fixme[9]: buf has type `memoryview`; used as `bytearray`.
buf = bytearray(BTC_HDR_COMMON_OFF + BTC_SHA_HASH_LEN +
BTC_VARINT_MIN_SIZE +
BTC_VARINT_MIN_SIZE * len(indices))
off = BTC_HDR_COMMON_OFF
buf[off:off + BTC_SHA_HASH_LEN] = block_hash.get_big_endian()
off += BTC_SHA_HASH_LEN
off += pack_int_to_btc_varint(len(indices), buf, off)
last_real_index = -1
for real_index in indices:
diff = real_index - last_real_index - 1
last_real_index = real_index
off += pack_int_to_btc_varint(diff, buf, off)
self.buf = buf
super(GetBlockTransactionsBtcMessage,
self).__init__(magic, self.MESSAGE_TYPE,
off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._block_hash = None
# pyre-fixme[8]: Attribute has type `List[int]`; used as `None`.
self._indices: List[int] = None
def __init__(self, magic=None, headers=None, buf=None):
if buf is None:
buf = bytearray(BTC_HDR_COMMON_OFF + 9 + len(headers) * 81)
self.buf = buf
off = BTC_HDR_COMMON_OFF
off += pack_int_to_btc_varint(len(headers), buf, off)
for header in headers:
buf[off:off + 81] = header
off += 81
BtcMessage.__init__(self, magic, self.MESSAGE_TYPE,
off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(self.buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._headers = self._header_count = self._block_hash = None
def __init__(self,
magic=None,
version=None,
prev_block=None,
merkle_root=None,
timestamp=None,
bits=None,
nonce=None,
txns=None,
buf=None):
if buf is None:
total_tx_size = sum(len(tx) for tx in txns)
buf = bytearray(BTC_HDR_COMMON_OFF + 80 + 9 + total_tx_size)
self.buf = buf
off = pack_block_header(buf, version, prev_block, merkle_root,
timestamp, bits, nonce)
off += pack_int_to_btc_varint(len(txns), buf, off)
for tx in txns:
buf[off:off + len(tx)] = tx
off += len(tx)
super(BlockBtcMessage,
self).__init__(magic, self.MESSAGE_TYPE,
off - BTC_HDR_COMMON_OFF, buf)
else:
assert not isinstance(buf, str)
self.buf = buf
self._memoryview = memoryview(self.buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._version = None
self._prev_block: Optional[BtcObjectHash] = None
self._merkle_root = None
self._bits = self._nonce = self._txn_count = self._txns = self._hash_val = None
self._header = self._tx_offset = None
self._timestamp = 0
def _recovered_compact_block_to_bx_block(
self, compression_result: CompactBlockCompressionResult,
recovery_item: CompactBlockRecoveryData
) -> CompactBlockCompressionResult:
"""
Handle recovery of Bitcoin compact block message.
"""
missing_indices = compression_result.missing_indices
recovered_transactions = compression_result.recovered_transactions
block_transactions = recovery_item.block_transactions
if len(missing_indices) != len(recovered_transactions):
logger.debug(
"Number of transactions missing in compact block does not match number of recovered transactions."
"Missing transactions - {}. Recovered transactions - {}",
len(missing_indices), len(recovered_transactions))
return CompactBlockCompressionResult(False, None, None, None,
missing_indices,
recovered_transactions)
for i in range(len(missing_indices)):
missing_index = missing_indices[i]
block_transactions[missing_index] = recovered_transactions[i]
size = 0
total_txs_count = len(block_transactions)
block_msg_parts = deque()
block_header = recovery_item.block_header
block_msg_parts.append(block_header)
size += len(block_header)
tx_count_size = btc_messages_util.get_sizeof_btc_varint(
total_txs_count)
tx_count_buf = bytearray(tx_count_size)
btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf,
0)
block_msg_parts.append(tx_count_buf)
size += tx_count_size
for transaction in block_transactions:
block_msg_parts.append(transaction)
size += len(transaction) # pyre-ignore
msg_header = bytearray(btc_constants.BTC_HDR_COMMON_OFF)
struct.pack_into("<L12sL", msg_header, 0, recovery_item.magic,
BtcMessageType.BLOCK, size)
block_msg_parts.appendleft(msg_header)
size += btc_constants.BTC_HDR_COMMON_OFF
block_msg_bytes = bytearray(size)
off = 0
for blob in block_msg_parts:
next_off = off + len(blob)
block_msg_bytes[off:next_off] = blob
off = next_off
checksum = crypto.bitcoin_hash(
block_msg_bytes[btc_constants.BTC_HDR_COMMON_OFF:size])
block_msg_bytes[btc_constants.BTC_HEADER_MINUS_CHECKSUM:btc_constants.
BTC_HDR_COMMON_OFF] = checksum[0:4]
bx_block, compression_block_info = self.block_to_bx_block(
BlockBtcMessage(buf=block_msg_bytes), recovery_item.tx_service,
True, 0) # TODO need to think about a better algorithm
compress_start_datetime = compression_block_info.start_datetime
compress_end_datetime = datetime.utcnow()
block_info = BlockInfo(
compression_block_info.block_hash,
compression_block_info.short_ids, compress_start_datetime,
compress_end_datetime,
(compress_end_datetime - compress_start_datetime).total_seconds() *
1000, compression_block_info.txn_count,
compression_block_info.compressed_block_hash,
compression_block_info.prev_block_hash,
compression_block_info.original_size,
compression_block_info.compressed_size,
compression_block_info.compression_rate,
compression_block_info.ignored_short_ids)
return CompactBlockCompressionResult(True, block_info, bx_block, None,
[], [])
def compact_block_to_bx_block(
self, compact_block: CompactBlockBtcMessage,
transaction_service: TransactionService
) -> CompactBlockCompressionResult:
"""
Handle decompression of Bitcoin compact block.
Decompression converts compact block message to full block message.
"""
compress_start_datetime = datetime.utcnow()
block_header = compact_block.block_header()
sha256_hash = hashlib.sha256()
sha256_hash.update(block_header)
sha256_hash.update(compact_block.short_nonce_buf())
hex_digest = sha256_hash.digest()
key = hex_digest[0:16]
short_ids = compact_block.short_ids()
short_id_to_tx_contents = {}
for tx_hash in transaction_service.iter_transaction_hashes():
tx_hash_binary = tx_hash.binary[::-1]
tx_short_id = compute_short_id(key, tx_hash_binary)
if tx_short_id in short_ids:
tx_content = transaction_service.get_transaction_by_hash(
tx_hash)
if tx_content is None:
logger.debug(
"Hash {} is known by transactions service but content is missing.",
tx_hash)
else:
short_id_to_tx_contents[tx_short_id] = tx_content
if len(short_id_to_tx_contents) == len(short_ids):
break
block_transactions = []
missing_transactions_indices = []
pre_filled_transactions = compact_block.pre_filled_transactions()
total_txs_count = len(pre_filled_transactions) + len(short_ids)
size = 0
block_msg_parts = deque()
block_msg_parts.append(block_header)
size += len(block_header)
tx_count_size = btc_messages_util.get_sizeof_btc_varint(
total_txs_count)
tx_count_buf = bytearray(tx_count_size)
btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf,
0)
block_msg_parts.append(tx_count_buf)
size += tx_count_size
short_ids_iter = iter(short_ids.keys())
for index in range(total_txs_count):
if index not in pre_filled_transactions:
short_id = next(short_ids_iter)
if short_id in short_id_to_tx_contents:
short_tx = short_id_to_tx_contents[short_id]
block_msg_parts.append(short_tx)
block_transactions.append(short_tx)
size += len(short_tx)
else:
missing_transactions_indices.append(index)
block_transactions.append(None)
else:
pre_filled_transaction = pre_filled_transactions[index]
block_msg_parts.append(pre_filled_transaction)
block_transactions.append(pre_filled_transaction)
size += len(pre_filled_transaction)
recovered_item = CompactBlockRecoveryData(block_transactions,
block_header,
compact_block.magic(),
transaction_service)
block_info = BlockInfo(compact_block.block_hash(), [],
compress_start_datetime,
compress_start_datetime, 0, None, None, None,
len(compact_block.rawbytes()), None, None, [])
if len(missing_transactions_indices) > 0:
recovery_index = self._last_recovery_idx
self._last_recovery_idx += 1
self._recovery_items[
recovery_index] = recovered_item # pyre-ignore
return CompactBlockCompressionResult(False, block_info, None,
recovery_index,
missing_transactions_indices,
[])
result = CompactBlockCompressionResult(False, block_info, None, None,
[], [])
return self._recovered_compact_block_to_bx_block(
result, recovered_item)
def pack_int_to_ont_varint(val: int, buf: Union[memoryview, bytearray], off: int) -> int:
return pack_int_to_btc_varint(val, buf, off)
def __init__(
self,
magic: int = None,
version: int = None,
prev_block: BtcObjectHash = None,
# pyre-fixme[9]: merkle_root has type `BtcObjectHash`; used as `None`.
# pyre-fixme[9]: timestamp has type `int`; used as `None`.
# pyre-fixme[9]: bits has type `int`; used as `None`.
merkle_root: BtcObjectHash = None,
timestamp: int = None,
bits: int = None,
# pyre-fixme[9]: block_nonce has type `int`; used as `None`.
# pyre-fixme[9]: short_nonce has type `int`; used as `None`.
# pyre-fixme[9]: short_ids has type `List[memoryview]`; used as `None`.
block_nonce: int = None,
short_nonce: int = None,
short_ids: List[memoryview] = None,
# pyre-fixme[9]: prefilled_txns has type `List[memoryview]`; used
# as `None`.
# pyre-fixme[9]: buf has type `bytearray`; used as `None`.
prefilled_txns: List[memoryview] = None,
buf: bytearray = None):
if buf is None:
prefilled_tx_size = sum(len(tx) for tx in prefilled_txns)
buf = bytearray(BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE +
BTC_SHORT_NONCE_SIZE + 2 * BTC_VARINT_MIN_SIZE +
BTC_COMPACT_BLOCK_SHORT_ID_LEN * len(short_ids) +
prefilled_tx_size)
off = pack_block_header(buf, version, prev_block, merkle_root,
timestamp, bits, block_nonce)
struct.pack_into("<Q", buf, off, short_nonce)
off += BTC_SHORT_NONCE_SIZE
off += pack_int_to_btc_varint(len(short_ids), buf, off)
for shortid in short_ids:
buf[off:off + BTC_COMPACT_BLOCK_SHORT_ID_LEN] = shortid
off += BTC_COMPACT_BLOCK_SHORT_ID_LEN
off += pack_int_to_btc_varint(len(prefilled_txns), buf, off)
for index, tx in prefilled_txns:
# pyre-fixme[6]: Expected `Sized` for 1st param but got `int`.
# pyre-fixme[6]: Expected `Union[typing.Iterable[int], bytes]` for
# 2nd param but got `int`.
buf[off:off + len(tx)] = tx
# pyre-fixme[6]: Expected `Sized` for 1st param but got `int`.
off += len(tx)
self.buf = buf
super(CompactBlockBtcMessage,
self).__init__(magic, self.MESSAGE_TYPE,
off - BTC_HDR_COMMON_OFF, buf)
else:
self.buf = buf
self._memoryview = memoryview(buf)
self._magic = self._command = self._payload_len = self._checksum = None
self._payload = None
self._version = self._prev_block = self._merkle_root = self._timestamp = None
# pyre-fixme[8]: Attribute has type `int`; used as `None`.
self._bits: int = None
# pyre-fixme[8]: Attribute has type `int`; used as `None`.
self._block_nonce: int = None
# pyre-fixme[8]: Attribute has type `int`; used as `None`.
self._short_nonce: int = None
# pyre-fixme[8]: Attribute has type `int`; used as `None`.
self._short_id_c: int = None
self._short_nonce_buf = None
# pyre-fixme[8]: Attribute has type `Dict[bytes, int]`; used as `None`.
self._short_ids: Dict[bytes, int] = None
# pyre-fixme[8]: Attribute has type `Dict[int, memoryview]`; used as `None`.
self._pre_filled_transactions: Dict[int, memoryview] = None
self._block_header = None
# pyre-fixme[8]: Attribute has type `BtcObjectHash`; used as `None`.
self._hash_val: BtcObjectHash = None
self._pre_filled_tx_count = None
