def __init__(self, magic=None, version=None, tx_in=None, tx_out=None, lock_time=None, is_segwit=None, buf=None): if buf is None: tot_size = sum([len(x.rawbytes()) for x in tx_in]) + sum( [len(x.rawbytes()) for x in tx_out]) buf = bytearray(BTC_HDR_COMMON_OFF + 2 * 9 + 8 + tot_size) self.buf = buf off = BTC_HDR_COMMON_OFF struct.pack_into("<I", buf, off, version) off += 4 off += btc_messages_util.pack_int_to_btc_varint( len(tx_in), buf, off) for inp in tx_in: rawbytes = inp.rawbytes() size = len(rawbytes) buf[off:off + size] = rawbytes off += size off += btc_messages_util.pack_int_to_btc_varint( len(tx_out), buf, off) for out in tx_out: rawbytes = out.rawbytes() size = len(rawbytes) buf[off:off + size] = rawbytes off += size struct.pack_into("<I", buf, off, lock_time) off += 4 super(TxBtcMessage, self).__init__(magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None self._version = None self._tx_in = None self._tx_out = None self._lock_time = None self._tx_hash = None self._tx_id = None self._tx_out_count = None self._tx_in_count = None self._is_segwit_tx = is_segwit
def __init__(self, magic: int = None, block_hash: BtcObjectHash = None, transactions: List[memoryview] = None, # pyre-fixme[9]: buf has type `memoryview`; used as `None`. buf: memoryview = None): if buf is None: total_tx_size = sum(len(tx) for tx in transactions) # pyre-fixme[9]: buf has type `memoryview`; used as `bytearray`. buf = bytearray(BTC_HDR_COMMON_OFF + BTC_SHA_HASH_LEN + BTC_VARINT_MIN_SIZE + total_tx_size) off = BTC_HDR_COMMON_OFF buf[off:off + BTC_SHA_HASH_LEN] = block_hash.get_big_endian() off += BTC_SHA_HASH_LEN off += pack_int_to_btc_varint(len(transactions), buf, off) for tx in transactions: buf[off:off + len(tx)] = tx off += len(tx) self.buf = buf super(BlockTransactionsBtcMessage, self).__init__(magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None self._block_hash = None # pyre-fixme[8]: Attribute has type `List[int]`; used as `None`. self._transactions: List[int] = None
def __init__(self, value=None, pk_script=None, buf=None, off=None, length=None): if buf is None: pk_script_len = len(pk_script) buf = bytearray(8 + 9 + pk_script_len) self.buf = buf off = 0 struct.pack_into("<Q", buf, off, value) off += 8 off += btc_messages_util.pack_int_to_btc_varint( pk_script_len, buf, off) buf[off:off + pk_script_len] = pk_script self.size = off + pk_script_len self.off = 0 else: self.buf = buf self.size = length self.off = off self._memoryview = memoryview(buf)
def __init__(self, prev_outpoint_hash=None, prev_out_index=None, sig_script=None, sequence=None, buf=None, off=None, length=None): if buf is None: buf = bytearray(36 + 9 + len(sig_script) + 4) self.buf = buf off = 0 pack_outpoint(prev_outpoint_hash, prev_out_index, buf, off) off += 36 off += btc_messages_util.pack_int_to_btc_varint( len(sig_script), buf, off) buf[off:off + len(sig_script)] = sig_script off += len(sig_script) struct.pack_into("<I", buf, off, sequence) off += 4 self.size = off self.off = 0 else: self.buf = buf self.size = length self.off = off self._memoryview = memoryview(buf)
def __init__(self, magic=None, inv_vect=None, command=None, request_witness_data=False, buf=None): if buf is None: buf = bytearray( btc_constants.BTC_HDR_COMMON_OFF + 9 + 36 * len(inv_vect) ) # we conservatively allocate the max space for varint self.buf = buf off = btc_constants.BTC_HDR_COMMON_OFF off += btc_messages_util.pack_int_to_btc_varint( len(inv_vect), buf, off) for inv_item in inv_vect: inv_type = inv_item[0] if request_witness_data: inv_type = _inv_type_to_witness_inv_type(inv_item[0]) struct.pack_into("<L", buf, off, inv_type) off += 4 buf[off:off + 32] = inv_item[1].get_big_endian() off += 32 super(InventoryBtcMessage, self).__init__(magic, command, off - btc_constants.BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None
def __init__(self, magic=None, addrs=None, buf=None): if addrs is None: addrs = [] if buf is None: buf = bytearray(BTC_HDR_COMMON_OFF + 9 + len(addrs) * (4 + 18)) self.buf = buf off = BTC_HDR_COMMON_OFF off += pack_int_to_btc_varint(len(addrs), buf, off) for triplet in addrs: # pack the timestamp struct.pack_into('<L', buf, off, triplet[0]) off += 4 # pack the host ip and port pair buf[off:off + 18] = ipaddrport_to_btcbytearray( triplet[1], triplet[2]) off += 18 BtcMessage.__init__(self, magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None
def __init__(self, magic=None, version=None, hashes=None, hash_stop=None, command=None, buf=None): if hashes is None: hashes = [] if buf is None: buf = bytearray(BTC_HDR_COMMON_OFF + 9 + (len(hashes) + 1) * 32) self.buf = buf off = BTC_HDR_COMMON_OFF struct.pack_into("<I", buf, off, version) off += 4 off += pack_int_to_btc_varint(len(hashes), buf, off) for hash_val in hashes: buf[off:off + 32] = hash_val.get_big_endian() off += 32 buf[off:off + 32] = hash_stop.get_big_endian() off += 32 BtcMessage.__init__(self, magic, command, off - BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None self._version = self._hash_count = self._hashes = self._hash_stop = None
def __init__( self, magic: int = None, block_hash: BtcObjectHash = None, indices: List[int] = None, # pyre-fixme[9]: buf has type `memoryview`; used as `None`. buf: memoryview = None): if buf is None: # pyre-fixme[9]: buf has type `memoryview`; used as `bytearray`. buf = bytearray(BTC_HDR_COMMON_OFF + BTC_SHA_HASH_LEN + BTC_VARINT_MIN_SIZE + BTC_VARINT_MIN_SIZE * len(indices)) off = BTC_HDR_COMMON_OFF buf[off:off + BTC_SHA_HASH_LEN] = block_hash.get_big_endian() off += BTC_SHA_HASH_LEN off += pack_int_to_btc_varint(len(indices), buf, off) last_real_index = -1 for real_index in indices: diff = real_index - last_real_index - 1 last_real_index = real_index off += pack_int_to_btc_varint(diff, buf, off) self.buf = buf super(GetBlockTransactionsBtcMessage, self).__init__(magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None self._block_hash = None # pyre-fixme[8]: Attribute has type `List[int]`; used as `None`. self._indices: List[int] = None
def __init__(self, magic=None, headers=None, buf=None): if buf is None: buf = bytearray(BTC_HDR_COMMON_OFF + 9 + len(headers) * 81) self.buf = buf off = BTC_HDR_COMMON_OFF off += pack_int_to_btc_varint(len(headers), buf, off) for header in headers: buf[off:off + 81] = header off += 81 BtcMessage.__init__(self, magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(self.buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None self._headers = self._header_count = self._block_hash = None
def __init__(self, magic=None, version=None, prev_block=None, merkle_root=None, timestamp=None, bits=None, nonce=None, txns=None, buf=None): if buf is None: total_tx_size = sum(len(tx) for tx in txns) buf = bytearray(BTC_HDR_COMMON_OFF + 80 + 9 + total_tx_size) self.buf = buf off = pack_block_header(buf, version, prev_block, merkle_root, timestamp, bits, nonce) off += pack_int_to_btc_varint(len(txns), buf, off) for tx in txns: buf[off:off + len(tx)] = tx off += len(tx) super(BlockBtcMessage, self).__init__(magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf) else: assert not isinstance(buf, str) self.buf = buf self._memoryview = memoryview(self.buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None self._version = None self._prev_block: Optional[BtcObjectHash] = None self._merkle_root = None self._bits = self._nonce = self._txn_count = self._txns = self._hash_val = None self._header = self._tx_offset = None self._timestamp = 0
def _recovered_compact_block_to_bx_block( self, compression_result: CompactBlockCompressionResult, recovery_item: CompactBlockRecoveryData ) -> CompactBlockCompressionResult: """ Handle recovery of Bitcoin compact block message. """ missing_indices = compression_result.missing_indices recovered_transactions = compression_result.recovered_transactions block_transactions = recovery_item.block_transactions if len(missing_indices) != len(recovered_transactions): logger.debug( "Number of transactions missing in compact block does not match number of recovered transactions." "Missing transactions - {}. Recovered transactions - {}", len(missing_indices), len(recovered_transactions)) return CompactBlockCompressionResult(False, None, None, None, missing_indices, recovered_transactions) for i in range(len(missing_indices)): missing_index = missing_indices[i] block_transactions[missing_index] = recovered_transactions[i] size = 0 total_txs_count = len(block_transactions) block_msg_parts = deque() block_header = recovery_item.block_header block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size for transaction in block_transactions: block_msg_parts.append(transaction) size += len(transaction) # pyre-ignore msg_header = bytearray(btc_constants.BTC_HDR_COMMON_OFF) struct.pack_into("<L12sL", msg_header, 0, recovery_item.magic, BtcMessageType.BLOCK, size) block_msg_parts.appendleft(msg_header) size += btc_constants.BTC_HDR_COMMON_OFF block_msg_bytes = bytearray(size) off = 0 for blob in block_msg_parts: next_off = off + len(blob) block_msg_bytes[off:next_off] = blob off = next_off checksum = crypto.bitcoin_hash( block_msg_bytes[btc_constants.BTC_HDR_COMMON_OFF:size]) block_msg_bytes[btc_constants.BTC_HEADER_MINUS_CHECKSUM:btc_constants. BTC_HDR_COMMON_OFF] = checksum[0:4] bx_block, compression_block_info = self.block_to_bx_block( BlockBtcMessage(buf=block_msg_bytes), recovery_item.tx_service, True, 0) # TODO need to think about a better algorithm compress_start_datetime = compression_block_info.start_datetime compress_end_datetime = datetime.utcnow() block_info = BlockInfo( compression_block_info.block_hash, compression_block_info.short_ids, compress_start_datetime, compress_end_datetime, (compress_end_datetime - compress_start_datetime).total_seconds() * 1000, compression_block_info.txn_count, compression_block_info.compressed_block_hash, compression_block_info.prev_block_hash, compression_block_info.original_size, compression_block_info.compressed_size, compression_block_info.compression_rate, compression_block_info.ignored_short_ids) return CompactBlockCompressionResult(True, block_info, bx_block, None, [], [])
def compact_block_to_bx_block( self, compact_block: CompactBlockBtcMessage, transaction_service: TransactionService ) -> CompactBlockCompressionResult: """ Handle decompression of Bitcoin compact block. Decompression converts compact block message to full block message. """ compress_start_datetime = datetime.utcnow() block_header = compact_block.block_header() sha256_hash = hashlib.sha256() sha256_hash.update(block_header) sha256_hash.update(compact_block.short_nonce_buf()) hex_digest = sha256_hash.digest() key = hex_digest[0:16] short_ids = compact_block.short_ids() short_id_to_tx_contents = {} for tx_hash in transaction_service.iter_transaction_hashes(): tx_hash_binary = tx_hash.binary[::-1] tx_short_id = compute_short_id(key, tx_hash_binary) if tx_short_id in short_ids: tx_content = transaction_service.get_transaction_by_hash( tx_hash) if tx_content is None: logger.debug( "Hash {} is known by transactions service but content is missing.", tx_hash) else: short_id_to_tx_contents[tx_short_id] = tx_content if len(short_id_to_tx_contents) == len(short_ids): break block_transactions = [] missing_transactions_indices = [] pre_filled_transactions = compact_block.pre_filled_transactions() total_txs_count = len(pre_filled_transactions) + len(short_ids) size = 0 block_msg_parts = deque() block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size short_ids_iter = iter(short_ids.keys()) for index in range(total_txs_count): if index not in pre_filled_transactions: short_id = next(short_ids_iter) if short_id in short_id_to_tx_contents: short_tx = short_id_to_tx_contents[short_id] block_msg_parts.append(short_tx) block_transactions.append(short_tx) size += len(short_tx) else: missing_transactions_indices.append(index) block_transactions.append(None) else: pre_filled_transaction = pre_filled_transactions[index] block_msg_parts.append(pre_filled_transaction) block_transactions.append(pre_filled_transaction) size += len(pre_filled_transaction) recovered_item = CompactBlockRecoveryData(block_transactions, block_header, compact_block.magic(), transaction_service) block_info = BlockInfo(compact_block.block_hash(), [], compress_start_datetime, compress_start_datetime, 0, None, None, None, len(compact_block.rawbytes()), None, None, []) if len(missing_transactions_indices) > 0: recovery_index = self._last_recovery_idx self._last_recovery_idx += 1 self._recovery_items[ recovery_index] = recovered_item # pyre-ignore return CompactBlockCompressionResult(False, block_info, None, recovery_index, missing_transactions_indices, []) result = CompactBlockCompressionResult(False, block_info, None, None, [], []) return self._recovered_compact_block_to_bx_block( result, recovered_item)
def pack_int_to_ont_varint(val: int, buf: Union[memoryview, bytearray], off: int) -> int: return pack_int_to_btc_varint(val, buf, off)
def __init__( self, magic: int = None, version: int = None, prev_block: BtcObjectHash = None, # pyre-fixme[9]: merkle_root has type `BtcObjectHash`; used as `None`. # pyre-fixme[9]: timestamp has type `int`; used as `None`. # pyre-fixme[9]: bits has type `int`; used as `None`. merkle_root: BtcObjectHash = None, timestamp: int = None, bits: int = None, # pyre-fixme[9]: block_nonce has type `int`; used as `None`. # pyre-fixme[9]: short_nonce has type `int`; used as `None`. # pyre-fixme[9]: short_ids has type `List[memoryview]`; used as `None`. block_nonce: int = None, short_nonce: int = None, short_ids: List[memoryview] = None, # pyre-fixme[9]: prefilled_txns has type `List[memoryview]`; used # as `None`. # pyre-fixme[9]: buf has type `bytearray`; used as `None`. prefilled_txns: List[memoryview] = None, buf: bytearray = None): if buf is None: prefilled_tx_size = sum(len(tx) for tx in prefilled_txns) buf = bytearray(BTC_HDR_COMMON_OFF + BTC_BLOCK_HDR_SIZE + BTC_SHORT_NONCE_SIZE + 2 * BTC_VARINT_MIN_SIZE + BTC_COMPACT_BLOCK_SHORT_ID_LEN * len(short_ids) + prefilled_tx_size) off = pack_block_header(buf, version, prev_block, merkle_root, timestamp, bits, block_nonce) struct.pack_into("<Q", buf, off, short_nonce) off += BTC_SHORT_NONCE_SIZE off += pack_int_to_btc_varint(len(short_ids), buf, off) for shortid in short_ids: buf[off:off + BTC_COMPACT_BLOCK_SHORT_ID_LEN] = shortid off += BTC_COMPACT_BLOCK_SHORT_ID_LEN off += pack_int_to_btc_varint(len(prefilled_txns), buf, off) for index, tx in prefilled_txns: # pyre-fixme[6]: Expected `Sized` for 1st param but got `int`. # pyre-fixme[6]: Expected `Union[typing.Iterable[int], bytes]` for # 2nd param but got `int`. buf[off:off + len(tx)] = tx # pyre-fixme[6]: Expected `Sized` for 1st param but got `int`. off += len(tx) self.buf = buf super(CompactBlockBtcMessage, self).__init__(magic, self.MESSAGE_TYPE, off - BTC_HDR_COMMON_OFF, buf) else: self.buf = buf self._memoryview = memoryview(buf) self._magic = self._command = self._payload_len = self._checksum = None self._payload = None self._version = self._prev_block = self._merkle_root = self._timestamp = None # pyre-fixme[8]: Attribute has type `int`; used as `None`. self._bits: int = None # pyre-fixme[8]: Attribute has type `int`; used as `None`. self._block_nonce: int = None # pyre-fixme[8]: Attribute has type `int`; used as `None`. self._short_nonce: int = None # pyre-fixme[8]: Attribute has type `int`; used as `None`. self._short_id_c: int = None self._short_nonce_buf = None # pyre-fixme[8]: Attribute has type `Dict[bytes, int]`; used as `None`. self._short_ids: Dict[bytes, int] = None # pyre-fixme[8]: Attribute has type `Dict[int, memoryview]`; used as `None`. self._pre_filled_transactions: Dict[int, memoryview] = None self._block_header = None # pyre-fixme[8]: Attribute has type `BtcObjectHash`; used as `None`. self._hash_val: BtcObjectHash = None self._pre_filled_tx_count = None