def test_propagate_block_to_network_unencrypted_block(self): self.node.opts.encrypt_blocks = False block_message = helpers.generate_bytearray(50) block_info = BlockInfo( Sha256Hash(helpers.generate_bytearray(crypto.SHA256_HASH_LEN)), [], datetime.datetime.utcnow(), datetime.datetime.utcnow(), 0, 1, helpers.generate_bytearray(crypto.SHA256_HASH_LEN), helpers.generate_bytearray(crypto.SHA256_HASH_LEN), 0, 0, 0, []) connection = MockConnection( MockSocketConnection(1, self.node, ip_address=LOCALHOST, port=9000), self.node) self.neutrality_service.propagate_block_to_network( block_message, connection, block_info) self.assertEqual(1, len(self.node.broadcast_messages)) broadcast_message, connection_types = self.node.broadcast_messages[0] # self.assertTrue(any(ConnectionType.RELAY_BLOCK & connection_type for connection_type in connection_types)) self.assertTrue( all(ConnectionType.RELAY_BLOCK in connection_type for connection_type in connection_types)) self.assertEqual(block_info.block_hash, broadcast_message.block_hash()) self.assertNotIn(block_info.block_hash, self.node.in_progress_blocks._cache) self.assertNotIn(broadcast_message.block_hash(), self.neutrality_service._receipt_tracker)
def compact_block_to_bx_block( self, compact_block: CompactBlockBtcMessage, transaction_service: ExtensionTransactionService ) -> CompactBlockCompressionResult: compress_start_datetime = datetime.utcnow() tsk = self.compact_mapping_tasks.borrow_task() tsk.init(tpe.InputBytes(compact_block.buf), transaction_service.proxy, compact_block.magic()) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compact_mapping_tasks.return_task(tsk) raise message_conversion_error.btc_compact_block_compression_error( compact_block.block_hash(), e) success = tsk.success() recovered_item = ExtensionCompactBlockRecoveryData( transaction_service, tsk) block_info = BlockInfo(compact_block.block_hash(), [], compress_start_datetime, compress_start_datetime, 0, None, None, None, len(compact_block.rawbytes()), None, None, []) if success: result = CompactBlockCompressionResult( False, block_info, None, None, [], create_recovered_transactions()) return self._extension_recovered_compact_block_to_bx_block( result, recovered_item) else: recovery_index = self._last_recovery_idx self._extension_recovered_items[recovery_index] = recovered_item self._last_recovery_idx += 1 return CompactBlockCompressionResult( False, block_info, None, recovery_index, tsk.missing_indices(), create_recovered_transactions())
def block_to_bx_block( self, block_msg: InternalEthBlockInfo, tx_service: ExtensionTransactionService ) -> Tuple[memoryview, BlockInfo]: compress_start_datetime = datetime.datetime.utcnow() compress_start_timestamp = time.time() self._default_block_size = max(self._default_block_size, len(block_msg.rawbytes())) tsk = self.compression_tasks.borrow_task() tsk.init(tpe.InputBytes(block_msg.rawbytes()), tx_service.proxy) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compression_tasks.return_task(tsk) raise message_conversion_error.eth_block_compression_error( block_msg.block_hash(), e) bx_block = tsk.bx_block() starting_offset = tsk.starting_offset() block = memoryview(bx_block)[starting_offset:] compressed_size = len(block) original_size = len(block_msg.rawbytes()) - starting_offset block_hash = block_msg.block_hash() block_info = BlockInfo( block_hash, tsk.short_ids(), compress_start_datetime, datetime.datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tsk.txn_count(), tsk.compressed_block_hash().hex_string(), tsk.prev_block_hash().hex_string(), original_size, compressed_size, 100 - float(compressed_size) / original_size * 100) self.compression_tasks.return_task(tsk) return block, block_info
def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() self._default_block_size = max(self._default_block_size, len(block_msg.buf)) tsk = self.compression_tasks.borrow_task() tsk.init(tpe.InputBytes(block_msg.buf), tx_service.proxy, enable_block_compression, min_tx_age_seconds) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: self.compression_tasks.return_task(tsk) raise message_conversion_error.btc_block_compression_error( block_msg.block_hash(), e) bx_block = tsk.bx_block() block = memoryview(bx_block) compressed_size = len(block) original_size = len(block_msg.rawbytes()) block_hash = OntObjectHash( binary=convert.hex_to_bytes(tsk.block_hash().hex_string())) block_info = BlockInfo( block_hash, tsk.short_ids(), compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tsk.txn_count(), tsk.compressed_block_hash().hex_string(), tsk.prev_block_hash().hex_string(), original_size, compressed_size, 100 - float(compressed_size) / original_size * 100, tsk.ignored_short_ids()) self.compression_tasks.return_task(tsk) return block, block_info
def recovered_compact_block_to_bx_block( self, failed_compression_result: CompactBlockCompressionResult, ) -> CompactBlockCompressionResult: failed_block_info = failed_compression_result.block_info start_datetime = datetime.utcnow() block_info = BlockInfo( failed_block_info.block_hash, # pyre-ignore failed_block_info.short_ids, # pyre-ignore start_datetime, start_datetime, 0, None, None, None, failed_block_info.original_size, # pyre-ignore None, None, []) failed_compression_result.block_info = block_info return self._recovered_compact_block_to_bx_block( failed_compression_result, self._recovery_items.pop( failed_compression_result.recovery_index) # pyre-ignore )
def get_block_info( bx_block: memoryview, block_hash: Sha256Hash, short_ids: List[int], decompress_start_datetime: datetime, decompress_start_timestamp: float, total_tx_count: Optional[int] = None, btc_block_msg: Optional[BlockBtcMessage] = None ) -> BlockInfo: if btc_block_msg is not None: bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(bx_block)) compressed_size = len(bx_block) prev_block_hash = convert.bytes_to_hex(btc_block_msg.prev_block_hash().binary) btc_block_len = len(btc_block_msg.rawbytes()) compression_rate = 100 - float(compressed_size) / btc_block_len * 100 else: bx_block_hash = None compressed_size = None prev_block_hash = None btc_block_len = None compression_rate = None return BlockInfo( block_hash, short_ids, decompress_start_datetime, datetime.utcnow(), (time.time() - decompress_start_timestamp) * 1000, total_tx_count, bx_block_hash, prev_block_hash, btc_block_len, compressed_size, compression_rate )
def block_to_bx_block( self, block_msg, tx_service ) -> Tuple[memoryview, BlockInfo]: """ Compresses a Bitcoin block's transactions and packs it into a bloXroute block. """ compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] header = block_msg.header() size += len(header) buf.append(header) for tx in block_msg.txns(): tx_hash = btc_common_utils.get_txid(tx) short_id = tx_service.get_short_id(tx_hash) if short_id == constants.NULL_TX_SID: buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off prev_block_hash = convert.bytes_to_hex(block_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) original_size = len(block_msg.rawbytes()) block_info = BlockInfo( block_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, block_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100 ) return memoryview(block), block_info
def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: """ Compresses a Bitcoin block's transactions and packs it into a bloXroute block. """ compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] original_size = len(block_msg.rawbytes()) ignored_sids = [] header = block_msg.header() size += len(header) buf.append(header) max_timestamp_for_compression = time.time() - min_tx_age_seconds for tx in block_msg.txns(): tx_hash = btc_common_utils.get_txid(tx) transaction_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(transaction_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time( short_id) if short_id == constants.NULL_TX_SID or \ not enable_block_compression or \ short_id_assign_time > max_timestamp_for_compression: if short_id != constants.NULL_TX_SIDS: ignored_sids.append(short_id) buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 block = finalize_block_bytes(buf, size, short_ids) prev_block_hash = convert.bytes_to_hex( block_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo(block_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, block_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100, ignored_sids) return memoryview(block), block_info
def block_to_bx_block(self, block_msg: BlockOntMessage, tx_service: TransactionService) \ -> Tuple[memoryview, BlockInfo]: start_datetime = datetime.utcnow() start_time = time.time() block_info = BlockInfo( block_msg.block_hash(), [], start_datetime, datetime.utcnow(), (time.time() - start_time) * 1000, block_msg.txn_count(), str(block_msg.block_hash()), convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), len(block_msg.rawbytes()), 0 ) return block_msg.rawbytes(), block_info
def _extension_recovered_compact_block_to_bx_block( self, mapping_result: CompactBlockCompressionResult, recovery_item: ExtensionCompactBlockRecoveryData ) -> CompactBlockCompressionResult: mapping_task = recovery_item.mapping_task compression_task: tpe.BtcCompactBlockCompressionTask = mapping_task.compression_task() # pyre-fixme[16]: `List` has no attribute `vector`. compression_task.add_recovered_transactions(mapping_result.recovered_transactions.vector) mapping_block_info = mapping_result.block_info try: task_pool_proxy.run_task(compression_task) except tpe.AggregatedException as e: self.compact_mapping_tasks.return_task(mapping_task) # pyre-fixme[16]: `Optional` has no attribute `block_hash`. raise message_conversion_error.btc_compact_block_compression_error(mapping_block_info.block_hash, e) bx_block = memoryview(compression_task.bx_block()) block_hash = mapping_block_info.block_hash txn_count = compression_task.txn_count() compressed_block_hash = compression_task.compressed_block_hash().hex_string() prev_block_hash = compression_task.prev_block_hash().hex_string() short_ids = compression_task.short_ids() compress_end_datetime = datetime.utcnow() # pyre-fixme[16]: `Optional` has no attribute `start_datetime`. compress_start_datetime = mapping_block_info.start_datetime # pyre-fixme[16]: `Optional` has no attribute `original_size`. original_size = mapping_block_info.original_size compressed_size = len(bx_block) block_info = BlockInfo( block_hash, short_ids, compress_start_datetime, compress_end_datetime, (compress_end_datetime - compress_start_datetime).total_seconds() * 1000, txn_count, compressed_block_hash, prev_block_hash, original_size, compressed_size, 100 - float(compressed_size) / original_size * 100 ) self.compact_mapping_tasks.return_task(mapping_task) return CompactBlockCompressionResult(True, block_info, bx_block, None, [], [])
def bx_block_to_block(self, bx_block_msg: memoryview, tx_service: TransactionService) -> BlockDecompressionResult: start_datetime = datetime.utcnow() start_time = time.time() block_msg = BlockOntMessage(buf=bytearray(bx_block_msg)) block_info = BlockInfo( block_msg.block_hash(), [], start_datetime, datetime.utcnow(), (time.time() - start_time) * 1000, block_msg.txn_count(), str(block_msg.block_hash()), convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), len(block_msg.rawbytes()), 0 ) return BlockDecompressionResult(block_msg, block_info, [], [])
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult: start_datetime = datetime.datetime.utcnow() start_time = time.time() block_msg = BlockBtcMessage(buf=bx_block_msg) block_info = BlockInfo( block_msg.block_hash(), [], start_datetime, datetime.datetime.utcnow(), (time.time() - start_time) * 1000, block_msg.txn_count(), # pyre-fixme[6]: Expected `Optional[str]` for 7th param but got # `BtcObjectHash`. block_msg.block_hash(), convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), len(block_msg.rawbytes()), 0 ) return BlockDecompressionResult(block_msg, block_info, [], [])
def block_to_bx_block(self, block_msg, tx_service) -> Tuple[memoryview, BlockInfo]: start_datetime = datetime.datetime.utcnow() start_time = time.time() block_msg = cast(BlockBtcMessage, block_msg) block_info = BlockInfo( block_msg.block_hash(), [], start_datetime, datetime.datetime.utcnow(), (time.time() - start_time) * 1000, block_msg.txn_count(), # pyre-fixme[6]: Expected `Optional[str]` for 7th param but got # `BtcObjectHash`. block_msg.block_hash(), convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), len(block_msg.rawbytes()), 0 ) return block_msg.rawbytes(), block_info
def recovered_compact_block_to_bx_block( # pyre-ignore self, failed_mapping_result: CompactBlockCompressionResult, ) -> CompactBlockCompressionResult: failed_block_info = failed_mapping_result.block_info block_info = BlockInfo( failed_block_info.block_hash, # pyre-ignore failed_block_info.short_ids, # pyre-ignore datetime.utcnow(), datetime.utcnow(), 0, None, None, None, failed_block_info.original_size, # pyre-ignore None, None, []) failed_mapping_result.block_info = block_info recovered_item = self._extension_recovered_items.pop( failed_mapping_result.recovery_index) # pyre-ignore return self._extension_recovered_compact_block_to_bx_block( failed_mapping_result, recovered_item)
def block_to_bx_block( self, block_msg: InternalEthBlockInfo, tx_service: TransactionService, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: """ Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids The code is optimized and does not make copies of bytes :param block_msg: Ethereum new block message :param tx_service: Transactions service :param enable_block_compression :param min_tx_age_seconds :return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash) """ compress_start_datetime = datetime.datetime.utcnow() compress_start_timestamp = time.time() txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message( block_msg) used_short_ids = [] # creating transactions content content_size = 0 buf = deque() ignored_sids = [] tx_start_index = 0 tx_count = 0 original_size = len(block_msg.rawbytes()) max_timestamp_for_compression = time.time() - min_tx_age_seconds while True: if tx_start_index >= len(txs_bytes): break _, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix( txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length] tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes) tx_hash = Sha256Hash(tx_hash_bytes) tx_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(tx_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time( short_id) if short_id <= constants.NULL_TX_SID or \ not enable_block_compression or short_id_assign_time > max_timestamp_for_compression: if short_id > constants.NULL_TX_SID: ignored_sids.append(short_id) is_full_tx_bytes = rlp_utils.encode_int(1) tx_content_bytes = tx_bytes else: is_full_tx_bytes = rlp_utils.encode_int(0) used_short_ids.append(short_id) tx_content_bytes = bytes() tx_content_prefix = rlp_utils.get_length_prefix_str( len(tx_content_bytes)) short_tx_content_size = len(is_full_tx_bytes) + len( tx_content_prefix) + len(tx_content_bytes) short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list( short_tx_content_size) buf.append(short_tx_content_prefix_bytes) buf.append(is_full_tx_bytes) buf.append(tx_content_prefix) buf.append(tx_content_bytes) content_size += len( short_tx_content_prefix_bytes) + short_tx_content_size tx_start_index = tx_item_start + tx_item_length tx_count += 1 list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(list_of_txs_prefix_bytes) content_size += len(list_of_txs_prefix_bytes) buf.appendleft(block_hdr_full_bytes) content_size += len(block_hdr_full_bytes) buf.append(remaining_bytes) content_size += len(remaining_bytes) compact_block_msg_prefix = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(compact_block_msg_prefix) content_size += len(compact_block_msg_prefix) block = finalize_block_bytes(buf, content_size, used_short_ids) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo(block_msg.block_hash(), used_short_ids, compress_start_datetime, datetime.datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tx_count, bx_block_hash, convert.bytes_to_hex(prev_block_bytes), original_size, content_size, 100 - float(content_size) / original_size * 100, ignored_sids) return memoryview(block), block_info
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult: """ Converts internal broadcast message to Ethereum new block message The code is optimized and does not make copies of bytes :param bx_block_msg: internal broadcast message bytes :param tx_service: Transactions service :return: tuple (new block message, block hash, unknown transaction short id, unknown transaction hashes) """ if not isinstance(bx_block_msg, (bytearray, memoryview)): raise TypeError( "Type bytearray is expected for arg block_bytes but was {0}". format(type(bx_block_msg))) decompress_start_datetime = datetime.datetime.utcnow() decompress_start_timestamp = time.time() block_msg_bytes = bx_block_msg if isinstance( bx_block_msg, memoryview) else memoryview(bx_block_msg) block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets( bx_block_msg) short_ids, short_ids_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer( bx_block_msg, block_offsets.short_id_offset) block_bytes = block_msg_bytes[ block_offsets.block_begin_offset:block_offsets.short_id_offset] _, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix( block_bytes, 0) block_itm_bytes = block_bytes[block_itm_start:] _, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix( block_itm_bytes, 0) full_hdr_bytes = block_itm_bytes[0:block_hdr_start + block_hdr_len] block_hash_bytes = eth_common_utils.keccak_hash(full_hdr_bytes) block_hash = Sha256Hash(block_hash_bytes) _, block_txs_len, block_txs_start = rlp_utils.consume_length_prefix( block_itm_bytes, block_hdr_start + block_hdr_len) txs_bytes = block_itm_bytes[block_txs_start:block_txs_start + block_txs_len] remaining_bytes = block_itm_bytes[block_txs_start + block_txs_len:] # parse statistics variables short_tx_index = 0 unknown_tx_sids = [] unknown_tx_hashes = [] # creating transactions content content_size = 0 buf = deque() tx_count = 0 tx_start_index = 0 while True: if tx_start_index >= len(txs_bytes): break _, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix( txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len] is_full_tx_start = 0 is_full_tx, is_full_tx_len, = rlp_utils.decode_int( tx_bytes, is_full_tx_start) _, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix( tx_bytes, is_full_tx_start + is_full_tx_len) tx_content_bytes = tx_bytes[tx_content_start:tx_content_start + tx_content_len] if is_full_tx: tx_bytes = tx_content_bytes else: short_id = short_ids[short_tx_index] tx_hash, tx_bytes, _ = tx_service.get_transaction(short_id) if tx_hash is None: unknown_tx_sids.append(short_id) elif tx_bytes is None: unknown_tx_hashes.append(tx_hash) short_tx_index += 1 if tx_bytes is not None and not unknown_tx_sids and not unknown_tx_hashes: buf.append(tx_bytes) content_size += len(tx_bytes) tx_count += 1 tx_start_index = tx_itm_start + tx_itm_len if not unknown_tx_sids and not unknown_tx_hashes: txs_prefix = rlp_utils.get_length_prefix_list(content_size) buf.appendleft(txs_prefix) content_size += len(txs_prefix) buf.appendleft(full_hdr_bytes) content_size += len(full_hdr_bytes) buf.append(remaining_bytes) content_size += len(remaining_bytes) msg_len_prefix = rlp_utils.get_length_prefix_list(content_size) buf.appendleft(msg_len_prefix) block_msg_bytes = bytearray(content_size) off = 0 for blob in buf: next_off = off + len(blob) block_msg_bytes[off:next_off] = blob off = next_off block_msg = InternalEthBlockInfo(block_msg_bytes) logger.debug( "Successfully parsed block broadcast message. {} " "transactions in block {}", tx_count, block_hash) bx_block_hash = convert.bytes_to_hex( crypto.double_sha256(bx_block_msg)) compressed_size = len(bx_block_msg) block_info = BlockInfo( block_hash, short_ids, decompress_start_datetime, datetime.datetime.utcnow(), (time.time() - decompress_start_timestamp) * 1000, tx_count, bx_block_hash, convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), compressed_size, 100 - float(compressed_size) / content_size * 100, []) return BlockDecompressionResult(block_msg, block_info, unknown_tx_sids, unknown_tx_hashes) else: logger.debug( "Block recovery needed for {}. Missing {} sids, {} tx hashes. " "Total txs in block: {}", block_hash, len(unknown_tx_sids), len(unknown_tx_hashes), tx_count) return BlockDecompressionResult( None, BlockInfo(block_hash, short_ids, decompress_start_datetime, datetime.datetime.utcnow(), (time.time() - decompress_start_timestamp) * 1000, None, None, None, None, None, None, []), unknown_tx_sids, unknown_tx_hashes)
def block_to_bx_block( self, block_msg: InternalEthBlockInfo, tx_service: TransactionService) -> Tuple[memoryview, BlockInfo]: """ Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids The code is optimized and does not make copies of bytes :param block_msg: Ethereum new block message :param tx_service: Transactions service :return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash) """ compress_start_datetime = datetime.datetime.utcnow() compress_start_timestamp = time.time() msg_bytes = memoryview(block_msg.rawbytes()) _, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix( msg_bytes, 0) block_msg_bytes = msg_bytes[block_msg_itm_start:block_msg_itm_start + block_msg_itm_len] _, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix( block_msg_bytes, 0) block_hdr_full_bytes = block_msg_bytes[0:block_hdr_itm_start + block_hdr_itm_len] block_hdr_bytes = block_msg_bytes[ block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len] _, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix( block_hdr_bytes, 0) prev_block_bytes = block_hdr_bytes[ prev_block_itm_start:prev_block_itm_start + prev_block_itm_len] _, txs_itm_len, txs_itm_start = rlp_utils.consume_length_prefix( block_msg_bytes, block_hdr_itm_start + block_hdr_itm_len) txs_bytes = block_msg_bytes[txs_itm_start:txs_itm_start + txs_itm_len] remaining_bytes = block_msg_bytes[txs_itm_start + txs_itm_len:] used_short_ids = [] # creating transactions content content_size = 0 buf = deque() tx_start_index = 0 tx_count = 0 while True: if tx_start_index >= len(txs_bytes): break _, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix( txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length] tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes) tx_hash = Sha256Hash(tx_hash_bytes) short_id = tx_service.get_short_id(tx_hash) if short_id <= 0: is_full_tx_bytes = rlp_utils.encode_int(1) tx_content_bytes = tx_bytes else: is_full_tx_bytes = rlp_utils.encode_int(0) used_short_ids.append(short_id) tx_content_bytes = bytes() tx_content_prefix = rlp_utils.get_length_prefix_str( len(tx_content_bytes)) short_tx_content_size = len(is_full_tx_bytes) + len( tx_content_prefix) + len(tx_content_bytes) short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list( short_tx_content_size) buf.append(short_tx_content_prefix_bytes) buf.append(is_full_tx_bytes) buf.append(tx_content_prefix) buf.append(tx_content_bytes) content_size += len( short_tx_content_prefix_bytes) + short_tx_content_size tx_start_index = tx_item_start + tx_item_length tx_count += 1 list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(list_of_txs_prefix_bytes) content_size += len(list_of_txs_prefix_bytes) buf.appendleft(block_hdr_full_bytes) content_size += len(block_hdr_full_bytes) buf.append(remaining_bytes) content_size += len(remaining_bytes) compact_block_msg_prefix = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(compact_block_msg_prefix) content_size += len(compact_block_msg_prefix) short_ids_bytes = compact_block_short_ids_serializer.serialize_short_ids_into_bytes( used_short_ids) buf.append(short_ids_bytes) content_size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", content_size) buf.appendleft(offset_buf) content_size += len(short_ids_bytes) # Parse it into the bloXroute message format and send it along block = bytearray(content_size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) original_size = len(block_msg.rawbytes()) block_info = BlockInfo(block_msg.block_hash(), used_short_ids, compress_start_datetime, datetime.datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tx_count, bx_block_hash, convert.bytes_to_hex(prev_block_bytes), original_size, content_size, 100 - float(content_size) / original_size * 100) return memoryview(block), block_info
def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: """ Pack an Ontology block's transactions into a bloXroute block. """ compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] original_size = len(block_msg.rawbytes()) header = block_msg.txn_header() size += len(header) buf.append(header) max_timestamp_for_compression = time.time() - min_tx_age_seconds ignored_sids = [] for tx in block_msg.txns(): tx_hash, _ = ont_messages_util.get_txid(tx) transaction_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(transaction_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time( short_id) if short_id == constants.NULL_TX_SID or \ not enable_block_compression or \ short_id_assign_time > max_timestamp_for_compression: if short_id != constants.NULL_TX_SIDS: ignored_sids.append(ignored_sids) buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes( short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES merkle_root = block_msg.merkle_root() buf.appendleft(merkle_root) size += ont_constants.ONT_HASH_LEN is_consensus_msg_buf = struct.pack("?", False) buf.appendleft(is_consensus_msg_buf) size += 1 offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off prev_block_hash = convert.bytes_to_hex( block_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo(block_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, block_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100, ignored_sids) return memoryview(block), block_info
def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: return block_msg.rawbytes(), \ BlockInfo(convert.bytes_to_hex(self.PREV_BLOCK.binary), [], datetime.datetime.utcnow(), datetime.datetime.utcnow(), 0, 0, None, None, 0, 0, 0, [])
def _recovered_compact_block_to_bx_block( self, compression_result: CompactBlockCompressionResult, recovery_item: CompactBlockRecoveryData ) -> CompactBlockCompressionResult: """ Handle recovery of Bitcoin compact block message. """ missing_indices = compression_result.missing_indices recovered_transactions = compression_result.recovered_transactions block_transactions = recovery_item.block_transactions if len(missing_indices) != len(recovered_transactions): logger.debug( "Number of transactions missing in compact block does not match number of recovered transactions." "Missing transactions - {}. Recovered transactions - {}", len(missing_indices), len(recovered_transactions)) return CompactBlockCompressionResult(False, None, None, None, missing_indices, recovered_transactions) for i in range(len(missing_indices)): missing_index = missing_indices[i] block_transactions[missing_index] = recovered_transactions[i] size = 0 total_txs_count = len(block_transactions) block_msg_parts = deque() block_header = recovery_item.block_header block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size for transaction in block_transactions: block_msg_parts.append(transaction) size += len(transaction) # pyre-ignore msg_header = bytearray(btc_constants.BTC_HDR_COMMON_OFF) struct.pack_into("<L12sL", msg_header, 0, recovery_item.magic, BtcMessageType.BLOCK, size) block_msg_parts.appendleft(msg_header) size += btc_constants.BTC_HDR_COMMON_OFF block_msg_bytes = bytearray(size) off = 0 for blob in block_msg_parts: next_off = off + len(blob) block_msg_bytes[off:next_off] = blob off = next_off checksum = crypto.bitcoin_hash( block_msg_bytes[btc_constants.BTC_HDR_COMMON_OFF:size]) block_msg_bytes[btc_constants.BTC_HEADER_MINUS_CHECKSUM:btc_constants. BTC_HDR_COMMON_OFF] = checksum[0:4] bx_block, compression_block_info = self.block_to_bx_block( BlockBtcMessage(buf=block_msg_bytes), recovery_item.tx_service, True, 0) # TODO need to think about a better algorithm compress_start_datetime = compression_block_info.start_datetime compress_end_datetime = datetime.utcnow() block_info = BlockInfo( compression_block_info.block_hash, compression_block_info.short_ids, compress_start_datetime, compress_end_datetime, (compress_end_datetime - compress_start_datetime).total_seconds() * 1000, compression_block_info.txn_count, compression_block_info.compressed_block_hash, compression_block_info.prev_block_hash, compression_block_info.original_size, compression_block_info.compressed_size, compression_block_info.compression_rate, compression_block_info.ignored_short_ids) return CompactBlockCompressionResult(True, block_info, bx_block, None, [], [])
def block_to_bx_block(self, block_msg, tx_service) -> Tuple[memoryview, BlockInfo]: return block_msg.rawbytes(), \ BlockInfo(convert.bytes_to_hex(self.PREV_BLOCK.binary), [], datetime.datetime.utcnow(), datetime.datetime.utcnow(), 0, 0, None, None, 0, 0, 0)
def compact_block_to_bx_block( self, compact_block: CompactBlockBtcMessage, transaction_service: TransactionService ) -> CompactBlockCompressionResult: """ Handle decompression of Bitcoin compact block. Decompression converts compact block message to full block message. """ compress_start_datetime = datetime.utcnow() block_header = compact_block.block_header() sha256_hash = hashlib.sha256() sha256_hash.update(block_header) sha256_hash.update(compact_block.short_nonce_buf()) hex_digest = sha256_hash.digest() key = hex_digest[0:16] short_ids = compact_block.short_ids() short_id_to_tx_contents = {} for tx_hash in transaction_service.iter_transaction_hashes(): tx_hash_binary = tx_hash.binary[::-1] tx_short_id = compute_short_id(key, tx_hash_binary) if tx_short_id in short_ids: tx_content = transaction_service.get_transaction_by_hash( tx_hash) if tx_content is None: logger.debug( "Hash {} is known by transactions service but content is missing.", tx_hash) else: short_id_to_tx_contents[tx_short_id] = tx_content if len(short_id_to_tx_contents) == len(short_ids): break block_transactions = [] missing_transactions_indices = [] pre_filled_transactions = compact_block.pre_filled_transactions() total_txs_count = len(pre_filled_transactions) + len(short_ids) size = 0 block_msg_parts = deque() block_msg_parts.append(block_header) size += len(block_header) tx_count_size = btc_messages_util.get_sizeof_btc_varint( total_txs_count) tx_count_buf = bytearray(tx_count_size) btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf, 0) block_msg_parts.append(tx_count_buf) size += tx_count_size short_ids_iter = iter(short_ids.keys()) for index in range(total_txs_count): if index not in pre_filled_transactions: short_id = next(short_ids_iter) if short_id in short_id_to_tx_contents: short_tx = short_id_to_tx_contents[short_id] block_msg_parts.append(short_tx) block_transactions.append(short_tx) size += len(short_tx) else: missing_transactions_indices.append(index) block_transactions.append(None) else: pre_filled_transaction = pre_filled_transactions[index] block_msg_parts.append(pre_filled_transaction) block_transactions.append(pre_filled_transaction) size += len(pre_filled_transaction) recovered_item = CompactBlockRecoveryData(block_transactions, block_header, compact_block.magic(), transaction_service) block_info = BlockInfo(compact_block.block_hash(), [], compress_start_datetime, compress_start_datetime, 0, None, None, None, len(compact_block.rawbytes()), None, None, []) if len(missing_transactions_indices) > 0: recovery_index = self._last_recovery_idx self._last_recovery_idx += 1 self._recovery_items[ recovery_index] = recovered_item # pyre-ignore return CompactBlockCompressionResult(False, block_info, None, recovery_index, missing_transactions_indices, []) result = CompactBlockCompressionResult(False, block_info, None, None, [], []) return self._recovered_compact_block_to_bx_block( result, recovered_item)
def block_to_bx_block( self, block_msg: OntConsensusMessage, tx_service: TransactionService, enable_block_compression: bool, min_tx_age_seconds: float ) -> Tuple[memoryview, BlockInfo]: """ Pack an Ontology consensus message's transactions into a bloXroute block. """ consensus_msg = block_msg compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] ignored_sids = [] original_size = len(consensus_msg.rawbytes()) consensus_payload_header = consensus_msg.consensus_payload_header() consensus_payload_header_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_payload_header_len, 0, len(consensus_payload_header)) size += ont_constants.ONT_INT_LEN buf.append(consensus_payload_header_len) size += len(consensus_payload_header) buf.append(consensus_payload_header) consensus_data_type = bytearray(ont_constants.ONT_CHAR_LEN) struct.pack_into("<B", consensus_data_type, 0, consensus_msg.consensus_data_type()) size += ont_constants.ONT_CHAR_LEN buf.append(consensus_data_type) consensus_data_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_data_len, 0, consensus_msg.consensus_data_len()) size += ont_constants.ONT_INT_LEN buf.append(consensus_data_len) block_start_len = consensus_msg.block_start_len_memoryview() txn_header = consensus_msg.txn_header() block_start_len_and_txn_header_total_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", block_start_len_and_txn_header_total_len, 0, len(block_start_len) + len(txn_header)) size += ont_constants.ONT_INT_LEN buf.append(block_start_len_and_txn_header_total_len) size += len(block_start_len) buf.append(block_start_len) size += len(txn_header) buf.append(txn_header) max_timestamp_for_compression = time.time() - min_tx_age_seconds for tx in consensus_msg.txns(): tx_hash, _ = ont_messages_util.get_txid(tx) short_id = tx_service.get_short_id(tx_hash) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time(short_id) if short_id == constants.NULL_TX_SID or \ not enable_block_compression or \ short_id_assign_time > max_timestamp_for_compression: if short_id != constants.NULL_TX_SIDS: ignored_sids.append(short_id) buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 # Prepend owner and signature, consensus payload tail, tx count and block hash to bx_block owner_and_signature = consensus_msg.owner_and_signature() owner_and_signature_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", owner_and_signature_len, 0, len(owner_and_signature)) size += len(owner_and_signature) buf.appendleft(owner_and_signature) size += ont_constants.ONT_INT_LEN buf.appendleft(owner_and_signature_len) payload_tail = consensus_msg.payload_tail() payload_tail_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", payload_tail_len, 0, len(payload_tail)) size += len(payload_tail) buf.appendleft(payload_tail) size += ont_constants.ONT_INT_LEN buf.appendleft(payload_tail_len) txn_count = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", txn_count, 0, consensus_msg.txn_count()) size += ont_constants.ONT_INT_LEN buf.appendleft(txn_count) block_hash = consensus_msg.block_hash().binary size += ont_constants.ONT_HASH_LEN buf.appendleft(block_hash) is_consensus_msg_buf = struct.pack("?", True) buf.appendleft(is_consensus_msg_buf) size += 1 block = finalize_block_bytes(buf, size, short_ids) prev_block_hash = convert.bytes_to_hex(consensus_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo( consensus_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, consensus_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100, ignored_sids ) return memoryview(block), block_info
def block_to_bx_block( self, block_msg: OntConsensusMessage, tx_service: TransactionService ) -> Tuple[memoryview, BlockInfo]: """ Compresses a Ontology consensus message's transactions and packs it into a bloXroute block. """ consensus_msg = block_msg compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] consensus_payload_header = consensus_msg.consensus_payload_header() consensus_payload_header_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_payload_header_len, 0, len(consensus_payload_header)) size += ont_constants.ONT_INT_LEN buf.append(consensus_payload_header_len) size += len(consensus_payload_header) buf.append(consensus_payload_header) consensus_data_type = bytearray(ont_constants.ONT_CHAR_LEN) struct.pack_into("<B", consensus_data_type, 0, consensus_msg.consensus_data_type()) size += ont_constants.ONT_CHAR_LEN buf.append(consensus_data_type) consensus_data_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_data_len, 0, consensus_msg.consensus_data_len()) size += ont_constants.ONT_INT_LEN buf.append(consensus_data_len) block_start_len = consensus_msg.block_start_len_memoryview() txn_header = consensus_msg.txn_header() block_start_len_and_txn_header_total_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", block_start_len_and_txn_header_total_len, 0, len(block_start_len) + len(txn_header)) size += ont_constants.ONT_INT_LEN buf.append(block_start_len_and_txn_header_total_len) size += len(block_start_len) buf.append(block_start_len) size += len(txn_header) buf.append(txn_header) for tx in consensus_msg.txns(): tx_hash, _ = ont_messages_util.get_txid(tx) short_id = tx_service.get_short_id(tx_hash) if short_id == constants.NULL_TX_SID: buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 # Prepend owner and signature, consensus payload tail, tx count and block hash to bx_block owner_and_signature = consensus_msg.owner_and_signature() owner_and_signature_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", owner_and_signature_len, 0, len(owner_and_signature)) size += len(owner_and_signature) buf.appendleft(owner_and_signature) size += ont_constants.ONT_INT_LEN buf.appendleft(owner_and_signature_len) payload_tail = consensus_msg.payload_tail() payload_tail_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", payload_tail_len, 0, len(payload_tail)) size += len(payload_tail) buf.appendleft(payload_tail) size += ont_constants.ONT_INT_LEN buf.appendleft(payload_tail_len) txn_count = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", txn_count, 0, consensus_msg.txn_count()) size += ont_constants.ONT_INT_LEN buf.appendleft(txn_count) block_hash = consensus_msg.block_hash().binary size += ont_constants.ONT_HASH_LEN buf.appendleft(block_hash) is_consensus_msg_buf = struct.pack("?", True) buf.appendleft(is_consensus_msg_buf) size += 1 serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off prev_block_hash = convert.bytes_to_hex(consensus_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) original_size = len(consensus_msg.rawbytes()) block_info = BlockInfo( consensus_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, consensus_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100 ) return memoryview(block), block_info
def bx_block_to_block(self, bx_block_msg, tx_service) -> BlockDecompressionResult: decompress_start_datetime = datetime.datetime.utcnow() decompress_start_timestamp = time.time() tsk = self.decompression_tasks.borrow_task() tsk.init(tpe.InputBytes(bx_block_msg), tx_service.proxy) try: task_pool_proxy.run_task(tsk) except tpe.AggregatedException as e: block_hash = Sha256Hash( convert.hex_to_bytes(tsk.block_hash().hex_string())) self.decompression_tasks.return_task(tsk) # TODO find a better solution raise message_conversion_error.eth_block_decompression_error( block_hash, e) total_tx_count = tsk.tx_count() unknown_tx_hashes = [ Sha256Hash(bytearray(unknown_tx_hash.binary())) for unknown_tx_hash in tsk.unknown_tx_hashes() ] unknown_tx_sids = tsk.unknown_tx_sids() block_hash = Sha256Hash( convert.hex_to_bytes(tsk.block_hash().hex_string())) if tsk.success(): starting_offset = tsk.starting_offset() block = memoryview(tsk.block_message())[starting_offset:] block_msg = InternalEthBlockInfo(block) content_size = len(block_msg.rawbytes()) logger.debug( "Successfully parsed block broadcast message. {} " "transactions in block {}", total_tx_count, block_hash) bx_block_hash = convert.bytes_to_hex( crypto.double_sha256(bx_block_msg)) compressed_size = len(bx_block_msg) block_info = BlockInfo( block_hash, tsk.short_ids(), decompress_start_datetime, datetime.datetime.utcnow(), (time.time() - decompress_start_timestamp) * 1000, total_tx_count, bx_block_hash, convert.bytes_to_hex(block_msg.prev_block_hash().binary), len(block_msg.rawbytes()), compressed_size, 100 - float(compressed_size) / content_size * 100) else: block_msg = None logger.debug( "Block recovery needed for {}. Missing {} sids, {} tx hashes. " "Total txs in block: {}", block_hash, len(unknown_tx_sids), len(unknown_tx_hashes), total_tx_count) block_info = BlockInfo( block_hash, tsk.short_ids(), decompress_start_datetime, datetime.datetime.utcnow(), (time.time() - decompress_start_timestamp) * 1000, None, None, None, None, None, None) self.decompression_tasks.return_task(tsk) return BlockDecompressionResult(block_msg, block_info, unknown_tx_sids, unknown_tx_hashes)