def block_to_bx_block( self, block_msg: OntConsensusMessage, tx_service: TransactionService, enable_block_compression: bool, min_tx_age_seconds: float ) -> Tuple[memoryview, BlockInfo]: """ Pack an Ontology consensus message's transactions into a bloXroute block. """ consensus_msg = block_msg compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] ignored_sids = [] original_size = len(consensus_msg.rawbytes()) consensus_payload_header = consensus_msg.consensus_payload_header() consensus_payload_header_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_payload_header_len, 0, len(consensus_payload_header)) size += ont_constants.ONT_INT_LEN buf.append(consensus_payload_header_len) size += len(consensus_payload_header) buf.append(consensus_payload_header) consensus_data_type = bytearray(ont_constants.ONT_CHAR_LEN) struct.pack_into("<B", consensus_data_type, 0, consensus_msg.consensus_data_type()) size += ont_constants.ONT_CHAR_LEN buf.append(consensus_data_type) consensus_data_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_data_len, 0, consensus_msg.consensus_data_len()) size += ont_constants.ONT_INT_LEN buf.append(consensus_data_len) block_start_len = consensus_msg.block_start_len_memoryview() txn_header = consensus_msg.txn_header() block_start_len_and_txn_header_total_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", block_start_len_and_txn_header_total_len, 0, len(block_start_len) + len(txn_header)) size += ont_constants.ONT_INT_LEN buf.append(block_start_len_and_txn_header_total_len) size += len(block_start_len) buf.append(block_start_len) size += len(txn_header) buf.append(txn_header) max_timestamp_for_compression = time.time() - min_tx_age_seconds for tx in consensus_msg.txns(): tx_hash, _ = ont_messages_util.get_txid(tx) short_id = tx_service.get_short_id(tx_hash) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time(short_id) if short_id == constants.NULL_TX_SID or \ not enable_block_compression or \ short_id_assign_time > max_timestamp_for_compression: if short_id != constants.NULL_TX_SIDS: ignored_sids.append(short_id) buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 # Prepend owner and signature, consensus payload tail, tx count and block hash to bx_block owner_and_signature = consensus_msg.owner_and_signature() owner_and_signature_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", owner_and_signature_len, 0, len(owner_and_signature)) size += len(owner_and_signature) buf.appendleft(owner_and_signature) size += ont_constants.ONT_INT_LEN buf.appendleft(owner_and_signature_len) payload_tail = consensus_msg.payload_tail() payload_tail_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", payload_tail_len, 0, len(payload_tail)) size += len(payload_tail) buf.appendleft(payload_tail) size += ont_constants.ONT_INT_LEN buf.appendleft(payload_tail_len) txn_count = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", txn_count, 0, consensus_msg.txn_count()) size += ont_constants.ONT_INT_LEN buf.appendleft(txn_count) block_hash = consensus_msg.block_hash().binary size += ont_constants.ONT_HASH_LEN buf.appendleft(block_hash) is_consensus_msg_buf = struct.pack("?", True) buf.appendleft(is_consensus_msg_buf) size += 1 block = finalize_block_bytes(buf, size, short_ids) prev_block_hash = convert.bytes_to_hex(consensus_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo( consensus_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, consensus_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100, ignored_sids ) return memoryview(block), block_info
def block_to_bx_block( self, block_msg: InternalEthBlockInfo, tx_service: TransactionService, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: """ Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids The code is optimized and does not make copies of bytes :param block_msg: Ethereum new block message :param tx_service: Transactions service :param enable_block_compression :param min_tx_age_seconds :return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash) """ compress_start_datetime = datetime.datetime.utcnow() compress_start_timestamp = time.time() txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message( block_msg) used_short_ids = [] # creating transactions content content_size = 0 buf = deque() ignored_sids = [] tx_start_index = 0 tx_count = 0 original_size = len(block_msg.rawbytes()) max_timestamp_for_compression = time.time() - min_tx_age_seconds while True: if tx_start_index >= len(txs_bytes): break _, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix( txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length] tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes) tx_hash = Sha256Hash(tx_hash_bytes) tx_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(tx_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time( short_id) if short_id <= constants.NULL_TX_SID or \ not enable_block_compression or short_id_assign_time > max_timestamp_for_compression: if short_id > constants.NULL_TX_SID: ignored_sids.append(short_id) is_full_tx_bytes = rlp_utils.encode_int(1) tx_content_bytes = tx_bytes else: is_full_tx_bytes = rlp_utils.encode_int(0) used_short_ids.append(short_id) tx_content_bytes = bytes() tx_content_prefix = rlp_utils.get_length_prefix_str( len(tx_content_bytes)) short_tx_content_size = len(is_full_tx_bytes) + len( tx_content_prefix) + len(tx_content_bytes) short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list( short_tx_content_size) buf.append(short_tx_content_prefix_bytes) buf.append(is_full_tx_bytes) buf.append(tx_content_prefix) buf.append(tx_content_bytes) content_size += len( short_tx_content_prefix_bytes) + short_tx_content_size tx_start_index = tx_item_start + tx_item_length tx_count += 1 list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(list_of_txs_prefix_bytes) content_size += len(list_of_txs_prefix_bytes) buf.appendleft(block_hdr_full_bytes) content_size += len(block_hdr_full_bytes) buf.append(remaining_bytes) content_size += len(remaining_bytes) compact_block_msg_prefix = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(compact_block_msg_prefix) content_size += len(compact_block_msg_prefix) block = finalize_block_bytes(buf, content_size, used_short_ids) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo(block_msg.block_hash(), used_short_ids, compress_start_datetime, datetime.datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tx_count, bx_block_hash, convert.bytes_to_hex(prev_block_bytes), original_size, content_size, 100 - float(content_size) / original_size * 100, ignored_sids) return memoryview(block), block_info
def _log_compressed_block_debug_info_eth(transaction_service: TransactionService, block_msg_bytes: Union[memoryview, bytearray]): is_block_relay = transaction_service.node.NODE_TYPE == NodeType.RELAY_BLOCK block_hash, short_ids, txs_bytes = _parse_block_eth(block_msg_bytes) # parse statistics variables short_tx_index = 0 tx_start_index = 0 tx_index_in_block = 0 txs_info = [] missing_short_ids = [] while True: if tx_start_index >= len(txs_bytes): break short_id = 0 has_contents = False assignmnet_time = 0 _, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len] is_full_tx_start = 0 is_full_tx, is_full_tx_len, = rlp_utils.decode_int(tx_bytes, is_full_tx_start) _, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix( tx_bytes, is_full_tx_start + is_full_tx_len) tx_content_bytes = tx_bytes[tx_content_start:tx_content_start + tx_content_len] if is_full_tx: tx_hash = Sha256Hash(eth_common_utils.keccak_hash(tx_content_bytes)) else: short_id = short_ids[short_tx_index] tx_hash, tx_bytes, _ = transaction_service.get_transaction(short_id) has_contents = tx_bytes is not None if tx_hash is not None: assignmnet_time = transaction_service.get_short_id_assign_time(short_id) short_tx_index += 1 if is_block_relay: txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash)) else: txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash, has_contents, assignmnet_time)) tx_index_in_block += 1 tx_start_index = tx_itm_start + tx_itm_len if not is_full_tx and not has_contents: missing_short_ids.append(short_id) if is_block_relay: log_message = \ "Block content (from block relay) {} from (index, is compressed, short id, hash is full) : {}" else: log_message = \ "Block content (full) {} (index, compressed, short id, hash, has contents, assignment time) : {}" logger.debug( log_message, block_hash, ",".join(str(tx_info) for tx_info in txs_info) ) node_type = transaction_service.node.NODE_TYPE assert node_type is not None log_can_decompress_block(node_type, block_hash, missing_short_ids)