def block_to_bx_block( self, block_msg, tx_service ) -> Tuple[memoryview, BlockInfo]: """ Compresses a Bitcoin block's transactions and packs it into a bloXroute block. """ compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] header = block_msg.header() size += len(header) buf.append(header) for tx in block_msg.txns(): tx_hash = btc_common_utils.get_txid(tx) short_id = tx_service.get_short_id(tx_hash) if short_id == constants.NULL_TX_SID: buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(btc_constants.BTC_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off prev_block_hash = convert.bytes_to_hex(block_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) original_size = len(block_msg.rawbytes()) block_info = BlockInfo( block_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, block_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100 ) return memoryview(block), block_info
def broadcast_key_pair(short_ids=None, network_num=0): if short_ids is None: short_ids = [1, 10, 99, 187] broadcast_message_hash = Sha256Hash(helpers.generate_bytearray(crypto.SHA256_HASH_LEN)) broadcast_message_bytes = bytearray(constants.UL_ULL_SIZE_IN_BYTES) broadcast_message_bytes.extend(helpers.generate_bytearray(500)) struct.pack_into("<Q", broadcast_message_bytes, 0, len(broadcast_message_bytes)) broadcast_message_bytes.extend(compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids)) key_bytes, enc_broadcast_message_bytes = crypto.symmetric_encrypt(bytes(broadcast_message_bytes)) broadcast_message = BroadcastMessage(broadcast_message_hash, network_num, "", True, enc_broadcast_message_bytes) key_message = KeyMessage(broadcast_message_hash, network_num, "", key_bytes) return broadcast_message, key_message
def test_serialize_and_deserialize_short_ids(self): dummy_short_ids = [1, 203, 997, 890333, 5] expected_bytes_len = compact_block_short_ids_serializer.get_serialized_short_ids_bytes_len(dummy_short_ids) buffer = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(dummy_short_ids) deserialized_short_ids, deserialized_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer( buffer, 0) self.assertEqual(expected_bytes_len, len(buffer)) self.assertEqual(expected_bytes_len, deserialized_bytes_len) self.assertEqual(dummy_short_ids, deserialized_short_ids)
def finalize_block_bytes(buf: Deque[Union[bytes, bytearray, memoryview]], size: int, short_ids: List[int]) -> memoryview: serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes( short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off return memoryview(block)
def block_to_bx_block( self, block_msg: InternalEthBlockInfo, tx_service: TransactionService) -> Tuple[memoryview, BlockInfo]: """ Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids The code is optimized and does not make copies of bytes :param block_msg: Ethereum new block message :param tx_service: Transactions service :return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash) """ compress_start_datetime = datetime.datetime.utcnow() compress_start_timestamp = time.time() msg_bytes = memoryview(block_msg.rawbytes()) _, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix( msg_bytes, 0) block_msg_bytes = msg_bytes[block_msg_itm_start:block_msg_itm_start + block_msg_itm_len] _, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix( block_msg_bytes, 0) block_hdr_full_bytes = block_msg_bytes[0:block_hdr_itm_start + block_hdr_itm_len] block_hdr_bytes = block_msg_bytes[ block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len] _, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix( block_hdr_bytes, 0) prev_block_bytes = block_hdr_bytes[ prev_block_itm_start:prev_block_itm_start + prev_block_itm_len] _, txs_itm_len, txs_itm_start = rlp_utils.consume_length_prefix( block_msg_bytes, block_hdr_itm_start + block_hdr_itm_len) txs_bytes = block_msg_bytes[txs_itm_start:txs_itm_start + txs_itm_len] remaining_bytes = block_msg_bytes[txs_itm_start + txs_itm_len:] used_short_ids = [] # creating transactions content content_size = 0 buf = deque() tx_start_index = 0 tx_count = 0 while True: if tx_start_index >= len(txs_bytes): break _, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix( txs_bytes, tx_start_index) tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length] tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes) tx_hash = Sha256Hash(tx_hash_bytes) short_id = tx_service.get_short_id(tx_hash) if short_id <= 0: is_full_tx_bytes = rlp_utils.encode_int(1) tx_content_bytes = tx_bytes else: is_full_tx_bytes = rlp_utils.encode_int(0) used_short_ids.append(short_id) tx_content_bytes = bytes() tx_content_prefix = rlp_utils.get_length_prefix_str( len(tx_content_bytes)) short_tx_content_size = len(is_full_tx_bytes) + len( tx_content_prefix) + len(tx_content_bytes) short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list( short_tx_content_size) buf.append(short_tx_content_prefix_bytes) buf.append(is_full_tx_bytes) buf.append(tx_content_prefix) buf.append(tx_content_bytes) content_size += len( short_tx_content_prefix_bytes) + short_tx_content_size tx_start_index = tx_item_start + tx_item_length tx_count += 1 list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(list_of_txs_prefix_bytes) content_size += len(list_of_txs_prefix_bytes) buf.appendleft(block_hdr_full_bytes) content_size += len(block_hdr_full_bytes) buf.append(remaining_bytes) content_size += len(remaining_bytes) compact_block_msg_prefix = rlp_utils.get_length_prefix_list( content_size) buf.appendleft(compact_block_msg_prefix) content_size += len(compact_block_msg_prefix) short_ids_bytes = compact_block_short_ids_serializer.serialize_short_ids_into_bytes( used_short_ids) buf.append(short_ids_bytes) content_size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", content_size) buf.appendleft(offset_buf) content_size += len(short_ids_bytes) # Parse it into the bloXroute message format and send it along block = bytearray(content_size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) original_size = len(block_msg.rawbytes()) block_info = BlockInfo(block_msg.block_hash(), used_short_ids, compress_start_datetime, datetime.datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, tx_count, bx_block_hash, convert.bytes_to_hex(prev_block_bytes), original_size, content_size, 100 - float(content_size) / original_size * 100) return memoryview(block), block_info
def block_to_bx_block( self, block_msg, tx_service, enable_block_compression: bool, min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]: """ Pack an Ontology block's transactions into a bloXroute block. """ compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] original_size = len(block_msg.rawbytes()) header = block_msg.txn_header() size += len(header) buf.append(header) max_timestamp_for_compression = time.time() - min_tx_age_seconds ignored_sids = [] for tx in block_msg.txns(): tx_hash, _ = ont_messages_util.get_txid(tx) transaction_key = tx_service.get_transaction_key(tx_hash) short_id = tx_service.get_short_id_by_key(transaction_key) short_id_assign_time = 0 if short_id != constants.NULL_TX_SID: short_id_assign_time = tx_service.get_short_id_assign_time( short_id) if short_id == constants.NULL_TX_SID or \ not enable_block_compression or \ short_id_assign_time > max_timestamp_for_compression: if short_id != constants.NULL_TX_SIDS: ignored_sids.append(ignored_sids) buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes( short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES merkle_root = block_msg.merkle_root() buf.appendleft(merkle_root) size += ont_constants.ONT_HASH_LEN is_consensus_msg_buf = struct.pack("?", False) buf.appendleft(is_consensus_msg_buf) size += 1 offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off prev_block_hash = convert.bytes_to_hex( block_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) block_info = BlockInfo(block_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, block_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100, ignored_sids) return memoryview(block), block_info
def test_bx_block_to_block__full_txs_success(self): tx_count = 150 txs = [] short_txs = [] short_ids = [] for i in range(1, tx_count): tx = mock_eth_messages.get_dummy_transaction(1) txs.append(tx) tx_bytes = rlp.encode(tx, Transaction) if i % 2 == 0: tx_hash = hashlib.sha256(tx_bytes).digest() self.tx_service.assign_short_id(tx_hash, i) self.tx_service.set_transaction_contents(tx_hash, tx_bytes) short_tx = ShortTransaction(0, bytes()) short_txs.append(short_tx) short_ids.append(i) else: short_tx = ShortTransaction(1, tx_bytes) short_txs.append(short_tx) dummy_chain_difficulty = 20 dummy_block_number = 10 compact_block = CompactBlock(mock_eth_messages.get_dummy_block_header(8), short_txs, [ mock_eth_messages.get_dummy_block_header(2), mock_eth_messages.get_dummy_block_header(3) ], dummy_chain_difficulty, dummy_block_number) compact_block_msg_bytes = bytearray(constants.UL_ULL_SIZE_IN_BYTES) compact_block_msg_bytes.extend(rlp.encode(compact_block, CompactBlock)) short_ids_offset = len(compact_block_msg_bytes) struct.pack_into("<Q", compact_block_msg_bytes, 0, short_ids_offset) compact_block_bytes = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids) compact_block_msg_bytes.extend(compact_block_bytes) compact_block_hash_bytes = hashlib.sha256(compact_block_msg_bytes).digest() compact_block_hash = Sha256Hash(compact_block_hash_bytes) bx_block_msg = BroadcastMessage(compact_block_hash, self.test_network_num, is_encrypted=True, blob=compact_block_msg_bytes) block_msg, block_info, unknown_tx_sids, unknown_tx_hashes = self.eth_message_converter.bx_block_to_block( bx_block_msg.blob(), self.tx_service) self.assertTrue(block_msg) self.assertIsInstance(block_msg, InternalEthBlockInfo) new_block_msg = block_msg.to_new_block_msg() block = new_block_msg.get_block() self._assert_values_equal(compact_block.header, block.header) self._assert_values_equal(compact_block.uncles, block.uncles) self.assertEqual(len(compact_block.transactions), len(block.transactions)) for block_tx, i in zip(block.transactions, range(0, tx_count - 1)): self.assertIsInstance(block_tx, Transaction) self._assert_values_equal(block_tx, txs[i]) self.assertEqual(compact_block.chain_difficulty, new_block_msg.chain_difficulty)
def block_to_bx_block( self, block_msg: OntConsensusMessage, tx_service: TransactionService ) -> Tuple[memoryview, BlockInfo]: """ Compresses a Ontology consensus message's transactions and packs it into a bloXroute block. """ consensus_msg = block_msg compress_start_datetime = datetime.utcnow() compress_start_timestamp = time.time() size = 0 buf = deque() short_ids = [] consensus_payload_header = consensus_msg.consensus_payload_header() consensus_payload_header_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_payload_header_len, 0, len(consensus_payload_header)) size += ont_constants.ONT_INT_LEN buf.append(consensus_payload_header_len) size += len(consensus_payload_header) buf.append(consensus_payload_header) consensus_data_type = bytearray(ont_constants.ONT_CHAR_LEN) struct.pack_into("<B", consensus_data_type, 0, consensus_msg.consensus_data_type()) size += ont_constants.ONT_CHAR_LEN buf.append(consensus_data_type) consensus_data_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", consensus_data_len, 0, consensus_msg.consensus_data_len()) size += ont_constants.ONT_INT_LEN buf.append(consensus_data_len) block_start_len = consensus_msg.block_start_len_memoryview() txn_header = consensus_msg.txn_header() block_start_len_and_txn_header_total_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", block_start_len_and_txn_header_total_len, 0, len(block_start_len) + len(txn_header)) size += ont_constants.ONT_INT_LEN buf.append(block_start_len_and_txn_header_total_len) size += len(block_start_len) buf.append(block_start_len) size += len(txn_header) buf.append(txn_header) for tx in consensus_msg.txns(): tx_hash, _ = ont_messages_util.get_txid(tx) short_id = tx_service.get_short_id(tx_hash) if short_id == constants.NULL_TX_SID: buf.append(tx) size += len(tx) else: short_ids.append(short_id) buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY) size += 1 # Prepend owner and signature, consensus payload tail, tx count and block hash to bx_block owner_and_signature = consensus_msg.owner_and_signature() owner_and_signature_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", owner_and_signature_len, 0, len(owner_and_signature)) size += len(owner_and_signature) buf.appendleft(owner_and_signature) size += ont_constants.ONT_INT_LEN buf.appendleft(owner_and_signature_len) payload_tail = consensus_msg.payload_tail() payload_tail_len = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", payload_tail_len, 0, len(payload_tail)) size += len(payload_tail) buf.appendleft(payload_tail) size += ont_constants.ONT_INT_LEN buf.appendleft(payload_tail_len) txn_count = bytearray(ont_constants.ONT_INT_LEN) struct.pack_into("<L", txn_count, 0, consensus_msg.txn_count()) size += ont_constants.ONT_INT_LEN buf.appendleft(txn_count) block_hash = consensus_msg.block_hash().binary size += ont_constants.ONT_HASH_LEN buf.appendleft(block_hash) is_consensus_msg_buf = struct.pack("?", True) buf.appendleft(is_consensus_msg_buf) size += 1 serialized_short_ids = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(short_ids) buf.append(serialized_short_ids) size += constants.UL_ULL_SIZE_IN_BYTES offset_buf = struct.pack("<Q", size) buf.appendleft(offset_buf) size += len(serialized_short_ids) block = bytearray(size) off = 0 for blob in buf: next_off = off + len(blob) block[off:next_off] = blob off = next_off prev_block_hash = convert.bytes_to_hex(consensus_msg.prev_block_hash().binary) bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block)) original_size = len(consensus_msg.rawbytes()) block_info = BlockInfo( consensus_msg.block_hash(), short_ids, compress_start_datetime, datetime.utcnow(), (time.time() - compress_start_timestamp) * 1000, consensus_msg.txn_count(), bx_block_hash, prev_block_hash, original_size, size, 100 - float(size) / original_size * 100 ) return memoryview(block), block_info