def _parse_block_parameters(
self, full_block_header_bytes: Union[bytearray,
memoryview]) -> BlockParameters:
_, block_header_len, block_header_start = rlp_utils.consume_length_prefix(
full_block_header_bytes, 0)
block_header_bytes = full_block_header_bytes[
block_header_start:block_header_start + block_header_len]
block_hash = self._get_block_hash(full_block_header_bytes)
offset = eth_common_constants.FIXED_LENGTH_FIELD_OFFSET
difficulty, difficulty_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += difficulty_length
number, _ = rlp_utils.decode_int(block_header_bytes, offset)
_gas_limit, gas_limit_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += gas_limit_length
_gas_used, gas_used_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += gas_used_length
_timestamp, timestamp_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += timestamp_length
_, extra_data_length, extra_data_start = rlp_utils.consume_length_prefix(
block_header_bytes, offset)
offset = extra_data_start + extra_data_length
_, mix_hash_length, mix_hash_start = rlp_utils.consume_length_prefix(
block_header_bytes, offset)
mix_hash = block_header_bytes[mix_hash_start:mix_hash_start +
mix_hash_length]
offset = mix_hash_start + mix_hash_length
_, nonce_length, nonce_start = rlp_utils.consume_length_prefix(
block_header_bytes, offset)
nonce = block_header_bytes[nonce_start:nonce_start + nonce_length]
return BlockParameters(block_hash, number, difficulty, mix_hash, nonce)
def difficulty(self) -> int:
"""
:return: seconds since epoch
"""
if self._difficulty is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
difficulty, _difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
self._difficulty = difficulty
difficulty = self._difficulty
assert difficulty is not None
return difficulty
def parse_block_message(block_msg: InternalEthBlockInfo):
msg_bytes = memoryview(block_msg.rawbytes())
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
msg_bytes, 0)
block_msg_bytes = msg_bytes[block_msg_itm_start:block_msg_itm_start +
block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_full_bytes = block_msg_bytes[0:block_hdr_itm_start +
block_hdr_itm_len]
block_hdr_bytes = block_msg_bytes[block_hdr_itm_start:block_hdr_itm_start +
block_hdr_itm_len]
_, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix(
block_hdr_bytes, 0)
prev_block_bytes = block_hdr_bytes[
prev_block_itm_start:prev_block_itm_start + prev_block_itm_len]
_, txs_itm_len, txs_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_hdr_itm_start + block_hdr_itm_len)
txs_bytes = block_msg_bytes[txs_itm_start:txs_itm_start + txs_itm_len]
remaining_bytes = block_msg_bytes[txs_itm_start + txs_itm_len:]
return txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes
def _parse_block_eth(block_msg_bytes: Union[bytearray, memoryview]) -> Tuple[Sha256Hash, List[int], memoryview]:
block_msg_bytes = block_msg_bytes if isinstance(block_msg_bytes, memoryview) else memoryview(block_msg_bytes)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(block_msg_bytes)
short_ids, _short_ids_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
block_msg_bytes,
block_offsets.short_id_offset
)
block_bytes = block_msg_bytes[block_offsets.block_begin_offset: block_offsets.short_id_offset]
_, _block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(block_bytes, 0)
block_itm_bytes = block_bytes[block_itm_start:]
_, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix(block_itm_bytes, 0)
full_hdr_bytes = block_itm_bytes[0:block_hdr_start + block_hdr_len]
block_hash_bytes = eth_common_utils.keccak_hash(full_hdr_bytes)
block_hash = Sha256Hash(block_hash_bytes)
_, block_txs_len, block_txs_start = rlp_utils.consume_length_prefix(
block_itm_bytes, block_hdr_start + block_hdr_len
)
txs_bytes = block_itm_bytes[block_txs_start:block_txs_start + block_txs_len]
return block_hash, short_ids, txs_bytes
def prev_block_hash(self) -> Sha256Hash:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[
block_msg_itm_start:block_msg_itm_start + block_itm_len]
_, diff_itm_len, diff_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_itm_start + block_itm_len)
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
block_hdr_bytes = block_itm_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
_, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix(
block_hdr_bytes, 0)
prev_block_bytes = block_hdr_bytes[
prev_block_itm_start:prev_block_itm_start + prev_block_itm_len]
return Sha256Hash(prev_block_bytes)
def timestamp(self) -> int:
"""
:return: seconds since epoch
"""
timestamp = self._timestamp
if timestamp is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(self._memory_view, 0)
block_msg_bytes = self._memory_view[block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[block_msg_itm_start:block_msg_itm_start + block_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(block_itm_bytes, 0)
block_hdr_bytes = block_itm_bytes[block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += difficulty_length
_number, number_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += number_length
_gas_limit, gas_limit_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += gas_limit_length
_gas_used, gas_used_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += gas_used_length
timestamp, _timestamp_length = rlp_utils.decode_int(block_hdr_bytes, offset)
self._timestamp = timestamp
return timestamp
def number(self) -> int:
"""
:return: block height
"""
number = self._number
if number is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[block_itm_start:block_itm_start +
block_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
block_hdr_bytes = block_itm_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
offset += difficulty_length
number, _ = rlp_utils.decode_int(block_hdr_bytes, offset)
self._number = number
return number
def number(self) -> int:
"""
:return: block height
"""
if self._number is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[
block_msg_itm_start:block_msg_itm_start + block_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
block_hdr_bytes = block_itm_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
offset += difficulty_length
self._number, _ = rlp_utils.decode_int(block_hdr_bytes, offset)
assert self._number is not None
# pyre-fixme[7]: Expected `int` but got `Optional[int]`.
return self._number
def from_new_block_msg(
cls, new_block_msg: NewBlockEthProtocolMessage
) -> "InternalEthBlockInfo":
"""
Creates NewBlockInternalEthMessage from raw bytes of NewBlockEthProtocolMessage
:param new_block_msg: new block message
:return: NewBlockInternalEthMessage message
"""
new_block_msg_bytes = memoryview(new_block_msg.rawbytes())
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
new_block_msg_bytes, 0)
block_msg_bytes = new_block_msg_bytes[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
msg_size = 0
# block item already include header, transactions and uncles
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[
block_msg_itm_start:block_msg_itm_start + block_itm_len]
msg_size += len(block_itm_bytes)
difficulty_bytes = block_msg_bytes[block_msg_itm_start +
block_itm_len:]
msg_size += len(difficulty_bytes)
block_number_bytes = rlp_utils.encode_int(new_block_msg.number())
msg_size += len(block_number_bytes)
msg_prefix = rlp_utils.get_length_prefix_list(
len(block_itm_bytes) + len(difficulty_bytes) +
len(block_number_bytes))
msg_size += len(msg_prefix)
msg_bytes = bytearray(msg_size)
written_bytes = 0
msg_bytes[written_bytes:written_bytes + len(msg_prefix)] = msg_prefix
written_bytes += len(msg_prefix)
msg_bytes[written_bytes:written_bytes +
len(block_itm_bytes)] = block_itm_bytes
written_bytes += len(block_itm_bytes)
msg_bytes[written_bytes:written_bytes +
len(difficulty_bytes)] = difficulty_bytes
written_bytes += len(difficulty_bytes)
msg_bytes[written_bytes:written_bytes +
len(block_number_bytes)] = block_number_bytes
written_bytes += len(block_number_bytes)
assert written_bytes == msg_size
return cls(msg_bytes)
def get_chain_difficulty(self) -> int:
chain_difficulty = self._chain_difficulty
if chain_difficulty is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(self._memory_view, 0)
block_msg_bytes = self._memory_view[block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(block_msg_bytes, 0)
chain_difficulty, _ = rlp_utils.decode_int(block_msg_bytes, block_itm_start + block_itm_len)
self._chain_difficulty = chain_difficulty
return chain_difficulty
def raw_tx_gas_price(tx_bytes: memoryview, tx_start_index: int) -> int:
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
tx_bytes, tx_start_index)
tx_bytes = tx_bytes[tx_item_start:tx_item_start + tx_item_length]
# gas_price is the second field, need to skip the first field (nonce)
_, nonce_item_length, nonce_item_start = rlp_utils.consume_length_prefix(
tx_bytes, 0)
gas_price, _ = rlp_utils.decode_int(tx_bytes,
nonce_item_start + nonce_item_length)
return gas_price
def block_header(self) -> memoryview:
block_header = self._block_header
if block_header is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(self._memory_view, 0)
block_msg_bytes = self._memory_view[block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[block_msg_itm_start:block_msg_itm_start + block_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(block_itm_bytes, 0)
block_header = block_itm_bytes[0:block_hdr_itm_start + block_hdr_itm_len]
self._block_header = block_header
return block_header
def get_previous_block_hash(self) -> Optional[Sha256Hash]:
if self.block_header_bytes is None:
return None
_, header_items_len, header_items_start = rlp_utils.consume_length_prefix(
self.block_header_bytes, 0)
header_items_bytes = self.block_header_bytes[header_items_start:]
_, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix(
header_items_bytes, 0)
prev_block_bytes = header_items_bytes[
prev_block_itm_start:prev_block_itm_start + prev_block_itm_len]
return Sha256Hash(prev_block_bytes)
def has_total_difficulty(self) -> bool:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_header_len, block_header_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
_, txs_len, txs_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_header_start + block_header_len)
_, uncles_len, uncles_start = rlp_utils.consume_length_prefix(
block_msg_bytes, txs_start + txs_len)
chain_difficulty, _ = rlp_utils.decode_int(block_msg_bytes,
uncles_start + uncles_len)
return chain_difficulty > 0
def from_new_block_parts(
cls,
new_block_details: NewBlockParts,
total_difficulty: Optional[int] = 0) -> "InternalEthBlockInfo":
"""
Creates NewBlockInternalEthMessage from block header and block body bytes
:param new_block_details: new block details
:param total_difficulty: total difficulty of the block if known
:return: instance of NewBlockInternalEthMessage
"""
block_header_bytes = new_block_details.block_header_bytes
block_body_bytes = new_block_details.block_body_bytes
block_number = new_block_details.block_number
# block body content includes transactions and uncles
_, block_content_len, block_content_start = rlp_utils.consume_length_prefix(
block_body_bytes, 0)
block_body_conent = block_body_bytes[block_content_start:]
msg_size = len(block_header_bytes) + len(block_body_conent)
total_difficulty_item = rlp_utils.encode_int(
0 if total_difficulty is None else total_difficulty)
msg_size += len(total_difficulty_item)
block_number_bytes = rlp_utils.encode_int(block_number)
msg_size += len(block_number_bytes)
new_block_item_prefix = rlp_utils.get_length_prefix_list(msg_size)
msg_size += len(new_block_item_prefix)
written_bytes = 0
new_block_bytes = bytearray(msg_size)
new_block_bytes[written_bytes:written_bytes +
len(new_block_item_prefix)] = new_block_item_prefix
written_bytes += len(new_block_item_prefix)
new_block_bytes[written_bytes:written_bytes +
len(block_header_bytes)] = block_header_bytes
written_bytes += len(block_header_bytes)
new_block_bytes[written_bytes:written_bytes +
len(block_body_conent)] = block_body_conent
written_bytes += len(block_body_conent)
new_block_bytes[written_bytes:written_bytes +
len(total_difficulty_item)] = total_difficulty_item
written_bytes += len(total_difficulty_item)
new_block_bytes[written_bytes:written_bytes +
len(block_number_bytes)] = block_number_bytes
written_bytes += len(block_number_bytes)
assert msg_size == written_bytes
return cls(new_block_bytes)
def to_new_block_parts(self) -> NewBlockParts:
_, msg_itm_len, msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
msg_itm_bytes = self._memory_view[msg_itm_start:]
offset = 0
_, header_len, header_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
header_bytes = msg_itm_bytes[offset:header_start + header_len]
offset = header_start + header_len
_, txs_len, txs_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
txs_bytes = msg_itm_bytes[offset:txs_start + txs_len]
offset = txs_start + txs_len
_, uncles_len, uncles_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
uncles_bytes = msg_itm_bytes[offset:uncles_start + uncles_len]
offset = uncles_start + uncles_len
_, total_difficulty_len, total_difficulty_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
offset = total_difficulty_start + total_difficulty_len
block_number, _ = rlp_utils.decode_int(msg_itm_bytes, offset)
block_body_prefix = rlp_utils.get_length_prefix_list(
len(txs_bytes) + len(uncles_bytes))
block_body_bytes = bytearray(
len(block_body_prefix) + len(txs_bytes) + len(uncles_bytes))
block_body_bytes[:len(block_body_prefix)] = block_body_prefix
written_bytes = len(block_body_prefix)
block_body_bytes[written_bytes:written_bytes +
len(txs_bytes)] = txs_bytes
written_bytes += len(txs_bytes)
block_body_bytes[written_bytes:written_bytes +
len(uncles_bytes)] = uncles_bytes
written_bytes += len(uncles_bytes)
return NewBlockParts(header_bytes, block_body_bytes, block_number)
def block_header_difficulty(
full_block_header_bytes: Union[memoryview, bytearray]) -> int:
_, block_header_len, block_header_start = rlp_utils.consume_length_prefix(
full_block_header_bytes, 0)
block_header_bytes = full_block_header_bytes[
block_header_start:block_header_start + block_header_len]
offset = eth_common_constants.FIXED_LENGTH_FIELD_OFFSET
difficulty, _difficulty_length = rlp_utils.decode_int(
block_header_bytes, offset)
return difficulty
def _get_compressed_block_header_bytes(
self, compressed_block_bytes: Union[bytearray, memoryview]
) -> Union[bytearray, memoryview]:
block_msg_bytes = compressed_block_bytes if isinstance(
compressed_block_bytes,
memoryview) else memoryview(compressed_block_bytes)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
block_msg_bytes)
block_bytes = block_msg_bytes[
block_offsets.block_begin_offset:block_offsets.short_id_offset]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_bytes, 0)
block_itm_bytes = block_bytes[block_itm_start:]
_, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
return block_itm_bytes[0:block_hdr_start + block_hdr_len]
def normalize_typed_transaction(tx_bytes: memoryview) -> memoryview:
"""
RLP could be either [transaction_type][encoded transaction], or
RLP([transaction_type][encoded transaction]). The first form cannot be
`rlp.decode`d, as it will only return the transaction type or throw an
exception if strict=True. Force it to be of the second type.
"""
item_type, item_length, _item_start = rlp_utils.consume_length_prefix(tx_bytes, 0)
if item_type == str and item_length == 1:
tx_bytes = memoryview(rlp.encode(tx_bytes.tobytes()))
return tx_bytes
def get_block_difficulty(self) -> Optional[int]:
if self.block_header_bytes is None:
return None
_, header_items_len, header_items_start = rlp_utils.consume_length_prefix(
self.block_header_bytes, 0)
header_items_bytes = self.block_header_bytes[header_items_start:]
block_difficulty, _ = rlp_utils.decode_int(
header_items_bytes, BlockHeader.FIXED_LENGTH_FIELD_OFFSET)
return block_difficulty
def block_number(self) -> int:
if self._block_number is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
offset += difficulty_length
self._block_number, _ = rlp_utils.decode_int(
block_hdr_bytes, offset)
block_number = self._block_number
assert block_number is not None
return block_number
def tx_to_bx_txs(
self,
tx_msg,
network_num,
transaction_flag: Optional[TransactionFlag] = None,
min_tx_network_fee: int = 0,
account_id: str = common_constants.DECODED_EMPTY_ACCOUNT_ID
) -> List[Tuple[TxMessage, Sha256Hash, Union[bytearray, memoryview]]]:
"""
Converts Ethereum transactions message to array of internal transaction messages
The code is optimized and does not make copies of bytes
:param tx_msg: Ethereum transaction message
:param network_num: blockchain network number
:param transaction_flag: transaction flag to assign to the BDN transaction.
:param min_tx_network_fee: minimum transaction fee. excludes transactions with gas price below this value,
:param account_id: the account id of the gateway
:return: array of tuples (transaction message, transaction hash, transaction bytes)
"""
if not isinstance(tx_msg, TransactionsEthProtocolMessage):
raise TypeError(
f"TransactionsEthProtocolMessage is expected for arg tx_msg but was {type(tx_msg)}"
)
bx_tx_msgs = []
msg_bytes = memoryview(tx_msg.rawbytes())
_, length, start = rlp_utils.consume_length_prefix(msg_bytes, 0)
txs_bytes = msg_bytes[start:]
tx_start_index = 0
while True:
bx_tx, _, tx_item_length, tx_item_start = eth_common_utils.raw_tx_to_bx_tx(
txs_bytes, tx_start_index, network_num, transaction_flag,
account_id)
gas_price = eth_common_utils.raw_tx_gas_price(
txs_bytes, tx_start_index)
if gas_price >= min_tx_network_fee:
bx_tx_msgs.append(
(bx_tx, bx_tx.message_hash(), bx_tx.tx_val()))
tx_start_index = tx_item_start + tx_item_length
if tx_start_index == len(txs_bytes):
break
return bx_tx_msgs
def from_new_block_parts(
cls, new_block_parts: NewBlockParts,
total_difficulty: int) -> "NewBlockEthProtocolMessage":
block_header = memoryview(new_block_parts.block_header_bytes)
block_body = memoryview(new_block_parts.block_body_bytes)
_, block_content_len, block_content_start = rlp_utils.consume_length_prefix(
block_body, 0)
block_body_conent = block_body[block_content_start:]
content_size = len(block_header) + len(block_body_conent)
block_item_prefix = rlp_utils.get_length_prefix_list(content_size)
content_size += len(block_item_prefix)
total_difficulty_item = rlp_utils.encode_int(total_difficulty)
content_size += len(total_difficulty_item)
new_block_item_prefix = rlp_utils.get_length_prefix_list(content_size)
content_size += len(new_block_item_prefix)
written_bytes = 0
new_block_bytes = bytearray(content_size)
new_block_bytes[written_bytes:written_bytes +
len(new_block_item_prefix)] = new_block_item_prefix
written_bytes += len(new_block_item_prefix)
new_block_bytes[written_bytes:written_bytes +
len(block_item_prefix)] = block_item_prefix
written_bytes += len(block_item_prefix)
new_block_bytes[written_bytes:written_bytes +
len(block_header)] = block_header
written_bytes += len(block_header)
new_block_bytes[written_bytes:written_bytes +
len(block_body_conent)] = block_body_conent
written_bytes += len(block_body_conent)
new_block_bytes[written_bytes:written_bytes +
len(total_difficulty_item)] = total_difficulty_item
written_bytes += len(total_difficulty_item)
return cls(new_block_bytes)
def raw_tx_to_bx_tx(
tx_bytes: Union[bytearray, memoryview],
tx_start_index: int,
network_num: int,
transaction_flag: Optional[TransactionFlag] = None
) -> Tuple[TxMessage, int, int]:
if isinstance(tx_bytes, bytearray):
tx_bytes = memoryview(tx_bytes)
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
tx_bytes, tx_start_index)
tx_bytes = tx_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = keccak_hash(tx_bytes)
msg_hash = Sha256Hash(tx_hash_bytes)
bx_tx = TxMessage(message_hash=msg_hash,
network_num=network_num,
tx_val=tx_bytes,
transaction_flag=transaction_flag)
return bx_tx, tx_item_length, tx_item_start
def tx_to_bx_txs(
self,
tx_msg,
network_num,
quota_type: Optional[QuotaType] = None
) -> List[Tuple[TxMessage, Sha256Hash, Union[bytearray, memoryview]]]:
"""
Converts Ethereum transactions message to array of internal transaction messages
The code is optimized and does not make copies of bytes
:param tx_msg: Ethereum transaction message
:param network_num: blockchain network number
:param quota_type: the quota type to assign to the BDN transaction.
:return: array of tuples (transaction message, transaction hash, transaction bytes)
"""
if not isinstance(tx_msg, TransactionsEthProtocolMessage):
raise TypeError(
"TransactionsEthProtocolMessage is expected for arg tx_msg but was {0}"
.format(type(tx_msg)))
bx_tx_msgs = []
msg_bytes = memoryview(tx_msg.rawbytes())
_, length, start = rlp_utils.consume_length_prefix(msg_bytes, 0)
txs_bytes = msg_bytes[start:]
tx_start_index = 0
while True:
bx_tx, tx_item_length, tx_item_start = raw_tx_to_bx_tx(
txs_bytes, tx_start_index, network_num, quota_type)
bx_tx_msgs.append((bx_tx, bx_tx.message_hash(), bx_tx.tx_val()))
tx_start_index = tx_item_start + tx_item_length
if tx_start_index == len(txs_bytes):
break
return bx_tx_msgs
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:param enable_block_compression
:param min_tx_age_seconds
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message(
block_msg)
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
ignored_sids = []
tx_start_index = 0
tx_count = 0
original_size = len(block_msg.rawbytes())
max_timestamp_for_compression = time.time() - min_tx_age_seconds
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
tx_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(tx_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(
short_id)
if short_id <= constants.NULL_TX_SID or \
not enable_block_compression or short_id_assign_time > max_timestamp_for_compression:
if short_id > constants.NULL_TX_SID:
ignored_sids.append(short_id)
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
block = finalize_block_bytes(buf, content_size, used_short_ids)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100,
ignored_sids)
return memoryview(block), block_info
def bx_block_to_block(self, bx_block_msg,
tx_service) -> BlockDecompressionResult:
"""
Converts internal broadcast message to Ethereum new block message
The code is optimized and does not make copies of bytes
:param bx_block_msg: internal broadcast message bytes
:param tx_service: Transactions service
:return: tuple (new block message, block hash, unknown transaction short id, unknown transaction hashes)
"""
if not isinstance(bx_block_msg, (bytearray, memoryview)):
raise TypeError(
"Type bytearray is expected for arg block_bytes but was {0}".
format(type(bx_block_msg)))
decompress_start_datetime = datetime.datetime.utcnow()
decompress_start_timestamp = time.time()
block_msg_bytes = bx_block_msg if isinstance(
bx_block_msg, memoryview) else memoryview(bx_block_msg)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block_msg)
short_ids, short_ids_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block_msg, block_offsets.short_id_offset)
block_bytes = block_msg_bytes[
block_offsets.block_begin_offset:block_offsets.short_id_offset]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_bytes, 0)
block_itm_bytes = block_bytes[block_itm_start:]
_, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
full_hdr_bytes = block_itm_bytes[0:block_hdr_start + block_hdr_len]
block_hash_bytes = eth_common_utils.keccak_hash(full_hdr_bytes)
block_hash = Sha256Hash(block_hash_bytes)
_, block_txs_len, block_txs_start = rlp_utils.consume_length_prefix(
block_itm_bytes, block_hdr_start + block_hdr_len)
txs_bytes = block_itm_bytes[block_txs_start:block_txs_start +
block_txs_len]
remaining_bytes = block_itm_bytes[block_txs_start + block_txs_len:]
# parse statistics variables
short_tx_index = 0
unknown_tx_sids = []
unknown_tx_hashes = []
# creating transactions content
content_size = 0
buf = deque()
tx_count = 0
tx_start_index = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len]
is_full_tx_start = 0
is_full_tx, is_full_tx_len, = rlp_utils.decode_int(
tx_bytes, is_full_tx_start)
_, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix(
tx_bytes, is_full_tx_start + is_full_tx_len)
tx_content_bytes = tx_bytes[tx_content_start:tx_content_start +
tx_content_len]
if is_full_tx:
tx_bytes = tx_content_bytes
else:
short_id = short_ids[short_tx_index]
tx_hash, tx_bytes, _ = tx_service.get_transaction(short_id)
if tx_hash is None:
unknown_tx_sids.append(short_id)
elif tx_bytes is None:
unknown_tx_hashes.append(tx_hash)
short_tx_index += 1
if tx_bytes is not None and not unknown_tx_sids and not unknown_tx_hashes:
buf.append(tx_bytes)
content_size += len(tx_bytes)
tx_count += 1
tx_start_index = tx_itm_start + tx_itm_len
if not unknown_tx_sids and not unknown_tx_hashes:
txs_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(txs_prefix)
content_size += len(txs_prefix)
buf.appendleft(full_hdr_bytes)
content_size += len(full_hdr_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
msg_len_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(msg_len_prefix)
block_msg_bytes = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block_msg_bytes[off:next_off] = blob
off = next_off
block_msg = InternalEthBlockInfo(block_msg_bytes)
logger.debug(
"Successfully parsed block broadcast message. {} "
"transactions in block {}", tx_count, block_hash)
bx_block_hash = convert.bytes_to_hex(
crypto.double_sha256(bx_block_msg))
compressed_size = len(bx_block_msg)
block_info = BlockInfo(
block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000, tx_count,
bx_block_hash,
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()), compressed_size,
100 - float(compressed_size) / content_size * 100, [])
return BlockDecompressionResult(block_msg, block_info,
unknown_tx_sids, unknown_tx_hashes)
else:
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}", block_hash, len(unknown_tx_sids),
len(unknown_tx_hashes), tx_count)
return BlockDecompressionResult(
None,
BlockInfo(block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
None, None, None, None, None, None, []),
unknown_tx_sids, unknown_tx_hashes)
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
msg_bytes = memoryview(block_msg.rawbytes())
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
msg_bytes, 0)
block_msg_bytes = msg_bytes[block_msg_itm_start:block_msg_itm_start +
block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_full_bytes = block_msg_bytes[0:block_hdr_itm_start +
block_hdr_itm_len]
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
_, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix(
block_hdr_bytes, 0)
prev_block_bytes = block_hdr_bytes[
prev_block_itm_start:prev_block_itm_start + prev_block_itm_len]
_, txs_itm_len, txs_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_hdr_itm_start + block_hdr_itm_len)
txs_bytes = block_msg_bytes[txs_itm_start:txs_itm_start + txs_itm_len]
remaining_bytes = block_msg_bytes[txs_itm_start + txs_itm_len:]
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
tx_start_index = 0
tx_count = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
short_id = tx_service.get_short_id(tx_hash)
if short_id <= 0:
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
short_ids_bytes = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(
used_short_ids)
buf.append(short_ids_bytes)
content_size += constants.UL_ULL_SIZE_IN_BYTES
offset_buf = struct.pack("<Q", content_size)
buf.appendleft(offset_buf)
content_size += len(short_ids_bytes)
# Parse it into the bloXroute message format and send it along
block = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block[off:next_off] = blob
off = next_off
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
original_size = len(block_msg.rawbytes())
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100)
return memoryview(block), block_info
def test_consume_length_prefix(self):
self.assertRaises(ParseError, rlp_utils.consume_length_prefix, memoryview(b"\x81\x00"), 0)
self.assertEqual((str, 2, 1), rlp_utils.consume_length_prefix(memoryview(b"\x82"), 0))
self.assertEqual((str, 56, 2), rlp_utils.consume_length_prefix(memoryview(b"\xb8\x38"), 0))
self.assertEqual((str, 128, 2), rlp_utils.consume_length_prefix(memoryview(b"\xb8\x80"), 0))
self.assertEqual((str, 1024, 3), rlp_utils.consume_length_prefix(memoryview(b"\xb9\x04\x00"), 0))
self.assertEqual((list, 1, 1), rlp_utils.consume_length_prefix(memoryview(b"\xc1"), 0))
self.assertEqual((list, 56, 2), rlp_utils.consume_length_prefix(memoryview(b"\xf8\x38"), 0))
self.assertEqual((list, 128, 2), rlp_utils.consume_length_prefix(memoryview(b"\xf8\x80"), 0))
self.assertEqual((list, 1024, 3), rlp_utils.consume_length_prefix(memoryview(b"\xf9\x04\x00"), 0))
self.assertEqual((str, 2, 3), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\x82"), 2))
self.assertEqual((str, 56, 4), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\xb8\x38"), 2))
self.assertEqual((str, 128, 4), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\xb8\x80"), 2))
self.assertEqual((str, 1024, 5), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\xb9\x04\x00"), 2))
self.assertEqual((list, 1, 3), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\xc1"), 2))
self.assertEqual((list, 56, 4), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\xf8\x38"), 2))
self.assertEqual((list, 128, 4), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\xf8\x80"), 2))
self.assertEqual((list, 1024, 5), rlp_utils.consume_length_prefix(memoryview(b"\x00\x00\xf9\x04\x00"), 2))
self.assertRaises(TypeError, rlp_utils.consume_length_prefix, b"\x82", 0)
self.assertRaises(ValueError, rlp_utils.consume_length_prefix, memoryview(b"\x82"), None)
def to_new_block_msg(self) -> NewBlockEthProtocolMessage:
"""
Converts message to instance of NewBlockEthProtocolMessage
:return: instance of NewBlockEthProtocolMessage
"""
_, msg_itm_len, msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
msg_itm_bytes = self._memory_view[msg_itm_start:]
offset = 0
msg_size = 0
_, header_len, header_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
header_bytes = msg_itm_bytes[offset:header_start + header_len]
offset = header_start + header_len
msg_size += len(header_bytes)
_, txs_len, txs_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
txs_bytes = msg_itm_bytes[offset:txs_start + txs_len]
offset = txs_start + txs_len
msg_size += len(txs_bytes)
_, uncles_len, uncles_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
uncles_bytes = msg_itm_bytes[offset:uncles_start + uncles_len]
offset = uncles_start + uncles_len
msg_size += len(uncles_bytes)
_, total_difficulty_len, total_difficulty_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
total_difficulty_bytes = msg_itm_bytes[offset:total_difficulty_start +
total_difficulty_len]
msg_size += len(total_difficulty_bytes)
block_prefix = rlp_utils.get_length_prefix_list(
len(header_bytes) + len(txs_bytes) + len(uncles_bytes))
msg_size += len(block_prefix)
msg_prefix = rlp_utils.get_length_prefix_list(msg_size)
msg_size += len(msg_prefix)
result_msg_bytes = bytearray(msg_size)
written_bytes = 0
result_msg_bytes[written_bytes:written_bytes +
len(msg_prefix)] = msg_prefix
written_bytes += len(msg_prefix)
result_msg_bytes[written_bytes:written_bytes +
len(block_prefix)] = block_prefix
written_bytes += len(block_prefix)
result_msg_bytes[written_bytes:written_bytes +
len(header_bytes)] = header_bytes
written_bytes += len(header_bytes)
result_msg_bytes[written_bytes:written_bytes +
len(txs_bytes)] = txs_bytes
written_bytes += len(txs_bytes)
result_msg_bytes[written_bytes:written_bytes +
len(uncles_bytes)] = uncles_bytes
written_bytes += len(uncles_bytes)
result_msg_bytes[written_bytes:written_bytes +
len(total_difficulty_bytes)] = total_difficulty_bytes
written_bytes += len(total_difficulty_bytes)
assert written_bytes == msg_size
return NewBlockEthProtocolMessage(result_msg_bytes)
