def number(self) -> int:
"""
:return: block height
"""
number = self._number
if number is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[block_itm_start:block_itm_start +
block_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
block_hdr_bytes = block_itm_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
offset += difficulty_length
number, _ = rlp_utils.decode_int(block_hdr_bytes, offset)
self._number = number
return number
def _parse_block_parameters(
self, full_block_header_bytes: Union[bytearray,
memoryview]) -> BlockParameters:
_, block_header_len, block_header_start = rlp_utils.consume_length_prefix(
full_block_header_bytes, 0)
block_header_bytes = full_block_header_bytes[
block_header_start:block_header_start + block_header_len]
block_hash = self._get_block_hash(full_block_header_bytes)
offset = eth_common_constants.FIXED_LENGTH_FIELD_OFFSET
difficulty, difficulty_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += difficulty_length
number, _ = rlp_utils.decode_int(block_header_bytes, offset)
_gas_limit, gas_limit_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += gas_limit_length
_gas_used, gas_used_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += gas_used_length
_timestamp, timestamp_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += timestamp_length
_, extra_data_length, extra_data_start = rlp_utils.consume_length_prefix(
block_header_bytes, offset)
offset = extra_data_start + extra_data_length
_, mix_hash_length, mix_hash_start = rlp_utils.consume_length_prefix(
block_header_bytes, offset)
mix_hash = block_header_bytes[mix_hash_start:mix_hash_start +
mix_hash_length]
offset = mix_hash_start + mix_hash_length
_, nonce_length, nonce_start = rlp_utils.consume_length_prefix(
block_header_bytes, offset)
nonce = block_header_bytes[nonce_start:nonce_start + nonce_length]
return BlockParameters(block_hash, number, difficulty, mix_hash, nonce)
def timestamp(self) -> int:
"""
:return: seconds since epoch
"""
timestamp = self._timestamp
if timestamp is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(self._memory_view, 0)
block_msg_bytes = self._memory_view[block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[block_msg_itm_start:block_msg_itm_start + block_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(block_itm_bytes, 0)
block_hdr_bytes = block_itm_bytes[block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += difficulty_length
_number, number_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += number_length
_gas_limit, gas_limit_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += gas_limit_length
_gas_used, gas_used_length = rlp_utils.decode_int(block_hdr_bytes, offset)
offset += gas_used_length
timestamp, _timestamp_length = rlp_utils.decode_int(block_hdr_bytes, offset)
self._timestamp = timestamp
return timestamp
def number(self) -> int:
"""
:return: block height
"""
if self._number is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[
block_msg_itm_start:block_msg_itm_start + block_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
block_hdr_bytes = block_itm_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
offset += difficulty_length
self._number, _ = rlp_utils.decode_int(block_hdr_bytes, offset)
assert self._number is not None
# pyre-fixme[7]: Expected `int` but got `Optional[int]`.
return self._number
def block_header_number(
full_block_header_bytes: Union[memoryview, bytearray]) -> int:
_, block_header_len, block_header_start = rlp_utils.consume_length_prefix(
full_block_header_bytes, 0)
block_header_bytes = full_block_header_bytes[
block_header_start:block_header_start + block_header_len]
offset = eth_common_constants.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(
block_header_bytes, offset)
offset += difficulty_length
number, _ = rlp_utils.decode_int(block_header_bytes, offset)
return number
def difficulty(self) -> int:
"""
:return: seconds since epoch
"""
if self._difficulty is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
difficulty, _difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
self._difficulty = difficulty
difficulty = self._difficulty
assert difficulty is not None
return difficulty
def get_chain_difficulty(self) -> int:
chain_difficulty = self._chain_difficulty
if chain_difficulty is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(self._memory_view, 0)
block_msg_bytes = self._memory_view[block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(block_msg_bytes, 0)
chain_difficulty, _ = rlp_utils.decode_int(block_msg_bytes, block_itm_start + block_itm_len)
self._chain_difficulty = chain_difficulty
return chain_difficulty
def get_block_difficulty(self) -> Optional[int]:
if self.block_header_bytes is None:
return None
_, header_items_len, header_items_start = rlp_utils.consume_length_prefix(
self.block_header_bytes, 0)
header_items_bytes = self.block_header_bytes[header_items_start:]
block_difficulty, _ = rlp_utils.decode_int(
header_items_bytes, BlockHeader.FIXED_LENGTH_FIELD_OFFSET)
return block_difficulty
def raw_tx_gas_price(tx_bytes: memoryview, tx_start_index: int) -> int:
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
tx_bytes, tx_start_index)
tx_bytes = tx_bytes[tx_item_start:tx_item_start + tx_item_length]
# gas_price is the second field, need to skip the first field (nonce)
_, nonce_item_length, nonce_item_start = rlp_utils.consume_length_prefix(
tx_bytes, 0)
gas_price, _ = rlp_utils.decode_int(tx_bytes,
nonce_item_start + nonce_item_length)
return gas_price
def test_decode_int(self):
encoded_0 = memoryview(rlp_utils.encode_int(0))
decoded_0 = rlp_utils.decode_int(encoded_0, 0)
self.assertEqual(0, decoded_0[0])
self.assertEqual(1, decoded_0[1])
encoded_127 = memoryview(rlp_utils.encode_int(127))
decoded_127 = rlp_utils.decode_int(encoded_127, 0)
self.assertEqual(127, decoded_127[0])
self.assertEqual(1, decoded_127[1])
encoded_128 = memoryview(rlp_utils.encode_int(128))
decoded_128 = rlp_utils.decode_int(encoded_128, 0)
self.assertEqual(128, decoded_128[0])
self.assertEqual(2, decoded_128[1])
encoded_256 = memoryview(rlp_utils.encode_int(256))
decoded_256 = rlp_utils.decode_int(encoded_256, 0)
self.assertEqual(256, decoded_256[0])
self.assertEqual(3, decoded_256[1])
encoded_1M = memoryview(rlp_utils.encode_int(1000000))
decoded_1M = rlp_utils.decode_int(encoded_1M, 0)
self.assertEqual(1000000, decoded_1M[0])
self.assertEqual(4, decoded_1M[1])
encoded_1M_lpad = memoryview(b"0" + rlp_utils.encode_int(1000000))
decoded_1M_lpad = rlp_utils.decode_int(encoded_1M_lpad, 1)
self.assertEqual(1000000, decoded_1M_lpad[0])
self.assertEqual(4, decoded_1M_lpad[1])
def block_number(self) -> int:
if self._block_number is None:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
offset = BlockHeader.FIXED_LENGTH_FIELD_OFFSET
_difficulty, difficulty_length = rlp_utils.decode_int(
block_hdr_bytes, offset)
offset += difficulty_length
self._block_number, _ = rlp_utils.decode_int(
block_hdr_bytes, offset)
block_number = self._block_number
assert block_number is not None
return block_number
def has_total_difficulty(self) -> bool:
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
block_msg_bytes = self._memory_view[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
_, block_header_len, block_header_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
_, txs_len, txs_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_header_start + block_header_len)
_, uncles_len, uncles_start = rlp_utils.consume_length_prefix(
block_msg_bytes, txs_start + txs_len)
chain_difficulty, _ = rlp_utils.decode_int(block_msg_bytes,
uncles_start + uncles_len)
return chain_difficulty > 0
def to_new_block_parts(self) -> NewBlockParts:
_, msg_itm_len, msg_itm_start = rlp_utils.consume_length_prefix(
self._memory_view, 0)
msg_itm_bytes = self._memory_view[msg_itm_start:]
offset = 0
_, header_len, header_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
header_bytes = msg_itm_bytes[offset:header_start + header_len]
offset = header_start + header_len
_, txs_len, txs_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
txs_bytes = msg_itm_bytes[offset:txs_start + txs_len]
offset = txs_start + txs_len
_, uncles_len, uncles_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
uncles_bytes = msg_itm_bytes[offset:uncles_start + uncles_len]
offset = uncles_start + uncles_len
_, total_difficulty_len, total_difficulty_start = rlp_utils.consume_length_prefix(
msg_itm_bytes, offset)
offset = total_difficulty_start + total_difficulty_len
block_number, _ = rlp_utils.decode_int(msg_itm_bytes, offset)
block_body_prefix = rlp_utils.get_length_prefix_list(
len(txs_bytes) + len(uncles_bytes))
block_body_bytes = bytearray(
len(block_body_prefix) + len(txs_bytes) + len(uncles_bytes))
block_body_bytes[:len(block_body_prefix)] = block_body_prefix
written_bytes = len(block_body_prefix)
block_body_bytes[written_bytes:written_bytes +
len(txs_bytes)] = txs_bytes
written_bytes += len(txs_bytes)
block_body_bytes[written_bytes:written_bytes +
len(uncles_bytes)] = uncles_bytes
written_bytes += len(uncles_bytes)
return NewBlockParts(header_bytes, block_body_bytes, block_number)
def bx_block_to_block(self, bx_block_msg,
tx_service) -> BlockDecompressionResult:
"""
Converts internal broadcast message to Ethereum new block message
The code is optimized and does not make copies of bytes
:param bx_block_msg: internal broadcast message bytes
:param tx_service: Transactions service
:return: tuple (new block message, block hash, unknown transaction short id, unknown transaction hashes)
"""
if not isinstance(bx_block_msg, (bytearray, memoryview)):
raise TypeError(
"Type bytearray is expected for arg block_bytes but was {0}".
format(type(bx_block_msg)))
decompress_start_datetime = datetime.datetime.utcnow()
decompress_start_timestamp = time.time()
block_msg_bytes = bx_block_msg if isinstance(
bx_block_msg, memoryview) else memoryview(bx_block_msg)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block_msg)
short_ids, short_ids_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block_msg, block_offsets.short_id_offset)
block_bytes = block_msg_bytes[
block_offsets.block_begin_offset:block_offsets.short_id_offset]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_bytes, 0)
block_itm_bytes = block_bytes[block_itm_start:]
_, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
full_hdr_bytes = block_itm_bytes[0:block_hdr_start + block_hdr_len]
block_hash_bytes = eth_common_utils.keccak_hash(full_hdr_bytes)
block_hash = Sha256Hash(block_hash_bytes)
_, block_txs_len, block_txs_start = rlp_utils.consume_length_prefix(
block_itm_bytes, block_hdr_start + block_hdr_len)
txs_bytes = block_itm_bytes[block_txs_start:block_txs_start +
block_txs_len]
remaining_bytes = block_itm_bytes[block_txs_start + block_txs_len:]
# parse statistics variables
short_tx_index = 0
unknown_tx_sids = []
unknown_tx_hashes = []
# creating transactions content
content_size = 0
buf = deque()
tx_count = 0
tx_start_index = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len]
is_full_tx_start = 0
is_full_tx, is_full_tx_len, = rlp_utils.decode_int(
tx_bytes, is_full_tx_start)
_, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix(
tx_bytes, is_full_tx_start + is_full_tx_len)
tx_content_bytes = tx_bytes[tx_content_start:tx_content_start +
tx_content_len]
if is_full_tx:
tx_bytes = tx_content_bytes
else:
short_id = short_ids[short_tx_index]
tx_hash, tx_bytes, _ = tx_service.get_transaction(short_id)
if tx_hash is None:
unknown_tx_sids.append(short_id)
elif tx_bytes is None:
unknown_tx_hashes.append(tx_hash)
short_tx_index += 1
if tx_bytes is not None and not unknown_tx_sids and not unknown_tx_hashes:
buf.append(tx_bytes)
content_size += len(tx_bytes)
tx_count += 1
tx_start_index = tx_itm_start + tx_itm_len
if not unknown_tx_sids and not unknown_tx_hashes:
txs_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(txs_prefix)
content_size += len(txs_prefix)
buf.appendleft(full_hdr_bytes)
content_size += len(full_hdr_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
msg_len_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(msg_len_prefix)
block_msg_bytes = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block_msg_bytes[off:next_off] = blob
off = next_off
block_msg = InternalEthBlockInfo(block_msg_bytes)
logger.debug(
"Successfully parsed block broadcast message. {} "
"transactions in block {}", tx_count, block_hash)
bx_block_hash = convert.bytes_to_hex(
crypto.double_sha256(bx_block_msg))
compressed_size = len(bx_block_msg)
block_info = BlockInfo(
block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000, tx_count,
bx_block_hash,
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()), compressed_size,
100 - float(compressed_size) / content_size * 100, [])
return BlockDecompressionResult(block_msg, block_info,
unknown_tx_sids, unknown_tx_hashes)
else:
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}", block_hash, len(unknown_tx_sids),
len(unknown_tx_hashes), tx_count)
return BlockDecompressionResult(
None,
BlockInfo(block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
None, None, None, None, None, None, []),
unknown_tx_sids, unknown_tx_hashes)
def _log_compressed_block_debug_info_eth(transaction_service: TransactionService,
block_msg_bytes: Union[memoryview, bytearray]):
is_block_relay = transaction_service.node.NODE_TYPE == NodeType.RELAY_BLOCK
block_hash, short_ids, txs_bytes = _parse_block_eth(block_msg_bytes)
# parse statistics variables
short_tx_index = 0
tx_start_index = 0
tx_index_in_block = 0
txs_info = []
missing_short_ids = []
while True:
if tx_start_index >= len(txs_bytes):
break
short_id = 0
has_contents = False
assignmnet_time = 0
_, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len]
is_full_tx_start = 0
is_full_tx, is_full_tx_len, = rlp_utils.decode_int(tx_bytes, is_full_tx_start)
_, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix(
tx_bytes, is_full_tx_start + is_full_tx_len)
tx_content_bytes = tx_bytes[tx_content_start:tx_content_start + tx_content_len]
if is_full_tx:
tx_hash = Sha256Hash(eth_common_utils.keccak_hash(tx_content_bytes))
else:
short_id = short_ids[short_tx_index]
tx_hash, tx_bytes, _ = transaction_service.get_transaction(short_id)
has_contents = tx_bytes is not None
if tx_hash is not None:
assignmnet_time = transaction_service.get_short_id_assign_time(short_id)
short_tx_index += 1
if is_block_relay:
txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash))
else:
txs_info.append((tx_index_in_block, not is_full_tx, short_id, tx_hash, has_contents, assignmnet_time))
tx_index_in_block += 1
tx_start_index = tx_itm_start + tx_itm_len
if not is_full_tx and not has_contents:
missing_short_ids.append(short_id)
if is_block_relay:
log_message = \
"Block content (from block relay) {} from (index, is compressed, short id, hash is full) : {}"
else:
log_message = \
"Block content (full) {} (index, compressed, short id, hash, has contents, assignment time) : {}"
logger.debug(
log_message,
block_hash,
",".join(str(tx_info) for tx_info in txs_info)
)
node_type = transaction_service.node.NODE_TYPE
assert node_type is not None
log_can_decompress_block(node_type, block_hash, missing_short_ids)
