def deserialize(self):
if self._msg_bytes is None or len(
self._msg_bytes
) < eth_common_constants.MDC_LEN + eth_common_constants.SIGNATURE_LEN:
raise ValueError("Message bytes empty or too short.")
mdc = self._memory_view[:eth_common_constants.MDC_LEN].tobytes()
if mdc != eth_common_utils.keccak_hash(
self._memory_view[eth_common_constants.MDC_LEN:].tobytes()):
raise WrongMACError("Message hash does not match MDC")
mdc_sig_len = eth_common_constants.MDC_LEN + eth_common_constants.SIGNATURE_LEN
signature = self._memory_view[eth_common_constants.
MDC_LEN:mdc_sig_len].tobytes()
self._msg_type = rlp_utils.safe_ord(self._memory_view[mdc_sig_len])
encoded_data = self._memory_view[mdc_sig_len:].tobytes()
signed_data = eth_common_utils.keccak_hash(encoded_data)
remote_pubkey = crypto_utils.recover_public_key(signed_data, signature)
self._public_key = remote_pubkey
if not crypto_utils.verify_signature(remote_pubkey, signature,
signed_data):
raise InvalidSignatureError(
"Message signature does not match public key")
encoded_payload = self._memory_view[mdc_sig_len + eth_common_constants.
MSG_TYPE_LEN:].tobytes()
self._deserialize_rlp_payload(encoded_payload)
self._is_deserialized = True
def test_sha3(self):
dummy_msg1 = helpers.generate_bytearray(111)
dummy_msg2 = helpers.generate_bytearray(1111)
sha1 = eth_common_utils.keccak_hash(dummy_msg1)
sha2 = eth_common_utils.keccak_hash(dummy_msg2)
self.assertEqual(len(sha1), eth_common_constants.SHA3_LEN_BYTES)
self.assertEqual(len(sha2), eth_common_constants.SHA3_LEN_BYTES)
self.assertNotEqual(sha1, sha2)
def serialize(self):
cmd_id = rlp_utils.str_to_bytes(chr(self.msg_type))
encoded_payload = self._serialize_rlp_payload()
signed_data = eth_common_utils.keccak_hash(cmd_id + encoded_payload)
signature = crypto_utils.sign(signed_data, self._private_key)
assert len(signature) == eth_common_constants.SIGNATURE_LEN
mdc = eth_common_utils.keccak_hash(signature + cmd_id +
encoded_payload)
assert len(mdc) == eth_common_constants.MDC_LEN
msg_bytes = bytearray(mdc + signature + cmd_id + encoded_payload)
self._set_raw_bytes(msg_bytes)
def create_auth_ack_message(self):
"""
authRecipient = E(remote-pubk, remote-ephemeral-pubk || nonce || 0x1) // token found
authRecipient = E(remote-pubk, remote-ephemeral-pubk || nonce || 0x0) // token not found
nonce, ephemeral_pubkey, version are local!
"""
assert not self._is_initiator
ephemeral_pubkey = self._ephemeral_ecc.get_raw_public_key()
encoded_nonce = rlp.sedes.big_endian_int.serialize(
random.randint(0, eth_common_constants.MAX_NONCE))
self._responder_nonce = eth_common_utils.keccak_hash(encoded_nonce)
if self._is_eip8_auth:
msg = self.create_eip8_auth_ack_message(
ephemeral_pubkey, self._responder_nonce,
eth_common_constants.P2P_PROTOCOL_VERSION)
assert len(msg) > eth_common_constants.AUTH_ACK_MSG_LEN
else:
msg = self.create_plain_auth_ack_message(
ephemeral_pubkey, self._responder_nonce,
eth_common_constants.P2P_PROTOCOL_VERSION)
assert len(msg) == eth_common_constants.AUTH_ACK_MSG_LEN
return msg
def _parse_block_eth(block_msg_bytes: Union[bytearray, memoryview]) -> Tuple[Sha256Hash, List[int], memoryview]:
block_msg_bytes = block_msg_bytes if isinstance(block_msg_bytes, memoryview) else memoryview(block_msg_bytes)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(block_msg_bytes)
short_ids, _short_ids_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
block_msg_bytes,
block_offsets.short_id_offset
)
block_bytes = block_msg_bytes[block_offsets.block_begin_offset: block_offsets.short_id_offset]
_, _block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(block_bytes, 0)
block_itm_bytes = block_bytes[block_itm_start:]
_, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix(block_itm_bytes, 0)
full_hdr_bytes = block_itm_bytes[0:block_hdr_start + block_hdr_len]
block_hash_bytes = eth_common_utils.keccak_hash(full_hdr_bytes)
block_hash = Sha256Hash(block_hash_bytes)
_, block_txs_len, block_txs_start = rlp_utils.consume_length_prefix(
block_itm_bytes, block_hdr_start + block_hdr_len
)
txs_bytes = block_itm_bytes[block_txs_start:block_txs_start + block_txs_len]
return block_hash, short_ids, txs_bytes
def test_sign_and_verify__valid(self):
msg = helpers.generate_bytearray(111)
msg_hash = eth_common_utils.keccak_hash(msg)
signature = self._eccx.sign(msg_hash)
self.assertTrue(self._eccx.verify(signature, msg_hash))
def create_auth_message(self):
"""
Generates authentication message bytes
1. initiator generates ecdhe-random and nonce and creates auth
2. initiator connects to remote and sends auth
New:
E(remote-pubk,
S(ephemeral-privk, ecdh-shared-secret ^ nonce) ||
H(ephemeral-pubk) || pubk || nonce || 0x0
)
Known:
E(remote-pubk,
S(ephemeral-privk, token ^ nonce) || H(ephemeral-pubk) || pubk || nonce || 0x1)
:param remote_pubkey: public key of remote node
:param nonce: nonce
:return: authentication message bytes
"""
assert self._is_initiator
ecdh_shared_secret = self._ecc.get_ecdh_key(self._remote_pubkey)
token = ecdh_shared_secret
flag = 0x0
self._initiator_nonce = eth_common_utils.keccak_hash(
rlp_utils.int_to_big_endian(random.randint(0, sys.maxsize)))
assert len(
self._initiator_nonce) == eth_common_constants.SHA3_LEN_BYTES
token_xor_nonce = crypto_utils.string_xor(token, self._initiator_nonce)
assert len(token_xor_nonce) == eth_common_constants.SHA3_LEN_BYTES
ephemeral_pubkey = self._ephemeral_ecc.get_raw_public_key()
assert len(ephemeral_pubkey) == eth_common_constants.PUBLIC_KEY_LEN
if not self._ecc.is_valid_key(ephemeral_pubkey):
raise InvalidKeyError("Invalid ephemeral pubkey")
# S(ephemeral-privk, ecdh-shared-secret ^ nonce)
S = self._ephemeral_ecc.sign(token_xor_nonce)
assert len(S) == eth_common_constants.SIGNATURE_LEN
# S || H(ephemeral-pubk) || pubk || nonce || 0x0
auth_message = S + eth_common_utils.keccak_hash(ephemeral_pubkey) + self._ecc.get_raw_public_key() + \
self._initiator_nonce + rlp_utils.ascii_chr(flag)
return auth_message
def block_hash(self) -> Sha256Hash:
if self._block_hash is None:
raw_hash = eth_common_utils.keccak_hash(self.block_header())
self._block_hash = Sha256Hash(raw_hash)
block_hash = self._block_hash
assert block_hash is not None
return block_hash
def get_block_transaction_hashes(self,
block_index: int) -> List[Sha256Hash]:
tx_hashes = []
for tx_bytes in self.get_block_transaction_bytes(block_index):
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hashes.append(Sha256Hash(tx_hash_bytes))
return tx_hashes
def test_sign_and_verify__invalid(self):
msg = helpers.generate_bytearray(111)
msg_hash = eth_common_utils.keccak_hash(msg)
signature = self._eccx.sign(msg_hash)
other_private_key = crypto_utils.make_private_key(helpers.generate_bytearray(222))
another_eccx = ECCx(raw_private_key=other_private_key)
self.assertFalse(another_eccx.verify(signature, msg_hash))
def make_private_key(seed):
"""
Generates ECC private using provided seed value
:param seed: seed used to generate ECC private key
:return: ECC private key
"""
if not seed:
raise ValueError("Seed is required")
return eth_common_utils.keccak_hash(seed)
def test_sign_and_verify_signature__valid_signature(self):
dummy_private_key = crypto_utils.make_private_key(
helpers.generate_bytearray(111))
public_key = crypto_utils.private_to_public_key(dummy_private_key)
# generate random bytes
msg = helpers.generate_bytearray(222)
msg_hash = eth_common_utils.keccak_hash(msg)
signature = crypto_utils.sign(msg_hash, dummy_private_key)
self.assertTrue(
crypto_utils.verify_signature(public_key, signature, msg_hash))
def test_recover_public_key(self):
dummy_private_key = crypto_utils.make_private_key(
helpers.generate_bytearray(111))
public_key = crypto_utils.private_to_public_key(dummy_private_key)
msg = helpers.generate_bytearray(222)
msg_hash = eth_common_utils.keccak_hash(msg)
signature = crypto_utils.sign(msg_hash, dummy_private_key)
recovered_pub_key = crypto_utils.recover_public_key(
msg_hash, signature)
self.assertEqual(recovered_pub_key, public_key)
def verify_eth_transaction_signature(transaction: Transaction) -> bool:
"""
checks eth transaction signature
:param transaction:
:return: if signature matches public key
"""
try:
signature = transaction.signature()
unsigned_msg = transaction.get_unsigned()
public_key = crypto_utils.recover_public_key(unsigned_msg, signature, keccak_hash)
return crypto_utils.verify_signature(public_key, signature, keccak_hash(memoryview(unsigned_msg)))
# pylint: disable=broad-except
except Exception:
return False
def test_encode_decode_signature(self):
dummy_private_key = crypto_utils.make_private_key(
helpers.generate_bytearray(111))
msg = helpers.generate_bytearray(222)
msg_hash = eth_common_utils.keccak_hash(msg)
signature = crypto_utils.sign(msg_hash, dummy_private_key)
v, r, s = crypto_utils.decode_signature(signature)
self.assertIsNotNone(v)
self.assertIsNotNone(r)
self.assertIsNotNone(s)
encoded_signature = crypto_utils.encode_signature(v, r, s)
self.assertEqual(encoded_signature, signature)
def msg_block_headers(self, msg: BlockHeadersEthProtocolMessage):
if not self.node.should_process_block_hash():
return
block_headers = msg.get_block_headers()
if self.pending_new_block_parts.contents and len(block_headers) == 1:
header_bytes = msg.get_block_headers_bytes()[0]
block_hash_bytes = eth_common_utils.keccak_hash(header_bytes)
block_hash = Sha256Hash(block_hash_bytes)
if block_hash in self.pending_new_block_parts.contents:
logger.debug("Received block header for new block {}",
convert.bytes_to_hex(block_hash.binary))
self.pending_new_block_parts.contents[
block_hash].block_header_bytes = header_bytes
self._check_pending_new_block(block_hash)
self._process_ready_new_blocks()
return
if len(block_headers) > 0:
block_hashes = [blk.hash_object() for blk in block_headers]
block_hashes.insert(0, Sha256Hash(block_headers[0].prev_hash))
self.node.block_cleanup_service.mark_blocks_and_request_cleanup(
block_hashes)
latest_block_number = 0
latest_block_difficulty = 0
block_queuing_service = self.node.block_queuing_service_manager.get_block_queuing_service(
self.connection)
if block_queuing_service is not None:
for block_header in block_headers:
block_queuing_service.mark_block_seen_by_blockchain_node(
block_header.hash_object(), None, block_header.number)
if block_header.number > latest_block_number:
latest_block_number = block_header.number
latest_block_difficulty = block_header.difficulty
self.node.block_processing_service.set_last_confirmed_block_parameters(
latest_block_number, latest_block_difficulty)
def msg_block_headers(self, msg: BlockHeadersEthProtocolMessage):
if not self.node.should_process_block_hash():
return
block_headers = msg.get_block_headers()
if self._pending_new_blocks_parts.contents and len(block_headers) == 1:
header_bytes = msg.get_block_headers_bytes()[0]
block_hash_bytes = eth_common_utils.keccak_hash(header_bytes)
block_hash = Sha256Hash(block_hash_bytes)
if block_hash in self._pending_new_blocks_parts.contents:
logger.debug("Received block header for new block {}", convert.bytes_to_hex(block_hash.binary))
self._pending_new_blocks_parts.contents[block_hash].block_header_bytes = header_bytes
self._check_pending_new_block(block_hash)
self._process_ready_new_blocks()
return
if len(block_headers) > 0:
block_hashes = [blk.hash_object() for blk in block_headers]
block_hashes.insert(0, Sha256Hash(block_headers[0].prev_hash))
self.node.block_cleanup_service.mark_blocks_and_request_cleanup(block_hashes)
self.node.block_queuing_service.mark_blocks_seen_by_blockchain_node(block_hashes)
def setup_cipher(self):
"""
Sets up cipher parameters.
Needs to be called after initial authentication handshake
"""
assert self._responder_nonce
assert self._initiator_nonce
assert self._auth_init
assert self._auth_ack
assert self._remote_ephemeral_pubkey
if not self._ecc.is_valid_key(self._remote_ephemeral_pubkey):
raise InvalidKeyError("Invalid remote ephemeral pubkey")
# derive base secrets from ephemeral key agreement
# ecdhe-shared-secret = ecdh.agree(ephemeral-privkey, remote-ephemeral-pubk)
ecdhe_shared_secret = self._ephemeral_ecc.get_ecdh_key(
self._remote_ephemeral_pubkey)
# shared-secret = sha3(ecdhe-shared-secret || sha3(nonce || initiator-nonce))
shared_secret = eth_common_utils.keccak_hash(
ecdhe_shared_secret +
eth_common_utils.keccak_hash(self._responder_nonce +
self._initiator_nonce))
self.ecdhe_shared_secret = ecdhe_shared_secret # used in tests
self.shared_secret = shared_secret # used in tests
# token = sha3(shared-secret)
self._token = eth_common_utils.keccak_hash(shared_secret)
# aes-secret = sha3(ecdhe-shared-secret || shared-secret)
self._aes_secret = eth_common_utils.keccak_hash(ecdhe_shared_secret +
shared_secret)
# mac-secret = sha3(ecdhe-shared-secret || aes-secret)
self.mac_secret = eth_common_utils.keccak_hash(ecdhe_shared_secret +
self._aes_secret)
# setup sha3 instances for the MACs
# egress-mac = sha3.update(mac-secret ^ recipient-nonce || auth-sent-init)
mac1 = crypto_utils.get_sha3_calculator(
crypto_utils.string_xor(self.mac_secret, self._responder_nonce) +
self._auth_init)
# ingress-mac = sha3.update(mac-secret ^ initiator-nonce || auth-recvd-ack)
mac2 = crypto_utils.get_sha3_calculator(
crypto_utils.string_xor(self.mac_secret, self._initiator_nonce) +
self._auth_ack)
if self._is_initiator:
self._egress_mac, self._ingress_mac = mac1, mac2
else:
self._egress_mac, self._ingress_mac = mac2, mac1
iv = "\x00" * eth_common_constants.IV_LEN
self._aes_enc = pyelliptic.Cipher(
self._aes_secret,
iv,
eth_common_constants.CIPHER_ENCRYPT_DO,
ciphername=eth_common_constants.RLPX_CIPHER_NAME)
self._aes_dec = pyelliptic.Cipher(
self._aes_secret,
iv,
eth_common_constants.CIPHER_DECRYPT_DO,
ciphername=eth_common_constants.RLPX_CIPHER_NAME)
self._mac_enc = AES.new(self.mac_secret, AES.MODE_ECB).encrypt
self._is_ready = True
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:param enable_block_compression
:param min_tx_age_seconds
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message(
block_msg)
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
ignored_sids = []
tx_start_index = 0
tx_count = 0
original_size = len(block_msg.rawbytes())
max_timestamp_for_compression = time.time() - min_tx_age_seconds
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
tx_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(tx_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(
short_id)
if short_id <= constants.NULL_TX_SID or \
not enable_block_compression or short_id_assign_time > max_timestamp_for_compression:
if short_id > constants.NULL_TX_SID:
ignored_sids.append(short_id)
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
block = finalize_block_bytes(buf, content_size, used_short_ids)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100,
ignored_sids)
return memoryview(block), block_info
def bx_block_to_block(self, bx_block_msg,
tx_service) -> BlockDecompressionResult:
"""
Converts internal broadcast message to Ethereum new block message
The code is optimized and does not make copies of bytes
:param bx_block_msg: internal broadcast message bytes
:param tx_service: Transactions service
:return: tuple (new block message, block hash, unknown transaction short id, unknown transaction hashes)
"""
if not isinstance(bx_block_msg, (bytearray, memoryview)):
raise TypeError(
"Type bytearray is expected for arg block_bytes but was {0}".
format(type(bx_block_msg)))
decompress_start_datetime = datetime.datetime.utcnow()
decompress_start_timestamp = time.time()
block_msg_bytes = bx_block_msg if isinstance(
bx_block_msg, memoryview) else memoryview(bx_block_msg)
block_offsets = compact_block_short_ids_serializer.get_bx_block_offsets(
bx_block_msg)
short_ids, short_ids_bytes_len = compact_block_short_ids_serializer.deserialize_short_ids_from_buffer(
bx_block_msg, block_offsets.short_id_offset)
block_bytes = block_msg_bytes[
block_offsets.block_begin_offset:block_offsets.short_id_offset]
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_bytes, 0)
block_itm_bytes = block_bytes[block_itm_start:]
_, block_hdr_len, block_hdr_start = rlp_utils.consume_length_prefix(
block_itm_bytes, 0)
full_hdr_bytes = block_itm_bytes[0:block_hdr_start + block_hdr_len]
block_hash_bytes = eth_common_utils.keccak_hash(full_hdr_bytes)
block_hash = Sha256Hash(block_hash_bytes)
_, block_txs_len, block_txs_start = rlp_utils.consume_length_prefix(
block_itm_bytes, block_hdr_start + block_hdr_len)
txs_bytes = block_itm_bytes[block_txs_start:block_txs_start +
block_txs_len]
remaining_bytes = block_itm_bytes[block_txs_start + block_txs_len:]
# parse statistics variables
short_tx_index = 0
unknown_tx_sids = []
unknown_tx_hashes = []
# creating transactions content
content_size = 0
buf = deque()
tx_count = 0
tx_start_index = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_itm_len, tx_itm_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_itm_start:tx_itm_start + tx_itm_len]
is_full_tx_start = 0
is_full_tx, is_full_tx_len, = rlp_utils.decode_int(
tx_bytes, is_full_tx_start)
_, tx_content_len, tx_content_start = rlp_utils.consume_length_prefix(
tx_bytes, is_full_tx_start + is_full_tx_len)
tx_content_bytes = tx_bytes[tx_content_start:tx_content_start +
tx_content_len]
if is_full_tx:
tx_bytes = tx_content_bytes
else:
short_id = short_ids[short_tx_index]
tx_hash, tx_bytes, _ = tx_service.get_transaction(short_id)
if tx_hash is None:
unknown_tx_sids.append(short_id)
elif tx_bytes is None:
unknown_tx_hashes.append(tx_hash)
short_tx_index += 1
if tx_bytes is not None and not unknown_tx_sids and not unknown_tx_hashes:
buf.append(tx_bytes)
content_size += len(tx_bytes)
tx_count += 1
tx_start_index = tx_itm_start + tx_itm_len
if not unknown_tx_sids and not unknown_tx_hashes:
txs_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(txs_prefix)
content_size += len(txs_prefix)
buf.appendleft(full_hdr_bytes)
content_size += len(full_hdr_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
msg_len_prefix = rlp_utils.get_length_prefix_list(content_size)
buf.appendleft(msg_len_prefix)
block_msg_bytes = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block_msg_bytes[off:next_off] = blob
off = next_off
block_msg = InternalEthBlockInfo(block_msg_bytes)
logger.debug(
"Successfully parsed block broadcast message. {} "
"transactions in block {}", tx_count, block_hash)
bx_block_hash = convert.bytes_to_hex(
crypto.double_sha256(bx_block_msg))
compressed_size = len(bx_block_msg)
block_info = BlockInfo(
block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000, tx_count,
bx_block_hash,
convert.bytes_to_hex(block_msg.prev_block_hash().binary),
len(block_msg.rawbytes()), compressed_size,
100 - float(compressed_size) / content_size * 100, [])
return BlockDecompressionResult(block_msg, block_info,
unknown_tx_sids, unknown_tx_hashes)
else:
logger.debug(
"Block recovery needed for {}. Missing {} sids, {} tx hashes. "
"Total txs in block: {}", block_hash, len(unknown_tx_sids),
len(unknown_tx_hashes), tx_count)
return BlockDecompressionResult(
None,
BlockInfo(block_hash, short_ids, decompress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - decompress_start_timestamp) * 1000,
None, None, None, None, None, None, []),
unknown_tx_sids, unknown_tx_hashes)
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
msg_bytes = memoryview(block_msg.rawbytes())
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
msg_bytes, 0)
block_msg_bytes = msg_bytes[block_msg_itm_start:block_msg_itm_start +
block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_full_bytes = block_msg_bytes[0:block_hdr_itm_start +
block_hdr_itm_len]
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
_, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix(
block_hdr_bytes, 0)
prev_block_bytes = block_hdr_bytes[
prev_block_itm_start:prev_block_itm_start + prev_block_itm_len]
_, txs_itm_len, txs_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_hdr_itm_start + block_hdr_itm_len)
txs_bytes = block_msg_bytes[txs_itm_start:txs_itm_start + txs_itm_len]
remaining_bytes = block_msg_bytes[txs_itm_start + txs_itm_len:]
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
tx_start_index = 0
tx_count = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
short_id = tx_service.get_short_id(tx_hash)
if short_id <= 0:
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
short_ids_bytes = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(
used_short_ids)
buf.append(short_ids_bytes)
content_size += constants.UL_ULL_SIZE_IN_BYTES
offset_buf = struct.pack("<Q", content_size)
buf.appendleft(offset_buf)
content_size += len(short_ids_bytes)
# Parse it into the bloXroute message format and send it along
block = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block[off:next_off] = blob
off = next_off
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
original_size = len(block_msg.rawbytes())
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100)
return memoryview(block), block_info
def get_block_hash(self) -> Optional[Sha256Hash]:
if self.block_header_bytes is None:
return None
raw_hash = eth_common_utils.keccak_hash(self.block_header_bytes)
return Sha256Hash(raw_hash)
def hash(self) -> bytearray:
"""The binary block hash"""
return eth_common_utils.keccak_hash(rlp.encode(self))
def hash(self):
hash_bytes = eth_common_utils.keccak_hash(Transaction.serialize(self))
return Sha256Hash(hash_bytes)
def hash(self):
hash_bytes = eth_common_utils.keccak_hash(rlp.encode(self))
return Sha256Hash(hash_bytes)
def public_key_to_address(public_key_bytes: bytes) -> bytes:
return eth_common_utils.keccak_hash(memoryview(public_key_bytes))[-20:]
def block_hash(self) -> Sha256Hash:
if self._block_hash is None:
raw_hash = eth_common_utils.keccak_hash(self.block_header())
self._block_hash = Sha256Hash(raw_hash)
return self._block_hash
def hash_object(self) -> Sha256Hash:
return Sha256Hash(eth_common_utils.keccak_hash(rlp.encode(self)))
def hash(self):
"""Transaction hash"""
hash_bytes = eth_common_utils.keccak_hash(rlp.encode(self))
return Sha256Hash(hash_bytes)
def _get_block_hash(
self, block_header_bytes: Union[bytearray,
memoryview]) -> Sha256Hash:
raw_hash = eth_common_utils.keccak_hash(memoryview(block_header_bytes))
return Sha256Hash(raw_hash)
