def from_new_block_parts(
cls,
new_block_details: NewBlockParts,
total_difficulty: Optional[int] = 0) -> "InternalEthBlockInfo":
"""
Creates NewBlockInternalEthMessage from block header and block body bytes
:param new_block_details: new block details
:param total_difficulty: total difficulty of the block if known
:return: instance of NewBlockInternalEthMessage
"""
block_header_bytes = new_block_details.block_header_bytes
block_body_bytes = new_block_details.block_body_bytes
block_number = new_block_details.block_number
# block body content includes transactions and uncles
_, block_content_len, block_content_start = rlp_utils.consume_length_prefix(
block_body_bytes, 0)
block_body_conent = block_body_bytes[block_content_start:]
msg_size = len(block_header_bytes) + len(block_body_conent)
total_difficulty_item = rlp_utils.encode_int(
0 if total_difficulty is None else total_difficulty)
msg_size += len(total_difficulty_item)
block_number_bytes = rlp_utils.encode_int(block_number)
msg_size += len(block_number_bytes)
new_block_item_prefix = rlp_utils.get_length_prefix_list(msg_size)
msg_size += len(new_block_item_prefix)
written_bytes = 0
new_block_bytes = bytearray(msg_size)
new_block_bytes[written_bytes:written_bytes +
len(new_block_item_prefix)] = new_block_item_prefix
written_bytes += len(new_block_item_prefix)
new_block_bytes[written_bytes:written_bytes +
len(block_header_bytes)] = block_header_bytes
written_bytes += len(block_header_bytes)
new_block_bytes[written_bytes:written_bytes +
len(block_body_conent)] = block_body_conent
written_bytes += len(block_body_conent)
new_block_bytes[written_bytes:written_bytes +
len(total_difficulty_item)] = total_difficulty_item
written_bytes += len(total_difficulty_item)
new_block_bytes[written_bytes:written_bytes +
len(block_number_bytes)] = block_number_bytes
written_bytes += len(block_number_bytes)
assert msg_size == written_bytes
return cls(new_block_bytes)
def from_block_hash_number_pair(cls, block_hash: Sha256Hash, number: int) -> "NewBlockHashesEthProtocolMessage":
block_hash_bytes = block_hash.binary
msg_size = len(block_hash_bytes)
block_hash_prefix = rlp_utils.get_length_prefix_str(len(block_hash_bytes))
msg_size += len(block_hash_prefix)
number_bytes = rlp_utils.encode_int(number)
msg_size += len(number_bytes)
block_hash_and_number_prefix = rlp_utils.get_length_prefix_list(msg_size)
msg_size += len(block_hash_and_number_prefix)
block_hashes_list_prefix = rlp_utils.get_length_prefix_list(msg_size)
msg_size += len(block_hashes_list_prefix)
msg_bytes = bytearray(msg_size)
offset = 0
msg_bytes[offset:len(block_hashes_list_prefix)] = block_hashes_list_prefix
offset += len(block_hashes_list_prefix)
msg_bytes[offset:offset + len(block_hash_and_number_prefix)] = block_hash_and_number_prefix
offset += len(block_hash_and_number_prefix)
msg_bytes[offset:offset + len(block_hash_prefix)] = block_hash_prefix
offset += len(block_hash_prefix)
msg_bytes[offset:offset + len(block_hash_bytes)] = block_hash_bytes
offset += len(block_hash_bytes)
msg_bytes[offset:offset + len(number_bytes)] = number_bytes
return cls(msg_bytes)
def from_new_block_msg(
cls, new_block_msg: NewBlockEthProtocolMessage
) -> "InternalEthBlockInfo":
"""
Creates NewBlockInternalEthMessage from raw bytes of NewBlockEthProtocolMessage
:param new_block_msg: new block message
:return: NewBlockInternalEthMessage message
"""
new_block_msg_bytes = memoryview(new_block_msg.rawbytes())
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
new_block_msg_bytes, 0)
block_msg_bytes = new_block_msg_bytes[
block_msg_itm_start:block_msg_itm_start + block_msg_itm_len]
msg_size = 0
# block item already include header, transactions and uncles
_, block_itm_len, block_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_itm_bytes = block_msg_bytes[
block_msg_itm_start:block_msg_itm_start + block_itm_len]
msg_size += len(block_itm_bytes)
difficulty_bytes = block_msg_bytes[block_msg_itm_start +
block_itm_len:]
msg_size += len(difficulty_bytes)
block_number_bytes = rlp_utils.encode_int(new_block_msg.number())
msg_size += len(block_number_bytes)
msg_prefix = rlp_utils.get_length_prefix_list(
len(block_itm_bytes) + len(difficulty_bytes) +
len(block_number_bytes))
msg_size += len(msg_prefix)
msg_bytes = bytearray(msg_size)
written_bytes = 0
msg_bytes[written_bytes:written_bytes + len(msg_prefix)] = msg_prefix
written_bytes += len(msg_prefix)
msg_bytes[written_bytes:written_bytes +
len(block_itm_bytes)] = block_itm_bytes
written_bytes += len(block_itm_bytes)
msg_bytes[written_bytes:written_bytes +
len(difficulty_bytes)] = difficulty_bytes
written_bytes += len(difficulty_bytes)
msg_bytes[written_bytes:written_bytes +
len(block_number_bytes)] = block_number_bytes
written_bytes += len(block_number_bytes)
assert written_bytes == msg_size
return cls(msg_bytes)
def test_decode_int(self):
encoded_0 = memoryview(rlp_utils.encode_int(0))
decoded_0 = rlp_utils.decode_int(encoded_0, 0)
self.assertEqual(0, decoded_0[0])
self.assertEqual(1, decoded_0[1])
encoded_127 = memoryview(rlp_utils.encode_int(127))
decoded_127 = rlp_utils.decode_int(encoded_127, 0)
self.assertEqual(127, decoded_127[0])
self.assertEqual(1, decoded_127[1])
encoded_128 = memoryview(rlp_utils.encode_int(128))
decoded_128 = rlp_utils.decode_int(encoded_128, 0)
self.assertEqual(128, decoded_128[0])
self.assertEqual(2, decoded_128[1])
encoded_256 = memoryview(rlp_utils.encode_int(256))
decoded_256 = rlp_utils.decode_int(encoded_256, 0)
self.assertEqual(256, decoded_256[0])
self.assertEqual(3, decoded_256[1])
encoded_1M = memoryview(rlp_utils.encode_int(1000000))
decoded_1M = rlp_utils.decode_int(encoded_1M, 0)
self.assertEqual(1000000, decoded_1M[0])
self.assertEqual(4, decoded_1M[1])
encoded_1M_lpad = memoryview(b"0" + rlp_utils.encode_int(1000000))
decoded_1M_lpad = rlp_utils.decode_int(encoded_1M_lpad, 1)
self.assertEqual(1000000, decoded_1M_lpad[0])
self.assertEqual(4, decoded_1M_lpad[1])
def from_new_block_parts(
cls, new_block_parts: NewBlockParts,
total_difficulty: int) -> "NewBlockEthProtocolMessage":
block_header = memoryview(new_block_parts.block_header_bytes)
block_body = memoryview(new_block_parts.block_body_bytes)
_, block_content_len, block_content_start = rlp_utils.consume_length_prefix(
block_body, 0)
block_body_conent = block_body[block_content_start:]
content_size = len(block_header) + len(block_body_conent)
block_item_prefix = rlp_utils.get_length_prefix_list(content_size)
content_size += len(block_item_prefix)
total_difficulty_item = rlp_utils.encode_int(total_difficulty)
content_size += len(total_difficulty_item)
new_block_item_prefix = rlp_utils.get_length_prefix_list(content_size)
content_size += len(new_block_item_prefix)
written_bytes = 0
new_block_bytes = bytearray(content_size)
new_block_bytes[written_bytes:written_bytes +
len(new_block_item_prefix)] = new_block_item_prefix
written_bytes += len(new_block_item_prefix)
new_block_bytes[written_bytes:written_bytes +
len(block_item_prefix)] = block_item_prefix
written_bytes += len(block_item_prefix)
new_block_bytes[written_bytes:written_bytes +
len(block_header)] = block_header
written_bytes += len(block_header)
new_block_bytes[written_bytes:written_bytes +
len(block_body_conent)] = block_body_conent
written_bytes += len(block_body_conent)
new_block_bytes[written_bytes:written_bytes +
len(total_difficulty_item)] = total_difficulty_item
written_bytes += len(total_difficulty_item)
return cls(new_block_bytes)
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:param enable_block_compression
:param min_tx_age_seconds
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message(
block_msg)
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
ignored_sids = []
tx_start_index = 0
tx_count = 0
original_size = len(block_msg.rawbytes())
max_timestamp_for_compression = time.time() - min_tx_age_seconds
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
tx_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(tx_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(
short_id)
if short_id <= constants.NULL_TX_SID or \
not enable_block_compression or short_id_assign_time > max_timestamp_for_compression:
if short_id > constants.NULL_TX_SID:
ignored_sids.append(short_id)
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
block = finalize_block_bytes(buf, content_size, used_short_ids)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100,
ignored_sids)
return memoryview(block), block_info
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
msg_bytes = memoryview(block_msg.rawbytes())
_, block_msg_itm_len, block_msg_itm_start = rlp_utils.consume_length_prefix(
msg_bytes, 0)
block_msg_bytes = msg_bytes[block_msg_itm_start:block_msg_itm_start +
block_msg_itm_len]
_, block_hdr_itm_len, block_hdr_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, 0)
block_hdr_full_bytes = block_msg_bytes[0:block_hdr_itm_start +
block_hdr_itm_len]
block_hdr_bytes = block_msg_bytes[
block_hdr_itm_start:block_hdr_itm_start + block_hdr_itm_len]
_, prev_block_itm_len, prev_block_itm_start = rlp_utils.consume_length_prefix(
block_hdr_bytes, 0)
prev_block_bytes = block_hdr_bytes[
prev_block_itm_start:prev_block_itm_start + prev_block_itm_len]
_, txs_itm_len, txs_itm_start = rlp_utils.consume_length_prefix(
block_msg_bytes, block_hdr_itm_start + block_hdr_itm_len)
txs_bytes = block_msg_bytes[txs_itm_start:txs_itm_start + txs_itm_len]
remaining_bytes = block_msg_bytes[txs_itm_start + txs_itm_len:]
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
tx_start_index = 0
tx_count = 0
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
short_id = tx_service.get_short_id(tx_hash)
if short_id <= 0:
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
short_ids_bytes = compact_block_short_ids_serializer.serialize_short_ids_into_bytes(
used_short_ids)
buf.append(short_ids_bytes)
content_size += constants.UL_ULL_SIZE_IN_BYTES
offset_buf = struct.pack("<Q", content_size)
buf.appendleft(offset_buf)
content_size += len(short_ids_bytes)
# Parse it into the bloXroute message format and send it along
block = bytearray(content_size)
off = 0
for blob in buf:
next_off = off + len(blob)
block[off:next_off] = blob
off = next_off
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
original_size = len(block_msg.rawbytes())
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100)
return memoryview(block), block_info
def test_encode_decode_int(self):
self.assertEqual(1, len(rlp_utils.encode_int(0)))
self.assertEqual(1, len(rlp_utils.encode_int(127)))
self.assertEqual(2, len(rlp_utils.encode_int(128)))
self.assertEqual(3, len(rlp_utils.encode_int(256)))
self.assertEqual(4, len(rlp_utils.encode_int(1000000)))
