def create_txs_service_msg(
transaction_service: TransactionService,
tx_service_snap: List[Sha256Hash],
sync_tx_content: bool = True) -> List[TxContentShortIds]:
task_start = time.time()
txs_content_short_ids: List[TxContentShortIds] = []
txs_msg_len = 0
while tx_service_snap:
transaction_key = transaction_service.get_transaction_key(
tx_service_snap.pop())
short_ids = list(
transaction_service.get_short_ids_by_key(transaction_key))
if sync_tx_content:
tx_content = transaction_service.get_transaction_by_key(
transaction_key)
else:
tx_content = bytearray(0)
# TODO: evaluate short id quota type flag value
short_id_flags = [
transaction_service.get_short_id_transaction_type(short_id)
for short_id in short_ids
]
tx_content_short_ids: TxContentShortIds = TxContentShortIds(
transaction_key.transaction_hash, tx_content, short_ids,
short_id_flags)
txs_msg_len += txs_serializer.get_serialized_tx_content_short_ids_bytes_len(
tx_content_short_ids)
txs_content_short_ids.append(tx_content_short_ids)
if txs_msg_len >= constants.TXS_MSG_SIZE or time.time(
) - task_start > constants.TXS_SYNC_TASK_DURATION:
break
return txs_content_short_ids
def create_txs_service_msg_from_time(
transaction_service: TransactionService,
start_time: float = 0,
sync_tx_content: bool = True,
snapshot_cache_keys: Optional[Set[TransactionCacheKeyType]] = None
) -> Tuple[List[TxContentShortIds], float, bool, Set[TransactionCacheKeyType]]:
task_start = time.time()
txs_content_short_ids: List[TxContentShortIds] = []
txs_msg_len = 0
if snapshot_cache_keys is None:
snapshot_cache_keys = set()
done = False
timestamp = start_time
expire_short_ids = []
for short_id, timestamp in transaction_service._tx_assignment_expire_queue.queue.items(
):
if timestamp > start_time:
cache_key = transaction_service._short_id_to_tx_cache_key.get(
short_id, None)
if cache_key is not None:
transaction_key = transaction_service.get_transaction_key(
None, cache_key)
if cache_key not in snapshot_cache_keys:
snapshot_cache_keys.add(
transaction_key.transaction_cache_key)
short_ids = list(
transaction_service._tx_cache_key_to_short_ids[
transaction_key.transaction_cache_key])
if sync_tx_content and transaction_service.has_transaction_contents_by_key(
transaction_key):
tx_content = transaction_service._tx_cache_key_to_contents[
transaction_key.transaction_cache_key]
else:
tx_content = bytearray(0)
short_id_flags = [
transaction_service.get_short_id_transaction_type(
short_id) for short_id in short_ids
]
tx_content_short_ids: TxContentShortIds = TxContentShortIds(
transaction_key.transaction_hash, tx_content,
short_ids, short_id_flags)
txs_msg_len += txs_serializer.get_serialized_tx_content_short_ids_bytes_len(
tx_content_short_ids)
txs_content_short_ids.append(tx_content_short_ids)
if txs_msg_len >= constants.TXS_MSG_SIZE or (
time.time() - task_start >
constants.TXS_SYNC_TASK_DURATION):
break
else:
expire_short_ids.append(short_id)
else:
done = True
for short_id in expire_short_ids:
transaction_service._tx_assignment_expire_queue.remove(short_id)
return txs_content_short_ids, timestamp, done, snapshot_cache_keys
def contents_cleanup(
transaction_service: TransactionService,
block_confirmation_message: AbstractCleanupMessage
):
message_hash = block_confirmation_message.message_hash()
for short_id in block_confirmation_message.short_ids():
transaction_service.remove_transaction_by_short_id(
short_id,
remove_related_short_ids=True,
force=True,
removal_reason=TxRemovalReason.BLOCK_CLEANUP
)
for tx_hash in block_confirmation_message.transaction_hashes():
transaction_service.remove_transaction_by_key(transaction_service.get_transaction_key(tx_hash), force=True)
transaction_service.on_block_cleaned_up(message_hash)
logger.statistics(
{
"type": "MemoryCleanup",
"event": "CacheStateAfterBlockCleanup",
"message_hash": repr(message_hash),
"data": transaction_service.get_cache_state_json()
}
)
def block_to_bx_block(
self, block_msg: InternalEthBlockInfo,
tx_service: TransactionService, enable_block_compression: bool,
min_tx_age_seconds: float) -> Tuple[memoryview, BlockInfo]:
"""
Convert Ethereum new block message to internal broadcast message with transactions replaced with short ids
The code is optimized and does not make copies of bytes
:param block_msg: Ethereum new block message
:param tx_service: Transactions service
:param enable_block_compression
:param min_tx_age_seconds
:return: Internal broadcast message bytes (bytearray), tuple (txs count, previous block hash)
"""
compress_start_datetime = datetime.datetime.utcnow()
compress_start_timestamp = time.time()
txs_bytes, block_hdr_full_bytes, remaining_bytes, prev_block_bytes = parse_block_message(
block_msg)
used_short_ids = []
# creating transactions content
content_size = 0
buf = deque()
ignored_sids = []
tx_start_index = 0
tx_count = 0
original_size = len(block_msg.rawbytes())
max_timestamp_for_compression = time.time() - min_tx_age_seconds
while True:
if tx_start_index >= len(txs_bytes):
break
_, tx_item_length, tx_item_start = rlp_utils.consume_length_prefix(
txs_bytes, tx_start_index)
tx_bytes = txs_bytes[tx_start_index:tx_item_start + tx_item_length]
tx_hash_bytes = eth_common_utils.keccak_hash(tx_bytes)
tx_hash = Sha256Hash(tx_hash_bytes)
tx_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(tx_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(
short_id)
if short_id <= constants.NULL_TX_SID or \
not enable_block_compression or short_id_assign_time > max_timestamp_for_compression:
if short_id > constants.NULL_TX_SID:
ignored_sids.append(short_id)
is_full_tx_bytes = rlp_utils.encode_int(1)
tx_content_bytes = tx_bytes
else:
is_full_tx_bytes = rlp_utils.encode_int(0)
used_short_ids.append(short_id)
tx_content_bytes = bytes()
tx_content_prefix = rlp_utils.get_length_prefix_str(
len(tx_content_bytes))
short_tx_content_size = len(is_full_tx_bytes) + len(
tx_content_prefix) + len(tx_content_bytes)
short_tx_content_prefix_bytes = rlp_utils.get_length_prefix_list(
short_tx_content_size)
buf.append(short_tx_content_prefix_bytes)
buf.append(is_full_tx_bytes)
buf.append(tx_content_prefix)
buf.append(tx_content_bytes)
content_size += len(
short_tx_content_prefix_bytes) + short_tx_content_size
tx_start_index = tx_item_start + tx_item_length
tx_count += 1
list_of_txs_prefix_bytes = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(list_of_txs_prefix_bytes)
content_size += len(list_of_txs_prefix_bytes)
buf.appendleft(block_hdr_full_bytes)
content_size += len(block_hdr_full_bytes)
buf.append(remaining_bytes)
content_size += len(remaining_bytes)
compact_block_msg_prefix = rlp_utils.get_length_prefix_list(
content_size)
buf.appendleft(compact_block_msg_prefix)
content_size += len(compact_block_msg_prefix)
block = finalize_block_bytes(buf, content_size, used_short_ids)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(block_msg.block_hash(), used_short_ids,
compress_start_datetime,
datetime.datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
tx_count, bx_block_hash,
convert.bytes_to_hex(prev_block_bytes),
original_size, content_size,
100 - float(content_size) / original_size * 100,
ignored_sids)
return memoryview(block), block_info
def block_to_bx_block(
self,
block_msg: OntConsensusMessage,
tx_service: TransactionService,
enable_block_compression: bool,
min_tx_age_seconds: float
) -> Tuple[memoryview, BlockInfo]:
"""
Pack an Ontology consensus message's transactions into a bloXroute block.
"""
consensus_msg = block_msg
compress_start_datetime = datetime.utcnow()
compress_start_timestamp = time.time()
size = 0
buf = deque()
short_ids = []
ignored_sids = []
original_size = len(consensus_msg.rawbytes())
consensus_payload_header = consensus_msg.consensus_payload_header()
consensus_payload_header_len = bytearray(ont_constants.ONT_INT_LEN)
struct.pack_into("<L", consensus_payload_header_len, 0, len(consensus_payload_header))
size += ont_constants.ONT_INT_LEN
buf.append(consensus_payload_header_len)
size += len(consensus_payload_header)
buf.append(consensus_payload_header)
consensus_data_type = bytearray(ont_constants.ONT_CHAR_LEN)
struct.pack_into("<B", consensus_data_type, 0, consensus_msg.consensus_data_type())
size += ont_constants.ONT_CHAR_LEN
buf.append(consensus_data_type)
consensus_data_len = bytearray(ont_constants.ONT_INT_LEN)
struct.pack_into("<L", consensus_data_len, 0, consensus_msg.consensus_data_len())
size += ont_constants.ONT_INT_LEN
buf.append(consensus_data_len)
block_start_len = consensus_msg.block_start_len_memoryview()
txn_header = consensus_msg.txn_header()
block_start_len_and_txn_header_total_len = bytearray(ont_constants.ONT_INT_LEN)
struct.pack_into("<L", block_start_len_and_txn_header_total_len, 0, len(block_start_len) + len(txn_header))
size += ont_constants.ONT_INT_LEN
buf.append(block_start_len_and_txn_header_total_len)
size += len(block_start_len)
buf.append(block_start_len)
size += len(txn_header)
buf.append(txn_header)
max_timestamp_for_compression = time.time() - min_tx_age_seconds
for tx in consensus_msg.txns():
tx_hash, _ = ont_messages_util.get_txid(tx)
transaction_key = tx_service.get_transaction_key(tx_hash)
short_id = tx_service.get_short_id_by_key(transaction_key)
short_id_assign_time = 0
if short_id != constants.NULL_TX_SID:
short_id_assign_time = tx_service.get_short_id_assign_time(short_id)
if short_id == constants.NULL_TX_SID or \
not enable_block_compression or \
short_id_assign_time > max_timestamp_for_compression:
if short_id != constants.NULL_TX_SIDS:
ignored_sids.append(short_id)
buf.append(tx)
size += len(tx)
else:
short_ids.append(short_id)
buf.append(ont_constants.ONT_SHORT_ID_INDICATOR_AS_BYTEARRAY)
size += 1
# Prepend owner and signature, consensus payload tail, tx count and block hash to bx_block
owner_and_signature = consensus_msg.owner_and_signature()
owner_and_signature_len = bytearray(ont_constants.ONT_INT_LEN)
struct.pack_into("<L", owner_and_signature_len, 0, len(owner_and_signature))
size += len(owner_and_signature)
buf.appendleft(owner_and_signature)
size += ont_constants.ONT_INT_LEN
buf.appendleft(owner_and_signature_len)
payload_tail = consensus_msg.payload_tail()
payload_tail_len = bytearray(ont_constants.ONT_INT_LEN)
struct.pack_into("<L", payload_tail_len, 0, len(payload_tail))
size += len(payload_tail)
buf.appendleft(payload_tail)
size += ont_constants.ONT_INT_LEN
buf.appendleft(payload_tail_len)
txn_count = bytearray(ont_constants.ONT_INT_LEN)
struct.pack_into("<L", txn_count, 0, consensus_msg.txn_count())
size += ont_constants.ONT_INT_LEN
buf.appendleft(txn_count)
block_hash = consensus_msg.block_hash().binary
size += ont_constants.ONT_HASH_LEN
buf.appendleft(block_hash)
is_consensus_msg_buf = struct.pack("?", True)
buf.appendleft(is_consensus_msg_buf)
size += 1
block = finalize_block_bytes(buf, size, short_ids)
prev_block_hash = convert.bytes_to_hex(consensus_msg.prev_block_hash().binary)
bx_block_hash = convert.bytes_to_hex(crypto.double_sha256(block))
block_info = BlockInfo(
consensus_msg.block_hash(),
short_ids,
compress_start_datetime,
datetime.utcnow(),
(time.time() - compress_start_timestamp) * 1000,
consensus_msg.txn_count(),
bx_block_hash,
prev_block_hash,
original_size,
size,
100 - float(size) / original_size * 100,
ignored_sids
)
return memoryview(block), block_info
def compact_block_to_bx_block(
self, compact_block: CompactBlockBtcMessage,
transaction_service: TransactionService
) -> CompactBlockCompressionResult:
"""
Handle decompression of Bitcoin compact block.
Decompression converts compact block message to full block message.
"""
compress_start_datetime = datetime.utcnow()
block_header = compact_block.block_header()
sha256_hash = hashlib.sha256()
sha256_hash.update(block_header)
sha256_hash.update(compact_block.short_nonce_buf())
hex_digest = sha256_hash.digest()
key = hex_digest[0:16]
short_ids = compact_block.short_ids()
short_id_to_tx_contents = {}
for tx_hash in transaction_service.iter_transaction_hashes():
transaction_key = transaction_service.get_transaction_key(tx_hash)
tx_hash_binary = tx_hash.binary[::-1]
tx_short_id = compute_short_id(key, tx_hash_binary)
if tx_short_id in short_ids:
tx_content = transaction_service.get_transaction_by_key(
transaction_key)
if tx_content is None:
logger.debug(
"Hash {} is known by transactions service but content is missing.",
tx_hash)
else:
short_id_to_tx_contents[tx_short_id] = tx_content
if len(short_id_to_tx_contents) == len(short_ids):
break
block_transactions = []
missing_transactions_indices = []
pre_filled_transactions = compact_block.pre_filled_transactions()
total_txs_count = len(pre_filled_transactions) + len(short_ids)
size = 0
block_msg_parts = deque()
block_msg_parts.append(block_header)
size += len(block_header)
tx_count_size = btc_messages_util.get_sizeof_btc_varint(
total_txs_count)
tx_count_buf = bytearray(tx_count_size)
btc_messages_util.pack_int_to_btc_varint(total_txs_count, tx_count_buf,
0)
block_msg_parts.append(tx_count_buf)
size += tx_count_size
short_ids_iter = iter(short_ids.keys())
for index in range(total_txs_count):
if index not in pre_filled_transactions:
short_id = next(short_ids_iter)
if short_id in short_id_to_tx_contents:
short_tx = short_id_to_tx_contents[short_id]
block_msg_parts.append(short_tx)
block_transactions.append(short_tx)
size += len(short_tx)
else:
missing_transactions_indices.append(index)
block_transactions.append(None)
else:
pre_filled_transaction = pre_filled_transactions[index]
block_msg_parts.append(pre_filled_transaction)
block_transactions.append(pre_filled_transaction)
size += len(pre_filled_transaction)
recovered_item = CompactBlockRecoveryData(block_transactions,
block_header,
compact_block.magic(),
transaction_service)
block_info = BlockInfo(compact_block.block_hash(), [],
compress_start_datetime,
compress_start_datetime, 0, None, None, None,
len(compact_block.rawbytes()), None, None, [])
if len(missing_transactions_indices) > 0:
recovery_index = self._last_recovery_idx
self._last_recovery_idx += 1
self._recovery_items[
recovery_index] = recovered_item # pyre-ignore
return CompactBlockCompressionResult(False, block_info, None,
recovery_index,
missing_transactions_indices,
[])
result = CompactBlockCompressionResult(False, block_info, None, None,
[], [])
return self._recovered_compact_block_to_bx_block(
result, recovered_item)
class SyncTxServiceTest(MessageFactoryTestCase):
NETWORK_NUM = 12345
def setUp(self) -> None:
self.node = MockNode(helpers.get_common_opts(1234))
self.network_num = 4
self.transaction_service = TransactionService(self.node,
self.network_num)
def _add_transactions(self, tx_count, tx_size, short_id_offset=0):
short_id = short_id_offset
for i in range(int(tx_count)):
tx_hash = Sha256Hash(
binary=helpers.generate_bytearray(crypto.SHA256_HASH_LEN))
tx_content = helpers.generate_bytearray(tx_size)
transaction_key = self.transaction_service.get_transaction_key(
tx_hash)
self.transaction_service.set_transaction_contents_by_key(
transaction_key, tx_content)
for _ in range(random.randrange(1, 10)):
short_id += 1
self.transaction_service.assign_short_id(tx_hash, short_id)
self.transaction_service.set_short_id_transaction_type(
short_id, TransactionFlag.PAID_TX)
if short_id % 7 < 2:
self.transaction_service._short_id_to_tx_cache_key.pop(
short_id, None)
@skip("We don't sync tx service using time")
def test_create_tx_service_msg(self):
self._add_transactions(100000, tx_size=50)
done = False
msgs = []
timestamp = 0
snapshot_cache_keys = None
total_time = 0
total_txs = 0
while not done:
start_ = time.time()
txs_content_short_ids, timestamp, done, snapshot_cache_keys = \
tx_sync_service_helpers.create_txs_service_msg_from_time(
self.transaction_service,
timestamp,
False,
snapshot_cache_keys
)
duration = time.time() - start_
total_time += duration
total_txs += len(txs_content_short_ids)
msgs.append(txs_content_short_ids)
# print(f"txs:{len(txs_content_short_ids)}, time: {duration}")
print(f"total - msgs:{len(msgs)}, time:{total_time}")
msg_build_time = 0
for txs_content_short_ids in msgs:
start_ = time.time()
msg = TxServiceSyncTxsMessage(self.network_num,
txs_content_short_ids)
duration = time.time() - start_
msg_build_time += duration
print(f"total - message creation time: {msg_build_time}")
self.assertTrue(True)
@skip("We don't sync tx service using snapshot")
def test_create_tx_service_msg_snapshot(self):
self._add_transactions(100000, tx_size=50)
total_time = 0
start_ = time.time()
snapshot = self.transaction_service.get_snapshot(0)
duration = time.time() - start_
print(f"snapshot creation: {duration}")
total_time += duration
msgs = []
while snapshot:
start_ = time.time()
txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg(
self.transaction_service, snapshot, sync_tx_content=False)
msgs.append(txs_content_short_ids)
duration = time.time() - start_
total_time += duration
# print(len(txs_content_short_ids), duration)
print(f"total time: {total_time}")
@skip("We don't sync tx service using snapshot")
def test_create_tx_service_msg_snapshot_by_time(self):
self._add_transactions(100000, tx_size=50)
total_time = 0
start_ = time.time()
snapshot = self.transaction_service.get_snapshot(1.0)
duration = time.time() - start_
print(f"snapshot creation: {duration}")
total_time += duration
msgs = []
while snapshot:
start_ = time.time()
txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg(
self.transaction_service, snapshot, sync_tx_content=False)
msgs.append(txs_content_short_ids)
duration = time.time() - start_
total_time += duration
# print(len(txs_content_short_ids), duration)
print(f"total time: {total_time}")
class SyncTxServiceTest(MessageFactoryTestCase):
NETWORK_NUM = 12345
def setUp(self) -> None:
self.node = MockNode(helpers.get_common_opts(1234))
self.network_num = 4
self.transaction_service = TransactionService(self.node,
self.network_num)
def get_message_factory(self):
return bloxroute_message_factory
def test_create_message_success_tx_service_sync_txs_msg(self):
self._test_create_msg_success_tx_service_sync_with_tx_content_count(
100)
def test_create_message_success_tx_service_sync_txs_msg_with_exceeded_buf(
self):
self._test_create_msg_success_tx_service_sync_with_tx_content_count(
1000)
def _test_create_msg_success_tx_service_sync_with_tx_content_count(
self, tx_content_count, sync_tx_content=True):
short_ids = [
list(range(1, 6)),
list(range(11, 15)),
list(range(53, 250)), [31],
list(range(41, 48)), [51, 52]
]
transaction_hashes = list(
map(crypto.double_sha256, map(bytes, short_ids)))
for i in range(len(short_ids)):
transaction_content = bytearray(tx_content_count)
transaction_content[:32] = transaction_hashes[i]
transaction_key = self.transaction_service.get_transaction_key(
transaction_hashes[i])
self.transaction_service.set_transaction_contents_by_key(
transaction_key, transaction_content)
for short_id in short_ids[i]:
self.transaction_service.assign_short_id_by_key(
transaction_key, short_id)
# Six blocks received after
for i in range(len(short_ids)):
self.transaction_service.track_seen_short_ids(
Sha256Hash(helpers.generate_bytearray(32)), short_ids[i])
tx_service_snap = self.transaction_service.get_snapshot()
txs_content_short_ids = tx_sync_service_helpers.create_txs_service_msg(
self.transaction_service, tx_service_snap, sync_tx_content)
if txs_content_short_ids:
self._send_tx_msg(txs_content_short_ids, transaction_hashes)
def _send_tx_msg(self, txs_content_short_ids, transaction_hashes):
tx_service_sync_txs_msg: TxServiceSyncTxsMessage = \
self.create_message_successfully(
TxServiceSyncTxsMessage(
self.NETWORK_NUM, txs_content_short_ids
),
TxServiceSyncTxsMessage
)
self.assertEqual(self.NETWORK_NUM,
tx_service_sync_txs_msg.network_num())
self.assertEqual(len(txs_content_short_ids),
tx_service_sync_txs_msg.tx_count())
tx_service_txs_content_short_ids = tx_service_sync_txs_msg.txs_content_short_ids(
)
tx_contents = [
self.transaction_service.get_transaction(short_id).contents
for tx_content_short_id in tx_service_txs_content_short_ids
for short_id in tx_content_short_id.short_ids
]
for tx_content_short_id in tx_service_txs_content_short_ids:
self.assertIn(bytearray(tx_content_short_id.tx_hash),
transaction_hashes)
self.assertIn(bytearray(tx_content_short_id.tx_content),
tx_contents)
self.assertEqual(
tx_content_short_id.short_ids,
list(
self.transaction_service.get_short_ids_by_key(
self.transaction_service.get_transaction_key(
tx_content_short_id.tx_hash))))