def msg_txs(self, msg: TxsMessage):
if ConnectionType.RELAY_TRANSACTION not in self.CONNECTION_TYPE:
self.log_error(log_messages.UNEXPECTED_TXS_ON_NON_RELAY_CONN, msg)
return
transactions = msg.get_txs()
tx_service = self.node.get_tx_service()
tx_stats.add_txs_by_short_ids_event(
map(lambda x: x.short_id, transactions),
TransactionStatEventType.
TX_UNKNOWN_SHORT_IDS_REPLY_RECEIVED_BY_GATEWAY_FROM_RELAY,
network_num=self.node.network_num,
peer=stats_format.connection(self),
found_tx_hashes=map(lambda x: convert.bytes_to_hex(x.hash.binary),
transactions))
for transaction in transactions:
tx_hash, transaction_contents, short_id = transaction
assert tx_hash is not None
assert short_id is not None
self.node.block_recovery_service.check_missing_sid(short_id)
if not tx_service.has_short_id(short_id):
tx_service.assign_short_id(tx_hash, short_id)
self.node.block_recovery_service.check_missing_tx_hash(tx_hash)
if not tx_service.has_transaction_contents(tx_hash):
assert transaction_contents is not None
tx_service.set_transaction_contents(tx_hash,
transaction_contents)
tx_stats.add_tx_by_hash_event(
tx_hash,
TransactionStatEventType.
TX_UNKNOWN_TRANSACTION_RECEIVED_BY_GATEWAY_FROM_RELAY,
self.node.network_num,
short_id,
peer=stats_format.connection(self))
self.node.block_processing_service.retry_broadcast_recovered_blocks(
self)
for block_awaiting_recovery in self.node.block_recovery_service.get_blocks_awaiting_recovery(
):
self.node.block_processing_service.schedule_recovery_retry(
block_awaiting_recovery)
def record_block_receipt(self, cipher_hash, connection):
"""
Records a receipt of a block hash. Releases key if threshold reached.
:param cipher_hash encrypted block ObjectHash
:param connection posting block received receipt
"""
if cipher_hash in self._receipt_tracker:
self._receipt_tracker[cipher_hash] += 1
block_stats.add_block_event_by_block_hash(
cipher_hash,
BlockStatEventType.ENC_BLOCK_RECEIVED_BLOCK_RECEIPT,
network_num=self._node.network_num,
peers=[connection],
more_info="{}, {} receipts".format(
stats_format.connection(connection),
self._receipt_tracker[cipher_hash]))
if self._are_enough_receipts_received(cipher_hash):
logger.debug(
"Received enough block receipt messages. Releasing key for block with hash: {}",
convert.bytes_to_hex(cipher_hash.binary))
self._send_key(cipher_hash)
self._node.alarm_queue.unregister_alarm(
self._alarms[cipher_hash])
del self._receipt_tracker[cipher_hash]
del self._alarms[cipher_hash]
def place_hold(self, block_hash, connection):
"""
Places hold on block hash and propagates message.
:param block_hash: ObjectHash
:param connection:
"""
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_REQUESTED,
network_num=connection.network_num,
more_info=stats_format.connection(connection))
if block_hash in self._node.blocks_seen.contents:
return
if block_hash not in self._holds.contents:
self._holds.add(block_hash, BlockHold(time.time(), connection))
conns = self._node.broadcast(BlockHoldingMessage(
block_hash, self._node.network_num),
broadcasting_conn=connection,
connection_types=[
ConnectionType.RELAY_BLOCK,
ConnectionType.GATEWAY
])
if len(conns) > 0:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_SENT_BY_GATEWAY_TO_PEERS,
network_num=self._node.network_num,
more_info=stats_format.connections(conns))
def _holding_timeout(self, block_hash, hold):
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_TIMED_OUT,
network_num=hold.connection.network_num,
more_info=stats_format.connection(hold.connection))
self._process_and_broadcast_block(hold.block_message, hold.connection)
def msg_compressed_block_txs(self, msg: CompressedBlockTxsMessage):
start_time = time.time()
self.msg_txs(msg.to_txs_message(),
RecoveredTxsSource.COMPRESSED_BLOCK_TXS_RECEIVED)
block_stats.add_block_event_by_block_hash(
msg.block_hash(),
BlockStatEventType.ENC_BLOCK_COMPRESSED_TXS_RECEIVED_BY_GATEWAY,
network_num=msg.network_num(),
peers=[self],
more_info="{}. processing time: {}".format(
stats_format.connection(self),
stats_format.timespan(start_time, time.time())))
def cancel_hold_timeout(self, block_hash, connection):
"""
Lifts hold on block hash and cancels timeout.
:param block_hash: ObjectHash
:param connection: connection cancelling hold
"""
if block_hash in self._holds.contents:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_LIFTED,
network_num=connection.network_num,
more_info=stats_format.connection(connection))
hold = self._holds.contents[block_hash]
if hold.alarm is not None:
self._node.alarm_queue.unregister_alarm(hold.alarm)
del self._holds.contents[block_hash]
def queue_block_for_processing(self, block_message, connection):
"""
Queues up block for processing on timeout if hold message received.
If no hold exists, compress and broadcast block immediately.
:param block_message: block message to process
:param connection: receiving connection (AbstractBlockchainConnection)
"""
block_hash = block_message.block_hash()
connection.log_info("Processing block {} from local blockchain node.",
block_hash)
if block_hash in self._holds.contents:
hold: BlockHold = self._holds.contents[block_hash]
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_HELD_BLOCK,
network_num=connection.network_num,
more_info=stats_format.connection(hold.holding_connection))
if hold.alarm is None:
hold.alarm = self._node.alarm_queue.register_alarm(
self._node.opts.blockchain_block_hold_timeout_s,
self._holding_timeout, block_hash, hold)
hold.block_message = block_message
hold.connection = connection
else:
# Broadcast BlockHoldingMessage through relays and gateways
conns = self._node.broadcast(BlockHoldingMessage(
block_hash, self._node.network_num),
broadcasting_conn=connection,
prepend_to_queue=True,
connection_types=[
ConnectionType.RELAY_BLOCK,
ConnectionType.GATEWAY
])
if len(conns) > 0:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_SENT_BY_GATEWAY_TO_PEERS,
network_num=self._node.network_num,
more_info=stats_format.connections(conns))
self._process_and_broadcast_block(block_message, connection)
def process_block_broadcast(self, msg,
connection: AbstractRelayConnection) -> None:
"""
Handle broadcast message receive from bloXroute.
This is typically an encrypted block.
"""
# TODO handle the situation where txs that received from relays while syncing are in the blocks that were
# ignored while syncing, so these txs won't be cleaned for 3 days
if not self._node.should_process_block_hash(msg.block_hash()):
return
block_stats.add_block_event(
msg,
BlockStatEventType.ENC_BLOCK_RECEIVED_BY_GATEWAY_FROM_NETWORK,
network_num=connection.network_num,
more_info=stats_format.connection(connection))
block_hash = msg.block_hash()
is_encrypted = msg.is_encrypted()
self._node.track_block_from_bdn_handling_started(
block_hash, connection.peer_desc)
if not is_encrypted:
block = msg.blob()
self._handle_decrypted_block(block, connection)
return
cipherblob = msg.blob()
expected_hash = Sha256Hash(crypto.double_sha256(cipherblob))
if block_hash != expected_hash:
connection.log_warning(log_messages.BLOCK_WITH_INCONSISTENT_HASHES,
expected_hash, block_hash)
return
if self._node.in_progress_blocks.has_encryption_key_for_hash(
block_hash):
connection.log_trace(
"Already had key for received block. Sending block to node.")
decrypt_start_timestamp = time.time()
decrypt_start_datetime = datetime.datetime.utcnow()
block = self._node.in_progress_blocks.decrypt_ciphertext(
block_hash, cipherblob)
if block is not None:
block_stats.add_block_event(
msg,
BlockStatEventType.ENC_BLOCK_DECRYPTED_SUCCESS,
start_date_time=decrypt_start_datetime,
end_date_time=datetime.datetime.utcnow(),
network_num=connection.network_num,
more_info=stats_format.timespan(decrypt_start_timestamp,
time.time()))
self._handle_decrypted_block(
block,
connection,
encrypted_block_hash_hex=convert.bytes_to_hex(
block_hash.binary))
else:
block_stats.add_block_event(
msg,
BlockStatEventType.ENC_BLOCK_DECRYPTION_ERROR,
network_num=connection.network_num)
else:
connection.log_trace("Received encrypted block. Storing.")
self._node.in_progress_blocks.add_ciphertext(
block_hash, cipherblob)
block_received_message = BlockReceivedMessage(block_hash)
conns = self._node.broadcast(
block_received_message,
connection,
connection_types=(ConnectionType.GATEWAY, ))
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_SENT_BLOCK_RECEIPT,
network_num=connection.network_num,
peers=conns,
)
def process_block_key(self, msg, connection: AbstractRelayConnection):
"""
Handles key message receive from bloXroute.
Looks for the encrypted block and decrypts; otherwise stores for later.
"""
key = msg.key()
block_hash = msg.block_hash()
if not self._node.should_process_block_hash(block_hash):
return
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_KEY_RECEIVED_BY_GATEWAY_FROM_NETWORK,
network_num=connection.network_num,
connection_type=connection.CONNECTION_TYPE,
more_info=stats_format.connection(connection))
if self._node.in_progress_blocks.has_encryption_key_for_hash(
block_hash):
return
if self._node.in_progress_blocks.has_ciphertext_for_hash(block_hash):
connection.log_trace(
"Cipher text found. Decrypting and sending to node.")
decrypt_start_timestamp = time.time()
decrypt_start_datetime = datetime.datetime.utcnow()
block = self._node.in_progress_blocks.decrypt_and_get_payload(
block_hash, key)
if block is not None:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_DECRYPTED_SUCCESS,
start_date_time=decrypt_start_datetime,
end_date_time=datetime.datetime.utcnow(),
network_num=connection.network_num,
more_info=stats_format.timespan(decrypt_start_timestamp,
time.time()))
self._handle_decrypted_block(
block,
connection,
encrypted_block_hash_hex=convert.bytes_to_hex(
block_hash.binary))
else:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_DECRYPTION_ERROR,
network_num=connection.network_num)
else:
connection.log_trace(
"No cipher text found on key message. Storing.")
self._node.in_progress_blocks.add_key(block_hash, key)
conns = self._node.broadcast(msg,
connection,
connection_types=[ConnectionType.GATEWAY])
if len(conns) > 0:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_KEY_SENT_BY_GATEWAY_TO_PEERS,
network_num=self._node.network_num,
more_info=stats_format.connections(conns))
def msg_tx(self, msg):
"""
Handle a TX message by broadcasting to the entire network
"""
start_time = time.time()
txn_count = 0
broadcast_txs_count = 0
process_tx_msg_result = self.tx_service.process_transactions_message_from_node(
msg, self.node.get_network_min_transaction_fee(),
self.node.opts.transaction_validation)
if not self.node.opts.has_fully_updated_tx_service:
logger.debug(
"Gateway received {} transaction messages while syncing, skipping..",
len(process_tx_msg_result))
return
broadcast_start_time = time.time()
for tx_result in process_tx_msg_result:
txn_count += 1
if TxValidationStatus.INVALID_FORMAT in tx_result.tx_validation_status:
gateway_transaction_stats_service.log_tx_validation_failed_structure(
)
logger.warning(
log_messages.NODE_RECEIVED_TX_WITH_INVALID_FORMAT,
self.node.NODE_TYPE, tx_result.transaction_hash,
stats_format.connection(self.connection))
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.TX_VALIDATION_FAILED_STRUCTURE,
self.connection.network_num,
peers=[self.connection])
continue
if TxValidationStatus.INVALID_SIGNATURE in tx_result.tx_validation_status:
gateway_transaction_stats_service.log_tx_validation_failed_signature(
)
logger.warning(log_messages.NODE_RECEIVED_TX_WITH_INVALID_SIG,
self.node.NODE_TYPE, tx_result.transaction_hash,
stats_format.connection(self.connection))
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.TX_VALIDATION_FAILED_SIGNATURE,
self.connection.network_num,
peers=[self.connection])
continue
if TxValidationStatus.LOW_FEE in tx_result.tx_validation_status:
gateway_transaction_stats_service.log_tx_validation_failed_gas_price(
)
# log low fee transaction here for bdn_performance
gateway_bdn_performance_stats_service.log_tx_from_blockchain_node(
True)
logger.trace(
"transaction {} has gas price lower then the setting {}",
tx_result.transaction_hash,
self.node.get_network_min_transaction_fee())
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.TX_VALIDATION_FAILED_GAS_PRICE,
self.connection.network_num,
peers=[self.connection])
continue
if tx_result.seen:
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.
TX_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN,
self.connection.network_num,
peers=[self.connection])
gateway_transaction_stats_service.log_duplicate_transaction_from_blockchain(
)
continue
broadcast_txs_count += 1
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.TX_RECEIVED_FROM_BLOCKCHAIN_NODE,
self.connection.network_num,
peers=[self.connection])
gateway_transaction_stats_service.log_transaction_from_blockchain(
tx_result.transaction_hash)
# log transactions that passed validation, according to fee
gateway_bdn_performance_stats_service.log_tx_from_blockchain_node(
not self.node.is_gas_price_above_min_network_fee(
tx_result.transaction_contents))
# All connections outside of this one is a bloXroute server
broadcast_peers = self.node.broadcast(
tx_result.bdn_transaction_message,
self.connection,
connection_types=[ConnectionType.RELAY_TRANSACTION])
self.node.broadcast(
msg,
self.connection,
connection_types=[ConnectionType.BLOCKCHAIN_NODE])
if self.node.opts.ws:
self.publish_transaction(
tx_result.transaction_hash,
memoryview(tx_result.transaction_contents))
if broadcast_peers:
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.TX_SENT_FROM_GATEWAY_TO_PEERS,
self.connection.network_num,
peers=broadcast_peers)
else:
logger.trace(
"Tx Message: {} from BlockchainNode was dropped, no upstream relay connection available",
tx_result.transaction_hash)
set_content_start_time = time.time()
end_time = time.time()
total_duration_ms = (end_time - start_time) * 1000
duration_before_broadcast_ms = (broadcast_start_time -
start_time) * 1000
duration_broadcast_ms = (set_content_start_time -
broadcast_start_time) * 1000
duration_set_content_ms = (end_time - set_content_start_time) * 1000
gateway_transaction_stats_service.log_processed_node_transaction(
total_duration_ms, duration_before_broadcast_ms,
duration_broadcast_ms, duration_set_content_ms,
broadcast_txs_count)
performance_utils.log_operation_duration(
msg_handling_logger,
"Process single transaction from Blockchain",
start_time,
gateway_constants.
BLOCKCHAIN_TX_PROCESSING_TIME_WARNING_THRESHOLD_S,
connection=self,
message=msg,
total_duration_ms=total_duration_ms,
duration_before_broadcast_ms=duration_before_broadcast_ms,
duration_broadcast_ms=duration_broadcast_ms,
duration_set_content_ms=duration_set_content_ms)
def msg_tx(self, msg):
"""
Handle transactions receive from bloXroute network.
"""
start_time = time.time()
if ConnectionType.RELAY_TRANSACTION not in self.CONNECTION_TYPE:
self.log_error(log_messages.UNEXPECTED_TX_MESSAGE, msg)
return
tx_service = self.node.get_tx_service()
tx_hash = msg.tx_hash()
short_id = msg.short_id()
tx_contents = msg.tx_val()
is_compact = msg.is_compact()
network_num = msg.network_num()
attempt_recovery = False
ext_start_time = time.time()
processing_result = tx_service.process_gateway_transaction_from_bdn(
tx_hash, short_id, tx_contents, is_compact)
ext_end_time = time.time()
if processing_result.ignore_seen:
gateway_transaction_stats_service.log_duplicate_transaction_from_relay(
)
tx_stats.add_tx_by_hash_event(
tx_hash,
TransactionStatEventType.
TX_RECEIVED_BY_GATEWAY_FROM_PEER_IGNORE_SEEN,
network_num,
short_id,
peer=stats_format.connection(self),
is_compact_transaction=False)
self.log_trace("Transaction has already been seen: {}", tx_hash)
return
if processing_result.existing_short_id:
gateway_transaction_stats_service.log_duplicate_transaction_from_relay(
is_compact)
tx_stats.add_tx_by_hash_event(
tx_hash,
TransactionStatEventType.
TX_RECEIVED_BY_GATEWAY_FROM_PEER_IGNORE_SEEN,
network_num,
short_id,
peer=stats_format.connection(self),
is_compact_transaction=is_compact)
return
tx_stats.add_tx_by_hash_event(
tx_hash,
TransactionStatEventType.TX_RECEIVED_BY_GATEWAY_FROM_PEER,
network_num,
short_id,
peer=stats_format.connection(self),
is_compact_transaction=msg.is_compact())
gateway_transaction_stats_service.log_transaction_from_relay(
tx_hash, short_id is not None, msg.is_compact())
if processing_result.assigned_short_id:
was_missing = self.node.block_recovery_service.check_missing_sid(
short_id)
attempt_recovery |= was_missing
tx_stats.add_tx_by_hash_event(
tx_hash,
TransactionStatEventType.TX_SHORT_ID_STORED_BY_GATEWAY,
network_num,
short_id,
was_missing=was_missing)
gateway_transaction_stats_service.log_short_id_assignment_processed(
)
elif not short_id:
tx_stats.add_tx_by_hash_event(
tx_hash,
TransactionStatEventType.TX_SHORT_ID_EMPTY_IN_MSG_FROM_RELAY,
network_num,
short_id,
peer=stats_format.connection(self))
if not is_compact and processing_result.existing_contents:
gateway_transaction_stats_service.log_redundant_transaction_content(
)
if processing_result.set_content:
self.log_trace(
"Adding hash value to tx service and forwarding it to node")
gateway_bdn_performance_stats_service.log_tx_from_bdn()
attempt_recovery |= self.node.block_recovery_service.check_missing_tx_hash(
tx_hash)
if self.node.node_conn is not None:
blockchain_tx_message = self.node.message_converter.bx_tx_to_tx(
msg)
self.publish_new_transaction(tx_hash, tx_contents)
transaction_feed_stats_service.log_new_transaction(tx_hash)
self.node.send_msg_to_node(blockchain_tx_message)
tx_stats.add_tx_by_hash_event(
tx_hash, TransactionStatEventType.
TX_SENT_FROM_GATEWAY_TO_BLOCKCHAIN_NODE, network_num,
short_id)
if attempt_recovery:
self.node.block_processing_service.retry_broadcast_recovered_blocks(
self)
end_time = time.time()
total_duration_ms = (end_time - start_time) * 1000
duration_before_ext_ms = (ext_start_time - start_time) * 1000
duration_ext_ms = (ext_end_time - ext_start_time) * 1000
duration_after_ext_ms = (end_time - ext_end_time) * 1000
gateway_transaction_stats_service.log_processed_bdn_transaction(
total_duration_ms, duration_before_ext_ms, duration_ext_ms,
duration_after_ext_ms)
performance_utils.log_operation_duration(
msg_handling_logger,
"Process single transaction from BDN",
start_time,
gateway_constants.BDN_TX_PROCESSING_TIME_WARNING_THRESHOLD_S,
connection=self,
message=msg,
total_duration_ms=total_duration_ms,
duration_before_ext_ms=duration_before_ext_ms,
duration_ext_ms=duration_ext_ms,
duration_after_ext_ms=duration_after_ext_ms)
def msg_tx(self, msg):
"""
Handle a TX message by broadcasting to the entire network
"""
start_time = time.time()
txn_count = 0
broadcast_txs_count = 0
tx_service = self.connection.node.get_tx_service()
process_tx_msg_result = tx_service.process_transactions_message_from_node(msg)
broadcast_start_time = time.time()
for tx_result in process_tx_msg_result:
txn_count += 1
if tx_result.seen:
tx_stats.add_tx_by_hash_event(tx_result.transaction_hash,
TransactionStatEventType.TX_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN,
self.connection.network_num,
peer=stats_format.connection(self.connection))
gateway_transaction_stats_service.log_duplicate_transaction_from_blockchain()
continue
broadcast_txs_count += 1
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.TX_RECEIVED_FROM_BLOCKCHAIN_NODE,
self.connection.network_num,
peer=stats_format.connection(self.connection)
)
gateway_transaction_stats_service.log_transaction_from_blockchain(tx_result.transaction_hash)
gateway_bdn_performance_stats_service.log_tx_from_blockchain_node()
# All connections outside of this one is a bloXroute server
broadcast_peers = self.connection.node.broadcast(
tx_result.bdn_transaction_message,
self.connection,
connection_types=[ConnectionType.RELAY_TRANSACTION]
)
if self.connection.node.opts.ws:
self.publish_transaction(
tx_result.transaction_hash, memoryview(tx_result.transaction_contents)
)
if broadcast_peers:
tx_stats.add_tx_by_hash_event(
tx_result.transaction_hash,
TransactionStatEventType.TX_SENT_FROM_GATEWAY_TO_PEERS,
self.connection.network_num,
peers=map(lambda conn: (stats_format.connection(conn)), broadcast_peers)
)
else:
logger.trace(
"Tx Message: {} from BlockchainNode was dropped, no upstream relay connection available",
tx_result.transaction_hash
)
set_content_start_time = time.time()
end_time = time.time()
total_duration_ms = (end_time - start_time) * 1000
duration_before_broadcast_ms = (broadcast_start_time - start_time) * 1000
duration_broadcast_ms = (set_content_start_time - broadcast_start_time) * 1000
duration_set_content_ms = (end_time - set_content_start_time) * 1000
gateway_transaction_stats_service.log_processed_node_transaction(
total_duration_ms,
duration_before_broadcast_ms,
duration_broadcast_ms,
duration_set_content_ms,
broadcast_txs_count
)
performance_utils.log_operation_duration(
msg_handling_logger,
"Process single transaction from Blockchain",
start_time,
gateway_constants.BLOCKCHAIN_TX_PROCESSING_TIME_WARNING_THRESHOLD_S,
connection=self,
message=msg,
total_duration_ms=total_duration_ms,
duration_before_broadcast_ms=duration_before_broadcast_ms,
duration_broadcast_ms=duration_broadcast_ms,
duration_set_content_ms=duration_set_content_ms
)