def place_hold(self, block_hash, connection):
"""
Places hold on block hash and propagates message.
:param block_hash: ObjectHash
:param connection:
"""
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_REQUESTED,
network_num=connection.network_num,
more_info=stats_format.connection(connection))
if block_hash in self._node.blocks_seen.contents:
return
if block_hash not in self._holds.contents:
self._holds.add(block_hash, BlockHold(time.time(), connection))
conns = self._node.broadcast(BlockHoldingMessage(
block_hash, self._node.network_num),
broadcasting_conn=connection,
connection_types=[
ConnectionType.RELAY_BLOCK,
ConnectionType.GATEWAY
])
if len(conns) > 0:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_SENT_BY_GATEWAY_TO_PEERS,
network_num=self._node.network_num,
more_info=stats_format.connections(conns))
def _propagate_block_to_gateway_peers(self, cipher_hash, bx_block):
"""
Propagates unencrypted bx_block to all gateway peers for encryption and sending to bloXroute.
Also sends keys to bloXroute in case this was user error (e.g. no gateway peers).
Called after a timeout. This invalidates all future bx_block receipts.
"""
bx_block_hash = crypto.double_sha256(bx_block)
hex_bx_block_hash = convert.bytes_to_hex(bx_block_hash)
logger.debug("Did not receive enough receipts for: {}. Propagating compressed block to other gateways: {}",
cipher_hash, hex_bx_block_hash)
self._send_key(cipher_hash)
request = BlockPropagationRequestMessage(bx_block)
conns = self._node.broadcast(request, None, connection_types=(ConnectionType.GATEWAY,))
block_stats.add_block_event_by_block_hash(cipher_hash,
BlockStatEventType.ENC_BLOCK_PROPAGATION_NEEDED,
network_num=self._node.network_num,
compressed_block_hash=hex_bx_block_hash,
peers=conns,
more_info="Peers: {}, {} receipts".format(
stats_format.connections(conns),
self._receipt_tracker[cipher_hash]))
del self._receipt_tracker[cipher_hash]
del self._alarms[cipher_hash]
return constants.CANCEL_ALARMS
def _propagate_unencrypted_block_to_network(self, bx_block, connection,
block_info):
if block_info is None:
raise ValueError(
"Block info is required to propagate unencrypted block")
broadcast_message = BroadcastMessage(block_info.block_hash,
self._node.network_num,
is_encrypted=False,
blob=bx_block)
conns = self._node.broadcast(
broadcast_message,
connection,
connection_types=[ConnectionType.RELAY_BLOCK])
handling_duration = self._node.track_block_from_node_handling_ended(
block_info.block_hash)
block_stats.add_block_event_by_block_hash(
block_info.block_hash,
BlockStatEventType.ENC_BLOCK_SENT_FROM_GATEWAY_TO_NETWORK,
network_num=self._node.network_num,
requested_by_peer=False,
peers=conns,
more_info="Peers: {}; Unencrypted; {}; Handled in {}".format(
stats_format.connections(conns),
self._format_block_info_stats(block_info),
stats_format.duration(handling_duration)))
logger.info("Propagating block {} to the BDN.", block_info.block_hash)
return broadcast_message
def _propagate_encrypted_block_to_network(self, bx_block, connection,
block_info):
if block_info is None or block_info.block_hash is None:
block_hash = b"Unknown"
requested_by_peer = True
else:
block_hash = block_info.block_hash
requested_by_peer = False
encrypt_start_datetime = datetime.datetime.utcnow()
encrypt_start_timestamp = time.time()
encrypted_block, raw_cipher_hash = self._node.in_progress_blocks.encrypt_and_add_payload(
bx_block)
compressed_size = len(bx_block)
encrypted_size = len(encrypted_block)
encryption_details = "Encryption: {}; Size change: {}->{}bytes, {}".format(
stats_format.timespan(encrypt_start_timestamp, time.time()),
compressed_size, encrypted_size,
stats_format.ratio(encrypted_size, compressed_size))
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_ENCRYPTED,
start_date_time=encrypt_start_datetime,
end_date_time=datetime.datetime.utcnow(),
network_num=self._node.network_num,
matching_block_hash=convert.bytes_to_hex(raw_cipher_hash),
matching_block_type=StatBlockType.ENCRYPTED.value,
more_info=encryption_details)
cipher_hash = Sha256Hash(raw_cipher_hash)
broadcast_message = BroadcastMessage(cipher_hash,
self._node.network_num,
is_encrypted=True,
blob=encrypted_block)
conns = self._node.broadcast(
broadcast_message,
connection,
connection_types=[ConnectionType.RELAY_BLOCK])
handling_duration = self._node.track_block_from_node_handling_ended(
block_hash)
block_stats.add_block_event_by_block_hash(
cipher_hash,
BlockStatEventType.ENC_BLOCK_SENT_FROM_GATEWAY_TO_NETWORK,
network_num=self._node.network_num,
requested_by_peer=requested_by_peer,
peers=conns,
more_info="Peers: {}; {}; {}; Requested by peer: {}; Handled in {}"
.format(stats_format.connections(conns), encryption_details,
self._format_block_info_stats(block_info),
requested_by_peer,
stats_format.duration(handling_duration)))
self.register_for_block_receipts(cipher_hash, bx_block)
return broadcast_message
def _send_key(self, cipher_hash):
key = self._node.in_progress_blocks.get_encryption_key(
bytes(cipher_hash.binary))
key_message = KeyMessage(cipher_hash, self._node.network_num, key=key)
conns = self._node.broadcast(key_message,
None,
connection_types=[
ConnectionType.RELAY_BLOCK,
ConnectionType.GATEWAY
])
block_stats.add_block_event_by_block_hash(
cipher_hash,
BlockStatEventType.ENC_BLOCK_KEY_SENT_FROM_GATEWAY_TO_NETWORK,
network_num=self._node.network_num,
more_info=stats_format.connections(conns))
def queue_block_for_processing(self, block_message, connection):
"""
Queues up block for processing on timeout if hold message received.
If no hold exists, compress and broadcast block immediately.
:param block_message: block message to process
:param connection: receiving connection (AbstractBlockchainConnection)
"""
block_hash = block_message.block_hash()
connection.log_info("Processing block {} from local blockchain node.",
block_hash)
if block_hash in self._holds.contents:
hold: BlockHold = self._holds.contents[block_hash]
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_HELD_BLOCK,
network_num=connection.network_num,
more_info=stats_format.connection(hold.holding_connection))
if hold.alarm is None:
hold.alarm = self._node.alarm_queue.register_alarm(
self._node.opts.blockchain_block_hold_timeout_s,
self._holding_timeout, block_hash, hold)
hold.block_message = block_message
hold.connection = connection
else:
# Broadcast BlockHoldingMessage through relays and gateways
conns = self._node.broadcast(BlockHoldingMessage(
block_hash, self._node.network_num),
broadcasting_conn=connection,
prepend_to_queue=True,
connection_types=[
ConnectionType.RELAY_BLOCK,
ConnectionType.GATEWAY
])
if len(conns) > 0:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.BLOCK_HOLD_SENT_BY_GATEWAY_TO_PEERS,
network_num=self._node.network_num,
more_info=stats_format.connections(conns))
self._process_and_broadcast_block(block_message, connection)
def __init__(self,
event_settings: StatEventTypeSettings,
event_subject_id: str,
node_id: str,
start_date_time: datetime.datetime,
end_date_time: Optional[datetime.datetime] = None,
peers: Optional[List["AbstractConnection"]] = None,
**kwargs):
self.event_name = event_settings.name
self.event_logic = event_settings.event_logic
self.event_subject_id = event_subject_id
self.node_id = node_id
self.start_date_time = start_date_time
self.end_date_time = end_date_time if end_date_time is not None else start_date_time
self.extra_data = kwargs
if peers:
peer_ids = defaultdict(list)
for peer in peers:
if peer and peer.peer_id:
peer_ids[peer.CONNECTION_TYPE.format_short()].append(
peer.peer_id)
self.extra_data["peer_ids"] = peer_ids
self.extra_data["peers"] = stats_format.connections(peers)
def _propagate_unencrypted_block_to_network(self, bx_block, connection, block_info):
if block_info is None:
raise ValueError("Block info is required to propagate unencrypted block")
is_consensus_msg, = struct.unpack_from("?", bx_block[8:9])
broadcast_type = BroadcastMessageType.CONSENSUS if is_consensus_msg else BroadcastMessageType.BLOCK
broadcast_message = BroadcastMessage(block_info.block_hash, self._node.network_num,
broadcast_type=broadcast_type, is_encrypted=False, blob=bx_block)
conns = self._node.broadcast(broadcast_message, connection, connection_types=[ConnectionType.RELAY_BLOCK])
handling_duration = self._node.track_block_from_node_handling_ended(block_info.block_hash)
block_stats.add_block_event_by_block_hash(block_info.block_hash,
BlockStatEventType.ENC_BLOCK_SENT_FROM_GATEWAY_TO_NETWORK,
network_num=self._node.network_num,
broadcast_type=broadcast_type,
requested_by_peer=False,
peers=map(lambda conn: (conn.peer_desc, conn.CONNECTION_TYPE), conns),
more_info="Peers: {}; Unencrypted; {}; Handled in {}".format(
stats_format.connections(conns),
self._format_block_info_stats(block_info),
stats_format.duration(handling_duration)))
logger.info("Propagating block {} to the BDN.", block_info.block_hash)
return broadcast_message
def process_block_key(self, msg, connection: AbstractRelayConnection):
"""
Handles key message receive from bloXroute.
Looks for the encrypted block and decrypts; otherwise stores for later.
"""
key = msg.key()
block_hash = msg.block_hash()
if not self._node.should_process_block_hash(block_hash):
return
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_KEY_RECEIVED_BY_GATEWAY_FROM_NETWORK,
network_num=connection.network_num,
connection_type=connection.CONNECTION_TYPE,
more_info=stats_format.connection(connection))
if self._node.in_progress_blocks.has_encryption_key_for_hash(
block_hash):
return
if self._node.in_progress_blocks.has_ciphertext_for_hash(block_hash):
connection.log_trace(
"Cipher text found. Decrypting and sending to node.")
decrypt_start_timestamp = time.time()
decrypt_start_datetime = datetime.datetime.utcnow()
block = self._node.in_progress_blocks.decrypt_and_get_payload(
block_hash, key)
if block is not None:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_DECRYPTED_SUCCESS,
start_date_time=decrypt_start_datetime,
end_date_time=datetime.datetime.utcnow(),
network_num=connection.network_num,
more_info=stats_format.timespan(decrypt_start_timestamp,
time.time()))
self._handle_decrypted_block(
block,
connection,
encrypted_block_hash_hex=convert.bytes_to_hex(
block_hash.binary))
else:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_DECRYPTION_ERROR,
network_num=connection.network_num)
else:
connection.log_trace(
"No cipher text found on key message. Storing.")
self._node.in_progress_blocks.add_key(block_hash, key)
conns = self._node.broadcast(msg,
connection,
connection_types=[ConnectionType.GATEWAY])
if len(conns) > 0:
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_KEY_SENT_BY_GATEWAY_TO_PEERS,
network_num=self._node.network_num,
more_info=stats_format.connections(conns))
def process_block_broadcast(self, msg,
connection: AbstractRelayConnection):
"""
Handle broadcast message receive from bloXroute.
This is typically an encrypted block.
"""
# TODO handle the situation where txs that received from relays while syncing are in the blocks that were
# ignored while syncing, so these txs won't be cleaned for 3 days
if not self._node.should_process_block_hash(msg.block_hash()):
return
block_stats.add_block_event(
msg,
BlockStatEventType.ENC_BLOCK_RECEIVED_BY_GATEWAY_FROM_NETWORK,
network_num=connection.network_num,
more_info=stats_format.connection(connection))
block_hash = msg.block_hash()
is_encrypted = msg.is_encrypted()
self._node.track_block_from_bdn_handling_started(
block_hash, connection.peer_desc)
if not is_encrypted:
block = msg.blob()
self._handle_decrypted_block(block, connection)
return
cipherblob = msg.blob()
expected_hash = Sha256Hash(crypto.double_sha256(cipherblob))
if block_hash != expected_hash:
connection.log_warning(log_messages.BLOCK_WITH_INCONSISTENT_HASHES,
expected_hash, block_hash)
return
if self._node.in_progress_blocks.has_encryption_key_for_hash(
block_hash):
connection.log_trace(
"Already had key for received block. Sending block to node.")
decrypt_start_timestamp = time.time()
decrypt_start_datetime = datetime.datetime.utcnow()
block = self._node.in_progress_blocks.decrypt_ciphertext(
block_hash, cipherblob)
if block is not None:
block_stats.add_block_event(
msg,
BlockStatEventType.ENC_BLOCK_DECRYPTED_SUCCESS,
start_date_time=decrypt_start_datetime,
end_date_time=datetime.datetime.utcnow(),
network_num=connection.network_num,
more_info=stats_format.timespan(decrypt_start_timestamp,
time.time()))
self._handle_decrypted_block(
block,
connection,
encrypted_block_hash_hex=convert.bytes_to_hex(
block_hash.binary))
else:
block_stats.add_block_event(
msg,
BlockStatEventType.ENC_BLOCK_DECRYPTION_ERROR,
network_num=connection.network_num)
else:
connection.log_trace("Received encrypted block. Storing.")
self._node.in_progress_blocks.add_ciphertext(
block_hash, cipherblob)
block_received_message = BlockReceivedMessage(block_hash)
conns = self._node.broadcast(
block_received_message,
connection,
connection_types=[ConnectionType.GATEWAY])
block_stats.add_block_event_by_block_hash(
block_hash,
BlockStatEventType.ENC_BLOCK_SENT_BLOCK_RECEIPT,
network_num=connection.network_num,
more_info=stats_format.connections(conns))
def msg_consensus(self, msg: OntConsensusMessage):
if not self.node.opts.is_consensus:
return
if msg.consensus_data_type() != ont_constants.BLOCK_PROPOSAL_CONSENSUS_MESSAGE_TYPE:
return
block_hash = msg.block_hash()
node = self.connection.node
if not node.should_process_block_hash(block_hash):
return
node.block_cleanup_service.on_new_block_received(block_hash, msg.prev_block_hash())
block_stats.add_block_event_by_block_hash(block_hash,
BlockStatEventType.BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE,
network_num=self.connection.network_num,
broadcast_type=BroadcastMessageType.CONSENSUS,
more_info="Protocol: {}, Network: {}".format(
node.opts.blockchain_protocol,
node.opts.blockchain_network
),
msg_size=len(msg.rawbytes())
)
if block_hash in self.connection.node.blocks_seen.contents:
block_stats.add_block_event_by_block_hash(block_hash,
BlockStatEventType.BLOCK_RECEIVED_FROM_BLOCKCHAIN_NODE_IGNORE_SEEN,
network_num=self.connection.network_num,
broadcast_type=BroadcastMessageType.CONSENSUS)
self.connection.log_info(
"Discarding duplicate consensus block {} from local blockchain node.",
block_hash
)
return
node.track_block_from_node_handling_started(block_hash)
self.connection.log_info(
"Processing consensus block {} from local blockchain node.",
block_hash
)
# Broadcast BlockHoldingMessage through relays and gateways
conns = self.node.broadcast(BlockHoldingMessage(block_hash, self.node.network_num),
broadcasting_conn=self.connection, prepend_to_queue=True,
connection_types=[ConnectionType.RELAY_BLOCK, ConnectionType.GATEWAY])
if len(conns) > 0:
block_stats.add_block_event_by_block_hash(block_hash,
BlockStatEventType.BLOCK_HOLD_SENT_BY_GATEWAY_TO_PEERS,
network_num=self.node.network_num,
broadcast_type=BroadcastMessageType.CONSENSUS,
more_info=stats_format.connections(conns))
try:
bx_block, block_info = self.node.consensus_message_converter.block_to_bx_block(
msg, self.node.get_tx_service()
)
except MessageConversionError as e:
block_stats.add_block_event_by_block_hash(
e.msg_hash,
BlockStatEventType.BLOCK_CONVERSION_FAILED,
network_num=self.connection.network_num,
broadcast_type=BroadcastMessageType.CONSENSUS,
conversion_type=e.conversion_type.value
)
self.connection.log_error(log_messages.BLOCK_COMPRESSION_FAIL_ONT_CONSENSUS, e.msg_hash, e)
return
block_stats.add_block_event_by_block_hash(block_hash,
BlockStatEventType.BLOCK_COMPRESSED,
start_date_time=block_info.start_datetime,
end_date_time=block_info.end_datetime,
network_num=self.connection.network_num,
broadcast_type=BroadcastMessageType.CONSENSUS,
prev_block_hash=block_info.prev_block_hash,
original_size=block_info.original_size,
txs_count=block_info.txn_count,
blockchain_network=self.node.opts.blockchain_network,
blockchain_protocol=self.node.opts.blockchain_protocol,
matching_block_hash=block_info.compressed_block_hash,
matching_block_type=StatBlockType.COMPRESSED.value,
more_info="Consensus compression: {}->{} bytes, {}, {}; "
"Tx count: {}".format(
block_info.original_size,
block_info.compressed_size,
stats_format.percentage(block_info.compression_rate),
stats_format.duration(block_info.duration_ms),
block_info.txn_count
)
)
self.node.block_processing_service._process_and_broadcast_compressed_block(bx_block, self.connection,
block_info, block_hash)