def recheck(self):
"""
Recheck all watchpoints.
Returns:
dict, metadata info.
"""
metadata_stream = self._metadata_stream
# validate backend status is able to recheck watchpoint
if not metadata_stream.enable_recheck:
log.error("Recheck is not available.")
raise DebuggerRecheckError("Recheck is not available.")
metadata_stream.state = ServerStatus.SENDING.value
metadata_stream.enable_recheck = False
# send updated watchpoint and recheck command
try:
event = self._construct_run_event({'level': 'recheck'})
self._send_watchpoints()
self._cache_store.put_command(event)
except MindInsightException as err:
log.error("Failed to send recheck event.")
log.exception(err)
metadata_stream.state = ServerStatus.WAITING.value
metadata_stream.enable_recheck = True
raise DebuggerContinueError("Failed to send recheck command.")
else:
log.debug("Send the recheck to command queue.")
return metadata_stream.get(['state', 'enable_recheck'])
def run(self):
"""Start the debugger offline server."""
log.info("Initialize Offline Debugger Server for dbg_dir: %s",
self._context.dbg_dir)
self._offline_server_manager.initialize()
log.info("Start Offline Debugger Server for dbg_dir: %s",
self._context.dbg_dir)
self._running.set()
try_count = 0
while self._running.is_set(
) and try_count < self._MAX_TRY_EXCEPT_COUNT:
try:
self._offline_server_manager.wait_for_termination()
if not self._offline_server_manager.is_runnable():
break
except MindInsightException as err:
log.exception(err)
log.warning(
"Error happens during listening on user commands. Restart listening again."
)
finally:
try_count += 1
# protect server from too much failure commands.
if try_count == self._MAX_TRY_EXCEPT_COUNT:
self._cache_store.clean()
metadata = self._cache_store.get_stream_handler(
Streams.METADATA).get()
self._cache_store.put_data(metadata)
log.warning("Exception exceed %d times, stop server.", try_count)
def continue_training(self, params):
"""
Send RunCMD to MindSpore.
Args:
params (dict): The control params.
Returns:
dict, metadata info.
"""
metadata_stream = self._metadata_stream
if metadata_stream.state != ServerStatus.WAITING.value:
log.error("MindSpore is not ready to run. Current state is: %s",
metadata_stream.state)
raise DebuggerContinueError(
"MindSpore is not ready to run or is running currently.")
metadata_stream.state = ServerStatus.RUNNING.value
try:
self._validate_continue_params(params)
event = self._construct_run_event(params)
self._send_watchpoints()
self._cache_store.put_command(event)
except MindInsightException as err:
log.error("Failed to send run event.")
log.exception(err)
metadata_stream.state = ServerStatus.WAITING.value
raise DebuggerContinueError("Failed to send run command.")
else:
metadata_stream.enable_recheck = False
log.debug("Send the RunCMD to command queue.")
return metadata_stream.get(['state', 'enable_recheck'])
def continue_training(self, params):
"""
Send RunCMD to MindSpore.
Args:
params (dict): The control params.
Returns:
dict, metadata info.
"""
metadata_stream = self._metadata_stream
if metadata_stream.state != ServerStatus.WAITING.value:
log.error("MindSpore is not ready to run. Current state is: %s",
metadata_stream.state)
raise DebuggerContinueError(
"MindSpore is not ready to run or is running currently.")
metadata_stream.state = ServerStatus.SENDING.value
try:
self._validate_continue_params(params)
event = self._construct_run_event(params)
# whether need to send recheck before continue, especially for initialization watchpoint
recheck_flag = bool(self._metadata_stream.step == 0
and self._watchpoint_stream.is_recheckable())
self._send_watchpoints()
if recheck_flag:
self._cache_store.put_command(
self._construct_run_event({'level': 'recheck'}))
log.info(
"Send recheck command for initialization watchpoints before continue command."
)
self._cache_store.put_command(event)
except MindInsightException as err:
log.error("Failed to send run event.")
log.exception(err)
metadata_stream.state = ServerStatus.WAITING.value
raise DebuggerContinueError("Failed to send run command.")
else:
metadata_stream.enable_recheck = False
log.debug("Send the RunCMD to command queue.")
return metadata_stream.get(['state', 'enable_recheck'])
def get_tensor_value_by_shape(self, shape=None):
"""
Get tensor value by shape.
Args:
shape (tuple): The specified shape.
Returns:
Union[None, str, numpy.ndarray], the value of parsed tensor.
"""
if self._value is None:
log.warning("%s has no value yet.", self.name)
return None
if shape is None or not isinstance(shape, tuple):
log.info("Get the whole tensor value with shape is %s", shape)
return self._value
if len(shape) != len(self.shape):
log.error("Invalid shape. Received: %s, tensor shape: %s", shape,
self.shape)
raise DebuggerParamValueError("Invalid shape. Shape unmatched.")
try:
value = self._value[shape]
except IndexError as err:
log.error("Invalid shape. Received: %s, tensor shape: %s", shape,
self.shape)
log.exception(err)
raise DebuggerParamValueError("Invalid shape. Shape unmatched.")
if isinstance(value, np.ndarray):
if value.size > self.max_number_data_show_on_ui:
log.info(
"The tensor size is %d, which is too large to show on UI.",
value.size)
value = "Too large to show."
else:
value = np.asarray(value)
return value
def record_log(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as err:
log.exception(err)
raise err
def record_log(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as err:
log.exception(err)
raise DebuggerServerRunningError(str(err))