def on_value_event(self, event):
"""Implementation of the tensor value-carrying Event proto callback.
Writes the Event proto to the file system for testing. The path written to
follows the same pattern as the file:// debug URLs of tfdbg, i.e., the
name scope of the op becomes the directory structure under the dump root
directory.
Args:
event: The Event proto carrying a tensor value.
Returns:
If the debug node belongs to the set of currently activated breakpoints,
a `EventReply` proto will be returned.
"""
if self._dump_dir:
self._write_value_event(event)
else:
value = event.summary.value[0]
tensor_value = debug_data.load_tensor_from_event(event)
self._event_listener_servicer.debug_tensor_values[value.node_name].append(
tensor_value)
items = event.summary.value[0].node_name.split(":")
node_name = items[0]
output_slot = int(items[1])
debug_op = items[2]
if ((node_name, output_slot, debug_op) in
self._event_listener_servicer.breakpoints):
return debug_service_pb2.EventReply()
def SendTracebacks(self, request, context):
self._call_types.append(request.call_type)
self._call_keys.append(request.call_key)
self._origin_stacks.append(request.origin_stack)
self._origin_id_to_strings.append(request.origin_id_to_string)
self._graph_tracebacks.append(request.graph_traceback)
self._graph_versions.append(request.graph_version)
return debug_service_pb2.EventReply()
def on_value_event(self, event):
"""Records the summary values based on an updated message from the
debugger.
Logs an error message if writing the event to disk fails.
Args:
event: The Event proto to be processed.
"""
if not event.summary.value:
logger.info("The summary of the event lacks a value.")
return None
# The node name property in the event proto is actually a watch key, which
# is a concatenation of several pieces of data.
watch_key = event.summary.value[0].node_name
tensor_value = debug_data.load_tensor_from_event(event)
device_name = _extract_device_name_from_event(event)
node_name, output_slot, debug_op = event.summary.value[
0
].node_name.split(":")
maybe_base_expanded_node_name = self._run_states.get_maybe_base_expanded_node_name(
node_name, self._run_key, device_name
)
self._tensor_store.add(watch_key, tensor_value)
self._outgoing_channel.put(
_comm_tensor_data(
device_name,
node_name,
maybe_base_expanded_node_name,
output_slot,
debug_op,
tensor_value,
event.wall_time,
)
)
logger.info("on_value_event(): waiting for client ack (tensors)...")
self._incoming_channel.get()
logger.info("on_value_event(): client ack received (tensor).")
# Determine if the particular debug watch key is in the current list of
# breakpoints. If it is, send an EventReply() to unblock the debug op.
if self._is_debug_node_in_breakpoints(event.summary.value[0].node_name):
logger.info(
"Sending empty EventReply for breakpoint: %s",
event.summary.value[0].node_name,
)
# TODO(cais): Support receiving and sending tensor value from front-end.
return debug_service_pb2.EventReply()
return None
def SendSourceFiles(self, request, context):
"""Base implementation of the handling of SendSourceFiles calls.
The base implementation does nothing with the incoming request.
Override in an implementation of the server if necessary.
Args:
request: A `DebuggedSourceFiles` proto, containing the path, content, size
and last-modified timestamp of source files.
context: Server context.
Returns:
A `EventReply` proto.
"""
return debug_service_pb2.EventReply()
def SendTracebacks(self, request, context):
"""Base implementation of the handling of SendTracebacks calls.
The base implementation does nothing with the incoming request.
Override in an implementation of the server if necessary.
Args:
request: A `CallTraceback` proto, containing information about the
type (e.g., graph vs. eager execution) and source-code traceback of the
call and (any) associated `tf.Graph`s.
context: Server context.
Returns:
A `EventReply` proto.
"""
return debug_service_pb2.EventReply()
def _process_debug_op_state_changes(self, event_reply=None):
"""Dequeue and process all the queued debug-op state change protos.
Include all the debug-op state change protos in a `EventReply` proto.
Args:
event_reply: An `EventReply` to add the `DebugOpStateChange` protos to,
or `None`.
Returns:
An `EventReply` proto with the dequeued `DebugOpStateChange` protos (if
any) added.
"""
if event_reply is None:
event_reply = debug_service_pb2.EventReply()
while not self._debug_ops_state_change_queue.empty():
state_change = self._debug_ops_state_change_queue.get()
debug_node_key = (state_change.node_name, state_change.output_slot,
state_change.debug_op)
if (state_change.state == debug_service_pb2.EventReply.
DebugOpStateChange.READ_WRITE):
logging.info("Adding breakpoint %s:%d:%s",
state_change.node_name, state_change.output_slot,
state_change.debug_op)
self._breakpoints.add(debug_node_key)
elif (state_change.state ==
debug_service_pb2.EventReply.DebugOpStateChange.READ_ONLY):
logging.info("Adding watchpoint %s:%d:%s",
state_change.node_name, state_change.output_slot,
state_change.debug_op)
if debug_node_key in self._breakpoints:
self._breakpoints.discard(debug_node_key)
elif (state_change.state ==
debug_service_pb2.EventReply.DebugOpStateChange.DISABLED):
logging.info("Removing watchpoint or breakpoint: %s:%d:%s",
state_change.node_name, state_change.output_slot,
state_change.debug_op)
if debug_node_key in self._breakpoints:
self._breakpoints.discard(debug_node_key)
else:
logging.warn(
"Attempting to remove a non-existent debug node key: %s",
debug_node_key)
new_state_change = event_reply.debug_op_state_changes.add()
new_state_change.CopyFrom(state_change)
return event_reply
def _watch_key_event_reply(to_enable, node_name, output_slot, debug_op):
"""Make EventReply proto to represent a request to watch/unwatch a debug op.
Args:
to_enable: (`bool`) whether the request is to enable the watch key.
node_name: (`str`) name of the node.
output_slot: (`int`) output slot of the tensor.
debug_op: (`str`) the debug op attached to node_name:output_slot tensor to
watch or unwatch.
Returns:
An EventReply proto.
"""
event_reply = debug_service_pb2.EventReply()
state_change = event_reply.debug_op_state_changes.add()
state_change.change = (
debug_service_pb2.EventReply.DebugOpStateChange.ENABLE if to_enable
else debug_service_pb2.EventReply.DebugOpStateChange.DISABLE)
state_change.node_name = node_name
state_change.output_slot = output_slot
state_change.debug_op = debug_op
return event_reply
def _watch_key_event_reply(new_state, node_name, output_slot, debug_op):
"""Make `EventReply` proto to represent a request to watch/unwatch a debug op.
Args:
new_state: (`debug_service_pb2.EventReply.DebugOpStateChange.State`) the new
state to set the debug node to, i.e., whether the debug node will become
disabled under the grpc mode (`DISABLED`), become a watchpoint
(`READ_ONLY`) or become a breakpoint (`READ_WRITE`).
node_name: (`str`) name of the node.
output_slot: (`int`) output slot of the tensor.
debug_op: (`str`) the debug op attached to node_name:output_slot tensor to
watch or unwatch.
Returns:
An EventReply proto.
"""
event_reply = debug_service_pb2.EventReply()
state_change = event_reply.debug_op_state_changes.add()
state_change.state = new_state
state_change.node_name = node_name
state_change.output_slot = output_slot
state_change.debug_op = debug_op
return event_reply
def SendSourceFiles(self, request, context):
# TODO(cais): Handle case in which the size of the request is greater than
# the 4-MB gRPC limit.
for source_file in request.source_files:
self._source_manager.add_debugged_source_file(source_file)
return debug_service_pb2.EventReply()
def SendTracebacks(self, request, context):
self._source_manager.add_graph_traceback(request.graph_version,
request.graph_traceback)
return debug_service_pb2.EventReply()
def SendSourceFiles(self, request, context):
self._source_files.append(request)
return debug_service_pb2.EventReply()
