def _get_node(self, element):
"""Get the node of a graph element.
Args:
element: A graph element (Op, Tensor or Node)
Returns:
The node associated with element in the graph.
"""
node_name, _ = debug_data.parse_node_or_tensor_name(element.name)
return self._sess.graph.as_graph_element(node_name)
def _get_node(self, element):
"""Get the node of a graph element.
Args:
element: A graph element (Op, Tensor or Node)
Returns:
The node associated with element in the graph.
"""
node_name, _ = debug_data.parse_node_or_tensor_name(element.name)
return self._sess.graph.as_graph_element(node_name)
def gradient_values_from_dump(grad_debugger, x_tensor, dump):
"""Find gradient values from a `DebugDumpDir` object.
Args:
grad_debugger: the `tf_debug.GradientsDebugger` instance to be used.
x_tensor: (`tf.Tensor`, `tf.Variable` or `str`) The x-tensor object or its
name. x-tensor refers to the independent `tf.Tensor`, i.e., the tensor
on the denominator of the differentiation.
dump: A `tfdbg.DebugDumpDir` object.
Returns:
If this `GradientsDebugger` instance has the gradient tensor of `x_tensor`
registered: a list of `numpy.ndarray` representing the value of the
gradient tensor from `dump`. The list could be empty, if the gradient
tensor is not executed in the `tf.Session.run()` call that generated
the `dump`. The list could also contain multiple values of the gradient
tensor, e.g., if gradient tensor is computed repeatedly in a
`tf.while_loop` during the run that generated the `dump`.
Raises:
LookupError: If this `GradientsDebugger` instance does not have the
gradient tensor of `x_tensor` registered.
ValueError: If this `GradientsDebugger` has a `tf.Graph` object that
does not match the `tf.Graph` object of the `dump`.
TypeError: If `x_tensor` is not a `tf.Tensor`, `tf.Variable` or `str`.
"""
# TODO(cais): Use this method in LocalCLIDebugWrapperSession to present the
# gradient tensors to the TFDBG CLI.
# If possible, verify that the Python graph of the dump and that of this
# GradientsDebugger match.
if (dump.python_graph and grad_debugger.graph and
dump.python_graph != grad_debugger.graph):
raise ValueError(
"This GradientsDebugger instance has a graph (%s) that differs from "
"the graph of the DebugDumpDir object (%s)." %
(grad_debugger.graph, dump.python_graph))
gradient_tensor = grad_debugger.gradient_tensor(x_tensor)
node_name, output_slot = debug_data.parse_node_or_tensor_name(
gradient_tensor.name)
try:
return dump.get_tensors(node_name, output_slot, "DebugIdentity")
except debug_data.WatchKeyDoesNotExistInDebugDumpDirError:
return []
def gradient_values_from_dump(grad_debugger, x_tensor, dump):
"""Find gradient values from a `DebugDumpDir` object.
Args:
grad_debugger: the `tf_debug.GradientsDebugger` instance to be used.
x_tensor: (`tf.Tensor`, `tf.Variable` or `str`) The x-tensor object or its
name. x-tensor refers to the independent `tf.Tensor`, i.e., the tensor
on the denominator of the differentiation.
dump: A `tfdbg.DebugDumpDir` object.
Returns:
If this `GradientsDebugger` instance has the gradient tensor of `x_tensor`
registered: a list of `numpy.ndarray` representing the value of the
gradient tensor from `dump`. The list could be empty, if the gradient
tensor is not executed in the `tf.Session.run()` call that generated
the `dump`. The list could also contain multiple values of the gradient
tensor, e.g., if gradient tensor is computed repeatedly in a
`tf.while_loop` during the run that generated the `dump`.
Raises:
LookupError: If this `GradientsDebugger` instance does not have the
gradient tensor of `x_tensor` registered.
ValueError: If this `GradientsDebugger` has a `tf.Graph` object that
does not match the `tf.Graph` object of the `dump`.
TypeError: If `x_tensor` is not a `tf.Tensor`, `tf.Variable` or `str`.
"""
# TODO(cais): Use this method in LocalCLIDebugWrapperSession to present the
# gradient tensors to the TFDBG CLI.
# If possible, verify that the Python graph of the dump and that of this
# GradientsDebugger match.
if (dump.python_graph and grad_debugger.graph
and dump.python_graph != grad_debugger.graph):
raise ValueError(
"This GradientsDebugger instance has a graph (%s) that differs from "
"the graph of the DebugDumpDir object (%s)." %
(grad_debugger.graph, dump.python_graph))
gradient_tensor = grad_debugger.gradient_tensor(x_tensor)
node_name, output_slot = debug_data.parse_node_or_tensor_name(
gradient_tensor.name)
try:
return dump.get_tensors(node_name, output_slot, "DebugIdentity")
except debug_data.WatchKeyDoesNotExistInDebugDumpDirError:
return []
def testParseTensorName(self):
node_name, slot = debug_data.parse_node_or_tensor_name(
"namespace1/node_2:3")
self.assertEqual("namespace1/node_2", node_name)
self.assertEqual(3, slot)
def testParseNodeName(self):
node_name, slot = debug_data.parse_node_or_tensor_name("namespace1/node_1")
self.assertEqual("namespace1/node_1", node_name)
self.assertIsNone(slot)
def testParseTensorName(self):
node_name, slot = debug_data.parse_node_or_tensor_name(
"namespace1/node_2:3")
self.assertEqual("namespace1/node_2", node_name)
self.assertEqual(3, slot)
def testParseNodeName(self):
node_name, slot = debug_data.parse_node_or_tensor_name("namespace1/node_1")
self.assertEqual("namespace1/node_1", node_name)
self.assertIsNone(slot)
def _tensor_to_grad_debug_op_name(tensor, grad_debugger_uuid): op_name, slot = debug_data.parse_node_or_tensor_name(tensor.name) return "%s_%d/%s%s" % (op_name, slot, _GRADIENT_DEBUG_TAG, grad_debugger_uuid)
def _list_inputs_or_outputs(self,
recursive,
node_name,
depth,
control,
op_type,
do_outputs=False):
"""Helper function used by list_inputs and list_outputs.
Format a list of lines to display the inputs or output recipients of a
given node.
Args:
recursive: Whether the listing is to be done recursively, as a boolean.
node_name: The name of the node in question, as a str.
depth: Maximum recursion depth, applies only if recursive == True, as an
int.
control: Whether control inputs or control recipients are included, as a
boolean.
op_type: Whether the op types of the nodes are to be included, as a
boolean.
do_outputs: Whether recipients, instead of input nodes are to be
listed, as a boolean.
Returns:
Input or recipient tree formatted as a RichTextLines object.
"""
if do_outputs:
tracker = self._debug_dump.node_recipients
type_str = "Recipients of"
short_type_str = "recipients"
else:
tracker = self._debug_dump.node_inputs
type_str = "Inputs to"
short_type_str = "inputs"
lines = []
font_attr_segs = {}
# Check if this is a tensor name, instead of a node name.
node_name, _ = debug_data.parse_node_or_tensor_name(node_name)
# Check if node exists.
if not self._debug_dump.node_exists(node_name):
return cli_shared.error(
"There is no node named \"%s\" in the partition graphs" % node_name)
if recursive:
max_depth = depth
else:
max_depth = 1
if control:
include_ctrls_str = ", control %s included" % short_type_str
else:
include_ctrls_str = ""
line = "%s node \"%s\"" % (type_str, node_name)
font_attr_segs[0] = [(len(line) - 1 - len(node_name), len(line) - 1, "bold")
]
lines.append(line + " (Depth limit = %d%s):" % (max_depth, include_ctrls_str
))
command_template = "lo -c -r %s" if do_outputs else "li -c -r %s"
self._dfs_from_node(
lines,
font_attr_segs,
node_name,
tracker,
max_depth,
1, [],
control,
op_type,
command_template=command_template)
# Include legend.
lines.append("")
lines.append("Legend:")
lines.append(" (d): recursion depth = d.")
if control:
lines.append(" (Ctrl): Control input.")
if op_type:
lines.append(" [Op]: Input node has op type Op.")
# TODO(cais): Consider appending ":0" at the end of 1st outputs of nodes.
return debugger_cli_common.RichTextLines(
lines, font_attr_segs=font_attr_segs)
def print_tensor(self, args, screen_info=None):
"""Command handler for print_tensor.
Print value of a given dumped tensor.
Args:
args: Command-line arguments, excluding the command prefix, as a list of
str.
screen_info: Optional dict input containing screen information such as
cols.
Returns:
Output text lines as a RichTextLines object.
"""
parsed = self._arg_parsers["print_tensor"].parse_args(args)
if screen_info and "cols" in screen_info:
np_printoptions = {"linewidth": screen_info["cols"]}
else:
np_printoptions = {}
# Determine if any range-highlighting is required.
highlight_options = cli_shared.parse_ranges_highlight(parsed.ranges)
tensor_name, tensor_slicing = (
command_parser.parse_tensor_name_with_slicing(parsed.tensor_name))
node_name, output_slot = debug_data.parse_node_or_tensor_name(tensor_name)
if (self._debug_dump.loaded_partition_graphs() and
not self._debug_dump.node_exists(node_name)):
output = cli_shared.error(
"Node \"%s\" does not exist in partition graphs" % node_name)
_add_main_menu(
output,
node_name=None,
enable_list_tensors=True,
enable_print_tensor=False)
return output
watch_keys = self._debug_dump.debug_watch_keys(node_name)
if output_slot is None:
output_slots = set()
for watch_key in watch_keys:
output_slots.add(int(watch_key.split(":")[1]))
if len(output_slots) == 1:
# There is only one dumped tensor from this node, so there is no
# ambiguity. Proceed to show the only dumped tensor.
output_slot = list(output_slots)[0]
else:
# There are more than one dumped tensors from this node. Indicate as
# such.
# TODO(cais): Provide an output screen with command links for
# convenience.
lines = [
"Node \"%s\" generated debug dumps from %s output slots:" %
(node_name, len(output_slots)),
"Please specify the output slot: %s:x." % node_name
]
output = debugger_cli_common.RichTextLines(lines)
_add_main_menu(
output,
node_name=node_name,
enable_list_tensors=True,
enable_print_tensor=False)
return output
# Find debug dump data that match the tensor name (node name + output
# slot).
matching_data = []
for watch_key in watch_keys:
debug_tensor_data = self._debug_dump.watch_key_to_data(watch_key)
for datum in debug_tensor_data:
if datum.output_slot == output_slot:
matching_data.append(datum)
if not matching_data:
# No dump for this tensor.
output = cli_shared.error("Tensor \"%s\" did not generate any dumps." %
parsed.tensor_name)
elif len(matching_data) == 1:
# There is only one dump for this tensor.
if parsed.number <= 0:
output = cli_shared.format_tensor(
matching_data[0].get_tensor(),
matching_data[0].watch_key,
np_printoptions,
print_all=parsed.print_all,
tensor_slicing=tensor_slicing,
highlight_options=highlight_options)
else:
output = cli_shared.error(
"Invalid number (%d) for tensor %s, which generated one dump." %
(parsed.number, parsed.tensor_name))
_add_main_menu(output, node_name=node_name, enable_print_tensor=False)
else:
# There are more than one dumps for this tensor.
if parsed.number < 0:
lines = [
"Tensor \"%s\" generated %d dumps:" % (parsed.tensor_name,
len(matching_data))
]
font_attr_segs = {}
for i, datum in enumerate(matching_data):
rel_time = (datum.timestamp - self._debug_dump.t0) / 1000.0
lines.append("#%d [%.3f ms] %s" % (i, rel_time, datum.watch_key))
command = "print_tensor %s -n %d" % (parsed.tensor_name, i)
font_attr_segs[len(lines) - 1] = [(
len(lines[-1]) - len(datum.watch_key), len(lines[-1]),
debugger_cli_common.MenuItem(None, command))]
lines.append("")
lines.append(
"You can use the -n (--number) flag to specify which dump to "
"print.")
lines.append("For example:")
lines.append(" print_tensor %s -n 0" % parsed.tensor_name)
output = debugger_cli_common.RichTextLines(
lines, font_attr_segs=font_attr_segs)
elif parsed.number >= len(matching_data):
output = cli_shared.error(
"Specified number (%d) exceeds the number of available dumps "
"(%d) for tensor %s" %
(parsed.number, len(matching_data), parsed.tensor_name))
else:
output = cli_shared.format_tensor(
matching_data[parsed.number].get_tensor(),
matching_data[parsed.number].watch_key + " (dump #%d)" %
parsed.number,
np_printoptions,
print_all=parsed.print_all,
tensor_slicing=tensor_slicing,
highlight_options=highlight_options)
_add_main_menu(output, node_name=node_name, enable_print_tensor=False)
return output
def node_info(self, args, screen_info=None):
"""Command handler for node_info.
Query information about a given node.
Args:
args: Command-line arguments, excluding the command prefix, as a list of
str.
screen_info: Optional dict input containing screen information such as
cols.
Returns:
Output text lines as a RichTextLines object.
"""
# TODO(cais): Add annotation of substrings for node names, to facilitate
# on-screen highlighting/selection of node names.
_ = screen_info
parsed = self._arg_parsers["node_info"].parse_args(args)
# Get a node name, regardless of whether the input is a node name (without
# output slot attached) or a tensor name (with output slot attached).
node_name, unused_slot = debug_data.parse_node_or_tensor_name(
parsed.node_name)
if not self._debug_dump.node_exists(node_name):
output = cli_shared.error(
"There is no node named \"%s\" in the partition graphs" % node_name)
_add_main_menu(
output,
node_name=None,
enable_list_tensors=True,
enable_node_info=False,
enable_list_inputs=False,
enable_list_outputs=False)
return output
# TODO(cais): Provide UI glossary feature to explain to users what the
# term "partition graph" means and how it is related to TF graph objects
# in Python. The information can be along the line of:
# "A tensorflow graph defined in Python is stripped of unused ops
# according to the feeds and fetches and divided into a number of
# partition graphs that may be distributed among multiple devices and
# hosts. The partition graphs are what's actually executed by the C++
# runtime during a run() call."
lines = ["Node %s" % node_name]
font_attr_segs = {
0: [(len(lines[-1]) - len(node_name), len(lines[-1]), "bold")]
}
lines.append("")
lines.append(" Op: %s" % self._debug_dump.node_op_type(node_name))
lines.append(" Device: %s" % self._debug_dump.node_device(node_name))
output = debugger_cli_common.RichTextLines(
lines, font_attr_segs=font_attr_segs)
# List node inputs (non-control and control).
inputs = self._debug_dump.node_inputs(node_name)
ctrl_inputs = self._debug_dump.node_inputs(node_name, is_control=True)
output.extend(self._format_neighbors("input", inputs, ctrl_inputs))
# List node output recipients (non-control and control).
recs = self._debug_dump.node_recipients(node_name)
ctrl_recs = self._debug_dump.node_recipients(node_name, is_control=True)
output.extend(self._format_neighbors("recipient", recs, ctrl_recs))
# Optional: List attributes of the node.
if parsed.attributes:
output.extend(self._list_node_attributes(node_name))
# Optional: List dumps available from the node.
if parsed.dumps:
output.extend(self._list_node_dumps(node_name))
if parsed.traceback:
output.extend(self._render_node_traceback(node_name))
_add_main_menu(output, node_name=node_name, enable_node_info=False)
return output
def _dfs_from_node(self,
lines,
attr_segs,
node_name,
tracker,
max_depth,
depth,
unfinished,
include_control=False,
show_op_type=False,
command_template=None):
"""Perform depth-first search (DFS) traversal of a node's input tree.
It recursively tracks the inputs (or output recipients) of the node called
node_name, and append these inputs (or output recipients) to a list of text
lines (lines) with proper indentation that reflects the recursion depth,
together with some formatting attributes (to attr_segs). The formatting
attributes can include command shortcuts, for example.
Args:
lines: Text lines to append to, as a list of str.
attr_segs: (dict) Attribute segments dictionary to append to.
node_name: Name of the node, as a str. This arg is updated during the
recursion.
tracker: A callable that takes one str as the node name input and
returns a list of str as the inputs/outputs.
This makes it this function general enough to be used with both
node-input and node-output tracking.
max_depth: Maximum recursion depth, as an int.
depth: Current recursion depth. This arg is updated during the
recursion.
unfinished: A stack of unfinished recursion depths, as a list of int.
include_control: Whether control dependencies are to be included as
inputs (and marked as such).
show_op_type: Whether op type of the input nodes are to be displayed
alongside the nodes' names.
command_template: (str) Template for command shortcut of the node names.
"""
# Make a shallow copy of the list because it may be extended later.
all_inputs = copy.copy(tracker(node_name, is_control=False))
is_ctrl = [False] * len(all_inputs)
if include_control:
# Sort control inputs or recipients in in alphabetical order of the node
# names.
ctrl_inputs = sorted(tracker(node_name, is_control=True))
all_inputs.extend(ctrl_inputs)
is_ctrl.extend([True] * len(ctrl_inputs))
if not all_inputs:
if depth == 1:
lines.append(" [None]")
return
unfinished.append(depth)
# Create depth-dependent hanging indent for the line.
hang = ""
for k in xrange(depth):
if k < depth - 1:
if k + 1 in unfinished:
hang += HANG_UNFINISHED
else:
hang += HANG_FINISHED
else:
hang += HANG_SUFFIX
if all_inputs and depth > max_depth:
lines.append(hang + ELLIPSIS)
unfinished.pop()
return
hang += DEPTH_TEMPLATE % depth
for i in xrange(len(all_inputs)):
inp = all_inputs[i]
if is_ctrl[i]:
ctrl_str = CTRL_LABEL
else:
ctrl_str = ""
op_type_str = ""
if show_op_type:
op_type_str = OP_TYPE_TEMPLATE % self._debug_dump.node_op_type(inp)
if i == len(all_inputs) - 1:
unfinished.pop()
line = hang + ctrl_str + op_type_str + inp
lines.append(line)
if command_template:
attr_segs[len(lines) - 1] = [(
len(line) - len(inp), len(line),
debugger_cli_common.MenuItem(None, command_template % inp))]
# Recursive call.
# The input's/output's name can be a tensor name, in the case of node
# with >1 output slots.
inp_node_name, _ = debug_data.parse_node_or_tensor_name(inp)
self._dfs_from_node(
lines,
attr_segs,
inp_node_name,
tracker,
max_depth,
depth + 1,
unfinished,
include_control=include_control,
show_op_type=show_op_type,
command_template=command_template)
def _tensor_to_grad_debug_op_name(tensor, grad_debugger_uuid):
op_name, slot = debug_data.parse_node_or_tensor_name(tensor.name)
return "%s_%d/%s%s" % (op_name, slot, _GRADIENT_DEBUG_TAG,
grad_debugger_uuid)