def testDebuggerExampleFilePathReturnsFalse(self): self.assertFalse( source_utils.guess_is_tensorflow_py_library(os.path.normpath( "site-packages/tensorflow/python/debug/examples/debug_mnist.py"))) self.assertFalse( source_utils.guess_is_tensorflow_py_library(os.path.normpath( "site-packages/tensorflow/python/debug/examples/v1/example_v1.py"))) self.assertFalse( source_utils.guess_is_tensorflow_py_library(os.path.normpath( "site-packages/tensorflow/python/debug/examples/v2/example_v2.py"))) self.assertFalse( source_utils.guess_is_tensorflow_py_library(os.path.normpath( "site-packages/tensorflow/python/debug/examples/v3/example_v3.py")))
def _get_profile_data_generator(self): """Get function that generates `ProfileDatum` objects. Returns: A function that generates `ProfileDatum` objects. """ node_to_file_line = {} node_to_op_type = {} for op in self._graph.get_operations(): file_line = "" for trace_entry in reversed(op.traceback): filepath = trace_entry[0] file_line = "%s:%d(%s)" % (os.path.basename(filepath), trace_entry[1], trace_entry[2]) if not source_utils.guess_is_tensorflow_py_library(filepath): break node_to_file_line[op.name] = file_line node_to_op_type[op.name] = op.type def profile_data_generator(device_step_stats): for node_stats in device_step_stats.node_stats: if node_stats.node_name == "_SOURCE" or node_stats.node_name == "_SINK": continue yield ProfileDatum( node_stats, node_to_file_line.get(node_stats.node_name, ""), node_to_op_type.get(node_stats.node_name, "")) return profile_data_generator
def _get_profile_data_generator(self): """Get function that generates `ProfileDatum` objects. Returns: A function that generates `ProfileDatum` objects. """ node_to_file_path = {} node_to_line_number = {} node_to_func_name = {} node_to_op_type = {} for op in self._graph.get_operations(): for trace_entry in reversed(op.traceback): file_path = trace_entry[0] line_num = trace_entry[1] func_name = trace_entry[2] if not source_utils.guess_is_tensorflow_py_library(file_path): break node_to_file_path[op.name] = file_path node_to_line_number[op.name] = line_num node_to_func_name[op.name] = func_name node_to_op_type[op.name] = op.type def profile_data_generator(device_step_stats): for node_stats in device_step_stats.node_stats: if node_stats.node_name == "_SOURCE" or node_stats.node_name == "_SINK": continue yield profiling.ProfileDatum( device_step_stats.device, node_stats, node_to_file_path.get(node_stats.node_name, ""), node_to_line_number.get(node_stats.node_name, 0), node_to_func_name.get(node_stats.node_name, ""), node_to_op_type.get(node_stats.node_name, "")) return profile_data_generator
def _get_profile_data_generator(self): """Get function that generates `ProfileDatum` objects. Returns: A function that generates `ProfileDatum` objects. """ node_to_file_line = {} node_to_op_type = {} for op in self._graph.get_operations(): file_line = "" for trace_entry in reversed(op.traceback): filepath = trace_entry[0] file_line = "%s:%d(%s)" % ( os.path.basename(filepath), trace_entry[1], trace_entry[2]) if not source_utils.guess_is_tensorflow_py_library(filepath): break node_to_file_line[op.name] = file_line node_to_op_type[op.name] = op.type def profile_data_generator(device_step_stats): for node_stats in device_step_stats.node_stats: if node_stats.node_name == "_SOURCE" or node_stats.node_name == "_SINK": continue yield ProfileDatum( node_stats, node_to_file_line.get(node_stats.node_name, ""), node_to_op_type.get(node_stats.node_name, "")) return profile_data_generator
def _get_profile_data_generator(self): """Get function that generates `ProfileDatum` objects. Returns: A function that generates `ProfileDatum` objects. """ node_to_file_path = {} node_to_line_number = {} node_to_func_name = {} node_to_op_type = {} for op in self._graph.get_operations(): for trace_entry in reversed(op.traceback): file_path = trace_entry[0] line_num = trace_entry[1] func_name = trace_entry[2] if not source_utils.guess_is_tensorflow_py_library(file_path): break node_to_file_path[op.name] = file_path node_to_line_number[op.name] = line_num node_to_func_name[op.name] = func_name node_to_op_type[op.name] = op.type def profile_data_generator(device_step_stats): for node_stats in device_step_stats.node_stats: if node_stats.node_name == "_SOURCE" or node_stats.node_name == "_SINK": continue yield profiling.ProfileDatum( device_step_stats.device, node_stats, node_to_file_path.get(node_stats.node_name, ""), node_to_line_number.get(node_stats.node_name, 0), node_to_func_name.get(node_stats.node_name, ""), node_to_op_type.get(node_stats.node_name, "")) return profile_data_generator
def _source_file_paths_outside_tensorflow_py_library(code_defs, id_to_string): """Extract source file paths outside TensorFlow Python library. Args: code_defs: An iterable of `CodeDef` protos, i.e., an iterable of stack traces. id_to_string: A proto map from integer ids to strings. Returns: An iterable of source file paths outside the TensorFlow Python library. """ file_ids = set() for code_def in code_defs: for trace in code_def.traces: file_ids.add(trace.file_id) non_tf_files = (id_to_string[file_id] for file_id in file_ids) non_tf_files = (f for f in non_tf_files if not source_utils.guess_is_tensorflow_py_library(f) and gfile.Exists(f)) return non_tf_files
def _source_file_paths_outside_tensorflow_py_library(code_defs, id_to_string): """Extract source file paths outside TensorFlow Python library. Args: code_defs: An iterable of `CodeDef` protos, i.e., an iterable of stack traces. id_to_string: A proto map from integer ids to strings. Returns: An iterable of source file paths outside the TensorFlow Python library. """ file_ids = set() for code_def in code_defs: for trace in code_def.traces: file_ids.add(trace.file_id) non_tf_files = (id_to_string[file_id] for file_id in file_ids) non_tf_files = ( f for f in non_tf_files if not source_utils.guess_is_tensorflow_py_library(f) and gfile.Exists(f)) return non_tf_files
def testNonPythonFileRaisesException(self): with self.assertRaisesRegexp(ValueError, r"is not a Python source file"): source_utils.guess_is_tensorflow_py_library( os.path.join(os.path.dirname(self.curr_file_path), "foo.cc"))
def testFileInPythonKernelsPathReturnsTrue(self): x = constant_op.constant(42.0, name="x") self.assertTrue( source_utils.guess_is_tensorflow_py_library(x.op.traceback[-1][0]))
def testSourceUtilModuleReturnsTrue(self): self.assertTrue( source_utils.guess_is_tensorflow_py_library(source_utils.__file__))
def testNonPythonFileRaisesException(self): with self.assertRaisesRegexp(ValueError, r"is not a Python source file"): source_utils.guess_is_tensorflow_py_library( os.path.join(os.path.dirname(self.curr_file_path), "foo.cc"))
def testSourceUtilModuleReturnsTrue(self): self.assertTrue( source_utils.guess_is_tensorflow_py_library(source_utils.__file__))
def testReturnsFalseForNonPythonFile(self): self.assertFalse( source_utils.guess_is_tensorflow_py_library( os.path.join(os.path.dirname(self.curr_file_path), "foo.cc")))
def get_check_numerics_error_message(slot, num_outputs, op_type, tensor, inputs, graph=None, traceback=None, stack_height_limit=30, path_length_limit=50): """Create a meaningful and user-friendly error message about offending tensor. The error message reveals the following info about the op that outputs NaN/Infinity: dtype, shape (to the extent known at graph-construction time), input tensors, stack trace for op creation (if is graph mode). Args: slot: (int) slot index of the tensor output. num_outputs: (int) total number of outputs of the op. op_type: (str) Type of the that generates `tensor`. tensor: (Tensor) the offending tensor, i.e., the tensor that contains Infinities or NaNs. inputs: (array of Tensor) inputs to the op that generates `tensor`. graph: (tf.Graph) the graph object that `tensor` belongs to. Available only under graph mode. traceback: (list of trace frames) the stack trace of the op's creation. Available only under graph model. stack_height_limit: (int or None) If int, limit to the height of the stack trace printed in the error message. If None, no limit to the height. path_length_limit: (int or None) Length limit for file paths included in the formatted stack trace. Returns: (str) A formatted error message. """ eager_vs_graph_qualifier = "graph" if graph else "eagerly-executing" message = "\n" message += ("\n!!! Detected Infinity or NaN in output %d of " "%s op \"%s\" (# of outputs: %d) !!!\n" % (slot, eager_vs_graph_qualifier, op_type, num_outputs)) message += " dtype: %s\n" % tensor.dtype message += " shape: %s\n" % (tensor.shape, ) if not graph: # This is an eager tensor. We can get its numpy value and count # NaNs and Infs. is_inf = np.isinf(tensor) num_neg_inf = np.sum(np.logical_and(np.less(tensor, 0.), is_inf)) num_pos_inf = np.sum(np.logical_and(np.greater(tensor, 0.), is_inf)) num_nan = np.sum(np.isnan(tensor)) if num_neg_inf > 0: message += " # of -Inf elements: %s\n" % num_neg_inf if num_pos_inf > 0: message += " # of +Inf elements: %s\n" % num_pos_inf if num_nan: message += " # of +NaN elements: %s\n" % num_nan if len(inputs) > 1: message += "\n Input tensors (%d):\n" % len(inputs) for slot, input_tensor in enumerate(inputs): message += " %d: %s\n" % ( slot, _maybe_lookup_original_input_tensor(graph, input_tensor)) elif len(inputs) == 1: message += "\n Input tensor: %s\n" % ( _maybe_lookup_original_input_tensor(graph, inputs[0])) if graph and hasattr(graph, "name") and graph.name: message += " Graph name: \"%s\"\n" % graph.name # Format the stack trace for the op's creation. We omit files that # belong to tensorflow itself. if graph and traceback: message += ( "\n Stack trace of op's creation (\"->\": inferred user code):\n") if stack_height_limit is not None and len( traceback) > stack_height_limit: num_omitted_frames = len(traceback) - stack_height_limit message += " + ... (Omitted %d frames)\n" % num_omitted_frames for filepath, lineno, function_name, source_line in traceback[ -stack_height_limit:]: user_code_indicator = " " if not source_utils.guess_is_tensorflow_py_library(filepath): user_code_indicator = " -> " message += " + %s (L%d) %s\n" % (limit_string_length( filepath, path_length_limit), lineno, function_name) if source_line is not None: message += "%s| %s\n" % (user_code_indicator, source_line) message += "\n" return message
def _findFirstTraceInsideTensorFlowPyLibrary(self, op): """Find the first trace of an op that belongs to the TF Python library.""" for trace in op.traceback: if source_utils.guess_is_tensorflow_py_library(trace.filename): return trace
def testUnitTestFileReturnsFalse(self): self.assertFalse( source_utils.guess_is_tensorflow_py_library(self.curr_file_path))
def testReturnsFalseForStdin(self): self.assertFalse( source_utils.guess_is_tensorflow_py_library("<stdin>"))
def testFileInPythonKernelsPathReturnsTrue(self): x = constant_op.constant(42.0, name="x") self.assertTrue( source_utils.guess_is_tensorflow_py_library(x.op.traceback[-1][0]))
def testReturnsFalseForEmptyFileName(self): self.assertFalse(source_utils.guess_is_tensorflow_py_library(""))
def testUnitTestFileReturnsFalse(self): self.assertFalse( source_utils.guess_is_tensorflow_py_library(self.curr_file_path))
def _findFirstTraceInsideTensorFlowPyLibrary(self, op): """Find the first trace of an op that belongs to the TF Python library.""" for trace in op.traceback: if source_utils.guess_is_tensorflow_py_library(trace[0]): return trace