def testLimitingInfNanMonitorAlertCountWorks(self):
        mock_reader = test.mock.MagicMock()
        monitor = debug_events_monitors.InfNanMonitor(mock_reader, limit=3)
        for i in range(10):
            execution_digest = debug_events_reader.ExecutionDigest(
                i * 1000, 1, "FooOp", output_tensor_device_ids=[0, 1])
            execution = debug_events_reader.Execution(
                execution_digest,
                "worker01", ["a1", "b2", "e3"],
                debug_event_pb2.TensorDebugMode.CURT_HEALTH,
                graph_id=None,
                input_tensor_ids=[12, 34],
                output_tensor_ids=[56, 78],
                debug_tensor_values=[[-1, 0],
                                     [-1, 1]])  # [tensor_id, any_inf_nan].
            monitor.on_execution(i, execution)

        alerts = monitor.alerts()
        self.assertLen(alerts, 3)
        for i, alert in enumerate(alerts):
            self.assertEqual(alert.wall_time, i * 1000)
            self.assertEqual(alert.op_type, "FooOp")
            self.assertEqual(alert.output_slot, 1)
            # The four fields below are unavailable under CURT_HEALTH mode by design.
            self.assertIsNone(alert.size)
            self.assertIsNone(alert.num_neg_inf)
            self.assertIsNone(alert.num_pos_inf)
            self.assertIsNone(alert.num_nan)
            self.assertEqual(alert.execution_index, i)
            self.assertIsNone(alert.graph_execution_trace_index)
    def testInfNanMonitorOnExecutionUnderHealthMode(self, tensor_debug_mode,
                                                    debug_tensor_values):
        mock_reader = test.mock.MagicMock()
        monitor = debug_events_monitors.InfNanMonitor(mock_reader)
        execution_digest = debug_events_reader.ExecutionDigest(
            1234, 1, "BarOp", output_tensor_device_ids=[0, 1])

        execution = debug_events_reader.Execution(
            execution_digest,
            "worker01", ["a1", "b2", "e3"],
            tensor_debug_mode,
            graph_id=None,
            input_tensor_ids=[12, 34],
            output_tensor_ids=[56, 78],
            debug_tensor_values=debug_tensor_values)
        monitor.on_execution(60, execution)

        self.assertLen(monitor.alerts(), 1)
        alert = monitor.alerts()[0]
        self.assertEqual(alert.wall_time, 1234)
        self.assertEqual(alert.op_type, "BarOp")
        self.assertEqual(alert.output_slot, 0)
        self.assertEqual(alert.size, 10)
        self.assertEqual(alert.num_neg_inf, 1)
        self.assertEqual(alert.num_pos_inf, 2)
        self.assertEqual(alert.num_nan, 3)
        self.assertEqual(alert.execution_index, 60)
        self.assertIsNone(alert.graph_execution_trace_index)
    def testInfNanMonitorOnExecutionUnderCurtHealthMode(self):
        mock_reader = test.mock.MagicMock()
        monitor = debug_events_monitors.InfNanMonitor(mock_reader)
        execution_digest = debug_events_reader.ExecutionDigest(
            1234, 1, "FooOp", output_tensor_device_ids=[0, 1])
        execution = debug_events_reader.Execution(
            execution_digest,
            "worker01", ["a1", "b2", "e3"],
            debug_event_pb2.TensorDebugMode.CURT_HEALTH,
            graph_id=None,
            input_tensor_ids=[12, 34],
            output_tensor_ids=[56, 78],
            debug_tensor_values=[[-1, 0], [-1,
                                           1]])  # [tensor_id, any_inf_nan].
        monitor.on_execution(50, execution)

        self.assertLen(monitor.alerts(), 1)
        alert = monitor.alerts()[0]
        self.assertEqual(alert.wall_time, 1234)
        self.assertEqual(alert.op_type, "FooOp")
        self.assertEqual(alert.output_slot, 1)
        # The four fields below are unavailable under CURT_HEALTH mode by design.
        self.assertIsNone(alert.size)
        self.assertIsNone(alert.num_neg_inf)
        self.assertIsNone(alert.num_pos_inf)
        self.assertIsNone(alert.num_nan)
        self.assertEqual(alert.execution_index, 50)
        self.assertIsNone(alert.graph_execution_trace_index)
 def testExecutionNoGraphNoInputButWithOutputToJson(self):
   execution_digest = debug_events_reader.ExecutionDigest(
       1234, 5678, "FooOp", output_tensor_device_ids=[1357])
   execution = debug_events_reader.Execution(
       execution_digest,
       "localhost",
       ("a1", "b2"),
       debug_event_pb2.TensorDebugMode.FULL_HEALTH,
       graph_id="abcd",
       input_tensor_ids=[13, 37],
       output_tensor_ids=None,
       debug_tensor_values=None)
   json = execution.to_json()
   self.jsonRoundTripCheck(json)
   self.assertEqual(json["wall_time"], 1234)
   self.assertEqual(json["op_type"], "FooOp")
   self.assertEqual(json["output_tensor_device_ids"], (1357,))
   self.assertEqual(json["host_name"], "localhost")
   self.assertEqual(json["stack_frame_ids"], ("a1", "b2"))
   self.assertEqual(json["tensor_debug_mode"],
                    debug_event_pb2.TensorDebugMode.FULL_HEALTH)
   self.assertEqual(json["graph_id"], "abcd")
   self.assertEqual(json["input_tensor_ids"], (13, 37))
   self.assertIsNone(json["output_tensor_ids"])
   self.assertIsNone(json["debug_tensor_values"])
 def testExecutionDigestWithTwoOutputsToJson(self):
   execution_digest = debug_events_reader.ExecutionDigest(
       1234, 5678, "FooOp", output_tensor_device_ids=[1357, 2468])
   json = execution_digest.to_json()
   self.jsonRoundTripCheck(json)
   self.assertEqual(json["wall_time"], 1234)
   self.assertEqual(json["op_type"], "FooOp")
   self.assertEqual(json["output_tensor_device_ids"], (1357, 2468))
 def testExecutionWithNoOutputTensorsReturnsZeroForNumOutputs(
     self, output_tensor_ids):
   execution = debug_events_reader.Execution(
       debug_events_reader.ExecutionDigest(1234, 5678, "FooOp"),
       "localhost", ("a1", "b2"),
       debug_event_pb2.TensorDebugMode.FULL_HEALTH,
       graph_id="abcd",
       input_tensor_ids=[13, 37],
       output_tensor_ids=output_tensor_ids,
       debug_tensor_values=None)
   self.assertEqual(execution.num_outputs, 0)
    def testInfNanMonitorOnExecutionUnderModeWithNoInfNanInfo(
            self, tensor_debug_mode, debug_tensor_values):
        mock_reader = test.mock.MagicMock()
        monitor = debug_events_monitors.InfNanMonitor(mock_reader)
        execution_digest = debug_events_reader.ExecutionDigest(
            1234, 1, "BarOp", output_tensor_device_ids=[0, 1])

        execution = debug_events_reader.Execution(
            execution_digest,
            "worker01", ["a1", "b2", "e3"],
            tensor_debug_mode,
            graph_id=None,
            input_tensor_ids=[12, 34],
            output_tensor_ids=[56, 78],
            debug_tensor_values=debug_tensor_values)
        monitor.on_execution(60, execution)

        self.assertEmpty(monitor.alerts())
    def testInfNanMonitorOnExecutionUnderFullTensorModeWorks(
            self, tensor_value, dtype, expected_size, expected_num_neg_inf,
            expected_num_pos_inf, expected_num_nan):
        mock_reader = test.mock.MagicMock()
        mock_reader.execution_to_tensor_values.return_value = [
            np.array([[0.0, -1.0, 1.0]]),
            np.array(tensor_value, dtype=dtype)
        ]
        monitor = debug_events_monitors.InfNanMonitor(mock_reader)
        execution_digest = debug_events_reader.ExecutionDigest(
            1234,
            1,
            "__inference_bar_function_1234",
            output_tensor_device_ids=[0, 1])
        execution = debug_events_reader.Execution(
            execution_digest,
            "worker01", ["a1", "b2", "e3"],
            debug_event_pb2.TensorDebugMode.FULL_TENSOR,
            graph_id=None,
            input_tensor_ids=[12, 34],
            output_tensor_ids=[56, 78])
        monitor.on_execution(70, execution)

        if expected_num_neg_inf or expected_num_pos_inf or expected_num_nan:
            self.assertLen(monitor.alerts(), 1)
            alert = monitor.alerts()[0]
            self.assertEqual(alert.wall_time, 1234)
            self.assertEqual(alert.op_type, "__inference_bar_function_1234")
            self.assertEqual(alert.output_slot, 1)
            self.assertEqual(alert.size, expected_size)
            self.assertEqual(alert.num_neg_inf, expected_num_neg_inf)
            self.assertEqual(alert.num_pos_inf, expected_num_pos_inf)
            self.assertEqual(alert.num_nan, expected_num_nan)
            self.assertEqual(alert.execution_index, 70)
            self.assertIsNone(alert.graph_execution_trace_index, 70)
        else:
            self.assertEmpty(monitor.alerts())