def setUp(self):
self.events_written = []
events_writer_manager = FakeEventsWriterManager(self.events_written)
self.stream_handler = debugger_server_lib.DebuggerDataStreamHandler(
events_writer_manager=events_writer_manager)
self.stream_handler.on_core_metadata_event(tf.Event())
def testSentinelStepValueAssignedWhenExecutorStepCountKeyIsMissing(self):
events_written = []
metadata_event = tf.Event()
metadata_event.log_message.message = json.dumps({})
stream_handler = debugger_server_lib.DebuggerDataStreamHandler(
events_writer_manager=FakeEventsWriterManager(events_written))
stream_handler.on_core_metadata_event(metadata_event)
health_pill_event = self._create_event_with_float_tensor(
"MatMul", 0, "DebugNumericSummary", list(range(1, 15)))
stream_handler.on_value_event(health_pill_event)
self.assertGreater(events_written[0].step, 0)
def testSentinelStepValueAssignedWhenMetadataJsonIsInvalid(self):
events_written = []
metadata_event = tf.Event()
metadata_event.log_message.message = "some invalid JSON string"
stream_handler = debugger_server_lib.DebuggerDataStreamHandler(
events_writer_manager=FakeEventsWriterManager(events_written))
stream_handler.on_core_metadata_event(metadata_event)
health_pill_event = self._create_event_with_float_tensor(
"MatMul", 0, "DebugNumericSummary", list(range(1, 15)))
stream_handler.on_value_event(health_pill_event)
self.assertGreater(events_written[0].step, 0)
def testAlertingEventCallback(self):
numerics_alert_callback = tf.test.mock.Mock()
stream_handler = debugger_server_lib.DebuggerDataStreamHandler(
events_writer_manager=FakeEventsWriterManager(
self.events_written),
numerics_alert_callback=numerics_alert_callback)
stream_handler.on_core_metadata_event(tf.Event())
# The stream handler receives 1 good event and 1 with an NaN value.
stream_handler.on_value_event(
self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary",
[0] * 14))
stream_handler.on_value_event(
self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [
0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]))
# The second event should have triggered the callback.
numerics_alert_callback.assert_called_once_with(
numerics_alert.NumericsAlert("/job:localhost/replica:0/task:0/cpu:0",
"Add:0", 0, 1, 0, 0))
# The stream handler receives an event with a -Inf value.
numerics_alert_callback.reset_mock()
stream_handler.on_value_event(
self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [
0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]))
numerics_alert_callback.assert_called_once_with(
numerics_alert.NumericsAlert("/job:localhost/replica:0/task:0/cpu:0",
"Add:0", 0, 0, 1, 0))
# The stream handler receives an event with a +Inf value.
numerics_alert_callback.reset_mock()
stream_handler.on_value_event(
self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [
0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0
]))
numerics_alert_callback.assert_called_once_with(
numerics_alert.NumericsAlert("/job:localhost/replica:0/task:0/cpu:0",
"Add:0", 0, 0, 0, 1))
# The stream handler receives an event without any pathetic values.
numerics_alert_callback.reset_mock()
stream_handler.on_value_event(
self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [
0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0
]))
# assert_not_called is not available in Python 3.4.
self.assertFalse(numerics_alert_callback.called)
def testCorrectStepIsWritten(self):
events_written = []
metadata_event = tf.Event()
metadata_event.log_message.message = json.dumps({"session_run_index": 42})
stream_handler = debugger_server_lib.DebuggerDataStreamHandler(
events_writer_manager=FakeEventsWriterManager(events_written))
stream_handler.on_core_metadata_event(metadata_event)
# The server receives 2 events. It should assign both the correct step.
stream_handler.on_value_event(
self._create_event_with_float_tensor("MatMul", 0, "DebugNumericSummary",
list(range(1, 15))))
stream_handler.on_value_event(
self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary",
list(range(2, 16))))
self.assertEqual(42, events_written[0].step)
self.assertEqual(42, events_written[1].step)
