def setUp(self): self.events_written = [] events_writer_manager = FakeEventsWriterManager(self.events_written) self.stream_handler = debugger_server_lib.DebuggerDataStreamHandler( events_writer_manager=events_writer_manager) self.stream_handler.on_core_metadata_event(tf.Event())
def testSentinelStepValueAssignedWhenExecutorStepCountKeyIsMissing(self): events_written = [] metadata_event = tf.Event() metadata_event.log_message.message = json.dumps({}) stream_handler = debugger_server_lib.DebuggerDataStreamHandler( events_writer_manager=FakeEventsWriterManager(events_written)) stream_handler.on_core_metadata_event(metadata_event) health_pill_event = self._create_event_with_float_tensor( "MatMul", 0, "DebugNumericSummary", list(range(1, 15))) stream_handler.on_value_event(health_pill_event) self.assertGreater(events_written[0].step, 0)
def testSentinelStepValueAssignedWhenMetadataJsonIsInvalid(self): events_written = [] metadata_event = tf.Event() metadata_event.log_message.message = "some invalid JSON string" stream_handler = debugger_server_lib.DebuggerDataStreamHandler( events_writer_manager=FakeEventsWriterManager(events_written)) stream_handler.on_core_metadata_event(metadata_event) health_pill_event = self._create_event_with_float_tensor( "MatMul", 0, "DebugNumericSummary", list(range(1, 15))) stream_handler.on_value_event(health_pill_event) self.assertGreater(events_written[0].step, 0)
def testAlertingEventCallback(self): numerics_alert_callback = tf.test.mock.Mock() stream_handler = debugger_server_lib.DebuggerDataStreamHandler( events_writer_manager=FakeEventsWriterManager( self.events_written), numerics_alert_callback=numerics_alert_callback) stream_handler.on_core_metadata_event(tf.Event()) # The stream handler receives 1 good event and 1 with an NaN value. stream_handler.on_value_event( self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [0] * 14)) stream_handler.on_value_event( self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [ 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])) # The second event should have triggered the callback. numerics_alert_callback.assert_called_once_with( numerics_alert.NumericsAlert("/job:localhost/replica:0/task:0/cpu:0", "Add:0", 0, 1, 0, 0)) # The stream handler receives an event with a -Inf value. numerics_alert_callback.reset_mock() stream_handler.on_value_event( self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [ 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])) numerics_alert_callback.assert_called_once_with( numerics_alert.NumericsAlert("/job:localhost/replica:0/task:0/cpu:0", "Add:0", 0, 0, 1, 0)) # The stream handler receives an event with a +Inf value. numerics_alert_callback.reset_mock() stream_handler.on_value_event( self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [ 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 ])) numerics_alert_callback.assert_called_once_with( numerics_alert.NumericsAlert("/job:localhost/replica:0/task:0/cpu:0", "Add:0", 0, 0, 0, 1)) # The stream handler receives an event without any pathetic values. numerics_alert_callback.reset_mock() stream_handler.on_value_event( self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", [ 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0 ])) # assert_not_called is not available in Python 3.4. self.assertFalse(numerics_alert_callback.called)
def testCorrectStepIsWritten(self): events_written = [] metadata_event = tf.Event() metadata_event.log_message.message = json.dumps({"session_run_index": 42}) stream_handler = debugger_server_lib.DebuggerDataStreamHandler( events_writer_manager=FakeEventsWriterManager(events_written)) stream_handler.on_core_metadata_event(metadata_event) # The server receives 2 events. It should assign both the correct step. stream_handler.on_value_event( self._create_event_with_float_tensor("MatMul", 0, "DebugNumericSummary", list(range(1, 15)))) stream_handler.on_value_event( self._create_event_with_float_tensor("Add", 0, "DebugNumericSummary", list(range(2, 16)))) self.assertEqual(42, events_written[0].step) self.assertEqual(42, events_written[1].step)