def testNumericsAlertReportResponse(self): """Tests that reports of bad values are returned.""" alerts = [ numerics_alert.NumericsAlertReportRow('cpu0', 'MatMul', 123, 2, 3, 4), numerics_alert.NumericsAlertReportRow('cpu1', 'Add', 124, 5, 6, 7), ] self.mock_debugger_data_server.numerics_alert_report.return_value = alerts response = self.server.get('/data/plugin/debugger/numerics_alert_report') self.assertEqual(200, response.status_code) retrieved_alerts = self._DeserializeResponse(response.get_data()) self.assertEqual(2, len(retrieved_alerts)) self.assertDictEqual({ 'device_name': 'cpu0', 'tensor_name': 'MatMul', 'first_timestamp': 123, 'nan_event_count': 2, 'neg_inf_event_count': 3, 'pos_inf_event_count': 4, }, retrieved_alerts[0]) self.assertDictEqual({ 'device_name': 'cpu1', 'tensor_name': 'Add', 'first_timestamp': 124, 'nan_event_count': 5, 'neg_inf_event_count': 6, 'pos_inf_event_count': 7, }, retrieved_alerts[1])
def testMultipleEventsFromDifferentDevicesAndSameTensorName(self): alert_1 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1) alert_2 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 1, 1) alert_3 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1634, 1, 1, 1) registry = numerics_alert.NumericsAlertRegistry() registry.register(alert_1) registry.register(alert_2) registry.register(alert_3) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 1, 2, 2, ), numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1, ), ], registry.report(), )
def testLoadFromJson(self): registry = numerics_alert.NumericsAlertRegistry(initialization_list=[ [ "/job:localhost/replica:0/task:0/cpu:0", "MatMul:0", { "pos_inf": [0, -1, -1], "nan": [1624, 1496818651573005, 1496818690371163], "neg_inf": [0, -1, -1], }, ], [ "/job:localhost/replica:0/task:0/cpu:0", "weight/Adagrad:0", { "pos_inf": [0, -1, -1], "nan": [1621, 1496818651607234, 1496818690370891], "neg_inf": [0, -1, -1], }, ], ]) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:localhost/replica:0/task:0/cpu:0", "MatMul:0", 1496818651573005, 1624, 0, 0, ), numerics_alert.NumericsAlertReportRow( "/job:localhost/replica:0/task:0/cpu:0", "weight/Adagrad:0", 1496818651607234, 1621, 0, 0, ), ], registry.report(), )
def testFilterReport(self): alert_1 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1) alert_2 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 1, 1) alert_3 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Mean:0", 1634, 1, 1, 1) registry = numerics_alert.NumericsAlertRegistry() registry.register(alert_1) registry.register(alert_2) registry.register(alert_3) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1, ) ], registry.report(device_name_filter=r".*\/task:1\/.*"), ) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:0/gpu:0", "xent/Mean:0", 1634, 1, 1, 1, ) ], registry.report(tensor_name_filter=r".*Mean.*"), )
def testNumericsAlertReportResponse(self): """Tests that reports of bad values are returned.""" alerts = [ numerics_alert.NumericsAlertReportRow("cpu0", "MatMul", 123, 2, 3, 4), numerics_alert.NumericsAlertReportRow("cpu1", "Add", 124, 5, 6, 7), ] self.mock_debugger_data_server.numerics_alert_report.return_value = ( alerts) response = self.server.get( "/data/plugin/debugger/numerics_alert_report") self.assertEqual(200, response.status_code) retrieved_alerts = self._DeserializeResponse(response.get_data()) self.assertEqual(2, len(retrieved_alerts)) self.assertDictEqual( { "device_name": "cpu0", "tensor_name": "MatMul", "first_timestamp": 123, "nan_event_count": 2, "neg_inf_event_count": 3, "pos_inf_event_count": 4, }, retrieved_alerts[0], ) self.assertDictEqual( { "device_name": "cpu1", "tensor_name": "Add", "first_timestamp": 124, "nan_event_count": 5, "neg_inf_event_count": 6, "pos_inf_event_count": 7, }, retrieved_alerts[1], )
def testSingleAlert(self): alert = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 10, 10) registry = numerics_alert.NumericsAlertRegistry() registry.register(alert) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 1, 1, ) ], registry.report(), )
def testMultipleEventsFromSameDeviceAndSameTensor(self): alert_1 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 10, 10) alert_2 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1634, 5, 5, 5) registry = numerics_alert.NumericsAlertRegistry() registry.register(alert_1) registry.register(alert_2) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 1, 2, 2, ) ], registry.report(), )
def testRegisterBeyondCapacityObeysCapacity(self): alert_1 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1) alert_2 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 1, 1) alert_3 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:2/gpu:0", "xent/Log:0", 1634, 0, 1, 1) alert_4 = numerics_alert.NumericsAlert( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1834, 1, 1, 1) registry = numerics_alert.NumericsAlertRegistry(capacity=2) registry.register(alert_1) registry.register(alert_2) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 1, 1, ), numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1, ), ], registry.report(), ) registry.register(alert_3) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 0, 1, 1, ), numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1, ), ], registry.report(), ) registry.register(alert_4) self.assertEqual( [ numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:0/gpu:0", "xent/Log:0", 1234, 1, 2, 2, ), numerics_alert.NumericsAlertReportRow( "/job:worker/replica:0/task:1/gpu:0", "xent/Log:0", 1434, 0, 1, 1, ), ], registry.report(), )