def test_prevents_fold_metrics_with_general_metric_names(self):
rb = rs.ObservationCollector()
rb.set_observation("loss", 0.5)
self.assertRaises(ValueError, rb.add_fold_observation, 0, "loss", 0.5)
self.assertRaises(ValueError, rb.add_fold_observation, 0, "loss", 0.2)
self.assertRaises(ValueError, rb.add_fold_observation, 0, "loss", 0.1)
rb.add_fold_observation(0, "fold_loss", 0.5)
def test_saves_multiple_fold_observations(self):
rb = rs.ObservationCollector()
rb.add_fold_observations(0, "loss", [0.5, 0.5, 0.5, 0.5])
self.assertEqual(rb.observations["loss"], [[0.5, 0.5, 0.5, 0.5]])
rb.add_fold_observations(1, "loss", [0.5, 0.44, 0.43, 0.4])
self.assertEqual(rb.observations["loss"],
[[0.5, 0.5, 0.5, 0.5], [0.5, 0.44, 0.43, 0.4]])
def test_prevents_general_metrics_with_fold_metric_names_multiple(self):
rb = rs.ObservationCollector()
rb.add_fold_observations(0, "loss", [0.5, 0.5, 0.5, 0.5])
rb.add_fold_observations(0, "loss", [0.2, 0.2, 0.2, 0.2])
rb.add_fold_observations(0, "loss", [0.1, 0.1, 0.1, 0.1])
self.assertRaises(ValueError, rb.set_observation, "loss", 0.5)
self.assertRaises(ValueError, rb.set_observation, "loss", 0.2)
self.assertRaises(ValueError, rb.set_observation, "loss", 0.1)
rb.set_observation("general_loss", [0.5, 4.7, 5.0])
def test_records_correctly_if_given_dict(self):
params = {
"title": "cool_experiment",
"learning_rate": 0.003,
"batch_size": 32,
"alpha": 2e-9,
"model": "rnn",
}
res = rs.ObservationCollector()
for i in range(3):
for j in range(1, 8):
res.add_fold_observation(i, "rmse", 0.98 / j)
rs.record_experiment(params, TEST_EXPERIMENT_PATH, observations=res.observations)
self.assertTrue(os.path.isfile(TEST_EXPERIMENT_PATH + "cool_experiment_d45dee5991986a5b8215706f5e904b3e.json"))
def test_records_NANs_as_zero(self):
params = {
"title": "cool_experiment",
"learning_rate": 0.003,
"batch_size": np.int64(32),
"alpha": 2e-9,
"model": "rnn",
}
res = rs.ObservationCollector()
for i in range(3):
for j in range(1, 8):
res.add_fold_observation(i, "rmse", float('nan'))
rs.record_experiment(params, TEST_EXPERIMENT_PATH, observations=res.observations)
self.assertTrue(os.path.isfile(TEST_EXPERIMENT_PATH + "cool_experiment_d45dee5991986a5b8215706f5e904b3e.json"))
e = rs.load_experiment(TEST_EXPERIMENT_PATH, "cool_experiment_d45dee5991986a5b8215706f5e904b3e.json")
def test_collects_dictionary(self):
rb = rs.ObservationCollector()
rb.set_observations({
"loss": 0.6,
"f-loss": 0.98,
"score": [5, 6, 7],
"flange-error": [[0.55, 0.66], [0.87, 0.78]]
})
self.assertAlmostEqual(rb.observations["loss"], 0.6, 7)
self.assertEqual(rb.observations["flange-error"],
[[0.55, 0.66], [0.87, 0.78]])
self.assertRaises(ValueError, rb.add_fold_observation, 0, "loss", 0.4)
self.assertRaises(ValueError, rb.add_fold_observation, 1, "loss", 0.4)
self.assertRaises(ValueError, rb.add_fold_observation, 0,
"flange-error", 0.4)
rb.add_fold_observation(0, "new-metric", 0.4)
def test_prevents_general_metrics_being_overwritten(self):
rb = rs.ObservationCollector()
rb.set_observation("loss", 0.5)
self.assertRaises(ValueError, rb.set_observation, "loss", 0.5)
self.assertRaises(ValueError, rb.set_observation, "loss", 0.2)
self.assertRaises(ValueError, rb.set_observation, "loss", 0.1)