def test_read_eval_metrics_when_no_events(self):
eval_dir = tempfile.mkdtemp()
self.assertTrue(os.path.exists(eval_dir))
# No error should be raised when eval directory exists with no event files.
self.assertEqual({}, early_stopping.read_eval_metrics(eval_dir))
os.rmdir(eval_dir)
self.assertFalse(os.path.exists(eval_dir))
# No error should be raised when eval directory does not exist.
self.assertEqual({}, early_stopping.read_eval_metrics(eval_dir))
def test_read_eval_metrics(self):
eval_dir = tempfile.mkdtemp()
_write_events(
eval_dir,
[
# steps, loss, accuracy
(1000, 1, 2),
(2000, 3, 4),
(3000, 5, 6),
])
self.assertEqual(
{
1000: {
'loss': 1,
'accuracy': 2
},
2000: {
'loss': 3,
'accuracy': 4
},
3000: {
'loss': 5,
'accuracy': 6
},
}, early_stopping.read_eval_metrics(eval_dir))
def test_data_loss_error_ignored(self):
eval_dir = tempfile.mkdtemp()
_write_events(
eval_dir,
[
# steps, loss, accuracy
(1000, 1, 2),
(2000, 3, 4),
(3000, 5, 6),
])
orig_tf_train_summary_iterator = tf.compat.v1.train.summary_iterator
def _summary_iterator(*args, **kwargs):
for event in orig_tf_train_summary_iterator(*args, **kwargs):
yield event
# Raise an error for one of the files after yielding a summary event.
if event.HasField('summary'):
raise tf.errors.DataLossError(None, None,
'testing data loss')
with mock.patch.object(tf.compat.v1.train,
'summary_iterator') as mock_summary_iterator:
mock_summary_iterator.side_effect = _summary_iterator
eval_results = early_stopping.read_eval_metrics(eval_dir)
self.assertEqual({1000: {'loss': 1, 'accuracy': 2}}, eval_results)
def after_run(
self,
run_context: "tf.estimator.SessionRunContext",
run_values: "tf.estimator.SessionRunValues",
) -> None:
global_step = run_values.results
# Get eval metrics every n steps.
if self._timer.should_trigger_for_step(global_step):
self._timer.update_last_triggered_step(global_step)
eval_metrics = read_eval_metrics(self._estimator.eval_dir())
else:
eval_metrics = None
if eval_metrics:
summary_step = next(reversed(eval_metrics))
latest_eval_metrics = eval_metrics[summary_step]
# If there exists a new evaluation summary.
if summary_step > self._current_summary_step:
current_score = latest_eval_metrics[self._metric]
if current_score is None:
current_score = float("nan")
self._trial.report(float(current_score), step=summary_step)
self._current_summary_step = summary_step
if self._trial.should_prune():
message = "Trial was pruned at iteration {}.".format(self._current_summary_step)
raise optuna.TrialPruned(message)
def after_run(self, run_context, run_values):
# type: (tf.train.SessionRunContext, tf.train.SessionRunValues) -> None
global_step = run_values.results
# Get eval metrics every n steps.
if self.timer.should_trigger_for_step(global_step):
self.timer.update_last_triggered_step(global_step)
eval_metrics = read_eval_metrics(self.estimator.eval_dir())
else:
eval_metrics = None
if eval_metrics:
summary_step = next(reversed(eval_metrics))
latest_eval_metrics = eval_metrics[summary_step]
# If there exists a new evaluation summary.
if summary_step > self.current_summary_step:
current_score = latest_eval_metrics[self.metric]
self.trial.report(current_score, step=summary_step)
self.current_summary_step = summary_step
if self.trial.should_prune():
message = "Trial was pruned at iteration {}.".format(
self.current_summary_step)
raise optuna.structs.TrialPruned(message)
