def on_epoch_end(self, epoch, logs={}):
# update epoch counters
self.execution_state.epochs += 1
self.tuner_state.remaining_budget -= 1
objective = canonicalize_metric_name(self.tuner_state.objective)
# update metrics and checkpoint if needed
for metric, value in logs.items():
improved = self.execution_state.metrics.update(metric, value)
metric = canonicalize_metric_name(metric)
if objective == metric and improved:
# Compute classification metrics and store in execution state
if self.validation_data:
report = compute_common_classification_metrics(
self.model, self.validation_data,
self.tuner_state.label_names)
self.execution_state.update_performance_metrics(report)
# TODO - figure out the race condition that causes us to clear
# the session before we finish the writes when we try to
# apply_async here.
# self.thread_pool.apply_async(self._checkpoint_model)
self._checkpoint_model()
self._write_result_file()
# reset epoch history
self.epoch_history = defaultdict(list)
# update status
self._report_status(force=True)
def test_continuous_single_classification_metrics_int():
model = _single_output_model(dtype=tf.float32)
x_val = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0])
y_val = np.array([0, 0, 0, 0, 1, 0, 1, 1, 1, 1])
results = compute_common_classification_metrics(model, (x_val, y_val))
_test_classification_metrics(results)
def test_continuous_multi_classification_metrics():
model = _multi_output_model(2, dtype=tf.float32)
x_val = np.array([[-1, 1] for _ in range(5)] + [[1, -1] for _ in range(5)])
y_val = [0, 0, 0, 0, 1, 0, 1, 1, 1, 1]
y_val = np.array([(x, 1 - x) for x in y_val], dtype=np.float32)
results = compute_common_classification_metrics(model, (x_val, y_val))
_test_classification_metrics(results)
def test_continuous_single_classification_metrics_training_end():
model = _single_output_model(dtype=tf.float32)
x_val = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0])
y_val = np.array([0, 0, 0, 0, 1, 0, 1, 1, 1, 1])
results = compute_common_classification_metrics(model, (x_val, y_val))
_test_classification_metrics(results)
assert np.allclose(results["roc_curve"]["fpr"], [0, 0.2, 1], atol=.05)
assert np.allclose(results["roc_curve"]["tpr"], [0, 0.8, 1], atol=.05)
assert np.allclose(results["roc_curve"]["thresholds"], [2, 1, 0], atol=.05)
def test_continuous_multi_classification_metrics_5way():
model = _multi_output_model(5, dtype=tf.float32)
x = [x for x in range(5)] + [-x for x in range(5)]
x = tf.keras.utils.to_categorical(x, num_classes=5)
y = x
results = compute_common_classification_metrics(model, (x, y))
metrics = results["classification_metrics"]
assert np.isclose(1, metrics["macro avg"]["f1-score"])
assert np.isclose(1, metrics["weighted avg"]["f1-score"])
def test_continuous_multi_classification_metrics_float():
model = _multi_output_model(2, dtype=tf.float32)
model.summary()
x_val = np.array([[0, 1] for _ in range(5)] + [[0, -1] for _ in range(5)],
dtype=np.float32)
y_val = [0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0]
y_val = np.array([[x, 1 - x] for x in y_val], dtype=np.float32)
results = compute_common_classification_metrics(model, (x_val, y_val))
_test_classification_metrics(results)