def format_objective(objective, direction=None):
"""Formats objective to a list of oracle_module.Objective.
Arguments:
objective: If a string, the direction of the optimization (min or max) will
be inferred.
direction: Optional. e.g. 'min' or 'max'.
Returns:
A list of oracle_module.Objective.
Raises:
TypeError: indicates wrong objective format.
"""
if isinstance(objective, oracle_module.Objective):
return [objective]
if isinstance(objective, str):
if direction:
return [oracle_module.Objective(objective, direction)]
return [
oracle_module.Objective(
objective, metrics_tracking.infer_metric_direction(objective))
]
if isinstance(objective, list):
if isinstance(objective[0], oracle_module.Objective):
return objective
if isinstance(objective[0], str):
return [
oracle_module.Objective(
m, metrics_tracking.infer_metric_direction(m))
for m in objective
]
raise TypeError(
'Objective should be either string or oracle_module.Objective, found {}'
.format(objective))
def update_trial(self,
trial_id: Text,
metrics: Mapping[Text, Union[int, float]],
step: int = 0):
"""Used by a worker to report the status of a trial."""
# Constructs the measurement.
# Adds the measurement of the objective functions to a trial.
elapsed_secs = time.time() - self._start_time
if elapsed_secs < 0 or step < 0:
raise ValueError(
"Both elapsed_secs and step must be non-negative.")
if elapsed_secs == 0 and step == 0:
raise ValueError(
"At least one of {elapsed_secs, step} must be positive")
metric_list = []
for ob in self._get_objective():
if ob.name not in metrics:
ob_name = ob.name.replace("val_", "")
if ob_name in metrics:
metric_list.append({
"metric": ob_name,
"value": float(metrics.get(ob_name))
})
tf.get_logger().info(
'Objective "{}" is not found in metrics.'.format(ob.name))
continue
metric_list.append({
"metric": ob.name,
"value": float(metrics.get(ob.name))
})
self.service.report_intermediate_objective_value(
step, elapsed_secs, metric_list, trial_id)
# Ensure metrics of trials are updated locally.
keras_tuner_trial = self.trials[trial_id]
for metric_name, metric_value in metrics.items():
if not keras_tuner_trial.metrics.exists(metric_name):
direction = metrics_tracking.infer_metric_direction(
metric_name)
keras_tuner_trial.metrics.register(metric_name,
direction=direction)
keras_tuner_trial.metrics.update(metric_name,
metric_value,
step=step)
# Checks whether a trial should stop or not.
tf.get_logger().info("UpdateTrial: polls the stop decision.")
should_stop = self.service.should_trial_stop(trial_id)
if should_stop:
keras_tuner_trial.status = trial_module.TrialStatus.STOPPED
return keras_tuner_trial.status
def _format_objective(objective):
if objective is None:
return Objective("default_objective", "min")
if isinstance(objective, list):
return [_format_objective(obj) for obj in objective]
if isinstance(objective, Objective):
return objective
if isinstance(objective, str):
direction = metrics_tracking.infer_metric_direction(objective)
if direction is None:
error_msg = (
'Could not infer optimization direction ("min" or "max") '
'for unknown metric "{obj}". Please specify the objective as'
"a `keras_tuner.Objective`, for example `keras_tuner.Objective("
'"{obj}", direction="min")`.')
error_msg = error_msg.format(obj=objective)
raise ValueError(error_msg)
return Objective(name=objective, direction=direction)
else:
raise ValueError("`objective` not understood, expected str or "
"`Objective` object, found: {}".format(objective))
def test_metric_direction_inference():
# Test min metrics.
assert metrics_tracking.infer_metric_direction("MAE") == "min"
assert (metrics_tracking.infer_metric_direction(
metrics.binary_crossentropy) == "min")
assert metrics_tracking.infer_metric_direction(
metrics.FalsePositives()) == "min"
# All losses in keras.losses are considered as 'min'.
assert metrics_tracking.infer_metric_direction("squared_hinge") == "min"
assert metrics_tracking.infer_metric_direction(losses.hinge) == "min"
assert (metrics_tracking.infer_metric_direction(
losses.CategoricalCrossentropy()) == "min")
# Test max metrics.
assert metrics_tracking.infer_metric_direction("binary_accuracy") == "max"
assert (metrics_tracking.infer_metric_direction(
metrics.categorical_accuracy) == "max")
assert metrics_tracking.infer_metric_direction(
metrics.Precision()) == "max"
# Test unknown metrics.
assert metrics_tracking.infer_metric_direction("my_metric") is None
def my_metric_fn(x, y):
return x
assert metrics_tracking.infer_metric_direction(my_metric_fn) is None
class MyMetric(metrics.Metric):
def update_state(self, x, y):
return 1
def result(self):
return 1
assert metrics_tracking.infer_metric_direction(MyMetric()) is None
# Test special cases.
assert metrics_tracking.infer_metric_direction("loss") == "min"
assert metrics_tracking.infer_metric_direction("acc") == "max"
assert metrics_tracking.infer_metric_direction("val_acc") == "max"
assert metrics_tracking.infer_metric_direction("crossentropy") == "min"
assert metrics_tracking.infer_metric_direction("ce") == "min"
assert metrics_tracking.infer_metric_direction("weighted_acc") == "max"
assert metrics_tracking.infer_metric_direction("val_weighted_ce") == "min"
assert (metrics_tracking.infer_metric_direction("weighted_binary_accuracy")
== "max")