def objective(trial: Trial) -> Tuple[float, float]:
with pytest.raises(NotImplementedError):
trial.should_prune()
return 1.0, 1.0
def objective(trial: Trial) -> float:
"""Compute objective value
Parameter
---------
trial : `Trial`
Current trial
Returns
-------
loss : `float`
Loss
"""
# use pyannote.metrics metric when available
try:
metric = self.pipeline.get_metric()
except NotImplementedError as e:
metric = None
losses = []
processing_time = []
evaluation_time = []
# instantiate pipeline with value suggested in current trial
pipeline = self.pipeline.instantiate(
self.pipeline.parameters(trial=trial))
if show_progress != False:
progress_bar = tqdm(total=len(inputs), **show_progress)
progress_bar.update(0)
# accumulate loss for each input
for i, input in enumerate(inputs):
# process input with pipeline
# (and keep track of processing time)
before_processing = time.time()
output = pipeline(input)
after_processing = time.time()
processing_time.append(after_processing - before_processing)
# evaluate output (and keep track of evaluation time)
before_evaluation = time.time()
# when metric is not available, use loss method instead
if metric is None:
loss = pipeline.loss(input, output)
losses.append(loss)
# when metric is available,`input` is expected to be provided
# by a `pyannote.database` protocol
else:
from pyannote.database import get_annotated
_ = metric(input["annotation"],
output,
uem=get_annotated(input))
after_evaluation = time.time()
evaluation_time.append(after_evaluation - before_evaluation)
if show_progress != False:
progress_bar.update(1)
if self.pruner is None:
continue
trial.report(
np.mean(losses) if metric is None else abs(metric), i)
if trial.should_prune(i):
raise optuna.structs.TrialPruned()
if show_progress != False:
progress_bar.close()
trial.set_user_attr("processing_time", sum(processing_time))
trial.set_user_attr("evaluation_time", sum(evaluation_time))
return np.mean(losses) if metric is None else abs(metric)
