def test_should_prune() -> None:
pruner = DeterministicPruner(True)
study = create_study(pruner=pruner)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
trial.report(1, 1)
assert trial.should_prune()
def _cross_validate_with_pruning(
self, trial: Trial,
estimator: "BaseEstimator") -> Mapping[str, OneDimArrayLikeType]:
if is_classifier(estimator):
partial_fit_params = self.fit_params.copy()
y = self.y.values if isinstance(self.y, pd.Series) else self.y
classes = np.unique(y)
partial_fit_params.setdefault("classes", classes)
else:
partial_fit_params = self.fit_params
n_splits = self.cv.get_n_splits(self.X, self.y, groups=self.groups)
estimators = [clone(estimator) for _ in range(n_splits)]
scores = {
"fit_time": np.zeros(n_splits),
"score_time": np.zeros(n_splits),
"test_score": np.empty(n_splits),
}
if self.return_train_score:
scores["train_score"] = np.empty(n_splits)
for step in range(self.max_iter):
for i, (train, test) in enumerate(
self.cv.split(self.X, self.y, groups=self.groups)):
out = self._partial_fit_and_score(estimators[i], train, test,
partial_fit_params)
if self.return_train_score:
scores["train_score"][i] = out.pop(0)
scores["test_score"][i] = out[0]
scores["fit_time"][i] += out[1]
scores["score_time"][i] += out[2]
intermediate_value = np.nanmean(scores["test_score"])
trial.report(intermediate_value, step=step)
if trial.should_prune():
self._store_scores(trial, scores)
raise TrialPruned(
"trial was pruned at iteration {}.".format(step))
return scores
def objective(trial: Trial) -> float:
x = trial.suggest_int("x", 5, 5)
if trial.number == 0:
return x
elif trial.number == 1:
trial.report(1, 4)
trial.report(2, 7)
raise TrialPruned()
elif trial.number == 2:
trial.report(float("nan"), 3)
raise TrialPruned()
elif trial.number == 3:
raise TrialPruned()
else:
raise RuntimeError()
def objective(trial: Trial) -> float:
x = trial.suggest_int("x", 5, 5)
z = trial.suggest_categorical("z", [None])
if trial.number == 0:
return x * int(z is None)
elif trial.number == 1:
trial.report(1, 4)
trial.report(2, 7)
raise TrialPruned()
elif trial.number == 2:
trial.report(float("nan"), 3)
raise TrialPruned()
elif trial.number == 3:
raise TrialPruned()
else:
raise RuntimeError()
def objective(trial: Trial) -> Tuple[float, float]:
with pytest.raises(NotImplementedError):
trial.report(1.0, 0)
return 1.0, 1.0
def test_report() -> None:
study = create_study()
trial = Trial(study, study._storage.create_new_trial(study._study_id))
# Report values that can be cast to `float` (OK).
trial.report(1.23, 1)
trial.report(float("nan"), 2)
trial.report("1.23", 3) # type: ignore
trial.report("inf", 4) # type: ignore
trial.report(1, 5)
trial.report(np.array([1], dtype=np.float32)[0], 6)
# Report values that cannot be cast to `float` or steps that are negative (Error).
with pytest.raises(TypeError):
trial.report(None, 7) # type: ignore
with pytest.raises(TypeError):
trial.report("foo", 7) # type: ignore
with pytest.raises(TypeError):
trial.report([1, 2, 3], 7) # type: ignore
with pytest.raises(TypeError):
trial.report("foo", -1) # type: ignore
with pytest.raises(ValueError):
trial.report(1.23, -1)
def objective(trial: Trial, report_intermediate_values: bool) -> float:
if report_intermediate_values:
trial.report(1.0, step=0)
trial.report(2.0, step=1)
return 0.0
def test_trial_report():
# type: () -> None
study = create_study()
trial = Trial(study, study._storage.create_new_trial(study.study_id))
# Report values that can be cast to `float` (OK).
trial.report(1.23)
trial.report(float('nan'))
trial.report('1.23') # type: ignore
trial.report('inf') # type: ignore
trial.report(1)
trial.report(np.array([1], dtype=np.float32)[0])
# Report values that cannot be cast to `float` (Error).
with pytest.raises(TypeError):
trial.report(None) # type: ignore
with pytest.raises(TypeError):
trial.report('foo') # type: ignore
with pytest.raises(TypeError):
trial.report([1, 2, 3]) # type: ignore
def objective(trial: Trial) -> float:
"""Compute objective value
Parameter
---------
trial : `Trial`
Current trial
Returns
-------
loss : `float`
Loss
"""
# use pyannote.metrics metric when available
try:
metric = self.pipeline.get_metric()
except NotImplementedError as e:
metric = None
losses = []
processing_time = []
evaluation_time = []
# instantiate pipeline with value suggested in current trial
pipeline = self.pipeline.instantiate(
self.pipeline.parameters(trial=trial))
if show_progress != False:
progress_bar = tqdm(total=len(inputs), **show_progress)
progress_bar.update(0)
# accumulate loss for each input
for i, input in enumerate(inputs):
# process input with pipeline
# (and keep track of processing time)
before_processing = time.time()
output = pipeline(input)
after_processing = time.time()
processing_time.append(after_processing - before_processing)
# evaluate output (and keep track of evaluation time)
before_evaluation = time.time()
# when metric is not available, use loss method instead
if metric is None:
loss = pipeline.loss(input, output)
losses.append(loss)
# when metric is available,`input` is expected to be provided
# by a `pyannote.database` protocol
else:
from pyannote.database import get_annotated
_ = metric(input["annotation"],
output,
uem=get_annotated(input))
after_evaluation = time.time()
evaluation_time.append(after_evaluation - before_evaluation)
if show_progress != False:
progress_bar.update(1)
if self.pruner is None:
continue
trial.report(
np.mean(losses) if metric is None else abs(metric), i)
if trial.should_prune(i):
raise optuna.structs.TrialPruned()
if show_progress != False:
progress_bar.close()
trial.set_user_attr("processing_time", sum(processing_time))
trial.set_user_attr("evaluation_time", sum(evaluation_time))
return np.mean(losses) if metric is None else abs(metric)