def _compute_model(self, skip_optimization: bool = None):
args = self._model_args
if skip_optimization is None:
skip_optimization = self.skip_optimization(self._state)
fit_parameters = not skip_optimization
if fit_parameters and self._candidate_evaluations:
# Did the labeled data really change since the last recent refit?
# If not, skip the refitting
if self._state.candidate_evaluations == self._candidate_evaluations:
fit_parameters = False
logger.warning(
"Skipping the refitting of GP hyperparameters, since the "
"labeled data did not change since the last recent fit")
self._model = GPMXNetModel(
state=self._state,
active_metric=args.active_metric,
random_seed=args.random_seed,
gpmodel=self._gpmodel,
fit_parameters=fit_parameters,
num_fantasy_samples=args.num_fantasy_samples,
normalize_targets=args.normalize_targets,
profiler=self._profiler,
debug_log=self._debug_log)
# Note: This may be different than self._gpmodel.get_params(), since
# the GPMXNetModel may append additional info
self._model_params = self._model.get_params()
if fit_parameters:
# Keep copy of labeled data in order to avoid unnecessary
# refitting
self._candidate_evaluations = copy.copy(
self._state.candidate_evaluations)
def test_gp_mcmc_fit(tuning_job_state):
def tuning_job_state_mcmc(X, Y) -> TuningJobState:
Y = [dictionarize_objective(y) for y in Y]
return TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', -4., 4., LinearScaling())),
[CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
_set_seeds(0)
def f(x):
return 0.1 * np.power(x, 3)
X = np.concatenate((np.random.uniform(-4., -1.,
10), np.random.uniform(1., 4., 10)))
Y = f(X)
X_test = np.sort(np.random.uniform(-1., 1., 10))
X = [(x, ) for x in X]
X_test = [(x, ) for x in X_test]
tuning_job_state = tuning_job_state_mcmc(X, Y)
# checks if fitting is running
random_seed = 0
gpmodel = default_gpmodel_mcmc(tuning_job_state,
random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
model = GPMXNetModel(tuning_job_state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=20)
X = [tuning_job_state.hp_ranges.to_ndarray(x) for x in X]
predictions = model.predict(np.array(X))
Y_std_list = [stds for means, stds in predictions]
Y_mean_list = [means for means, stds in predictions]
Y_mean = np.mean(Y_mean_list, axis=0)
Y_std = np.mean(Y_std_list, axis=0)
assert np.all(np.abs(Y_mean - Y) < 1e-1), \
"in a noiseless setting, mean of GP should coincide closely with outputs at training points"
X_test = [tuning_job_state.hp_ranges.to_ndarray(x) for x in X_test]
predictions_test = model.predict(np.array(X_test))
Y_std_test_list = [stds for means, stds in predictions_test]
Y_std_test = np.mean(Y_std_test_list, axis=0)
assert np.max(Y_std) < np.min(Y_std_test), \
"Standard deviation on un-observed points should be greater than at observed ones"
def default_models() -> List[GPMXNetModel]:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
(0.0, 0.0
), # same evals are added multiple times to force GP to unlearn prior
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
state = TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
),
[CandidateEvaluation(x, y) for x, y in zip(X, Y)],
[],
[],
)
random_seed = 0
gpmodel = default_gpmodel(state,
random_seed=random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
gpmodel_mcmc = default_gpmodel_mcmc(state,
random_seed=random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
return [
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=20),
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel_mcmc,
fit_parameters=True,
num_fantasy_samples=20)
]
def default_models(do_mcmc=True) -> List[GPMXNetModel]:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
state = TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
random_seed = 0
gpmodel = default_gpmodel(state,
random_seed=random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
result = [
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=20)
]
if do_mcmc:
gpmodel_mcmc = default_gpmodel_mcmc(state,
random_seed=random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
result.append(
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel_mcmc,
fit_parameters=True,
num_fantasy_samples=20))
return result
def test_gp_fit(tuning_job_state):
_set_seeds(0)
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [np.sum(x) * 10.0 for x in X]
# checks if fitting is running
random_seed = 0
gpmodel = default_gpmodel(tuning_job_state,
random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
model = GPMXNetModel(tuning_job_state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=20)
X = [tuning_job_state.hp_ranges.to_ndarray(x) for x in X]
Y_mean, Y_std = model.predict(np.array(X))[0]
assert np.all(np.abs(Y_mean - Y) < 1e-1), \
"in a noiseless setting, mean of GP should coincide closely with outputs at training points"
X_test = [
(0.2, 0.2),
(0.4, 0.2),
(0.1, 0.9),
(0.5, 0.5),
]
X_test = [tuning_job_state.hp_ranges.to_ndarray(x) for x in X_test]
Y_mean_test, Y_std_test = model.predict(np.array(X_test))[0]
assert np.min(Y_std) < np.min(Y_std_test), \
"Standard deviation on un-observed points should be greater than at observed ones"
def fit_predict_ours(data: dict,
random_seed: int,
optimization_config: OptimizationConfig,
test_intermediates: Optional[dict] = None) -> dict:
# Create surrogate model
num_dims = len(data['ss_limits'])
_gpmodel = GaussianProcessRegression(
kernel=Matern52(num_dims, ARD=True),
mean=ZeroMeanFunction(), # Instead of ScalarMeanFunction
optimization_config=optimization_config,
random_seed=random_seed,
test_intermediates=test_intermediates)
model = GPMXNetModel(data['state'],
DEFAULT_METRIC,
random_seed,
_gpmodel,
fit_parameters=True,
num_fantasy_samples=20)
model_params = model.get_params()
print('Hyperparameters: {}'.format(model_params))
# Prediction
means, stddevs = model.predict(data['test_inputs'])[0]
return {'means': means, 'stddevs': stddevs}
def test_gp_fantasizing():
"""
Compare whether acquisition function evaluations (values, gradients) with
fantasizing are the same as averaging them by hand.
"""
random_seed = 4567
_set_seeds(random_seed)
num_fantasy_samples = 10
num_pending = 5
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()))
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
num_data = len(X)
Y = [
dictionarize_objective(np.random.randn(1, 1)) for _ in range(num_data)
]
# Draw fantasies. This is done for a number of fixed pending candidates
# The model parameters are fit in the first iteration, when there are
# no pending candidates
# Note: It is important to not normalize targets, because this would be
# done on the observed targets only, not the fantasized ones, so it
# would be hard to compare below.
pending_evaluations = []
for _ in range(num_pending):
pending_cand = tuple(np.random.rand(2, ))
pending_evaluations.append(PendingEvaluation(pending_cand))
state = TuningJobState(hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X, Y)],
failed_candidates=[],
pending_evaluations=pending_evaluations)
gpmodel = default_gpmodel(state,
random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
model = GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
fantasy_samples = model.fantasy_samples
# Evaluate acquisition function and gradients with fantasizing
num_test = 50
X_test = np.vstack([
hp_ranges.to_ndarray(tuple(np.random.rand(2, )))
for _ in range(num_test)
])
acq_func = EIAcquisitionFunction(model)
fvals, grads = acq_func.compute_acq_with_gradients(X_test)
# Do the same computation by averaging by hand
fvals_cmp = np.empty((num_fantasy_samples, ) + fvals.shape)
grads_cmp = np.empty((num_fantasy_samples, ) + grads.shape)
X_full = X + state.pending_candidates
for it in range(num_fantasy_samples):
Y_full = Y + [
dictionarize_objective(eval.fantasies[DEFAULT_METRIC][:, it])
for eval in fantasy_samples
]
state2 = TuningJobState(
hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X_full, Y_full)],
failed_candidates=[],
pending_evaluations=[])
# We have to skip parameter optimization here
model2 = GPMXNetModel(state2,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=False,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
acq_func2 = EIAcquisitionFunction(model2)
fvals_, grads_ = acq_func2.compute_acq_with_gradients(X_test)
fvals_cmp[it, :] = fvals_
grads_cmp[it, :] = grads_
# Comparison
fvals2 = np.mean(fvals_cmp, axis=0)
grads2 = np.mean(grads_cmp, axis=0)
assert np.allclose(fvals, fvals2)
assert np.allclose(grads, grads2)
class GPMXNetPendingCandidateStateTransformer(PendingCandidateStateTransformer):
"""
This class maintains the TuningJobState along an asynchronous GP-based
HPO experiment, and manages the reaction to changes of this state.
In particular, it provides a GPMXNetModel on demand, which encapsulates
the GP posterior.
Note: The GPMXNetModel can be accessed only once the state has at least
one labeled case, since otherwise no posterior can be computed.
skip_optimization is a predicate depending on TuningJobState, determining
what is done at the next recent GPMXNetModel computation. If False, the
GP hyperparameters are optimized. Otherwise, the current ones are not
changed.
Safeguard against multiple GP hyperparameter optimization while labeled
data does not change:
The posterior has to be recomputed every time the state changes, even if
this only concerns pending evaluations. The expensive part of this is
refitting the GP hyperparameters, which makes sense only when the labeled
data in the state changes. We put a safeguard in place to avoid refitting
when the labeled data is unchanged.
"""
def __init__(
self, gpmodel: GPModel, init_state: TuningJobState,
model_args: GPMXNetModelArgs,
skip_optimization: SkipOptimizationPredicate = None,
profiler: GPMXNetSimpleProfiler = None,
debug_log: Optional[DebugLogPrinter] = None):
self._gpmodel = gpmodel
self._state = copy.copy(init_state)
self._model_args = model_args
if skip_optimization is None:
self.skip_optimization = NeverSkipPredicate()
else:
self.skip_optimization = skip_optimization
self._profiler = profiler
self._debug_log = debug_log
# GPMXNetModel computed on demand
self._model: GPMXNetModel = None
self._candidate_evaluations = None
# _model_params is returned by get_params. Careful: This is not just
# self._gpmodel.get_params(), since the current GPMXNetModel may append
# additional parameters
self._model_params = gpmodel.get_params()
@property
def state(self) -> TuningJobState:
return self._state
def model(self, **kwargs) -> GPMXNetModel:
"""
If skip_optimization is given, it overrides the self.skip_optimization
predicate.
:return: GPMXNetModel for current state
"""
if self._model is None:
skip_optimization = kwargs.get('skip_optimization')
self._compute_model(skip_optimization=skip_optimization)
return self._model
def get_params(self):
return self._model_params
def set_params(self, param_dict):
self._gpmodel.set_params(param_dict)
self._model_params = self._gpmodel.get_params()
def append_candidate(self, candidate: Candidate):
"""
Appends new pending candidate to the state.
:param candidate: New pending candidate
"""
self._model = None # Invalidate
self._state.pending_evaluations.append(PendingEvaluation(candidate))
@staticmethod
def _find_candidate(candidate: Candidate, lst: List):
try:
pos = next(
i for i, x in enumerate(lst)
if x.candidate == candidate)
except StopIteration:
pos = -1
return pos
def drop_candidate(self, candidate: Candidate):
"""
Drop candidate (labeled or pending) from state.
:param candidate: Candidate to be dropped
"""
# Candidate may be labeled or pending. First, try labeled
pos = self._find_candidate(
candidate, self._state.candidate_evaluations)
if pos != -1:
self._model = None # Invalidate
self._state.candidate_evaluations.pop(pos)
if self._debug_log is not None:
deb_msg = "[GPMXNetAsyncPendingCandidateStateTransformer.drop_candidate]\n"
deb_msg += ("- len(candidate_evaluations) afterwards = {}".format(
len(self.state.candidate_evaluations)))
logger.info(deb_msg)
else:
# Try pending
pos = self._find_candidate(
candidate, self._state.pending_evaluations)
assert pos != -1, \
"Candidate {} not registered (neither labeled, nor pending)".format(
candidate)
self._model = None # Invalidate
self._state.pending_evaluations.pop(pos)
if self._debug_log is not None:
deb_msg = "[GPMXNetAsyncPendingCandidateStateTransformer.drop_candidate]\n"
deb_msg += ("- len(pending_evaluations) afterwards = {}\n".format(
len(self.state.pending_evaluations)))
logger.info(deb_msg)
def label_candidate(self, data: CandidateEvaluation):
"""
Adds a labeled candidate. If it was pending before, it is removed as
pending candidate.
:param data: New labeled candidate
"""
pos = self._find_candidate(
data.candidate, self._state.pending_evaluations)
if pos != -1:
self._state.pending_evaluations.pop(pos)
self._state.candidate_evaluations.append(data)
self._model = None # Invalidate
def filter_pending_evaluations(
self, filter_pred: Callable[[PendingEvaluation], bool]):
"""
Filters state.pending_evaluations with filter_pred.
:param filter_pred Filtering predicate
"""
new_pending_evaluations = list(filter(
filter_pred, self._state.pending_evaluations))
if len(new_pending_evaluations) != len(self._state.pending_evaluations):
if self._debug_log is not None:
deb_msg = "[GPMXNetAsyncPendingCandidateStateTransformer.filter_pending_evaluations]\n"
deb_msg += ("- from len {} to {}".format(
len(self.state.pending_evaluations), len(new_pending_evaluations)))
logger.info(deb_msg)
self._model = None # Invalidate
del self._state.pending_evaluations[:]
self._state.pending_evaluations.extend(new_pending_evaluations)
def mark_candidate_failed(self, candidate: Candidate):
self._state.failed_candidates.append(candidate)
def _compute_model(self, skip_optimization: bool = None):
args = self._model_args
if skip_optimization is None:
skip_optimization = self.skip_optimization(self._state)
fit_parameters = not skip_optimization
if fit_parameters and self._candidate_evaluations:
# Did the labeled data really change since the last recent refit?
# If not, skip the refitting
if self._state.candidate_evaluations == self._candidate_evaluations:
fit_parameters = False
logger.warning(
"Skipping the refitting of GP hyperparameters, since the "
"labeled data did not change since the last recent fit")
self._model = GPMXNetModel(
state=self._state,
active_metric=args.active_metric,
random_seed=args.random_seed,
gpmodel=self._gpmodel,
fit_parameters=fit_parameters,
num_fantasy_samples=args.num_fantasy_samples,
normalize_targets=args.normalize_targets,
profiler=self._profiler,
debug_log=self._debug_log)
# Note: This may be different than self._gpmodel.get_params(), since
# the GPMXNetModel may append additional info
self._model_params = self._model.get_params()
if fit_parameters:
# Keep copy of labeled data in order to avoid unnecessary
# refitting
self._candidate_evaluations = copy.copy(
self._state.candidate_evaluations)