def tuning_job_state():
return {
'algo-1':
TuningJobState(hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('a1_hp_1', -5.0, 5.0,
LinearScaling()),
HyperparameterRangeCategorical('a1_hp_2', ('a', 'b', 'c'))),
candidate_evaluations=[
CandidateEvaluation(candidate=(-3.0, 'a'),
value=1.0),
CandidateEvaluation(candidate=(-1.9, 'c'),
value=2.0),
CandidateEvaluation(candidate=(-3.5, 'a'),
value=0.3)
],
failed_candidates=[],
pending_evaluations=[]),
'algo-2':
TuningJobState(hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('a2_hp_1', -5.0, 5.0,
LinearScaling()),
HyperparameterRangeInteger('a2_hp_2', -5, 5, LinearScaling(), -5,
5)),
candidate_evaluations=[
CandidateEvaluation(candidate=(-1.9, -1),
value=0.0),
CandidateEvaluation(candidate=(-3.5, 3), value=2.0)
],
failed_candidates=[],
pending_evaluations=[])
}
def tuning_job_state_mcmc(X, Y) -> TuningJobState:
Y = [dictionarize_objective(y) for y in Y]
return TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', -4., 4., LinearScaling())),
[CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def data_to_state(data: dict) -> TuningJobState:
cs = CS.ConfigurationSpace()
cs_names = ['x{}'.format(i) for i in range(len(data['ss_limits']))]
cs.add_hyperparameters([
CSH.UniformFloatHyperparameter(
name=name, lower=lims['min'], upper=lims['max'])
for name, lims in zip(cs_names, data['ss_limits'])])
_evaluations = []
x_mult = []
x_add = []
for lim in data['ss_limits']:
mn, mx = lim['min'], lim['max']
x_mult.append(mx - mn)
x_add.append(mn)
x_mult = np.array(x_mult)
x_add = np.array(x_add)
for x, y in zip(data['train_inputs'], data['train_targets']):
x_decoded = x * x_mult + x_add
config_dct = dict(zip(cs_names, x_decoded))
config = CS.Configuration(cs, values=config_dct)
_evaluations.append(CandidateEvaluation(
config, dictionarize_objective(y)))
return TuningJobState(
hp_ranges=HyperparameterRanges_CS(cs),
candidate_evaluations=_evaluations,
failed_candidates=[],
pending_evaluations=[])
def default_models() -> List[GPMXNetModel]:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
(0.0, 0.0
), # same evals are added multiple times to force GP to unlearn prior
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
state = TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
),
[CandidateEvaluation(x, y) for x, y in zip(X, Y)],
[],
[],
)
random_seed = 0
gpmodel = default_gpmodel(state,
random_seed=random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
gpmodel_mcmc = default_gpmodel_mcmc(state,
random_seed=random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
return [
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=20),
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel_mcmc,
fit_parameters=True,
num_fantasy_samples=20)
]
def tuning_job_state() -> TuningJobState:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
return TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def update(self, config: Candidate, reward: float):
"""
Registers new datapoint at config, with reward reward.
Note that in general, config should previously have been registered as
pending (register_pending). If so, it is switched from pending
to labeled. If not, it is considered directly labeled.
:param config:
:param reward:
"""
crit_val = self.map_reward(reward)
self.state_transformer.label_candidate(CandidateEvaluation(
candidate=copy.deepcopy(config),
metrics=dictionarize_objective(crit_val)))
if self.debug_log is not None:
config_id = self.debug_log.config_id(config)
msg = "Update for config_id {}: reward = {}, crit_val = {}".format(
config_id, reward, crit_val)
logger.info(msg)
def decode_state(enc_state: dict, hp_ranges: HyperparameterRanges_CS) \
-> TuningJobState:
assert isinstance(hp_ranges, HyperparameterRanges_CS), \
"Must have hp_ranges of HyperparameterRanges_CS type"
config_space = hp_ranges.config_space
def to_cs(x):
return CS.Configuration(config_space, values=x)
candidate_evaluations = [
CandidateEvaluation(to_cs(x['candidate']), x['metrics'])
for x in enc_state['candidate_evaluations']]
failed_candidates = [to_cs(x) for x in enc_state['failed_candidates']]
pending_evaluations = [
PendingEvaluation(to_cs(x)) for x in enc_state['pending_evaluations']]
return TuningJobState(
hp_ranges=hp_ranges,
candidate_evaluations=candidate_evaluations,
failed_candidates=failed_candidates,
pending_evaluations=pending_evaluations)
def default_models(do_mcmc=True) -> List[GPMXNetModel]:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
state = TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
random_seed = 0
gpmodel = default_gpmodel(state,
random_seed=random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
result = [
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=20)
]
if do_mcmc:
gpmodel_mcmc = default_gpmodel_mcmc(state,
random_seed=random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
result.append(
GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel_mcmc,
fit_parameters=True,
num_fantasy_samples=20))
return result
def _candidate_evaluations(num):
return [
CandidateEvaluation(candidate=(i, ),
metrics=dictionarize_objective(float(i)))
for i in range(num)
]
TuningJobState(hp_ranges=HyperparameterRanges_Impl(),
candidate_evaluations=_candidate_evaluations(6),
failed_candidates=[],
pending_evaluations=[]),
'4':
TuningJobState(hp_ranges=HyperparameterRanges_Impl(),
candidate_evaluations=_candidate_evaluations(120),
failed_candidates=[],
pending_evaluations=[]),
}
@pytest.mark.parametrize(
'candidate_evaluations,failed_candidates,pending_evaluations,expected',
[([], [], [], set()),
([CandidateEvaluation((123, 'a'), {'': 9.87})], [], [], {(123, 'a')}),
([], [(123, 'a')], [], {(123, 'a')}),
([], [], [PendingEvaluation((123, 'a'))], {(123, 'a')}),
([CandidateEvaluation(
(1, 'a'), {'': 9.87})], [(2, 'b')], [PendingEvaluation(
(3, 'c'))], {(1, 'a'), (2, 'b'), (3, 'c')})])
def test_compute_blacklisted_candidates(
hp_ranges: HyperparameterRanges_Impl,
candidate_evaluations: List[CandidateEvaluation],
failed_candidates: List[Candidate],
pending_evaluations: List[PendingEvaluation],
expected: Set[Candidate]):
state = TuningJobState(hp_ranges, candidate_evaluations, failed_candidates,
pending_evaluations)
actual = compute_blacklisted_candidates(state)
assert set(expected) == set(actual)
def test_gp_fantasizing():
"""
Compare whether acquisition function evaluations (values, gradients) with
fantasizing are the same as averaging them by hand.
"""
random_seed = 4567
_set_seeds(random_seed)
num_fantasy_samples = 10
num_pending = 5
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()))
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
num_data = len(X)
Y = [
dictionarize_objective(np.random.randn(1, 1)) for _ in range(num_data)
]
# Draw fantasies. This is done for a number of fixed pending candidates
# The model parameters are fit in the first iteration, when there are
# no pending candidates
# Note: It is important to not normalize targets, because this would be
# done on the observed targets only, not the fantasized ones, so it
# would be hard to compare below.
pending_evaluations = []
for _ in range(num_pending):
pending_cand = tuple(np.random.rand(2, ))
pending_evaluations.append(PendingEvaluation(pending_cand))
state = TuningJobState(hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X, Y)],
failed_candidates=[],
pending_evaluations=pending_evaluations)
gpmodel = default_gpmodel(state,
random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
model = GPMXNetModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
fantasy_samples = model.fantasy_samples
# Evaluate acquisition function and gradients with fantasizing
num_test = 50
X_test = np.vstack([
hp_ranges.to_ndarray(tuple(np.random.rand(2, )))
for _ in range(num_test)
])
acq_func = EIAcquisitionFunction(model)
fvals, grads = acq_func.compute_acq_with_gradients(X_test)
# Do the same computation by averaging by hand
fvals_cmp = np.empty((num_fantasy_samples, ) + fvals.shape)
grads_cmp = np.empty((num_fantasy_samples, ) + grads.shape)
X_full = X + state.pending_candidates
for it in range(num_fantasy_samples):
Y_full = Y + [
dictionarize_objective(eval.fantasies[DEFAULT_METRIC][:, it])
for eval in fantasy_samples
]
state2 = TuningJobState(
hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X_full, Y_full)],
failed_candidates=[],
pending_evaluations=[])
# We have to skip parameter optimization here
model2 = GPMXNetModel(state2,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=False,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
acq_func2 = EIAcquisitionFunction(model2)
fvals_, grads_ = acq_func2.compute_acq_with_gradients(X_test)
fvals_cmp[it, :] = fvals_
grads_cmp[it, :] = grads_
# Comparison
fvals2 = np.mean(fvals_cmp, axis=0)
grads2 = np.mean(grads_cmp, axis=0)
assert np.allclose(fvals, fvals2)
assert np.allclose(grads, grads2)