def tuning_job_state(): return { 'algo-1': TuningJobState(hp_ranges=HyperparameterRanges_Impl( HyperparameterRangeContinuous('a1_hp_1', -5.0, 5.0, LinearScaling()), HyperparameterRangeCategorical('a1_hp_2', ('a', 'b', 'c'))), candidate_evaluations=[ CandidateEvaluation(candidate=(-3.0, 'a'), value=1.0), CandidateEvaluation(candidate=(-1.9, 'c'), value=2.0), CandidateEvaluation(candidate=(-3.5, 'a'), value=0.3) ], failed_candidates=[], pending_evaluations=[]), 'algo-2': TuningJobState(hp_ranges=HyperparameterRanges_Impl( HyperparameterRangeContinuous('a2_hp_1', -5.0, 5.0, LinearScaling()), HyperparameterRangeInteger('a2_hp_2', -5, 5, LinearScaling(), -5, 5)), candidate_evaluations=[ CandidateEvaluation(candidate=(-1.9, -1), value=0.0), CandidateEvaluation(candidate=(-3.5, 3), value=2.0) ], failed_candidates=[], pending_evaluations=[]) }
def tuning_job_state_mcmc(X, Y) -> TuningJobState: Y = [dictionarize_objective(y) for y in Y] return TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', -4., 4., LinearScaling())), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def data_to_state(data: dict) -> TuningJobState: cs = CS.ConfigurationSpace() cs_names = ['x{}'.format(i) for i in range(len(data['ss_limits']))] cs.add_hyperparameters([ CSH.UniformFloatHyperparameter( name=name, lower=lims['min'], upper=lims['max']) for name, lims in zip(cs_names, data['ss_limits'])]) _evaluations = [] x_mult = [] x_add = [] for lim in data['ss_limits']: mn, mx = lim['min'], lim['max'] x_mult.append(mx - mn) x_add.append(mn) x_mult = np.array(x_mult) x_add = np.array(x_add) for x, y in zip(data['train_inputs'], data['train_targets']): x_decoded = x * x_mult + x_add config_dct = dict(zip(cs_names, x_decoded)) config = CS.Configuration(cs, values=config_dct) _evaluations.append(CandidateEvaluation( config, dictionarize_objective(y))) return TuningJobState( hp_ranges=HyperparameterRanges_CS(cs), candidate_evaluations=_evaluations, failed_candidates=[], pending_evaluations=[])
def default_models() -> List[GPMXNetModel]: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), (0.0, 0.0 ), # same evals are added multiple times to force GP to unlearn prior (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] state = TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [], ) random_seed = 0 gpmodel = default_gpmodel(state, random_seed=random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) gpmodel_mcmc = default_gpmodel_mcmc(state, random_seed=random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) return [ GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20), GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel_mcmc, fit_parameters=True, num_fantasy_samples=20) ]
def tuning_job_state() -> TuningJobState: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] return TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def update(self, config: Candidate, reward: float): """ Registers new datapoint at config, with reward reward. Note that in general, config should previously have been registered as pending (register_pending). If so, it is switched from pending to labeled. If not, it is considered directly labeled. :param config: :param reward: """ crit_val = self.map_reward(reward) self.state_transformer.label_candidate(CandidateEvaluation( candidate=copy.deepcopy(config), metrics=dictionarize_objective(crit_val))) if self.debug_log is not None: config_id = self.debug_log.config_id(config) msg = "Update for config_id {}: reward = {}, crit_val = {}".format( config_id, reward, crit_val) logger.info(msg)
def decode_state(enc_state: dict, hp_ranges: HyperparameterRanges_CS) \ -> TuningJobState: assert isinstance(hp_ranges, HyperparameterRanges_CS), \ "Must have hp_ranges of HyperparameterRanges_CS type" config_space = hp_ranges.config_space def to_cs(x): return CS.Configuration(config_space, values=x) candidate_evaluations = [ CandidateEvaluation(to_cs(x['candidate']), x['metrics']) for x in enc_state['candidate_evaluations']] failed_candidates = [to_cs(x) for x in enc_state['failed_candidates']] pending_evaluations = [ PendingEvaluation(to_cs(x)) for x in enc_state['pending_evaluations']] return TuningJobState( hp_ranges=hp_ranges, candidate_evaluations=candidate_evaluations, failed_candidates=failed_candidates, pending_evaluations=pending_evaluations)
def default_models(do_mcmc=True) -> List[GPMXNetModel]: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] state = TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], []) random_seed = 0 gpmodel = default_gpmodel(state, random_seed=random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) result = [ GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20) ] if do_mcmc: gpmodel_mcmc = default_gpmodel_mcmc(state, random_seed=random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) result.append( GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel_mcmc, fit_parameters=True, num_fantasy_samples=20)) return result
def _candidate_evaluations(num): return [ CandidateEvaluation(candidate=(i, ), metrics=dictionarize_objective(float(i))) for i in range(num) ]
TuningJobState(hp_ranges=HyperparameterRanges_Impl(), candidate_evaluations=_candidate_evaluations(6), failed_candidates=[], pending_evaluations=[]), '4': TuningJobState(hp_ranges=HyperparameterRanges_Impl(), candidate_evaluations=_candidate_evaluations(120), failed_candidates=[], pending_evaluations=[]), } @pytest.mark.parametrize( 'candidate_evaluations,failed_candidates,pending_evaluations,expected', [([], [], [], set()), ([CandidateEvaluation((123, 'a'), {'': 9.87})], [], [], {(123, 'a')}), ([], [(123, 'a')], [], {(123, 'a')}), ([], [], [PendingEvaluation((123, 'a'))], {(123, 'a')}), ([CandidateEvaluation( (1, 'a'), {'': 9.87})], [(2, 'b')], [PendingEvaluation( (3, 'c'))], {(1, 'a'), (2, 'b'), (3, 'c')})]) def test_compute_blacklisted_candidates( hp_ranges: HyperparameterRanges_Impl, candidate_evaluations: List[CandidateEvaluation], failed_candidates: List[Candidate], pending_evaluations: List[PendingEvaluation], expected: Set[Candidate]): state = TuningJobState(hp_ranges, candidate_evaluations, failed_candidates, pending_evaluations) actual = compute_blacklisted_candidates(state) assert set(expected) == set(actual)
def test_gp_fantasizing(): """ Compare whether acquisition function evaluations (values, gradients) with fantasizing are the same as averaging them by hand. """ random_seed = 4567 _set_seeds(random_seed) num_fantasy_samples = 10 num_pending = 5 hp_ranges = HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling())) X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] num_data = len(X) Y = [ dictionarize_objective(np.random.randn(1, 1)) for _ in range(num_data) ] # Draw fantasies. This is done for a number of fixed pending candidates # The model parameters are fit in the first iteration, when there are # no pending candidates # Note: It is important to not normalize targets, because this would be # done on the observed targets only, not the fantasized ones, so it # would be hard to compare below. pending_evaluations = [] for _ in range(num_pending): pending_cand = tuple(np.random.rand(2, )) pending_evaluations.append(PendingEvaluation(pending_cand)) state = TuningJobState(hp_ranges, [CandidateEvaluation(x, y) for x, y in zip(X, Y)], failed_candidates=[], pending_evaluations=pending_evaluations) gpmodel = default_gpmodel(state, random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) model = GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=num_fantasy_samples, normalize_targets=False) fantasy_samples = model.fantasy_samples # Evaluate acquisition function and gradients with fantasizing num_test = 50 X_test = np.vstack([ hp_ranges.to_ndarray(tuple(np.random.rand(2, ))) for _ in range(num_test) ]) acq_func = EIAcquisitionFunction(model) fvals, grads = acq_func.compute_acq_with_gradients(X_test) # Do the same computation by averaging by hand fvals_cmp = np.empty((num_fantasy_samples, ) + fvals.shape) grads_cmp = np.empty((num_fantasy_samples, ) + grads.shape) X_full = X + state.pending_candidates for it in range(num_fantasy_samples): Y_full = Y + [ dictionarize_objective(eval.fantasies[DEFAULT_METRIC][:, it]) for eval in fantasy_samples ] state2 = TuningJobState( hp_ranges, [CandidateEvaluation(x, y) for x, y in zip(X_full, Y_full)], failed_candidates=[], pending_evaluations=[]) # We have to skip parameter optimization here model2 = GPMXNetModel(state2, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=False, num_fantasy_samples=num_fantasy_samples, normalize_targets=False) acq_func2 = EIAcquisitionFunction(model2) fvals_, grads_ = acq_func2.compute_acq_with_gradients(X_test) fvals_cmp[it, :] = fvals_ grads_cmp[it, :] = grads_ # Comparison fvals2 = np.mean(fvals_cmp, axis=0) grads2 = np.mean(grads_cmp, axis=0) assert np.allclose(fvals, fvals2) assert np.allclose(grads, grads2)