def test_hyperband_api(): import random params = Space.from_dict({'a': 'uniform(0, 1)'}) hpo = Hyperband(Fidelity(0, 100, 10, 'epochs'), params) assert not hpo.is_done() for rung in range(3): params_set = hpo.suggest() for i, params in enumerate(params_set): print(i, params) try: hpo.suggest() raise RuntimeError() except WaitingForTrials: pass except OptimizationIsDone: pass for i, params in enumerate(params_set): v = random.uniform(0, 1) if i == len(params_set) - 1: v = 1e-10 hpo.observe(params, v) print('-------') assert hpo.is_done() print(hpo.result()) print(hpo.info())
def __init__(self, fidelity: Fidelity, space: Union[Space, Dict], seed=0, **kwargs): self.identity = 'uid' for k, v in kwargs.items(): debug(f'used parameter ({k}: {v})') if isinstance(space, dict): space = Space.from_dict(space) if space is not None: space.identity(self.identity) if isinstance(fidelity, dict): fidelity = Fidelity.from_dict(fidelity) self.fidelity = fidelity self.space = space self.seed = seed self.seed_time = 0 self.manual_insert = 0 self.manual_samples = [] self.manual_fidelity = [] self.trials = OrderedDict()
def test_build_bounds(): space = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'uniform(1, 15)' }) lower, upper = build_bounds(space.instantiate('Orion')) assert lower.tolist() == [-5, 1] assert upper.tolist() == [10, 15] space = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'loguniform(1, 15)' }) lower, upper = build_bounds(space.instantiate('Orion')) assert lower.tolist() == [-5, numpy.log(1)] assert upper.tolist() == [10, numpy.log(15)]
def test_serialization_is_same(backend): import copy space = make_big_space(backend) serialized = space.serialize() new_space = Space.from_dict(copy.deepcopy(serialized)) new_serialized = new_space.serialize() assert serialized == new_serialized new_space.sample(**{'sub.epoch': 1, 'epoch': 2})
def test_serialization_sample_big(): import copy space = make_big_space() serialized = space.serialize() print(json.dumps(serialized, indent=2)) new_space = Space.from_dict(copy.deepcopy(serialized)) new_serialized = new_space.serialize() assert serialized == new_serialized print(json.dumps(new_space.sample(2, **{'sub.epoch': 1, 'epoch': 2}), indent=2))
def test_sample_in_log_space(robo): params = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'loguniform(12, 15)' }) robo = RoBO(FIDELITY, params, count=15, n_init=10) while not robo.is_done(): samples = robo.suggest() for sample in samples: robo.observe(sample['uid'], branin(**sample)) assert 12 <= sample['y'] <= 15
def load_state_dict(self, state): state = decompress_dict(state) self.space = Space.from_dict(state['space']) self.seed = state['seed'] self.manual_samples = state['manual_samples'] self.manual_fidelity = state['manual_fidelity'] self.manual_insert = state['manual_insert'] self.seed_time = state['seed_time'] self.fidelity = Fidelity.from_dict(state['fidelity']) self.trials = OrderedDict( (k, Trial.from_dict(t)) for k, t in state['trials']) return self
def test_hyperband_simple_sequential(): import random params = Space.from_dict({'a': 'uniform(0, 1)'}) hpo = Hyperband(Fidelity(0, 100, 10, 'epochs'), params) for params in hpo: hpo.observe(params, result=random.uniform(0, 1)) assert hpo.is_done() print(hpo.result()) print(hpo.info())
def test_convert_samples_to_x(): space = { 'a': 'uniform(lower=-1, upper=1)', 'b': 'uniform(lower=-1, upper=1)', 'c': 'loguniform(lower=1, upper=10)' } samples = Space.from_dict(space).sample(10, seed=1) X = convert_samples_to_x(samples, space) assert X.shape == (10, 3) assert X[0, 0] == samples[0]['a'] assert X[0, 1] == samples[0]['b'] assert X[0, 2] == numpy.log(samples[0]['c'])
def simulate_hpo(model, space, budget, seed): # NOTE: We hardcode for random search for now. # Sample points samples = Space.from_dict(space).sample(budget, seed=seed) X = convert_samples_to_x(samples, space) # Score points mean, std = model.predict(X) scores = mean # Select best min_index = numpy.argmin(scores) best_point = X[min_index] return convert_x_to_samples(X[min_index].reshape(1, -1), space)[0]
def test_X_with_log_dims(robo): params = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'loguniform(1, 15)' }) robo = RoBO(FIDELITY, params, count=5, n_init=2) while not robo.is_done(): samples = robo.suggest() for sample in samples: robo.observe(sample['uid'], branin(**sample)) assert robo.X.shape == (5, 2) assert robo.y.shape == (5,) assert robo.X.min(0)[0] >= -5 assert robo.X.min(0)[1] >= numpy.log(1) assert robo.X.max(0)[0] <= 10 assert robo.X.max(0)[1] <= numpy.log(15)
def fit_model(X, y, space, seed=1): n_samples = 50 # kernel = GPy.kern.Matern52(all_x.shape[1], variance=0.01, ARD=True) # model = GPModel_MCMC(n_samples=n_samples, kernel=kernel) # with all_logging_disabled(logging.ERROR): # model.updateModel(all_x, all_y, None, None) orion_space = Space.from_dict(space).instantiate('Orion') lower, upper = build_bounds(orion_space) # TODO set the seeds model = build_model(lower, upper, model_type='gp_mcmc', model_seed=1, prior_seed=1) model.train(X, y) return model
def build_data(budget, variables, defaults, space): epochs = 5 # defaults = {'a': 0, 'b': 1, 'c': 2, 'd': 3} # params = {'c': 2, 'd': 3, 'epoch': epochs} n_vars = len(variables) objectives = numpy.arange(budget * (epochs + 1)) numpy.random.RandomState(0).shuffle(objectives) objectives = objectives.reshape((epochs + 1, budget, 1)) params = Space.from_dict(space).sample(budget, seed=1) trials = OrderedDict() for trial_params in params: config = copy.deepcopy( dict(list(variables.items()) + list(defaults.items()))) config.update(trial_params) config['uid'] = compute_identity(config, IDENTITY_SIZE) # NOTE: We don't need objectives trials[config['uid']] = Trial(config) metrics = dict() for trial_i, trial_uid in enumerate(trials.keys()): metrics[trial_uid] = [{ 'epoch': i, 'obj': objectives[i, trial_i, 0] } for i in range(epochs + 1)] data = [] param_names = list(sorted(space.keys())) return create_valid_curves_xarray(trials, metrics, sorted(variables.keys()), epochs, param_names, seed=1)
def test_deserialize_orion(): cs = Space.from_dict(orion_space) print(cs) print(cs.sample(fid=1)) print(json.dumps(cs.serialize(), indent=2))