def test_optimizer_add_observations_calls_space_add_observations(self): optimizer = BOOptimizer(hptuning_config=self.manager1.hptuning_config) with patch.object(SearchSpace, 'add_observations') as add_observations_mock: optimizer.add_observations(configs=[], metrics=[]) assert add_observations_mock.call_count == 1
def get_suggestions(self, iteration_config=None): if not iteration_config: return get_random_suggestions(matrix=self.hptuning_config.matrix, n_suggestions=self.n_initial_trials, seed=self.hptuning_config.seed) # Use the iteration_config to construct observed point and metrics experiments_configs = dict( iteration_config.combined_experiments_configs) experiments_metrics = dict( iteration_config.combined_experiments_metrics) configs = [] metrics = [] for key in experiments_metrics.keys(): configs.append(experiments_configs[key]) metrics.append(experiments_metrics[key]) optimizer = BOOptimizer(hptuning_config=self.hptuning_config) optimizer.add_observations(configs=configs, metrics=metrics) suggestion = optimizer.get_suggestion() return [suggestion] if suggestion else None
def test_concrete_example(self): hptuning_config = HPTuningConfig.from_dict({ 'concurrency': 2, 'bo': { 'n_iterations': 5, 'n_initial_trials': 10, 'metric': { 'name': 'loss', 'optimization': 'minimize' }, 'utility_function': { 'acquisition_function': 'ucb', 'kappa': 2.576, 'gaussian_process': { 'kernel': 'matern', 'length_scale': 1.0, 'nu': 1.9, 'n_restarts_optimizer': 0 }, 'n_warmup': 1, 'n_iter': 1 } }, 'matrix': { 'learning_rate': {'uniform': [0.001, 0.01]}, 'dropout': {'values': [0.25, 0.3]}, 'activation': {'values': ['relu', 'sigmoid']} } }) optimizer = BOOptimizer(hptuning_config=hptuning_config) configs = [ { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.004544653508229265, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.005615296199690899, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.008784330869587902, "activation": "sigmoid", "dropout": 0.25 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.0058591075447430065, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.007464080062927171, "activation": "sigmoid", "dropout": 0.25 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.0024763129571936738, "activation": "relu", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.0074881581817925705, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.003360405779075163, "activation": "relu", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.009916904455792564, "activation": "sigmoid", "dropout": 0.25 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.000881723263162717, "activation": "sigmoid", "dropout": 0.3 } ] metrics = [ 2.3018131256103516, 2.302884340286255, 2.3071441650390625, 2.3034636974334717, 2.301487922668457, 0.05087224021553993, 2.3032383918762207, 0.06383182853460312, 2.3120572566986084, 0.7617478370666504 ] optimizer.add_observations(configs=configs, metrics=metrics) suggestion = optimizer.get_suggestion() assert 0.001 <= suggestion['learning_rate'] <= 0.01 assert suggestion['dropout'] in [0.25, 0.3] assert suggestion['activation'] in ['relu', 'sigmoid']
def test_optimizer_get_suggestion(self): # Manager 1 optimizer1 = BOOptimizer(hptuning_config=self.manager1.hptuning_config) optimizer1.N_ITER = 1 optimizer1.N_WARMUP = 1 configs = [ {'feature1': 1, 'feature2': 1, 'feature3': 1}, {'feature1': 1, 'feature2': 1.2, 'feature3': 1}, {'feature1': 1, 'feature2': 1, 'feature3': 1}, {'feature1': 1, 'feature2': 1.11, 'feature3': 1}, {'feature1': 1, 'feature2': 1.1, 'feature3': 1}, {'feature1': 1, 'feature2': 1.21, 'feature3': 1}, {'feature1': 2, 'feature2': 2, 'feature3': 2}, {'feature1': 3, 'feature2': 2, 'feature3': 2}, {'feature1': 2, 'feature2': 1.8, 'feature3': 3}, {'feature1': 3, 'feature2': 2, 'feature3': 3}, {'feature1': 2, 'feature2': 2, 'feature3': 2}, {'feature1': 3, 'feature2': 2, 'feature3': 2}, {'feature1': 2, 'feature2': 1.8, 'feature3': 3}, {'feature1': 3, 'feature2': 2, 'feature3': 3} ] metrics = [0, 1.1, 0.1, 0.1, 1.09, 0.4, 100, 200, 200, 300, 110, 210, 210, 310] optimizer1.add_observations(configs=configs, metrics=metrics) suggestion = optimizer1.get_suggestion() assert 1 <= suggestion['feature1'] <= 3 assert 1 <= suggestion['feature2'] <= 2 assert 1 <= suggestion['feature3'] <= 5 # Manager 2 optimizer2 = BOOptimizer(hptuning_config=self.manager2.hptuning_config) optimizer2.N_ITER = 1 optimizer2.N_WARMUP = 1 configs = [ {'feature1': 1, 'feature2': 1, 'feature3': 1, 'feature4': 1, 'feature5': 'a'}, {'feature1': 2, 'feature2': 1.2, 'feature3': 2, 'feature4': 4, 'feature5': 'b'}, {'feature1': 3, 'feature2': 1.3, 'feature3': 3, 'feature4': 3, 'feature5': 'a'} ] metrics = [1, 2, 3] optimizer2.add_observations(configs=configs, metrics=metrics) suggestion = optimizer2.get_suggestion() assert 1 <= suggestion['feature1'] <= 5 assert 1 <= suggestion['feature2'] <= 5 assert 1 <= suggestion['feature3'] <= 6 assert 1 <= suggestion['feature4'] <= 5 assert suggestion['feature5'] in ['a', 'b', 'c']
def test_concrete_example(self): hptuning_config = HPTuningConfig.from_dict({ 'concurrency': 2, 'bo': { 'n_iterations': 5, 'n_initial_trials': 10, 'metric': { 'name': 'loss', 'optimization': 'minimize' }, 'utility_function': { 'acquisition_function': 'ucb', 'kappa': 2.576, 'gaussian_process': { 'kernel': 'matern', 'length_scale': 1.0, 'nu': 1.9, 'n_restarts_optimizer': 0 }, 'n_warmup': 1, 'n_iter': 1 } }, 'matrix': { 'learning_rate': {'uniform': [0.001, 0.01]}, 'dropout': {'values': [0.25, 0.3]}, 'activation': {'values': ['relu', 'sigmoid']} } }) optimizer = BOOptimizer(hptuning_config=hptuning_config) configs = [ { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.004544653508229265, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.005615296199690899, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.008784330869587902, "activation": "sigmoid", "dropout": 0.25 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.0058591075447430065, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.007464080062927171, "activation": "sigmoid", "dropout": 0.25 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.0024763129571936738, "activation": "relu", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.0074881581817925705, "activation": "sigmoid", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.003360405779075163, "activation": "relu", "dropout": 0.3 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.009916904455792564, "activation": "sigmoid", "dropout": 0.25 }, { "num_epochs": 1, "num_steps": 300, "batch_size": 128, "learning_rate": 0.000881723263162717, "activation": "sigmoid", "dropout": 0.3 } ] metrics = [ 2.3018131256103516, 2.302884340286255, 2.3071441650390625, 2.3034636974334717, 2.301487922668457, 0.05087224021553993, 2.3032383918762207, 0.06383182853460312, 2.3120572566986084, 0.7617478370666504 ] optimizer.add_observations(configs=configs, metrics=metrics) suggestion = optimizer.get_suggestion() assert 0.001 <= suggestion['learning_rate'] <= 0.01 assert suggestion['dropout'] in [0.25, 0.3] assert suggestion['activation'] in ['relu', 'sigmoid']
def test_optimizer_get_suggestion(self): # Manager 1 optimizer1 = BOOptimizer(hptuning_config=self.manager1.hptuning_config) optimizer1.N_ITER = 1 optimizer1.N_WARMUP = 1 configs = [ {'feature1': 1, 'feature2': 1, 'feature3': 1}, {'feature1': 1, 'feature2': 1.2, 'feature3': 1}, {'feature1': 1, 'feature2': 1, 'feature3': 1}, {'feature1': 1, 'feature2': 1.11, 'feature3': 1}, {'feature1': 1, 'feature2': 1.1, 'feature3': 1}, {'feature1': 1, 'feature2': 1.21, 'feature3': 1}, {'feature1': 2, 'feature2': 2, 'feature3': 2}, {'feature1': 3, 'feature2': 2, 'feature3': 2}, {'feature1': 2, 'feature2': 1.8, 'feature3': 3}, {'feature1': 3, 'feature2': 2, 'feature3': 3}, {'feature1': 2, 'feature2': 2, 'feature3': 2}, {'feature1': 3, 'feature2': 2, 'feature3': 2}, {'feature1': 2, 'feature2': 1.8, 'feature3': 3}, {'feature1': 3, 'feature2': 2, 'feature3': 3} ] metrics = [0, 1.1, 0.1, 0.1, 1.09, 0.4, 100, 200, 200, 300, 110, 210, 210, 310] optimizer1.add_observations(configs=configs, metrics=metrics) suggestion = optimizer1.get_suggestion() assert 1 <= suggestion['feature1'] <= 3 assert 1 <= suggestion['feature2'] <= 2 assert 1 <= suggestion['feature3'] <= 5 # Manager 2 optimizer2 = BOOptimizer(hptuning_config=self.manager2.hptuning_config) optimizer2.N_ITER = 1 optimizer2.N_WARMUP = 1 configs = [ {'feature1': 1, 'feature2': 1, 'feature3': 1, 'feature4': 1, 'feature5': 'a'}, {'feature1': 2, 'feature2': 1.2, 'feature3': 2, 'feature4': 4, 'feature5': 'b'}, {'feature1': 3, 'feature2': 1.3, 'feature3': 3, 'feature4': 3, 'feature5': 'a'} ] metrics = [1, 2, 3] optimizer2.add_observations(configs=configs, metrics=metrics) suggestion = optimizer2.get_suggestion() assert 1 <= suggestion['feature1'] <= 5 assert 1 <= suggestion['feature2'] <= 5 assert 1 <= suggestion['feature3'] <= 6 assert 1 <= suggestion['feature4'] <= 5 assert suggestion['feature5'] in ['a', 'b', 'c']
def test_optimizer_add_observations_calls_space_add_observations(self): optimizer = BOOptimizer(hptuning_config=self.manager1.hptuning_config) with patch.object(SearchSpace, 'add_observations') as add_observations_mock: optimizer.add_observations(configs=[], metrics=[]) assert add_observations_mock.call_count == 1