def setUp(self):
study_configuration_json = {
"goal":
"MAXIMIZE",
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [{
"parameterName": "hidden1",
"type": "INTEGER",
"minValue": 40,
"maxValue": 400,
"scallingType": "LINEAR"
}]
}
study_configuration = json.dumps(study_configuration_json)
self.study = Study.create("RandomSearchStudy", study_configuration)
trial1 = Trial.create(self.study.id, "RandomSearchTrial1")
trial2 = Trial.create(self.study.id, "RandomSearchTrial2")
self.trials = [trial1, trial2]
TrialMetric.create(trial1.id, 10, 0.5)
TrialMetric.create(trial1.id, 20, 0.6)
TrialMetric.create(trial2.id, 10, 0.6)
TrialMetric.create(trial2.id, 20, 0.5)
def setUp(self):
study_configuration_json = {
"goal":
"MAXIMIZE",
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [{
"parameterName": "hidden1",
"type": "INTEGER",
"minValue": 40,
"maxValue": 400,
"scallingType": "LINEAR"
}]
}
study_configuration = json.dumps(study_configuration_json)
self.study = Study.create("RandomSearchStudy", study_configuration)
trial1 = Trial.create(self.study.id, "RandomSearchTrial1")
trial2 = Trial.create(self.study.id, "RandomSearchTrial2")
self.trials = [trial1, trial2]
TrialMetric.create(trial1.id, 10, 0.5)
TrialMetric.create(trial1.id, 20, 0.6)
TrialMetric.create(trial2.id, 10, 0.6)
TrialMetric.create(trial2.id, 20, 0.5)
def get_new_suggestions(self, study_id, trials=[], number=1):
"""
Get the new suggested trials with grid search.
"""
study = Study.objects.get(id=study_id)
result = []
for i in range(number):
trial = Trial.create(study.id, "RandomSearchTrial")
parameter_values_json = {}
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
min_value = param["minValue"]
max_value = param["maxValue"]
if number > 1:
value_step = (max_value - min_value) / (number - 1)
else:
value_step = max_value - min_value
parameter_value = min_value + value_step * i
parameter_values_json[param["parameterName"]] = parameter_value
trial.parameter_values = json.dumps(parameter_values_json)
trial.save()
result.append(trial)
return result
def get_new_suggestions(self, study_id, trials, number=1):
"""
Get the new suggested trials with grid search.
"""
study = Study.objects.get(id=study_id)
result = []
for i in range(number):
trial = Trial.create(study.id, "GridSearchTrial")
parameter_values_json = {}
# TODO: Support different type of parameters
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
min_value = param["minValue"]
max_value = param["maxValue"]
if number > 1:
value_step = (max_value - min_value) / (number - 1)
else:
value_step = max_value - min_value
parameter_value = min_value + value_step * i
parameter_values_json[param["parameterName"]] = parameter_value
trial.parameter_values = json.dumps(parameter_values_json)
trial.save()
result.append(trial)
return result
def get_new_suggestions(self, study_id, trials, number=1):
"""
Get the new suggested trials with random search.
"""
study = Study.objects.get(id=study_id)
result = []
for i in range(number):
trial = Trial.create(study.id, "RandomSearchTrial")
parameter_values_json = {}
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
if param["type"] == "DOUBLE":
min_value = param["minValue"]
max_value = param["maxValue"]
selected_value = self.get_random_value(
min_value, max_value)
parameter_values_json[
param["parameterName"]] = selected_value
elif param["type"] == "INTEGER":
min_value = param["minValue"]
max_value = param["maxValue"]
random_value = self.get_random_value(min_value, max_value)
selected_value = int(round(random_value))
parameter_values_json[
param["parameterName"]] = selected_value
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
feasible_points.sort()
min_value = feasible_points[0]
max_value = feasible_points[-1]
random_value = self.get_random_value(min_value, max_value)
selected_value = self.find_closest_value_in_list(
feasible_points, random_value)
parameter_values_json[
param["parameterName"]] = selected_value
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
value.strip()
for value in feasible_points_string.split(",")
]
random_value = random.randint(0, len(feasible_points) - 1)
selected_value = feasible_points[random_value]
parameter_values_json[
param["parameterName"]] = selected_value
trial.parameter_values = json.dumps(parameter_values_json)
trial.save()
result.append(trial)
return result
def get_new_suggestions(self, study_id, trials, number=1):
"""
Get the new suggested trials with random search.
"""
study = Study.objects.get(id=study_id)
result = []
for i in range(number):
trial = Trial.create(study.id, "RandomSearchTrial")
parameter_values_json = {}
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
if param["type"] == "DOUBLE":
min_value = param["minValue"]
max_value = param["maxValue"]
selected_value = self.get_random_value(min_value, max_value)
parameter_values_json[param["parameterName"]] = selected_value
elif param["type"] == "INTEGER":
min_value = param["minValue"]
max_value = param["maxValue"]
random_value = self.get_random_value(min_value, max_value)
selected_value = int(round(random_value))
parameter_values_json[param["parameterName"]] = selected_value
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
feasible_points.sort()
min_value = feasible_points[0]
max_value = feasible_points[-1]
random_value = self.get_random_value(min_value, max_value)
selected_value = self.find_closest_value_in_list(
feasible_points, random_value)
parameter_values_json[param["parameterName"]] = selected_value
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
value.strip() for value in feasible_points_string.split(",")
]
random_value = random.randint(0, len(feasible_points) - 1)
selected_value = feasible_points[random_value]
parameter_values_json[param["parameterName"]] = selected_value
trial.parameter_values = json.dumps(parameter_values_json)
trial.save()
result.append(trial)
return result
def get_new_suggestions(self, study_id, trials=[], number=1):
"""
Get the new suggested trials with random search.
"""
return_trial_list = []
study = Study.objects.get(id=study_id)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for i in range(number):
trial = Trial.create(study.name, "RandomSearchTrial")
parameter_values_json = {}
for param in params:
if param["type"] == "DOUBLE":
suggest_value = AlgorithmUtil.get_random_value(
param["minValue"], param["maxValue"])
elif param["type"] == "INTEGER":
suggest_value = AlgorithmUtil.get_random_int_value(
param["minValue"], param["maxValue"])
elif param["type"] == "DISCRETE":
feasible_point_list = [
float(value.strip())
for value in param["feasiblePoints"].split(",")
]
suggest_value = AlgorithmUtil.get_random_item_from_list(
feasible_point_list)
elif param["type"] == "CATEGORICAL":
feasible_point_list = [
value.strip()
for value in param["feasiblePoints"].split(",")
]
suggest_value = AlgorithmUtil.get_random_item_from_list(
feasible_point_list)
parameter_values_json[param["parameterName"]] = suggest_value
trial.parameter_values = json.dumps(parameter_values_json)
trial.save()
return_trial_list.append(trial)
return return_trial_list
def v1_study_trials(request, study_id):
# Create the trial
if request.method == "POST":
data = json.loads(request.body)
name = data["name"]
trial = Trial.create(study_id, name)
return JsonResponse({"data": trial.to_json()})
# List the studies
elif request.method == "GET":
trials = Trial.objects.filter(study_id=study_id)
response_data = [trial.to_json() for trial in trials]
return JsonResponse({"data": response_data})
else:
return JsonResponse({"error": "Unsupported http method"})
def get_new_suggestions(self, study_id, trials, number=1):
"""
Get the new suggested trials with random search.
"""
study = Study.objects.get(id=study_id)
result = []
for i in range(number):
trial = Trial.create(study.id, "RandomSearchTrial")
parameter_values_json = {}
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
min_value = param["minValue"]
max_value = param["maxValue"]
random_value = self.get_random_value(min_value, max_value)
parameter_values_json[param["parameterName"]] = random_value
trial.parameter_values = json.dumps(parameter_values_json)
trial.save()
result.append(trial)
return result
def get_new_suggestions(self, study_name, input_trials=[], number=1):
"""
Get the new suggested trials with Chocolate algorithm.
"""
# 1. Construct search space
# Example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)}
chocolate_search_space = {}
study = Study.objects.get(name=study_name)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
param_name = param["parameterName"]
if param["type"] == "INTEGER":
# TODO: Support int type of search space)
pass
elif param["type"] == "DOUBLE":
chocolate_search_space[param_name] = choco.uniform(
param["minValue"], param["maxValue"])
elif param["type"] == "DISCRETE" or param["type"] == "CATEGORICAL":
feasible_point_list = [
value.strip() for value in param["feasiblePoints"].split(",")
]
chocolate_search_space[param_name] = choco.choice(feasible_point_list)
conn = choco.SQLiteConnection("sqlite:///my_db.db")
# Refer to https://chocolate.readthedocs.io/tutorials/algo.html
if self.algorithm_name == "Grid":
sampler = choco.Grid(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "Random":
sampler = choco.Random(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "QuasiRandom":
sampler = choco.QuasiRandom(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "Bayes":
sampler = choco.Bayes(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "CMAES":
sampler = choco.CMAES(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "MOCMAES":
mu = 1
sampler = choco.MOCMAES(
conn, chocolate_search_space, mu=mu, clear_db=True)
# 2. Update with completed advisor trials
completed_advisor_trials = Trial.objects.filter(
study_name=study_name, status="Completed")
for index, advisor_trial in enumerate(completed_advisor_trials):
parameter_values_json = json.loads(advisor_trial.parameter_values)
loss = advisor_trial.objective_value
if study_configuration_json["goal"] == "MAXIMIZE":
loss = -1 * loss
entry = {"_chocolate_id": index, "_loss": loss}
entry.update(parameter_values_json)
# Should not use sampler.update(token, loss)
conn.insert_result(entry)
# 3. Run algorithm and construct return advisor trials
return_trial_list = []
for i in range(number):
# Example: {'_chocolate_id': 1}
# Example: {u'hidden2': u'32', u'learning_rate': 0.07122424534644338, u'l1_normalization': 0.8402644688674471, u'optimizer': u'adam'}
token, chocolate_params = sampler.next()
parameter_values_json = {}
for param in params:
if param["type"] == "INTEGER" or param["type"] == "DOUBLE" or param["type"] == "CATEGORICAL":
parameter_values_json[param["parameterName"]] = chocolate_params[
param["parameterName"]]
elif param["type"] == "DISCRETE":
parameter_values_json[param["parameterName"]] = int(
chocolate_params[param["parameterName"]])
new_advisor_trial = Trial.create(study.name, "ChocolateTrial")
new_advisor_trial.parameter_values = json.dumps(parameter_values_json)
new_advisor_trial.save()
return_trial_list.append(new_advisor_trial)
return return_trial_list
def get_new_suggestions(self, study_name, input_trials=[], number=1):
"""
Get the new suggested trials with TPE algorithm.
"""
# Construct search space, example: {"x": hyperopt.hp.uniform('x', -10, 10), "x2": hyperopt.hp.uniform('x2', -10, 10)}
hyperopt_search_space = {}
study = Study.objects.get(name=study_name)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
param_name = param["parameterName"]
if param["type"] == "INTEGER":
# TODO: Support int type of search space)
pass
elif param["type"] == "DOUBLE":
hyperopt_search_space[param_name] = hyperopt.hp.uniform(
param_name, param["minValue"], param["maxValue"])
elif param["type"] == "DISCRETE" or param["type"] == "CATEGORICAL":
feasible_point_list = [
value.strip() for value in param["feasiblePoints"].split(",")
]
hyperopt_search_space[param_name] = hyperopt.hp.choice(
param_name, feasible_point_list)
# New hyperopt variables
hyperopt_rstate = np.random.RandomState()
hyperopt_domain = hyperopt.Domain(
None, hyperopt_search_space, pass_expr_memo_ctrl=None)
hyperopt_trial_specs = []
hyperopt_trial_results = []
# Example: # Example: [{'tid': 0, 'idxs': {'l1_normalization': [0], 'learning_rate': [0], 'hidden2': [0], 'optimizer': [0]}, 'cmd': ('domain_attachment', 'FMinIter_Domain'), 'vals': {'l1_normalization': [0.1], 'learning_rate': [0.1], 'hidden2': [1], 'optimizer': [1]}, 'workdir': None}]
hyperopt_trial_miscs = []
hyperopt_trial_new_ids = []
# Update hyperopt for trained trials with completed advisor trials
completed_hyperopt_trials = hyperopt.Trials()
completed_advisor_trials = Trial.objects.filter(
study_name=study_name, status="Completed")
for index, advisor_trial in enumerate(completed_advisor_trials):
# Example: {"learning_rate": 0.01, "optimizer": "ftrl"}
parameter_values_json = json.loads(advisor_trial.parameter_values)
# Example: {'l1_normalization': [0], 'learning_rate': [0], 'hidden2': [0], 'optimizer': [0]}
hyperopt_trial_miscs_idxs = {}
# Example: {'l1_normalization': [0.1], 'learning_rate': [0.1], 'hidden2': [1], 'optimizer': [1]}
hyperopt_trial_miscs_vals = {}
new_id = index
hyperopt_trial_new_ids.append(new_id)
hyperopt_trial_misc = dict(
tid=new_id, cmd=hyperopt_domain.cmd, workdir=hyperopt_domain.workdir)
for param in params:
if param["type"] == "INTEGER":
pass
elif param["type"] == "DOUBLE":
parameter_value = parameter_values_json[param["parameterName"]]
hyperopt_trial_miscs_idxs[param["parameterName"]] = [index]
hyperopt_trial_miscs_vals[param["parameterName"]] = [parameter_value]
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
parameter_value = parameter_values_json[param["parameterName"]]
index_of_value_in_list = feasible_points.index(parameter_value)
hyperopt_trial_miscs_idxs[param["parameterName"]] = [index]
hyperopt_trial_miscs_vals[param["parameterName"]] = [
index_of_value_in_list
]
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
value.strip() for value in feasible_points_string.split(",")
]
# Example: "ftrl"
parameter_value = parameter_values_json[param["parameterName"]]
index_of_value_in_list = feasible_points.index(parameter_value)
hyperopt_trial_miscs_idxs[param["parameterName"]] = [index]
hyperopt_trial_miscs_vals[param["parameterName"]] = [
index_of_value_in_list
]
hyperopt_trial_specs.append(None)
hyperopt_trial_misc["idxs"] = hyperopt_trial_miscs_idxs
hyperopt_trial_misc["vals"] = hyperopt_trial_miscs_vals
hyperopt_trial_miscs.append(hyperopt_trial_misc)
# TODO: Use negative objective value for loss or not
loss_for_hyperopt = advisor_trial.objective_value
if study_configuration_json["goal"] == "MAXIMIZE":
# Now hyperopt only supports fmin and we need to reverse objective value for maximization
loss_for_hyperopt = -1 * advisor_trial.objective_value
hyperopt_trial_result = {
"loss": loss_for_hyperopt,
"status": hyperopt.STATUS_OK
}
hyperopt_trial_results.append(hyperopt_trial_result)
if len(completed_advisor_trials) > 0:
# Example: {'refresh_time': datetime.datetime(2018, 9, 18, 12, 6, 41, 922000), 'book_time': datetime.datetime(2018, 9, 18, 12, 6, 41, 922000), 'misc': {'tid': 0, 'idxs': {'x2': [0], 'x': [0]}, 'cmd': ('domain_attachment', 'FMinIter_Domain'), 'vals': {'x2': [-8.137088361136204], 'x': [-4.849028446711832]}, 'workdir': None}, 'state': 2, 'tid': 0, 'exp_key': None, 'version': 0, 'result': {'status': 'ok', 'loss': 14.849028446711833}, 'owner': None, 'spec': None}
hyperopt_trials = completed_hyperopt_trials.new_trial_docs(
hyperopt_trial_new_ids, hyperopt_trial_specs, hyperopt_trial_results,
hyperopt_trial_miscs)
for current_hyperopt_trials in hyperopt_trials:
current_hyperopt_trials["state"] = hyperopt.JOB_STATE_DONE
completed_hyperopt_trials.insert_trial_docs(hyperopt_trials)
completed_hyperopt_trials.refresh()
rval = hyperopt.FMinIter(
self.hyperopt_algorithm,
hyperopt_domain,
completed_hyperopt_trials,
max_evals=-1,
rstate=hyperopt_rstate,
verbose=0)
rval.catch_eval_exceptions = False
new_ids = rval.trials.new_trial_ids(number)
rval.trials.refresh()
random_state = rval.rstate.randint(2**31 - 1)
new_trials = self.hyperopt_algorithm(
new_ids, rval.domain, completed_hyperopt_trials, random_state)
rval.trials.refresh()
# Construct return advisor trials from new hyperopt trials
return_trial_list = []
for i in range(number):
# Example: {u'hidden2': [2], u'learning_rate': [0.04633366105812467], u'l1_normalization': [0.16858448611765364], u'optimizer': [3]}
vals = new_trials[0]['misc']['vals']
new_advisor_trial = Trial.create(study.name, "TpeTrial")
parameter_values_json = {}
for param in params:
if param["type"] == "INTEGER":
pass
elif param["type"] == "DOUBLE":
suggest_value = vals[param["parameterName"]][0]
parameter_values_json[param["parameterName"]] = suggest_value
elif param["type"] == "DISCRETE":
feasible_point_list = [
float(value.strip())
for value in param["feasiblePoints"].split(",")
]
suggest_index = vals[param["parameterName"]][0]
suggest_value = feasible_point_list[suggest_index]
elif param["type"] == "CATEGORICAL":
feasible_point_list = [
value.strip() for value in param["feasiblePoints"].split(",")
]
suggest_index = vals[param["parameterName"]][0]
suggest_value = feasible_point_list[suggest_index]
parameter_values_json[param["parameterName"]] = suggest_value
new_advisor_trial.parameter_values = json.dumps(parameter_values_json)
return_trial_list.append(new_advisor_trial)
return return_trial_list
def get_new_suggestions(self, study_id, trials=[], number=1):
"""
Get the new suggested trials with random search.
"""
search_space = hyperopt.hp.uniform('x', -10, 10)
search_space_instance = search_space
rstate = np.random.RandomState()
trials = hyperopt.Trials()
domain = hyperopt.Domain(None,
search_space_instance,
pass_expr_memo_ctrl=None)
algorithm = hyperopt.tpe.suggest
rval = hyperopt.FMinIter(algorithm,
domain,
trials,
max_evals=-1,
rstate=rstate,
verbose=0)
rval.catch_eval_exceptions = False
algorithm = rval.algo
new_ids = rval.trials.new_trial_ids(1)
rval.trials.refresh()
random_state = rval.rstate.randint(2**31 - 1)
new_trials = algorithm(new_ids, rval.domain, trials, random_state)
rval.trials.refresh()
# Example: {'x': [8.721658602103911]}
vals = new_trials[0]['misc']['vals']
#import ipdb;ipdb.set_trace()
"""
parameter = dict()
for key in vals:
try:
parameter[key] = vals[key][0].item()
except Exception:
parameter[key] = None
"""
"""
trials =rval.trials
trial = trials.new_trial_docs([new_id], rval_specs, rval_results, rval_miscs)[0]
trial['result'] = {'loss': reward, 'status': 'ok'}
trial['state'] = hp.JOB_STATE_DONE
trials.insert_trial_docs([trial])
trials.refresh()
"""
"""
def _choose_tuner(self, algorithm_name):
if algorithm_name == 'tpe':
return hp.tpe.suggest
if algorithm_name == 'random_search':
return hp.rand.suggest
if algorithm_name == 'anneal':
return hp.anneal.suggest
raise RuntimeError('Not support tuner algorithm in hyperopt.')
"""
return_trial_list = []
study = Study.objects.get(id=study_id)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for i in range(number):
trial = Trial.create(study.id, "TpeTrial")
parameter_values_json = {}
for param in params:
if param["type"] == "INTEGER" or param[
"type"] == "DISCRETE" or param["type"] == "CATEGORICAL":
pass
elif param["type"] == "DOUBLE":
# TODO: Get the specified value from hyperopt
suggest_value = vals["x"][0]
parameter_values_json[
param["parameterName"]] = suggest_value
parameter_values_json[param["parameterName"]] = suggest_value
trial.parameter_values = json.dumps(parameter_values_json)
trial.save()
return_trial_list.append(trial)
return return_trial_list
def get_new_suggestions(self, study_name, input_trials=[], number=1):
"""
Get the new suggested trials with skopt algorithm.
"""
# Construct search space, example: {"x": hyperopt.hp.uniform('x', -10, 10), "x2": hyperopt.hp.uniform('x2', -10, 10)}
hyperopt_search_space = {}
study = Study.objects.get(name=study_name)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
skopt_search_space = []
for param in params:
param_name = param["parameterName"]
if param["type"] == "INTEGER":
skopt_search_space.append(
skopt.space.Integer(
param["minValue"], param["maxValue"], name="min_samples_leaf"
)
)
elif param["type"] == "DOUBLE":
skopt_search_space.append(
skopt.space.Real(
param["minValue"],
param["maxValue"],
"log-uniform",
name="learning_rate",
)
)
elif param["type"] == "DISCRETE" or param["type"] == "CATEGORICAL":
pass
if self.algorithm_name == "bayesian_optimization":
skopt_optimizer = skopt.Optimizer([(-2.0, 2.0)])
else:
print("Unsupport skopt algorithm: {}".format(self.algorithm_name))
completed_advisor_trials = Trial.objects.filter(
study_name=study_name, status="Completed"
)
for index, advisor_trial in enumerate(completed_advisor_trials):
# Example: {"learning_rate": 0.01, "optimizer": "ftrl"}
parameter_values_json = json.loads(advisor_trial.parameter_values)
# Example: [(-2.0, 2.0)]
skopt_suggested = []
for param in params:
if param["type"] == "INTEGER" or param["type"] == "DOUBLE":
parameter_value = parameter_values_json[param["parameterName"]]
skopt_suggested.append(parameter_value)
elif param["type"] == "DISCRETE":
pass
elif param["type"] == "CATEGORICAL":
pass
loss_for_skopt = advisor_trial.objective_value
if study_configuration_json["goal"] == "MAXIMIZE":
# Now hyperopt only supports fmin and we need to reverse objective value for maximization
loss_for_skopt = -1 * advisor_trial.objective_value
skopt_optimizer.tell(skopt_suggested, loss_for_skopt)
return_trial_list = []
for i in range(number):
skopt_suggested = skopt_optimizer.ask()
new_advisor_trial = Trial.create(study.name, "SkoptTrial")
parameter_values_json = {}
index = 0
for param in params:
if param["type"] == "INTEGER" or param["type"] == "DOUBLE":
parameter_values_json[param["parameterName"]] = skopt_suggested[
index
]
elif param["type"] == "DISCRETE":
pass
elif param["type"] == "CATEGORICAL":
pass
index += 1
new_advisor_trial.parameter_values = json.dumps(parameter_values_json)
new_advisor_trial.save()
return_trial_list.append(new_advisor_trial)
return return_trial_list
def get_new_suggestions(self, study_id, trials, number=1):
# TODO: Only support retuning one trial
study = Study.objects.get(id=study_id)
completed_trials = Trial.objects.filter(study_id=study_id,
status="Completed")
study_configuration_json = json.loads(study.study_configuration)
random_init_trials = study_configuration_json.get(
"randomInitTrials", 3)
params = study_configuration_json["params"]
# Use random search if it has less dataset
if len(completed_trials) < random_init_trials:
randomSearchAlgorithm = RandomSearchAlgorithm()
return_trials = randomSearchAlgorithm.get_new_suggestions(
study_id, trials)
return return_trials
else:
return_trial = Trial.create(study.id, "BayesianOptimizationTrial")
acquisition_fucntion_kappa = 5
# Example: {'x': (-4, 4), 'y': (-3, 3)}
bound_dict = {}
for param in params:
if param["type"] == "DOUBLE" or param["type"] == "INTEGER":
min_value = param["minValue"]
max_value = param["maxValue"]
bound_dict[param["parameterName"]] = (min_value, max_value)
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
feasible_points.sort()
min_value = feasible_points[0]
max_value = feasible_points[-1]
bound_dict[param["parameterName"]] = (min_value, max_value)
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
value.strip()
for value in feasible_points_string.split(",")
]
for feasible_point in feasible_points:
parameter_name = "{}_{}".format(
param["parameterName"], feasible_point)
bound_dict[parameter_name] = (0, 1)
bounds = []
for key in bound_dict.keys():
bounds.append(bound_dict[key])
bounds = np.asarray(bounds)
gp = GaussianProcessRegressor(
kernel=Matern(nu=2.5),
n_restarts_optimizer=25,
)
init_points = []
init_labels = []
"""
parametername_type_map = {}
for param in params:
parametername_type_map[param["parameterName"]] = param["type"]
"""
for trial in completed_trials:
# Example: {"learning_rate": 0.01, "optimizer": "ftrl"}
parameter_values_json = json.loads(trial.parameter_values)
# Example: [0.01]
instance_features = []
instance_label = trial.objective_value
for param in params:
if param["type"] == "DOUBLE" or param[
"type"] == "INTEGER" or param["type"] == "DISCRETE":
instance_feature = parameter_values_json[
param["parameterName"]]
instance_features.append(instance_feature)
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
# Example: ["sgd", "adagrad", "adam", "ftrl"]
feasible_points = [
value.strip()
for value in feasible_points_string.split(",")
]
# Example: "ftrl"
parameter_value = parameter_values_json[
param["parameterName"]]
for feasible_point in feasible_points:
if feasible_point == parameter_value:
instance_features.append(1)
else:
instance_features.append(0)
init_points.append(instance_features)
init_labels.append(instance_label)
#import ipdb;ipdb.set_trace()
train_features = np.asarray(init_points)
train_labels = np.asarray(init_labels)
current_max_label = train_labels.max()
gp.fit(train_features, train_labels)
# Example: [[-3.66909025, -0.84486644], [-1.93270006, -0.95367483], [1.36095631, 0.61358525], ...], shape is [100000, 2]
x_tries = np.random.uniform(bounds[:, 0],
bounds[:, 1],
size=(100000, bounds.shape[0]))
mean, std = gp.predict(x_tries, return_std=True)
# Confidence bound criteria
acquisition_fucntion_values = mean + acquisition_fucntion_kappa * std
x_max = x_tries[acquisition_fucntion_values.argmax()]
max_acquision_fucntion_value = acquisition_fucntion_values.max()
# Example: [3993.864683994805, 44.15441513231316]
x_max = np.clip(x_max, bounds[:, 0], bounds[:, 1])
print("Current max acquision function choose: {}".format(x_max))
# Example: {"hidden2": 3993.864683994805, "hidden1": 44.15441513231316}
suggested_parameter_values_json = {}
index = 0
"""
# Example: [0.1, 0.5, 0.3, 0.9]
# Example: {"learning_rate": (0.01, 0.5), "hidden1": (40, 400), "optimizer_sgd": (0, 1), "optimizer_ftrl": (0, 1)}
for key in bound_dict.keys():
parameter_values_json[key] = x_max[index]
index += 1
"""
for param in params:
if param["type"] == "DOUBLE" or param["type"] == "DISCRETE":
suggested_parameter_values_json[
param["parameterName"]] = x_max[index]
index += 1
elif param["type"] == "INTEGER":
suggested_parameter_values_json[
param["parameterName"]] = int(round(x_max[index]))
index += 1
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
feasible_points.sort()
selected_value = self.find_closest_value_in_list(
feasible_points, x_max[index])
suggested_parameter_values_json[
param["parameterName"]] = selected_value
index += 1
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
# Example: ["sgd", "adagrad", "adam", "ftrl"]
feasible_points = [
value.strip()
for value in feasible_points_string.split(",")
]
# 记录这4个值中数最大的，然后取到对应的字符串
current_max = x_max[index]
suggested_parameter_value = feasible_points[0]
for feasible_point in feasible_points:
if x_max[index] > current_max:
current_max = x_max[index]
suggested_parameter_value = feasible_point
index += 1
suggested_parameter_values_json[
param["parameterName"]] = suggested_parameter_value
return_trial.parameter_values = json.dumps(
suggested_parameter_values_json)
return_trial.save()
return [return_trial]
def get_new_suggestions(self, study_id, trials, number=1):
# TODO: Only support retuning one trial
study = Study.objects.get(id=study_id)
completed_trials = Trial.objects.filter(
study_id=study_id, status="Completed")
study_configuration_json = json.loads(study.study_configuration)
random_init_trials = study_configuration_json.get("randomInitTrials", 3)
params = study_configuration_json["params"]
# Use random search if it has less dataset
if len(completed_trials) < random_init_trials:
randomSearchAlgorithm = RandomSearchAlgorithm()
return_trials = randomSearchAlgorithm.get_new_suggestions(
study_id, trials)
return return_trials
else:
return_trial = Trial.create(study.id, "BayesianOptimizationTrial")
acquisition_fucntion_kappa = 5
# Example: {'x': (-4, 4), 'y': (-3, 3)}
bound_dict = {}
for param in params:
if param["type"] == "DOUBLE" or param["type"] == "INTEGER":
min_value = param["minValue"]
max_value = param["maxValue"]
bound_dict[param["parameterName"]] = (min_value, max_value)
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
feasible_points.sort()
min_value = feasible_points[0]
max_value = feasible_points[-1]
bound_dict[param["parameterName"]] = (min_value, max_value)
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
value.strip() for value in feasible_points_string.split(",")
]
for feasible_point in feasible_points:
parameter_name = "{}_{}".format(param["parameterName"],
feasible_point)
bound_dict[parameter_name] = (0, 1)
bounds = []
for key in bound_dict.keys():
bounds.append(bound_dict[key])
bounds = np.asarray(bounds)
gp = GaussianProcessRegressor(
kernel=Matern(nu=2.5),
n_restarts_optimizer=25, )
init_points = []
init_labels = []
"""
parametername_type_map = {}
for param in params:
parametername_type_map[param["parameterName"]] = param["type"]
"""
for trial in completed_trials:
# Example: {"learning_rate": 0.01, "optimizer": "ftrl"}
parameter_values_json = json.loads(trial.parameter_values)
# Example: [0.01]
instance_features = []
instance_label = trial.objective_value
for param in params:
if param["type"] == "DOUBLE" or param["type"] == "INTEGER" or param["type"] == "DISCRETE":
instance_feature = parameter_values_json[param["parameterName"]]
instance_features.append(instance_feature)
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
# Example: ["sgd", "adagrad", "adam", "ftrl"]
feasible_points = [
value.strip() for value in feasible_points_string.split(",")
]
# Example: "ftrl"
parameter_value = parameter_values_json[param["parameterName"]]
for feasible_point in feasible_points:
if feasible_point == parameter_value:
instance_features.append(1)
else:
instance_features.append(0)
init_points.append(instance_features)
init_labels.append(instance_label)
#import ipdb;ipdb.set_trace()
train_features = np.asarray(init_points)
train_labels = np.asarray(init_labels)
current_max_label = train_labels.max()
gp.fit(train_features, train_labels)
# Example: [[-3.66909025, -0.84486644], [-1.93270006, -0.95367483], [1.36095631, 0.61358525], ...], shape is [100000, 2]
x_tries = np.random.uniform(
bounds[:, 0], bounds[:, 1], size=(100000, bounds.shape[0]))
mean, std = gp.predict(x_tries, return_std=True)
# Confidence bound criteria
acquisition_fucntion_values = mean + acquisition_fucntion_kappa * std
x_max = x_tries[acquisition_fucntion_values.argmax()]
max_acquision_fucntion_value = acquisition_fucntion_values.max()
# Example: [3993.864683994805, 44.15441513231316]
x_max = np.clip(x_max, bounds[:, 0], bounds[:, 1])
print("Current max acquision function choose: {}".format(x_max))
# Example: {"hidden2": 3993.864683994805, "hidden1": 44.15441513231316}
suggested_parameter_values_json = {}
index = 0
"""
# Example: [0.1, 0.5, 0.3, 0.9]
# Example: {"learning_rate": (0.01, 0.5), "hidden1": (40, 400), "optimizer_sgd": (0, 1), "optimizer_ftrl": (0, 1)}
for key in bound_dict.keys():
parameter_values_json[key] = x_max[index]
index += 1
"""
for param in params:
if param["type"] == "DOUBLE" or param["type"] == "DISCRETE":
suggested_parameter_values_json[param["parameterName"]] = x_max[
index]
index += 1
elif param["type"] == "INTEGER":
suggested_parameter_values_json[param["parameterName"]] = int(
round(x_max[index]))
index += 1
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
feasible_points.sort()
selected_value = self.find_closest_value_in_list(
feasible_points, x_max[index])
suggested_parameter_values_json[param[
"parameterName"]] = selected_value
index += 1
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
# Example: ["sgd", "adagrad", "adam", "ftrl"]
feasible_points = [
value.strip() for value in feasible_points_string.split(",")
]
# 记录这4个值中数最大的，然后取到对应的字符串
current_max = x_max[index]
suggested_parameter_value = feasible_points[0]
for feasible_point in feasible_points:
if x_max[index] > current_max:
current_max = x_max[index]
suggested_parameter_value = feasible_point
index += 1
suggested_parameter_values_json[param[
"parameterName"]] = suggested_parameter_value
return_trial.parameter_values = json.dumps(
suggested_parameter_values_json)
return_trial.save()
return [return_trial]
def get_new_suggestions(self, study_name, trials=[], number=1):
# TODO: Only support returning one trial
number = 1
# Get study and completed data
study = Study.objects.get(name=study_name)
completed_trials = Trial.objects.filter(study_name=study_name,
status="Completed")
study_configuration_json = json.loads(study.study_configuration)
random_init_trial_number = study_configuration_json.get(
"randomInitTrials", 3)
params = study_configuration_json["params"]
study_goal = study_configuration_json["goal"]
# Use random search if it has less dataset
if len(completed_trials) < random_init_trial_number:
randomSearchAlgorithm = RandomSearchAlgorithm()
return_trials = randomSearchAlgorithm.get_new_suggestions(
study_name, trials, number)
return return_trials
# Construct the map of name and scope to compute gaussian process
acquisition_function_kappa = 5
# Example: {'x': (-4, 4), 'y': (-3, 3)}
# name_scope_map = {}
# Construct the list with only scope, Example: [(40, 400)]
bounds = []
for param in params:
if param["type"] == "DOUBLE" or param["type"] == "INTEGER":
min_value = param["minValue"]
max_value = param["maxValue"]
# name_scope_map[param["parameterName"]] = (min_value, max_value)
bounds.append((min_value, max_value))
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
for feasible_point in feasible_points:
parameter_name = "{}_{}".format(param["parameterName"],
feasible_point)
# name_scope_map[parameter_name] = (0, 1)
bounds.append((0, 1))
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
value.strip()
for value in feasible_points_string.split(",")
]
for feasible_point in feasible_points:
parameter_name = "{}_{}".format(param["parameterName"],
feasible_point)
# name_scope_map[parameter_name] = (0, 1)
bounds.append((0, 1))
# Make sure it is numpy ndarry
bounds = np.asarray(bounds)
# Construct data to train gaussian process, Example: [[50], [150], [250]]
init_points = []
# Example: [0.6, 0.8, 0.6]
init_labels = []
# Construct train data with completed trials
for trial in completed_trials:
# Example: {"learning_rate": 0.01, "optimizer": "ftrl"}
parameter_values_json = json.loads(trial.parameter_values)
# Example: [0.01, "ftrl"]
instance_features = []
instance_label = trial.objective_value
for param in params:
if param["type"] == "DOUBLE" or param["type"] == "INTEGER":
instance_feature = parameter_values_json[
param["parameterName"]]
instance_features.append(instance_feature)
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
parameter_value = parameter_values_json[
param["parameterName"]]
for feasible_point in feasible_points:
if feasible_point == parameter_value:
instance_features.append(1)
else:
instance_features.append(0)
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
# Example: ["sgd", "adagrad", "adam", "ftrl"]
feasible_points = [
value.strip()
for value in feasible_points_string.split(",")
]
# Example: "ftrl"
parameter_value = parameter_values_json[
param["parameterName"]]
for feasible_point in feasible_points:
if feasible_point == parameter_value:
instance_features.append(1)
else:
instance_features.append(0)
init_points.append(instance_features)
init_labels.append(instance_label)
# Example: ndarray([[ 50], [150], [250]])
train_features = np.asarray(init_points)
# Example: ndarray([0.6, 0.8, 0.6])
train_labels = np.asarray(init_labels)
# current_max_label = train_labels.max()
# Train with gaussian process
gp = GaussianProcessRegressor(
kernel=Matern(nu=2.5),
n_restarts_optimizer=25,
)
gp.fit(train_features, train_labels)
# Example: [[-3.66909025, -0.84486644], [-1.93270006, -0.95367483], [1.36095631, 0.61358525], ...], shape is [100000, 2]
x_tries = np.random.uniform(bounds[:, 0],
bounds[:, 1],
size=(100000, bounds.shape[0]))
mean, std = gp.predict(x_tries, return_std=True)
# Confidence bound criteria
acquisition_fucntion_values = mean + acquisition_function_kappa * std
#x_max = x_tries[acquisition_fucntion_values.argmax()]
# tobe
#x_max = x_tries[acquisition_fucntion_values.argmin()]
if study_goal == "MAXIMIZE":
x_max = x_tries[acquisition_fucntion_values.argmax()]
#max_acquision_fucntion_value = acquisition_fucntion_values.max()
elif study_goal == "MINIMIZE":
x_max = x_tries[acquisition_fucntion_values.argmin()]
#max_acquision_fucntion_value = acquisition_fucntion_values.min()
else:
# TODO: Throw the error
x_max = []
# Example: [3993.864683994805, 44.15441513231316]
x_max = np.clip(x_max, bounds[:, 0], bounds[:, 1])
print("Current max acquision function choose: {}".format(x_max))
# Example: {"hidden2": 3993.864683994805, "hidden1": 44.15441513231316}
suggested_parameter_values_json = {}
index = 0
"""
Construct the suggested params according to the result of gaussian process
# Example prior result: [0.1, 0.5, 0.3, 0.9]
# Example param scope: {"learning_rate": (0.01, 0.5), "hidden1": (40, 400), "optimizer_sgd": (0, 1), "optimizer_ftrl": (0, 1)}
for key in bound_dict.keys():
parameter_values_json[key] = x_max[index]
index += 1
"""
for param in params:
if param["type"] == "DOUBLE":
suggested_parameter_values_json[
param["parameterName"]] = x_max[index]
index += 1
elif param["type"] == "INTEGER":
suggested_parameter_values_json[param["parameterName"]] = int(
x_max[index])
index += 1
elif param["type"] == "DISCRETE":
feasible_points_string = param["feasiblePoints"]
feasible_points = [
float(value.strip())
for value in feasible_points_string.split(",")
]
# Find the max value of these and get its string
current_max = x_max[index]
suggested_parameter_value = feasible_points[0]
for feasible_point in feasible_points:
if x_max[index] > current_max:
current_max = x_max[index]
suggested_parameter_value = feasible_point
index += 1
suggested_parameter_values_json[
param["parameterName"]] = suggested_parameter_value
elif param["type"] == "CATEGORICAL":
feasible_points_string = param["feasiblePoints"]
# Example: ["sgd", "adagrad", "adam", "ftrl"]
feasible_points = [
value.strip()
for value in feasible_points_string.split(",")
]
# Find the max value of these and get its string
current_max = x_max[index]
suggested_parameter_value = feasible_points[0]
for feasible_point in feasible_points:
if x_max[index] > current_max:
current_max = x_max[index]
suggested_parameter_value = feasible_point
index += 1
suggested_parameter_values_json[
param["parameterName"]] = suggested_parameter_value
return_trial = Trial.create(study.name, "BayesianOptimizationTrial")
return_trial.parameter_values = json.dumps(
suggested_parameter_values_json)
return_trial.save()
return [return_trial]
def get_new_suggestions(self, study_name, trials=[], number=1):
"""
Get the new suggested trials with grid search.
"""
return_trial_list = []
study = Study.objects.get(name=study_name)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
param_number = len(params)
# [['8', '16', '32', '64'], ['sgd', 'adagrad', 'adam', 'ftrl'], ['true', 'false']]
param_values_list = []
for param in params:
# Check param type
if param["type"] == "DOUBLE" or param["type"] == "INTEGER":
raise Exception("Grid search does not support DOUBLE and INTEGER")
feasible_point_list = [
value.strip() for value in param["feasiblePoints"].split(",")
]
param_values_list.append(feasible_point_list)
# Example: [('8', 'sgd', 'true'), ('8', 'sgd', 'false'), ('8', 'adagrad', 'true'), ('8', 'adagrad', 'false'), ('8', 'adam', 'true'), ('8', 'adam', 'false'), ('8', 'ftrl', 'true'), ('8', 'ftrl', 'false'), ('16', 'sgd', 'true'), ('16', 'sgd', 'false'), ('16', 'adagrad', 'true'), ('16', 'adagrad', 'false'), ('16', 'adam', 'true'), ('16', 'adam', 'false'), ('16', 'ftrl', 'true'), ('16', 'ftrl', 'false'), ('32', 'sgd', 'true'), ('32', 'sgd', 'false'), ('32', 'adagrad', 'true'), ('32', 'adagrad', 'false'), ('32', 'adam', 'true'), ('32', 'adam', 'false'), ('32', 'ftrl', 'true'), ('32', 'ftrl', 'false'), ('64', 'sgd', 'true'), ('64', 'sgd', 'false'), ('64', 'adagrad', 'true'), ('64', 'adagrad', 'false'), ('64', 'adam', 'true'), ('64', 'adam', 'false'), ('64', 'ftrl', 'true'), ('64', 'ftrl', 'false')]
combination_values_list = list(itertools.product(*param_values_list))
# Example: [{"hidden2": "8", "optimizer": "sgd", "batch_normalization": "true"}, ......]
all_combination_values_json = []
for combination_values in combination_values_list:
combination_values_json = {}
# Example: (u'8', u'sgd', u'true')
for i in range(param_number):
# Example: "sgd"
combination_values_json[params[i][
"parameterName"]] = combination_values[i]
all_combination_values_json.append(combination_values_json)
all_combination_number = len(all_combination_values_json)
# Compute how many grid search params have been allocated
allocated_trials = Trial.objects.filter(study_name=study_name)
return_trials_start_index = len(allocated_trials)
if return_trials_start_index > all_combination_number:
return_trials_start_index = 0
elif return_trials_start_index + number > all_combination_number:
return_trials_start_index = all_combination_number - number
for i in range(number):
trial = Trial.create(study.name, "GridSearchTrial")
trial.parameter_values = json.dumps(
all_combination_values_json[return_trials_start_index + i])
trial.save()
return_trial_list.append(trial)
return return_trial_list