def create_optimizer(self, algorithm_name):
# Search Space example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)}
chocolate_search_space = {}
for param in self.search_space.params:
key = BaseChocolateService.encode(param.name)
if param.type == INTEGER:
chocolate_search_space[key] = choco.quantized_uniform(
int(param.min), int(param.max), int(param.step))
elif param.type == DOUBLE:
chocolate_search_space[key] = choco.quantized_uniform(
float(param.min), float(param.max), float(param.step))
elif param.type == CATEGORICAL:
chocolate_search_space[key] = choco.choice(param.list)
else:
chocolate_search_space[key] = choco.choice(
[float(e) for e in param.list])
# Refer to https://chocolate.readthedocs.io/tutorials/algo.html
if algorithm_name == "grid":
self.chocolate_optimizer = choco.Grid(self.conn,
chocolate_search_space,
clear_db=True)
# hyperopt-random is the default option in katib.
elif algorithm_name == "chocolate-random":
self.chocolate_optimizer = choco.Random(self.conn,
chocolate_search_space,
clear_db=True)
elif algorithm_name == "chocolate-quasirandom":
self.chocolate_optimizer = choco.QuasiRandom(
self.conn, chocolate_search_space, clear_db=True)
elif algorithm_name == "chocolate-bayesian-optimization":
self.chocolate_optimizer = choco.Bayes(self.conn,
chocolate_search_space,
clear_db=True)
# elif self.algorithm_name == "chocolate-CMAES":
# self.chocolate_optimizer = choco.CMAES(self.conn, chocolate_search_space, clear_db=True)
elif algorithm_name == "chocolate-mocmaes":
mu = 1
self.chocolate_optimizer = choco.MOCMAES(self.conn,
chocolate_search_space,
mu=mu,
clear_db=True)
else:
raise Exception(
'"Failed to create Chocolate optimizer for the algorithm: {}'.
format(algorithm_name))
def create_optimizer(self, algorithm_name):
# Search Space example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)}
chocolate_search_space = {}
for param in self.search_space.params:
key = BaseChocolateService.encode(param.name)
# Chocolate quantized_uniform distribution uses half-open interval: [low, high).
if param.type == INTEGER:
chocolate_search_space[key] = choco.quantized_uniform(
int(param.min),
int(param.max) + int(param.step), int(param.step))
elif param.type == DOUBLE:
chocolate_search_space[key] = choco.quantized_uniform(
float(param.min),
float(param.max) + float(param.step), float(param.step))
# For Categorical and Discrete insert indexes to DB from list of values
elif param.type == CATEGORICAL or param.type == DISCRETE:
chocolate_search_space[key] = choco.choice(
[idx for idx, _ in enumerate(param.list)])
if algorithm_name in DEPRECATED_ALGORITHM_NAME:
warnings.warn(
"Algorithm name '{}' is deprecated. Please use '{}'.".format(
algorithm_name,
DEPRECATED_ALGORITHM_NAME[algorithm_name],
),
DeprecationWarning,
)
algorithm_name = DEPRECATED_ALGORITHM_NAME[algorithm_name]
# Refer to https://chocolate.readthedocs.io/tutorials/algo.html
if algorithm_name == "grid":
self.chocolate_optimizer = choco.Grid(self.conn,
chocolate_search_space,
clear_db=True)
# hyperopt-random is the default option in katib.
elif algorithm_name == "random":
self.chocolate_optimizer = choco.Random(self.conn,
chocolate_search_space,
clear_db=True)
elif algorithm_name == "quasirandom":
self.chocolate_optimizer = choco.QuasiRandom(
self.conn, chocolate_search_space, clear_db=True)
elif algorithm_name == "bayesianoptimization":
self.chocolate_optimizer = choco.Bayes(self.conn,
chocolate_search_space,
clear_db=True)
# elif self.algorithm_name == "chocolate-CMAES":
# self.chocolate_optimizer = choco.CMAES(self.conn, chocolate_search_space, clear_db=True)
elif algorithm_name == "mocmaes":
mu = 1
self.chocolate_optimizer = choco.MOCMAES(self.conn,
chocolate_search_space,
mu=mu,
clear_db=True)
else:
raise Exception(
'"Failed to create Chocolate optimizer for the algorithm: {}'.
format(algorithm_name))
def convert_param_to_choco(param):
"""Convert a single search parameter suitably for ``chocolate``.
"""
from math import log10
import chocolate as choco
if param['type'] == 'BOOL':
return choco.choice([False, True])
if param['type'] == 'INT':
return choco.quantized_uniform(low=param['min'],
high=param['max'] + 1,
step=1)
if param['type'] == 'STRING':
return choco.choice(param['options'])
if param['type'] == 'FLOAT':
return choco.uniform(low=param['min'], high=param['max'])
if param['type'] == 'FLOAT_EXP':
return choco.log(low=log10(param['min']),
high=log10(param['max']),
base=10)
else:
raise ValueError("Didn't understand space {}.".format(param))
def get_new_suggestions(self, study, trials=[], number=1):
"""
Get the new suggested trials with Chocolate algorithm.
"""
# 1. Construct search space
# Example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)}
chocolate_search_space = {}
# study = Study.objects.get(name=study_name)
study_configuration_json = json.loads(study.study_configuration)
params = study_configuration_json["params"]
for param in params:
param_name = param["parameterName"]
if param["type"] == "INTEGER":
# TODO: Support int type of search space)
pass
elif param["type"] == "DOUBLE":
chocolate_search_space[param_name] = choco.uniform(
param["minValue"], param["maxValue"])
elif param["type"] == "DISCRETE" or param["type"] == "CATEGORICAL":
feasible_point_list = [
value.strip()
for value in param["feasiblePoints"].split(",")
]
chocolate_search_space[param_name] = choco.choice(
feasible_point_list)
conn = choco.SQLiteConnection("sqlite:///my_db.db")
# Refer to https://chocolate.readthedocs.io/tutorials/algo.html
if self.algorithm_name == "Grid":
sampler = choco.Grid(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "Random":
sampler = choco.Random(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "QuasiRandom":
sampler = choco.QuasiRandom(conn,
chocolate_search_space,
clear_db=True)
elif self.algorithm_name == "Bayes":
sampler = choco.Bayes(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "CMAES":
sampler = choco.CMAES(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "MOCMAES":
mu = 1
sampler = choco.MOCMAES(conn,
chocolate_search_space,
mu=mu,
clear_db=True)
# 2. Update with completed advisor trials
# completed_advisor_trials = Trial.objects.filter(
# study_name=study_name, status="Completed")
completed_advisor_trials = [
i for i in trials if i.status == "Completed"
]
for index, advisor_trial in enumerate(completed_advisor_trials):
parameter_values_json = json.loads(advisor_trial.parameter_values)
loss = advisor_trial.objective_value
if study_configuration_json["goal"] == "MAXIMIZE":
loss = -1 * loss
entry = {"_chocolate_id": index, "_loss": loss}
entry.update(parameter_values_json)
# Should not use sampler.update(token, loss)
conn.insert_result(entry)
# 3. Run algorithm and construct return advisor trials
return_trial_list = []
for i in range(number):
# Example: {'_chocolate_id': 1}
# Example: {u'hidden2': u'32', u'learning_rate': 0.07122424534644338, u'l1_normalization': 0.8402644688674471, u'optimizer': u'adam'}
token, chocolate_params = sampler.next()
parameter_values_json = {}
for param in params:
if (param["type"] == "INTEGER" or param["type"] == "DOUBLE"
or param["type"] == "CATEGORICAL"):
parameter_values_json[
param["parameterName"]] = chocolate_params[
param["parameterName"]]
elif param["type"] == "DISCRETE":
parameter_values_json[param["parameterName"]] = int(
chocolate_params[param["parameterName"]])
new_advisor_trial = Trial.create(study.name, "ChocolateTrial")
new_advisor_trial.parameter_values = json.dumps(
parameter_values_json)
# new_advisor_trial.save()
return_trial_list.append(new_advisor_trial)
return return_trial_list
def getSuggestions(self, search_space, trials, request_number):
"""
Get the new suggested trials with chocolate algorithm.
"""
# Example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)}
chocolate_search_space = {}
for param in search_space.params:
key = BaseChocolateService.encode(param.name)
if param.type == INTEGER:
chocolate_search_space[key] = choco.quantized_uniform(
int(param.min), int(param.max), 1)
elif param.type == DOUBLE:
chocolate_search_space[key] = choco.quantized_uniform(
float(param.min), float(param.max), float(param.step))
elif param.type == CATEGORICAL:
chocolate_search_space[key] = choco.choice(param.list)
else:
chocolate_search_space[key] = choco.choice(
[float(e) for e in param.list])
conn = choco.SQLiteConnection("sqlite:///my_db.db")
# Refer to https://chocolate.readthedocs.io/tutorials/algo.html
if self.algorithm_name == "grid":
sampler = choco.Grid(conn, chocolate_search_space, clear_db=True)
# hyperopt-random is the default option in katib.
elif self.algorithm_name == "chocolate-random":
sampler = choco.Random(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "chocolate-quasirandom":
sampler = choco.QuasiRandom(conn,
chocolate_search_space,
clear_db=True)
elif self.algorithm_name == "chocolate-bayesian-optimization":
sampler = choco.Bayes(conn, chocolate_search_space, clear_db=True)
# elif self.algorithm_name == "chocolate-CMAES":
# sampler = choco.CMAES(conn, chocolate_search_space, clear_db=True)
elif self.algorithm_name == "chocolate-MOCMAES":
mu = 1
sampler = choco.MOCMAES(conn,
chocolate_search_space,
mu=mu,
clear_db=True)
else:
raise Exception('"Failed to create the algortihm: {}'.format(
self.algorithm_name))
for index, trial in enumerate(trials):
loss_for_choco = float(trial.target_metric.value)
if search_space.goal == MAX_GOAL:
loss_for_choco = -1 * loss_for_choco
entry = {"_chocolate_id": index, "_loss": loss_for_choco}
for param in search_space.params:
param_assignment = None
for assignment in trial.assignments:
if param.name == assignment.name:
param_assignment = assignment.value
break
if param.type == INTEGER:
param_assignment = int(param_assignment)
elif param.type == DOUBLE:
param_assignment = float(param_assignment)
entry.update({
BaseChocolateService.encode(param.name):
param_assignment
})
logger.info(entry)
# Should not use sampler.update(token, loss), because we will create
# a new BaseChocolateService instance for every request. Thus we need
# to insert all previous trials every time.
conn.insert_result(entry)
list_of_assignments = []
for i in range(request_number):
try:
token, chocolate_params = sampler.next()
list_of_assignments.append(
BaseChocolateService.convert(search_space,
chocolate_params))
except StopIteration:
logger.info(
"Chocolate db is exhausted, increase Search Space or decrease maxTrialCount!"
)
return list_of_assignments
"num_val_trials" : 10,
"batch_size" : 32,
"gamma" : 0.99,
"cnn": False,
"num_val_trials" : 10,
"batch_size" : 32,
"gamma" : 0.99,
"num_trials" : 5,
"USE_CUDA" : True,
"device" : "",
"eps": 1.,
"num_workers": 12,
"num_gpus": 1,
# Searched over
"hyperparam_mutations":{
"method": choco.choice(['our', 'PER']),
"return_latent":choco.choice['state', 'first_hidden', 'second_hidden', 'last']),
"var": choco.choice([.3, 1., 3., 10.]),
"mean": choco.choice([0.]),
"decision_eps": choco.choice([1.]),
"theta" : choco.choice([1.]),
"invert_actions" : choco.choice([False]),
"lr": choco.choice([1e-3, 5e-4, 1e-4, 5e-5, 1e-5]),
},
}
# initialize chocolate
i = 0
sampler = None
while sampler is None:
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
m = models[model](**params)
m.fit(trn_x, trn_y)
y_pred = m.predict(tst_x)
return -1*skm.f1_score(tst_y, y_pred, average='macro')
space = [
# {'model': 'RandomForestClassifier',
# "max_depth" : choco.quantized_uniform(2, 32, 2),
# "min_samples_split": choco.quantized_uniform(2, 600, 2),
# "n_estimators" : choco.quantized_uniform(125, 800, 25),},
{'model': 'SVC',
"gamma": 'auto',
"C": choco.log(-3, 3, 10),
"kernel": choco.choice(['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']),
"tol": choco.log(-5, -2, 10),},
{'model': 'XGBClassifier',
"learning_rate" : choco.uniform(0.001, 0.1),
"max_depth" : choco.quantized_uniform(2, 16, 2),
"min_child_weight": choco.quantized_uniform(2, 10, 2),
"subsample" : choco.quantized_uniform(0.7, 1.05, 0.05),
"n_estimators" : choco.quantized_uniform(25, 525, 25),},
{'model': 'LogisticRegression',
"penalty" : choco.choice(['l1', 'l2']),
"C" : choco.log(-2, 1, 10),},
]
models = {
'RandomForestClassifier': RandomForestClassifier,
'SVC': SVC,
def f1_score_model(trn_x, trn_y, tst_x, tst_y, model, **params):
m = models[model](**params)
m.fit(trn_x, trn_y)
y_pred = m.predict(tst_x)
return -1 * skm.f1_score(tst_y, y_pred, average='macro')
space = [
{
'model': 'SVC',
"gamma": 'auto',
"C": choco.log(-3, 3, 10),
"kernel":
choco.choice(['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']),
"tol": choco.log(-5, -2, 10),
},
{
'model': 'XGBClassifier',
"learning_rate": choco.uniform(0.001, 0.1),
"max_depth": choco.quantized_uniform(2, 16, 2),
"min_child_weight": choco.quantized_uniform(2, 10, 2),
"subsample": choco.quantized_uniform(0.7, 1.05, 0.05),
"n_estimators": choco.quantized_uniform(25, 525, 25),
},
{
'model': 'RandomForestClassifier',
"max_depth": choco.quantized_uniform(2, 10, 2),
"min_samples_leaf": choco.quantized_uniform(2, 10, 2),
"n_estimators": choco.quantized_uniform(25, 525, 25),