def main(train_function):
client = AdvisorClient()
# Get or create the study
study_configuration = {
"goal":
"MINIMIZE",
"randomInitTrials":
3,
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [
{
"parameterName": "min_child_weight",
"type": "INTEGER",
"minValue": 1,
"maxValue": 20,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "max_depth",
"type": "INTEGER",
"minValue": 5,
"maxValue": 15,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "gamma",
"type": "DOUBLE",
"minValue": 0.0,
"maxValue": 10.0,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "alpha",
"type": "DOUBLE",
"minValue": 0.0,
"maxValue": 10.0,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
]
}
study = client.create_study("Study", study_configuration,
"BayesianOptimization")
#study = client.get_study_by_id(21)
# Get suggested trials
trials = client.get_suggestions(study.id, 3)
# Generate parameters
parameter_value_dicts = []
for trial in trials:
parameter_value_dict = json.loads(trial.parameter_values)
print("The suggested parameters: {}".format(parameter_value_dict))
parameter_value_dicts.append(parameter_value_dict)
# Run training
metrics = []
for i in range(len(trials)):
metric = train_function(**parameter_value_dicts[i])
metrics.append(metric)
# Complete the trial
for i in range(len(trials)):
trial = trials[i]
client.complete_trial_with_one_metric(trial, metrics[i])
is_done = client.is_study_done(study.id)
best_trial = client.get_best_trial(study.id)
print("The study: {}, best trial: {}".format(study, best_trial))
def main():
client = AdvisorClient()
# Create the study
name = "Study"
study_configuration = {
"goal":
"MINIMIZE",
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [{
"parameterName": "learning_rate",
"type": "INTEGER",
"minValue": 0.01,
"maxValue": 0.1,
"scallingType": "LINEAR"
}]
}
study = client.create_study(name, study_configuration)
# Get suggested trials
trials = client.get_suggestions(study.id, 3)
# Generate command-line parameters
commandline_parameters = []
i = 0
for trial in trials:
parameter = "--output_path=output/{}".format(i)
parameter_value_dict = json.loads(trial.parameter_values)
# Example: {"learning_rate": 0.05943265431983244}
for k, v in parameter_value_dict.items():
parameter += " --{}={}".format(k, v)
print(parameter)
commandline_parameters.append(parameter)
i += 1
# Run training
for i in range(3):
module_args = commandline_parameters[i]
module_name = "trainer.task"
# Example: python -m trainer.task --output_path=0 --learning_rate=0.0796523079087
shell_command = "python -m {} {}".format(module_name, module_args)
print(shell_command)
subprocess.call(shell_command, shell=True)
# Complete the trial
for i in range(3):
trial = trials[i]
logdir = "output/{}".format(i)
tensorboard_metrics = tensorboard_util.get_hyperparameters_metric(
logdir)
client.complete_trial(trial, tensorboard_metrics)
# Check if study done
is_done = client.is_study_done(study.id)
best_trial = client.get_best_trial(study.id)
print("The study: {}, if it is done: {}, best trial: {}".format(
study, is_done, best_trial))
def main(train_function):
client = AdvisorClient()
# Get or create the study
study_configuration = {
"goal":
"MINIMIZE",
"randomInitTrials":
1,
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [
{
"parameterName": "gamma",
"type": "DOUBLE",
"minValue": 0.001,
"maxValue": 0.01,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
{
"parameterName": "C",
"type": "DOUBLE",
"minValue": 0.5,
"maxValue": 1.0,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
{
"parameterName": "kernel",
"type": "CATEGORICAL",
"minValue": 0,
"maxValue": 0,
"feasiblePoints": "linear, poly, rbf, sigmoid, precomputed",
"scalingType": "LINEAR"
},
{
"parameterName": "coef0",
"type": "DOUBLE",
"minValue": 0.0,
"maxValue": 0.5,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
]
}
study = client.create_study("Study", study_configuration,
"BayesianOptimization")
#study = client.get_study_by_id(6)
num_trials = 20
for i in range(num_trials):
# Get suggested trials
trials = client.get_suggestions(study.name, 3)
# Generate parameters
parameter_value_dicts = []
for trial in trials:
parameter_value_dict = json.loads(trial.parameter_values)
print("The suggested parameters: {}".format(parameter_value_dict))
parameter_value_dicts.append(parameter_value_dict)
# Run training
metrics = []
for i in range(len(trials)):
metric = train_function(**parameter_value_dicts[i])
#metric = train_function(parameter_value_dicts[i])
metrics.append(metric)
# Complete the trial
for i in range(len(trials)):
trial = trials[i]
client.complete_trial_with_one_metric(trial, metrics[i])
is_done = client.is_study_done(study.name)
best_trial = client.get_best_trial(study.name)
print("The study: {}, best trial: {}".format(study, best_trial))
print(best_trial.parameter_values)
def run(self):
client = AdvisorClient()
self.run_config_dict
# TODO: move the logic into local runner
runner = LocalRunner()
if "runner" in self.run_config_dict:
if self.run_config_dict["runner"] == "local_runner":
runner = LocalRunner()
logging.info("Run with local runner")
study_name = self.run_config_dict["name"].encode("utf-8")
study = client.get_or_create_study(study_name,
self.run_config_dict["search_space"],
self.run_config_dict["algorithm"])
logging.info("Create study: {}".format(study))
for i in range(self.run_config_dict["trialNumber"]):
logging.info("-------------------- Start Trial --------------------")
# Get suggested trials
trials = client.get_suggestions(study.name, 1)
logging.info("Get trial: {}".format(trials[0]))
#import ipdb;ipdb.set_trace()
# Generate parameters
parameter_value_dicts = []
for trial in trials:
parameter_value_dict = json.loads(trial.parameter_values)
logging.info(
"The suggested parameters: {}".format(parameter_value_dict))
parameter_value_dicts.append(parameter_value_dict)
# Run training
for trial in trials:
#metric = train_function(**parameter_value_dicts[i])
# Example: {"gamma": 0.0063987614450157415}
parameters_dict = json.loads(trials[0].parameter_values)
parameter_string = ""
for k, v in parameters_dict.items():
parameter_string += " -{}={}".format(k, v)
command_string = "cd {} && {} {}".format(
self.run_config_dict["path"], self.run_config_dict["command"],
parameter_string)
#exit_code = subprocess.call(command_string, shell=True)
logging.info("Run the command: {}".format(command_string))
# Example: '0.0\n'
# Example: 'Compute y = x * x - 3 * x + 2\nIput x is: 1.0\nOutput is: 0.0\n0.0\n'
command_output = subprocess.check_output(command_string, shell=True)
# TODO: Log the output in the directory
#logging.info("Get output of command: {}".format(command_output))
metric = float(command_output.split("\n")[-2].strip())
# Complete the trial
client.complete_trial_with_one_metric(trial, metric)
logging.info("Update the trial with metrics: {}".format(metric))
logging.info("--------------------- End Trial ---------------------")
is_done = client.is_study_done(study.name)
best_trial = client.get_best_trial(study.name)
logging.info("The study: {}, best trial: {}".format(study, best_trial))
def main():
client = AdvisorClient()
model = MyModel()
model.load_dataset()
# Get or create the study
study_configuration = {
"goal":
"MINIMIZE",
"randomInitTrials":
5,
"maxTrials":
30,
"maxParallelTrials":
1,
"params": [
{
"parameterName": "max_bin",
"type": "INTEGER",
"minValue": 63,
"maxValue": 511,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "bin_construct_sample_cnt",
"type": "INTEGER",
"minValue": 3,
"maxValue": 1000,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "num_leaves",
"type": "INTEGER",
"minValue": 63,
"maxValue": 512,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
# {
# "parameterName": "lambda_l2",
# "type": "DOUBLE",
# "minValue": 0.0,
# "maxValue": 10.0,
# "feasiblePoints": "",
# "scallingType": "LINEAR"
# },
# {
# "parameterName": "lambda_l1",
# "type": "DOUBLE",
# "minValue": 0.0,
# "maxValue": 10.0,
# "feasiblePoints": "",
# "scallingType": "LINEAR"
# },
{
"parameterName": "cat_l2",
"type": "DOUBLE",
"minValue": 1.0,
"maxValue": 100.0,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "cat_smooth",
"type": "DOUBLE",
"minValue": 1.0,
"maxValue": 100.0,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
]
}
# study = client.create_study("lightgbm_search", study_configuration,
# "BayesianOptimization")
study = client.get_study_by_id(44)
# Get suggested trials
# trials = client.get_suggestions(study.id, 5)
# for i in range(5):
# trial = trials[i]
# parameter_value_dict = json.loads(trial.parameter_values)
# logger.info("The suggested parameters: {}".format(parameter_value_dict))
# metric = model.train(**parameter_value_dict)
# client.complete_trial_with_one_metric(trial, metric)
for i in range(30):
# Get suggested trials
trials = client.get_suggestions(study.id, 1)
# Generate parameters
parameter_value_dicts = []
for trial in trials:
parameter_value_dict = json.loads(trial.parameter_values)
logger.info("The suggested parameters: {}".format(parameter_value_dict))
parameter_value_dicts.append(parameter_value_dict)
# Run training
metrics = []
for i in range(len(trials)):
metric = model.train(**parameter_value_dicts[i])
metrics.append(metric)
# Complete the trial
for i in range(len(trials)):
trial = trials[i]
client.complete_trial_with_one_metric(trial, metrics[i])
is_done = client.is_study_done(study.id)
assert is_done
best_trial = client.get_best_trial(study.id)
logger.info("The study: {}, best trial: {}".format(study, best_trial))
def main(train_function):
client = AdvisorClient()
# Get or create the study
study_configuration = {
"goal":
"MINIMIZE",
"randomInitTrials":
1,
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [
{
"parameterName": "gamma",
"type": "DOUBLE",
"minValue": 0.001,
"maxValue": 0.01,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "C",
"type": "DOUBLE",
"minValue": 0.5,
"maxValue": 1.0,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
{
"parameterName": "kernel",
"type": "CATEGORICAL",
"minValue": 0,
"maxValue": 0,
"feasiblePoints": "linear, poly, rbf, sigmoid, precomputed",
"scallingType": "LINEAR"
},
{
"parameterName": "coef0",
"type": "DOUBLE",
"minValue": 0.0,
"maxValue": 0.5,
"feasiblePoints": "",
"scallingType": "LINEAR"
},
]
}
study = client.create_study("Study", study_configuration,
"BayesianOptimization")
#study = client.get_study_by_id(6)
# Get suggested trials
trials = client.get_suggestions(study.id, 3)
# Generate parameters
parameter_value_dicts = []
for trial in trials:
parameter_value_dict = json.loads(trial.parameter_values)
print("The suggested parameters: {}".format(parameter_value_dict))
parameter_value_dicts.append(parameter_value_dict)
# Run training
metrics = []
for i in range(len(trials)):
metric = train_function(**parameter_value_dicts[i])
#metric = train_function(parameter_value_dicts[i])
metrics.append(metric)
# Complete the trial
for i in range(len(trials)):
trial = trials[i]
client.complete_trial_with_one_metric(trial, metrics[i])
is_done = client.is_study_done(study.id)
best_trial = client.get_best_trial(study.id)
print("The study: {}, best trial: {}".format(study, best_trial))
def main(train_function):
client = AdvisorClient()
# Get or create the study
study_configuration = {
"goal":
"MINIMIZE",
"randomInitTrials":
1,
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [
{
"parameterName": "filter_number0",
"type": "INTEGER",
"minValue": 16,
"maxValue": 32,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
{
"parameterName": "convolution_kernel_size0",
"type": "INTEGER",
"minValue": 3,
"maxValue": 5,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
{
"parameterName": "max_polling_size0",
"type": "INTEGER",
"minValue": 2,
"maxValue": 3,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
{
"parameterName": "filter_number1",
"type": "INTEGER",
"minValue": 16,
"maxValue": 32,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
{
"parameterName": "convolution_kernel_size1",
"type": "INTEGER",
"minValue": 3,
"maxValue": 5,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
{
"parameterName": "max_polling_size1",
"type": "INTEGER",
"minValue": 2,
"maxValue": 3,
"feasiblePoints": "",
"scalingType": "LINEAR"
},
]
}
#study = client.create_study("Study", study_configuration, "BayesianOptimization")
study = client.create_study("Study", study_configuration,
"RandomSearchAlgorithm")
#study = client.get_study_by_id(6)
# Get suggested trials
trials = client.get_suggestions(study.id, 3)
#import ipdb;ipdb.set_trace()
# Generate parameters
parameter_value_dicts = []
for trial in trials:
parameter_value_dict = json.loads(trial.parameter_values)
print("The suggested parameters: {}".format(parameter_value_dict))
parameter_value_dicts.append(parameter_value_dict)
# Run training
metrics = []
for i in range(len(trials)):
metric = train_function(**parameter_value_dicts[i])
#metric = train_function(parameter_value_dicts[i])
metrics.append(metric)
trial = trials[i]
# Complete the trial
client.complete_trial_with_one_metric(trial, metrics[i])
is_done = client.is_study_done(study.id)
best_trial = client.get_best_trial(study.id)
print("The study: {}, best trial: {}".format(study, best_trial))
def main(train_function):
client = AdvisorClient()
# Get or create the study
study_configuration = {
"goal":
"MINIMIZE",
"randomInitTrials":
1,
"maxTrials":
5,
"maxParallelTrials":
1,
"params": [{
"parameterName":
"operation_name1",
"type":
"CATEGORICAL",
"minValue":
0,
"maxValue":
0,
"feasiblePoints":
"Fliplr, Crop, GaussianBlur, ContrastNormalization, AdditiveGaussianNoise, Multiply",
"scallingType":
"LINEAR"
}, {
"parameterName": "magnitude1",
"type": "INTEGER",
"minValue": 1,
"maxValue": 11,
"scallingType": "LINEAR"
}, {
"parameterName": "probability1",
"type": "INTEGER",
"minValue": 1,
"maxValue": 10,
"scallingType": "LINEAR"
}, {
"parameterName":
"operation_name2",
"type":
"CATEGORICAL",
"minValue":
0,
"maxValue":
0,
"feasiblePoints":
"Fliplr, Crop, GaussianBlur, ContrastNormalization, AdditiveGaussianNoise, Multiply",
"scallingType":
"LINEAR"
}, {
"parameterName": "magnitude2",
"type": "INTEGER",
"minValue": 1,
"maxValue": 11,
"scallingType": "LINEAR"
}, {
"parameterName": "probability2",
"type": "INTEGER",
"minValue": 1,
"maxValue": 10,
"scallingType": "LINEAR"
}, {
"parameterName":
"operation_name3",
"type":
"CATEGORICAL",
"minValue":
0,
"maxValue":
0,
"feasiblePoints":
"Fliplr, Crop, GaussianBlur, ContrastNormalization, AdditiveGaussianNoise, Multiply",
"scallingType":
"LINEAR"
}, {
"parameterName": "magnitude3",
"type": "INTEGER",
"minValue": 1,
"maxValue": 11,
"scallingType": "LINEAR"
}, {
"parameterName": "probability3",
"type": "INTEGER",
"minValue": 1,
"maxValue": 10,
"scallingType": "LINEAR"
}]
}
#study = client.create_study("Study", study_configuration, "BayesianOptimization")
study = client.create_study("Study", study_configuration,
"RandomSearchAlgorithm")
#study = client.get_study_by_id(6)
# Get suggested trials
trials = client.get_suggestions(study.id, 3)
#import ipdb;ipdb.set_trace()
# Generate parameters
parameter_value_dicts = []
for trial in trials:
parameter_value_dict = json.loads(trial.parameter_values)
print("The suggested parameters: {}".format(parameter_value_dict))
parameter_value_dicts.append(parameter_value_dict)
# Run training
metrics = []
for i in range(len(trials)):
metric = train_function(**parameter_value_dicts[i])
#metric = train_function(parameter_value_dicts[i])
metrics.append(metric)
trial = trials[i]
# Complete the trial
client.complete_trial_with_one_metric(trial, metrics[i])
is_done = client.is_study_done(study.id)
best_trial = client.get_best_trial(study.id)
print("The study: {}, best trial: {}".format(study, best_trial))