def main(unused_argv): client = AdvisorClient() with open(FLAGS.config, "r") as cfg: config = json.load(cfg) assert "study_name" in config, "The config needs a 'study_name' parameter (str)" study_name = config.pop("study_name") study_config = config.pop("hyper-parameter-config") algorithm = config.pop("algorithm") fixed_parameter_config = config.pop("fixed-parameter-config") exp_dir = config.pop("study_dir") if not os.path.isdir(os.path.join(exp_dir, study_name)): os.mkdir(os.path.join(exp_dir, study_name)) assert algorithm in ALGORITHMS, "algorithm {} is not supported. Select one of {}".format( algorithm, ", ".join([algo for algo in ALGORITHMS])) study = client.get_or_create_study(study_name=study_name, study_configuration=study_config) print(study) max_trials = study_config["maxTrials"] for i in range(max_trials): trial = client.get_suggestions(study.name, 1)[0] parameter_value_dict = json.loads(trial.parameter_values) parameter_value_dict["skip_gru_units"] = int( parameter_value_dict["skip_gru_units"]) parameter_value_dict["gru_units"] = int( parameter_value_dict["gru_units"]) parameter_value_dict["cnn_batch_normalization"] = int( parameter_value_dict["cnn_batch_normalization"]) merged_config = {**fixed_parameter_config, **parameter_value_dict} while os.path.isdir( os.path.join(exp_dir, study_name, "trial_{}".format(i))): i += 1 os.mkdir(os.path.join(exp_dir, study_name, "trial_{}").format(i)) merged_config["model_dir"] = os.path.join(exp_dir, study_name, "trial_{}".format(i)) manager = Manager(**merged_config) ret = manager.run() corr_score = ret["corr"] rmse_score = ret["rmse"] metric = rmse_score + (1 - corr_score) trial = client.complete_trial_with_one_metric(trial, metric) print(trial) best_trial = client.get_best_trial(study.name) print("Best trial: {}".format(best_trial))
def run(self): client = AdvisorClient() self.run_config_dict # TODO: move the logic into local runner runner = LocalRunner() if "runner" in self.run_config_dict: if self.run_config_dict["runner"] == "local_runner": runner = LocalRunner() logging.info("Run with local runner") study_name = self.run_config_dict["name"].encode("utf-8") study = client.get_or_create_study(study_name, self.run_config_dict["search_space"], self.run_config_dict["algorithm"]) logging.info("Create study: {}".format(study)) for i in range(self.run_config_dict["trialNumber"]): logging.info("-------------------- Start Trial --------------------") # Get suggested trials trials = client.get_suggestions(study.name, 1) logging.info("Get trial: {}".format(trials[0])) #import ipdb;ipdb.set_trace() # Generate parameters parameter_value_dicts = [] for trial in trials: parameter_value_dict = json.loads(trial.parameter_values) logging.info( "The suggested parameters: {}".format(parameter_value_dict)) parameter_value_dicts.append(parameter_value_dict) # Run training for trial in trials: #metric = train_function(**parameter_value_dicts[i]) # Example: {"gamma": 0.0063987614450157415} parameters_dict = json.loads(trials[0].parameter_values) parameter_string = "" for k, v in parameters_dict.items(): parameter_string += " -{}={}".format(k, v) command_string = "cd {} && {} {}".format( self.run_config_dict["path"], self.run_config_dict["command"], parameter_string) #exit_code = subprocess.call(command_string, shell=True) logging.info("Run the command: {}".format(command_string)) # Example: '0.0\n' # Example: 'Compute y = x * x - 3 * x + 2\nIput x is: 1.0\nOutput is: 0.0\n0.0\n' command_output = subprocess.check_output(command_string, shell=True) # TODO: Log the output in the directory #logging.info("Get output of command: {}".format(command_output)) metric = float(command_output.split("\n")[-2].strip()) # Complete the trial client.complete_trial_with_one_metric(trial, metric) logging.info("Update the trial with metrics: {}".format(metric)) logging.info("--------------------- End Trial ---------------------") is_done = client.is_study_done(study.name) best_trial = client.get_best_trial(study.name) logging.info("The study: {}, best trial: {}".format(study, best_trial))