def generate_plot_from_dataset(dataset, experiments, output_path): module_path = os.path.join(os.path.dirname(HERE), "nas_big_data", dataset) # check baseline data: baseline_data = None if "baseline" in experiments: baseline_file = experiments["baseline"] baseline_path = os.path.join(module_path, "baseline", baseline_file) baseline_data = load_json(baseline_path) bests_data = None bests_output_path = None if "bests" in experiments: bests_output_path = os.path.join(output_path, "bests") create_dir(bests_output_path) bests_data = {} for best_name in experiments["bests"]: hists_path = os.path.join(module_path, "best", best_name, "history") hists = [name for name in os.listdir(hists_path) if "json" in name] bests_data[best_name] = [load_json(os.path.join(hists_path, h)) for h in hists] # load the data for each experiment experiments = {exp:None for exp in experiments["experiments"]} for exp in experiments: experiment_path = os.path.join(module_path, "exp", exp) data = load_data_from_exp(experiment_path) experiments[exp] = data plot_objective_multi(experiments, output_path, baseline_data) plot_best_objective(experiments, output_path) plot_number_of_evaluations(experiments, output_path) plot_scatter_objective_multi(experiments, output_path) plot_usage_training_time(experiments, output_path) # plot best if bests_data is not None: plot_best_networks(bests_data, baseline_data, bests_output_path)
def main(): output_path = os.path.join(HERE, "outputs") create_dir(output_path) experiments_path = os.path.join(HERE, "experiments.yaml") experiments = yaml_load(experiments_path) for dataset in experiments: dataset_path = os.path.join(output_path, dataset) create_dir(dataset_path) for experiment in experiments[dataset]["experiments"]: experiment_path = os.path.join(dataset_path, experiment) create_dir(experiment_path) generate_plot_from_exp(dataset, experiment, output_path=experiment_path) # comparative plots between experiments of the same dataset generate_plot_from_dataset(dataset, experiments[dataset], output_path=dataset_path)
horovodrun -np 4 python -m deephyper.benchmark.nas.covertype.train """ import os from deephyper.nas.run.tf_distributed import run from deephyper.nas.run.util import create_dir from nas_big_data.combo.load_data import load_data_test from nas_big_data.combo.problem_agebo import Problem os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(i) for i in range(4)]) HERE = os.path.dirname(os.path.abspath(__file__)) output_dir = os.path.join(HERE, __file__[:-3]) create_dir(output_dir) Problem.load_data(load_data_test) config = Problem.space config["log_dir"] = output_dir config["hyperparameters"]["num_epochs"] = 200 config["hyperparameters"]["verbose"] = 1 config["hyperparameters"]["learning_rate"] = 0.0003233381265748 config["hyperparameters"]["batch_size"] = 166 config["hyperparameters"]["optimizer"] = "adam" config["hyperparameters"]["patience_ReduceLROnPlateau"] = 10 config["hyperparameters"]["patience_EarlyStopping"] = 29 config["loss"] = "mae"