def generate_plot_from_dataset(dataset, experiments, output_path):
module_path = os.path.join(os.path.dirname(HERE), "nas_big_data", dataset)
# check baseline data:
baseline_data = None
if "baseline" in experiments:
baseline_file = experiments["baseline"]
baseline_path = os.path.join(module_path, "baseline", baseline_file)
baseline_data = load_json(baseline_path)
bests_data = None
bests_output_path = None
if "bests" in experiments:
bests_output_path = os.path.join(output_path, "bests")
create_dir(bests_output_path)
bests_data = {}
for best_name in experiments["bests"]:
hists_path = os.path.join(module_path, "best", best_name, "history")
hists = [name for name in os.listdir(hists_path) if "json" in name]
bests_data[best_name] = [load_json(os.path.join(hists_path, h)) for h in hists]
# load the data for each experiment
experiments = {exp:None for exp in experiments["experiments"]}
for exp in experiments:
experiment_path = os.path.join(module_path, "exp", exp)
data = load_data_from_exp(experiment_path)
experiments[exp] = data
plot_objective_multi(experiments, output_path, baseline_data)
plot_best_objective(experiments, output_path)
plot_number_of_evaluations(experiments, output_path)
plot_scatter_objective_multi(experiments, output_path)
plot_usage_training_time(experiments, output_path)
# plot best
if bests_data is not None:
plot_best_networks(bests_data, baseline_data, bests_output_path)
def main():
output_path = os.path.join(HERE, "outputs")
create_dir(output_path)
experiments_path = os.path.join(HERE, "experiments.yaml")
experiments = yaml_load(experiments_path)
for dataset in experiments:
dataset_path = os.path.join(output_path, dataset)
create_dir(dataset_path)
for experiment in experiments[dataset]["experiments"]:
experiment_path = os.path.join(dataset_path, experiment)
create_dir(experiment_path)
generate_plot_from_exp(dataset, experiment, output_path=experiment_path)
# comparative plots between experiments of the same dataset
generate_plot_from_dataset(dataset, experiments[dataset], output_path=dataset_path)
horovodrun -np 4 python -m deephyper.benchmark.nas.covertype.train """ import os from deephyper.nas.run.tf_distributed import run from deephyper.nas.run.util import create_dir from nas_big_data.combo.load_data import load_data_test from nas_big_data.combo.problem_agebo import Problem os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(i) for i in range(4)]) HERE = os.path.dirname(os.path.abspath(__file__)) output_dir = os.path.join(HERE, __file__[:-3]) create_dir(output_dir) Problem.load_data(load_data_test) config = Problem.space config["log_dir"] = output_dir config["hyperparameters"]["num_epochs"] = 200 config["hyperparameters"]["verbose"] = 1 config["hyperparameters"]["learning_rate"] = 0.0003233381265748 config["hyperparameters"]["batch_size"] = 166 config["hyperparameters"]["optimizer"] = "adam" config["hyperparameters"]["patience_ReduceLROnPlateau"] = 10 config["hyperparameters"]["patience_EarlyStopping"] = 29 config["loss"] = "mae"