def optim_dehb(args):
if args.runcount_limit == 0:
args.runcount_limit = None
f = snake_from_config_wrapper("DEHB", args.num_opponents,
args.num_games_per_eval, args.timeout)
dehb = DEHB(f=f,
cs=cs,
dimensions=len(cs.get_hyperparameters()),
min_budget=1,
max_budget=2,
n_workers=args.n_jobs,
output_path=args.output_dir,
save_smac=True)
def_value = f(cs.get_default_configuration())
print(
f"Default Configuration evaluates to a win percentage of {(1 - def_value['fitness']) * 100:.2f}%"
)
print(f"Starting Opimization of {args.runcount_limit} configurations...")
trajectory, runtime, history = dehb.run(
fevals=args.runcount_limit,
total_cost=args.walltime,
verbose=True,
# parameters expected as **kwargs in target_function is passed here (surpassed by wrapper)
)
best_config = dehb.vector_to_configspace(dehb.inc_config)
inc_value = f(best_config)
print(
f"Optimized Configuration {best_config} evaluates to a win percentage of {(1 - inc_value['fitness']) * 100:.2f}%"
)
config,
budget=int(budget))
else:
fitness, cost = search_space.objective_function(nasbench, config)
fitness = 1 - fitness
return fitness, cost
dehbs = {None: DEHB, "0": DEHB_0, "1": DEHB_1, "2": DEHB_2, "3": DEHB_3}
DEHB = dehbs[args.version]
# Initializing DEHB object
dehb = DEHB(cs=cs,
dimensions=dimensions,
f=f,
strategy=args.strategy,
mutation_factor=args.mutation_factor,
crossover_prob=args.crossover_prob,
eta=args.eta,
min_budget=min_budget,
max_budget=max_budget,
generations=args.gens)
if args.runs is None: # for a single run
if not args.fix_seed:
np.random.seed(0)
# Running DEHB iterations
traj, runtime, history = dehb.run(iterations=args.iter,
verbose=args.verbose)
fh = open(
os.path.join(
output_path,
'DEHB_{}_ssp_{}_seed_0.obj'.format(args.run_id, space)), 'wb')
def main():
args = input_arguments()
use_cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
torch.manual_seed(args.seed)
# Data Preparation
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
train_set = torchvision.datasets.MNIST(root='./data',
train=True,
download=True,
transform=transform)
train_set, valid_set = torch.utils.data.random_split(
train_set, [50000, 10000])
test_set = torchvision.datasets.MNIST(root='./data',
train=False,
download=True,
transform=transform)
# Get configuration space
cs = get_configspace(args.seed)
dimensions = len(cs.get_hyperparameters())
# Some insights into Dask interfaces to DEHB and handling GPU devices for parallelism:
# * if args.scheduler_file is specified, args.n_workers need not be specifed --- since
# args.scheduler_file indicates a Dask client/server is active
# * if args.scheduler_file is not specified and args.n_workers > 1 --- the DEHB object
# creates a Dask client as at instantiation and dies with the associated DEHB object
# * if args.single_node_with_gpus is True --- assumes that all GPU devices indicated
# through the environment variable "CUDA_VISIBLE_DEVICES" resides on the same machine
# Dask checks and setups
single_node_with_gpus = args.single_node_with_gpus
if args.scheduler_file is not None and os.path.isfile(args.scheduler_file):
client = Client(scheduler_file=args.scheduler_file)
# explicitly delegating GPU handling to Dask workers defined
single_node_with_gpus = False
else:
client = None
###########################
# DEHB optimisation block #
###########################
np.random.seed(args.seed)
dehb = DEHB(
f=objective_function,
cs=cs,
dimensions=dimensions,
min_budget=args.min_budget,
max_budget=args.max_budget,
eta=args.eta,
output_path=args.output_path,
# if client is not None and of type Client, n_workers is ignored
# if client is None, a Dask client with n_workers is set up
client=client,
n_workers=args.n_workers)
traj, runtime, history = dehb.run(
total_cost=args.runtime,
verbose=args.verbose,
# arguments below are part of **kwargs shared across workers
train_set=train_set,
valid_set=valid_set,
test_set=test_set,
single_node_with_gpus=single_node_with_gpus,
device=device)
# end of DEHB optimisation
# Saving optimisation trace history
name = time.strftime("%x %X %Z", time.localtime(dehb.start))
name = name.replace("/", '-').replace(":", '-').replace(" ", '_')
dehb.logger.info("Saving optimisation trace history...")
with open(os.path.join(args.output_path, "history_{}.pkl".format(name)),
"wb") as f:
pickle.dump(history, f)
# Retrain and evaluate best found configuration
if args.refit_training:
dehb.logger.info(
"Retraining on complete training data to compute test metrics...")
train_set = torchvision.datasets.MNIST(root='./data',
train=True,
download=True,
transform=transform)
incumbent = dehb.vector_to_configspace(dehb.inc_config)
acc = train_and_evaluate(incumbent,
args.max_budget,
verbose=True,
train_set=train_set,
test_set=test_set,
device=device)
dehb.logger.info(
"Test accuracy of {:.3f} for the best found configuration: ".
format(acc))
dehb.logger.info(incumbent)
os.makedirs(str(output_path), exist_ok=True)
benchmark = TabularBenchmark(data_dir=args.data_dir,
model=args.model,
task_id=args.task_id)
loss_df, cost_df = create_datasets_from_benchmark(benchmark)
benchmark = TabularBenchmark(data_dir=args.data_dir,
model="rf",
task_id=args.task_id)
fidelity = benchmark.original_fs.get_hyperparameter("n_estimators")
min_budget, max_budget = fidelity.lower, fidelity.upper
dehb = DEHB(cs=benchmark.original_cs,
f=target_function,
min_budget=min_budget,
max_budget=max_budget,
n_workers=1,
output_path="./dehb_output/")
fevals = args.fevals
# Loss fitting
train_X, train_y, valid_X, valid_y = create_splits(loss_df,
val_size=0.1,
seed=1)
_ = dehb.run(
fevals=fevals,
save_history=False,
save_intermediate=False,
verbose=True,
# kwargs
train_X=train_X,
)
task_id = benchmark.exp_args['task_id']
space = benchmark.exp_args['space']
max_fidelity = benchmark.get_max_fidelity()
fidelity_info = benchmark.get_fidelity_range()
if len(fidelity_info) > 1:
assert "Supports only multi-fidelity (1-d) and not multi-multi-fidelity (>1-d)!"
fidelity_name, min_budget, max_budget = fidelity_info[0]
global_best = benchmark.get_global_min("val")["acc"]
eval_type = "test" if args.test else "val"
output_path = os.path.join(args.output_path, space, str(task_id))
os.makedirs(output_path, exist_ok=True)
dehb = DEHB(
f=target_function, cs=benchmark.configuration_space,
min_budget=min_budget, max_budget=max_budget, eta=3,
n_workers=1, output_path="./dehb_dump"
)
trace, costs, full_trace = dehb.run(
fevals=args.fevals, verbose=args.verbose, save_history=False, save_intermediate=False,
# **kwargs list
benchmark=benchmark, metric=args.metric, test=args.test, fidelity_name=fidelity_name
)
plt.tight_layout()
plt.plot(np.cumsum(costs), np.array(trace) - global_best, label="DEHB")
plt.xscale("log")
plt.yscale("log")
plt.title("Task ID {} for {}".format(task_id, space))
plt.xlabel("Wallclock time in seconds")
plt.ylabel("Loss regret")