def test_build_new_GreaterThanIntCondition(self):
expected = "a real [0.0, 1.0] [0.5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 0.5"
cs = ConfigurationSpace()
a = UniformFloatHyperparameter("a", 0, 1, 0.5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 0.5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
expected = "a integer [0, 10] [5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 5"
cs = ConfigurationSpace()
a = UniformIntegerHyperparameter("a", 0, 10, 5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_build_new_GreaterThanIntCondition(self):
expected = "a real [0.0, 1.0] [0.5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 0.5"
cs = ConfigurationSpace()
a = UniformFloatHyperparameter("a", 0, 1, 0.5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 0.5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
expected = "a integer [0, 10] [5]\n" \
"b integer [0, 10] [5]\n\n" \
"b | a > 5"
cs = ConfigurationSpace()
a = UniformIntegerHyperparameter("a", 0, 10, 5)
b = UniformIntegerHyperparameter("b", 0, 10, 5)
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
cond = GreaterThanCondition(b, a, 5)
cs.add_condition(cond)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_q_float(self):
expected = "Q16_float_a real [16.0, 1024.0] [520.0]"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformFloatHyperparameter("float_a", 16, 1024, q=16))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_log10(self):
expected = "a real [10.0, 1000.0] [100.0]log"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformFloatHyperparameter("a", 10, 1000, log=True))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_log10(self):
expected = "a real [10.0, 1000.0] [100.0]log"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformFloatHyperparameter("a", 10, 1000, log=True))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_q_float(self):
expected = "Q16_float_a real [16.0, 1024.0] [520.0]"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformFloatHyperparameter("float_a", 16, 1024, q=16))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_q_int(self):
expected = "Q16_int_a integer [16, 1024] [520]"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("int_a", 16, 1024, q=16))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_new_q_int(self):
expected = "Q16_int_a integer [16, 1024] [520]"
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("int_a", 16, 1024, q=16))
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def cache_runhistory_configspace(save_folder, flow_id, task_id, model_type, required_setups, reverse=False, fixed_parameters=None, ignore_parameters=None):
if fixed_parameters:
save_folder_suffix = [param + '_' + value for param, value in fixed_parameters.items()]
save_folder_suffix = '/' + '__'.join(save_folder_suffix)
else:
save_folder_suffix = '/vanilla'
runhistory_path = save_folder + save_folder_suffix + '/runhistory.json'
configspace_path = save_folder + save_folder_suffix + '/config_space.pcs'
print(runhistory_path, configspace_path)
if not os.path.isfile(runhistory_path) or not os.path.isfile(configspace_path):
runhistory, configspace = openmlpimp.utils.obtain_runhistory_and_configspace(flow_id, task_id, model_type,
required_setups=required_setups,
fixed_parameters=fixed_parameters,
ignore_parameters=ignore_parameters,
reverse=reverse)
os.makedirs(save_folder + save_folder_suffix, exist_ok=True)
with open(runhistory_path, 'w') as outfile:
json.dump(runhistory, outfile, indent=2)
with open(configspace_path, 'w') as outfile:
outfile.write(write(configspace))
else:
print('[Obtained from cache]')
# now the files are guaranteed to exists
return runhistory_path, configspace_path
def test_write_restrictions(self):
s = "c integer [0, 2] [0]\n" + \
"d ordinal {cold, luke-warm, hot} [cold]\n" + \
"e real [0.0, 1.0] [0.0]\n" + \
"b real [0.0, 1.0] [0.0]\n" + \
"a real [0.0, 1.0] [0.0]\n" + \
"\n" + \
"b | d in {luke-warm, hot} || c > 1\n" + \
"a | b == 0.5 && e > 0.5"
a = pcs_new.read(s.split('\n'))
out = pcs_new.write(a)
self.assertEqual(out, s)
def test_build_new_forbidden(self):
expected = "a categorical {a, b, c} [a]\nb categorical {a, b, c} [c]\n\n" \
"{a=a, b=a}\n{a=a, b=b}\n{a=b, b=a}\n{a=b, b=b}\n"
cs = ConfigurationSpace()
a = CategoricalHyperparameter("a", ["a", "b", "c"], "a")
b = CategoricalHyperparameter("b", ["a", "b", "c"], "c")
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
fb = ForbiddenAndConjunction(ForbiddenInClause(a, ["a", "b"]),
ForbiddenInClause(b, ["a", "b"]))
cs.add_forbidden_clause(fb)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def test_write_restrictions(self):
s = "c integer [0, 2] [0]\n" + \
"d ordinal {cold, luke-warm, hot} [cold]\n" + \
"e real [0.0, 1.0] [0.0]\n" + \
"b real [0.0, 1.0] [0.0]\n" + \
"a real [0.0, 1.0] [0.0]\n" + \
"\n" + \
"b | d in {luke-warm, hot} || c > 1\n" + \
"a | b == 0.5 && e > 0.5"
a = pcs_new.read(s.split('\n'))
out = pcs_new.write(a)
self.assertEqual(out, s)
def test_build_new_forbidden(self):
expected = "a categorical {a, b, c} [a]\nb categorical {a, b, c} [c]\n\n" \
"{a=a, b=a}\n{a=a, b=b}\n{a=b, b=a}\n{a=b, b=b}\n"
cs = ConfigurationSpace()
a = CategoricalHyperparameter("a", ["a", "b", "c"], "a")
b = CategoricalHyperparameter("b", ["a", "b", "c"], "c")
cs.add_hyperparameter(a)
cs.add_hyperparameter(b)
fb = ForbiddenAndConjunction(ForbiddenInClause(a, ["a", "b"]),
ForbiddenInClause(b, ["a", "b"]))
cs.add_forbidden_clause(fb)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def run_experiment(args,
worker,
dest_dir,
smac_deterministic,
store_all_runs=False):
print("Running experiment (args: %s)" % str(args))
# make sure the working and dest directory exist
os.makedirs(args.working_directory, exist_ok=True)
os.makedirs(dest_dir, exist_ok=True)
if args.opt_method in ['randomsearch', 'bohb', 'hyperband']:
print("Using hpbandster-optimizer (%s)" % args.opt_method)
# Every process has to lookup the hostname
host = hpns.nic_name_to_host(args.nic_name)
print("Host: %s" % str(host))
# setup a nameserver
NS = hpns.NameServer(run_id=args.run_id,
nic_name=args.nic_name,
port=0,
host=host,
working_directory=args.working_directory)
ns_host, ns_port = NS.start()
print("Initialized nameserver (ns_host: %s; ns_port: %s)" %
(str(ns_host), str(ns_port)))
if args.worker:
print("This is a pure worker-thread.")
worker = get_worker(args, host=host)
worker.load_nameserver_credentials(
working_directory=args.working_directory)
worker.run(background=False)
print("Exiting...")
exit(0)
print(
"This is the name-server thread, however there will be a worker running in the background."
)
worker = get_worker(args, host=host)
# start worker in the background
worker.load_nameserver_credentials(
working_directory=args.working_directory)
worker.run(background=True)
if args.exp_name == 'paramnet_surrogates':
print(
"This is the paramnet_surrogates experiment, so any custom budgets will be replaced by the "
"dataset-specific budgets.")
args.min_budget, args.max_budget = worker.budgets[
args.dataset_paramnet_surrogates]
print(
"Background-worker is running, grabbing configspace from worker and initializing result_logger "
"(with dest_dir %s)" % dest_dir)
configspace = worker.configspace
result_logger = hpres.json_result_logger(directory=dest_dir,
overwrite=True)
print("Getting optimizer.")
opt = get_optimizer(
args,
configspace,
working_directory=args.working_directory,
run_id=args.run_id,
min_budget=args.min_budget,
max_budget=args.max_budget,
host=host,
nameserver=ns_host,
nameserver_port=ns_port,
ping_interval=30,
result_logger=result_logger,
)
print("Initialization successful, starting optimization.")
from ConfigSpace.read_and_write import pcs_new
with open(os.path.join(dest_dir, 'configspace.pcs'), 'w') as fh:
fh.write(pcs_new.write(opt.config_generator.configspace))
result = opt.run(n_iterations=args.num_iterations,
min_n_workers=args.n_workers)
print("Finished optimization")
# shutdown the worker and the dispatcher
opt.shutdown(shutdown_workers=True)
NS.shutdown()
if args.exp_name == 'paramnet_surrogates':
# This if block is necessary to set budgets for paramnet_surrogates - for nothing else
args_tmp = copy.deepcopy(args)
args_tmp.opt_method = 'bohb'
worker = get_worker(args_tmp)
args.min_budget, args.max_budget = worker.budgets[
args.dataset_paramnet_surrogates]
# the number of iterations for the blackbox optimizers must be increased so they have comparable total budgets
bb_iterations = int(args.num_iterations *
(1 +
(np.log(args.max_budget) - np.log(args.min_budget)) /
np.log(args.eta)))
#if args.opt_method == 'tpe':
# result = worker.run_tpe(bb_iterations)
if args.opt_method == 'smac':
result = worker.run_smac(bb_iterations,
deterministic=smac_deterministic,
working_directory=args.dest_dir)
if result is None:
raise ValueError("Unknown method %s!" % args.method)
return result
def test_write_new_log_int(self):
expected = "int_log_a integer [1, 6] [2]log"
cs = ConfigurationSpace()
cs.add_hyperparameter(int_log_a)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
def fit(self, pipeline_config, data_manager, instance, autonet, run_number,
run_id, task_id):
instance_name, run_name = get_names(instance, run_id, run_number)
run_result_dir = get_run_result_dir(pipeline_config, instance, run_id,
run_number, autonet)
instance_run_id = str(run_name) + "-" + str(instance_name)
instance_run_id = '_'.join(instance_run_id.split(':'))
autonet.autonet_config = None #clean results of last fit
autonet.update_autonet_config(task_id=task_id,
run_id=instance_run_id,
result_logger_dir=run_result_dir)
if (task_id not in [-1, 1]):
return {'result_dir': run_result_dir}
if not os.path.exists(run_result_dir):
try:
os.makedirs(run_result_dir)
except Exception as e:
print(e)
logging.getLogger('benchmark').debug(
"Create config and info files for current run " + str(run_name))
instance_info = dict()
instance_info['path'] = instance
instance_info['is_classification'] = data_manager.is_classification
instance_info['is_multilabel'] = data_manager.is_multilabel
instance_info['instance_shape'] = data_manager.X_train.shape
instance_info[
'categorical_features'] = data_manager.categorical_features
autonet_config = autonet.get_current_autonet_config()
if autonet_config["hyperparameter_search_space_updates"] is not None:
autonet_config["hyperparameter_search_space_updates"].save_as_file(
os.path.join(run_result_dir,
"hyperparameter_search_space_updates.txt"))
if 'user_updates_config' in pipeline_config:
user_updates_config = pipeline_config['user_updates_config']
if user_updates_config:
from shutil import copyfile
copyfile(
user_updates_config,
os.path.join(run_result_dir, 'user_updates_config.csv'))
self.write_config_to_file(run_result_dir, "instance.info",
instance_info)
self.write_config_to_file(run_result_dir, "benchmark.config",
pipeline_config)
self.write_config_to_file(run_result_dir, "autonet.config",
autonet_config)
# save refit config - add indent and sort keys
if 'refit_config' in pipeline_config and pipeline_config[
'refit_config'] is not None:
import json
with open(pipeline_config['refit_config'], 'r') as f:
refit_config = json.loads(f.read())
with open(os.path.join(run_result_dir, 'refit_config.json'),
'w+') as f:
f.write(json.dumps(refit_config, indent=4, sort_keys=True))
# save search space
search_space = autonet.pipeline.get_hyperparameter_search_space(
**autonet_config)
with open(os.path.join(run_result_dir, "configspace.json"), "w") as f:
f.write(cs_json.write(search_space))
# save search space without constants - used by bohb - as pcs (simple)
simplified_search_space, _ = remove_constant_hyperparameter(
search_space)
with open(os.path.join(run_result_dir, "configspace_simple.pcs"),
"w") as f:
f.write(cs_pcs.write(simplified_search_space))
return {'result_dir': run_result_dir}
def test_write_new_log_int(self):
expected = "int_log_a integer [1, 6] [2]log"
cs = ConfigurationSpace()
cs.add_hyperparameter(int_log_a)
value = pcs_new.write(cs)
self.assertEqual(expected, value)
