def test_build_new_GreaterThanIntCondition(self): expected = "a real [0.0, 1.0] [0.5]\n" \ "b integer [0, 10] [5]\n\n" \ "b | a > 0.5" cs = ConfigurationSpace() a = UniformFloatHyperparameter("a", 0, 1, 0.5) b = UniformIntegerHyperparameter("b", 0, 10, 5) cs.add_hyperparameter(a) cs.add_hyperparameter(b) cond = GreaterThanCondition(b, a, 0.5) cs.add_condition(cond) value = pcs_new.write(cs) self.assertEqual(expected, value) expected = "a integer [0, 10] [5]\n" \ "b integer [0, 10] [5]\n\n" \ "b | a > 5" cs = ConfigurationSpace() a = UniformIntegerHyperparameter("a", 0, 10, 5) b = UniformIntegerHyperparameter("b", 0, 10, 5) cs.add_hyperparameter(a) cs.add_hyperparameter(b) cond = GreaterThanCondition(b, a, 5) cs.add_condition(cond) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_build_new_GreaterThanIntCondition(self): expected = "a real [0.0, 1.0] [0.5]\n" \ "b integer [0, 10] [5]\n\n" \ "b | a > 0.5" cs = ConfigurationSpace() a = UniformFloatHyperparameter("a", 0, 1, 0.5) b = UniformIntegerHyperparameter("b", 0, 10, 5) cs.add_hyperparameter(a) cs.add_hyperparameter(b) cond = GreaterThanCondition(b, a, 0.5) cs.add_condition(cond) value = pcs_new.write(cs) self.assertEqual(expected, value) expected = "a integer [0, 10] [5]\n" \ "b integer [0, 10] [5]\n\n" \ "b | a > 5" cs = ConfigurationSpace() a = UniformIntegerHyperparameter("a", 0, 10, 5) b = UniformIntegerHyperparameter("b", 0, 10, 5) cs.add_hyperparameter(a) cs.add_hyperparameter(b) cond = GreaterThanCondition(b, a, 5) cs.add_condition(cond) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_write_new_q_float(self): expected = "Q16_float_a real [16.0, 1024.0] [520.0]" cs = ConfigurationSpace() cs.add_hyperparameter( UniformFloatHyperparameter("float_a", 16, 1024, q=16)) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_write_new_log10(self): expected = "a real [10.0, 1000.0] [100.0]log" cs = ConfigurationSpace() cs.add_hyperparameter( UniformFloatHyperparameter("a", 10, 1000, log=True)) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_write_new_log10(self): expected = "a real [10.0, 1000.0] [100.0]log" cs = ConfigurationSpace() cs.add_hyperparameter( UniformFloatHyperparameter("a", 10, 1000, log=True)) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_write_new_q_float(self): expected = "Q16_float_a real [16.0, 1024.0] [520.0]" cs = ConfigurationSpace() cs.add_hyperparameter( UniformFloatHyperparameter("float_a", 16, 1024, q=16)) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_write_new_q_int(self): expected = "Q16_int_a integer [16, 1024] [520]" cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("int_a", 16, 1024, q=16)) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_write_new_q_int(self): expected = "Q16_int_a integer [16, 1024] [520]" cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("int_a", 16, 1024, q=16)) value = pcs_new.write(cs) self.assertEqual(expected, value)
def cache_runhistory_configspace(save_folder, flow_id, task_id, model_type, required_setups, reverse=False, fixed_parameters=None, ignore_parameters=None): if fixed_parameters: save_folder_suffix = [param + '_' + value for param, value in fixed_parameters.items()] save_folder_suffix = '/' + '__'.join(save_folder_suffix) else: save_folder_suffix = '/vanilla' runhistory_path = save_folder + save_folder_suffix + '/runhistory.json' configspace_path = save_folder + save_folder_suffix + '/config_space.pcs' print(runhistory_path, configspace_path) if not os.path.isfile(runhistory_path) or not os.path.isfile(configspace_path): runhistory, configspace = openmlpimp.utils.obtain_runhistory_and_configspace(flow_id, task_id, model_type, required_setups=required_setups, fixed_parameters=fixed_parameters, ignore_parameters=ignore_parameters, reverse=reverse) os.makedirs(save_folder + save_folder_suffix, exist_ok=True) with open(runhistory_path, 'w') as outfile: json.dump(runhistory, outfile, indent=2) with open(configspace_path, 'w') as outfile: outfile.write(write(configspace)) else: print('[Obtained from cache]') # now the files are guaranteed to exists return runhistory_path, configspace_path
def test_write_restrictions(self): s = "c integer [0, 2] [0]\n" + \ "d ordinal {cold, luke-warm, hot} [cold]\n" + \ "e real [0.0, 1.0] [0.0]\n" + \ "b real [0.0, 1.0] [0.0]\n" + \ "a real [0.0, 1.0] [0.0]\n" + \ "\n" + \ "b | d in {luke-warm, hot} || c > 1\n" + \ "a | b == 0.5 && e > 0.5" a = pcs_new.read(s.split('\n')) out = pcs_new.write(a) self.assertEqual(out, s)
def test_build_new_forbidden(self): expected = "a categorical {a, b, c} [a]\nb categorical {a, b, c} [c]\n\n" \ "{a=a, b=a}\n{a=a, b=b}\n{a=b, b=a}\n{a=b, b=b}\n" cs = ConfigurationSpace() a = CategoricalHyperparameter("a", ["a", "b", "c"], "a") b = CategoricalHyperparameter("b", ["a", "b", "c"], "c") cs.add_hyperparameter(a) cs.add_hyperparameter(b) fb = ForbiddenAndConjunction(ForbiddenInClause(a, ["a", "b"]), ForbiddenInClause(b, ["a", "b"])) cs.add_forbidden_clause(fb) value = pcs_new.write(cs) self.assertEqual(expected, value)
def test_write_restrictions(self): s = "c integer [0, 2] [0]\n" + \ "d ordinal {cold, luke-warm, hot} [cold]\n" + \ "e real [0.0, 1.0] [0.0]\n" + \ "b real [0.0, 1.0] [0.0]\n" + \ "a real [0.0, 1.0] [0.0]\n" + \ "\n" + \ "b | d in {luke-warm, hot} || c > 1\n" + \ "a | b == 0.5 && e > 0.5" a = pcs_new.read(s.split('\n')) out = pcs_new.write(a) self.assertEqual(out, s)
def test_build_new_forbidden(self): expected = "a categorical {a, b, c} [a]\nb categorical {a, b, c} [c]\n\n" \ "{a=a, b=a}\n{a=a, b=b}\n{a=b, b=a}\n{a=b, b=b}\n" cs = ConfigurationSpace() a = CategoricalHyperparameter("a", ["a", "b", "c"], "a") b = CategoricalHyperparameter("b", ["a", "b", "c"], "c") cs.add_hyperparameter(a) cs.add_hyperparameter(b) fb = ForbiddenAndConjunction(ForbiddenInClause(a, ["a", "b"]), ForbiddenInClause(b, ["a", "b"])) cs.add_forbidden_clause(fb) value = pcs_new.write(cs) self.assertEqual(expected, value)
def run_experiment(args, worker, dest_dir, smac_deterministic, store_all_runs=False): print("Running experiment (args: %s)" % str(args)) # make sure the working and dest directory exist os.makedirs(args.working_directory, exist_ok=True) os.makedirs(dest_dir, exist_ok=True) if args.opt_method in ['randomsearch', 'bohb', 'hyperband']: print("Using hpbandster-optimizer (%s)" % args.opt_method) # Every process has to lookup the hostname host = hpns.nic_name_to_host(args.nic_name) print("Host: %s" % str(host)) # setup a nameserver NS = hpns.NameServer(run_id=args.run_id, nic_name=args.nic_name, port=0, host=host, working_directory=args.working_directory) ns_host, ns_port = NS.start() print("Initialized nameserver (ns_host: %s; ns_port: %s)" % (str(ns_host), str(ns_port))) if args.worker: print("This is a pure worker-thread.") worker = get_worker(args, host=host) worker.load_nameserver_credentials( working_directory=args.working_directory) worker.run(background=False) print("Exiting...") exit(0) print( "This is the name-server thread, however there will be a worker running in the background." ) worker = get_worker(args, host=host) # start worker in the background worker.load_nameserver_credentials( working_directory=args.working_directory) worker.run(background=True) if args.exp_name == 'paramnet_surrogates': print( "This is the paramnet_surrogates experiment, so any custom budgets will be replaced by the " "dataset-specific budgets.") args.min_budget, args.max_budget = worker.budgets[ args.dataset_paramnet_surrogates] print( "Background-worker is running, grabbing configspace from worker and initializing result_logger " "(with dest_dir %s)" % dest_dir) configspace = worker.configspace result_logger = hpres.json_result_logger(directory=dest_dir, overwrite=True) print("Getting optimizer.") opt = get_optimizer( args, configspace, working_directory=args.working_directory, run_id=args.run_id, min_budget=args.min_budget, max_budget=args.max_budget, host=host, nameserver=ns_host, nameserver_port=ns_port, ping_interval=30, result_logger=result_logger, ) print("Initialization successful, starting optimization.") from ConfigSpace.read_and_write import pcs_new with open(os.path.join(dest_dir, 'configspace.pcs'), 'w') as fh: fh.write(pcs_new.write(opt.config_generator.configspace)) result = opt.run(n_iterations=args.num_iterations, min_n_workers=args.n_workers) print("Finished optimization") # shutdown the worker and the dispatcher opt.shutdown(shutdown_workers=True) NS.shutdown() if args.exp_name == 'paramnet_surrogates': # This if block is necessary to set budgets for paramnet_surrogates - for nothing else args_tmp = copy.deepcopy(args) args_tmp.opt_method = 'bohb' worker = get_worker(args_tmp) args.min_budget, args.max_budget = worker.budgets[ args.dataset_paramnet_surrogates] # the number of iterations for the blackbox optimizers must be increased so they have comparable total budgets bb_iterations = int(args.num_iterations * (1 + (np.log(args.max_budget) - np.log(args.min_budget)) / np.log(args.eta))) #if args.opt_method == 'tpe': # result = worker.run_tpe(bb_iterations) if args.opt_method == 'smac': result = worker.run_smac(bb_iterations, deterministic=smac_deterministic, working_directory=args.dest_dir) if result is None: raise ValueError("Unknown method %s!" % args.method) return result
def test_write_new_log_int(self): expected = "int_log_a integer [1, 6] [2]log" cs = ConfigurationSpace() cs.add_hyperparameter(int_log_a) value = pcs_new.write(cs) self.assertEqual(expected, value)
def fit(self, pipeline_config, data_manager, instance, autonet, run_number, run_id, task_id): instance_name, run_name = get_names(instance, run_id, run_number) run_result_dir = get_run_result_dir(pipeline_config, instance, run_id, run_number, autonet) instance_run_id = str(run_name) + "-" + str(instance_name) instance_run_id = '_'.join(instance_run_id.split(':')) autonet.autonet_config = None #clean results of last fit autonet.update_autonet_config(task_id=task_id, run_id=instance_run_id, result_logger_dir=run_result_dir) if (task_id not in [-1, 1]): return {'result_dir': run_result_dir} if not os.path.exists(run_result_dir): try: os.makedirs(run_result_dir) except Exception as e: print(e) logging.getLogger('benchmark').debug( "Create config and info files for current run " + str(run_name)) instance_info = dict() instance_info['path'] = instance instance_info['is_classification'] = data_manager.is_classification instance_info['is_multilabel'] = data_manager.is_multilabel instance_info['instance_shape'] = data_manager.X_train.shape instance_info[ 'categorical_features'] = data_manager.categorical_features autonet_config = autonet.get_current_autonet_config() if autonet_config["hyperparameter_search_space_updates"] is not None: autonet_config["hyperparameter_search_space_updates"].save_as_file( os.path.join(run_result_dir, "hyperparameter_search_space_updates.txt")) if 'user_updates_config' in pipeline_config: user_updates_config = pipeline_config['user_updates_config'] if user_updates_config: from shutil import copyfile copyfile( user_updates_config, os.path.join(run_result_dir, 'user_updates_config.csv')) self.write_config_to_file(run_result_dir, "instance.info", instance_info) self.write_config_to_file(run_result_dir, "benchmark.config", pipeline_config) self.write_config_to_file(run_result_dir, "autonet.config", autonet_config) # save refit config - add indent and sort keys if 'refit_config' in pipeline_config and pipeline_config[ 'refit_config'] is not None: import json with open(pipeline_config['refit_config'], 'r') as f: refit_config = json.loads(f.read()) with open(os.path.join(run_result_dir, 'refit_config.json'), 'w+') as f: f.write(json.dumps(refit_config, indent=4, sort_keys=True)) # save search space search_space = autonet.pipeline.get_hyperparameter_search_space( **autonet_config) with open(os.path.join(run_result_dir, "configspace.json"), "w") as f: f.write(cs_json.write(search_space)) # save search space without constants - used by bohb - as pcs (simple) simplified_search_space, _ = remove_constant_hyperparameter( search_space) with open(os.path.join(run_result_dir, "configspace_simple.pcs"), "w") as f: f.write(cs_pcs.write(simplified_search_space)) return {'result_dir': run_result_dir}
def test_write_new_log_int(self): expected = "int_log_a integer [1, 6] [2]log" cs = ConfigurationSpace() cs.add_hyperparameter(int_log_a) value = pcs_new.write(cs) self.assertEqual(expected, value)