def get_method_options(prob, capital_type): """ Returns a dictionary of method options. """ methods = prob.methods all_method_options = {} for meth in methods: # wrap up curr_options = load_options(blackbox_opt_args) if meth in ['hyperopt', 'smac', 'gpyopt', 'pdoo']: curr_options.redo_evals_for_true_val = True if meth == 'hyperopt': curr_options.algo = tpe.suggest curr_options.space = hp.uniform elif meth == 'spearmint': e1_file_path = os.path.dirname(os.path.realpath(__file__)) curr_options.redo_evals_for_true_val = True curr_options.noisy = NOISY_EVALS curr_options.exp_dir = os.path.join( e1_file_path, 'Spearmint/' + prob.study_name.split('-')[0]) print(curr_options.exp_dir) curr_options.pkg_dir = '/zfsauton3/home/kkandasa/projects/Boss/Spearmint/spearmint' curr_options.pkg_dir = 'Insert-location-of-spearmint-dir-here' elif meth.startswith('dragonfly-mf'): euc_mf_gpb_args = get_all_mf_euc_gp_bandit_args() curr_options = load_options(euc_mf_gpb_args) elif meth == 'dragonfly': euc_gpb_args = get_all_euc_gp_bandit_args() curr_options = load_options(euc_gpb_args) else: # Assuming its one of ml, ml+post_sampling etc. euc_gpb_args = get_all_euc_gp_bandit_args() curr_options = load_options(euc_gpb_args) curr_options.capital_type = capital_type # Add to all_method_options all_method_options[meth] = curr_options return all_method_options
def get_method_options(prob, capital_type): """ Returns the method options. """ methods = prob.methods all_method_options = {} for meth in methods: if meth == 'rand': curr_options = load_options(cp_random_optimiser_args) elif meth == 'ga': curr_options = load_options(cpga_opt_args) elif meth.startswith('dragonfly'): if meth.startswith('dragonfly-mf'): curr_options = load_options(get_all_mf_cp_gp_bandit_args()) else: curr_options = load_options(get_all_cp_gp_bandit_args()) meth_parts = meth.split('_') if len(meth_parts) == 2: curr_options.acq_opt_method = meth_parts[-1] elif meth in ['hyperopt', 'gpyopt', 'smac']: curr_options = load_options(blackbox_opt_args) curr_options = Namespace(redo_evals_for_true_val=False) else: raise ValueError('Unknown method %s.'%(meth)) curr_options.capital_type = capital_type all_method_options[meth] = curr_options return all_method_options
def get_method_options(prob, capital_type): """ Returns the method options. """ methods = prob.methods all_method_options = {} for meth in methods: if meth == 'rand': curr_options = load_options(cp_random_optimiser_args) elif meth == 'ga': curr_options = load_options(cpga_opt_args) elif meth.startswith('dragonfly'): if meth.startswith('dragonfly-mf'): curr_options = load_options(get_all_mf_cp_gp_bandit_args()) else: curr_options = load_options(get_all_cp_gp_bandit_args()) meth_parts = meth.split('_') if len(meth_parts) == 2: curr_options.acq_opt_method = meth_parts[-1] elif meth in ['hyperopt', 'gpyopt', 'smac', 'spearmint']: curr_options = load_options(blackbox_opt_args) curr_options = Namespace(redo_evals_for_true_val=True) else: raise ValueError('Unknown method %s.' % (meth)) # Some additional data we will need for Spearmint if meth == 'spearmint': # Swap in the following after you download and install Spearmint curr_options.exp_dir = '/home/karun/boss/e3_cp/Spearmint/' + \ prob.study_name.split('-')[0] curr_options.pkg_dir = '/home/karun/Spearmint/spearmint' curr_options.capital_type = capital_type all_method_options[meth] = curr_options return all_method_options
def build_local_cp_ga_optimiser(func_caller, cp_domain, nn_mutate_op, worker_manager, max_capital, mode='asy', orderings=None, options=None, reporter="silent"): """ A GA optimiser on Cartesian product space from the function caller. """ log = logging.getLogger("LocalGA Optimiser") if not isinstance(func_caller, ExperimentCaller): func_caller = CPFunctionCaller(func_caller, cp_domain, domain_orderings=orderings) options = load_options(blackbox_opt_args, partial_options=options) options.mode = mode options.capital_type = 'return_value' start_time = timer() log.info( f"Starting GA optimisation with capital {max_capital} ({options.capital_type})." ) result = LocalCPGAOptimiser(func_caller, nn_mutate_op, worker_manager, options=options, reporter=reporter).optimise(max_capital) end_time = timer() log.info(f"GA optimisation finished, took {(end_time - start_time):.2f}s.") return result
def __init__(self, func_caller, nn_morphs_op, worker_manager=None, options=None, reporter=None, ask_tell_mode=False): """ Constructor. """ self.nn_morphs_op = nn_morphs_op options = load_options(blackbox_opt_args, partial_options=options) assert options.prev_evaluations is not None self.pivots = [ qinfo.point for qinfo in options.prev_evaluations.qinfos ] options.prev_evaluations = None options.init_capital = 0 super().__init__(func_caller, worker_manager, model=None, options=options, reporter=reporter, ask_tell_mode=ask_tell_mode) self.nn_idx = \ [dom.get_type() for dom in self.domain.list_of_domains].index("neural_network") self.single_mutation_ops = \ [get_default_mutation_op(dom) for dom in self.domain.list_of_domains] self.points_to_evaluate = None
def __init__(self, func_caller, worker_manager=None, single_mutation_ops=None, single_crossover_ops=None, options=None, reporter=None, ask_tell_mode=False): """ Constructor. """ options = load_options(ga_opt_args, partial_options=options) super(CPGAOptimiser, self).__init__(func_caller, worker_manager, mutation_op=self._mutation_op, options=options, reporter=reporter, ask_tell_mode=ask_tell_mode) self._set_up_single_mutation_ops(single_mutation_ops) self._set_up_single_crossover_ops(single_crossover_ops)
def __init__(self, func_caller, worker_manager=None, mutation_op=None, options=None, reporter=None, ask_tell_mode=False): options = load_options(ga_opt_args, partial_options=options) options.init_capital = options.population_size super().__init__(func_caller, worker_manager, model=None, options=options, reporter=reporter, ask_tell_mode=ask_tell_mode) self.mutation_op = mutation_op self.pool = []
def prepare_chemist_options(self, chemist_args, domain_config): """ Resets default gp_bandit options with chemist arguments """ dflt_list_of_options = get_all_cp_gp_bandit_args() list_of_options = self.reset_default_options(dflt_list_of_options, chemist_args) for name, value in domain_config.items(): list_of_options += [{'name': name, 'default': value}] options = load_options(list_of_options, reporter=self.reporter) if self.mf_strategy is not None: options.mf_strategy = self.mf_strategy if isinstance(self.worker_manager, RealWorkerManager): options.capital_type = 'realtime' elif isinstance(self.worker_manager, SyntheticWorkerManager): options.capital_type = 'return_value' options.get_initial_qinfos = \ lambda num: get_cp_domain_initial_qinfos(self.func_caller.domain, num) return options
def build_aging_cp_ga_optimiser(func_caller, cp_domain, worker_manager, max_capital, mode='asy', orderings=None, single_mutation_ops=None, single_crossover_ops=None, options=None, reporter="silent"): """ A GA optimiser on Cartesian product space from the function caller. """ log = logging.getLogger("AgingGA Optimiser") if not isinstance(func_caller, ExperimentCaller): func_caller = CPFunctionCaller(func_caller, cp_domain, domain_orderings=orderings) options = load_options(ga_opt_args, partial_options=options) options.mode = mode options.capital_type = 'return_value' options.population_size = 100 options.sample_size = 25 start_time = timer() log.info( f"Starting GA optimisation with capital {max_capital} ({options.capital_type})." ) result = CPGAOptimiser(func_caller, worker_manager, single_mutation_ops=single_mutation_ops, single_crossover_ops=single_crossover_ops, options=options, reporter=reporter).optimise(max_capital) end_time = timer() log.info(f"GA optimisation finished, took {(end_time - start_time):.2f}s.") return result
{'type': 'discrete', 'items': place_pruned_graph_list}, {'type': 'discrete', 'items': enable_bfloat16_sendrecv_list}, {'type': 'discrete', 'items': do_common_subexpression_elimination_list}, {'type': 'discrete_numeric', 'items': max_folded_constant_list}, {'type': 'discrete', 'items': do_function_inlining_list}, {'type': 'discrete_numeric', 'items': global_jit_level_list}, {'type': 'discrete', 'items': optimizer_list} ] dragonfly_args = [ get_option_specs('report_results_every', False, 2, 'Path to the json or pb config file. '), get_option_specs('init_capital', False, None, 'Path to the json or pb config file. '), get_option_specs('init_capital_frac', False, 0.07, 'Path to the json or pb config file. '), get_option_specs('num_init_evals', False, 2, 'Path to the json or pb config file. ')] options = load_options(dragonfly_args) config_params = {'domain': domain_vars} config = load_config(config_params) max_num_evals = 60*60*12 moo_objectives = [runtime_eval, acc_eval] pareto_opt_vals, pareto_opt_pts, history = multiobjective_maximise_functions(moo_objectives, config.domain,max_num_evals,capital_type='realtime',config=config,options=options) f = open("./output.log","w+") print(pareto_opt_pts,file=f) print("\n",file=f) print(pareto_opt_vals,file=f) print("\n",file=f) print(history,file=f)
def get_config(self, budget): """Function to sample a new configuration This function is called inside BOHB to query a new configuration Parameters: ----------- budget: float the budget for which this configuration is scheduled Returns ------- config return a valid configuration with parameters and budget """ if not self.is_moo: return self.get_config_old(budget) logger.debug('start sampling a new configuration.') if not self.configs: print( f"[vincent] self.configs is empty! Use a random config instead." ) sample = self.configspace.sample_configuration() sample = ConfigSpace.util.deactivate_inactive_hyperparameters( configuration_space=self.configspace, configuration=sample.get_dictionary()).get_dictionary() sample['TRIAL_BUDGET'] = budget return sample domain_vars = list() for name in self.search_space.keys(): if isinstance(self.search_space[name][0], (float, int)): var_type = 'discrete_numeric' else: var_type = 'discrete' domain_var = {'type': var_type, 'items': self.search_space[name]} domain_vars.append(domain_var) points = list() vals = list() true_vals = list() print(f"[vincent] self.configs:{self.configs} budget:{budget}") print(f"{list(self.search_space.keys())}") for conf_array in self.configs[0]: first, second = [], [] for i in range(len(conf_array)): item = self.search_space[list( self.search_space.keys())[i]][int(conf_array[i])] if isinstance(item, (float, int)): second.append(item) else: first.append(item) points.append([first, second]) for idx in range(len(self.losses[0])): vals.append([-self.losses[0][idx], -self.runtime[0][idx]]) true_vals.append([-self.losses[0][idx], -self.runtime[0][idx]]) print(f"[vincent] len of points:{len(points)}") if len(points) > 10: vals_array = np.array(vals) pareto_index = is_pareto_efficient_simple(vals_array) p_idx = [] np_idx = [] np_items = [] for j in range(len(pareto_index)): if pareto_index[j] == True: p_idx.append(j) else: np_idx.append(j) np_items.append(vals[j]) print(f"[vincent] pareto_index:{p_idx}") print(f"[vincent] not pareto_index:{np_idx}") if len(p_idx) >= 5: tmp_idx = [] for j in range(5): tmp_idx.append(p_idx[j]) points = [points[i] for i in tmp_idx] vals = [vals[i] for i in tmp_idx] true_vals = [true_vals[i] for i in tmp_idx] else: num_diff = 5 - len(p_idx) print(f"[vincent] diff num:{num_diff}") print(f"[vincent] search space:{self.search_space}") if self.search_space['PREFERENCE'][0] == "accuracy": acc_items = [-item[0] for item in np_items] sort_n_idx = np.argsort(acc_items) for i in range(num_diff): p_idx.append(sort_n_idx[i]) print(f"[vincent] final pareto_index:{p_idx}") points = [points[i] for i in p_idx] vals = [vals[i] for i in p_idx] true_vals = [true_vals[i] for i in p_idx] elif self.search_space['PREFERENCE'][0] == "runtime": time_items = [-item[1] for item in np_items] sort_n_idx = np.argsort(time_items) for i in range(num_diff): p_idx.append(sort_n_idx[i]) print(f"[vincent] final pareto_index:{p_idx}") points = [points[i] for i in p_idx] vals = [vals[i] for i in p_idx] true_vals = [true_vals[i] for i in p_idx] # import random # idx_list = random.sample(range(len(points)), 10) # print(f"[vincent] random selections list idx_list:{idx_list}") # points = [points[i] for i in idx_list] # vals = [vals[i] for i in idx_list] # true_vals = [true_vals[i] for i in idx_list] ## vals = [[acc,-spent time],[acc,-spent time]] ## load from memory previous_eval = {'qinfos': []} for i in range(len(points)): tmp = Namespace(point=points[i], val=vals[i], true_val=true_vals[i]) previous_eval['qinfos'].append(tmp) p = Namespace(**previous_eval) load_args = [ get_option_specs('init_capital', False, 1, 'Path to the json or pb config file. '), get_option_specs( 'init_capital_frac', False, None, 'The fraction of the total capital to be used for initialisation.' ), get_option_specs( 'num_init_evals', False, 1, 'The number of evaluations for initialisation. If <0, will use default.' ), get_option_specs('prev_evaluations', False, p, 'Data for any previous evaluations.') ] options = load_options(load_args) config_params = {'domain': domain_vars} config = load_config(config_params) max_num_evals = 1 self.dragonfly_config = None def fake_func(x): if not self.dragonfly_config: self.dragonfly_config = x print( f"[vincent] x is assigned to self.dragonfly_config:{self.dragonfly_config}" ) return 0 moo_objectives = [fake_func, fake_func] _, _, _ = multiobjective_maximise_functions(moo_objectives, config.domain, max_num_evals, capital_type='num_evals', config=config, options=options) print( f"[vincent] self.dragonfly_config after dragonfly:{self.dragonfly_config}" ) ## load prev from the file # data_to_save = {'points': points, # 'vals': vals, # 'true_vals': true_vals} # print(f"[vincent] data_to_save:{data_to_save}") # temp_save_path = './dragonfly.saved' # with open(temp_save_path, 'wb') as save_file_handle: # pickle.dump(data_to_save, save_file_handle) # load_args = [ # get_option_specs('progress_load_from', False, temp_save_path, # 'Load progress (from possibly a previous run) from this file.') # ] # options = load_options(load_args) # config_params = {'domain': domain_vars} # config = load_config(config_params) # max_num_evals = 1 # self.dragonfly_config = None # def fake_func(x): # if not self.dragonfly_config: # self.dragonfly_config = x # print(f"[vincent] x is assigned to self.dragonfly_config:{self.dragonfly_config}") # return 0 # moo_objectives = [fake_func, fake_func] # _, _, _ = multiobjective_maximise_functions(moo_objectives, config.domain,max_num_evals,capital_type='num_evals',config=config,options=options) # print(f"[vincent] self.dragonfly_config after dragonfly:{self.dragonfly_config}") # import os # if os.path.exists(temp_save_path): # os.remove(temp_save_path) if not self.dragonfly_config: print( f"[vincent] Get empty config from dragonfly! Use a random config instead." ) sample = self.configspace.sample_configuration() else: sample = dict() df_idx = 0 for name in self.search_space.keys(): sample[name] = self.dragonfly_config[df_idx] df_idx += 1 logger.debug('done sampling a new configuration.') sample['TRIAL_BUDGET'] = budget print(f'[vincent] sample from get_config:{sample}') return sample
def main(): """ Main function. """ options = load_options(get_command_line_args(), cmd_line=True) # Load domain and objective config = load_config_file(options.config) if hasattr(config, 'fidel_space'): is_mf = True else: is_mf = False # Load module expt_dir = os.path.dirname( os.path.abspath(os.path.realpath(options.config))) if not os.path.exists(expt_dir): raise ValueError("Experiment directory does not exist.") sys.path.append(expt_dir) obj_module = import_module(config.name, expt_dir) sys.path.remove(expt_dir) # Set capital if options.max_capital < 0: raise ValueError( 'max_capital (time or number of evaluations) must be positive.') # Call optimiser _print_prefix = 'Maximising' if options.max_or_min == 'max' else 'Minimising' call_to_optimise = { 'single': { 'max': maximise_function, 'min': minimise_function }, 'single_mf': { 'max': maximise_multifidelity_function, 'min': minimise_multifidelity_function }, 'multi': { 'max': multiobjective_maximise_functions, 'min': multiobjective_minimise_functions }, } if not options.is_multi_objective: if is_mf: print( '%s multi-fidelity function on\n Fidelity-Space: %s.\n Domain: %s.\n' % (_print_prefix, config.fidel_space, config.domain)) opt_val, opt_pt, history = call_to_optimise['single_mf'][ options.max_or_min](obj_module.objective, fidel_space=None, domain=None, fidel_to_opt=config.fidel_to_opt, fidel_cost_func=obj_module.cost, max_capital=options.max_capital, capital_type=options.capital_type, opt_method=options.opt_method, config=config, options=options, reporter=options.report_progress) else: print('%s function on Domain: %s.\n' % (_print_prefix, config.domain)) opt_val, opt_pt, history = call_to_optimise['single'][ options.max_or_min](obj_module.objective, domain=None, max_capital=options.max_capital, capital_type=options.capital_type, opt_method=options.opt_method, config=config, options=options, reporter=options.report_progress) print('Optimum Value in %d evals: %0.4f' % (len(history.curr_opt_points), opt_val)) print('Optimum Point: %s.' % (opt_pt)) else: if is_mf: raise ValueError( 'Multi-objective multi-fidelity optimisation has not been ' + 'implemented yet.') else: # Check format of function caller if hasattr(obj_module, 'objectives'): objectives_to_pass = obj_module.objectives num_objectives = len(objectives_to_pass) else: num_objectives = obj_module.num_objectives objectives_to_pass = (obj_module.compute_objectives, obj_module.num_objectives) print('%s %d multiobjective functions on Domain: %s.\n' % (_print_prefix, num_objectives, config.domain)) print(objectives_to_pass) pareto_values, pareto_points, history = \ call_to_optimise['multi'][options.max_or_min](objectives_to_pass, domain=None, max_capital=options.max_capital, capital_type=options.capital_type, opt_method=options.opt_method, config=config, options=options, reporter=options.report_progress) num_pareto_points = len(pareto_points) print('Found %d Pareto Points: %s.' % (num_pareto_points, pareto_points)) print('Corresponding Pareto Values: %s.' % (pareto_values))
def get_optimiser_options(): options = load_options(get_all_mf_cp_gp_bandit_args()) options.acq = "custom_mobo_ts" return options
def main(): """ Main function. """ # First load arguments all_args = dragonfly_args + get_all_euc_gp_bandit_args() + get_all_cp_gp_bandit_args() \ + get_all_mf_euc_gp_bandit_args() + get_all_mf_cp_gp_bandit_args() \ + get_all_euc_moo_gp_bandit_args() + get_all_cp_moo_gp_bandit_args() all_args = get_unique_list_of_option_args(all_args) options = load_options(all_args, cmd_line=True) # Load domain and objective config = load_config_file(options.config) if hasattr(config, 'fidel_space'): is_mf = True else: is_mf = False expt_dir = os.path.dirname( os.path.abspath(os.path.realpath(options.config))) if not os.path.exists(expt_dir): raise ValueError("Experiment directory does not exist.") objective_file_name = config.name obj_module = imp.load_source( objective_file_name, os.path.join(expt_dir, objective_file_name + '.py')) # Set capital options.capital_type = 'return_value' if options.budget < 0: budget = options.max_capital else: budget = options.budget if budget < 0: raise ValueError( 'Specify the budget via argument budget or max_capital.') options.max_capital = budget # Call optimiser _print_prefix = 'Maximising' if options.max_or_min == 'max' else 'Minimising' call_to_optimise = { 'single': { 'max': maximise_function, 'min': minimise_function }, 'single_mf': { 'max': maximise_multifidelity_function, 'min': minimise_multifidelity_function }, 'multi': { 'max': multiobjective_maximise_functions, 'min': multiobjective_minimise_functions }, } if not options.is_multi_objective: if is_mf: print('%s function on fidel_space: %s, domain %s.' % (_print_prefix, config.fidel_space, config.domain)) opt_val, opt_pt, history = call_to_optimise['single_mf'][ options.max_or_min](obj_module.objective, domain=None, fidel_space=None, fidel_to_opt=config.fidel_to_opt, fidel_cost_func=obj_module.cost, max_capital=options.max_capital, config=config, options=options) else: print('%s function on domain %s.' % (_print_prefix, config.domain)) opt_val, opt_pt, history = call_to_optimise['single'][ options.max_or_min](obj_module.objective, domain=None, max_capital=options.max_capital, config=config, options=options) print('Optimum Value in %d evals: %0.4f' % (len(history.curr_opt_points), opt_val)) print('Optimum Point: %s.' % (opt_pt)) else: if is_mf: raise ValueError( 'Multi-objective multi-fidelity optimisation has not been ' + 'implemented yet.') else: print( '%s multiobjective functions on domain %s with %d functions.' % (_print_prefix, config.domain, len(obj_module.objectives))) pareto_values, pareto_points, history = \ call_to_optimise['multi'][options.max_or_min](obj_module.objectives, domain=None, max_capital=options.max_capital, config=config, options=options) num_pareto_points = len(pareto_points) print('Found %d Pareto Points: %s.' % (num_pareto_points, pareto_points)) print('Corresponding Pareto Values: %s.' % (pareto_values))