def test_multi_config_design(self): stats = Stats(scenario=self.scenario) stats.start_timing() self.ta.stats = stats tj = TrajLogger(output_dir=None, stats=stats) rh = RunHistory(aggregate_func=average_cost) self.ta.runhistory = rh rng = np.random.RandomState(seed=12345) intensifier = Intensifier(tae_runner=self.ta, stats=stats, traj_logger=tj, rng=rng, instances=[None], run_obj_time=False) configs = [ Configuration(configuration_space=self.cs, values={"x1": 4}), Configuration(configuration_space=self.cs, values={"x1": 2}) ] dc = MultiConfigInitialDesign(tae_runner=self.ta, scenario=self.scenario, stats=stats, traj_logger=tj, runhistory=rh, rng=rng, configs=configs, intensifier=intensifier, aggregate_func=average_cost) inc = dc.run() self.assertTrue(stats.ta_runs == 2) self.assertTrue(len(rh.data) == 2) self.assertTrue(rh.get_cost(inc) == 4)
def __init__( self, scenario: Scenario, tae_runner: typing.Union[ExecuteTARun, typing.Callable] = None, runhistory: RunHistory = None, intensifier: Intensifier = None, acquisition_function: AbstractAcquisitionFunction = None, acquisition_function_optimizer: AcquisitionFunctionMaximizer = None, model: AbstractEPM = None, runhistory2epm: AbstractRunHistory2EPM = None, initial_design: InitialDesign = None, initial_configurations: typing.List[Configuration] = None, stats: Stats = None, restore_incumbent: Configuration = None, rng: typing.Union[np.random.RandomState, int] = None, smbo_class: SMBO = None, run_id: int = 1): """Constructor Parameters ---------- scenario : ~smac.scenario.scenario.Scenario Scenario object tae_runner : ~smac.tae.execute_ta_run.ExecuteTARun or callable Callable or implementation of :class:`~smac.tae.execute_ta_run.ExecuteTARun`. In case a callable is passed it will be wrapped by :class:`~smac.tae.execute_func.ExecuteTAFuncDict`. If not set, it will be initialized with the :class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`. runhistory : RunHistory runhistory to store all algorithm runs intensifier : Intensifier intensification object to issue a racing to decide the current incumbent acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction Object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`. Will use :class:`~smac.optimizer.acquisition.EI` if not set. acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`. Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set. model : AbstractEPM Model that implements train() and predict(). Will use a :class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set. runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost` if objective is cost or :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost` if objective is runtime. initial_design : InitialDesign initial sampling design initial_configurations : typing.List[Configuration] list of initial configurations for initial design -- cannot be used together with initial_design stats : Stats optional stats object rng : np.random.RandomState Random number generator restore_incumbent : Configuration incumbent used if restoring to previous state smbo_class : ~smac.optimizer.smbo.SMBO Class implementing the SMBO interface which will be used to instantiate the optimizer class. run_id: int, (default: 1) Run ID will be used as subfolder for output_dir. """ self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) aggregate_func = average_cost self.output_dir = create_output_directory(scenario, run_id) scenario.write() # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario) # initialize empty runhistory if runhistory is None: runhistory = RunHistory(aggregate_func=aggregate_func) # inject aggr_func if necessary if runhistory.aggregate_func is None: runhistory.aggregate_func = aggregate_func # initial random number generator num_run, rng = self._get_rng(rng=rng) # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed(rng.randint(MAXINT)) # initial Trajectory Logger traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats) # initial EPM types, bounds = get_types(scenario.cs, scenario.feature_array) if model is None: model = RandomForestWithInstances( types=types, bounds=bounds, instance_features=scenario.feature_array, seed=rng.randint(MAXINT), pca_components=scenario.PCA_DIM) # initial acquisition function if acquisition_function is None: if scenario.run_obj == "runtime": acquisition_function = LogEI(model=model) else: acquisition_function = EI(model=model) # inject model if necessary if acquisition_function.model is None: acquisition_function.model = model # initialize optimizer on acquisition function if acquisition_function_optimizer is None: acquisition_function_optimizer = InterleavedLocalAndRandomSearch( acquisition_function, scenario.cs, np.random.RandomState(seed=rng.randint(MAXINT))) elif not isinstance( acquisition_function_optimizer, AcquisitionFunctionMaximizer, ): raise ValueError( "Argument 'acquisition_function_optimizer' must be of type" "'AcquisitionFunctionMaximizer', but is '%s'" % type(acquisition_function_optimizer)) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file if tae_runner is None: tae_runner = ExecuteTARunOld( ta=scenario.ta, stats=self.stats, run_obj=scenario.run_obj, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash) # Second case, the tae_runner is a function to be optimized elif callable(tae_runner): tae_runner = ExecuteTAFuncDict( ta=tae_runner, stats=self.stats, run_obj=scenario.run_obj, memory_limit=scenario.memory_limit, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash) # Third case, if it is an ExecuteTaRun we can simply use the # instance. Otherwise, the next check raises an exception elif not isinstance(tae_runner, ExecuteTARun): raise TypeError("Argument 'tae_runner' is %s, but must be " "either a callable or an instance of " "ExecuteTaRun. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # Check that overall objective and tae objective are the same if tae_runner.run_obj != scenario.run_obj: raise ValueError("Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner.run_obj, scenario.run_obj)) # inject stats if necessary if tae_runner.stats is None: tae_runner.stats = self.stats # inject runhistory if necessary if tae_runner.runhistory is None: tae_runner.runhistory = runhistory # inject cost_for_crash if tae_runner.crash_cost != scenario.cost_for_crash: tae_runner.crash_cost = scenario.cost_for_crash # initialize intensification if intensifier is None: intensifier = Intensifier(tae_runner=tae_runner, stats=self.stats, traj_logger=traj_logger, rng=rng, instances=scenario.train_insts, cutoff=scenario.cutoff, deterministic=scenario.deterministic, run_obj_time=scenario.run_obj == "runtime", always_race_against=scenario.cs.get_default_configuration() \ if scenario.always_race_default else None, instance_specifics=scenario.instance_specific, minR=scenario.minR, maxR=scenario.maxR) # inject deps if necessary if intensifier.tae_runner is None: intensifier.tae_runner = tae_runner if intensifier.stats is None: intensifier.stats = self.stats if intensifier.traj_logger is None: intensifier.traj_logger = traj_logger # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both" ) if initial_configurations is not None: initial_design = MultiConfigInitialDesign( tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, configs=initial_configurations, intensifier=intensifier, aggregate_func=aggregate_func) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": initial_design = DefaultConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) elif scenario.initial_incumbent == "RANDOM": initial_design = RandomConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) # inject deps if necessary if initial_design.tae_runner is None: initial_design.tae_runner = tae_runner if initial_design.scenario is None: initial_design.scenario = scenario if initial_design.stats is None: initial_design.stats = self.stats if initial_design.traj_logger is None: initial_design.traj_logger = traj_logger # initial conversion of runhistory into EPM data if runhistory2epm is None: num_params = len(scenario.cs.get_hyperparameters()) if scenario.run_obj == "runtime": # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log10(scenario.cutoff) threshold = np.log10(scenario.cutoff * scenario.par_factor) imputor = RFRImputator(rng=rng, cutoff=cutoff, threshold=threshold, model=model, change_threshold=0.01, max_iter=2) runhistory2epm = RunHistory2EPM4LogCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.CAPPED, ], imputor=imputor) elif scenario.run_obj == 'quality': runhistory2epm = RunHistory2EPM4Cost( scenario=scenario, num_params=num_params, success_states=[StatusType.SUCCESS, StatusType.CRASHED], impute_censored_data=False, impute_state=None) else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) # inject scenario if necessary: if runhistory2epm.scenario is None: runhistory2epm.scenario = scenario smbo_args = { 'scenario': scenario, 'stats': self.stats, 'initial_design': initial_design, 'runhistory': runhistory, 'runhistory2epm': runhistory2epm, 'intensifier': intensifier, 'aggregate_func': aggregate_func, 'num_run': num_run, 'model': model, 'acq_optimizer': acquisition_function_optimizer, 'acquisition_func': acquisition_function, 'rng': rng, 'restore_incumbent': restore_incumbent } if smbo_class is None: self.solver = SMBO(**smbo_args) else: self.solver = smbo_class(**smbo_args)
def __init__( self, scenario: Scenario, # TODO: once we drop python3.4 add type hint # typing.Union[ExecuteTARun, callable] tae_runner=None, runhistory: RunHistory = None, intensifier: Intensifier = None, acquisition_function: AbstractAcquisitionFunction = None, model: AbstractEPM = None, runhistory2epm: AbstractRunHistory2EPM = None, initial_design: InitialDesign = None, initial_configurations: typing.List[Configuration] = None, stats: Stats = None, rng: np.random.RandomState = None, run_id: int = 1): """Constructor""" self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) aggregate_func = average_cost self.runhistory = None self.trajectory = None # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario) self.output_dir = create_output_directory(scenario, run_id) scenario.write() # initialize empty runhistory if runhistory is None: runhistory = RunHistory(aggregate_func=aggregate_func) # inject aggr_func if necessary if runhistory.aggregate_func is None: runhistory.aggregate_func = aggregate_func # initial random number generator num_run, rng = self._get_rng(rng=rng) # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed(rng.randint(MAXINT)) # initial Trajectory Logger traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats) # initial EPM types, bounds = get_types(scenario.cs, scenario.feature_array) if model is None: model = RandomForestWithInstances( types=types, bounds=bounds, instance_features=scenario.feature_array, seed=rng.randint(MAXINT), pca_components=scenario.PCA_DIM, num_trees=scenario.rf_num_trees, do_bootstrapping=scenario.rf_do_bootstrapping, ratio_features=scenario.rf_ratio_features, min_samples_split=scenario.rf_min_samples_split, min_samples_leaf=scenario.rf_min_samples_leaf, max_depth=scenario.rf_max_depth) # initial acquisition function if acquisition_function is None: if scenario.run_obj == "runtime": acquisition_function = LogEI(model=model) else: acquisition_function = EI(model=model) # inject model if necessary if acquisition_function.model is None: acquisition_function.model = model # initialize optimizer on acquisition function local_search = LocalSearch( acquisition_function, scenario.cs, max_steps=scenario.sls_max_steps, n_steps_plateau_walk=scenario.sls_n_steps_plateau_walk) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file if tae_runner is None: tae_runner = ExecuteTARunOld( ta=scenario.ta, stats=self.stats, run_obj=scenario.run_obj, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash) # Second case, the tae_runner is a function to be optimized elif callable(tae_runner): tae_runner = ExecuteTAFuncDict( ta=tae_runner, stats=self.stats, run_obj=scenario.run_obj, memory_limit=scenario.memory_limit, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash) # Third case, if it is an ExecuteTaRun we can simply use the # instance. Otherwise, the next check raises an exception elif not isinstance(tae_runner, ExecuteTARun): raise TypeError("Argument 'tae_runner' is %s, but must be " "either a callable or an instance of " "ExecuteTaRun. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # Check that overall objective and tae objective are the same if tae_runner.run_obj != scenario.run_obj: raise ValueError("Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner.run_obj, scenario.run_obj)) # inject stats if necessary if tae_runner.stats is None: tae_runner.stats = self.stats # inject runhistory if necessary if tae_runner.runhistory is None: tae_runner.runhistory = runhistory # inject cost_for_crash if tae_runner.crash_cost != scenario.cost_for_crash: tae_runner.crash_cost = scenario.cost_for_crash # initialize intensification if intensifier is None: intensifier = Intensifier( tae_runner=tae_runner, stats=self.stats, traj_logger=traj_logger, rng=rng, instances=scenario.train_insts, cutoff=scenario.cutoff, deterministic=scenario.deterministic, run_obj_time=scenario.run_obj == "runtime", always_race_against=scenario.cs.get_default_configuration() if scenario.always_race_default else None, instance_specifics=scenario.instance_specific, minR=scenario.minR, maxR=scenario.maxR, adaptive_capping_slackfactor=scenario. intens_adaptive_capping_slackfactor, min_chall=scenario.intens_min_chall) # inject deps if necessary if intensifier.tae_runner is None: intensifier.tae_runner = tae_runner if intensifier.stats is None: intensifier.stats = self.stats if intensifier.traj_logger is None: intensifier.traj_logger = traj_logger # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both" ) if initial_configurations is not None: initial_design = MultiConfigInitialDesign( tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, configs=initial_configurations, intensifier=intensifier, aggregate_func=aggregate_func) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": initial_design = DefaultConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) elif scenario.initial_incumbent == "RANDOM": initial_design = RandomConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) # inject deps if necessary if initial_design.tae_runner is None: initial_design.tae_runner = tae_runner if initial_design.scenario is None: initial_design.scenario = scenario if initial_design.stats is None: initial_design.stats = self.stats if initial_design.traj_logger is None: initial_design.traj_logger = traj_logger # initial conversion of runhistory into EPM data if runhistory2epm is None: num_params = len(scenario.cs.get_hyperparameters()) if scenario.run_obj == "runtime": # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log(scenario.cutoff) threshold = np.log(scenario.cutoff * scenario.par_factor) imputor = RFRImputator(rng=rng, cutoff=cutoff, threshold=threshold, model=model, change_threshold=0.01, max_iter=2) runhistory2epm = RunHistory2EPM4LogCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.CAPPED, ], imputor=imputor) elif scenario.run_obj == 'quality': runhistory2epm = RunHistory2EPM4Cost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=False, impute_state=None) else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) # inject scenario if necessary: if runhistory2epm.scenario is None: runhistory2epm.scenario = scenario self.solver = EPILS_Solver(scenario=scenario, stats=self.stats, initial_design=initial_design, runhistory=runhistory, runhistory2epm=runhistory2epm, intensifier=intensifier, aggregate_func=aggregate_func, num_run=num_run, model=model, acq_optimizer=local_search, acquisition_func=acquisition_function, rng=rng)
def __init__( self, scenario: Scenario, tae_runner: typing.Optional[typing.Union[ExecuteTARun, typing.Callable]] = None, runhistory: typing.Optional[RunHistory] = None, intensifier: typing.Optional[Intensifier] = None, acquisition_function: typing. Optional[AbstractAcquisitionFunction] = None, acquisition_function_optimizer: typing. Optional[AcquisitionFunctionMaximizer] = None, model: typing.Optional[AbstractEPM] = None, runhistory2epm: typing.Optional[AbstractRunHistory2EPM] = None, initial_design: typing.Optional[InitialDesign] = None, initial_configurations: typing.Optional[ typing.List[Configuration]] = None, stats: typing.Optional[Stats] = None, restore_incumbent: typing.Optional[Configuration] = None, rng: typing.Optional[typing.Union[np.random.RandomState, int]] = None, smbo_class: typing.Optional[SMBO] = None, run_id: typing.Optional[int] = None, random_configuration_chooser: typing. Optional[RandomConfigurationChooser] = None): """ Constructor Parameters ---------- scenario : ~smac.scenario.scenario.Scenario Scenario object tae_runner : ~smac.tae.execute_ta_run.ExecuteTARun or callable Callable or implementation of :class:`~smac.tae.execute_ta_run.ExecuteTARun`. In case a callable is passed it will be wrapped by :class:`~smac.tae.execute_func.ExecuteTAFuncDict`. If not set, it will be initialized with the :class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`. runhistory : RunHistory runhistory to store all algorithm runs intensifier : Intensifier intensification object to issue a racing to decide the current incumbent acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction Object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`. Will use :class:`~smac.optimizer.acquisition.EI` if not set. acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`. Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set. model : AbstractEPM Model that implements train() and predict(). Will use a :class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set. runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost` if objective is cost or :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost` if objective is runtime. initial_design : InitialDesign initial sampling design initial_configurations : typing.List[Configuration] list of initial configurations for initial design -- cannot be used together with initial_design stats : Stats optional stats object rng : np.random.RandomState Random number generator restore_incumbent : Configuration incumbent used if restoring to previous state smbo_class : ~smac.optimizer.smbo.SMBO Class implementing the SMBO interface which will be used to instantiate the optimizer class. run_id : int (optional) Run ID will be used as subfolder for output_dir. If no ``run_id`` is given, a random ``run_id`` will be chosen. random_configuration_chooser : ~smac.optimizer.random_configuration_chooser.RandomConfigurationChooser How often to choose a random configuration during the intensification procedure. """ self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) aggregate_func = average_cost self.scenario = scenario self.output_dir = "" if not restore_incumbent: # restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an # output directory. This is the default path. # initial random number generator run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) self.output_dir = create_output_directory(scenario, run_id) elif scenario.output_dir is not None: run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) # output-directory is created in CLI when restoring from a # folder. calling the function again in the facade results in two # folders being created: run_X and run_X.OLD. if we are # restoring, the output-folder exists already and we omit creating it, # but set the self-output_dir to the dir. # necessary because we want to write traj to new output-dir in CLI. self.output_dir = scenario.output_dir_for_this_run if (scenario.deterministic is True and getattr(scenario, 'tuner_timeout', None) is None and scenario.run_obj == 'quality'): self.logger.info('Optimizing a deterministic scenario for ' 'quality without a tuner timeout - will make ' 'SMAC deterministic!') scenario.intensification_percentage = 1e-10 scenario.write() # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario) if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG": self.logger.warn( "Runtime as objective automatically activates log(y) transformation" ) self.scenario.transform_y = "LOG" # initialize empty runhistory if runhistory is None: runhistory = RunHistory(aggregate_func=aggregate_func) # inject aggr_func if necessary if runhistory.aggregate_func is None: runhistory.aggregate_func = aggregate_func if not random_configuration_chooser: random_configuration_chooser = ChooserProb(prob=scenario.rand_prob, rng=rng) # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed(rng.randint(MAXINT)) # initial Trajectory Logger traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats) # initial EPM types, bounds = get_types(scenario.cs, scenario.feature_array) if model is None: model = RandomForestWithInstances( types=types, bounds=bounds, instance_features=scenario.feature_array, seed=rng.randint(MAXINT), pca_components=scenario.PCA_DIM, log_y=scenario.transform_y in ["LOG", "LOGS"], num_trees=scenario.rf_num_trees, do_bootstrapping=scenario.rf_do_bootstrapping, ratio_features=scenario.rf_ratio_features, min_samples_split=scenario.rf_min_samples_split, min_samples_leaf=scenario.rf_min_samples_leaf, max_depth=scenario.rf_max_depth) # initial acquisition function if acquisition_function is None: if scenario.transform_y in ["LOG", "LOGS"]: acquisition_function = LogEI(model=model) else: acquisition_function = EI(model=model) # inject model if necessary if acquisition_function.model is None: acquisition_function.model = model # initialize optimizer on acquisition function if acquisition_function_optimizer is None: acquisition_function_optimizer = InterleavedLocalAndRandomSearch( acquisition_function=acquisition_function, config_space=scenario.cs, rng=np.random.RandomState(seed=rng.randint(MAXINT)), max_steps=scenario.sls_max_steps, n_steps_plateau_walk=scenario.sls_n_steps_plateau_walk) elif not isinstance( acquisition_function_optimizer, AcquisitionFunctionMaximizer, ): raise ValueError( "Argument 'acquisition_function_optimizer' must be of type" "'AcquisitionFunctionMaximizer', but is '%s'" % type(acquisition_function_optimizer)) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file if tae_runner is None: tae_runner = ExecuteTARunOld( ta=scenario.ta, stats=self.stats, run_obj=scenario.run_obj, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash, abort_on_first_run_crash=scenario.abort_on_first_run_crash) # Second case, the tae_runner is a function to be optimized elif callable(tae_runner): tae_runner = ExecuteTAFuncDict( ta=tae_runner, stats=self.stats, run_obj=scenario.run_obj, memory_limit=scenario.memory_limit, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash, abort_on_first_run_crash=scenario.abort_on_first_run_crash) # Third case, if it is an ExecuteTaRun we can simply use the # instance. Otherwise, the next check raises an exception elif not isinstance(tae_runner, ExecuteTARun): raise TypeError("Argument 'tae_runner' is %s, but must be " "either a callable or an instance of " "ExecuteTaRun. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # Check that overall objective and tae objective are the same if tae_runner.run_obj != scenario.run_obj: raise ValueError("Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner.run_obj, scenario.run_obj)) # inject stats if necessary if tae_runner.stats is None: tae_runner.stats = self.stats # inject runhistory if necessary if tae_runner.runhistory is None: tae_runner.runhistory = runhistory # inject cost_for_crash if tae_runner.crash_cost != scenario.cost_for_crash: tae_runner.crash_cost = scenario.cost_for_crash # initialize intensification if intensifier is None: intensifier = Intensifier( tae_runner=tae_runner, stats=self.stats, traj_logger=traj_logger, rng=rng, instances=scenario.train_insts, cutoff=scenario.cutoff, deterministic=scenario.deterministic, run_obj_time=scenario.run_obj == "runtime", always_race_against=scenario.cs.get_default_configuration() if scenario.always_race_default else None, use_ta_time_bound=scenario.use_ta_time, instance_specifics=scenario.instance_specific, minR=scenario.minR, maxR=scenario.maxR, adaptive_capping_slackfactor=scenario. intens_adaptive_capping_slackfactor, min_chall=scenario.intens_min_chall) # inject deps if necessary if intensifier.tae_runner is None: intensifier.tae_runner = tae_runner if intensifier.stats is None: intensifier.stats = self.stats if intensifier.traj_logger is None: intensifier.traj_logger = traj_logger # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both" ) if initial_configurations is not None: initial_design = MultiConfigInitialDesign( tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, configs=initial_configurations, intensifier=intensifier, aggregate_func=aggregate_func) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": initial_design = DefaultConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) elif scenario.initial_incumbent == "RANDOM": initial_design = RandomConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) elif scenario.initial_incumbent == "LHD": initial_design = LHDesign(runhistory=runhistory, intensifier=intensifier, aggregate_func=aggregate_func, tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) elif scenario.initial_incumbent == "FACTORIAL": initial_design = FactorialInitialDesign( runhistory=runhistory, intensifier=intensifier, aggregate_func=aggregate_func, tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) elif scenario.initial_incumbent == "SOBOL": initial_design = SobolDesign(runhistory=runhistory, intensifier=intensifier, aggregate_func=aggregate_func, tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) # inject deps if necessary if initial_design.tae_runner is None: initial_design.tae_runner = tae_runner if initial_design.scenario is None: initial_design.scenario = scenario if initial_design.stats is None: initial_design.stats = self.stats if initial_design.traj_logger is None: initial_design.traj_logger = traj_logger # initial conversion of runhistory into EPM data if runhistory2epm is None: num_params = len(scenario.cs.get_hyperparameters()) if scenario.run_obj == 'runtime': # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log(scenario.cutoff) threshold = np.log(scenario.cutoff * scenario.par_factor) imputor = RFRImputator(rng=rng, cutoff=cutoff, threshold=threshold, model=model, change_threshold=0.01, max_iter=2) runhistory2epm = RunHistory2EPM4LogCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.CAPPED, ], imputor=imputor) elif scenario.run_obj == 'quality': if scenario.transform_y == "NONE": runhistory2epm = RunHistory2EPM4Cost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, StatusType.CRASHED ], impute_censored_data=False, impute_state=None) elif scenario.transform_y == "LOG": runhistory2epm = RunHistory2EPM4LogCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, StatusType.CRASHED ], impute_censored_data=False, impute_state=None) elif scenario.transform_y == "LOGS": runhistory2epm = RunHistory2EPM4LogScaledCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, StatusType.CRASHED ], impute_censored_data=False, impute_state=None) elif scenario.transform_y == "INVS": runhistory2epm = RunHistory2EPM4InvScaledCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, StatusType.CRASHED ], impute_censored_data=False, impute_state=None) else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) # inject scenario if necessary: if runhistory2epm.scenario is None: runhistory2epm.scenario = scenario smbo_args = { 'scenario': scenario, 'stats': self.stats, 'initial_design': initial_design, 'runhistory': runhistory, 'runhistory2epm': runhistory2epm, 'intensifier': intensifier, 'aggregate_func': aggregate_func, 'num_run': run_id, 'model': model, 'acq_optimizer': acquisition_function_optimizer, 'acquisition_func': acquisition_function, 'rng': rng, 'restore_incumbent': restore_incumbent, 'random_configuration_chooser': random_configuration_chooser } if smbo_class is None: self.solver = SMBO(**smbo_args) else: self.solver = smbo_class(**smbo_args)
def build_pc_smbo(self, tae_runner, stats, scenario, runhistory, aggregate_func, acq_func_name, model_target_names, logging_directory, double_intensification=False, constant_pipeline_steps=None, variable_pipeline_steps=None, cached_pipeline_steps=None, seed=None, intensification_instances=None, num_marginalized_configurations_by_random_search=20, num_configs_for_marginalization=40, random_splitting_number=5, random_splitting_enabled=False): # Build intensifier rng = np.random.RandomState(seed) traj_logger = TrajLogger(logging_directory, stats) intensifier = Intensifier(tae_runner=tae_runner, stats=stats, traj_logger=traj_logger, rng=rng, cutoff=scenario.cutoff, deterministic=scenario.deterministic, run_obj_time=scenario.run_obj == "runtime", run_limit=scenario.ta_run_limit, instances=intensification_instances, maxR=len(intensification_instances)) # Build model types, bounds = get_types(scenario.cs, scenario.feature_array) #types = get_types(scenario.cs) if len(model_target_names) > 1: # model_target_names = ['cost','time'] model = UncorrelatedMultiObjectiveRandomForestWithInstances( target_names=model_target_names, bounds=bounds, types=types) # UncorrelatedMultiObjectiveRandomForestWithInstances(target_names=model_target_names, # types=types) elif len(model_target_names) == 1: model = RandomForestWithInstances(types=types, bounds=bounds) else: model = RandomEPM(rng=rng) # model = RandomForestWithInstances(types=types) # Build acquisition function, runhistory2epm and local search num_params = len(scenario.cs.get_hyperparameters()) if acq_func_name in ["ei", "pc-ei"]: acquisition_func = EI(model) acq_func_wrapper = PCAquisitionFunctionWrapper( acquisition_func=acquisition_func, config_space=scenario.cs, runhistory=runhistory, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps) runhistory2epm = RunHistory2EPM4Cost( scenario, num_params, success_states=[StatusType.SUCCESS]) local_search = LocalSearch(acquisition_function=acq_func_wrapper, config_space=scenario.cs) select_configuration = SelectConfigurations( scenario=scenario, stats=stats, runhistory=runhistory, model=model, acq_optimizer=local_search, acquisition_func=acq_func_wrapper, rng=rng, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps) elif acq_func_name in ["m-ei", "pc-m-ei"]: #acquisition_func = MEI(model) acquisition_func = EI(model) acq_func_wrapper = PCAquisitionFunctionWrapper( acquisition_func=acquisition_func, config_space=scenario.cs, runhistory=runhistory, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps) runhistory2epm = RunHistory2EPM4Cost( scenario, num_params, success_states=[StatusType.SUCCESS]) local_search = LocalSearch(acquisition_function=acq_func_wrapper, config_space=scenario.cs) # TODO: num_configs_for_marginalization select_configuration = SelectConfigurationsWithMarginalization( scenario=scenario, stats=stats, runhistory=runhistory, model=model, acq_optimizer=local_search, acquisition_func=acq_func_wrapper, rng=rng, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps, num_marginalized_configurations_by_random_search= num_marginalized_configurations_by_random_search, num_configs_for_marginalization=num_configs_for_marginalization ) elif acq_func_name in ['eips', 'pc-eips']: acquisition_func = EIPS(model) acq_func_wrapper = PCAquisitionFunctionWrapper( acquisition_func=acquisition_func, config_space=scenario.cs, runhistory=runhistory, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps) runhistory2epm = RunHistory2EPM4EIPS( scenario, num_params, success_states=[StatusType.SUCCESS]) local_search = LocalSearch(acquisition_function=acq_func_wrapper, config_space=scenario.cs) select_configuration = SelectConfigurations( scenario=scenario, stats=stats, runhistory=runhistory, model=model, acq_optimizer=local_search, acquisition_func=acq_func_wrapper, rng=rng, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps) elif acq_func_name in ["m-eips", "pc-m-eips"]: acquisition_func = EIPS(model) acq_func_wrapper = PCAquisitionFunctionWrapper( acquisition_func=acquisition_func, config_space=scenario.cs, runhistory=runhistory, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps) runhistory2epm = RunHistory2EPM4EIPS( scenario, num_params, success_states=[StatusType.SUCCESS]) local_search = LocalSearch(acquisition_function=acq_func_wrapper, config_space=scenario.cs) # TODO: num_configs_for_marginalization select_configuration = SelectConfigurationsWithMarginalization( scenario=scenario, stats=stats, runhistory=runhistory, model=model, acq_optimizer=local_search, acquisition_func=acq_func_wrapper, rng=rng, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps, num_marginalized_configurations_by_random_search= num_marginalized_configurations_by_random_search, num_configs_for_marginalization=num_configs_for_marginalization ) elif acq_func_name == 'pceips': acquisition_func = PCEIPS(model) acq_func_wrapper = PCAquisitionFunctionWrapperWithCachingReduction( acquisition_func=acquisition_func, config_space=scenario.cs, runhistory=runhistory, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps, cached_pipeline_steps=cached_pipeline_steps) runhistory2epm = RunHistory2EPM4EIPS( scenario, num_params, success_states=[StatusType.SUCCESS]) local_search = LocalSearch(acquisition_function=acq_func_wrapper, config_space=scenario.cs) if constant_pipeline_steps == None or variable_pipeline_steps == None or cached_pipeline_steps == None: raise ValueError( "Constant_pipeline_steps and variable pipeline steps should not be none\ when using PCEIPS") select_configuration = SelectConfigurations( scenario=scenario, stats=stats, runhistory=runhistory, model=model, acq_optimizer=local_search, acquisition_func=acq_func_wrapper, rng=rng, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps) elif acq_func_name == 'pc-m-pceips': acquisition_func = PCEIPS(model) acq_func_wrapper = PCAquisitionFunctionWrapperWithCachingReduction( acquisition_func=acquisition_func, config_space=scenario.cs, runhistory=runhistory, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps, cached_pipeline_steps=cached_pipeline_steps) runhistory2epm = RunHistory2EPM4EIPS( scenario, num_params, success_states=[StatusType.SUCCESS]) local_search = LocalSearch(acquisition_function=acq_func_wrapper, config_space=scenario.cs) if constant_pipeline_steps == None or variable_pipeline_steps == None or cached_pipeline_steps == None: raise ValueError( "Constant_pipeline_steps and variable pipeline steps should not be none\ when using PCEIPS") select_configuration = SelectConfigurationsWithMarginalization( scenario=scenario, stats=stats, runhistory=runhistory, model=model, acq_optimizer=local_search, acquisition_func=acq_func_wrapper, rng=rng, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps, num_marginalized_configurations_by_random_search= num_marginalized_configurations_by_random_search, num_configs_for_marginalization=num_configs_for_marginalization ) elif acq_func_name == "roar": runhistory2epm = RunHistory2EPM4Cost( scenario, num_params, success_states=[StatusType.SUCCESS]) select_configuration = SelectConfigurationsRandom( scenario=scenario) elif acq_func_name == "pc-roar-mrs": runhistory2epm = RunHistory2EPM4Cost( scenario, num_params, success_states=[StatusType.SUCCESS]) select_configuration = SelectConfigurationsMRS( scenario=scenario, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps, splitting_number=random_splitting_number, random_splitting_enabled=random_splitting_enabled) elif acq_func_name == "pc-roar-sigmoid-rs": runhistory2epm = RunHistory2EPM4Cost( scenario, num_params, success_states=[StatusType.SUCCESS]) select_configuration = SelectConfigurationsSigmoidRS( scenario=scenario, constant_pipeline_steps=constant_pipeline_steps, variable_pipeline_steps=variable_pipeline_steps, fraction=random_splitting_number) else: # Not a valid acquisition function raise ValueError("The provided acquisition function is not valid") # Build initial design # initial_design = RandomConfiguration(tae_runner=tae_runner, # scenario=scenario, # stats=stats, # traj_logger=traj_logger, # rng=rng) initial_configs = scenario.cs.sample_configuration(size=2) for config in initial_configs: config._populate_values() initial_design = MultiConfigInitialDesign( tae_runner=tae_runner, scenario=scenario, stats=stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, configs=initial_configs, intensifier=intensifier, aggregate_func=aggregate_func) # run id num_run = rng.randint(1234567980) # Build pc_smbo if acq_func_name not in ['pc-roar-sigmoid-rs']: smbo = PCSMBO(scenario=scenario, stats=stats, initial_design=initial_design, runhistory=runhistory, runhistory2epm=runhistory2epm, intensifier=intensifier, aggregate_func=aggregate_func, num_run=num_run, model=model, rng=rng, select_configuration=select_configuration, double_intensification=double_intensification) else: smbo = PCSMBOSigmoidRandomSearch( scenario=scenario, stats=stats, initial_design=initial_design, runhistory=runhistory, runhistory2epm=runhistory2epm, intensifier=intensifier, aggregate_func=aggregate_func, num_run=num_run, model=model, rng=rng, select_configuration=select_configuration) return smbo
def __init__( self, scenario: Scenario, # TODO: once we drop python3.4 add type hint # typing.Union[ExecuteTARun, callable] tae_runner=None, runhistory: RunHistory = None, intensifier: Intensifier = None, acquisition_function: AbstractAcquisitionFunction = None, model: AbstractEPM = None, runhistory2epm: AbstractRunHistory2EPM = None, initial_design: InitialDesign = None, initial_configurations: typing.List[Configuration] = None, stats: Stats = None, rng: np.random.RandomState = None): ''' Facade to use SMAC default mode Parameters ---------- scenario: smac.scenario.scenario.Scenario Scenario object tae_runner: ExecuteTARun or callable Callable or implementation of :class:`ExecuteTaRun`. In case a callable is passed it will be wrapped by tae.ExecuteTaFunc(). If not set, tae_runner will be initialized with the tae.ExecuteTARunOld() runhistory: RunHistory runhistory to store all algorithm runs intensifier: Intensifier intensification object to issue a racing to decide the current incumbent acquisition_function : AcquisitionFunction Object that implements the AbstractAcquisitionFunction. Will use EI if not set. model : AbstractEPM Model that implements train() and predict(). Will use a RandomForest if not set. runhistory2epm : RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use RunHistory2EPM4Cost if objective is cost or RunHistory2EPM4LogCost if objective is runtime. initial_design: InitialDesign initial sampling design initial_configurations: typing.List[Configuration] list of initial configurations for initial design -- cannot be used together with initial_design stats: Stats optional stats object rng: np.random.RandomState Random number generator ''' self.logger = logging.getLogger("SMAC") aggregate_func = average_cost # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario) # initialize empty runhistory if runhistory is None: runhistory = RunHistory(aggregate_func=aggregate_func) # initial random number generator num_run, rng = self._get_rng(rng=rng) # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed(rng.randint(MAXINT)) # initial Trajectory Logger traj_logger = TrajLogger(output_dir=scenario.output_dir, stats=self.stats) # initial EPM types = get_types(scenario.cs, scenario.feature_array) if model is None: model = RandomForestWithInstances( types=types, instance_features=scenario.feature_array, seed=rng.randint(MAXINT)) # initial acquisition function if acquisition_function is None: acquisition_function = EI(model=model) # initialize optimizer on acquisition function local_search = LocalSearch(acquisition_function, scenario.cs) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file if tae_runner is None: tae_runner = ExecuteTARunOld(ta=scenario.ta, stats=self.stats, run_obj=scenario.run_obj, runhistory=runhistory, par_factor=scenario.par_factor) # Second case, the tae_runner is a function to be optimized elif callable(tae_runner): tae_runner = ExecuteTAFuncDict(ta=tae_runner, stats=self.stats, run_obj=scenario.run_obj, memory_limit=scenario.memory_limit, runhistory=runhistory, par_factor=scenario.par_factor) # Third case, if it is an ExecuteTaRun we can simply use the # instance. Otherwise, the next check raises an exception elif not isinstance(tae_runner, ExecuteTARun): raise TypeError("Argument 'tae_runner' is %s, but must be " "either a callable or an instance of " "ExecuteTaRun. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # Check that overall objective and tae objective are the same if tae_runner.run_obj != scenario.run_obj: raise ValueError("Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner.run_obj, scenario.run_obj)) # inject stats if necessary if tae_runner.stats is None: tae_runner.stats = self.stats # inject runhistory if necessary if tae_runner.runhistory is None: tae_runner.runhistory = runhistory # initial intensification if intensifier is None: intensifier = Intensifier( tae_runner=tae_runner, stats=self.stats, traj_logger=traj_logger, rng=rng, instances=scenario.train_insts, cutoff=scenario.cutoff, deterministic=scenario.deterministic, run_obj_time=scenario.run_obj == "runtime", instance_specifics=scenario.instance_specific, minR=scenario.minR, maxR=scenario.maxR) # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both" ) if initial_configurations is not None: initial_design = MultiConfigInitialDesign( tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, configs=initial_configurations, intensifier=intensifier, aggregate_func=aggregate_func) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": initial_design = DefaultConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) elif scenario.initial_incumbent == "RANDOM": initial_design = RandomConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, rng=rng) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) # initial conversion of runhistory into EPM data if runhistory2epm is None: num_params = len(scenario.cs.get_hyperparameters()) if scenario.run_obj == "runtime": # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log10(scenario.cutoff) threshold = np.log10(scenario.cutoff * scenario.par_factor) imputor = RFRImputator(rs=rng, cutoff=cutoff, threshold=threshold, model=model, change_threshold=0.01, max_iter=2) runhistory2epm = RunHistory2EPM4LogCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.TIMEOUT, ], imputor=imputor) elif scenario.run_obj == 'quality': runhistory2epm = RunHistory2EPM4Cost\ (scenario=scenario, num_params=num_params, success_states=[StatusType.SUCCESS, ], impute_censored_data=False, impute_state=None) else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) self.solver = SMBO(scenario=scenario, stats=self.stats, initial_design=initial_design, runhistory=runhistory, runhistory2epm=runhistory2epm, intensifier=intensifier, aggregate_func=aggregate_func, num_run=num_run, model=model, acq_optimizer=local_search, acquisition_func=acquisition_function, rng=rng)