def register_result(self, config: Configuration, loss: float, status: StatusType, update_model: bool = True, **kwargs) -> None: try: key = str(config) info, start_time = self.run_infos[key] end_time = time.time() del self.run_infos[key] result = RunValue(loss, end_time - start_time, SmacStatus(status.value), start_time, end_time, {}) if update_model: self._incorporate_run_results(info, result, 10) except KeyError: # Configuration was sampled during structure search and not via SMAC if update_model: self.runhistory.add( config=config, cost=loss, time=0.0, status=SmacStatus(status.value), instance_id=None, seed=0, )
def test_process_results(self): """ Makes sure that we can process the results of a completed configuration """ intent, run_info = self.intensifier.get_next_run( challengers=[self.config1, self.config2], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) result = RunValue( cost=1, time=0.5, status=StatusType.SUCCESS, starttime=1, endtime=2, additional_info=None, ) self.rh.add( config=run_info.config, cost=1, time=0.5, status=StatusType.SUCCESS, instance_id=run_info.instance, seed=run_info.seed, additional_info=None) incumbent, inc_perf = self.intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(incumbent, run_info.config) self.assertEqual(inc_perf, 1)
def run_wrapper( self, run_info: RunInfo, ) -> Tuple[RunInfo, RunValue]: """ wrapper function for ExecuteTARun.run_wrapper() to cap the target algorithm runtime if it would run over the total allowed runtime. Parameters ---------- run_info : RunInfo Object that contains enough information to execute a configuration run in isolation. Returns ------- RunInfo: an object containing the configuration launched RunValue: Contains information about the status/performance of config """ if self.budget_type is None: if run_info.budget != 0: raise ValueError( 'If budget_type is None, budget must be.0, but is %f' % run_info.budget) else: if run_info.budget == 0: run_info = run_info._replace(budget=100) elif run_info.budget <= 0 or run_info.budget > 100: raise ValueError( 'Illegal value for budget, must be >0 and <=100, but is %f' % run_info.budget) if self.budget_type not in ('subsample', 'iterations', 'mixed'): raise ValueError( "Illegal value for budget type, must be one of " "('subsample', 'iterations', 'mixed'), but is : %s" % self.budget_type) remaining_time = self.stats.get_remaing_time_budget() if remaining_time - 5 < run_info.cutoff: run_info = run_info._replace(cutoff=int(remaining_time - 5)) if run_info.cutoff < 1.0: return run_info, RunValue( status=StatusType.STOP, cost=self.worst_possible_result, time=0.0, additional_info={}, starttime=time.time(), endtime=time.time(), ) elif (run_info.cutoff != int(np.ceil(run_info.cutoff)) and not isinstance(run_info.cutoff, int)): run_info = run_info._replace(cutoff=int(np.ceil(run_info.cutoff))) return super().run_wrapper(run_info=run_info)
def target_from_run_info(RunInfo): value_from_config = sum( [a for a in RunInfo.config.get_dictionary().values()]) return RunValue(cost=value_from_config, time=0.5, status=StatusType.SUCCESS, starttime=time.time(), endtime=time.time() + 1, additional_info={})
def ensemble_run_history(request): run_history = RunHistory() run_history._add( RunKey(config_id=3, instance_id='{"task_id": "breast_cancer"}', seed=1, budget=3.0), RunValue(cost=0.11347517730496459, time=0.21858787536621094, status=None, starttime=time.time(), endtime=time.time(), additional_info={ 'duration': 0.20323538780212402, 'num_run': 3, 'configuration_origin': 'Random Search' }), status=None, origin=None, ) run_history._add( RunKey(config_id=6, instance_id='{"task_id": "breast_cancer"}', seed=1, budget=6.0), RunValue(cost=2 * 0.11347517730496459, time=2 * 0.21858787536621094, status=None, starttime=time.time(), endtime=time.time(), additional_info={ 'duration': 0.20323538780212402, 'num_run': 6, 'configuration_origin': 'Random Search' }), status=None, origin=None, ) return run_history
def test_process_results(self): """Ensures that the results are processed by the pertinent intensifer, based on the source id""" scheduler = ParallelScheduler( stats=None, traj_logger=None, instances=[1, 2, 3], rng=np.random.RandomState(12345), deterministic=True, ) scheduler.intensifier_instances = { 0: mock.Mock(), 1: mock.Mock(), 2: mock.Mock(), } run_info = RunInfo( config=None, instance=0, instance_specific="0", cutoff=None, seed=0, capped=False, budget=0.0, source_id=2, ) result = RunValue(cost=1, time=0.5, status=StatusType.SUCCESS, starttime=1, endtime=2, additional_info={}) scheduler.process_results(run_info=run_info, result=result, incumbent=None, run_history=None, time_bound=None) self.assertIsNone( scheduler.intensifier_instances[0].process_results.call_args) self.assertIsNone( scheduler.intensifier_instances[1].process_results.call_args) self.assertEqual( scheduler.intensifier_instances[2].process_results.call_args[1] ['run_info'], run_info)
def make_dict_run_history_data(data): run_history_data = dict() for row in data: run_key = RunKey( config_id=row[0][0], instance_id=row[0][1], seed=row[0][2], budget=row[0][3]) run_value = RunValue( cost=row[1][0], time=row[1][1], status=getattr(StatusType, row[1][2]['__enum__'].split(".")[-1]), starttime=row[1][3], endtime=row[1][4], additional_info=row[1][5], ) run_history_data[run_key] = run_value return run_history_data
def test_incorporate_run_results_callback(self, process_results_mock): process_results_mock.return_value = None, None class TestCallback(IncorporateRunResultCallback): def __init__(self): self.num_call = 0 def __call__(self, smbo, run_info, result, time_left) -> None: self.num_call += 1 self.config = run_info.config callback = TestCallback() self.scenario.output_dir = None smac = SMAC4AC(self.scenario) smac.register_callback(callback) self.output_dirs.append(smac.output_dir) smbo = smac.solver config = self.scenario.cs.sample_configuration() run_info = RunInfo( config=config, instance=None, instance_specific=None, seed=1, cutoff=None, capped=False, budget=0.0, source_id=0, ) result = RunValue(1.2345, 2.3456, "status", "starttime", "endtime", "additional_info") time_left = 10 smbo._incorporate_run_results(run_info=run_info, result=result, time_left=time_left) self.assertEqual(callback.num_call, 1) self.assertEqual(callback.config, config)
def run_wrapper( self, run_info: RunInfo, ) -> typing.Tuple[RunInfo, RunValue]: """Wrapper around run() to exec and check the execution of a given config file This function encapsulates common handling/processing, so that run() implementation is simplified. Parameters ---------- run_info : RunInfo Object that contains enough information to execute a configuration run in isolation. Returns ------- RunInfo: an object containing the configuration launched RunValue: Contains information about the status/performance of config """ start = time.time() if run_info.cutoff is None and self.run_obj == "runtime": if self.logger: self.logger.critical( "For scenarios optimizing running time " "(run objective), a cutoff time is required, " "but not given to this call.") raise ValueError("For scenarios optimizing running time " "(run objective), a cutoff time is required, " "but not given to this call.") cutoff = None if run_info.cutoff is not None: cutoff = int(math.ceil(run_info.cutoff)) try: status, cost, runtime, additional_info = self.run( config=run_info.config, instance=run_info.instance, cutoff=cutoff, seed=run_info.seed, budget=run_info.budget, instance_specific=run_info.instance_specific) except Exception as e: status = StatusType.CRASHED cost = self.cost_for_crash runtime = time.time() - start # Add context information to the error message exception_traceback = traceback.format_exc() error_message = repr(e) additional_info = { 'traceback': exception_traceback, 'error': error_message } end = time.time() if run_info.budget == 0 and status == StatusType.DONOTADVANCE: raise ValueError( "Cannot handle DONOTADVANCE state when using intensify or SH/HB on " "instances.") # Catch NaN or inf. if (self.run_obj == 'runtime' and not np.isfinite(runtime) or self.run_obj == 'quality' and not np.isfinite(cost)): if self.logger: self.logger.warning( "Target Algorithm returned NaN or inf as {}. " "Algorithm run is treated as CRASHED, cost " "is set to {} for quality scenarios. " "(Change value through \"cost_for_crash\"" "-option.)".format(self.run_obj, self.cost_for_crash)) status = StatusType.CRASHED if self.run_obj == "runtime": # The following line pleases mypy - we already check for cutoff not being none above, # prior to calling run. However, mypy assumes that the data type of cutoff # is still Optional[int] assert cutoff is not None if runtime > self.par_factor * cutoff: self.logger.warning("Returned running time is larger " "than {0} times the passed cutoff time. " "Clamping to {0} x cutoff.".format( self.par_factor)) runtime = cutoff * self.par_factor status = StatusType.TIMEOUT if status == StatusType.SUCCESS: cost = runtime else: cost = cutoff * self.par_factor if status == StatusType.TIMEOUT and run_info.capped: status = StatusType.CAPPED else: if status == StatusType.CRASHED: cost = self.cost_for_crash return run_info, RunValue(status=status, cost=cost, time=runtime, additional_info=additional_info, starttime=start, endtime=end)
def test_process_results_via_sourceid(self): """Makes sure source id is honored when deciding which_HB instance will consume the result/run_info """ # Mock the_HB instance so we can make sure the correct item is passed for i in range(10): self.HB.intensifier_instances[i] = mock.Mock() self.HB.intensifier_instances[i].process_results.return_value = ( self.config1, 0.5) # make iter false so the mock object is not overwritten self.HB.intensifier_instances[i].iteration_done = False # randomly create run_infos and push into HB. Then we will make # sure they got properly allocated for i in np.random.choice(list(range(10)), 30): run_info = RunInfo( config=self.config1, instance=0, instance_specific="0", cutoff=None, seed=0, capped=False, budget=0.0, source_id=i, ) # make sure results aren't messed up via magic variable # That is we check only the proper_HB instance has this magic = time.time() result = RunValue(cost=1, time=0.5, status=StatusType.SUCCESS, starttime=1, endtime=2, additional_info=magic) self.HB.process_results(run_info=run_info, incumbent=None, run_history=self.rh, time_bound=None, result=result, log_traj=False) # Check the call arguments of each sh instance and make sure # it is the correct one # First the expected one self.assertEqual( self.HB.intensifier_instances[i].process_results.call_args[1] ['run_info'], run_info) self.assertEqual( self.HB.intensifier_instances[i].process_results.call_args[1] ['result'], result) all_other_run_infos, all_other_results = [], [] for j in range(len(self.HB.intensifier_instances)): # Skip the expected_HB instance if i == j: continue if self.HB.intensifier_instances[ j].process_results.call_args is None: all_other_run_infos.append(None) else: all_other_run_infos.append( self.HB.intensifier_instances[j].process_results. call_args[1]['run_info']) all_other_results.append( self.HB.intensifier_instances[j].process_results. call_args[1]['result']) self.assertNotIn(run_info, all_other_run_infos) self.assertNotIn(result, all_other_results)
def run_wrapper( self, run_info: RunInfo, ) -> Tuple[RunInfo, RunValue]: """ wrapper function for ExecuteTARun.run_wrapper() to cap the target algorithm runtime if it would run over the total allowed runtime. Args: run_info (RunInfo): Object that contains enough information to execute a configuration run in isolation. Returns: RunInfo: an object containing the configuration launched RunValue: Contains information about the status/performance of config """ if self.budget_type is None: if run_info.budget != 0: raise ValueError( 'If budget_type is None, budget must be.0, but is %f' % run_info.budget) else: if run_info.budget == 0: # SMAC can return budget zero for intensifiers that don't have a concept # of budget, for example a simple bayesian optimization intensifier. # Budget determines how our pipeline trains, which can be via runtime or epochs epochs_budget = self.pipeline_config.get('epochs', np.inf) runtime_budget = self.pipeline_config.get('runtime', np.inf) run_info = run_info._replace( budget=min(epochs_budget, runtime_budget)) elif run_info.budget <= 0: raise ValueError( 'Illegal value for budget, must be greater than zero but is %f' % run_info.budget) if self.budget_type not in ('epochs', 'runtime'): raise ValueError( "Illegal value for budget type, must be one of " "('epochs', 'runtime'), but is : %s" % self.budget_type) remaining_time = self.stats.get_remaing_time_budget() if remaining_time - 5 < run_info.cutoff: run_info = run_info._replace(cutoff=int(remaining_time - 5)) if run_info.cutoff < 1.0: return run_info, RunValue( status=StatusType.STOP, cost=self.worst_possible_result, time=0.0, additional_info={}, starttime=time.time(), endtime=time.time(), ) elif (run_info.cutoff != int(np.ceil(run_info.cutoff)) and not isinstance(run_info.cutoff, int)): run_info = run_info._replace(cutoff=int(np.ceil(run_info.cutoff))) self.logger.info("Starting to evaluate configuration %s" % run_info.config.config_id) run_info, run_value = super().run_wrapper(run_info=run_info) return run_info, run_value