def test_abort_on_runner(self, patch): def target(x): return 5 # should raise an error if abort_on_first_run_crash is True patch.side_effect = FirstRunCrashedException() scen = Scenario({ 'cs': test_helpers.get_branin_config_space(), 'run_obj': 'quality', 'output_dir': 'data-test_smbo-abort', 'abort_on_first_run_crash': True }) self.output_dirs.append(scen.output_dir) smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver self.assertRaises(FirstRunCrashedException, smbo.run) # should not raise an error if abort_on_first_run_crash is False patch.side_effect = FirstRunCrashedException() scen = Scenario({ 'cs': test_helpers.get_branin_config_space(), 'run_obj': 'quality', 'output_dir': 'data-test_smbo-abort', 'abort_on_first_run_crash': False, 'wallclock-limit': 1 }) self.output_dirs.append(scen.output_dir) smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver try: smbo.start() smbo.run() except FirstRunCrashedException: self.fail('Raises FirstRunCrashedException unexpectedly!')
def test_abort_on_initial_design(self, patch): def target(x): return 5 # should raise an error if abort_on_first_run_crash is True patch.side_effect = FirstRunCrashedException() scen = Scenario({ "cs": test_helpers.get_branin_config_space(), "run_obj": "quality", "output_dir": "data-test_smbo-abort", "abort_on_first_run_crash": True, "deterministic": False, "limit_resources": True, }) self.output_dirs.append(scen.output_dir) smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver with self.assertRaisesRegex(FirstRunCrashedException, "in _mock_call"): smbo.run() # should not raise an error if abort_on_first_run_crash is False patch.side_effect = FirstRunCrashedException() scen = Scenario({ "cs": test_helpers.get_branin_config_space(), "run_obj": "quality", "output_dir": "data-test_smbo-abort", "abort_on_first_run_crash": False, "wallclock-limit": 1, "deterministic": False, "limit_resources": True, }) self.output_dirs.append(scen.output_dir) smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver try: smbo.start() smbo.run() except FirstRunCrashedException: self.fail("Raises FirstRunCrashedException unexpectedly!")
def _incorporate_run_results(self, run_info: RunInfo, result: RunValue, time_left: float) -> None: """ The SMBO submits a config-run-request via a RunInfo object. When that config run is completed, a RunValue, which contains all the relevant information obtained after running a job, is returned. This method incorporates the status of that run into the stats/runhistory objects so that other consumers can advance with their task. Additionally, it checks for a new incumbent via the intensifier process results, which also has the side effect of moving the intensifier to a new state Parameters ---------- run_info: RunInfo Describes the run (config) from which to process the results result: RunValue Contains relevant information regarding the execution of a config time_left: float time in [sec] available to perform intensify """ # update SMAC stats self.stats.ta_time_used += float(result.time) self.stats.finished_ta_runs += 1 self.logger.debug( f"Return: Status: {result.status}, cost: {result.cost}, time: {result.time}, " f"additional: {result.additional_info}") self.runhistory.add( config=run_info.config, cost=result.cost, time=result.time, status=result.status, instance_id=run_info.instance, seed=run_info.seed, budget=run_info.budget, starttime=result.starttime, endtime=result.endtime, force_update=True, additional_info=result.additional_info, ) self.stats.n_configs = len(self.runhistory.config_ids) if result.status == StatusType.ABORT: raise TAEAbortException( "Target algorithm status ABORT - SMAC will " "exit. The last incumbent can be found " "in the trajectory-file.") elif result.status == StatusType.STOP: self._stop = True return if self.scenario.abort_on_first_run_crash: # type: ignore[attr-defined] # noqa F821 if self.stats.finished_ta_runs == 1 and result.status == StatusType.CRASHED: raise FirstRunCrashedException( "First run crashed, abort. Please check your setup -- we assume that your default " "configuration does not crashes. (To deactivate this exception, use the SMAC scenario option " "'abort_on_first_run_crash'). Additional run info: %s" % result.additional_info) # Update the intensifier with the result of the runs self.incumbent, inc_perf = self.intensifier.process_results( run_info=run_info, incumbent=self.incumbent, run_history=self.runhistory, time_bound=max(self._min_time, time_left), result=result, ) for callback in self._callbacks["_incorporate_run_results"]: response = callback(smbo=self, run_info=run_info, result=result, time_left=time_left) # If a callback returns False, the optimization loop should be interrupted # the other callbacks are still being called if response is False: self.logger.debug( "An IncorporateRunResultCallback returned False, requesting abort." ) self._stop = True if self.scenario.save_instantly: # type: ignore[attr-defined] # noqa F821 self.save() return