def test_abort_on_runner(self, patch):
def target(x):
return 5
# should raise an error if abort_on_first_run_crash is True
patch.side_effect = FirstRunCrashedException()
scen = Scenario({
'cs': test_helpers.get_branin_config_space(),
'run_obj': 'quality',
'output_dir': 'data-test_smbo-abort',
'abort_on_first_run_crash': True
})
self.output_dirs.append(scen.output_dir)
smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver
self.assertRaises(FirstRunCrashedException, smbo.run)
# should not raise an error if abort_on_first_run_crash is False
patch.side_effect = FirstRunCrashedException()
scen = Scenario({
'cs': test_helpers.get_branin_config_space(),
'run_obj': 'quality',
'output_dir': 'data-test_smbo-abort',
'abort_on_first_run_crash': False,
'wallclock-limit': 1
})
self.output_dirs.append(scen.output_dir)
smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver
try:
smbo.start()
smbo.run()
except FirstRunCrashedException:
self.fail('Raises FirstRunCrashedException unexpectedly!')
def test_abort_on_initial_design(self, patch):
def target(x):
return 5
# should raise an error if abort_on_first_run_crash is True
patch.side_effect = FirstRunCrashedException()
scen = Scenario({
"cs": test_helpers.get_branin_config_space(),
"run_obj": "quality",
"output_dir": "data-test_smbo-abort",
"abort_on_first_run_crash": True,
"deterministic": False,
"limit_resources": True,
})
self.output_dirs.append(scen.output_dir)
smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver
with self.assertRaisesRegex(FirstRunCrashedException, "in _mock_call"):
smbo.run()
# should not raise an error if abort_on_first_run_crash is False
patch.side_effect = FirstRunCrashedException()
scen = Scenario({
"cs": test_helpers.get_branin_config_space(),
"run_obj": "quality",
"output_dir": "data-test_smbo-abort",
"abort_on_first_run_crash": False,
"wallclock-limit": 1,
"deterministic": False,
"limit_resources": True,
})
self.output_dirs.append(scen.output_dir)
smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver
try:
smbo.start()
smbo.run()
except FirstRunCrashedException:
self.fail("Raises FirstRunCrashedException unexpectedly!")
def _incorporate_run_results(self, run_info: RunInfo, result: RunValue,
time_left: float) -> None:
"""
The SMBO submits a config-run-request via a RunInfo object.
When that config run is completed, a RunValue, which contains
all the relevant information obtained after running a job, is returned.
This method incorporates the status of that run into
the stats/runhistory objects so that other consumers
can advance with their task.
Additionally, it checks for a new incumbent via the intensifier process results,
which also has the side effect of moving the intensifier to a new state
Parameters
----------
run_info: RunInfo
Describes the run (config) from which to process the results
result: RunValue
Contains relevant information regarding the execution of a config
time_left: float
time in [sec] available to perform intensify
"""
# update SMAC stats
self.stats.ta_time_used += float(result.time)
self.stats.finished_ta_runs += 1
self.logger.debug(
f"Return: Status: {result.status}, cost: {result.cost}, time: {result.time}, "
f"additional: {result.additional_info}")
self.runhistory.add(
config=run_info.config,
cost=result.cost,
time=result.time,
status=result.status,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
starttime=result.starttime,
endtime=result.endtime,
force_update=True,
additional_info=result.additional_info,
)
self.stats.n_configs = len(self.runhistory.config_ids)
if result.status == StatusType.ABORT:
raise TAEAbortException(
"Target algorithm status ABORT - SMAC will "
"exit. The last incumbent can be found "
"in the trajectory-file.")
elif result.status == StatusType.STOP:
self._stop = True
return
if self.scenario.abort_on_first_run_crash: # type: ignore[attr-defined] # noqa F821
if self.stats.finished_ta_runs == 1 and result.status == StatusType.CRASHED:
raise FirstRunCrashedException(
"First run crashed, abort. Please check your setup -- we assume that your default "
"configuration does not crashes. (To deactivate this exception, use the SMAC scenario option "
"'abort_on_first_run_crash'). Additional run info: %s" %
result.additional_info)
# Update the intensifier with the result of the runs
self.incumbent, inc_perf = self.intensifier.process_results(
run_info=run_info,
incumbent=self.incumbent,
run_history=self.runhistory,
time_bound=max(self._min_time, time_left),
result=result,
)
for callback in self._callbacks["_incorporate_run_results"]:
response = callback(smbo=self,
run_info=run_info,
result=result,
time_left=time_left)
# If a callback returns False, the optimization loop should be interrupted
# the other callbacks are still being called
if response is False:
self.logger.debug(
"An IncorporateRunResultCallback returned False, requesting abort."
)
self._stop = True
if self.scenario.save_instantly: # type: ignore[attr-defined] # noqa F821
self.save()
return