def test_serial_runs(self):
# We use the funcdict as a mechanism to test SerialRunner
runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality')
self.assertIsInstance(runner, SerialRunner)
run_info = RunInfo(config=2, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
run_info = RunInfo(config=3, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
run_values = runner.get_finished_runs()
self.assertEqual(len(run_values), 2)
# To make sure runs launched serially, we just make sure that the end time of
# a run is later than the other
# Results are returned in left to right
self.assertLessEqual(int(run_values[1][1].endtime), int(run_values[0][1].starttime))
# No wait time in serial runs!
start = time.time()
runner.wait()
# The run takes a second, so 0.5 is sufficient
self.assertLess(time.time() - start, 0.5)
pass
def test_eval_with_limits_holdout_fail_silent(self, pynisher_mock):
pynisher_mock.return_value = None
config = unittest.mock.Mock()
config.origin = 'MOCK'
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=self.backend,
autosklearn_seed=1,
port=self.logger_port,
resampling_strategy='holdout',
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
pynisher_context='fork',
)
# The following should not fail because abort on first config crashed is false
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=60,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertEqual(
info[1].additional_info, {
'configuration_origin': 'MOCK',
'error': "Result queue is empty",
'exit_status': 0,
'exitcode': 0,
'subprocess_stdout': '',
'subprocess_stderr': ''
})
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertEqual(
info[1].additional_info, {
'configuration_origin': 'MOCK',
'error': "Result queue is empty",
'exit_status': 0,
'exitcode': 0,
'subprocess_stdout': '',
'subprocess_stderr': ''
})
def test_eval_with_limits_holdout_timeout_with_results_in_queue(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': {}})
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.side_effect = side_effect
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
# Test for a succesful run
ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1,
port=self.logger_port,
resampling_strategy='holdout',
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
pynisher_context='forkserver',
)
info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=None,
instance_specific=None, seed=1, capped=False))
self.assertEqual(info[1].status, StatusType.SUCCESS)
self.assertEqual(info[1].cost, 0.5)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
# And a crashed run which is in the queue
def side_effect(**kwargs):
queue = kwargs['queue']
queue.put({'status': StatusType.CRASHED,
'loss': 2.0,
'additional_run_info': {}})
m2.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1,
port=self.logger_port,
resampling_strategy='holdout',
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
pynisher_context='forkserver',
)
info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=None,
instance_specific=None, seed=1, capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
def run_wrapper(
self,
run_info: RunInfo,
) -> Tuple[RunInfo, RunValue]:
"""
wrapper function for ExecuteTARun.run_wrapper() to cap the target algorithm
runtime if it would run over the total allowed runtime.
Parameters
----------
run_info : RunInfo
Object that contains enough information to execute a configuration run in
isolation.
Returns
-------
RunInfo:
an object containing the configuration launched
RunValue:
Contains information about the status/performance of config
"""
if self.budget_type is None:
if run_info.budget != 0:
raise ValueError(
'If budget_type is None, budget must be.0, but is %f' %
run_info.budget)
else:
if run_info.budget == 0:
run_info = run_info._replace(budget=100)
elif run_info.budget <= 0 or run_info.budget > 100:
raise ValueError(
'Illegal value for budget, must be >0 and <=100, but is %f'
% run_info.budget)
if self.budget_type not in ('subsample', 'iterations', 'mixed'):
raise ValueError(
"Illegal value for budget type, must be one of "
"('subsample', 'iterations', 'mixed'), but is : %s" %
self.budget_type)
remaining_time = self.stats.get_remaing_time_budget()
if remaining_time - 5 < run_info.cutoff:
run_info = run_info._replace(cutoff=int(remaining_time - 5))
if run_info.cutoff < 1.0:
return run_info, RunValue(
status=StatusType.STOP,
cost=self.worst_possible_result,
time=0.0,
additional_info={},
starttime=time.time(),
endtime=time.time(),
)
elif (run_info.cutoff != int(np.ceil(run_info.cutoff))
and not isinstance(run_info.cutoff, int)):
run_info = run_info._replace(cutoff=int(np.ceil(run_info.cutoff)))
return super().run_wrapper(run_info=run_info)
def test_parallel_runs(self):
"""Make sure because there are 2 workers, the runs are launched
closely in time together"""
# We use the funcdict as a mechanism to test Runner
runner = ExecuteTAFuncDict(ta=target_delayed,
stats=self.stats,
run_obj='quality')
runner = DaskParallelRunner(runner, n_workers=2)
run_info = RunInfo(config=2,
instance='test',
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0)
runner.submit_run(run_info)
run_info = RunInfo(config=3,
instance='test',
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0)
runner.submit_run(run_info)
# At this stage, we submitted 2 jobs, that are running in remote
# workers. We have to wait for each one of them to complete. The
# runner provides a wait() method to do so, yet it can only wait for
# a single job to be completed. It does internally via dask wait(<list of futures>)
# so we take wait for the first job to complete, and take it out
runner.wait()
run_values = runner.get_finished_runs()
# To be on the safe side, we don't check for:
# self.assertEqual(len(run_values), 1)
# In the ideal world, two runs were launched which take the same time
# so waiting for one means, the second one is completed. But some
# overhead might cause it to be delayed.
# Above took the first run results and put it on run_values
# But for this check we need the second run we submitted
# Again, the runs might take slightly different time depending if we have
# heterogeneous workers, so calling wait 2 times is a guarantee that 2
# jobs are finished
runner.wait()
run_values.extend(runner.get_finished_runs())
self.assertEqual(len(run_values), 2)
# To make it is parallel, we just make sure that the start of the second
# run is earlier than the end of the first run
# Results are returned in left to right
self.assertLessEqual(int(run_values[0][1].starttime),
int(run_values[1][1].endtime))
def test_eval_with_limits_holdout_2(self, eval_houldout_mock):
config = unittest.mock.Mock()
config.config_id = 198
def side_effect(*args, **kwargs):
queue = kwargs['queue']
queue.put({'status': StatusType.SUCCESS,
'loss': 0.5,
'additional_run_info': kwargs['instance']})
eval_houldout_mock.side_effect = side_effect
ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1,
port=self.logger_port,
resampling_strategy='holdout',
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
pynisher_context='fork',
)
self.scenario.wallclock_limit = 180
instance = "{'subsample': 30}"
info = ta.run_wrapper(RunInfo(config=config, cutoff=30, instance=instance,
instance_specific=None, seed=1, capped=False))
self.assertEqual(info[1].status, StatusType.SUCCESS)
self.assertEqual(len(info[1].additional_info), 2)
self.assertIn('configuration_origin', info[1].additional_info)
self.assertEqual(info[1].additional_info['message'], "{'subsample': 30}")
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock):
pynisher_mock.side_effect = MemoryError
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.MEMOUT)
# For accuracy, worst possible result is MAXINT
worst_possible_result = 1
self.assertEqual(info[1].cost, worst_possible_result)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
def test_run(self):
"""Makes sure that we are able to run a configuration and
return the expected values/types"""
# We use the funcdict as a mechanism to test SerialRunner
runner = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj='quality')
self.assertIsInstance(runner, SerialRunner)
run_info = RunInfo(config=2,
instance='test',
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0)
# submit runs! then get the value
runner.submit_run(run_info)
run_values = runner.get_finished_runs()
self.assertEqual(len(run_values), 1)
self.assertIsInstance(run_values, list)
self.assertIsInstance(run_values[0][0], RunInfo)
self.assertIsInstance(run_values[0][1], RunValue)
self.assertEqual(run_values[0][1].cost, 4)
self.assertEqual(run_values[0][1].status, StatusType.SUCCESS)
def test_start_tae_return_abort(self, test_run):
'''
testing abort
'''
# Patch run-function for custom-return
test_run.return_value = StatusType.ABORT, 12345.0, 1.2345, {}
scen = Scenario(
scenario={
'cs': ConfigurationSpace(),
'run_obj': 'quality',
'output_dir': '',
},
cmd_options=None,
)
stats = Stats(scen)
stats.start_timing()
eta = SerialRunner(ta=lambda *args: None, stats=stats)
_, run_value = eta.run_wrapper(
RunInfo(config=None,
instance=1,
instance_specific=None,
cutoff=30,
seed=None,
capped=False,
budget=0.0))
self.assertEqual(run_value.status, StatusType.ABORT)
def test_eval_with_limits_holdout(self, pynisher_mock):
pynisher_mock.side_effect = safe_eval_success_mock
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[0].config.config_id, 198)
self.assertEqual(info[1].status, StatusType.SUCCESS, info)
self.assertEqual(info[1].cost, 0.5)
self.assertIsInstance(info[1].time, float)
def test_exception_in_target_function(self, eval_holdout_mock):
config = unittest.mock.Mock()
config.config_id = 198
eval_holdout_mock.side_effect = ValueError
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.CRASHED)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertEqual(info[1].additional_info['error'], 'ValueError()')
self.assertIn('traceback', info[1].additional_info)
self.assertNotIn('exitcode', info[1].additional_info)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
self.stats.ta_runs = 1
ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4)
self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'],
int)
def test_eval_with_limits_holdout_fail_timeout(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
m1 = unittest.mock.Mock()
m2 = unittest.mock.Mock()
m1.return_value = m2
pynisher_mock.return_value = m1
m2.exit_status = pynisher.TimeoutException
m2.wall_clock_time = 30
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.TIMEOUT)
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertNotIn('exitcode', info[1].additional_info)
def test_eval_with_limits_holdout_fail_memory_error(self, pynisher_mock):
pynisher_mock.side_effect = MemoryError
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
logger=self.logger,
stats=self.stats,
memory_limit=3072,
metric=log_loss,
cost_for_crash=get_cost_of_crash(log_loss),
abort_on_first_run_crash=False,
)
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status, StatusType.MEMOUT)
# For logloss, worst possible result is MAXINT
worst_possible_result = MAXINT
self.assertEqual(info[1].cost, worst_possible_result)
self.assertIsInstance(info[1].time, float)
def test_race_challenger_2(self):
"""
Makes sure that a racing configuration with better performance,
that is capped, doesn't substitute the incumbent.
"""
def target(x):
time.sleep(1.5)
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
)
self.rh.add(
config=self.config1,
cost=0.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None,
)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=True,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_additional_info_crash_msg(self):
"""
We want to make sure we catch errors as additional info,
and in particular when doing multiprocessing runs, we
want to make sure we capture dask exceptions
"""
def target_nonpickable(x, seed, instance):
return x**2, {'key': seed, 'instance': instance}
runner = ExecuteTAFuncDict(ta=target_nonpickable, stats=self.stats, run_obj='quality')
runner = DaskParallelRunner(runner, n_workers=2)
run_info = RunInfo(config=2, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
runner.wait()
run_info, result = runner.get_finished_runs()[0]
# Make sure the traceback message is included
self.assertIn('traceback', result.additional_info)
self.assertIn(
# We expect the problem to occur in the run wrapper
# So traceback should show this!
'target_nonpickable',
result.additional_info['traceback'])
# Make sure the error message is included
self.assertIn('error', result.additional_info)
self.assertIn(
'Can\'t pickle local object',
result.additional_info['error'])
def test_get_next_run_waits_if_no_workers(self):
"""
In the case all workers are busy, we wait so that we do
not saturate the process with configurations that will not
finish in time
"""
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config1, self.config2],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
# We can get the configuration 1
self.assertEqual(intent, RunInfoIntent.RUN)
self.assertEqual(run_info, RunInfo(
config=self.config1,
instance=1,
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0,
))
# We should not get configuration 2
# As there is just 1 worker
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config2],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
self.assertEqual(intent, RunInfoIntent.WAIT)
self.assertEqual(run_info, RunInfo(
config=None,
instance=None,
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0,
))
def test_crashed_cost_value(self, test_run):
'''
test cost on crashed runs
'''
# Patch run-function for custom-return
scen = Scenario(scenario={
'cs': ConfigurationSpace(),
'run_obj': 'quality'
},
cmd_options=None)
stats = Stats(scen)
stats.start_timing()
stats.submitted_ta_runs += 1
# Check quality
test_run.return_value = StatusType.CRASHED, np.nan, np.nan, {}
eta = SerialRunner(ta=lambda *args: None,
stats=stats,
run_obj='quality',
cost_for_crash=100)
run_info, result = eta.run_wrapper(
RunInfo(config={},
instance=1,
instance_specific="0",
cutoff=None,
seed=None,
capped=False,
budget=0.0))
self.assertEqual(100, result.cost)
# Check runtime
eta = SerialRunner(ta=lambda *args: None,
stats=stats,
run_obj='runtime',
cost_for_crash=10.7)
run_info, result = eta.run_wrapper(
RunInfo(config={},
instance=1,
instance_specific="0",
cutoff=20,
seed=None,
capped=False,
budget=0.0))
self.assertEqual(20.0, result.cost)
def mock_get_next_run(**kwargs):
config = cs.sample_configuration()
all_configs.append(config)
return (RunInfoIntent.RUN,
RunInfo(config=config,
instance=time.time() % 10,
instance_specific={},
seed=0,
cutoff=None,
capped=False,
budget=0.0))
def sample_config(self, recursion_depth: int = 5) -> Configuration:
# TODO even though initial_incumbent is set to DEFAULT, the first configuration is random
# sample next configuration for intensification
# Initial design runs are also included in the BO loop now.
intent, run_info = self.intensifier.get_next_run(
challengers=self.initial_design_configs,
incumbent=self.incumbent,
chooser=self.epm_chooser,
run_history=self.runhistory,
repeat_configs=self.intensifier.repeat_configs,
num_workers=self.tae_runner.num_workers(),
)
# remove config from initial design challengers to not repeat it again
self.initial_design_configs = [
c for c in self.initial_design_configs if c != run_info.config
]
if intent == RunInfoIntent.SKIP:
if recursion_depth > 0:
return self.sample_config(recursion_depth - 1)
else:
self.logger.warning(
'Repeatedly failed to sample configuration. Using random configuration instead.'
)
config = self.config_space.sample_configuration()
config.origin = 'Random Search'
run_info = RunInfo(config, run_info.instance,
run_info.instance_specific, run_info.seed,
run_info.cutoff, run_info.capped,
run_info.budget, run_info.source_id)
if run_info.config is None:
config = self.config_space.sample_configuration()
config.origin = 'Random Search'
else:
config = run_info.config
self.runhistory.add(
config=config,
cost=float(MAXINT),
time=0.0,
status=SmacStatus.RUNNING,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
self.stats.submitted_ta_runs += 1
self.run_infos[str(config)] = (run_info, time.time())
return config
def test_race_challenger_1(self):
"""
Makes sure that a racing configuration with better performance,
is selected as incumbent
No adaptive capping
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
intensifier.N = 1
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
cutoff=cutoff,
seed=seed,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def get_next_run(self,
challengers: typing.Optional[typing.List[Configuration]],
incumbent: Configuration,
chooser: typing.Optional[EPMChooser],
run_history: RunHistory,
repeat_configs: bool = True,
num_workers: int = 1,
) -> typing.Tuple[RunInfoIntent, RunInfo]:
"""
Selects which challenger to be used. As in a traditional BO loop,
we sample from the EPM, which is the next configuration based on
the acquisition function. The input data is read from the runhistory.
Parameters
----------
challengers : typing.List[Configuration]
promising configurations
incumbent: Configuration
incumbent configuration
chooser : smac.optimizer.epm_configuration_chooser.EPMChooser
optimizer that generates next configurations to use for racing
run_history : smac.runhistory.runhistory.RunHistory
stores all runs we ran so far
repeat_configs : bool
if False, an evaluated configuration will not be generated again
num_workers: int
the maximum number of workers available
at a given time.
Returns
-------
intent: RunInfoIntent
Indicator of how to consume the RunInfo object
run_info: RunInfo
An object that encapsulates the minimum information to
evaluate a configuration
"""
# We always sample from the configs provided or the EPM
challenger = self._next_challenger(challengers=challengers,
chooser=chooser,
run_history=run_history,
repeat_configs=repeat_configs)
return RunInfoIntent.RUN, RunInfo(
config=challenger,
instance=self.instances[-1],
instance_specific="0",
seed=0 if self.deterministic else self.rs.randint(low=0, high=MAXINT, size=1)[0],
cutoff=self.cutoff,
capped=False,
budget=0.0,
)
def test_zero_or_negative_cutoff(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(backend=self.backend, autosklearn_seed=1,
port=self.logger_port,
resampling_strategy='holdout',
stats=self.stats,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
self.scenario.wallclock_limit = 5
self.stats.submitted_ta_runs += 1
run_info, run_value = ta.run_wrapper(RunInfo(config=config, cutoff=9, instance=None,
instance_specific=None, seed=1, capped=False))
self.assertEqual(run_value.status, StatusType.STOP)
def test_process_results(self):
"""Ensures that the results are processed by the pertinent intensifer,
based on the source id"""
scheduler = ParallelScheduler(
stats=None,
traj_logger=None,
instances=[1, 2, 3],
rng=np.random.RandomState(12345),
deterministic=True,
)
scheduler.intensifier_instances = {
0: mock.Mock(),
1: mock.Mock(),
2: mock.Mock(),
}
run_info = RunInfo(
config=None,
instance=0,
instance_specific="0",
cutoff=None,
seed=0,
capped=False,
budget=0.0,
source_id=2,
)
result = RunValue(cost=1,
time=0.5,
status=StatusType.SUCCESS,
starttime=1,
endtime=2,
additional_info={})
scheduler.process_results(run_info=run_info,
result=result,
incumbent=None,
run_history=None,
time_bound=None)
self.assertIsNone(
scheduler.intensifier_instances[0].process_results.call_args)
self.assertIsNone(
scheduler.intensifier_instances[1].process_results.call_args)
self.assertEqual(
scheduler.intensifier_instances[2].process_results.call_args[1]
['run_info'], run_info)
def _validate_parallel(
self,
tae: BaseRunner,
runs: typing.List[_Run],
n_jobs: int,
backend: str,
runhistory: typing.Optional[RunHistory] = None,
) -> typing.List[RunValue]:
"""
Validate runs with joblibs Parallel-interface
Parameters
----------
tae: BaseRunner
tae to be used for validation
runs: list<_Run>
list with _Run-objects
[_Run(config=CONFIG1,inst=INSTANCE1,seed=SEED1,inst_specs=INST_SPECIFICS1), ...]
n_jobs: int
number of cpus to use for validation (-1 to use all)
backend: str
what backend to use for parallelization
runhistory: RunHistory
optional, RunHistory-object to reuse runs
Returns
-------
run_results: list<tuple(tae-returns)>
results as returned by tae
"""
# Runs with parallel
run_results = Parallel(n_jobs=n_jobs, backend=backend)(
delayed(_unbound_tae_starter)(
tae,
runhistory,
RunInfo(
config=run.config,
instance=run.inst,
instance_specific="0",
seed=run.seed,
cutoff=self.scen.cutoff, # type: ignore[attr-defined] # noqa F821
capped=False,
budget=0
)
) for run in runs)
return run_results
def test_silent_exception_in_target_function(self):
config = unittest.mock.Mock(spec=int)
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
seed=1,
stats=self.stats,
memory_limit=3072,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
logger_port=self.logger_port,
pynisher_context='fork',
)
ta.pynisher_logger = unittest.mock.Mock()
self.stats.submitted_ta_runs += 1
info = ta.run_wrapper(
RunInfo(config=config,
cutoff=3000,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(info[1].status,
StatusType.CRASHED,
msg=str(info[1].additional_info))
self.assertEqual(info[1].cost, 1.0)
self.assertIsInstance(info[1].time, float)
self.assertIn(info[1].additional_info['error'], (
"""AttributeError("'BackendMock' object has no attribute """
"""'save_targets_ensemble'",)""",
"""AttributeError("'BackendMock' object has no attribute """
"""'save_targets_ensemble'")""",
"""AttributeError('save_targets_ensemble')"""
"""AttributeError("'BackendMock' object has no attribute """
"""'setup_logger'",)""",
"""AttributeError("'BackendMock' object has no attribute """
"""'setup_logger'")""",
))
self.assertNotIn('exitcode', info[1].additional_info)
self.assertNotIn('exit_status', info[1].additional_info)
self.assertNotIn('traceback', info[1])
def test_incorporate_run_results_callback(self, process_results_mock):
process_results_mock.return_value = None, None
class TestCallback(IncorporateRunResultCallback):
def __init__(self):
self.num_call = 0
def __call__(self, smbo, run_info, result, time_left) -> None:
self.num_call += 1
self.config = run_info.config
callback = TestCallback()
self.scenario.output_dir = None
smac = SMAC4AC(self.scenario)
smac.register_callback(callback)
self.output_dirs.append(smac.output_dir)
smbo = smac.solver
config = self.scenario.cs.sample_configuration()
run_info = RunInfo(
config=config,
instance=None,
instance_specific=None,
seed=1,
cutoff=None,
capped=False,
budget=0.0,
source_id=0,
)
result = RunValue(1.2345, 2.3456, "status", "starttime", "endtime",
"additional_info")
time_left = 10
smbo._incorporate_run_results(run_info=run_info,
result=result,
time_left=time_left)
self.assertEqual(callback.num_call, 1)
self.assertEqual(callback.config, config)
def test_cutoff_lower_than_remaining_time(self, pynisher_mock):
config = unittest.mock.Mock()
config.config_id = 198
ta = ExecuteTaFuncWithQueue(
backend=BackendMock(),
autosklearn_seed=1,
resampling_strategy='holdout',
stats=self.stats,
metric=accuracy,
cost_for_crash=get_cost_of_crash(accuracy),
abort_on_first_run_crash=False,
)
self.stats.ta_runs = 1
ta.run_wrapper(
RunInfo(config=config,
cutoff=30,
instance=None,
instance_specific=None,
seed=1,
capped=False))
self.assertEqual(pynisher_mock.call_args[1]['wall_time_in_s'], 4)
self.assertIsInstance(pynisher_mock.call_args[1]['wall_time_in_s'],
int)
def test_get_next_run(self):
"""
Makes sure that sampling a configuration returns a valid
configuration
"""
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config1],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
self.assertEqual(intent, RunInfoIntent.RUN)
self.assertEqual(run_info, RunInfo(
config=self.config1,
instance=1,
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0,
))
resampling_strategy='test',
memory_limit=memory_lim,
disable_file_output=True,
logger=logger,
stats=stats,
scoring_functions=scoring_functions,
include=include,
metric=automl_arguments['metric'],
cost_for_crash=get_cost_of_crash(automl_arguments['metric']),
abort_on_first_run_crash=False,
)
run_info, run_value = ta.run_wrapper(
RunInfo(
config=config,
instance=None,
instance_specific=None,
seed=1,
cutoff=per_run_time_limit * 3,
capped=False,
))
if run_value.status == StatusType.SUCCESS:
assert len(
run_value.additional_info) > 1, run_value.additional_info
# print(additional_run_info)
validated_trajectory.append(
list(entry) + [task_id] + [run_value.additional_info])
print('Finished validating configuration %d/%d' %
(i + 1, len(trajectory)))
print('Finished to validate configurations')
