def test_race_challenger_2(self):
"""
Makes sure that a racing configuration with better performance,
that is capped, doesn't substitute the incumbent.
"""
def target(x):
time.sleep(1.5)
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
)
self.rh.add(
config=self.config1,
cost=0.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None,
)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=True,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_2(self):
'''
test _race_challenger with adaptive capping
'''
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
def test_run(self):
"""Makes sure that we are able to run a configuration and
return the expected values/types"""
# We use the funcdict as a mechanism to test SerialRunner
runner = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj='quality')
self.assertIsInstance(runner, SerialRunner)
run_info = RunInfo(config=2,
instance='test',
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0)
# submit runs! then get the value
runner.submit_run(run_info)
run_values = runner.get_finished_runs()
self.assertEqual(len(run_values), 1)
self.assertIsInstance(run_values, list)
self.assertIsInstance(run_values[0][0], RunInfo)
self.assertIsInstance(run_values[0][1], RunValue)
self.assertEqual(run_values[0][1].cost, 4)
self.assertEqual(run_values[0][1].status, StatusType.SUCCESS)
def test_add_inc_run_nondet(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_add_inc_run_det(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_race_challenger(self):
'''
test _race_challenger without adaptive capping
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
self.assertEqual(inc, self.config2)
def test_memout(self):
def fill_memory(*args):
a = np.random.random_sample((10000, 10000)).astype(np.float64)
return np.sum(a)
taf = ExecuteTAFuncDict(ta=fill_memory,
stats=self.stats,
memory_limit=1024)
rval = taf.run(config=None)
platform = os.getenv('TRAVIS_OS_NAME')
if platform is None:
platform = {'linux': 'linux', 'darwin': 'osx'}.get(sys.platform)
print(platform, sys.platform)
if platform == 'linux':
self.assertEqual(rval[0], StatusType.MEMOUT)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
elif platform == 'osx':
self.assertEqual(rval[0], StatusType.SUCCESS)
else:
raise ValueError('Test cannot be performed on platform %s' %
sys.platform)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run
# as it is the first evaluation of this intensifier
self.assertEqual(intensifier.num_run, 0)
self.assertEqual(intensifier.num_chall_run, 0)
def test_race_challenger_large_blocked_seed(self):
"""
test _race_challenger whether seeds are blocked for challenger runs
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
self.assertEqual(intensifier.num_run, 10)
self.assertEqual(intensifier.num_chall_run, 10)
def test_timeout(self):
def run_over_time(*args):
time.sleep(5)
taf = ExecuteTAFuncDict(ta=run_over_time, stats=self.stats)
rval = taf.run(config=None, cutoff=1)
self.assertEqual(rval[0], StatusType.TIMEOUT)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
def test_fail_silent(self):
def function(*args):
return
taf = ExecuteTAFuncDict(ta=function, stats=self.stats)
rval = taf.run(config=None, cutoff=1)
self.assertEqual(rval[0], StatusType.CRASHED)
self.assertEqual(rval[1], 2147483647.0)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
def test_serial_runs(self):
# We use the funcdict as a mechanism to test SerialRunner
runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality')
self.assertIsInstance(runner, SerialRunner)
run_info = RunInfo(config=2, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
run_info = RunInfo(config=3, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
run_values = runner.get_finished_runs()
self.assertEqual(len(run_values), 2)
# To make sure runs launched serially, we just make sure that the end time of
# a run is later than the other
# Results are returned in left to right
self.assertLessEqual(int(run_values[1][1].endtime), int(run_values[0][1].starttime))
# No wait time in serial runs!
start = time.time()
runner.wait()
# The run takes a second, so 0.5 is sufficient
self.assertLess(time.time() - start, 0.5)
pass
def test_run(self):
target = lambda x: x**2
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
rval = taf.run(config=2)
self.assertFalse(taf._accepts_instance)
self.assertFalse(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], dict())
target = lambda x, seed: (x**2, {'key': seed})
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
rval = taf.run(config=2, instance='test')
self.assertFalse(taf._accepts_instance)
self.assertTrue(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], {'key': 12345})
target = lambda x, seed, instance: (x**2, {
'key': seed,
'instance': instance
})
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
rval = taf.run(config=2, instance='test')
self.assertTrue(taf._accepts_instance)
self.assertTrue(taf._accepts_seed)
self.assertEqual(rval[0], StatusType.SUCCESS)
self.assertEqual(rval[1], 4)
self.assertGreaterEqual(rval[2], 0.0)
self.assertEqual(rval[3], {'key': 12345, 'instance': 'test'})
def test_race_challenger_1(self):
"""
Makes sure that a racing configuration with better performance,
is selected as incumbent
No adaptive capping
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
intensifier.N = 1
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
cutoff=cutoff,
seed=seed,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_3(self):
'''
test _race_challenger with adaptive capping on a previously capped configuration
'''
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1)
taf.runhistory = self.rh
intensifier = Intensifier(
tae_runner=taf, stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1])
self.rh.add(config=self.config1, cost=0.5, time=.5,
status=StatusType.SUCCESS, instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(config=self.config1, cost=2, time=2,
status=StatusType.TIMEOUT, instance_id=2,
seed=12345,
additional_info=None)
# give config2 a second chance
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.runs_per_config[2], 2)
def test_eval_challenger_3(self):
"""
test eval_challenger for a resumed SMAC run (first run with incumbent)
"""
def target(x):
return x['a']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=False,
always_race_against=None,
run_limit=1)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# adding run for incumbent configuration
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
# intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage
intent_, run_info = intensifier.get_next_run(
challengers=[self.config2],
incumbent=self.config1,
run_history=self.rh,
chooser=None)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)), 2)
def test_num_workers(self):
"""Make sure we can properly return the number of workers"""
# We use the funcdict as a mechanism to test Runner
runner = ExecuteTAFuncDict(ta=target_delayed, stats=self.stats, run_obj='quality')
runner = DaskParallelRunner(runner, n_workers=2)
self.assertEqual(runner.num_workers(), 2)
# Reduce the number of workers
# have to give time for the worker to be killed
runner.client.cluster.scale(1)
time.sleep(2)
self.assertEqual(runner.num_workers(), 1)
def test_timeout_runtime(self):
def run_over_time(*args):
time.sleep(5)
taf = ExecuteTAFuncDict(ta=run_over_time,
stats=self.stats,
run_obj='runtime',
par_factor=11)
rval = taf.run(config=None, cutoff=1)
self.assertEqual(rval[0], StatusType.TIMEOUT)
self.assertGreaterEqual(rval[1], 11)
self.assertGreaterEqual(rval[2], 1)
self.assertEqual(rval[3], dict())
def test_additional_info_crash_msg(self):
"""
We want to make sure we catch errors as additional info,
and in particular when doing multiprocessing runs, we
want to make sure we capture dask exceptions
"""
def target_nonpickable(x, seed, instance):
return x**2, {'key': seed, 'instance': instance}
runner = ExecuteTAFuncDict(ta=target_nonpickable, stats=self.stats, run_obj='quality')
runner = DaskParallelRunner(runner, n_workers=2)
run_info = RunInfo(config=2, instance='test', instance_specific="0",
seed=0, cutoff=None, capped=False, budget=0.0)
runner.submit_run(run_info)
runner.wait()
run_info, result = runner.get_finished_runs()[0]
# Make sure the traceback message is included
self.assertIn('traceback', result.additional_info)
self.assertIn(
# We expect the problem to occur in the run wrapper
# So traceback should show this!
'target_nonpickable',
result.additional_info['traceback'])
# Make sure the error message is included
self.assertIn('error', result.additional_info)
self.assertIn(
'Can\'t pickle local object',
result.additional_info['error'])
def test_eval_challenger(self):
"""
since hyperband uses eval_challenger and get_next_challenger of the internal successive halving,
we don't test these method extensively
"""
def target(x):
return 0.1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Hyperband(
tae_runner=taf, stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False,
instances=[None], initial_budget=0.5, max_budget=1, eta=2)
self.assertFalse(hasattr(intensifier, 's'))
# Testing get_next_challenger - get next configuration
config, _ = intensifier.get_next_challenger(challengers=[self.config2, self.config3],
chooser=None, run_history=self.rh)
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(config, self.config2)
# update to the last SH iteration of the given HB stage
self.assertEqual(intensifier.s, 1)
self.assertEqual(intensifier.s_max, 1)
self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS,
seed=0, budget=1)
self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS,
seed=0, budget=0.5)
self.rh.add(config=self.config3, cost=3, time=2, status=StatusType.SUCCESS,
seed=0, budget=0.5)
intensifier.sh_intensifier.success_challengers = {self.config2, self.config3}
intensifier.sh_intensifier._update_stage(self.rh)
# evaluation should change the incumbent to config2
inc, inc_value = intensifier.eval_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.s, 0)
self.assertEqual(inc_value, 0.1)
self.assertEqual(self.stats.ta_runs, 1)
self.assertEqual(list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2])
self.assertEqual(self.stats.inc_changed, 1)
def eval_challenger(
run_info: RunInfo,
taf: ExecuteTAFuncDict,
stats: Stats,
runhistory: RunHistory,
):
"""
Wrapper over challenger evaluation
SMBO objects handles run history now, but to keep
same testing functionality this function is a small
wrapper to launch the taf and add it to the history
"""
# evaluating configuration
run_info, result = taf.run_wrapper(
run_info=run_info,
)
stats.ta_time_used += float(result.time)
runhistory.add(
config=run_info.config,
cost=result.cost,
time=result.time,
status=result.status,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
stats.n_configs = len(runhistory.config_ids)
return result
def __init__(self,
cost_oracle: typing.Mapping[str, float],
tae: typing.Type[SerialRunner] = ExecuteTARunOld,
**kwargs: typing.Any) -> None:
'''
Constructor
Arguments
---------
cost_oracle: typing.Mapping[str,float]
cost of oracle per instance
'''
super().__init__(**kwargs)
self.cost_oracle = cost_oracle
if tae is ExecuteTARunAClib:
self.runner = ExecuteTARunAClib(**kwargs) # type: SerialRunner
elif tae is ExecuteTARunOld:
self.runner = ExecuteTARunOld(**kwargs)
elif tae is ExecuteTAFuncDict:
self.runner = ExecuteTAFuncDict(**kwargs)
elif tae is ExecuteTAFuncArray:
self.runner = ExecuteTAFuncArray(**kwargs)
else:
raise Exception('TAE not supported')
def setUp(self):
self.cs = ConfigurationSpace()
self.cs.add_hyperparameter(
UniformFloatHyperparameter(name="x1", lower=1, upper=10,
default=2))
self.scenario = Scenario({'cs': self.cs, 'run_obj': 'quality'})
self.ta = ExecuteTAFuncDict(lambda x: x["x1"]**2)
def test_inject_stats_and_runhistory_object_to_TAE(self):
ta = ExecuteTAFuncDict(lambda x: x**2)
self.assertIsNone(ta.stats)
self.assertIsNone(ta.runhistory)
SMAC(tae_runner=ta, scenario=self.scenario)
self.assertIsInstance(ta.stats, Stats)
self.assertIsInstance(ta.runhistory, RunHistory)
def test_pass_tae_runner_objective(self):
tae = ExecuteTAFuncDict(lambda: 1,
run_obj='runtime')
self.assertRaisesRegexp(ValueError, "Objective for the target algorithm"
" runner and the scenario must be "
"the same, but are 'runtime' and "
"'quality'",
SMAC, tae_runner=tae, scenario=self.scenario)
def test_race_challenger_large_blocked_seed(self):
'''
test _race_challenger whether seeds are blocked for challenger runs
'''
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
# tie on first instances and then challenger should always win
# and be returned as inc
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
def test_get_runhistory_and_trajectory(self):
ta = ExecuteTAFuncDict(lambda x: x ** 2)
smac = SMAC(tae_runner=ta, scenario=self.scenario)
self.assertRaises(ValueError, smac.get_runhistory)
self.assertRaises(ValueError, smac.get_trajectory)
smac.trajectory = 'dummy'
self.assertEqual(smac.get_trajectory(), 'dummy')
smac.runhistory = 'dummy'
self.assertEqual(smac.get_runhistory(), 'dummy')
def create_worker(self, worker_id) -> Worker:
"""Function to create a worker.
Parameters
----------
worker_id : int
A seed for the random generator. It should be the id of the worker.
Returns
-------
worker : Worker
Return a ps-lite worker.
"""
# 首先指定输出目录
output_dir = self.temp_folder + "worker-output_%s" % (
datetime.datetime.fromtimestamp(time.time()).strftime(
'%Y-%m-%d_%H:%M:%S_%f'))
# 然后创建scenario
scenario_dict = {
"cs": self.cs,
"run_obj": "quality",
"cutoff_time": self.cutoff,
"initial_incumbent": "RANDOM",
"output_dir": output_dir
}
scenario = Scenario(scenario_dict)
# 统计类
stats = Stats(scenario)
# 创建ta函数
ta = CustomizedTA(self.X_train, self.y_train, self.X_valid,
self.y_valid)
tae_runner = ExecuteTAFuncDict(ta=ta,
stats=stats,
run_obj=scenario.run_obj,
memory_limit=scenario.memory_limit,
runhistory=RunHistory(
aggregate_func=average_cost),
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# logger和rng
traj_logger = TrajLogger(output_dir=output_dir, stats=stats)
rng = np.random.RandomState(seed=worker_id)
# 创建intensifier
intensifier = Intensifier(tae_runner=tae_runner,
stats=stats,
traj_logger=traj_logger,
rng=rng,
instances=scenario.train_insts)
# 最终目的，创建worker并返回
worker = Worker(self.worker_args[worker_id], self.cs, intensifier,
worker_id=worker_id)
return worker
def setUp(self):
self.cs = ConfigurationSpace()
self.cs.add_hyperparameter(UniformFloatHyperparameter(
name="x1", lower=1, upper=10, default_value=1)
)
self.scenario = Scenario({
'cs': self.cs,
'run_obj': 'quality',
'output_dir': '',
'ta_run_limit': 100,
})
self.stats = Stats(scenario=self.scenario)
self.rh = RunHistory()
self.ta = ExecuteTAFuncDict(lambda x: x["x1"]**2, stats=self.stats, runhistory=self.rh)
def test_inject_dependencies(self):
# initialize objects with missing dependencies
ta = ExecuteTAFuncDict(lambda x: x**2)
rh = RunHistory(aggregate_func=None)
acqu_func = EI(model=None)
intensifier = Intensifier(tae_runner=None,
stats=None,
traj_logger=None,
rng=np.random.RandomState(),
instances=None)
init_design = DefaultConfiguration(tae_runner=None,
scenario=None,
stats=None,
traj_logger=None,
rng=np.random.RandomState())
rh2epm = RunHistory2EPM4Cost(scenario=self.scenario, num_params=0)
rh2epm.scenario = None
# assert missing dependencies
self.assertIsNone(rh.aggregate_func)
self.assertIsNone(acqu_func.model)
self.assertIsNone(intensifier.tae_runner)
self.assertIsNone(intensifier.stats)
self.assertIsNone(intensifier.traj_logger)
self.assertIsNone(init_design.tae_runner)
self.assertIsNone(init_design.scenario)
self.assertIsNone(init_design.stats)
self.assertIsNone(init_design.traj_logger)
self.assertIsNone(rh2epm.scenario)
# initialize smac-object
SMAC(scenario=self.scenario,
tae_runner=ta,
runhistory=rh,
intensifier=intensifier,
acquisition_function=acqu_func,
runhistory2epm=rh2epm,
initial_design=init_design)
# assert that missing dependencies are injected
self.assertIsNotNone(rh.aggregate_func, AbstractAcquisitionFunction)
self.assertIsInstance(acqu_func.model, AbstractEPM)
self.assertIsInstance(intensifier.tae_runner, ExecuteTARun)
self.assertIsInstance(intensifier.stats, Stats)
self.assertIsInstance(intensifier.traj_logger, TrajLogger)
self.assertIsInstance(init_design.tae_runner, ExecuteTARun)
self.assertIsInstance(init_design.scenario, Scenario)
self.assertIsInstance(init_design.stats, Stats)
self.assertIsInstance(init_design.traj_logger, TrajLogger)
self.assertIsInstance(rh2epm.scenario, Scenario)
