def test_parallel_same_as_serial_HB(self):
"""Makes sure we behave the same as a serial run at the end"""
# Get the run_history for a_HB instance run:
rh = RunHistory()
stats = Stats(scenario=self.scen)
stats.start_timing()
_HB = _Hyperband(
stats=stats,
traj_logger=TrajLogger(output_dir=None, stats=stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent, inc_perf = self._exhaust_run_and_get_incumbent(
_HB, rh, num_workers=1)
# Just to make sure nothing has changed from the_HB instance side to make
# this check invalid:
# We add config values, so config 3 with 0 and 7 should be the lesser cost
self.assertEqual(incumbent, self.config3)
self.assertEqual(inc_perf, 7.0)
# Do the same for HB, but have multiple_HB instance in there
# This_HB instance will be created via num_workers==2
# in self._exhaust_run_and_get_incumbent
HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent(
HB, self.rh)
self.assertEqual(incumbent, incumbent_phb)
# This makes sure there is a single incumbent in HB
self.assertEqual(inc_perf, inc_perf_phb)
# We don't want to loose any configuration, and particularly
# we want to make sure the values of_HB instance to HB match
self.assertEqual(len(self.rh.data), len(rh.data))
# Because it is a deterministic run, the run histories must be the
# same on exhaustion
self.assertDictEqual(self.rh.data, rh.data)
def test_compare_configs_no_joint_set(self):
intensifier = Intensifier(
tae_runner=None, stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=None, instances=[1])
for i in range(2):
self.rh.add(config=self.config1, cost=2, time=2,
status=StatusType.SUCCESS, instance_id=1,
seed=i, additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=i, additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=1, additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1, cost=i + 1, time=i + 1,
status=StatusType.SUCCESS, instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2, cost=i + 1, time=i + 1,
status=StatusType.SUCCESS, instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def test_init(self):
scen = Scenario(scenario={'run_obj': 'quality', 'cs': self.cs, 'output_dir': ''})
stats = Stats(scen)
with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, 'tmp_test_folder')
TrajLogger(output_dir=path, stats=stats)
self.assertTrue(os.path.exists(path))
def test_add_inc_run_nondet(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_get_next_challenger(self):
"""
test get_next_challenger()
"""
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# get a new challenger to evaluate
config, new = intensifier.get_next_challenger(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertEqual(intensifier.N, 1)
self.assertTrue(new)
# when already evaluating a challenger, return the same challenger
intensifier.to_run = [(1, 1, 0)]
config, new = intensifier.get_next_challenger(
challengers=[self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertFalse(new)
def _set_param_space(self, param_space):
self.param_space = param_space
from ConfigSpace.hyperparameters import UniformFloatHyperparameter
from smac.configspace import ConfigurationSpace
from smac.optimizer.objective import average_cost
from smac.runhistory.runhistory import RunHistory
from smac.scenario.scenario import Scenario
from smac.stats.stats import Stats
from smac.utils.io.traj_logging import TrajLogger
self.cs = ConfigurationSpace()
for param in param_space:
if param.type == 'continuous':
var = UniformFloatHyperparameter(param.name, param.low,
param.high)
self.cs.add_hyperparameter(var)
self.runhistory = RunHistory(aggregate_func=average_cost)
self.scenario = Scenario({
'run_obj': 'quality',
'runcount-limit': self.budget,
'cs': self.cs
})
self.stats = Stats(self.scenario)
self.traj_logger = TrajLogger(output_dir=__scratch__, stats=self.stats)
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1, cost=1, time=2,
status=StatusType.SUCCESS, instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_multi_config_design(self):
stats = Stats(scenario=self.scenario)
stats.start_timing()
self.ta.stats = stats
tj = TrajLogger(output_dir=None, stats=stats)
rh = RunHistory(aggregate_func=average_cost)
self.ta.runhistory = rh
rng = np.random.RandomState(seed=12345)
intensifier = Intensifier(tae_runner=self.ta,
stats=stats,
traj_logger=tj,
rng=rng,
instances=[None],
run_obj_time=False)
configs = [
Configuration(configuration_space=self.cs, values={"x1": 4}),
Configuration(configuration_space=self.cs, values={"x1": 2})
]
dc = MultiConfigInitialDesign(tae_runner=self.ta,
scenario=self.scenario,
stats=stats,
traj_logger=tj,
runhistory=rh,
rng=rng,
configs=configs,
intensifier=intensifier,
aggregate_func=average_cost)
inc = dc.run()
self.assertTrue(stats.ta_runs == 2)
self.assertTrue(len(rh.data) == 2)
self.assertTrue(rh.get_cost(inc) == 4)
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31})
self.config5 = Configuration(self.cs, values={'a': 31, 'b': 33})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=False,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
n_seeds=2,
initial_budget=2,
max_budget=5,
eta=2,
)
def test_ambigious_categoricals(self, mock_stats):
mock_stats.ta_time_used = 0.5
mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time
mock_stats.ta_runs = 1
with tempfile.TemporaryDirectory() as tmpdir:
tl = TrajLogger(output_dir=tmpdir, stats=mock_stats)
problem_config = Configuration(
self.cs, {
'param_a': 0.0,
'param_b': 2,
'param_c': 'value',
'ambigous_categorical': True
}) # not recoverable without json
tl.add_entry(0.9, 1, problem_config)
from_aclib2 = tl.read_traj_aclib_format(
os.path.join(tmpdir, 'traj_aclib2.json'), self.cs)
from_alljson = tl.read_traj_alljson_format(
os.path.join(tmpdir, 'traj.json'), self.cs)
# Wrong! but passes:
self.assertIsInstance(
from_aclib2[0]['incumbent']['ambigous_categorical'], str)
# Works good for alljson:
self.assertIsInstance(
from_alljson[0]['incumbent']['ambigous_categorical'], bool)
def test_compare_configs_inc(self):
'''
incumbent is better
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def optimize(optimizer, scenario, trajectory=None):
then = time.time()
best_conf = optimizer.optimize()
print(best_conf)
print('training time:', time.time() - then)
traj_logger = TrajLogger(None, Stats(scenario))
trajectory = trajectory or traj_logger.read_traj_aclib_format(
"smac-output/run_1/traj_aclib2.json", scenario.cs)
validator = Validator(scenario, trajectory, rng=np.random.RandomState(42))
# evaluate on test instances and calculate cpu time
then = time.time()
runhis_dev = validator.validate(config_mode="def", instance_mode="test")
runhis_inc = validator.validate(config_mode="inc", instance_mode="test")
print('validating time:', time.time() - then)
default_conf = runhis_dev.ids_config[1]
incumbent_conf = runhis_inc.ids_config[1]
dev_vals = get_instance_costs_for_config(runhis_dev, default_conf)
inc_vals = get_instance_costs_for_config(runhis_inc, incumbent_conf)
# ###### Filter runs for plotting #######
dev_x = []
inc_x = []
for key in set(dev_vals.keys()) & set(inc_vals.keys()):
dev_x.append(dev_vals[key])
inc_x.append(inc_vals[key])
# print(dev_vals)
# print(inc_vals)
print(dev_x)
print(inc_x)
print('PAR10:', np.mean(inc_x), '/', np.mean(dev_x))
max_x = 1000.0
par1er = lambda xx: np.mean([(x / 10 if x == max_x else x) for x in xx])
print('PAR1 :', par1er(inc_x), '/', par1er(dev_x))
to_counter = lambda xx: len([x for x in xx if x == max_x])
print('TOs :', to_counter(inc_x), '/', to_counter(dev_x))
print('wins :', len([i for i in range(len(dev_x)) if dev_x[i] > inc_x[i]]),
'/', len(dev_x))
fig, ax = plt.subplots()
ax.scatter(dev_x, inc_x, marker="x")
ax.set_xlabel("Default Configuration")
ax.set_ylabel("Incumbent Configuration")
lims = [
np.min([ax.get_xlim(), ax.get_ylim()]),
np.max([ax.get_xlim(), ax.get_ylim()])
]
ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
# ax.set_xlim(lims)
# ax.set_ylim(lims)
ax.set_xscale('log')
ax.set_yscale('log')
fig.savefig("fig-smac.png")
def test_race_challenger(self):
'''
test _race_challenger without adaptive capping
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
self.assertEqual(inc, self.config2)
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs,
values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs,
values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs,
values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs,
values={'a': 29, 'b': 31})
self.scen = Scenario({"cutoff_time": 2, 'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.intensifier = SimpleIntensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1],
)
def test_race_challenger_2(self):
"""
Makes sure that a racing configuration with better performance,
that is capped, doesn't substitute the incumbent.
"""
def target(x):
time.sleep(1.5)
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
)
self.rh.add(
config=self.config1,
cost=0.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None,
)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=True,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_2(self):
'''
test _race_challenger with adaptive capping
'''
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
def test_add_inc_run_det(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run
# as it is the first evaluation of this intensifier
self.assertEqual(intensifier.num_run, 0)
self.assertEqual(intensifier.num_chall_run, 0)
def test_race_challenger_large_blocked_seed(self):
"""
test _race_challenger whether seeds are blocked for challenger runs
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
self.assertEqual(intensifier.num_run, 10)
self.assertEqual(intensifier.num_chall_run, 10)
def get_trajectory(self, folder2result, output_path, scenario, rh, budget=None):
"""
If budget is specified, get trajectory for only that budget. Else use hpbandster's averaging.
If multiple results are specified, sort by times_finished and only add to combined trajectory if loss is better
"""
cs = scenario.cs
if not output_path:
output_path = tempfile.mkdtemp()
traj_logger = TrajLogger(output_path, Stats(scenario))
total_traj_dict = []
for f, result in folder2result.items():
if budget:
traj_dict = self.get_incumbent_trajectory_for_budget(result, budget)
else:
traj_dict = result.get_incumbent_trajectory()
id2config_mapping = result.get_id2config_mapping()
for config_id, time, budget, loss in zip(traj_dict['config_ids'], traj_dict['times_finished'], traj_dict['budgets'], traj_dict['losses']):
incumbent = self._get_config(config_id, id2config_mapping, cs)
try:
incumbent_id = rh.config_ids[incumbent]
except KeyError as e:
# This config was not evaluated on this budget, just skip it
continue
except:
raise
total_traj_dict.append({'config_id' : incumbent_id, 'time_finished' : time, 'budget' : budget, 'loss' : loss})
last_loss = np.inf
for element in sorted(total_traj_dict, key=lambda x: x['time_finished']):
incumbent_id = element["config_id"]
incumbent = rh.ids_config[incumbent_id]
time = element["time_finished"]
loss = element["loss"]
if loss > last_loss:
continue
ta_runs = -1
ta_time_used = -1
wallclock_time = time
train_perf = loss
# add
traj_logger.trajectory.append({"cpu_time": ta_time_used,
"total_cpu_time": None, # TODO: fix this
"wallclock_time": wallclock_time,
"evaluations": ta_runs,
"cost": train_perf,
"incumbent": incumbent
})
traj_logger._add_in_old_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time)
traj_logger._add_in_aclib_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time)
return traj_logger.trajectory
def test_init(self):
scen = Scenario(scenario={
'run_obj': 'quality',
'cs': self.cs,
'output_dir': ''
})
stats = Stats(scen)
TrajLogger(output_dir='./tmp_test_folder', stats=stats)
self.assertFalse(os.path.exists('smac3-output'))
self.assertTrue(os.path.exists('tmp_test_folder'))
def create_worker(self, worker_id) -> Worker:
"""Function to create a worker.
Parameters
----------
worker_id : int
A seed for the random generator. It should be the id of the worker.
Returns
-------
worker : Worker
Return a ps-lite worker.
"""
# 首先指定输出目录
output_dir = self.temp_folder + "worker-output_%s" % (
datetime.datetime.fromtimestamp(time.time()).strftime(
'%Y-%m-%d_%H:%M:%S_%f'))
# 然后创建scenario
scenario_dict = {
"cs": self.cs,
"run_obj": "quality",
"cutoff_time": self.cutoff,
"initial_incumbent": "RANDOM",
"output_dir": output_dir
}
scenario = Scenario(scenario_dict)
# 统计类
stats = Stats(scenario)
# 创建ta函数
ta = CustomizedTA(self.X_train, self.y_train, self.X_valid,
self.y_valid)
tae_runner = ExecuteTAFuncDict(ta=ta,
stats=stats,
run_obj=scenario.run_obj,
memory_limit=scenario.memory_limit,
runhistory=RunHistory(
aggregate_func=average_cost),
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# logger和rng
traj_logger = TrajLogger(output_dir=output_dir, stats=stats)
rng = np.random.RandomState(seed=worker_id)
# 创建intensifier
intensifier = Intensifier(tae_runner=tae_runner,
stats=stats,
traj_logger=traj_logger,
rng=rng,
instances=scenario.train_insts)
# 最终目的，创建worker并返回
worker = Worker(self.worker_args[worker_id], self.cs, intensifier,
worker_id=worker_id)
return worker
def test_add_multiple_entries(self, mock_stats):
tl = TrajLogger(output_dir='./tmp_test_folder', stats=mock_stats)
test_config = {'param_a': 0.5,
'param_b': 1,
'param_c': 'value'}
mock_stats.ta_time_used = 0.5
mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time
mock_stats.ta_runs = 1
tl.add_entry(0.9, 1, test_config)
mock_stats.ta_runs = 2
mock_stats.ta_time_used = 0
tl.add_entry(1.3, 1, test_config)
test_config['param_a'] = 0.
mock_stats.ta_time_used = 0
tl.add_entry(0.7, 2, test_config)
self.assertTrue(os.path.exists('tmp_test_folder/traj_old.csv'))
self.assertTrue(os.path.exists('tmp_test_folder/traj_aclib2.json'))
with open('tmp_test_folder/traj_old.csv') as to:
data = to.read().split('\n')[:-1]
header = data[0].split(',')
self.assertEquals(header[0], '"CPU Time Used"')
self.assertEquals(header[-1], '"Configuration..."')
data = list(map(lambda x: x.split(', '), data[1:]))
frmt_str = '%1.6f'
self.assertEquals(frmt_str % 0.5, data[0][0])
self.assertEquals(frmt_str % 0.9, data[0][1])
self.assertEquals(frmt_str % 0.5, data[0][-4])
self.assertEquals(frmt_str % 0, data[1][0])
self.assertEquals(frmt_str % 1.3, data[1][1])
self.assertEquals(frmt_str % 2, data[1][-4])
self.assertEquals(frmt_str % 0, data[2][0])
self.assertEquals(frmt_str % .7, data[2][1])
self.assertEquals(frmt_str % 3, data[2][-4])
json_dicts = []
with open('tmp_test_folder/traj_aclib2.json') as js:
data = js.read().split('\n')[:-1]
for d in data:
json_dicts.append(json.loads(d))
self.assertEquals(json_dicts[0]['cpu_time'], .5)
self.assertEquals(json_dicts[0]['cost'], 0.9)
self.assertEquals(len(json_dicts[0]['incumbent']), 3)
self.assertTrue("param_a='0.5'" in json_dicts[0]['incumbent'])
def test_race_challenger_1(self):
"""
Makes sure that a racing configuration with better performance,
is selected as incumbent
No adaptive capping
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
intensifier.N = 1
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
cutoff=cutoff,
seed=seed,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_3(self):
'''
test _race_challenger with adaptive capping on a previously capped configuration
'''
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1)
taf.runhistory = self.rh
intensifier = Intensifier(
tae_runner=taf, stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1])
self.rh.add(config=self.config1, cost=0.5, time=.5,
status=StatusType.SUCCESS, instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(config=self.config1, cost=2, time=2,
status=StatusType.TIMEOUT, instance_id=2,
seed=12345,
additional_info=None)
# give config2 a second chance
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.runs_per_config[2], 2)
def test_eval_challenger_3(self):
"""
test eval_challenger for a resumed SMAC run (first run with incumbent)
"""
def target(x):
return x['a']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=False,
always_race_against=None,
run_limit=1)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# adding run for incumbent configuration
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
# intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage
intent_, run_info = intensifier.get_next_run(
challengers=[self.config2],
incumbent=self.config1,
run_history=self.rh,
chooser=None)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)), 2)
def test_init_budget(self):
self.stats.start_timing()
tj = TrajLogger(output_dir=None, stats=self.stats)
_ = np.random.RandomState(seed=12345)
kwargs = dict(cs=self.cs,
traj_logger=tj,
rng=np.random.RandomState(seed=12345),
ta_run_limit=self.scenario.ta_run_limit)
configs = [
Configuration(configuration_space=self.cs, values={"x1": 4}),
Configuration(configuration_space=self.cs, values={"x1": 2})
]
dc = InitialDesign(
configs=configs,
init_budget=3,
**kwargs,
)
self.assertEqual(dc.init_budget, 3)
dc = InitialDesign(
init_budget=3,
**kwargs,
)
self.assertEqual(dc.init_budget, 3)
configs = [
Configuration(configuration_space=self.cs, values={"x1": 4}),
Configuration(configuration_space=self.cs, values={"x1": 2})
]
dc = InitialDesign(
configs=configs,
**kwargs,
)
self.assertEqual(dc.init_budget, 2)
dc = InitialDesign(**kwargs, )
self.assertEqual(dc.init_budget, 10)
with self.assertRaisesRegex(
ValueError,
'Initial budget 200 cannot be higher than the run limit 100.',
):
InitialDesign(init_budget=200, **kwargs)
with self.assertRaisesRegex(
ValueError,
'Need to provide either argument `init_budget`, `configs` or `n_configs_x_params`, '
'but provided none of them.',
):
InitialDesign(**kwargs, n_configs_x_params=None)
def test_single_default_config_design(self):
self.stats.start_timing()
tj = TrajLogger(output_dir=None, stats=self.stats)
dc = DefaultConfiguration(cs=self.cs,
traj_logger=tj,
rng=np.random.RandomState(seed=12345),
ta_run_limit=self.scenario.ta_run_limit)
# should return only the default config
configs = dc.select_configurations()
self.assertEqual(len(configs), 1)
self.assertEqual(configs[0]['x1'], 1)
