def test_multi_config_design(self):
stats = Stats(scenario=self.scenario)
stats.start_timing()
self.ta.stats = stats
tj = TrajLogger(output_dir=None, stats=stats)
rh = RunHistory(aggregate_func=average_cost)
self.ta.runhistory = rh
rng = np.random.RandomState(seed=12345)
intensifier = Intensifier(tae_runner=self.ta,
stats=stats,
traj_logger=tj,
rng=rng,
instances=[None],
run_obj_time=False)
configs = [
Configuration(configuration_space=self.cs, values={"x1": 4}),
Configuration(configuration_space=self.cs, values={"x1": 2})
]
dc = MultiConfigInitialDesign(tae_runner=self.ta,
scenario=self.scenario,
stats=stats,
traj_logger=tj,
runhistory=rh,
rng=rng,
configs=configs,
intensifier=intensifier,
aggregate_func=average_cost)
inc = dc.run()
self.assertTrue(stats.ta_runs == 2)
self.assertTrue(len(rh.data) == 2)
self.assertTrue(rh.get_cost(inc) == 4)
def test_crashed_cost_value(self, test_run):
'''
test cost on crashed runs
'''
# Patch run-function for custom-return
scen = Scenario(scenario={
'cs': ConfigurationSpace(),
'run_obj': 'quality'
},
cmd_options=None)
stats = Stats(scen)
stats.start_timing()
stats.ta_runs += 1
# Check quality
test_run.return_value = StatusType.CRASHED, np.nan, np.nan, {}
eta = ExecuteTARun(ta=lambda *args: None,
stats=stats,
run_obj='quality',
cost_for_crash=100)
self.assertEqual(100, eta.start(config={}, instance=1)[1])
# Check runtime
eta = ExecuteTARun(ta=lambda *args: None,
stats=stats,
run_obj='runtime',
cost_for_crash=10.7)
self.assertEqual(20.0, eta.start(config={}, instance=1, cutoff=20)[1])
def test_start_tae_return_abort(self, test_run):
'''
testing abort
'''
# Patch run-function for custom-return
test_run.return_value = StatusType.ABORT, 12345.0, 1.2345, {}
scen = Scenario(
scenario={
'cs': ConfigurationSpace(),
'run_obj': 'quality',
'output_dir': '',
},
cmd_options=None,
)
stats = Stats(scen)
stats.start_timing()
eta = SerialRunner(ta=lambda *args: None, stats=stats)
_, run_value = eta.run_wrapper(
RunInfo(config=None,
instance=1,
instance_specific=None,
cutoff=30,
seed=None,
capped=False,
budget=0.0))
self.assertEqual(run_value.status, StatusType.ABORT)
def test_parallel_same_as_serial_HB(self):
"""Makes sure we behave the same as a serial run at the end"""
# Get the run_history for a_HB instance run:
rh = RunHistory()
stats = Stats(scenario=self.scen)
stats.start_timing()
_HB = _Hyperband(
stats=stats,
traj_logger=TrajLogger(output_dir=None, stats=stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent, inc_perf = self._exhaust_run_and_get_incumbent(
_HB, rh, num_workers=1)
# Just to make sure nothing has changed from the_HB instance side to make
# this check invalid:
# We add config values, so config 3 with 0 and 7 should be the lesser cost
self.assertEqual(incumbent, self.config3)
self.assertEqual(inc_perf, 7.0)
# Do the same for HB, but have multiple_HB instance in there
# This_HB instance will be created via num_workers==2
# in self._exhaust_run_and_get_incumbent
HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent(
HB, self.rh)
self.assertEqual(incumbent, incumbent_phb)
# This makes sure there is a single incumbent in HB
self.assertEqual(inc_perf, inc_perf_phb)
# We don't want to loose any configuration, and particularly
# we want to make sure the values of_HB instance to HB match
self.assertEqual(len(self.rh.data), len(rh.data))
# Because it is a deterministic run, the run histories must be the
# same on exhaustion
self.assertDictEqual(self.rh.data, rh.data)
def test_single_default_config_design(self):
stats = Stats(scenario=self.scenario)
stats.start_timing()
self.ta.stats = stats
tj = TrajLogger(output_dir=None, stats=stats)
rh = RunHistory(aggregate_func=average_cost)
dc = DefaultConfiguration(tae_runner=self.ta, scenario=self.scenario,
stats=stats, traj_logger=tj,
rng=np.random.RandomState(seed=12345))
inc = dc.run()
self.assertTrue(stats.ta_runs==1)
self.assertTrue(len(rh.data)==0)
def get_tae(obj):
""" Create SerialRunner-object for testing. """
scen = Scenario(scenario={
'cs': ConfigurationSpace(),
'run_obj': obj,
'cutoff_time': '10'
},
cmd_options=None)
stats = Stats(scen)
stats.start_timing()
# Add first run to not trigger FirstRunCrashedException
stats.submitted_ta_runs += 1
eta = SerialRunner(ta=lambda *args: None, stats=stats, run_obj=obj)
return eta
def test_memory_limit_usage(self, run_mock):
run_mock.return_value = StatusType.SUCCESS, 12345.0, 1.2345, {}
stats = Stats(Scenario({}))
stats.start_timing()
tae = ExecuteTARun(lambda x: x**2, stats)
self.assertRaisesRegex(
ValueError, 'Target algorithm executor '
'ExecuteTARun does not support restricting the memory usage.',
tae.start, {'x': 2},
'a',
memory_limit=123)
tae._supports_memory_limit = True
rval = tae.start({'x': 2}, 'a', memory_limit=10)
self.assertEqual(rval, run_mock.return_value)
def setUp(self):
self.datamanager = get_multiclass_classification_datamanager()
self.tmp = os.path.join(os.getcwd(), '.test_evaluation')
self.logger = logging.getLogger()
scenario_mock = unittest.mock.Mock()
scenario_mock.wallclock_limit = 10
scenario_mock.algo_runs_timelimit = 1000
scenario_mock.ta_run_limit = 100
self.scenario = scenario_mock
stats = Stats(scenario_mock)
stats.start_timing()
self.stats = stats
try:
shutil.rmtree(self.tmp)
except:
pass
def setUp(self):
self.datamanager = get_multiclass_classification_datamanager()
self.tmp = os.path.join(os.getcwd(), '.test_evaluation')
self.logger = logging.getLogger()
scenario_mock = unittest.mock.Mock()
scenario_mock.wallclock_limit = 10
scenario_mock.algo_runs_timelimit = 1000
scenario_mock.ta_run_limit = 100
self.scenario = scenario_mock
stats = Stats(scenario_mock)
stats.start_timing()
self.stats = stats
try:
shutil.rmtree(self.tmp)
except:
pass
def test_start_tae_return_abort(self, test_run):
'''
testing abort
'''
# Patch run-function for custom-return
test_run.return_value = StatusType.ABORT, 12345.0, 1.2345, {}
scen = Scenario(scenario={
'cs': ConfigurationSpace(),
'output_dir': ''
},
cmd_args=None)
stats = Stats(scen)
stats.start_timing()
eta = ExecuteTARun(ta=lambda *args: None, stats=stats)
self.assertRaises(TAEAbortException, eta.start, config={}, instance=1)
def setUp(self):
self.datamanager = get_multiclass_classification_datamanager()
self.tmp = os.path.join(os.getcwd(), '.test_evaluation')
os.mkdir(self.tmp)
self.logger_port = logging.handlers.DEFAULT_TCP_LOGGING_PORT
scenario_mock = unittest.mock.Mock()
scenario_mock.wallclock_limit = 10
scenario_mock.algo_runs_timelimit = 1000
scenario_mock.ta_run_limit = 100
self.scenario = scenario_mock
stats = Stats(scenario_mock)
stats.start_timing()
self.stats = stats
try:
shutil.rmtree(self.tmp)
except Exception:
pass
def test_crashed_cost_value(self, test_run):
'''
test cost on crashed runs
'''
# Patch run-function for custom-return
scen = Scenario(scenario={
'cs': ConfigurationSpace(),
'run_obj': 'quality'
},
cmd_options=None)
stats = Stats(scen)
stats.start_timing()
stats.submitted_ta_runs += 1
# Check quality
test_run.return_value = StatusType.CRASHED, np.nan, np.nan, {}
eta = SerialRunner(ta=lambda *args: None,
stats=stats,
run_obj='quality',
cost_for_crash=100)
run_info, result = eta.run_wrapper(
RunInfo(config={},
instance=1,
instance_specific="0",
cutoff=None,
seed=None,
capped=False,
budget=0.0))
self.assertEqual(100, result.cost)
# Check runtime
eta = SerialRunner(ta=lambda *args: None,
stats=stats,
run_obj='runtime',
cost_for_crash=10.7)
run_info, result = eta.run_wrapper(
RunInfo(config={},
instance=1,
instance_specific="0",
cutoff=20,
seed=None,
capped=False,
budget=0.0))
self.assertEqual(20.0, result.cost)
def test_start_crash_first_run(self, test_run):
'''
testing crash-on-first-run
'''
# Patch run-function for custom-return
test_run.return_value = StatusType.CRASHED, 12345.0, 1.2345, {}
scen = Scenario(scenario={
'cs': ConfigurationSpace(),
'output_dir': ''
},
cmd_args=None)
stats = Stats(scen)
stats.start_timing()
eta = ExecuteTARun(ta=lambda *args: None, stats=stats)
self.assertRaises(FirstRunCrashedException,
eta.start,
config={},
instance=1)
def test_start_exhausted_budget(self):
'''
testing exhausted budget
'''
# Set time-limit negative in scenario-options to trigger exception
scen = Scenario(scenario={
'wallclock_limit': -1,
'cs': ConfigurationSpace(),
'output_dir': ''
},
cmd_args=None)
stats = Stats(scen)
stats.start_timing()
eta = ExecuteTARun(
ta=lambda *args: None, # Dummy-function
stats=stats)
self.assertRaises(BudgetExhaustedException,
eta.start,
config={},
instance=1)
class TestSimpleIntensifier(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs,
values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs,
values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs,
values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs,
values={'a': 29, 'b': 31})
self.scen = Scenario({"cutoff_time": 2, 'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.intensifier = SimpleIntensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1],
)
def test_get_next_run(self):
"""
Makes sure that sampling a configuration returns a valid
configuration
"""
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config1],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
self.assertEqual(intent, RunInfoIntent.RUN)
self.assertEqual(run_info, RunInfo(
config=self.config1,
instance=1,
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0,
))
def test_get_next_run_waits_if_no_workers(self):
"""
In the case all workers are busy, we wait so that we do
not saturate the process with configurations that will not
finish in time
"""
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config1, self.config2],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
# We can get the configuration 1
self.assertEqual(intent, RunInfoIntent.RUN)
self.assertEqual(run_info, RunInfo(
config=self.config1,
instance=1,
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0,
))
# We should not get configuration 2
# As there is just 1 worker
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config2],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
self.assertEqual(intent, RunInfoIntent.WAIT)
self.assertEqual(run_info, RunInfo(
config=None,
instance=None,
instance_specific="0",
seed=0,
cutoff=None,
capped=False,
budget=0.0,
))
def test_process_results(self):
"""
Makes sure that we can process the results of a completed
configuration
"""
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config1, self.config2],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
result = RunValue(
cost=1,
time=0.5,
status=StatusType.SUCCESS,
starttime=1,
endtime=2,
additional_info=None,
)
self.rh.add(
config=run_info.config, cost=1, time=0.5,
status=StatusType.SUCCESS, instance_id=run_info.instance,
seed=run_info.seed, additional_info=None)
incumbent, inc_perf = self.intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(incumbent, run_info.config)
self.assertEqual(inc_perf, 1)
class TestAbstractIntensifier(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_get_next_challenger(self):
"""
test get_next_challenger - pick from list/chooser
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
# Error when nothing to choose from
with self.assertRaisesRegex(ValueError,
"No configurations/chooser provided"):
intensifier.get_next_challenger(challengers=None,
chooser=None,
run_history=self.rh)
# next challenger from a list
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh)
self.assertEqual(config, self.config1)
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3],
chooser=None,
run_history=self.rh)
self.assertEqual(config, self.config2)
# next challenger from a chooser
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=chooser,
run_history=self.rh)
self.assertEqual(len(list(config.get_dictionary().values())), 2)
self.assertTrue(24 in config.get_dictionary().values())
self.assertTrue(68 in config.get_dictionary().values())
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=chooser,
run_history=self.rh)
self.assertEqual(len(list(config.get_dictionary().values())), 2)
self.assertTrue(95 in config.get_dictionary().values())
self.assertTrue(38 in config.get_dictionary().values())
def test_get_next_challenger_repeat(self):
"""
test get_next_challenger - repeat configurations
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
# should not repeat configurations
self.rh.add(self.config1, 1, 1, StatusType.SUCCESS)
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh,
repeat_configs=False)
self.assertEqual(config, self.config2)
# should repeat configurations
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh,
repeat_configs=True)
self.assertEqual(config, self.config1)
def test_compare_configs_no_joint_set(self):
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
"""
incumbent is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(
self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(
config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
class TestIntensify(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_race_challenger(self):
"""
test _race_challenger without adaptive capping
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
intensifier.N = 1
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_2(self):
"""
test _race_challenger with adaptive capping
"""
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_3(self):
"""
test _race_challenger with adaptive capping on a previously capped configuration
"""
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="runtime",
par_factor=1)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1])
self.rh.add(config=self.config1,
cost=0.5,
time=.5,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.TIMEOUT,
instance_id=2,
seed=12345,
additional_info=None)
# give config2 a second chance - now it should run on both instances
# run on instance 1
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# run on instance 2
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
self.assertEqual(config, self.config2)
self.assertTrue(intensifier.continue_challenger)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2)
self.assertFalse(intensifier.continue_challenger)
self.assertEqual(intensifier.num_run, 3)
self.assertEqual(intensifier.num_chall_run, 3)
def test_race_challenger_large(self):
"""
test _race_challenger using solution_quality
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=True)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=12345,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
self.assertEqual(intensifier.num_run, 10)
self.assertEqual(intensifier.num_chall_run, 10)
def test_race_challenger_large_blocked_seed(self):
"""
test _race_challenger whether seeds are blocked for challenger runs
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
self.assertEqual(intensifier.num_run, 10)
self.assertEqual(intensifier.num_chall_run, 10)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run
# as it is the first evaluation of this intensifier
self.assertEqual(intensifier.num_run, 0)
self.assertEqual(intensifier.num_chall_run, 0)
def test_add_inc_run_nondet(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
runhistory=self.rh,
run_obj="solution_quality")
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1,
only_max_observed_budget=True)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
self.assertEqual(intensifier.num_run, 2)
self.assertEqual(intensifier.num_chall_run, 0)
def test_get_next_challenger(self):
"""
test get_next_challenger()
"""
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# get a new challenger to evaluate
config, new = intensifier.get_next_challenger(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertEqual(intensifier.N, 1)
self.assertTrue(new)
# when already evaluating a challenger, return the same challenger
intensifier.to_run = [(1, 1, 0)]
config, new = intensifier.get_next_challenger(
challengers=[self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertFalse(new)
def test_generate_challenger(self):
"""
test generate_challenger()
"""
# test get generator from a list of challengers
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
gen = intensifier._generate_challengers(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(next(gen), self.config1)
self.assertEqual(next(gen), self.config2)
self.assertRaises(StopIteration, next, gen)
# test get generator from a chooser - would return only 1 configuration
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser
gen = intensifier._generate_challengers(challengers=None,
chooser=chooser)
self.assertEqual(next(gen).get_dictionary(), {'a': 24, 'b': 68})
self.assertRaises(StopIteration, next, gen)
# when both are none, raise error
with self.assertRaisesRegex(ValueError,
"No configurations/chooser provided"):
intensifier._generate_challengers(challengers=None, chooser=None)
def test_eval_challenger_1(self):
"""
test eval_challenger() - a complete intensification run with a `always_race_against` configuration
"""
def target(x):
if x['a'] == 100:
time.sleep(1)
return x['a']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
run_obj_time=True,
cutoff=2,
deterministic=False,
always_race_against=self.config3,
run_limit=1)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# intensification iteration #1
# run first config as incumbent if incumbent is None
config, _ = intensifier.get_next_challenger(challengers=[self.config2],
chooser=None)
self.assertEqual(config, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# eval config 2 (=first run)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=None,
run_history=self.rh,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# intensification iteration #2
# regular intensification begins - run incumbent first
config, _ = intensifier.get_next_challenger(
challengers=None, # don't need a new list here as old one is cont'd
chooser=None)
self.assertEqual(config, inc)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(self.stats.ta_runs, 2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(self.stats.inc_changed, 1)
# run challenger now that the incumbent has been executed
config, _ = intensifier.get_next_challenger(challengers=[self.config1],
chooser=None)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(config, self.config1)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# challenger has a better performance, but not run on all instances yet. so incumbent stays the same
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertTrue(intensifier.continue_challenger)
# run challenger again on the other instance
config, _ = intensifier.get_next_challenger(
challengers=None, # don't need a new list here as old one is cont'd
chooser=None)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(config, self.config1)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# challenger better than incumbent in both instances. so incumbent changed
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS)
self.assertFalse(intensifier.continue_challenger)
# run basis configuration (`always_race_against`)
config, _ = intensifier.get_next_challenger(
challengers=None, # don't need a new list here as old one is cont'd
chooser=None)
self.assertEqual(config, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# the basis configuration (config3) not better than incumbent, so can move on
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 2)
self.assertEqual(self.stats.ta_runs, 5)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED)
self.assertEqual(
intensifier.n_iters,
1) # iteration continues as `min_chall` condition is not met
self.assertIsInstance(intensifier.configs_to_run,
collections.abc.Iterator)
# no configs should be left at the end
with self.assertRaises(StopIteration):
next(intensifier.configs_to_run)
# intensification continues running incumbent again in same iteration...
config, _ = intensifier.get_next_challenger(
challengers=None, # don't need a new list here as old one is cont'd
chooser=None)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)), 3)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)), 2)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)),
0) # capped
def test_eval_challenger_2(self):
"""
test eval_challenger() - a complete intensification run without a `always_race_against` configuration
"""
def target(x):
return 2 * x['a'] + x['b']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=True,
always_race_against=None,
run_limit=1)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# intensification iteration #1
# run first config as incumbent if incumbent is None
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
self.assertEqual(config, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# eval config 2 (=first run)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=None,
run_history=self.rh,
)
self.assertEqual(inc, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# regular intensification begins - run incumbent
config, _ = intensifier.get_next_challenger(
challengers=None, # since incumbent is run, no configs required
chooser=None)
self.assertEqual(config, inc)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# no new TA runs as there are no more instances to run
self.assertEqual(inc, self.config3)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(self.stats.ta_runs, 1)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)), 1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
# run challenger now that the incumbent has been executed
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config1], chooser=None)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(config, self.config2)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# challenger has a better performance, so incumbent has changed
self.assertEqual(inc, self.config2)
self.assertEqual(self.stats.inc_changed, 2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT
) # since there is no `always_race_against`
self.assertFalse(intensifier.continue_challenger)
self.assertEqual(
intensifier.n_iters,
1) # iteration continues as `min_chall` condition is not met
# intensification continues running incumbent again in same iteration...
# run incumbent
config, _ = intensifier.get_next_challenger(
challengers=None, # don't need a new list here as old one is cont'd
chooser=None)
self.assertEqual(config, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# run challenger
config, _ = intensifier.get_next_challenger(
challengers=None, # don't need a new list here as old one is cont'd
chooser=None)
self.assertEqual(config, self.config1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(intensifier.n_iters,
2) # 2 intensification run complete!
# no configs should be left at the end
with self.assertRaises(StopIteration):
next(intensifier.configs_to_run)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)), 1)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)), 1)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)), 1)
def test_eval_challenger_3(self):
"""
test eval_challenger for a resumed SMAC run (first run with incumbent)
"""
def target(x):
return x['a']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=False,
always_race_against=None,
run_limit=1)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# adding run for incumbent configuration
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
# intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage
config, _ = intensifier.get_next_challenger(challengers=[self.config2],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)), 2)
def test_no_new_intensification_wo_challenger_run(self):
"""
This test ensures that no new iteration is started if no challenger run was conducted
"""
def target(x):
return 2 * x['a'] + x['b']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(
tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=True,
always_race_against=None,
run_limit=1,
min_chall=1,
)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
self.assertEqual(config, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=None,
run_history=self.rh,
)
self.assertEqual(inc, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# regular intensification begins - run incumbent
config, _ = intensifier.get_next_challenger(
challengers=None, # since incumbent is run, no configs required
chooser=None)
self.assertEqual(config, inc)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(intensifier.n_iters, 1)
# Check that we don't walk into the next iteration if the challenger is passed again
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(intensifier.n_iters, 1)
intensifier._next_iteration()
# Add a configuration, then try to execute it afterwards
self.assertEqual(intensifier.n_iters, 2)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=0,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
config, _ = intensifier.get_next_challenger(challengers=[self.config1],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.n_iters, 2)
self.assertEqual(intensifier.num_chall_run, 0)
# This returns the config evaluating the incumbent again
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# This doesn't return a config because the array of configs is exhausted
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=None)
self.assertIsNone(config)
# This finally gives a runable configuration
config, _ = intensifier.get_next_challenger(challengers=[self.config2],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.n_iters, 3)
self.assertEqual(intensifier.num_chall_run, 1)
class TestAbstractRacer(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_compare_configs_no_joint_set(self):
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
"""
incumbent is better
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(
self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(
config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
class TestIntensify(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory(aggregate_func=average_cost)
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"output_dir": ""
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger("Test")
def test_compare_configs_chall(self):
'''
challenger is better but has not enough runs
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
'''
incumbent is better
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
'''
challenger is better but has less runs;
-> no decision (None)
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_race_challenger(self):
'''
test _race_challenger without adaptive capping
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
self.assertEqual(inc, self.config2)
def test_race_challenger(self):
'''
test _race_challenger with adaptive capping
'''
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
self.assertLess(self.rh.get_cost(self.config2), 2,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
run = self.rh.get_runs_for_config(self.config2)[0]
config_id = self.rh.config_ids[self.config2]
self.assertEqual(run.instance, 1, run.instance)
self.assertEqual(run.seed, 12345, run.seed)
def test_race_challenger_large(self):
'''
test _race_challenger using solution_quality
'''
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
deterministic=True)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=12345,
additional_info=None)
# tie on first instances and then challenger should always win
# and be returned as inc
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2)
self.assertEqual(len(runs), 10)
def test_race_challenger_large_blocked_seed(self):
'''
test _race_challenger whether seeds are blocked for challenger runs
'''
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
# tie on first instances and then challenger should always win
# and be returned as inc
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
def test_add_inc_run_det(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_add_inc_run_nondet(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_adaptive_capping(self):
'''
test _adapt_cutoff()
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(self.config1)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=self.config1,
instance_seed_pairs=inst_seed_pairs,
run_history=self.rh)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def run_configuration(backend, config_id, task_id, configuration, run_args,
memory_limit, per_run_time_limit):
evaluation, iterative_fit, early_stopping, N_FOLDS, searchspace = run_args
# TODO make this an argument from the command line!
scenario_mock = unittest.mock.Mock()
scenario_mock.wallclock_limit = per_run_time_limit * 100
scenario_mock.algo_runs_timelimit = per_run_time_limit * 100
scenario_mock.ta_run_limit = np.inf
stats = Stats(scenario_mock)
stats.ta_runs = 2
# Resampling strategies
kwargs = {}
if evaluation == "holdout" and iterative_fit:
resampling_strategy = 'holdout-iterative-fit'
elif evaluation == "holdout" and not iterative_fit:
resampling_strategy = 'holdout'
elif evaluation == "CV" and not iterative_fit:
resampling_strategy = 'cv'
kwargs = {'folds': N_FOLDS}
elif evaluation == "CV" and iterative_fit:
resampling_strategy = 'cv-iterative-fit'
kwargs = {'folds': N_FOLDS}
else:
raise ValueError("Unknown resampling strategy", evaluation)
iterative_wo_early_stopping = [
'extra_trees', 'PassiveAggressiveWOEarlyStopping', 'random_forest',
'SGDWOEarlyStopping', 'GradientBoostingClassifierWOEarlyStopping'
]
iterative_w_early_stopping = [
'extra_trees', 'passive_aggressive', 'random_forest', 'sgd',
'gradient_boosting'
]
if not early_stopping:
add_classifier_wo_early_stopping()
if searchspace == "iterative":
include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
include_preprocessor = [
"no_preprocessing",
]
elif searchspace == "full":
assert early_stopping is True
include_estimator = None
include_preprocessor = None
# elif searchspace == 'only-iterative-nopreproc':
# include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
# include_preprocessor = ["no_preprocessing", ]
# elif searchspace == 'only-iterative-cheappreproc':
# include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
# include_preprocessor = ["no_preprocessing", 'kitchen_sinks', 'polynomial', 'select_percentile_classification', 'select_rates']
# elif searchspace == 'only-iterative':
# include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping
# include_preprocessor = None
# elif searchspace == "gb":
# include_estimator = ['GradientBoostingClassifierWOEarlyStopping'] if early_stopping else ['GradientBoostingClassifierWEarlyStopping']
# include_preprocessor = None
else:
raise ValueError(searchspace)
stats.start_timing()
tae = ExecuteTaFuncWithQueue(
backend=backend,
autosklearn_seed=3,
resampling_strategy=resampling_strategy,
metric=balanced_accuracy,
logger=logging.getLogger(name="%s_%s" % (task_id, config_id)),
initial_num_run=2,
stats=stats,
runhistory=None,
run_obj='quality',
par_factor=1,
all_scoring_functions=False,
output_y_hat_optimization=True,
include={
"classifier": include_estimator,
"feature_preprocessor": include_preprocessor
},
exclude=None,
memory_limit=memory_limit,
disable_file_output=True,
init_params=None,
**kwargs)
# Finally run configuration
status, cost, runtime, additional_run_info = tae.start(
config=configuration,
instance=None,
cutoff=per_run_time_limit,
instance_specific=None,
capped=False,
)
return status, cost, runtime, additional_run_info
class TestIntensify(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={"a": 0, "b": 100})
self.config2 = Configuration(self.cs, values={"a": 100, "b": 0})
self.config3 = Configuration(self.cs, values={"a": 100, "b": 100})
self.scen = Scenario({
"cutoff_time": 2,
"cs": self.cs,
"run_obj": "runtime",
"output_dir": "",
"deterministic": False,
"limit_resources": True,
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_race_challenger_1(self):
"""
Makes sure that a racing configuration with better performance,
is selected as incumbent
No adaptive capping
"""
def target(x):
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
)
self.rh.add(
config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None,
)
intensifier.N = 1
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
cutoff=cutoff,
seed=seed,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_2(self):
"""
Makes sure that a racing configuration with better performance,
that is capped, doesn't substitute the incumbent.
"""
def target(x):
time.sleep(1.5)
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
)
self.rh.add(
config=self.config1,
cost=0.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None,
)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=True,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_3(self):
"""
test _race_challenger with adaptive capping on a previously capped configuration
"""
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(
use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="runtime",
par_factor=1,
)
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1],
)
self.rh.add(
config=self.config1,
cost=0.5,
time=0.5,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None,
)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=config, incumbent=self.config1, run_history=self.rh)
run_info = RunInfo(
config=config,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=True,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(
config=self.config1,
cost=2,
time=2,
status=StatusType.TIMEOUT,
instance_id=2,
seed=12345,
additional_info=None,
)
# give config2 a second chance - now it should run on both instances
# run on instance 1
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=config, incumbent=self.config1, run_history=self.rh)
run_info = RunInfo(
config=config,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# run on instance 2
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
self.assertEqual(config, self.config2)
self.assertTrue(intensifier.continue_challenger)
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=config, incumbent=self.config1, run_history=self.rh)
run_info = RunInfo(
config=config,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2)
self.assertFalse(intensifier.continue_challenger)
self.assertEqual(intensifier.num_run, 3)
self.assertEqual(intensifier.num_chall_run, 3)
def test_race_challenger_large(self):
"""
test _race_challenger using solution_quality
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=True,
)
for i in range(10):
self.rh.add(
config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=12345,
additional_info=None,
)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
if intensifier.continue_challenger:
config = intensifier.current_challenger
else:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=config, incumbent=self.config1, run_history=self.rh)
run_info = RunInfo(
config=config,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
self.assertEqual(intensifier.num_run, 10)
self.assertEqual(intensifier.num_chall_run, 10)
def test_race_challenger_large_blocked_seed(self):
"""
test _race_challenger whether seeds are blocked for challenger runs
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=False,
)
for i in range(10):
self.rh.add(
config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None,
)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
if intensifier.continue_challenger:
config = intensifier.current_challenger
else:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=config, incumbent=self.config1, run_history=self.rh)
run_info = RunInfo(
config=config,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
self.assertEqual(intensifier.num_run, 10)
self.assertEqual(intensifier.num_chall_run, 10)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True,
)
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
# So the returned seed/instance is None so that a new
# run to be triggered is not launched
available_insts = intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh)
# Make sure that the list is empty, and hence no new call
# of incumbent will be triggered
self.assertFalse(available_insts)
# The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run
# as it is the first evaluation of this intensifier
# After the above incumbent run, the stage is
# IntensifierStage.RUN_CHALLENGER. Change it to test next iteration
intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(intensifier.num_run, 0)
self.assertEqual(intensifier.num_chall_run, 0)
def test_add_inc_run_nondet(self):
"""
test _add_inc_run()
"""
def target(x):
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="solution_quality")
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False,
)
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1,
only_max_observed_budget=True)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
# The number of runs performed should be 3
# No Next iteration call as an incumbent is provided
self.assertEqual(intensifier.num_run, 2)
self.assertEqual(intensifier.num_chall_run, 0)
def testget_next_challenger(self):
"""
test get_next_challenger()
"""
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True,
)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# get a new challenger to evaluate
config, new = intensifier.get_next_challenger(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertEqual(intensifier.N, 1)
self.assertTrue(new)
# when already evaluating a challenger, return the same challenger
intensifier.to_run = [(1, 1, 0)]
config, new = intensifier.get_next_challenger(
challengers=[self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertFalse(new)
def test_generate_challenger(self):
"""
test generate_challenger()
"""
# test get generator from a list of challengers
intensifier = Intensifier(
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True,
)
gen = intensifier._generate_challengers(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(next(gen), self.config1)
self.assertEqual(next(gen), self.config2)
self.assertRaises(StopIteration, next, gen)
# test get generator from a chooser - would return only 1 configuration
intensifier = Intensifier(
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True,
)
chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser
gen = intensifier._generate_challengers(challengers=None,
chooser=chooser)
self.assertEqual(next(gen).get_dictionary(), {"a": 24, "b": 68})
self.assertRaises(StopIteration, next, gen)
# when both are none, raise error
with self.assertRaisesRegex(ValueError,
"No configurations/chooser provided"):
intensifier._generate_challengers(challengers=None, chooser=None)
def test_eval_challenger_1(self):
"""
test eval_challenger() - a complete intensification run with a `always_race_against` configuration
"""
print(self.rh)
def target(x):
if x["a"] == 100:
time.sleep(1)
return x["a"]
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
run_obj_time=True,
cutoff=2,
deterministic=False,
always_race_against=self.config3,
run_limit=1,
)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# intensification iteration #1
# run first config as incumbent if incumbent is None
intent, run_info = intensifier.get_next_run(
incumbent=None,
run_history=self.rh,
challengers=[self.config2],
chooser=None,
)
self.assertEqual(run_info.config, self.config2)
self.assertEqual(intensifier.stage,
IntensifierStage.PROCESS_FIRST_CONFIG_RUN)
# eval config 2 (=first run)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# intensification iteration #2
# regular intensification begins - run incumbent first
intent, run_info = intensifier.get_next_run(
challengers=None, # don't need a new list here as old one is cont'd
incumbent=inc,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, inc)
self.assertEqual(intensifier.stage,
IntensifierStage.PROCESS_INCUMBENT_RUN)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(self.stats.inc_changed, 1)
# run challenger now that the incumbent has been executed
intent, run_info = intensifier.get_next_run(challengers=[self.config1],
incumbent=inc,
run_history=self.rh,
chooser=None)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(run_info.config, self.config1)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# challenger has a better performance, but not run on all instances yet. so incumbent stays the same
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertTrue(intensifier.continue_challenger)
# run challenger again on the other instance
intent, run_info = intensifier.get_next_run(
challengers=None, # don't need a new list here as old one is cont'd
incumbent=inc,
run_history=self.rh,
chooser=None,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(run_info.config, self.config1)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# challenger better than incumbent in both instances. so incumbent changed
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS)
self.assertFalse(intensifier.continue_challenger)
# run basis configuration (`always_race_against`)
intent, run_info = intensifier.get_next_run(
challengers=None, # don't need a new list here as old one is cont'd
incumbent=inc,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# the basis configuration (config3) not better than incumbent, so can move on
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 2)
self.assertEqual(intensifier.stage,
IntensifierStage.RUN_INCUMBENT,
msg=self.rh.data.items())
self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED)
self.assertEqual(
intensifier.n_iters,
1) # iteration continues as `min_chall` condition is not met
self.assertIsInstance(intensifier.configs_to_run,
collections.abc.Iterator)
# no configs should be left at the end
with self.assertRaises(StopIteration):
next(intensifier.configs_to_run)
# intensification continues running incumbent again in same iteration...
intent, run_info = intensifier.get_next_run(
challengers=None, # don't need a new list here as old one is cont'd
incumbent=inc,
run_history=self.rh,
chooser=None,
)
self.assertEqual(intensifier.stage,
IntensifierStage.PROCESS_INCUMBENT_RUN)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)),
3,
)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)),
2,
)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)),
0,
) # capped
def test_eval_challenger_2(self):
"""
test eval_challenger() - a complete intensification run without a `always_race_against` configuration
"""
def target(x):
return 2 * x["a"] + x["b"]
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=True,
always_race_against=None,
run_limit=1,
)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# intensification iteration #1
# run first config as incumbent if incumbent is None
intent, run_info = intensifier.get_next_run(
challengers=[self.config3],
incumbent=None,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, self.config3)
self.assertEqual(intensifier.stage,
IntensifierStage.PROCESS_FIRST_CONFIG_RUN)
# eval config 2 (=first run)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# regular intensification begins - run incumbent
# Normally a challenger will be given, which in this case is the incumbent
# But no more instances are available. So to prevent cicles
# where No iteration happens, provide the challengers
intent, run_info = intensifier.get_next_run(
challengers=[
self.config2,
self.config1,
], # since incumbent is run, no configs required
incumbent=inc,
run_history=self.rh,
chooser=None,
)
# no new TA runs as there are no more instances to run
self.assertEqual(inc, self.config3)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)),
1,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
# run challenger now that the incumbent has been executed
# So this call happen above, to save one iteration
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(run_info.config, self.config2)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# challenger has a better performance, so incumbent has changed
self.assertEqual(inc, self.config2)
self.assertEqual(self.stats.inc_changed, 2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT
) # since there is no `always_race_against`
self.assertFalse(intensifier.continue_challenger)
self.assertEqual(
intensifier.n_iters,
1) # iteration continues as `min_chall` condition is not met
# intensification continues running incumbent again in same iteration...
# run incumbent
# Same here, No further instance-seed pairs for incumbent available
# so above call gets the new config to run
self.assertEqual(run_info.config, self.config2)
# There is a transition from:
# IntensifierStage.RUN_FIRST_CONFIG-> IntensifierStage.RUN_INCUMBENT
# Because after the first run, incumbent is run.
# Nevertheless, there is now a transition:
# IntensifierStage.RUN_INCUMBENT->IntensifierStage.RUN_CHALLENGER
# because in add_inc_run, there are more available instance pairs
# FROM: IntensifierStage.RUN_INCUMBENT TO: IntensifierStage.RUN_INCUMBENT WHY: no more to run
# if all <instance, seed> have been run, compare challenger performance
# self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# run challenger
intent, run_info = intensifier.get_next_run(
challengers=None, # don't need a new list here as old one is cont'd
incumbent=inc,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, self.config1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(intensifier.n_iters,
2) # 2 intensification run complete!
# no configs should be left at the end
with self.assertRaises(StopIteration):
next(intensifier.configs_to_run)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)),
1,
)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)),
1,
)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)),
1,
)
def test_eval_challenger_3(self):
"""
test eval_challenger for a resumed SMAC run (first run with incumbent)
"""
def target(x):
return x["a"]
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=False,
always_race_against=None,
run_limit=1,
)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
# adding run for incumbent configuration
self.rh.add(
config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None,
)
# intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage
intent_, run_info = intensifier.get_next_run(
challengers=[self.config2],
incumbent=self.config1,
run_history=self.rh,
chooser=None,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config1,
only_max_observed_budget=True)),
2,
)
def test_no_new_intensification_wo_challenger_run(self):
"""
This test ensures that no new iteration is started if no challenger run was conducted
"""
def target(x):
return 2 * x["a"] + x["b"]
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=True,
always_race_against=None,
run_limit=1,
min_chall=1,
)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
intent, run_info = intensifier.get_next_run(
challengers=[self.config3],
incumbent=None,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, self.config3)
self.assertEqual(intensifier.stage,
IntensifierStage.PROCESS_FIRST_CONFIG_RUN)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# regular intensification begins - run incumbent
# No further instance-seed pairs for incumbent available
# Here None challenger is suggested. Code jumps to next iteration
# This causes a transition from RUN_INCUMBENT->RUN_CHALLENGER
# But then, the next configuration to run is the incumbent
# We don't rerun the incumbent (see message):
# Challenger was the same as the current incumbent; Skipping challenger
# Then, we try to get more challengers, but below all challengers
# Provided are config3, the incumbent which means nothing more to run
intent, run_info = intensifier.get_next_run(
challengers=[self.config3
], # since incumbent is run, no configs required
incumbent=inc,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, None)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
intensifier._next_iteration()
# Add a configuration, then try to execute it afterwards
self.assertEqual(intensifier.n_iters, 2)
self.rh.add(
config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=0,
additional_info=None,
)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# In the upcoming get next run, the stage is RUN_CHALLENGER
# so the intensifier tries to run config1. Nevertheless,
# there are no further instances for this configuration available.
# In this scenario, the intensifier produces a SKIP intent as an indication
# that a new iteration must be initiated, and for code simplicity,
# relies on a new call to get_next_run to yield more configurations
intent, run_info = intensifier.get_next_run(challengers=[self.config1],
incumbent=inc,
run_history=self.rh,
chooser=None)
self.assertEqual(intent, RunInfoIntent.SKIP)
# This doesn't return a config because the array of configs is exhausted
intensifier.stage = IntensifierStage.RUN_CHALLENGER
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=None)
self.assertIsNone(config)
# This finally gives a runable configuration
intent, run_info = intensifier.get_next_run(challengers=[self.config2],
incumbent=inc,
run_history=self.rh,
chooser=None)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# 4 Iterations due to the proactive runs
# of get next challenger
self.assertEqual(intensifier.n_iters, 3)
self.assertEqual(intensifier.num_chall_run, 1)
class Test_Hyperband(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_update_stage(self):
"""
test initialization of all parameters and tracking variables
"""
intensifier = _Hyperband(stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1],
initial_budget=0.1,
max_budget=1,
eta=2)
intensifier._update_stage()
self.assertEqual(intensifier.s, 3)
self.assertEqual(intensifier.s_max, 3)
self.assertEqual(intensifier.hb_iters, 0)
self.assertIsInstance(intensifier.sh_intensifier, _SuccessiveHalving)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage,
[8.0, 4.0, 2.0, 1.0])
# next HB stage
intensifier._update_stage()
self.assertEqual(intensifier.s, 2)
self.assertEqual(intensifier.hb_iters, 0)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.25)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage,
[4.0, 2.0, 1.0])
intensifier._update_stage() # s = 1
intensifier._update_stage() # s = 0
# HB iteration completed
intensifier._update_stage()
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(intensifier.hb_iters, 1)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage,
[8.0, 4.0, 2.0, 1.0])
def test_eval_challenger(self):
"""
since hyperband uses eval_challenger and get_next_run of the internal successive halving,
we don't test these method extensively
"""
def target(x):
return 0.1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = _Hyperband(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[None],
initial_budget=0.5,
max_budget=1,
eta=2)
self.assertFalse(hasattr(intensifier, 's'))
# Testing get_next_run - get next configuration
intent, run_info = intensifier.get_next_run(
challengers=[self.config2, self.config3],
chooser=None,
incumbent=None,
run_history=self.rh)
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(run_info.config, self.config2)
# update to the last SH iteration of the given HB stage
self.assertEqual(intensifier.s, 1)
self.assertEqual(intensifier.s_max, 1)
# We assume now that process results was called with below successes.
# We track closely run execution through run_tracker, so this also
# has to be update -- the fact that the succesive halving inside hyperband
# processed the given configurations
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
seed=0,
budget=1)
intensifier.sh_intensifier.run_tracker[(self.config1, None, 0,
1)] = True
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
seed=0,
budget=0.5)
intensifier.sh_intensifier.run_tracker[(self.config2, None, 0,
0.5)] = True
self.rh.add(config=self.config3,
cost=3,
time=2,
status=StatusType.SUCCESS,
seed=0,
budget=0.5)
intensifier.sh_intensifier.run_tracker[(self.config3, None, 0,
0.5)] = True
intensifier.sh_intensifier.success_challengers = {
self.config2, self.config3
}
intensifier.sh_intensifier._update_stage(self.rh)
intent, run_info = intensifier.get_next_run(
challengers=[self.config2, self.config3],
chooser=None,
incumbent=None,
run_history=self.rh)
# evaluation should change the incumbent to config2
self.assertIsNotNone(run_info.config)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, inc_value = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.s, 0)
self.assertEqual(inc_value, 0.1)
self.assertEqual(
list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2])
self.assertEqual(self.stats.inc_changed, 1)
def runhistory_builder(ta,scenario_dic,rng):
tae_runner = ExecuteTARun(ta=ta)
scenario = Scenario(scenario_dic)
stats = Stats(scenario=scenario)
traj_logger = TrajLogger(stats=stats,output_dir="/home/dfki/Desktop/temp")
# if tae_runner.stats is None:
# new_smac =SMAC(scenario=scenario,tae_runner=tae_runner)
# tae_runner.stats = new_smac.stats
stats.start_timing()
deful_config_builder = DefaultConfiguration(tae_runner,scenario,stats,traj_logger,rng)
config_milad =deful_config_builder._select_configuration()
config_milad._values = None
config_milad._values = {'balancing:strategy': 'none', 'categorical_encoding:__choice__': 'one_hot_encoding', 'classifier:__choice__': 'random_forest', 'imputation:strategy': 'mean', 'preprocessor:__choice__': 'no_preprocessing', 'rescaling:__choice__': 'standardize', 'categorical_encoding:one_hot_encoding:use_minimum_fraction': 'True', 'classifier:random_forest:bootstrap': 'True', 'classifier:random_forest:criterion': 'gini', 'classifier:random_forest:max_depth': 10, 'classifier:random_forest:max_features': 0.5, 'classifier:random_forest:max_leaf_nodes': 'None', 'classifier:random_forest:min_impurity_decrease': 0.0, 'classifier:random_forest:min_samples_leaf': 1, 'classifier:random_forest:min_samples_split': 2, 'classifier:random_forest:min_weight_fraction_leaf': 0.0, 'classifier:random_forest:n_estimators': 100, 'categorical_encoding:one_hot_encoding:minimum_fraction': 0.01}
# config_milad._values = {'balancing:strategy': 'none',
# 'categorical_encoding:__choice__': 'no_encoding',
# 'classifier:__choice__': 'random_forest',
# 'imputation:strategy': 'mean',
# 'preprocessor:__choice__': 'pca',
# 'preprocessor:copy':True,
# 'preprocessor:iterated_power':'auto',
# 'preprocessor:n_components':'None',
# 'preprocessor:random_state':'None',
# 'preprocessor:svd_solver':'auto',
# 'preprocessor:tol':0.0,
# 'preprocessor:whiten':'False',
# 'rescaling:__choice__': 'None',
# 'classifier:random_forest:bootstrap': 'True',
# 'classifier:random_forest:class_weight': 'None',
# 'classifier:random_forest:criterion': 'gini',
# 'classifier:random_forest:max_depth': 'None',
# 'classifier:random_forest:max_features': 'auto',
# 'classifier:random_forest:max_leaf_nodes': 'None',
# 'classifier:random_forest:min_impurity_decrease': 0.0,
# 'classifier:random_forest:min_impurity_split': '1e-07',
# 'classifier:random_forest:min_samples_leaf': 1,
# 'classifier:random_forest:min_samples_split': 2,
# 'classifier:random_forest:min_weight_fraction_leaf': 0.0,
# 'classifier:random_forest:n_estimators': 10,
# 'classifier:random_forest:n_jobs': 1,
# 'classifier:random_forest:oob_score': 'False',
# 'classifier:random_forest:random_state': 'None',
# 'classifier:random_forest:verbose': 0,
# 'classifier:random_forest:warm_start': 'False',
# }
# config_milad._vector =None
status, cost, runtime, additional_info = tae_runner.start(config=config_milad,instance=None)
print(status, cost, runtime, additional_info)
runhistory = RunHistory(aggregate_func=average_cost)
runhistory.add( config=config_milad,
cost=cost,
time=runtime,
status=status,
instance_id=None,
additional_info=additional_info)
return runhistory
print('Starting to validate configurations')
for i, entry in enumerate(trajectory):
print('Starting to validate configuration %d/%d' %
(i + 1, len(trajectory)))
incumbent_id = entry.incumbent_id
train_performance = entry.train_perf
if incumbent_id not in incumbent_id_to_model:
config = entry.incumbent
logger = logging.getLogger('Testing:)')
stats = Stats(
Scenario({
'cutoff_time': per_run_time_limit * 2,
'run_obj': 'quality',
}))
stats.start_timing()
# To avoid the output "first run crashed"...
stats.submitted_ta_runs += 1
stats.finished_ta_runs += 1
memory_lim = memory_limit_factor * automl_arguments['memory_limit']
ta = ExecuteTaFuncWithQueue(
backend=automl.automl_._backend,
autosklearn_seed=seed,
resampling_strategy='test',
memory_limit=memory_lim,
disable_file_output=True,
logger=logger,
stats=stats,
scoring_functions=scoring_functions,
include=include,
metric=automl_arguments['metric'],
class TestHyperband(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_update_stage(self):
"""
test initialization of all parameters and tracking variables
"""
intensifier = Hyperband(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1],
initial_budget=0.1,
max_budget=1,
eta=2)
intensifier._update_stage()
self.assertEqual(intensifier.s, 3)
self.assertEqual(intensifier.s_max, 3)
self.assertEqual(intensifier.hb_iters, 0)
self.assertIsInstance(intensifier.sh_intensifier, SuccessiveHalving)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage,
[8.0, 4.0, 2.0, 1.0])
# next HB stage
intensifier._update_stage()
self.assertEqual(intensifier.s, 2)
self.assertEqual(intensifier.hb_iters, 0)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.25)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage,
[4.0, 2.0, 1.0])
intensifier._update_stage() # s = 1
intensifier._update_stage() # s = 0
# HB iteration completed
intensifier._update_stage()
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(intensifier.hb_iters, 1)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage,
[8.0, 4.0, 2.0, 1.0])
def test_eval_challenger(self):
"""
since hyperband uses eval_challenger and get_next_challenger of the internal successive halving,
we don't test these method extensively
"""
def target(x):
return 0.1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Hyperband(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[None],
initial_budget=0.5,
max_budget=1,
eta=2)
self.assertFalse(hasattr(intensifier, 's'))
# Testing get_next_challenger - get next configuration
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3],
chooser=None,
run_history=self.rh)
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(config, self.config2)
# update to the last SH iteration of the given HB stage
self.assertEqual(intensifier.s, 1)
self.assertEqual(intensifier.s_max, 1)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
seed=0,
budget=1)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
seed=0,
budget=0.5)
self.rh.add(config=self.config3,
cost=3,
time=2,
status=StatusType.SUCCESS,
seed=0,
budget=0.5)
intensifier.sh_intensifier.curr_challengers = {
self.config2, self.config3
}
intensifier.sh_intensifier._update_stage(self.rh)
# evaluation should change the incumbent to config2
inc, inc_value = intensifier.eval_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.s, 0)
self.assertEqual(inc_value, 0.1)
self.assertEqual(self.stats.ta_runs, 1)
self.assertEqual(
list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2])
self.assertEqual(self.stats.inc_changed, 1)
def validate(self,
config_mode: str = 'def',
instance_mode: str = 'test',
repetitions: int = 1,
n_jobs: int = 1,
backend: str = 'threading',
runhistory: RunHistory = None,
tae: ExecuteTARun = None):
"""
Validate configs on instances and save result in runhistory.
Parameters
----------
config_mode: string
what configurations to validate
from [def, inc, def+inc, time, all], time means evaluation at
timesteps 2^-4, 2^-3, 2^-2, 2^-1, 2^0, 2^1, ...
instance_mode: string
what instances to use for validation, from [train, test, train+test]
repetitions: int
number of repetitions in nondeterministic algorithms
n_jobs: int
number of parallel processes used by joblib
runhistory: RunHistory or string or None
runhistory to take data from
tae: ExecuteTARun
tae to be used. if none, will initialize ExecuteTARunOld
Returns
-------
runhistory: RunHistory
runhistory with validated runs
"""
self.logger.debug(
"Validating configs '%s' on instances '%s', repeating %d times"
" with %d parallel runs on backend '%s'.", config_mode,
instance_mode, repetitions, n_jobs, backend)
# Reset runhistory
self.rh = RunHistory(average_cost)
# Get relevant configurations and instances
configs = self._get_configs(config_mode)
instances = self._get_instances(instance_mode)
# If runhistory is given as string, load into memory
if isinstance(runhistory, str):
fn = runhistory
runhistory = RunHistory(average_cost)
runhistory.load_json(fn, self.scen.cs)
# Get all runs needed as list
runs = self.get_runs(configs,
instances,
repetitions=repetitions,
runhistory=runhistory)
# Create new Stats without limits
inf_scen = Scenario({
'run_obj': self.scen.run_obj,
'cutoff_time': self.scen.cutoff,
'output_dir': None
})
inf_stats = Stats(inf_scen)
inf_stats.start_timing()
# Create TAE
if not tae:
tae = ExecuteTARunOld(ta=self.scen.ta,
stats=inf_stats,
run_obj=self.scen.run_obj,
par_factor=self.scen.par_factor,
cost_for_crash=self.scen.cost_for_crash)
else:
# Inject endless-stats
tae.stats = inf_stats
# Validate!
run_results = self._validate_parallel(tae, runs, n_jobs, backend)
# tae returns (status, cost, runtime, additional_info)
# Add runs to RunHistory
idx = 0
for result in run_results:
self.rh.add(config=runs[idx]['config'],
cost=result[1],
time=result[2],
status=result[0],
instance_id=runs[idx]['inst'],
seed=runs[idx]['seed'],
additional_info=result[3])
idx += 1
# Save runhistory
if not self.output.endswith('.json'):
old = self.output
self.output = os.path.join(self.output,
'validated_runhistory.json')
self.logger.debug("Output is \"%s\", changing to \"%s\"!", old,
self.output)
base = os.path.split(self.output)[0]
if not os.path.exists(base):
self.logger.debug("Folder (\"%s\") doesn't exist, creating.", base)
os.makedirs(base)
self.logger.info("Saving validation-results in %s", self.output)
self.rh.save_json(self.output)
return self.rh
class TestHyperband(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31})
self.config5 = Configuration(self.cs, values={'a': 31, 'b': 33})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=False,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
n_seeds=2,
initial_budget=2,
max_budget=5,
eta=2,
)
def test_initialization(self):
"""Makes sure that a proper_HB is created"""
# We initialize the HB with zero intensifier_instances
self.assertEqual(len(self.HB.intensifier_instances), 0)
# Add an instance to check the_HB initialization
self.assertTrue(self.HB._add_new_instance(num_workers=1))
# Some default init
self.assertEqual(self.HB.intensifier_instances[0].hb_iters, 0)
self.assertEqual(self.HB.intensifier_instances[0].max_budget, 5)
self.assertEqual(self.HB.intensifier_instances[0].initial_budget, 2)
# Update the stage
(self.HB.intensifier_instances[0]._update_stage(self.rh))
# Parameters properly passed to SH
self.assertEqual(
len(self.HB.intensifier_instances[0].sh_intensifier.inst_seed_pairs
), 10)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.max_budget, 5)
def test_process_results_via_sourceid(self):
"""Makes sure source id is honored when deciding
which_HB instance will consume the result/run_info
"""
# Mock the_HB instance so we can make sure the correct item is passed
for i in range(10):
self.HB.intensifier_instances[i] = mock.Mock()
self.HB.intensifier_instances[i].process_results.return_value = (
self.config1, 0.5)
# make iter false so the mock object is not overwritten
self.HB.intensifier_instances[i].iteration_done = False
# randomly create run_infos and push into HB. Then we will make
# sure they got properly allocated
for i in np.random.choice(list(range(10)), 30):
run_info = RunInfo(
config=self.config1,
instance=0,
instance_specific="0",
cutoff=None,
seed=0,
capped=False,
budget=0.0,
source_id=i,
)
# make sure results aren't messed up via magic variable
# That is we check only the proper_HB instance has this
magic = time.time()
result = RunValue(cost=1,
time=0.5,
status=StatusType.SUCCESS,
starttime=1,
endtime=2,
additional_info=magic)
self.HB.process_results(run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=None,
result=result,
log_traj=False)
# Check the call arguments of each sh instance and make sure
# it is the correct one
# First the expected one
self.assertEqual(
self.HB.intensifier_instances[i].process_results.call_args[1]
['run_info'], run_info)
self.assertEqual(
self.HB.intensifier_instances[i].process_results.call_args[1]
['result'], result)
all_other_run_infos, all_other_results = [], []
for j in range(len(self.HB.intensifier_instances)):
# Skip the expected_HB instance
if i == j:
continue
if self.HB.intensifier_instances[
j].process_results.call_args is None:
all_other_run_infos.append(None)
else:
all_other_run_infos.append(
self.HB.intensifier_instances[j].process_results.
call_args[1]['run_info'])
all_other_results.append(
self.HB.intensifier_instances[j].process_results.
call_args[1]['result'])
self.assertNotIn(run_info, all_other_run_infos)
self.assertNotIn(result, all_other_results)
def test_get_next_run_single_HB_instance(self):
"""Makes sure that a single_HB instance returns a valid config"""
challengers = [self.config1, self.config2, self.config3, self.config4]
for i in range(30):
intent, run_info = self.HB.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=self.rh,
num_workers=1,
)
# Regenerate challenger list
challengers = [c for c in challengers if c != run_info.config]
if intent == RunInfoIntent.WAIT:
break
# Add the config to self.rh in order to make HB aware that this
# config/instance was launched
self.rh.add(
config=run_info.config,
cost=10,
time=0.0,
status=StatusType.RUNNING,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
# smax==1 (int(np.floor(np.log(self.max_budget / self.initial_budget) / np.log(self. eta))))
self.assertEqual(self.HB.intensifier_instances[0].s_max, 1)
# And we do not even complete 1 iteration, so s has to be 1
self.assertEqual(self.HB.intensifier_instances[0].s, 1)
# We should not create more_HB instance intensifier_instances
self.assertEqual(len(self.HB.intensifier_instances), 1)
# We are running with:
# 'all_budgets': array([2.5, 5. ]) -> 2 intensifier_instances per config top
# 'n_configs_in_stage': [2.0, 1.0],
# This means we run int(2.5) + 2.0 = 4 runs before waiting
self.assertEqual(i, 4)
# Also, check the internals of the unique sh instance
# sh_initial_budget==2.5 (self.eta ** -self.s * self.max_budget)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2)
# n_challengers=2 (int(np.floor((self.s_max + 1) / (self.s + 1)) * self.eta ** self.s))
self.assertEqual(
len(self.HB.intensifier_instances[0].sh_intensifier.
n_configs_in_stage), 2)
def test_get_next_run_multiple_HB_instances(self):
"""Makes sure that two _HB instance can properly coexist and tag
run_info properly"""
# We allow 2_HB instance to be created. This means, we have a newer iteration
# to expect in hyperband
challengers = [self.config1, self.config2, self.config3, self.config4]
run_infos = []
for i in range(30):
intent, run_info = self.HB.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=self.rh,
num_workers=2,
)
run_infos.append(run_info)
# Regenerate challenger list
challengers = [c for c in challengers if c != run_info.config]
# Add the config to self.rh in order to make HB aware that this
# config/instance was launched
if intent == RunInfoIntent.WAIT:
break
self.rh.add(
config=run_info.config,
cost=10,
time=0.0,
status=StatusType.RUNNING,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
# We have not completed an iteration
self.assertEqual(self.HB.intensifier_instances[0].hb_iters, 0)
# Because n workers is now 2, we expect 2 sh intensifier_instances
self.assertEqual(len(self.HB.intensifier_instances), 2)
# Each of the intensifier_instances should have s equal to 1
# As no iteration has been completed
self.assertEqual(self.HB.intensifier_instances[0].s_max, 1)
self.assertEqual(self.HB.intensifier_instances[0].s, 1)
self.assertEqual(self.HB.intensifier_instances[1].s_max, 1)
self.assertEqual(self.HB.intensifier_instances[1].s, 1)
# First let us check everything makes sense in_HB-SH-0 HB-SH-0
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.max_budget, 5)
self.assertEqual(
len(self.HB.intensifier_instances[0].sh_intensifier.
n_configs_in_stage), 2)
self.assertEqual(
self.HB.intensifier_instances[1].sh_intensifier.initial_budget, 2)
self.assertEqual(
self.HB.intensifier_instances[1].sh_intensifier.max_budget, 5)
self.assertEqual(
len(self.HB.intensifier_instances[1].sh_intensifier.
n_configs_in_stage), 2)
# We are running with:
# + 4 runs for sh instance 0 ('all_budgets': array([2.5, 5. ]), 'n_configs_in_stage': [2.0, 1.0])
# that is, for SH0 we run in stage==0 int(2.5) intensifier_instances * 2.0 configs
# And this times 2 because we have 2_HB intensifier_instances
self.assertEqual(i, 8)
# Adding a new worker is not possible as we already have 2 intensifier_instances
# and n_workers==2
intent, run_info = self.HB.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=self.rh,
num_workers=2,
)
self.assertEqual(intent, RunInfoIntent.WAIT)
def test_add_new_instance(self):
"""Test whether we can add a instance and when we should not"""
# By default we do not create a_HB
# test adding the first instance!
self.assertEqual(len(self.HB.intensifier_instances), 0)
self.assertTrue(self.HB._add_new_instance(num_workers=1))
self.assertEqual(len(self.HB.intensifier_instances), 1)
self.assertIsInstance(self.HB.intensifier_instances[0], _Hyperband)
# A second call should not add a new_HB instance
self.assertFalse(self.HB._add_new_instance(num_workers=1))
# We try with 2_HB instance active
# We effectively return true because we added a new_HB instance
self.assertTrue(self.HB._add_new_instance(num_workers=2))
self.assertEqual(len(self.HB.intensifier_instances), 2)
self.assertIsInstance(self.HB.intensifier_instances[1], _Hyperband)
# Trying to add a third one should return false
self.assertFalse(self.HB._add_new_instance(num_workers=2))
self.assertEqual(len(self.HB.intensifier_instances), 2)
def _exhaust_run_and_get_incumbent(self, sh, rh, num_workers=2):
"""
Runs all provided configs on all intensifier_instances and return the incumbent
as a nice side effect runhistory/stats are properly filled
"""
challengers = [self.config1, self.config2, self.config3, self.config4]
incumbent = None
for i in range(100):
try:
intent, run_info = sh.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=rh,
num_workers=num_workers,
)
except ValueError as e:
# Get configurations until you run out of them
print(e)
break
# Regenerate challenger list
challengers = [c for c in challengers if c != run_info.config]
if intent == RunInfoIntent.WAIT:
break
result = target_from_run_info(run_info)
rh.add(
config=run_info.config,
cost=result.cost,
time=result.time,
status=result.status,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
incumbent, inc_perf = sh.process_results(
run_info=run_info,
incumbent=incumbent,
run_history=rh,
time_bound=100.0,
result=result,
log_traj=False,
)
return incumbent, inc_perf
def test_parallel_same_as_serial_HB(self):
"""Makes sure we behave the same as a serial run at the end"""
# Get the run_history for a_HB instance run:
rh = RunHistory()
stats = Stats(scenario=self.scen)
stats.start_timing()
_HB = _Hyperband(
stats=stats,
traj_logger=TrajLogger(output_dir=None, stats=stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent, inc_perf = self._exhaust_run_and_get_incumbent(
_HB, rh, num_workers=1)
# Just to make sure nothing has changed from the_HB instance side to make
# this check invalid:
# We add config values, so config 3 with 0 and 7 should be the lesser cost
self.assertEqual(incumbent, self.config3)
self.assertEqual(inc_perf, 7.0)
# Do the same for HB, but have multiple_HB instance in there
# This_HB instance will be created via num_workers==2
# in self._exhaust_run_and_get_incumbent
HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent(
HB, self.rh)
self.assertEqual(incumbent, incumbent_phb)
# This makes sure there is a single incumbent in HB
self.assertEqual(inc_perf, inc_perf_phb)
# We don't want to loose any configuration, and particularly
# we want to make sure the values of_HB instance to HB match
self.assertEqual(len(self.rh.data), len(rh.data))
# Because it is a deterministic run, the run histories must be the
# same on exhaustion
self.assertDictEqual(self.rh.data, rh.data)
class SMBO(BaseSolver):
def __init__(self,
scenario,
tae_runner=None,
acquisition_function=None,
model=None,
runhistory2epm=None,
stats=None,
rng=None):
'''
Interface that contains the main Bayesian optimization loop
Parameters
----------
scenario: smac.scenario.scenario.Scenario
Scenario object
tae_runner: object
object that implements the following method to call the target
algorithm (or any other arbitrary function):
run(self, config)
If not set, it will be initialized with the tae.ExecuteTARunOld()
acquisition_function : AcquisitionFunction
Object that implements the AbstractAcquisitionFunction. Will use
EI if not set.
model : object
Model that implements train() and predict(). Will use a
RandomForest if not set.
runhistory2epm : RunHistory2EMP
Object that implements the AbstractRunHistory2EPM. If None,
will use RunHistory2EPM4Cost if objective is cost or
RunHistory2EPM4LogCost if objective is runtime.
stats: Stats
optional stats object
rng: numpy.random.RandomState
Random number generator
'''
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
self.runhistory = RunHistory()
self.logger = logging.getLogger("smbo")
if rng is None:
self.num_run = np.random.randint(1234567980)
self.rng = np.random.RandomState(seed=self.num_run)
elif isinstance(rng, int):
self.num_run = rng
self.rng = np.random.RandomState(seed=rng)
elif isinstance(rng, np.random.RandomState):
self.num_run = rng.randint(1234567980)
self.rng = rng
else:
raise TypeError('Unknown type %s for argument rng. Only accepts '
'None, int or np.random.RandomState' %
str(type(rng)))
self.scenario = scenario
self.config_space = scenario.cs
self.traj_logger = TrajLogger(output_dir=self.scenario.output_dir,
stats=self.stats)
self.types = get_types(self.config_space, scenario.feature_array)
if model is None:
self.model = RandomForestWithInstances(
self.types,
scenario.feature_array,
seed=self.rng.randint(1234567980))
else:
self.model = model
if acquisition_function is None:
self.acquisition_func = EI(self.model)
else:
self.acquisition_func = acquisition_function
self.local_search = LocalSearch(self.acquisition_func,
self.config_space)
self.incumbent = None
if tae_runner is None:
self.executor = ExecuteTARunOld(ta=scenario.ta,
stats=self.stats,
run_obj=scenario.run_obj,
par_factor=scenario.par_factor)
else:
self.executor = tae_runner
self.inten = Intensifier(
executor=self.executor,
stats=self.stats,
traj_logger=self.traj_logger,
instances=self.scenario.train_insts,
cutoff=self.scenario.cutoff,
deterministic=self.scenario.deterministic,
run_obj_time=self.scenario.run_obj == "runtime",
instance_specifics=self.scenario.instance_specific)
num_params = len(self.config_space.get_hyperparameters())
self.objective = average_cost
if self.scenario.run_obj == "runtime":
if runhistory2epm is None:
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log10(self.scenario.cutoff)
threshold = np.log10(self.scenario.cutoff *
self.scenario.par_factor)
imputor = RFRImputator(cs=self.config_space,
rs=self.rng,
cutoff=cutoff,
threshold=threshold,
model=self.model,
change_threshold=0.01,
max_iter=10)
self.rh2EPM = RunHistory2EPM4LogCost(scenario=self.scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.TIMEOUT,
],
imputor=imputor)
else:
self.rh2EPM = runhistory2epm
elif self.scenario.run_obj == 'quality':
if runhistory2epm is None:
self.rh2EPM = RunHistory2EPM4Cost\
(scenario=self.scenario, num_params=num_params,
success_states=[StatusType.SUCCESS, ],
impute_censored_data=False, impute_state=None)
else:
self.rh2EPM = runhistory2epm
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
def run_initial_design(self):
'''
runs algorithm runs for a initial design;
default implementation: running the default configuration on
a random instance-seed pair
Side effect: adds runs to self.runhistory
Returns
-------
incumbent: Configuration()
initial incumbent configuration
'''
default_conf = self.config_space.get_default_configuration()
self.incumbent = default_conf
# add this incumbent right away to have an entry to time point 0
self.traj_logger.add_entry(train_perf=2**31,
incumbent_id=1,
incumbent=self.incumbent)
rand_inst_id = self.rng.randint(0, len(self.scenario.train_insts))
# ignore instance specific values
rand_inst = self.scenario.train_insts[rand_inst_id]
if self.scenario.deterministic:
initial_seed = 0
else:
initial_seed = random.randint(0, MAXINT)
status, cost, runtime, additional_info = self.executor.start(
default_conf,
instance=rand_inst,
cutoff=self.scenario.cutoff,
seed=initial_seed,
instance_specific=self.scenario.instance_specific.get(
rand_inst, "0"))
if status in [StatusType.CRASHED or StatusType.ABORT]:
self.logger.critical("First run crashed -- Abort")
sys.exit(1)
self.runhistory.add(config=default_conf,
cost=cost,
time=runtime,
status=status,
instance_id=rand_inst,
seed=initial_seed,
additional_info=additional_info)
defaul_inst_seeds = set(
self.runhistory.get_runs_for_config(default_conf))
default_perf = self.objective(default_conf, self.runhistory,
defaul_inst_seeds)
self.runhistory.update_cost(default_conf, default_perf)
self.stats.inc_changed += 1 # first incumbent
self.traj_logger.add_entry(train_perf=default_perf,
incumbent_id=self.stats.inc_changed,
incumbent=self.incumbent)
return default_conf
def run(self, max_iters=10):
'''
Runs the Bayesian optimization loop for max_iters iterations
Parameters
----------
max_iters: int
The maximum number of iterations
Returns
----------
incumbent: np.array(1, H)
The best found configuration
'''
self.stats.start_timing()
#self.runhistory = RunHisory()
self.incumbent = self.run_initial_design()
# Main BO loop
iteration = 1
while True:
if self.scenario.shared_model:
pSMAC.read(run_history=self.runhistory,
output_directory=self.scenario.output_dir,
configuration_space=self.config_space,
logger=self.logger)
start_time = time.time()
X, Y = self.rh2EPM.transform(self.runhistory)
self.logger.debug("Search for next configuration")
# get all found configurations sorted according to acq
challengers = self.choose_next(X, Y)
time_spend = time.time() - start_time
logging.debug(
"Time spend to choose next configurations: %.2f sec" %
(time_spend))
self.logger.debug("Intensify")
self.incumbent, inc_perf = self.inten.intensify(
challengers=challengers,
incumbent=self.incumbent,
run_history=self.runhistory,
objective=self.objective,
time_bound=max(0.01, time_spend))
# TODO: Write run history into database
if self.scenario.shared_model:
pSMAC.write(run_history=self.runhistory,
output_directory=self.scenario.output_dir,
num_run=self.num_run)
if iteration == max_iters:
break
iteration += 1
logging.debug(
"Remaining budget: %f (wallclock), %f (ta costs), %f (target runs)"
% (self.stats.get_remaing_time_budget(),
self.stats.get_remaining_ta_budget(),
self.stats.get_remaining_ta_runs()))
if self.stats.is_budget_exhausted():
break
self.stats.print_stats(debug_out=True)
return self.incumbent
def choose_next(self,
X,
Y,
num_interleaved_random=1010,
num_configurations_by_random_search_sorted=1000,
num_configurations_by_local_search=10):
"""Choose next candidate solution with Bayesian optimization.
Parameters
----------
X : (N, D) numpy array
Each row contains a configuration and one set of
instance features.
Y : (N, O) numpy array
The function values for each configuration instance pair.
Returns
-------
list
List of 2020 suggested configurations to evaluate.
"""
self.model.train(X, Y)
if self.runhistory.empty():
incumbent_value = 0.0
elif self.incumbent is None:
# TODO try to calculate an incumbent from the runhistory!
incumbent_value = 0.0
else:
incumbent_value = self.runhistory.get_cost(self.incumbent)
self.acquisition_func.update(model=self.model, eta=incumbent_value)
# Remove dummy acquisition function value
next_configs_by_random_search = [
x[1] for x in self._get_next_by_random_search(
num_points=num_interleaved_random)
]
# Get configurations sorted by EI
next_configs_by_random_search_sorted = \
self._get_next_by_random_search(
num_configurations_by_random_search_sorted, _sorted=True)
next_configs_by_local_search = \
self._get_next_by_local_search(num_configurations_by_local_search)
next_configs_by_acq_value = next_configs_by_random_search_sorted + \
next_configs_by_local_search
next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0])
self.logger.debug(
"First 10 acq func values of selected configurations: %s" %
(str([_[0] for _ in next_configs_by_acq_value[:10]])))
next_configs_by_acq_value = [_[1] for _ in next_configs_by_acq_value]
challengers = list(
itertools.chain(*zip(next_configs_by_acq_value,
next_configs_by_random_search)))
return challengers
def _get_next_by_random_search(self, num_points=1000, _sorted=False):
"""Get candidate solutions via local search.
Parameters
----------
num_points : int, optional (default=10)
Number of local searches and returned values.
_sorted : bool, optional (default=True)
Whether to sort the candidate solutions by acquisition function
value.
Returns
-------
list : (acquisition value, Candidate solutions)
"""
rand_configs = self.config_space.sample_configuration(size=num_points)
if _sorted:
imputed_rand_configs = map(ConfigSpace.util.impute_inactive_values,
rand_configs)
imputed_rand_configs = [
x.get_array() for x in imputed_rand_configs
]
imputed_rand_configs = np.array(imputed_rand_configs,
dtype=np.float64)
acq_values = self.acquisition_func(imputed_rand_configs)
# From here
# http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values
random = self.rng.rand(len(acq_values))
# Last column is primary sort key!
indices = np.lexsort((random.flatten(), acq_values.flatten()))
for i in range(len(rand_configs)):
rand_configs[i].origin = 'Random Search (sorted)'
# Cannot use zip here because the indices array cannot index the
# rand_configs list, because the second is a pure python list
return [(acq_values[ind][0], rand_configs[ind])
for ind in indices[::-1]]
else:
for i in range(len(rand_configs)):
rand_configs[i].origin = 'Random Search'
return [(0, rand_configs[i]) for i in range(len(rand_configs))]
def _get_next_by_local_search(self, num_points=10):
"""Get candidate solutions via local search.
In case acquisition function values tie, these will be broken randomly.
Parameters
----------
num_points : int, optional (default=10)
Number of local searches and returned values.
Returns
-------
list : (acquisition value, Candidate solutions),
ordered by their acquisition function value
"""
configs_acq = []
# Start N local search from different random start points
for i in range(num_points):
if i == 0 and self.incumbent is not None:
start_point = self.incumbent
else:
start_point = self.config_space.sample_configuration()
configuration, acq_val = self.local_search.maximize(start_point)
configuration.origin = 'Local Search'
configs_acq.append((acq_val[0][0], configuration))
# shuffle for random tie-break
random.shuffle(configs_acq, self.rng.rand)
# sort according to acq value
# and return n best configurations
configs_acq.sort(reverse=True, key=lambda x: x[0])
return configs_acq
class TestIntensify(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_race_challenger(self):
"""
test _race_challenger without adaptive capping
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
intensifier.N = 1
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
self.assertEqual(inc, self.config2)
def test_race_challenger_2(self):
"""
test _race_challenger with adaptive capping
"""
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
def test_race_challenger_3(self):
"""
test _race_challenger with adaptive capping on a previously capped configuration
"""
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="runtime",
par_factor=1)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1])
self.rh.add(config=self.config1,
cost=0.5,
time=.5,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.TIMEOUT,
instance_id=2,
seed=12345,
additional_info=None)
# give config2 a second chance - now it should run on both instances
# run on instance 1
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# run on instance 2
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
self.assertEqual(config, self.config2)
self.assertTrue(intensifier.continue_challenger)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2)
self.assertFalse(intensifier.continue_challenger)
def test_race_challenger_large(self):
"""
test _race_challenger using solution_quality
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=True)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=12345,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
def test_race_challenger_large_blocked_seed(self):
"""
test _race_challenger whether seeds are blocked for challenger runs
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_add_inc_run_nondet(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
runhistory=self.rh,
run_obj="solution_quality")
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1,
only_max_observed_budget=True)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_get_next_challenger(self):
"""
test get_next_challenger()
"""
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# get a new challenger to evaluate
config, new = intensifier.get_next_challenger(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertEqual(intensifier.N, 1)
self.assertTrue(new)
# when already evaluating a challenger, return the same challenger
intensifier.to_run = [(1, 1, 0)]
config, new = intensifier.get_next_challenger(
challengers=[self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertFalse(new)
def test_generate_challenger(self):
"""
test generate_challenger()
"""
# test get generator from a list of challengers
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
gen = intensifier._generate_challengers(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(next(gen), self.config1)
self.assertEqual(next(gen), self.config2)
self.assertRaises(StopIteration, next, gen)
# test get generator from a chooser - would return only 1 configuration
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser
gen = intensifier._generate_challengers(challengers=None,
chooser=chooser)
self.assertEqual(next(gen).get_dictionary(), {'a': 24, 'b': 68})
self.assertRaises(StopIteration, next, gen)
# when both are none, raise error
with self.assertRaisesRegex(ValueError,
"No configurations/chooser provided"):
intensifier._generate_challengers(challengers=None, chooser=None)
def test_eval_challenger(self):
"""
test eval_challenger() - a complete intensification run
"""
def target(x):
return x['a']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=False,
always_race_against=self.config3,
run_limit=1)
# run incumbent first if it was not run before
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config1, self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=None,
run_history=self.rh,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
# run challenger now that the incumbent has been executed
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config1, self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# challenger should have a better performance, so incumbent should have changed
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_DEFAULT)
self.assertFalse(intensifier.continue_challenger)
# run `always_race_against` now since the incumbent has changed
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config1, self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)), 1)
self.assertEqual(intensifier.n_iters, 1)
self.assertIsInstance(intensifier.configs_to_run, collections.Iterator)
with self.assertRaises(StopIteration):
next(intensifier.configs_to_run)
def validate(
self,
config_mode: Union[str, typing.List[Configuration]] = 'def',
instance_mode: Union[str, typing.List[str]] = 'test',
repetitions: int = 1,
n_jobs: int = 1,
backend: str = 'threading',
runhistory: typing.Optional[RunHistory] = None,
tae: BaseRunner = None,
output_fn: typing.Optional[str] = None,
) -> RunHistory:
"""
Validate configs on instances and save result in runhistory.
If a runhistory is provided as input it is important that you run it on the same/comparable hardware.
side effect: if output is specified, saves runhistory to specified
output directory.
Parameters
----------
config_mode: str or list<Configuration>
string or directly a list of Configuration.
string from [def, inc, def+inc, wallclock_time, cpu_time, all].
time evaluates at cpu- or wallclock-timesteps of:
[max_time/2^0, max_time/2^1, max_time/2^3, ..., default]
with max_time being the highest recorded time
instance_mode: str or list<str>
what instances to use for validation, either from
[train, test, train+test] or directly a list of instances
repetitions: int
number of repetitions in nondeterministic algorithms
n_jobs: int
number of parallel processes used by joblib
backend: str
what backend joblib should use for parallel runs
runhistory: RunHistory
optional, RunHistory-object to reuse runs
tae: BaseRunner
tae to be used. if None, will initialize ExecuteTARunOld
output_fn: str
path to runhistory to be saved. if the suffix is not '.json', will
be interpreted as directory and filename will be
'validated_runhistory.json'
Returns
-------
runhistory: RunHistory
runhistory with validated runs
"""
self.logger.debug(
"Validating configs '%s' on instances '%s', repeating %d times"
" with %d parallel runs on backend '%s'.", config_mode,
instance_mode, repetitions, n_jobs, backend)
# Get all runs to be evaluated as list
runs, validated_rh = self._get_runs(config_mode, instance_mode,
repetitions, runhistory)
# Create new Stats without limits
inf_scen = Scenario({
'run_obj': self.scen.run_obj,
'cutoff_time':
self.scen.cutoff, # type: ignore[attr-defined] # noqa F821
'output_dir': ""
})
inf_stats = Stats(inf_scen)
inf_stats.start_timing()
# Create TAE
if not tae:
tae = ExecuteTARunOld(
ta=self.scen.ta, # type: ignore[attr-defined] # noqa F821
stats=inf_stats,
run_obj=self.scen.run_obj,
par_factor=self.scen.
par_factor, # type: ignore[attr-defined] # noqa F821
cost_for_crash=self.scen.cost_for_crash
) # type: ignore[attr-defined] # noqa F821
else:
# Inject endless-stats
tae.stats = inf_stats
# Validate!
run_results = self._validate_parallel(tae, runs, n_jobs, backend,
runhistory)
assert len(run_results) == len(runs), (run_results, runs)
# tae returns (status, cost, runtime, additional_info)
# Add runs to RunHistory
for run, result in zip(runs, run_results):
validated_rh.add(config=run.config,
cost=result.cost,
time=result.time,
status=result.status,
instance_id=run.inst,
seed=run.seed,
additional_info=result.additional_info)
self._save_results(validated_rh,
output_fn,
backup_fn="validated_runhistory.json")
return validated_rh
class TestIntensify(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory(aggregate_func=average_cost)
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_compare_configs_no_joint_set(self):
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
'''
challenger is better
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
'''
incumbent is better
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
'''
challenger is better but has less runs;
-> no decision (None)
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_race_challenger(self):
'''
test _race_challenger without adaptive capping
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
self.assertEqual(inc, self.config2)
def test_race_challenger_2(self):
'''
test _race_challenger with adaptive capping
'''
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
def test_race_challenger_3(self):
'''
test _race_challenger with adaptive capping on a previously capped configuration
'''
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="runtime",
par_factor=1)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1])
self.rh.add(config=self.config1,
cost=0.5,
time=.5,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.TIMEOUT,
instance_id=2,
seed=12345,
additional_info=None)
# give config2 a second chance
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.runs_per_config[2], 2)
def test_race_challenger_large(self):
'''
test _race_challenger using solution_quality
'''
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
deterministic=True)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=12345,
additional_info=None)
# tie on first instances and then challenger should always win
# and be returned as inc
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2)
self.assertEqual(len(runs), 10)
def test_race_challenger_large_blocked_seed(self):
'''
test _race_challenger whether seeds are blocked for challenger runs
'''
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
# tie on first instances and then challenger should always win
# and be returned as inc
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
def test_add_inc_run_det(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_add_inc_run_nondet(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_adaptive_capping(self):
'''
test _adapt_cutoff()
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(self.config1)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=self.config1,
instance_seed_pairs=inst_seed_pairs,
run_history=self.rh)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
memory_limit_factor = 2
for entry in trajectory:
incumbent_id = entry.incumbent_id
train_performance = entry.train_perf
if incumbent_id not in incumbent_id_to_model:
config = entry.incumbent
logger = logging.getLogger('Testing:)')
stats = Stats(
Scenario({
'cutoff_time': per_run_time_limit * 2,
'run_obj': 'quality',
})
)
stats.start_timing()
# To avoid the output "first run crashed"...
stats.ta_runs += 1
ta = ExecuteTaFuncWithQueue(backend=automl._automl._backend,
autosklearn_seed=seed,
resampling_strategy='test',
memory_limit=memory_limit_factor * automl_arguments['ml_memory_limit'],
disable_file_output=True,
logger=logger,
stats=stats,
all_scoring_functions=True,
metric=metric)
status, cost, runtime, additional_run_info = ta.start(
config=config, instance=None, cutoff=per_run_time_limit*3)
if status == StatusType.SUCCESS: