def optimize(optimizer, scenario, trajectory=None):
then = time.time()
best_conf = optimizer.optimize()
print(best_conf)
print('training time:', time.time() - then)
traj_logger = TrajLogger(None, Stats(scenario))
trajectory = trajectory or traj_logger.read_traj_aclib_format(
"smac-output/run_1/traj_aclib2.json", scenario.cs)
validator = Validator(scenario, trajectory, rng=np.random.RandomState(42))
# evaluate on test instances and calculate cpu time
then = time.time()
runhis_dev = validator.validate(config_mode="def", instance_mode="test")
runhis_inc = validator.validate(config_mode="inc", instance_mode="test")
print('validating time:', time.time() - then)
default_conf = runhis_dev.ids_config[1]
incumbent_conf = runhis_inc.ids_config[1]
dev_vals = get_instance_costs_for_config(runhis_dev, default_conf)
inc_vals = get_instance_costs_for_config(runhis_inc, incumbent_conf)
# ###### Filter runs for plotting #######
dev_x = []
inc_x = []
for key in set(dev_vals.keys()) & set(inc_vals.keys()):
dev_x.append(dev_vals[key])
inc_x.append(inc_vals[key])
# print(dev_vals)
# print(inc_vals)
print(dev_x)
print(inc_x)
print('PAR10:', np.mean(inc_x), '/', np.mean(dev_x))
max_x = 1000.0
par1er = lambda xx: np.mean([(x / 10 if x == max_x else x) for x in xx])
print('PAR1 :', par1er(inc_x), '/', par1er(dev_x))
to_counter = lambda xx: len([x for x in xx if x == max_x])
print('TOs :', to_counter(inc_x), '/', to_counter(dev_x))
print('wins :', len([i for i in range(len(dev_x)) if dev_x[i] > inc_x[i]]),
'/', len(dev_x))
fig, ax = plt.subplots()
ax.scatter(dev_x, inc_x, marker="x")
ax.set_xlabel("Default Configuration")
ax.set_ylabel("Incumbent Configuration")
lims = [
np.min([ax.get_xlim(), ax.get_ylim()]),
np.max([ax.get_xlim(), ax.get_ylim()])
]
ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
# ax.set_xlim(lims)
# ax.set_ylim(lims)
ax.set_xscale('log')
ax.set_yscale('log')
fig.savefig("fig-smac.png")
def test_parallel_same_as_serial_HB(self):
"""Makes sure we behave the same as a serial run at the end"""
# Get the run_history for a_HB instance run:
rh = RunHistory()
stats = Stats(scenario=self.scen)
stats.start_timing()
_HB = _Hyperband(
stats=stats,
traj_logger=TrajLogger(output_dir=None, stats=stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent, inc_perf = self._exhaust_run_and_get_incumbent(
_HB, rh, num_workers=1)
# Just to make sure nothing has changed from the_HB instance side to make
# this check invalid:
# We add config values, so config 3 with 0 and 7 should be the lesser cost
self.assertEqual(incumbent, self.config3)
self.assertEqual(inc_perf, 7.0)
# Do the same for HB, but have multiple_HB instance in there
# This_HB instance will be created via num_workers==2
# in self._exhaust_run_and_get_incumbent
HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent(
HB, self.rh)
self.assertEqual(incumbent, incumbent_phb)
# This makes sure there is a single incumbent in HB
self.assertEqual(inc_perf, inc_perf_phb)
# We don't want to loose any configuration, and particularly
# we want to make sure the values of_HB instance to HB match
self.assertEqual(len(self.rh.data), len(rh.data))
# Because it is a deterministic run, the run histories must be the
# same on exhaustion
self.assertDictEqual(self.rh.data, rh.data)
def get_trajectory(self, cs):
def alternative_configuration_recovery(config_list: typing.List[str], cs: ConfigurationSpace):
""" Used to recover ints and bools as categoricals or constants from trajectory """
config_dict = {}
for param in config_list:
k,v = param.split("=")
v = v.strip("'")
hp = cs.get_hyperparameter(k)
if isinstance(hp, FloatHyperparameter):
v = float(v)
elif isinstance(hp, IntegerHyperparameter):
v = int(v)
################# DIFFERENCE: ################
elif isinstance(hp, CategoricalHyperparameter) or isinstance(hp, Constant):
if isinstance(hp.default_value, bool):
v = True if v == 'True' else False
elif isinstance(hp.default_value, int):
v = int(v)
elif isinstance(hp.default_value, float):
v = float(v)
else:
v = v
##############################################
config_dict[k] = v
config = Configuration(configuration_space=cs, values=config_dict)
config.origin = "External Trajectory"
return config
TrajLogger._convert_dict_to_config = alternative_configuration_recovery
# Try to find trajectory in "alljson"-format todo instead just convert "old" smac data to new smac data
traj_fn = os.path.join(self.folder, 'traj.json')
if os.path.isfile(traj_fn):
self.logger.debug("Found trajectory file in alljson-format at %s", traj_fn)
return TrajLogger.read_traj_alljson_format(fn=traj_fn, cs=cs)
self.logger.debug("%s not found. Trying to find in subfolders.")
try:
return TrajLogger.read_traj_alljson_format(fn=self.get_glob_file(self.folder, 'traj.json'), cs=cs)
except FileNotFoundError:
self.logger.info("Globbed approach failed. Trying old format.")
old_traj_fn = os.path.join(self.folder, 'traj_aclib2.json')
if os.path.isfile(old_traj_fn):
self.logger.debug("Found trajectory file in aclib2-format (deprecated) at %s", old_traj_fn)
return TrajLogger.read_traj_aclib_format(fn=old_traj_fn, cs=cs)
try:
return TrajLogger.read_traj_aclib_format(fn=self.get_glob_file(self.folder, 'traj_aclib2.json'), cs=cs)
except FileNotFoundError:
raise FileNotFoundError("Neither 'traj.json' nor 'traj_aclib2.json in %s or subdirectories.", self.folder)
traj = TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=cs)
return traj
def test_add_entry(self, mock_stats):
tl = TrajLogger(output_dir='./tmp_test_folder', stats=mock_stats)
test_config = {'param_a': 0.5, 'param_b': 1, 'param_c': 'value'}
mock_stats.ta_time_used = .5
mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time
mock_stats.ta_runs = 1
tl.add_entry(0.9, 1, test_config)
self.assertTrue(os.path.exists('tmp_test_folder/traj_old.csv'))
self.assertTrue(os.path.exists('tmp_test_folder/traj_aclib2.json'))
with open('tmp_test_folder/traj_old.csv') as to:
data = to.read().split('\n')[:-1]
header = data[0].split(',')
self.assertEquals(header[0], '"CPU Time Used"')
self.assertEquals(header[-1], '"Configuration..."')
data = list(map(lambda x: x.split(', '), data[1:]))
frmt_str = '%1.6f'
self.assertEquals(frmt_str % .5, data[0][0])
self.assertEquals(frmt_str % .9, data[0][1])
self.assertEquals(frmt_str % 0.5, data[0][-4])
with open('tmp_test_folder/traj_aclib2.json') as js:
json_dict = json.load(js)
self.assertEquals(json_dict['cpu_time'], .5)
self.assertEquals(json_dict['cost'], 0.9)
self.assertEquals(len(json_dict['incumbent']), 3)
self.assertTrue("param_a='0.5'" in json_dict['incumbent'])
# And finally, test the list that's added to the trajectory class
self.assertEqual(
tl.trajectory[0],
TrajEntry(0.9, 1, {
'param_c': 'value',
'param_b': 1,
'param_a': 0.5
}, 1, 0.5, 1))
# Test named-tuple-access:
self.assertEqual(tl.trajectory[0].train_perf, 0.9)
self.assertEqual(tl.trajectory[0].incumbent_id, 1)
self.assertEqual(tl.trajectory[0].ta_runs, 1)
self.assertEqual(tl.trajectory[0].ta_time_used, 0.5)
self.assertEqual(tl.trajectory[0].wallclock_time, 1)
self.assertEqual(len(tl.trajectory), 1)
def restore_state_after_output_dir(
self,
scen: Scenario,
stats: Stats,
traj_list_aclib: typing.List,
traj_list_old: typing.List,
) -> Configuration:
"""Finish processing files for state-restoration. Trajectory
is read in, but needs to be written to new output-folder. Therefore, the
output-dir is created. This needs to be considered in the SMAC-facade."""
# write trajectory-list
traj_path_aclib = os.path.join(
scen.output_dir,
"traj_aclib2.json") # type: ignore[attr-defined] # noqa F821
traj_path_old = os.path.join(
scen.output_dir,
"traj_old.csv") # type: ignore[attr-defined] # noqa F821
with open(traj_path_aclib, 'w') as traj_fn:
traj_fn.writelines(traj_list_aclib)
with open(traj_path_old, 'w') as traj_fn:
traj_fn.writelines(traj_list_old)
# read trajectory to retrieve incumbent
# TODO replace this with simple traj_path_aclib?
trajectory = TrajLogger.read_traj_aclib_format(
fn=traj_path_aclib,
cs=scen.cs) # type: ignore[attr-defined] # noqa F821
incumbent = trajectory[-1]["incumbent"]
self.logger.debug("Restored incumbent %s from %s", incumbent,
traj_path_aclib)
return incumbent
def main():
# Initialize scenario, using runcount_limit as budget.
orig_scen_dict = {
'algo': 'python cmdline_wrapper.py',
'paramfile': 'param_config_space.pcs',
'run_obj': 'quality',
'runcount_limit': 25,
'deterministic': True,
'output_dir': 'restore_me'
}
original_scenario = Scenario(orig_scen_dict)
smac = SMAC(scenario=original_scenario)
smac.optimize()
print(
"\n########## BUDGET EXHAUSTED! Restoring optimization: ##########\n")
# Now the output is in the folder 'restore_me'
#
# We could simply modify the scenario-object, stored in
# 'smac.solver.scenario' and start optimization again:
#smac.solver.scenario.ta_run_limit = 50
#smac.optimize()
# Or, to show the whole process of recovering a SMAC-run from the output
# directory, create a new scenario with an extended budget:
new_scenario = Scenario(
orig_scen_dict,
cmd_args={
'runcount_limit': 50, # overwrite these args
'output_dir': 'restored'
})
# We load the runhistory, ...
rh_path = os.path.join(original_scenario.output_dir, "runhistory.json")
runhistory = RunHistory(aggregate_func=None)
runhistory.load_json(rh_path, new_scenario.cs)
# ... stats, ...
stats_path = os.path.join(original_scenario.output_dir, "stats.json")
stats = Stats(new_scenario)
stats.load(stats_path)
# ... and trajectory.
traj_path = os.path.join(original_scenario.output_dir, "traj_aclib2.json")
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path,
cs=new_scenario.cs)
incumbent = trajectory[-1]["incumbent"]
# Because we changed the output_dir, we might want to copy the old
# trajectory-file (runhistory and stats will be complete)
new_traj_path = os.path.join(new_scenario.output_dir, "traj_aclib2.json")
shutil.copy(traj_path, new_traj_path)
# Now we can initialize SMAC with the recovered objects and restore the
# state where we left off. By providing stats and a restore_incumbent, SMAC
# automatically detects the intention of restoring a state.
smac = SMAC(scenario=new_scenario,
runhistory=runhistory,
stats=stats,
restore_incumbent=incumbent)
smac.optimize()
def test_add_inc_run_det(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_race_challenger(self):
'''
test _race_challenger without adaptive capping
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
self.assertEqual(inc, self.config2)
def _set_param_space(self, param_space):
self.param_space = param_space
from ConfigSpace.hyperparameters import UniformFloatHyperparameter
from smac.configspace import ConfigurationSpace
from smac.optimizer.objective import average_cost
from smac.runhistory.runhistory import RunHistory
from smac.scenario.scenario import Scenario
from smac.stats.stats import Stats
from smac.utils.io.traj_logging import TrajLogger
self.cs = ConfigurationSpace()
for param in param_space:
if param.type == 'continuous':
var = UniformFloatHyperparameter(param.name, param.low,
param.high)
self.cs.add_hyperparameter(var)
self.runhistory = RunHistory(aggregate_func=average_cost)
self.scenario = Scenario({
'run_obj': 'quality',
'runcount-limit': self.budget,
'cs': self.cs
})
self.stats = Stats(self.scenario)
self.traj_logger = TrajLogger(output_dir=__scratch__, stats=self.stats)
def restore_state(self, args_, scen, root_logger):
# Check for folder and files
rh_path = os.path.join(args_.restore_state, "runhistory.json")
stats_path = os.path.join(args_.restore_state, "stats.json")
traj_path = os.path.join(args_.restore_state, "traj_aclib2.json")
scen_path = os.path.join(args_.restore_state, "scenario.txt")
if not os.path.isdir(args_.restore_state):
raise FileNotFoundError(
"Could not find folder from which to restore.")
# Load runhistory and stats
rh = RunHistory(aggregate_func=None)
rh.load_json(rh_path, scen.cs)
root_logger.debug("Restored runhistory from %s", rh_path)
stats = Stats(scen)
stats.load(stats_path)
root_logger.debug("Restored stats from %s", stats_path)
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path,
cs=scen.cs)
incumbent = trajectory[-1]["incumbent"]
root_logger.debug("Restored incumbent %s from %s", incumbent,
traj_path)
# Copy traj if output_dir of specified scenario-file is different than
# the output_dir of the scenario-file in the folder from which to restore.
if scen.output_dir != InputReader().read_scenario_file(
scen_path)['output_dir']:
new_traj_path = os.path.join(scen.output_dir, "traj_aclib2.json")
shutil.copy(traj_path, new_traj_path)
root_logger.debug("Copied traj %s", rh_path)
return rh, stats, incumbent
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31})
self.config5 = Configuration(self.cs, values={'a': 31, 'b': 33})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=False,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
n_seeds=2,
initial_budget=2,
max_budget=5,
eta=2,
)
def test_compare_configs_no_joint_set(self):
intensifier = Intensifier(
tae_runner=None, stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=None, instances=[1])
for i in range(2):
self.rh.add(config=self.config1, cost=2, time=2,
status=StatusType.SUCCESS, instance_id=1,
seed=i, additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=i, additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=1, additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run
# as it is the first evaluation of this intensifier
self.assertEqual(intensifier.num_run, 0)
self.assertEqual(intensifier.num_chall_run, 0)
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1, cost=i + 1, time=i + 1,
status=StatusType.SUCCESS, instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2, cost=i + 1, time=i + 1,
status=StatusType.SUCCESS, instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1, cost=1, time=2,
status=StatusType.SUCCESS, instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def setUp(self):
base_directory = os.path.split(__file__)[0]
base_directory = os.path.abspath(
os.path.join(base_directory, '..', '..'))
self.current_dir = os.getcwd()
os.chdir(base_directory)
logging.basicConfig()
self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__)
self.logger.setLevel(logging.DEBUG)
self.rng = np.random.RandomState(seed=42)
self.scen_fn = 'test/test_files/validation/scenario.txt'
self.train_insts = ['0', '1', '2']
self.test_insts = ['3', '4', '5']
self.inst_specs = {'0': 'null', '1': 'one', '2': 'two',
'3': 'three', '4': 'four', '5': 'five'}
self.feature_dict = {'0': np.array((1, 2, 3)),
'1': np.array((1, 2, 3)),
'2': np.array((1, 2, 3)),
'3': np.array((1, 2, 3)),
'4': np.array((1, 2, 3)),
'5': np.array((1, 2, 3))}
self.output_rh = 'test/test_files/validation/'
scen = Scenario(self.scen_fn, cmd_options={'run_obj': 'quality'})
self.stats = Stats(scen)
self.trajectory = TrajLogger.read_traj_aclib_format(
fn='test/test_files/validation/test_validation_traj.json', cs=scen.cs)
self.output_dirs = [self.output_rh + 'test']
self.output_files = [self.output_rh + 'validated_runhistory_EPM.json',
self.output_rh + 'validated_runhistory.json']
self.maxDiff = None
def test_get_next_challenger(self):
"""
test get_next_challenger()
"""
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# get a new challenger to evaluate
config, new = intensifier.get_next_challenger(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertEqual(intensifier.N, 1)
self.assertTrue(new)
# when already evaluating a challenger, return the same challenger
intensifier.to_run = [(1, 1, 0)]
config, new = intensifier.get_next_challenger(
challengers=[self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertFalse(new)
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def setUp(self):
logging.basicConfig()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
self.logger.setLevel(logging.DEBUG)
self.rng = np.random.RandomState(seed=42)
self.scen_fn = 'test/test_files/validation/scenario.txt'
self.train_insts = ['0', '1', '2']
self.test_insts = ['3', '4', '5']
self.inst_specs = {
'0': 'null',
'1': 'one',
'2': 'two',
'3': 'three',
'4': 'four',
'5': 'five'
}
self.feature_dict = {
'0': np.array((1, 2, 3)),
'1': np.array((1, 2, 3)),
'2': np.array((1, 2, 3)),
'3': np.array((1, 2, 3)),
'4': np.array((1, 2, 3)),
'5': np.array((1, 2, 3))
}
self.output_rh = 'test/test_files/validation/'
scen = Scenario(self.scen_fn, cmd_args={'run_obj': 'quality'})
self.trajectory = TrajLogger.read_traj_aclib_format(
fn='test/test_files/validation/test_validation_traj.json',
cs=scen.cs)
def setUp(self):
logging.basicConfig()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
self.logger.setLevel(logging.DEBUG)
self.rng = np.random.RandomState(seed=42)
self.scen = Scenario({
'run_obj': 'quality',
'param_file': 'test/test_files/validation/test_validation_pcs.pcs',
'algo': 'python -u test/test_files/example_ta.py'
})
self.train_insts = ['0', '1', '2']
self.test_insts = ['3', '4', '5']
self.scen.instance_specific = {
'0': 'null',
'1': 'one',
'2': 'two',
'3': 'three',
'4': 'four',
'5': 'five'
}
self.output_rh = 'test/test_files/validation/test_validation_rh.json'
self.trajectory = TrajLogger.read_traj_aclib_format(
fn='test/test_files/validation/test_validation_traj.json',
cs=self.scen.cs)
def test_multi_config_design(self):
stats = Stats(scenario=self.scenario)
stats.start_timing()
self.ta.stats = stats
tj = TrajLogger(output_dir=None, stats=stats)
rh = RunHistory(aggregate_func=average_cost)
self.ta.runhistory = rh
rng = np.random.RandomState(seed=12345)
intensifier = Intensifier(tae_runner=self.ta,
stats=stats,
traj_logger=tj,
rng=rng,
instances=[None],
run_obj_time=False)
configs = [
Configuration(configuration_space=self.cs, values={"x1": 4}),
Configuration(configuration_space=self.cs, values={"x1": 2})
]
dc = MultiConfigInitialDesign(tae_runner=self.ta,
scenario=self.scenario,
stats=stats,
traj_logger=tj,
runhistory=rh,
rng=rng,
configs=configs,
intensifier=intensifier,
aggregate_func=average_cost)
inc = dc.run()
self.assertTrue(stats.ta_runs == 2)
self.assertTrue(len(rh.data) == 2)
self.assertTrue(rh.get_cost(inc) == 4)
def get_trajectory(self, cs):
def alternative_configuration_recovery(config_list: typing.List[str],
cs: ConfigurationSpace):
""" Used to recover ints and bools as categoricals or constants from trajectory """
config_dict = {}
for param in config_list:
k, v = param.split("=")
v = v.strip("'")
hp = cs.get_hyperparameter(k)
if isinstance(hp, FloatHyperparameter):
v = float(v)
elif isinstance(hp, IntegerHyperparameter):
v = int(v)
################# DIFFERENCE: ################
elif isinstance(hp, CategoricalHyperparameter) or isinstance(
hp, Constant):
if isinstance(hp.default_value, bool):
v = True if v == 'True' else False
elif isinstance(hp.default_value, int):
v = int(v)
else:
v = v
##############################################
config_dict[k] = v
config = Configuration(configuration_space=cs, values=config_dict)
config.origin = "External Trajectory"
return config
TrajLogger._convert_dict_to_config = alternative_configuration_recovery
traj_fn = os.path.join(self.folder, 'traj_aclib2.json')
traj = TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=cs)
return traj
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs,
values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs,
values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs,
values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs,
values={'a': 29, 'b': 31})
self.scen = Scenario({"cutoff_time": 2, 'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.intensifier = SimpleIntensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1],
)
def test_race_challenger_2(self):
"""
Makes sure that a racing configuration with better performance,
that is capped, doesn't substitute the incumbent.
"""
def target(x):
time.sleep(1.5)
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
)
self.rh.add(
config=self.config1,
cost=0.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None,
)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc, instance, seed, cutoff = intensifier._get_next_racer(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
run_info = RunInfo(
config=self.config2,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=True,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.num_run, 1)
self.assertEqual(intensifier.num_chall_run, 1)
def test_compare_configs_inc(self):
'''
incumbent is better
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def restore_state(scenario: typing.Union[Scenario, ScenarioProperties]):
r"""Read in files for state-restoration: runhistory, stats, trajectory.
:param scenario: Scenario whose state shall be loaded.
:return: (RunHistory, Stats, dict)-tuple
"""
# Check for folder and files
rh_path = os.path.join(scenario.output_dir_for_this_run, 'runhistory.json')
stats_path = os.path.join(scenario.output_dir_for_this_run, 'stats.json')
traj_path_aclib = os.path.join(scenario.output_dir_for_this_run, 'traj_aclib2.json')
if not os.path.isdir(scenario.output_dir_for_this_run):
raise FileNotFoundError('Could not find folder from which to restore.')
# Load runhistory and stats
rh = RunHistory(aggregate_func=None)
rh.load_json(rh_path, scenario.cs)
log.debug('Restored runhistory from %s', rh_path)
stats = Stats(scenario)
stats.load(stats_path)
log.debug('Restored stats from %s', stats_path)
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path_aclib, cs=scenario.cs)
incumbent = trajectory[-1]['incumbent']
log.debug('Restored incumbent %s from %s', incumbent, traj_path_aclib)
return rh, stats, incumbent
def test_race_challenger_2(self):
'''
test _race_challenger with adaptive capping
'''
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
def test_init(self):
scen = Scenario(scenario={'run_obj': 'quality', 'cs': self.cs, 'output_dir': ''})
stats = Stats(scen)
with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, 'tmp_test_folder')
TrajLogger(output_dir=path, stats=stats)
self.assertTrue(os.path.exists(path))
def test_add_inc_run_nondet(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def validate(self, config_mode='inc', instance_mode='train+test',
repetitions=1, n_jobs=-1, backend='threading'):
"""Create validator-object and run validation, using
scenario-information, runhistory from smbo and tae_runner from intensify
Parameters
----------
config_mode: string
what configurations to validate
from [def, inc, def+inc, time, all], time means evaluation at
timesteps 2^-4, 2^-3, 2^-2, 2^-1, 2^0, 2^1, ...
instance_mode: string
what instances to use for validation, from [train, test, train+test]
repetitions: int
number of repetitions in nondeterministic algorithms (in
deterministic will be fixed to 1)
n_jobs: int
number of parallel processes used by joblib
Returns
-------
runhistory: RunHistory
runhistory containing all specified runs
"""
traj_fn = os.path.join(self.scenario.output_dir, "traj_aclib2.json")
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=self.scenario.cs)
new_rh_path = os.path.join(self.scenario.output_dir, "validated_runhistory.json")
validator = Validator(self.scenario, trajectory, new_rh_path, self.rng)
new_rh = validator.validate(config_mode, instance_mode, repetitions, n_jobs,
backend, self.runhistory,
self.intensifier.tae_runner)
return new_rh
