def register_result(self,
config: Configuration,
loss: float,
status: StatusType,
update_model: bool = True,
**kwargs) -> None:
try:
key = str(config)
info, start_time = self.run_infos[key]
end_time = time.time()
del self.run_infos[key]
result = RunValue(loss, end_time - start_time,
SmacStatus(status.value), start_time, end_time,
{})
if update_model:
self._incorporate_run_results(info, result, 10)
except KeyError:
# Configuration was sampled during structure search and not via SMAC
if update_model:
self.runhistory.add(
config=config,
cost=loss,
time=0.0,
status=SmacStatus(status.value),
instance_id=None,
seed=0,
)
def test_process_results(self):
"""
Makes sure that we can process the results of a completed
configuration
"""
intent, run_info = self.intensifier.get_next_run(
challengers=[self.config1, self.config2],
incumbent=None,
run_history=self.rh,
num_workers=1,
chooser=None,
)
result = RunValue(
cost=1,
time=0.5,
status=StatusType.SUCCESS,
starttime=1,
endtime=2,
additional_info=None,
)
self.rh.add(
config=run_info.config, cost=1, time=0.5,
status=StatusType.SUCCESS, instance_id=run_info.instance,
seed=run_info.seed, additional_info=None)
incumbent, inc_perf = self.intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(incumbent, run_info.config)
self.assertEqual(inc_perf, 1)
def run_wrapper(
self,
run_info: RunInfo,
) -> Tuple[RunInfo, RunValue]:
"""
wrapper function for ExecuteTARun.run_wrapper() to cap the target algorithm
runtime if it would run over the total allowed runtime.
Parameters
----------
run_info : RunInfo
Object that contains enough information to execute a configuration run in
isolation.
Returns
-------
RunInfo:
an object containing the configuration launched
RunValue:
Contains information about the status/performance of config
"""
if self.budget_type is None:
if run_info.budget != 0:
raise ValueError(
'If budget_type is None, budget must be.0, but is %f' %
run_info.budget)
else:
if run_info.budget == 0:
run_info = run_info._replace(budget=100)
elif run_info.budget <= 0 or run_info.budget > 100:
raise ValueError(
'Illegal value for budget, must be >0 and <=100, but is %f'
% run_info.budget)
if self.budget_type not in ('subsample', 'iterations', 'mixed'):
raise ValueError(
"Illegal value for budget type, must be one of "
"('subsample', 'iterations', 'mixed'), but is : %s" %
self.budget_type)
remaining_time = self.stats.get_remaing_time_budget()
if remaining_time - 5 < run_info.cutoff:
run_info = run_info._replace(cutoff=int(remaining_time - 5))
if run_info.cutoff < 1.0:
return run_info, RunValue(
status=StatusType.STOP,
cost=self.worst_possible_result,
time=0.0,
additional_info={},
starttime=time.time(),
endtime=time.time(),
)
elif (run_info.cutoff != int(np.ceil(run_info.cutoff))
and not isinstance(run_info.cutoff, int)):
run_info = run_info._replace(cutoff=int(np.ceil(run_info.cutoff)))
return super().run_wrapper(run_info=run_info)
def target_from_run_info(RunInfo):
value_from_config = sum(
[a for a in RunInfo.config.get_dictionary().values()])
return RunValue(cost=value_from_config,
time=0.5,
status=StatusType.SUCCESS,
starttime=time.time(),
endtime=time.time() + 1,
additional_info={})
def ensemble_run_history(request):
run_history = RunHistory()
run_history._add(
RunKey(config_id=3,
instance_id='{"task_id": "breast_cancer"}',
seed=1,
budget=3.0),
RunValue(cost=0.11347517730496459,
time=0.21858787536621094,
status=None,
starttime=time.time(),
endtime=time.time(),
additional_info={
'duration': 0.20323538780212402,
'num_run': 3,
'configuration_origin': 'Random Search'
}),
status=None,
origin=None,
)
run_history._add(
RunKey(config_id=6,
instance_id='{"task_id": "breast_cancer"}',
seed=1,
budget=6.0),
RunValue(cost=2 * 0.11347517730496459,
time=2 * 0.21858787536621094,
status=None,
starttime=time.time(),
endtime=time.time(),
additional_info={
'duration': 0.20323538780212402,
'num_run': 6,
'configuration_origin': 'Random Search'
}),
status=None,
origin=None,
)
return run_history
def test_process_results(self):
"""Ensures that the results are processed by the pertinent intensifer,
based on the source id"""
scheduler = ParallelScheduler(
stats=None,
traj_logger=None,
instances=[1, 2, 3],
rng=np.random.RandomState(12345),
deterministic=True,
)
scheduler.intensifier_instances = {
0: mock.Mock(),
1: mock.Mock(),
2: mock.Mock(),
}
run_info = RunInfo(
config=None,
instance=0,
instance_specific="0",
cutoff=None,
seed=0,
capped=False,
budget=0.0,
source_id=2,
)
result = RunValue(cost=1,
time=0.5,
status=StatusType.SUCCESS,
starttime=1,
endtime=2,
additional_info={})
scheduler.process_results(run_info=run_info,
result=result,
incumbent=None,
run_history=None,
time_bound=None)
self.assertIsNone(
scheduler.intensifier_instances[0].process_results.call_args)
self.assertIsNone(
scheduler.intensifier_instances[1].process_results.call_args)
self.assertEqual(
scheduler.intensifier_instances[2].process_results.call_args[1]
['run_info'], run_info)
def make_dict_run_history_data(data):
run_history_data = dict()
for row in data:
run_key = RunKey(
config_id=row[0][0],
instance_id=row[0][1],
seed=row[0][2],
budget=row[0][3])
run_value = RunValue(
cost=row[1][0],
time=row[1][1],
status=getattr(StatusType, row[1][2]['__enum__'].split(".")[-1]),
starttime=row[1][3],
endtime=row[1][4],
additional_info=row[1][5],
)
run_history_data[run_key] = run_value
return run_history_data
def test_incorporate_run_results_callback(self, process_results_mock):
process_results_mock.return_value = None, None
class TestCallback(IncorporateRunResultCallback):
def __init__(self):
self.num_call = 0
def __call__(self, smbo, run_info, result, time_left) -> None:
self.num_call += 1
self.config = run_info.config
callback = TestCallback()
self.scenario.output_dir = None
smac = SMAC4AC(self.scenario)
smac.register_callback(callback)
self.output_dirs.append(smac.output_dir)
smbo = smac.solver
config = self.scenario.cs.sample_configuration()
run_info = RunInfo(
config=config,
instance=None,
instance_specific=None,
seed=1,
cutoff=None,
capped=False,
budget=0.0,
source_id=0,
)
result = RunValue(1.2345, 2.3456, "status", "starttime", "endtime",
"additional_info")
time_left = 10
smbo._incorporate_run_results(run_info=run_info,
result=result,
time_left=time_left)
self.assertEqual(callback.num_call, 1)
self.assertEqual(callback.config, config)
def run_wrapper(
self,
run_info: RunInfo,
) -> typing.Tuple[RunInfo, RunValue]:
"""Wrapper around run() to exec and check the execution of a given config file
This function encapsulates common handling/processing, so that run() implementation
is simplified.
Parameters
----------
run_info : RunInfo
Object that contains enough information to execute a configuration run in
isolation.
Returns
-------
RunInfo:
an object containing the configuration launched
RunValue:
Contains information about the status/performance of config
"""
start = time.time()
if run_info.cutoff is None and self.run_obj == "runtime":
if self.logger:
self.logger.critical(
"For scenarios optimizing running time "
"(run objective), a cutoff time is required, "
"but not given to this call.")
raise ValueError("For scenarios optimizing running time "
"(run objective), a cutoff time is required, "
"but not given to this call.")
cutoff = None
if run_info.cutoff is not None:
cutoff = int(math.ceil(run_info.cutoff))
try:
status, cost, runtime, additional_info = self.run(
config=run_info.config,
instance=run_info.instance,
cutoff=cutoff,
seed=run_info.seed,
budget=run_info.budget,
instance_specific=run_info.instance_specific)
except Exception as e:
status = StatusType.CRASHED
cost = self.cost_for_crash
runtime = time.time() - start
# Add context information to the error message
exception_traceback = traceback.format_exc()
error_message = repr(e)
additional_info = {
'traceback': exception_traceback,
'error': error_message
}
end = time.time()
if run_info.budget == 0 and status == StatusType.DONOTADVANCE:
raise ValueError(
"Cannot handle DONOTADVANCE state when using intensify or SH/HB on "
"instances.")
# Catch NaN or inf.
if (self.run_obj == 'runtime' and not np.isfinite(runtime)
or self.run_obj == 'quality' and not np.isfinite(cost)):
if self.logger:
self.logger.warning(
"Target Algorithm returned NaN or inf as {}. "
"Algorithm run is treated as CRASHED, cost "
"is set to {} for quality scenarios. "
"(Change value through \"cost_for_crash\""
"-option.)".format(self.run_obj, self.cost_for_crash))
status = StatusType.CRASHED
if self.run_obj == "runtime":
# The following line pleases mypy - we already check for cutoff not being none above,
# prior to calling run. However, mypy assumes that the data type of cutoff
# is still Optional[int]
assert cutoff is not None
if runtime > self.par_factor * cutoff:
self.logger.warning("Returned running time is larger "
"than {0} times the passed cutoff time. "
"Clamping to {0} x cutoff.".format(
self.par_factor))
runtime = cutoff * self.par_factor
status = StatusType.TIMEOUT
if status == StatusType.SUCCESS:
cost = runtime
else:
cost = cutoff * self.par_factor
if status == StatusType.TIMEOUT and run_info.capped:
status = StatusType.CAPPED
else:
if status == StatusType.CRASHED:
cost = self.cost_for_crash
return run_info, RunValue(status=status,
cost=cost,
time=runtime,
additional_info=additional_info,
starttime=start,
endtime=end)
def test_process_results_via_sourceid(self):
"""Makes sure source id is honored when deciding
which_HB instance will consume the result/run_info
"""
# Mock the_HB instance so we can make sure the correct item is passed
for i in range(10):
self.HB.intensifier_instances[i] = mock.Mock()
self.HB.intensifier_instances[i].process_results.return_value = (
self.config1, 0.5)
# make iter false so the mock object is not overwritten
self.HB.intensifier_instances[i].iteration_done = False
# randomly create run_infos and push into HB. Then we will make
# sure they got properly allocated
for i in np.random.choice(list(range(10)), 30):
run_info = RunInfo(
config=self.config1,
instance=0,
instance_specific="0",
cutoff=None,
seed=0,
capped=False,
budget=0.0,
source_id=i,
)
# make sure results aren't messed up via magic variable
# That is we check only the proper_HB instance has this
magic = time.time()
result = RunValue(cost=1,
time=0.5,
status=StatusType.SUCCESS,
starttime=1,
endtime=2,
additional_info=magic)
self.HB.process_results(run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=None,
result=result,
log_traj=False)
# Check the call arguments of each sh instance and make sure
# it is the correct one
# First the expected one
self.assertEqual(
self.HB.intensifier_instances[i].process_results.call_args[1]
['run_info'], run_info)
self.assertEqual(
self.HB.intensifier_instances[i].process_results.call_args[1]
['result'], result)
all_other_run_infos, all_other_results = [], []
for j in range(len(self.HB.intensifier_instances)):
# Skip the expected_HB instance
if i == j:
continue
if self.HB.intensifier_instances[
j].process_results.call_args is None:
all_other_run_infos.append(None)
else:
all_other_run_infos.append(
self.HB.intensifier_instances[j].process_results.
call_args[1]['run_info'])
all_other_results.append(
self.HB.intensifier_instances[j].process_results.
call_args[1]['result'])
self.assertNotIn(run_info, all_other_run_infos)
self.assertNotIn(result, all_other_results)
def run_wrapper(
self,
run_info: RunInfo,
) -> Tuple[RunInfo, RunValue]:
"""
wrapper function for ExecuteTARun.run_wrapper() to cap the target algorithm
runtime if it would run over the total allowed runtime.
Args:
run_info (RunInfo): Object that contains enough information
to execute a configuration run in isolation.
Returns:
RunInfo:
an object containing the configuration launched
RunValue:
Contains information about the status/performance of config
"""
if self.budget_type is None:
if run_info.budget != 0:
raise ValueError(
'If budget_type is None, budget must be.0, but is %f' %
run_info.budget)
else:
if run_info.budget == 0:
# SMAC can return budget zero for intensifiers that don't have a concept
# of budget, for example a simple bayesian optimization intensifier.
# Budget determines how our pipeline trains, which can be via runtime or epochs
epochs_budget = self.pipeline_config.get('epochs', np.inf)
runtime_budget = self.pipeline_config.get('runtime', np.inf)
run_info = run_info._replace(
budget=min(epochs_budget, runtime_budget))
elif run_info.budget <= 0:
raise ValueError(
'Illegal value for budget, must be greater than zero but is %f'
% run_info.budget)
if self.budget_type not in ('epochs', 'runtime'):
raise ValueError(
"Illegal value for budget type, must be one of "
"('epochs', 'runtime'), but is : %s" % self.budget_type)
remaining_time = self.stats.get_remaing_time_budget()
if remaining_time - 5 < run_info.cutoff:
run_info = run_info._replace(cutoff=int(remaining_time - 5))
if run_info.cutoff < 1.0:
return run_info, RunValue(
status=StatusType.STOP,
cost=self.worst_possible_result,
time=0.0,
additional_info={},
starttime=time.time(),
endtime=time.time(),
)
elif (run_info.cutoff != int(np.ceil(run_info.cutoff))
and not isinstance(run_info.cutoff, int)):
run_info = run_info._replace(cutoff=int(np.ceil(run_info.cutoff)))
self.logger.info("Starting to evaluate configuration %s" %
run_info.config.config_id)
run_info, run_value = super().run_wrapper(run_info=run_info)
return run_info, run_value
