def test_validation(self):
with mock.patch.object(TrajLogger,
"read_traj_aclib_format",
return_value=None) as traj_mock:
self.scenario.output_dir = "test"
smbo = SMAC(self.scenario).solver
with mock.patch.object(Validator, "validate",
return_value=None) as validation_mock:
smbo.validate(config_mode='inc',
instance_mode='train+test',
repetitions=1,
use_epm=False,
n_jobs=-1,
backend='threading')
self.assertTrue(validation_mock.called)
with mock.patch.object(Validator,
"validate_epm",
return_value=None) as epm_validation_mock:
smbo.validate(config_mode='inc',
instance_mode='train+test',
repetitions=1,
use_epm=True,
n_jobs=-1,
backend='threading')
self.assertTrue(epm_validation_mock.called)
def validate_incs(self, incs: np.ndarray):
"""
Validation
"""
solver = SMAC(scenario=self.scenario,
tae_runner=self.tae,
rng=self.rng,
run_id=MAXINT,
**self.kwargs)
self.logger.info('*' * 120)
self.logger.info('Validating')
new_rh = solver.validate(config_mode=incs,
instance_mode=self.val_set,
repetitions=1,
use_epm=False,
n_jobs=self.n_optimizers)
return self._get_mean_costs(incs, new_rh)
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 200, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
# We can also validate our results (though this makes a lot more sense with instances)
smac.validate(
config_mode='inc', # We can choose which configurations to evaluate
#instance_mode='train+test', # Defines what instances to validate
repetitions=
100, # Ignored, unless you set "deterministic" to "false" in line 95
n_jobs=1) # How many cores to use in parallel for optimization