def test_pruning_parameters_config(self):
"""Tests that parsing the config is done properly and that the BBOptimizer class can be
properly instanciated.
"""
shaman_config = SHAManConfig.from_yaml(PRUNING_CONFIG, "component_1")
BBOptimizer(black_box=FakeBlackBox,
parameter_space=shaman_config.component_parameter_space,
async_optim=True,
max_step_cost=shaman_config.pruning.max_step_duration,
**shaman_config.bbo_parameters)
def test_noise_reduction(self):
"""Tests that the noise reduction parameters are properly parsed.
"""
shaman_config = SHAManConfig.from_yaml(NOISE_REDUCTION_CONFIG,
"component_1")
expected_noise_reduction_parameters = {
"resampling_policy": "simple_resampling",
"estimator": "numpy.median",
"nbr_resamples": 3,
"fitness_aggregation": "simple_fitness_aggregation",
}
self.assertDictEqual(shaman_config.noise_reduction,
expected_noise_reduction_parameters)
def test_bbo_kwargs(self):
"""Tests that the BBO kwargs are properly parsed."""
shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
expected_kwargs = {
"heuristic": "genetic_algorithm",
"initial_sample_size": 2,
"selection_method": tournament_pick,
"crossover_method": single_point_crossover,
"mutation_method": mutate_chromosome_to_neighbor,
"pool_size": 5,
"mutation_rate": 0.4,
"elitism": False,
}
self.assertDictEqual(shaman_config.bbo_parameters, expected_kwargs)
def test_parameter_space(self):
"""Tests that the parameter space is properly parsed, for both multiplicative
and additive"""
shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_1")
expected_parameter_space = numpy.array([
numpy.array([1, 2]),
numpy.array([2, 4, 8]),
numpy.array([1, 4, 12])
],
dtype=object)
assert_array_equal(expected_parameter_space[0],
shaman_config.component_parameter_space[0])
assert_array_equal(expected_parameter_space[1],
shaman_config.component_parameter_space[1])
assert_array_equal(expected_parameter_space[2],
shaman_config.component_parameter_space[2])
def test_noise_reduction_bbo(self):
"""Tests that the noise reduction parameters are properly added to the bbo_kwargs and
the BBOptimizer object can be initialized.
"""
shaman_config = SHAManConfig.from_yaml(NOISE_REDUCTION_CONFIG,
"component_1")
# Check that the dictionary has been properly updated
assert "nbr_resamples" in shaman_config.bbo_parameters
assert "fitness_aggregation" in shaman_config.bbo_parameters
assert "resampling_policy" in shaman_config.bbo_parameters
# Check that the estimator function is properly parsed
self.assertEqual(shaman_config.bbo_parameters["estimator"],
numpy.median)
# Check that the BBOptimizer class can be properly instanciated
BBOptimizer(black_box=FakeBlackBox,
parameter_space=shaman_config.component_parameter_space,
**shaman_config.bbo_parameters)
def test_buid_configuration(self):
"""Tests that building the configuration worzks as expected.
Writes down the configuration and then try loading it using the Pydantic model.
"""
self.shaman_config.build_configuration(self.POST_DATA)
SHAManConfig.from_yaml(OUTPUT_FILE, "component_1")
def test_empty_pruning(self):
"""Tests that when there is no pruning specified, it returns None."""
shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
self.assertEqual(shaman_config.pruning, None)
def test_bbo_init(self):
"""Tests that the BBO kwargs allow to initialize an object of class BBOptimizer"""
shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
BBOptimizer(black_box=FakeBlackBox,
parameter_space=shaman_config.component_parameter_space,
**shaman_config.bbo_parameters)
def test_load_config_vanilla_component_2(self):
"""Tests that loading the config from yaml behaves as expected for component_2."""
shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
self.assertEqual(shaman_config.component_parameter_names, ["param_1"])
def test_unknown_component(self):
"""Tests that loading the config from yaml with an unknown component, a key error is raised."""
with self.assertRaises(KeyError):
SHAManConfig.from_yaml(VANILLA_CONFIG, "do_not_exist")
def test_wrong_parameter_space(self):
"""Tests that when the parametric space makes no sense, an error is raised."""
with self.assertRaises(ValueError):
shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG_WRONG,
"component_1")
def test_empty_noise_reduction(self):
"""Tests that when there is no noise reduction, it returns None."""
shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
self.assertEqual(shaman_config.noise_reduction, None)
def __init__(
self,
component_name: str,
nbr_iteration: int,
sbatch_file: str,
experiment_name: str,
configuration_file: str,
sbatch_dir: str = None,
slurm_dir: str = None,
result_file: str = None,
) -> None:
"""Builds an object of class SHAManExperiment. This experiment is
defined by giving:
- The name of the component to tune.
- The number of allowed iterations for the optimization process.
- The name of the sbatch file to run.
- The name of the experiment.
- A configuration file to setup the experiment.
- Where the slurm outputs should be stored
(optional, if not specified, the outputs are deleted)
- The path to the result file (optional, if not specified,
no file is created).
Args:
component_name (str): The name of the component to use.
nbr_iteration (int): The number of iterations.
sbatch_file (str): The path to the sbatch file.
experiment_name (str): The name of the experiment.
sbatch_dir (str or Path): The working directory,
where the shaman sbatch will be stored.
If not specified, the directory where the command is called.
slurm_dir (str or Path): The directory where the slurm outputs
will be stored.
If set to None, all the slurm outputs are removed after the
end of the experiment.
result_file (str): The path to the result file.
If set to None, no such file is created and the results are
debuged to the screen.
configuration_file (str): The path to the configuration file.
Defaults to the configuration file present in the package
and called config.cfg.
"""
# The name of the component that will be tuned through the experiment
self.component_name = component_name
# The maximum number of iterations
self.nbr_iteration = nbr_iteration
# The name of the sbatch to use, after ensuring its existence
if Path(sbatch_file).exists():
self.sbatch_file = sbatch_file
else:
raise FileNotFoundError(f"Could not find sbatch {sbatch_file}.")
# Store information about the experiment
self.experiment_name = experiment_name
# Store the sbatch directory and convert to Path if not already
if sbatch_dir:
if isinstance(sbatch_dir, Path):
self.sbatch_dir = sbatch_dir
else:
self.sbatch_dir = Path(sbatch_dir)
else:
self.sbatch_dir = Path.cwd()
# Store the slurm directory and convert to Path if not already
if slurm_dir:
if isinstance(slurm_dir, Path):
self.slurm_dir = slurm_dir
else:
self.slurm_dir = Path(slurm_dir)
else:
self.slurm_dir = None
self.result_file = result_file
self.experiment_id = ""
# Parse the configuration
self.configuration = SHAManConfig.from_yaml(
configuration_file, self.component_name
)
# Create API client using the configuration information
self.api_client = Client(base_url=self.api_url, proxies={})
# Create the black box object using the informations
self.bb_wrapper = BBWrapper(
self.component_name,
self.configuration.component_parameter_names,
self.sbatch_file,
sbatch_dir=self.sbatch_dir,
component_configuration=self.api_url
+ "/"
+ api_settings.component_endpoint,
)
logger.debug(self.api_url + "/" + api_settings.component_endpoint)
# Setup black box optimizer using configuration information
self.bb_optimizer = self.setup_bb_optimizer()
# Compute the start of the experiment
self.experiment_start = \
datetime.datetime.utcnow().strftime("%y/%m/%d %H:%M:%S")