Пример #1
0
 def test_pruning_parameters_config(self):
     """Tests that parsing the config is done properly and that the BBOptimizer class can be
     properly instanciated.
     """
     shaman_config = SHAManConfig.from_yaml(PRUNING_CONFIG, "component_1")
     BBOptimizer(black_box=FakeBlackBox,
                 parameter_space=shaman_config.component_parameter_space,
                 async_optim=True,
                 max_step_cost=shaman_config.pruning.max_step_duration,
                 **shaman_config.bbo_parameters)
Пример #2
0
 def test_noise_reduction(self):
     """Tests that the noise reduction parameters are properly parsed.
     """
     shaman_config = SHAManConfig.from_yaml(NOISE_REDUCTION_CONFIG,
                                            "component_1")
     expected_noise_reduction_parameters = {
         "resampling_policy": "simple_resampling",
         "estimator": "numpy.median",
         "nbr_resamples": 3,
         "fitness_aggregation": "simple_fitness_aggregation",
     }
     self.assertDictEqual(shaman_config.noise_reduction,
                          expected_noise_reduction_parameters)
Пример #3
0
 def test_bbo_kwargs(self):
     """Tests that the BBO kwargs are properly parsed."""
     shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
     expected_kwargs = {
         "heuristic": "genetic_algorithm",
         "initial_sample_size": 2,
         "selection_method": tournament_pick,
         "crossover_method": single_point_crossover,
         "mutation_method": mutate_chromosome_to_neighbor,
         "pool_size": 5,
         "mutation_rate": 0.4,
         "elitism": False,
     }
     self.assertDictEqual(shaman_config.bbo_parameters, expected_kwargs)
Пример #4
0
 def test_parameter_space(self):
     """Tests that the parameter space is properly parsed, for both multiplicative
     and additive"""
     shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_1")
     expected_parameter_space = numpy.array([
         numpy.array([1, 2]),
         numpy.array([2, 4, 8]),
         numpy.array([1, 4, 12])
     ],
                                            dtype=object)
     assert_array_equal(expected_parameter_space[0],
                        shaman_config.component_parameter_space[0])
     assert_array_equal(expected_parameter_space[1],
                        shaman_config.component_parameter_space[1])
     assert_array_equal(expected_parameter_space[2],
                        shaman_config.component_parameter_space[2])
Пример #5
0
 def test_noise_reduction_bbo(self):
     """Tests that the noise reduction parameters are properly added to the bbo_kwargs and
     the BBOptimizer object can be initialized.
     """
     shaman_config = SHAManConfig.from_yaml(NOISE_REDUCTION_CONFIG,
                                            "component_1")
     # Check that the dictionary has been properly updated
     assert "nbr_resamples" in shaman_config.bbo_parameters
     assert "fitness_aggregation" in shaman_config.bbo_parameters
     assert "resampling_policy" in shaman_config.bbo_parameters
     # Check that the estimator function is properly parsed
     self.assertEqual(shaman_config.bbo_parameters["estimator"],
                      numpy.median)
     # Check that the BBOptimizer class can be properly instanciated
     BBOptimizer(black_box=FakeBlackBox,
                 parameter_space=shaman_config.component_parameter_space,
                 **shaman_config.bbo_parameters)
Пример #6
0
 def test_buid_configuration(self):
     """Tests that building the configuration worzks as expected.
     Writes down the configuration and then try loading it using the Pydantic model.
     """
     self.shaman_config.build_configuration(self.POST_DATA)
     SHAManConfig.from_yaml(OUTPUT_FILE, "component_1")
Пример #7
0
 def test_empty_pruning(self):
     """Tests that when there is no pruning specified, it returns None."""
     shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
     self.assertEqual(shaman_config.pruning, None)
Пример #8
0
 def test_bbo_init(self):
     """Tests that the BBO kwargs allow to initialize an object of class BBOptimizer"""
     shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
     BBOptimizer(black_box=FakeBlackBox,
                 parameter_space=shaman_config.component_parameter_space,
                 **shaman_config.bbo_parameters)
Пример #9
0
 def test_load_config_vanilla_component_2(self):
     """Tests that loading the config from yaml behaves as expected for component_2."""
     shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
     self.assertEqual(shaman_config.component_parameter_names, ["param_1"])
Пример #10
0
 def test_unknown_component(self):
     """Tests that loading the config from yaml with an unknown component, a key error is raised."""
     with self.assertRaises(KeyError):
         SHAManConfig.from_yaml(VANILLA_CONFIG, "do_not_exist")
Пример #11
0
 def test_wrong_parameter_space(self):
     """Tests that when the parametric space makes no sense, an error is raised."""
     with self.assertRaises(ValueError):
         shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG_WRONG,
                                                "component_1")
Пример #12
0
 def test_empty_noise_reduction(self):
     """Tests that when there is no noise reduction, it returns None."""
     shaman_config = SHAManConfig.from_yaml(VANILLA_CONFIG, "component_2")
     self.assertEqual(shaman_config.noise_reduction, None)
Пример #13
0
    def __init__(
        self,
        component_name: str,
        nbr_iteration: int,
        sbatch_file: str,
        experiment_name: str,
        configuration_file: str,
        sbatch_dir: str = None,
        slurm_dir: str = None,
        result_file: str = None,
    ) -> None:
        """Builds an object of class SHAManExperiment. This experiment is
        defined by giving:

            - The name of the component to tune.
            - The number of allowed iterations for the optimization process.
            - The name of the sbatch file to run.
            - The name of the experiment.
            - A configuration file to setup the experiment.
            - Where the slurm outputs should be stored
                (optional, if not specified, the outputs are deleted)
            - The path to the result file (optional, if not specified,
                no file is created).

        Args:
            component_name (str): The name of the component to use.
            nbr_iteration (int): The number of iterations.
            sbatch_file (str): The path to the sbatch file.
            experiment_name (str): The name of the experiment.
            sbatch_dir (str or Path): The working directory,
                where the shaman sbatch will be stored.
                If not specified, the directory where the command is called.
            slurm_dir (str or Path): The directory where the slurm outputs
                will be stored.
                If set to None, all the slurm outputs are removed after the
                end of the experiment.
            result_file (str): The path to the result file.
                If set to None, no such file is created and the results are
                debuged to the screen.
            configuration_file (str): The path to the configuration file.
                Defaults to the configuration file present in the package
                and called config.cfg.
        """
        # The name of the component that will be tuned through the experiment
        self.component_name = component_name
        # The maximum number of iterations
        self.nbr_iteration = nbr_iteration
        # The name of the sbatch to use, after ensuring its existence
        if Path(sbatch_file).exists():
            self.sbatch_file = sbatch_file
        else:
            raise FileNotFoundError(f"Could not find sbatch {sbatch_file}.")
        # Store information about the experiment
        self.experiment_name = experiment_name
        # Store the sbatch directory and convert to Path if not already
        if sbatch_dir:
            if isinstance(sbatch_dir, Path):
                self.sbatch_dir = sbatch_dir
            else:
                self.sbatch_dir = Path(sbatch_dir)
        else:
            self.sbatch_dir = Path.cwd()
        # Store the slurm directory and convert to Path if not already
        if slurm_dir:
            if isinstance(slurm_dir, Path):
                self.slurm_dir = slurm_dir
            else:
                self.slurm_dir = Path(slurm_dir)
        else:
            self.slurm_dir = None
        self.result_file = result_file
        self.experiment_id = ""

        # Parse the configuration
        self.configuration = SHAManConfig.from_yaml(
            configuration_file, self.component_name
        )
        # Create API client using the configuration information
        self.api_client = Client(base_url=self.api_url, proxies={})
        # Create the black box object using the informations
        self.bb_wrapper = BBWrapper(
            self.component_name,
            self.configuration.component_parameter_names,
            self.sbatch_file,
            sbatch_dir=self.sbatch_dir,
            component_configuration=self.api_url
            + "/"
            + api_settings.component_endpoint,
        )
        logger.debug(self.api_url + "/" + api_settings.component_endpoint)

        # Setup black box optimizer using configuration information
        self.bb_optimizer = self.setup_bb_optimizer()
        # Compute the start of the experiment
        self.experiment_start = \
            datetime.datetime.utcnow().strftime("%y/%m/%d %H:%M:%S")