def put_default_configuration(self, default_configuration: Configuration): if self._is_valid_configuration_instance(default_configuration): if not self.default_configuration: self.default_configuration = default_configuration self.api.send("default", "configuration", configurations=[default_configuration.get_parameters()], results=[default_configuration.get_average_result()]) if default_configuration not in self.all_configurations: self.all_configurations.append(default_configuration) self._calculate_current_best_configurations() else: raise ValueError("The default Configuration was registered already.")
def _add_configuration_to_experiment(self, configuration: Configuration) -> None: """ Save configuration after passing all checks. This method also sends an update to API (front-end). :param configuration: Configuration object. :return: None """ self.all_configurations.append(configuration) self.api.send("new", "configuration", configurations=[configuration.get_parameters()], results=[configuration.get_average_result()]) self.logger.info("Adding to Experiment: %s" % configuration)
def compute_avg_results_over_configurations( configurations: List[Configuration]) -> List[float]: # Make sure, that all Configurations are the same points in a search space: assert all([ config.parameters == configurations[0].parameters for config in configurations ]) tasks = BRISEBenchmarkAnalyser.collect_tasks_from_configurations( configurations) previous_task_configuration = Configuration.TaskConfiguration Configuration.set_task_config(configurations[0].TaskConfiguration) tmp_configuration = Configuration(configurations[0].parameters, Configuration.Type.TEST) tmp_configuration.add_tasks(task=tasks) result = tmp_configuration.get_average_result() Configuration.set_task_config(previous_task_configuration) return result
def evaluate(self, current_configuration: Configuration, experiment: Experiment): """ Return number of measurements to finish Configuration or 0 if it finished. In other case - compute result as average between all experiments. :param current_configuration: instance of Configuration class :param experiment: instance of 'experiment' is required for model-awareness. :return: int min_tasks_per_configuration if Configuration was not measured at all or 1 if Configuration was not measured precisely or 0 if it finished """ tasks_data = current_configuration.get_tasks() if len(tasks_data) < self.min_tasks_per_configuration: return self.min_tasks_per_configuration - len(tasks_data) elif len(tasks_data) >= self.max_tasks_per_configuration: return 0 else: average_result = current_configuration.get_average_result() # Calculating standard deviation all_dim_std = current_configuration.get_standard_deviation() # The number of Degrees of Freedom generally equals the number of observations (Tasks) minus # the number of estimated parameters. degrees_of_freedom = len(tasks_data) - len(average_result) # Calculate the critical t-student value from the t distribution student_coefficients = [ t.ppf(c_l, df=degrees_of_freedom) for c_l in self.confidence_levels ] # Calculating confidence interval for each dimension, that contains a confidence intervals for # singular measurements and confidence intervals for multiple measurements. # First - singular measurements errors: conf_intervals_sm = [] for c_l, d_s_a, d_a_c, avg in zip(self.confidence_levels, self.device_scale_accuracies, self.device_accuracy_classes, average_result): d = sqrt((c_l * d_s_a / 2)**2 + (d_a_c * avg / 100)**2) conf_intervals_sm.append(c_l * d) # Calculation of confidence interval for multiple measurements: conf_intervals_mm = [] for student_coefficient, dim_skd in zip(student_coefficients, all_dim_std): conf_intervals_mm.append(student_coefficient * dim_skd / sqrt(len(tasks_data))) # confidence interval, or in other words absolute error absolute_errors = [] for c_i_ss, c_i_mm in zip(conf_intervals_sm, conf_intervals_mm): absolute_errors.append(sqrt(pow(c_i_ss, 2) + pow(c_i_mm, 2))) # Calculating relative error for each dimension relative_errors = [] for interval, avg_res in zip(absolute_errors, average_result): if not avg_res: # it is 0 or 0.0 # if new use-cases appear with the same behaviour. if interval == 0: avg_res = 1 # Anyway relative error will be 0 and avg will not be changed. else: return 1 relative_errors.append(interval / avg_res * 100) # Thresholds for relative errors that should not be exceeded for accurate measurement. thresholds = [] if self.is_model_aware: # We adapt thresholds current_solution = experiment.get_current_solution( ).get_average_result() minimization_experiment = experiment.is_minimization() for b_t, max_t, r_max, avg_res, cur_solution_avg in \ zip(self.base_acceptable_errors, self.max_acceptable_errors, self.ratios_max, average_result, current_solution): if minimization_experiment: if not cur_solution_avg: ratio = 1 else: ratio = avg_res / cur_solution_avg else: if not avg_res: ratio = 1 else: ratio = cur_solution_avg / avg_res adopted_threshold = b_t + (max_t - b_t) / ( 1 + exp(-1 * (ratio - r_max / 2))) thresholds.append(adopted_threshold) else: # Or we don't adapt thresholds for acceptable_error in self.base_acceptable_errors: thresholds.append(acceptable_error) # Simple implementation of possible multi-dim Repeater decision making: # If any of resulting dimensions are not accurate - just terminate. for threshold, error in zip(thresholds, relative_errors): if error > threshold: return 1 return 0