def evaluate(self, current_configuration: Configuration, experiment: Experiment): """ Return max_tasks_per_configuration to measure default Configuration or 0. :param current_configuration: instance of Configuration class. :param experiment: instance of 'experiment' is required for model-awareness. :return: max_tasks_per_configuration or 0 """ if len(current_configuration.get_tasks() ) < self.max_tasks_per_configuration: return self.max_tasks_per_configuration - len( current_configuration.get_tasks()) else: return 0
def evaluate(self, current_configuration: Configuration, experiment: Experiment): """ Return max_tasks_per_configuration to measure default Configuration or 0. :param current_configuration: instance of Configuration class. :param experiment: instance of 'experiment' required by the abstract class. Not used in this strategy. :return: max_tasks_per_configuration or 0 """ if len(current_configuration.get_tasks() ) < self.max_tasks_per_configuration: return self.max_tasks_per_configuration - len( current_configuration.get_tasks()) else: return 0
def evaluate(self, current_configuration: Configuration, experiment: Experiment): """ Return number of measurements to finish Configuration or 0 if it finished. In other case - compute result as average between all experiments. :param current_configuration: instance of Configuration class :param experiment: instance of 'experiment' is required for experiment-awareness. :return: int min_tasks_per_configuration if Configuration was not measured at all or 1 if Configuration was not measured precisely or 0 if it finished """ tasks_data = current_configuration.get_tasks() if len(tasks_data) == 0: return 1 c_c_results = current_configuration.results c_s_results = experiment.get_current_solution().results c_c_results_l = [] c_s_results_l = [] for key in experiment.get_objectives(): c_c_results_l.append(c_c_results[key]) c_s_results_l.append(c_s_results[key]) if len(tasks_data) < self.min_tasks_per_configuration: if self.is_experiment_aware: ratios = [ cur_config_dim / cur_solution_dim for cur_config_dim, cur_solution_dim in zip( c_c_results_l, c_s_results_l) ] if all([ ratio >= ratio_max for ratio, ratio_max in zip(ratios, self.ratios_max) ]): return 0 return self.min_tasks_per_configuration - len(tasks_data) elif len(tasks_data) >= self.max_tasks_per_configuration: return 0 else: # Calculating standard deviation all_dim_std = current_configuration.get_standard_deviation() # The number of Degrees of Freedom generally equals the number of observations (Tasks) minus # the number of estimated parameters. degrees_of_freedom = len(tasks_data) - len(c_c_results_l) # Calculate the critical t-student value from the t distribution student_coefficients = [ t.ppf(c_l, df=degrees_of_freedom) for c_l in self.confidence_levels ] # Calculating confidence interval for each dimension, that contains a confidence intervals for # singular measurements and confidence intervals for multiple measurements. # First - singular measurements errors: conf_intervals_sm = [] for c_l, d_s_a, d_a_c, avg in zip(self.confidence_levels, self.device_scale_accuracies, self.device_accuracy_classes, c_c_results_l): d = sqrt((c_l * d_s_a / 2)**2 + (d_a_c * avg / 100)**2) conf_intervals_sm.append(c_l * d) # Calculation of confidence interval for multiple measurements: conf_intervals_mm = [] for student_coefficient, dim_skd in zip(student_coefficients, all_dim_std): conf_intervals_mm.append(student_coefficient * dim_skd / sqrt(len(tasks_data))) # confidence interval, or in other words absolute error absolute_errors = [] for c_i_ss, c_i_mm in zip(conf_intervals_sm, conf_intervals_mm): absolute_errors.append(sqrt(pow(c_i_ss, 2) + pow(c_i_mm, 2))) # Calculating relative error for each dimension relative_errors = [] for interval, avg_res in zip(absolute_errors, c_c_results_l): if not avg_res: # it is 0 or 0.0 # if new use-cases appear with the same behaviour. if interval == 0: avg_res = 1 # Anyway relative error will be 0 and avg will not be changed. else: return 1 relative_errors.append(interval / avg_res * 100) # Thresholds for relative errors that should not be exceeded for accurate measurement. thresholds = [] if self.is_experiment_aware: # We adapt thresholds objectives_minimization = experiment.get_objectives_minimization( ) for i in range(len(objectives_minimization)): if objectives_minimization[i]: if not c_s_results_l[i]: ratio = 1 else: ratio = c_c_results_l[i] / c_s_results_l[i] else: if not c_c_results_l[i]: ratio = 1 else: ratio = c_s_results_l[i] / c_c_results_l[i] adopted_threshold = \ self.base_acceptable_errors[i] \ + (self.max_acceptable_errors[i] - self.base_acceptable_errors[i]) \ / (1 + exp(- (10 / self.ratios_max[i]) * (ratio - self.ratios_max[i] / 2))) thresholds.append(adopted_threshold) else: # Or we don't adapt thresholds for acceptable_error in self.base_acceptable_errors: thresholds.append(acceptable_error) # Simple implementation of possible multi-dim Repeater decision making: # If any of resulting dimensions are not accurate - just terminate. for threshold, error in zip(thresholds, relative_errors): if error > threshold: return 1 return 0