def test_paired_permutation(self): """ Testing paired permutation test. """ rng = np.random.RandomState(42) a, b = rng.normal(loc=0, size=100), rng.normal(loc=0, size=100) result = paired_permutation(a, a, rng, 100, self.logger) self.assertGreater(result, 0.9999) result = paired_permutation(a, b, rng, 100, self.logger) self.assertGreater(result, 0.3) a, b = rng.normal(loc=-1, size=100), rng.normal(loc=1, size=100) result = paired_permutation(a, b, rng, 1000, self.logger) self.assertLess(result, 0.001)
def _permutation_test(self, epm_rh, default, incumbent, num_permutations, par=1): if par != 1 and not self.scenario.cutoff: return np.nan cutoff = self.scenario.cutoff def_cost = get_cost_dict_for_config(epm_rh, default, par=par, cutoff=cutoff) inc_cost = get_cost_dict_for_config(epm_rh, incumbent, par=par, cutoff=cutoff) data1, data2 = zip(*[(def_cost[i], inc_cost[i]) for i in def_cost.keys()]) p = paired_permutation(data1, data2, self.rng, num_permutations=num_permutations, logger=self.logger) self.logger.debug( "p-value for def/inc-difference: %f (permutation test " "with %d permutations and par %d)", p, num_permutations, par) return p
def create_performance_table(self, default, incumbent, epm_rh, oracle): """Create table, compare default against incumbent on train-, test- and combined instances. Listing PAR10, PAR1 and timeouts. Distinguishes between train and test, if available.""" self.logger.info("... create performance table") cost_dict_def = get_cost_dict_for_config(epm_rh, default) cost_dict_inc = get_cost_dict_for_config(epm_rh, incumbent) def_par1, inc_par1 = self.get_parX(cost_dict_def, 1), self.get_parX(cost_dict_inc, 1) def_par10, inc_par10 = self.get_parX(cost_dict_def, 10), self.get_parX( cost_dict_inc, 10) ora_par1, ora_par10 = self.get_parX(oracle, 1), self.get_parX(oracle, 10) def_timeouts = get_timeout(epm_rh, default, self.scenario.cutoff) inc_timeouts = get_timeout(epm_rh, incumbent, self.scenario.cutoff) def_timeouts_tuple = self.timeouts_to_tuple(def_timeouts) inc_timeouts_tuple = self.timeouts_to_tuple(inc_timeouts) if self.scenario.cutoff: ora_timeout = self.timeouts_to_tuple( {i: c < self.scenario.cutoff for i, c in oracle.items()}) data1, data2 = zip(*[(int(def_timeouts[i]), int(inc_timeouts[i])) for i in def_timeouts.keys()]) p_value_timeouts = "%.5f" % paired_permutation( data1, data2, self.rng, num_permutations=10000, logger=self.logger) else: ora_timeout = self.timeouts_to_tuple({}) p_value_timeouts = "N/A" # p-values (paired permutation) p_value_par10 = self._permutation_test(epm_rh, default, incumbent, 10000, 10) p_value_par10 = "%.5f" % p_value_par10 if np.isfinite( p_value_par10) else 'N/A' p_value_par1 = self._permutation_test(epm_rh, default, incumbent, 10000, 1) p_value_par1 = "%.5f" % p_value_par1 if np.isfinite( p_value_par1) else 'N/A' dec_place = 3 metrics = [] if self.scenario.run_obj == 'runtime': metrics.append('PAR10') metrics.append('PAR1') else: metrics.append('Quality') if self.scenario.cutoff: metrics.append('Timeouts') train, test = len(self.scenario.train_insts) > 1, len( self.scenario.test_insts) > 1 oracle = train or test # oracle only makes sense with instances # Create table array = [] if 'PAR10' in metrics: if train and test: values = [ def_par10[0], inc_par10[0], ora_par10[0], def_par10[1], inc_par10[1], ora_par10[1] ] elif oracle: values = [def_par10, inc_par10, ora_par10] # oracle only with instances else: values = [def_par10, inc_par10] values = [ round(value, dec_place) if np.isfinite(value) else 'N/A' for value in values ] if train or test: values.append(p_value_par10) array.append(values) if 'PAR1' in metrics or 'Quality' in metrics: if train and test: values = [ def_par1[0], inc_par1[0], ora_par1[0], def_par1[1], inc_par1[1], ora_par1[1] ] elif oracle: values = [def_par1, inc_par1, ora_par1] # oracle only with instances else: values = [def_par1, inc_par1] values = [ round(value, dec_place) if np.isfinite(value) else 'N/A' for value in values ] if train or test: values.append(p_value_par1) array.append(values) if 'Timeouts' in metrics: if train and test: values = [ "{}/{}".format(def_timeouts_tuple[0][0], def_timeouts_tuple[0][1]), "{}/{}".format(inc_timeouts_tuple[0][0], inc_timeouts_tuple[0][1]), "{}/{}".format(ora_timeout[0][0], ora_timeout[0][1]), "{}/{}".format(def_timeouts_tuple[1][0], def_timeouts_tuple[1][1]), "{}/{}".format(inc_timeouts_tuple[1][0], inc_timeouts_tuple[1][1]), "{}/{}".format(ora_timeout[1][0], ora_timeout[1][1]), ] elif oracle: values = [ "{}/{}".format(def_timeouts_tuple[0], def_timeouts_tuple[1]), "{}/{}".format(inc_timeouts_tuple[0], inc_timeouts_tuple[1]), "{}/{}".format(ora_timeout[0], ora_timeout[1]) ] else: values = [ "{}/{}".format(def_timeouts_tuple[0], def_timeouts_tuple[1]), "{}/{}".format(inc_timeouts_tuple[0], inc_timeouts_tuple[1]), ] if train or test: values.append(p_value_timeouts) array.append(values) array = np.array(array) columns = ['Default', 'Incumbent'] if oracle: columns.append('Oracle') if train and test: columns = columns + columns if train or test: columns.append('p-value') self.logger.debug(array) self.logger.debug(columns) df = DataFrame(data=array, index=metrics, columns=columns) table = df.to_html() if train and test: # Insert two-column-header table = table.split(sep='</thead>', maxsplit=1)[1] new_table = "<table border=\"3\" class=\"dataframe\">\n"\ " <col>\n"\ " <colgroup span=\"2\"></colgroup>\n"\ " <colgroup span=\"2\"></colgroup>\n"\ " <thead>\n"\ " <tr>\n"\ " <td rowspan=\"2\"></td>\n"\ " <th colspan=\"3\" scope=\"colgroup\">Train</th>\n"\ " <th colspan=\"3\" scope=\"colgroup\">Test</th>\n"\ " <th colspan=\"1\" scope=\"colgroup\">p-value</th>\n"\ " </tr>\n"\ " <tr>\n"\ " <th scope=\"col\">Default</th>\n"\ " <th scope=\"col\">Incumbent</th>\n"\ " <th scope=\"col\">Oracle</th>\n"\ " <th scope=\"col\">Default</th>\n"\ " <th scope=\"col\">Incumbent</th>\n"\ " <th scope=\"col\">Oracle</th>\n"\ " </tr>\n"\ "</thead>\n" table = new_table + table self.table = table self.dataframe = df return df