def _fill_table(self, names, scores): table = self.comparison_table for row, row_name, row_scores in zip(count(), names, scores): for col, col_name, col_scores in zip(range(row), names, scores): if row_scores is None or col_scores is None: continue if self.use_rope and self.rope: p0, rope, p1 = baycomp.two_on_single( row_scores, col_scores, self.rope) if np.isnan(p0) or np.isnan(rope) or np.isnan(p1): self._set_cells_na(table, row, col) continue self._set_cell( table, row, col, f"{p0:.3f}<br/><small>{rope:.3f}</small>", f"p({row_name} > {col_name}) = {p0:.3f}\n" f"p({row_name} = {col_name}) = {rope:.3f}") self._set_cell( table, col, row, f"{p1:.3f}<br/><small>{rope:.3f}</small>", f"p({col_name} > {row_name}) = {p1:.3f}\n" f"p({col_name} = {row_name}) = {rope:.3f}") else: p0, p1 = baycomp.two_on_single(row_scores, col_scores) if np.isnan(p0) or np.isnan(p1): self._set_cells_na(table, row, col) continue self._set_cell(table, row, col, f"{p0:.3f}", f"p({row_name} > {col_name}) = {p0:.3f}") self._set_cell(table, col, row, f"{p1:.3f}", f"p({col_name} > {row_name}) = {p1:.3f}")
def compare_dir(_dir1, _dir2): m1 = get_met_from_dir(_dir1) m2 = get_met_from_dir(_dir2) ropes = [0.01, 0.001, 0.01] mets = ['roc', 'bri', 'pr'] res = {} res_np = {} for met, rope in zip(mets, ropes): #print("met", met) res[met] = baycomp.two_on_single(m1[met], m2[met], rope=rope, runs=50) return res
def bayesian_comparison(scores1, scores2, n_repeats, alpha, rope=0): """ Bayesian comparison between pipelines to assess relative performance. Perform Bayesian analysis to predict the probability that pipe(line)1 outperforms pipe(line)2 based on repeated kfold cross validation results using a correlated t-test. If prob[X] > 1.0 - alpha, then you make the decision that X is better. If no prob's reach this threshold, make no decision about the super(infer)iority of the pipelines relative to each other. Notes ----- See https://baycomp.readthedocs.io/en/latest/functions.html. Parameters ---------- scores1 : array-like List of scores from each repeat of each CV fold for pipe1. scores2 : array-like List of scores from each repeat of each CV fold for pipe2. n_repeats : int Number of repetitions of cross validation. alpha : float Statistical significance level. rope : float The width of the region of practical equivalence. Returns ------- probs Tuple of (prob_1, p_equiv, prob_2) """ scores1 = np.array(scores1).flatten() scores2 = np.array(scores2).flatten() probs = two_on_single(scores1, scores2, rope=rope, runs=n_repeats, names=None, plot=False) if rope == 0: probs = np.array([probs[0], 0, probs[1]]) return probs > (1.0 - alpha), probs
def bayes_wins(a, b, width=0.1, independant=False, score=False): """ Compare a and b using a Bayesian signed-ranks test. Args: a: Ballot representing one candidate (array-like). b: Ballot representing one candidate (array-like). width: the width of the region of practical equivalence. independant: True if the different scores are correlated (e.g. bootstraps or cross-validation scores). score: If True, returns the probability of winning instead of a boolean. """ a, b = np.array(a), np.array(b) if independant: p_a, p_tie, p_b = two_on_multiple(a, b, rope=width) else: p_a, p_tie, p_b = two_on_single(a, b, rope=width) if score: res = p_a else: res = p_a == max([p_a, p_tie, p_b]) return res
# teste de levene para igualdade de variância entre as amostras stat, p = ttest_ind(modelo1, modelo2) print('Comparação de igualdade das médias entre os 2 modelos mais acurados...') print('stat=%.3f, p=%.3f' % (stat, p)) if p > 0.05: print('Provavelmente as médias são iguais') else: print('Provavelmente as médias são diferentes') # In[56]: # teste Bayesiano hierárquico para comparação de modelos print('Teste Bayesiano Hierárquico') print('Probabilidade do modelo1 ser melhor do que o modelo2...') print(baycomp.two_on_single(modelo1, modelo2)) # In[57]: # avaliando a possibilidade de igualdade dos modelos names = ("UCV_nb", "BCV_tree") probs, fig = baycomp.two_on_single(modelo1, modelo2, rope=0.1, plot=True, names=names) print(probs) plt.title('Distribuição da probabilidade de igualdade dos modelos', fontsize=13) fig.show()