def _fill_table(self, names, scores):
table = self.comparison_table
for row, row_name, row_scores in zip(count(), names, scores):
for col, col_name, col_scores in zip(range(row), names, scores):
if row_scores is None or col_scores is None:
continue
if self.use_rope and self.rope:
p0, rope, p1 = baycomp.two_on_single(
row_scores, col_scores, self.rope)
if np.isnan(p0) or np.isnan(rope) or np.isnan(p1):
self._set_cells_na(table, row, col)
continue
self._set_cell(
table, row, col,
f"{p0:.3f}<br/><small>{rope:.3f}</small>",
f"p({row_name} > {col_name}) = {p0:.3f}\n"
f"p({row_name} = {col_name}) = {rope:.3f}")
self._set_cell(
table, col, row,
f"{p1:.3f}<br/><small>{rope:.3f}</small>",
f"p({col_name} > {row_name}) = {p1:.3f}\n"
f"p({col_name} = {row_name}) = {rope:.3f}")
else:
p0, p1 = baycomp.two_on_single(row_scores, col_scores)
if np.isnan(p0) or np.isnan(p1):
self._set_cells_na(table, row, col)
continue
self._set_cell(table, row, col, f"{p0:.3f}",
f"p({row_name} > {col_name}) = {p0:.3f}")
self._set_cell(table, col, row, f"{p1:.3f}",
f"p({col_name} > {row_name}) = {p1:.3f}")
def compare_dir(_dir1, _dir2):
m1 = get_met_from_dir(_dir1)
m2 = get_met_from_dir(_dir2)
ropes = [0.01, 0.001, 0.01]
mets = ['roc', 'bri', 'pr']
res = {}
res_np = {}
for met, rope in zip(mets, ropes):
#print("met", met)
res[met] = baycomp.two_on_single(m1[met], m2[met], rope=rope, runs=50)
return res
def bayesian_comparison(scores1, scores2, n_repeats, alpha, rope=0):
"""
Bayesian comparison between pipelines to assess relative performance.
Perform Bayesian analysis to predict the probability that pipe(line)1
outperforms pipe(line)2 based on repeated kfold cross validation
results using a correlated t-test.
If prob[X] > 1.0 - alpha, then you make the decision that X is better.
If no prob's reach this threshold, make no decision about the
super(infer)iority of the pipelines relative to each other.
Notes
-----
See https://baycomp.readthedocs.io/en/latest/functions.html.
Parameters
----------
scores1 : array-like
List of scores from each repeat of each CV fold for pipe1.
scores2 : array-like
List of scores from each repeat of each CV fold for pipe2.
n_repeats : int
Number of repetitions of cross validation.
alpha : float
Statistical significance level.
rope : float
The width of the region of practical equivalence.
Returns
-------
probs
Tuple of (prob_1, p_equiv, prob_2)
"""
scores1 = np.array(scores1).flatten()
scores2 = np.array(scores2).flatten()
probs = two_on_single(scores1,
scores2,
rope=rope,
runs=n_repeats,
names=None,
plot=False)
if rope == 0:
probs = np.array([probs[0], 0, probs[1]])
return probs > (1.0 - alpha), probs
def bayes_wins(a, b, width=0.1, independant=False, score=False):
""" Compare a and b using a Bayesian signed-ranks test.
Args:
a: Ballot representing one candidate (array-like).
b: Ballot representing one candidate (array-like).
width: the width of the region of practical equivalence.
independant: True if the different scores are correlated (e.g. bootstraps or cross-validation scores).
score: If True, returns the probability of winning instead of a boolean.
"""
a, b = np.array(a), np.array(b)
if independant:
p_a, p_tie, p_b = two_on_multiple(a, b, rope=width)
else:
p_a, p_tie, p_b = two_on_single(a, b, rope=width)
if score:
res = p_a
else:
res = p_a == max([p_a, p_tie, p_b])
return res
# teste de levene para igualdade de variância entre as amostras
stat, p = ttest_ind(modelo1, modelo2)
print('Comparação de igualdade das médias entre os 2 modelos mais acurados...')
print('stat=%.3f, p=%.3f' % (stat, p))
if p > 0.05:
print('Provavelmente as médias são iguais')
else:
print('Provavelmente as médias são diferentes')
# In[56]:
# teste Bayesiano hierárquico para comparação de modelos
print('Teste Bayesiano Hierárquico')
print('Probabilidade do modelo1 ser melhor do que o modelo2...')
print(baycomp.two_on_single(modelo1, modelo2))
# In[57]:
# avaliando a possibilidade de igualdade dos modelos
names = ("UCV_nb", "BCV_tree")
probs, fig = baycomp.two_on_single(modelo1,
modelo2,
rope=0.1,
plot=True,
names=names)
print(probs)
plt.title('Distribuição da probabilidade de igualdade dos modelos',
fontsize=13)
fig.show()