def test_runs():
ary = np.random.random((6, 4))
checkerboard_plot(ary,
col_labels=['abc', 'def', 'ghi', 'jkl'],
row_labels=['sample %d' % i for i in range(1, 6)],
cell_colors=['skyblue', 'whitesmoke'],
font_colors=['black', 'black'],
figsize=(5, 5))
def run_mcnemar_test(y_test, model_1_class_predictions,
model_2_class_predictions, model_1_name, model_2_name):
"""
Runs the McNemar test to determine if there is a statistically significant difference in the class predictions.
Writes the results and associated contingency table locally.
:param y_test: y_test series
:param model_1_class_predictions: class predictions from model 1
:param model_2_class_predictions: class predictions from model 2
:param model_1_name: name of the first model
:param model_2_name: name of the second model
"""
results_table = mcnemar_table(y_target=y_test,
y_model1=model_1_class_predictions,
y_model2=model_2_class_predictions)
chi2, p = mcnemar(ary=results_table, corrected=True)
pd.DataFrame({
'chi2': [chi2],
'p': [p]
}).to_csv(os.path.join(f'{model_1_name}_{model_2_name}_mcnemar_test.csv'))
board = checkerboard_plot(
results_table,
figsize=(6, 6),
fmt='%d',
col_labels=[f'{model_2_name} wrong', f'{model_2_name} right'],
row_labels=[f'{model_1_name} wrong', f'{model_1_name} right'])
plt.tight_layout()
plt.savefig(
os.path.join('modeling', 'comparison_files',
f'{model_1_name}_{model_2_name}_mcnemar_test.png'))
plt.clf()
def plot_board(board, file):
board = np.char.mod('%d', board)
board = np.where(board == str(BLACK_PIECES), 'B', board)
board = np.where(board == str(WHITE_PIECES), 'W', board)
board = np.where(board == str(0), '', board)
brd = checkerboard_plot(board)
plt.savefig(f"{file}.svg")
plt.savefig(f"{file}.png")
print('matthews_corr_coef (condel): ',
matthews_corrcoef(true_class_binary, condel_binary))
#################################################################
'''SIFT'''
sift_and_model = mcnemar_table(y_target=np.array(true_class_binary),
y_model1=np.array(model_binary),
y_model2=np.array(sift_binary))
print('model & sift: ', '\n', sift_and_model)
chi2, p = mcnemar(ary=sift_and_model, corrected=True)
print(' chi_squared: ', chi2)
print(' p-value: ', p)
brd = checkerboard_plot(sift_and_model,
figsize=(2, 2),
fmt='%d',
col_labels=['model 2 wrong', 'model 2 right'],
row_labels=['model 1 wrong', 'model 1 right'])
plt.show()
'''PPH2'''
pph2_and_model = mcnemar_table(y_target=np.array(true_class_binary),
y_model1=np.array(model_binary),
y_model2=np.array(pph2_binary))
print('model & pph2: ', '\n', pph2_and_model)
chi2, p = mcnemar(ary=pph2_and_model, corrected=True)
print(' chi_squared: ', chi2)
print(' p-value: ', p)
brd = checkerboard_plot(pph2_and_model,
figsize=(2, 2),