def cross_validate_naive_b_smoothing_sign_test(data, classes, folds, unigrams):
""" Test difference in two cross-validation accuracy sequences, one obtained from a smoothed and the other from an unsmoothed system. """
systemA_accuracies = cross_validate_naiveB(data,
classes,
folds,
smooth=True,
lowercase=False,
unigrams=unigrams,
return_raw=True)
systemB_accuracies = cross_validate_naiveB(data,
classes,
folds,
smooth=False,
lowercase=False,
unigrams=unigrams,
return_raw=True)
return sign_test_lists(systemA_accuracies, systemB_accuracies)
def cross_validate_naive_b_bigrams_sign_test(data, classes, folds, unigrams,
bigrams):
""" Test difference in two cross-validation accuracy sequences, one obtained using unigrams and the other using bigrams. """
systemA_accuracies = cross_validate_naiveB(data,
classes,
folds,
smooth=False,
lowercase=False,
unigrams=unigrams,
return_raw=True)
systemB_accuracies = cross_validate_naiveB(data,
classes,
folds,
smooth=False,
lowercase=False,
bigrams=bigrams,
return_raw=True)
return sign_test_lists(systemA_accuracies, systemB_accuracies)
def test_calculating_p_value_from_experiments(self):
# this is an example case from Siegel and Castellan (1986)
A = [5, 4, 6, 6, 3, 2, 5, 3, 1, 4, 5, 4, 4, 7, 5, 5, 5]
B = [3, 3, 4, 5, 3, 3, 2, 3, 2, 3, 2, 2, 5, 2, 5, 3, 1]
self.assertEqual(0.0574, round(sign_test_lists(A, B), 4))