def __testStatistic(self, classifier1: ExperimentPerformance,
classifier2: ExperimentPerformance):
if classifier1.numberOfExperiments(
) != classifier2.numberOfExperiments():
raise StatisticalTestNotApplicable(
"In order to apply a paired test, you need to have the same number of "
"experiments in both algorithms.")
if classifier1.numberOfExperiments() != 10:
raise StatisticalTestNotApplicable(
"In order to apply a 5x2 test, you need to have 10 experiments."
)
numerator = 0
difference = []
for i in range(classifier1.numberOfExperiments()):
difference.append(
classifier1.getErrorRate(i) - classifier2.getErrorRate(i))
numerator += difference[i] * difference[i]
denominator = 0
for i in range(classifier1.numberOfExperiments() // 2):
mean = (difference[2 * i] + difference[2 * i + 1]) / 2
variance = (difference[2 * i] - mean) * (difference[2 * i] - mean) + (difference[2 * i + 1] - mean) \
* (difference[2 * i + 1] - mean)
denominator += variance
denominator *= 2
if denominator == 0:
raise StatisticalTestNotApplicable("Variance is 0.")
return numerator / denominator
def __testStatistic(self, classifier1: ExperimentPerformance, classifier2: ExperimentPerformance):
if classifier1.numberOfExperiments() != classifier2.numberOfExperiments():
raise StatisticalTestNotApplicable("In order to apply a paired test, you need to have the same number of "
"experiments in both algorithms.")
difference = []
total = 0
for i in range(classifier1.numberOfExperiments()):
difference.append(classifier1.getErrorRate(i) - classifier2.getErrorRate(i))
total += difference[i]
mean = total / classifier1.numberOfExperiments()
total = 0
for i in range(classifier1.numberOfExperiments()):
total += (difference[i] - mean) * (difference[i] - mean)
standardDeviation = math.sqrt(total / (classifier1.numberOfExperiments() - 1))
if standardDeviation == 0:
raise StatisticalTestNotApplicable("Variance is 0.")
return math.sqrt(classifier1.numberOfExperiments()) * mean / standardDeviation
def compare(self, classifier1: ExperimentPerformance, classifier2: ExperimentPerformance) -> StatisticalTestResult:
if classifier1.numberOfExperiments() != classifier2.numberOfExperiments():
raise StatisticalTestNotApplicable("In order to apply a paired test, you need to have the same number of "
"experiments in both algorithms.")
plus = 0
minus = 0
for i in range(classifier1.numberOfExperiments()):
if classifier1.getErrorRate(i) < classifier2.getErrorRate(i):
plus = plus + 1
else:
if classifier1.getErrorRate(i) > classifier2.getErrorRate(i):
minus = minus + 1
total = plus + minus
pValue = 0
if total == 0:
raise StatisticalTestNotApplicable("Variance is 0.")
for i in range(plus + 1):
pValue += self.__binomial(total, i) / math.pow(2, total)
return StatisticalTestResult(pValue, False)