def randomization_test(self, labels, y1, y2, epoch=1000):
import random
e = Evaluate(labels, y1, self.num_of_cls)
f_1 = e.calc_micro_metrics()['f1']
e = Evaluate(labels, y2, self.num_of_cls)
f_2 = e.calc_micro_metrics()['f1']
s = abs(f_1-f_2)
cnt = 0
for i in range(0, epoch):
temp_y1 = []
temp_y2 = []
for idx in range(len(labels)):
if random.uniform(0, 1) > 0.5:
temp_y1.append(y2[idx])
temp_y2.append(y1[idx])
else:
temp_y1.append(y1[idx])
temp_y2.append(y2[idx])
e = Evaluate(labels, temp_y1, self.num_of_cls)
f_1 = e.calc_micro_metrics()['f1']
e = Evaluate(labels, temp_y2, self.num_of_cls)
f_2 = e.calc_micro_metrics()['f1']
s_prime = abs(f_1-f_2)
if s_prime > s:
cnt += 1
p_value = (cnt+1)/(epoch+1)
return p_value
def test(self, data, labels):
predictions = []
cnt_correct = 0
for idx, doc in enumerate(data):
_, prediction = self.predict(doc)
predictions.append(prediction)
true_label = labels[idx]
if true_label == prediction:
cnt_correct += 1
eval = Evaluate(predictions, labels, self.num_of_cls)
metrics = {'macro':eval.calc_macro_metrics(), 'micro':eval.calc_micro_metrics()}
return predictions, metrics