def test_atom_filter(initialized_filter, train_dir, test_dir):
train_corp = TrainingCorpus(train_dir)
test_corp = Corpus(test_dir)
filter = initialized_filter
filter.train(train_corp)
prediction = dict()
for name, mail in test_corp.emails():
result = filter.test(mail)
if result == -1:
continue
elif result > POSITIVITY_THRESHOLD:
prediction[name] = POSITIVE
else:
prediction[name] = NEGATIVE
truth = read_classification_from_file(test_dir + '/' + TRUTHFILE)
conf_matrix = BinaryConfusionMatrix(POSITIVE, NEGATIVE)
conf_matrix.compute_from_dicts(truth, prediction)
matrix_dict = conf_matrix.as_dict()
# For testing purposes
print(matrix_dict)
score = quality_score(matrix_dict['tp'], \
matrix_dict['tn'], \
matrix_dict['fp'], \
matrix_dict['fn'])
return score
def compute_quality_for_corpus(corpus_dir):
path = os.getcwd()
os.chdir(corpus_dir)
truth_dict = read_classification_from_file('!truth.txt')
pred_dict = read_classification_from_file('!prediction.txt')
os.chdir(path)
cm = BinaryConfusionMatrix(pos_tag='SPAM', neg_tag='OK')
cm.compute_from_dicts(truth_dict, pred_dict)
return quality_score(**cm.as_dict())
def compute_quality_for_corpus(corpus_dir): """ Compute quality_score() for predictions in corpus """ matrix = BinaryConfusionMatrix(pos_tag="SPAM", neg_tag="OK") matrix.compute_from_dicts( dict(read_classification_from_file(os.path.join(corpus_dir, "!truth.txt"))), dict(read_classification_from_file(os.path.join(corpus_dir, "!prediction.txt"))) ) return quality_score(**matrix.as_dict())
def compute_quality_for_corpus(corpus_dir):
truth_dict = utils.read_classification_from_file(
os.path.join(corpus_dir, '!truth.txt'))
prediction_dict = utils.read_classification_from_file(
os.path.join(corpus_dir, '!prediction.txt'))
confusion_matrix = BinaryConfusionMatrix(pos_tag, neg_tag)
confusion_matrix.compute_from_dicts(truth_dict, prediction_dict)
conf_dict = confusion_matrix.as_dict()
return quality_score(**conf_dict)
def compute_quality_for_corpus(corpus_dir):
truth_dic = methods.read_classification_from_file(methods.add_slash(corpus_dir)+"!truth.txt")
pred_dic = methods.read_classification_from_file(methods.add_slash(corpus_dir)+"!prediction.txt")
bc1 = BinaryConfusionMatrix('SPAM', 'OK')
bc1.compute_from_dicts(truth_dic, pred_dic)
dict_score = bc1.as_dict()
fn=dict_score['fn']
tn=dict_score['tn']
fp=dict_score['fp']
tp=dict_score['tp']
return quality_score(tp, tn, fp, fn), tp, tn, fp, fn
def compute_quality_for_corpus(corpus_dir):
""" Compute quality_score() for predictions in corpus """
matrix = BinaryConfusionMatrix(pos_tag="SPAM", neg_tag="OK")
matrix.compute_from_dicts(
dict(
read_classification_from_file(
os.path.join(corpus_dir, "!truth.txt"))),
dict(
read_classification_from_file(
os.path.join(corpus_dir, "!prediction.txt"))))
return quality_score(**matrix.as_dict())
def compute_quality_for_corpus(corpus_dir):
truth_clasf = read_classification_from_file(corpus_dir + '/' + TRUTHFILE)
pred_clasf = read_classification_from_file(corpus_dir + '/' + PREDFILE)
conf_matrix = BinaryConfusionMatrix(POSITIVE, NEGATIVE)
conf_matrix.compute_from_dicts(truth_clasf, pred_clasf)
matrix_dict = conf_matrix.as_dict()
# Testing purposes
print(matrix_dict)
score = quality_score(matrix_dict['tp'], \
matrix_dict['tn'], \
matrix_dict['fp'], \
matrix_dict['fn'])
return score
def compute_quality_for_corpus(corpus_dir):
dirs = os.listdir(corpus_dir)
for file in dirs:
if '!' in file:
if file == "!truth.txt":
truth_dict = read_classification_from_file(corpus_dir + '/' +
file)
elif file == "!prediction.txt":
pred_dict = read_classification_from_file(corpus_dir + '/' +
file)
else:
truth_dict = None
pred_dict = None
else:
pass
cm1 = BinaryConfusionMatrix(pos_tag="SPAM", neg_tag="OK")
cm1.compute_from_dicts(truth_dict, pred_dict)
final_dict = cm1.as_dict()
return quality_score(final_dict['tp'], final_dict['tn'], final_dict['fp'],
final_dict['fn'])
def compute_quality_for_corpus(corpus_dir):
'''Return the quality score for tested corpus (with truth and prediction files).'''
from utils import read_classification_from_file as load_as_dict
truth_file = '!truth.txt'
pred_file = '!prediction.txt'
truth_dict = load_as_dict(os.path.join(corpus_dir, truth_file))
pred_dict = load_as_dict(os.path.join(corpus_dir, pred_file))
from confmat import BinaryConfusionMatrix
pos_tag = 'SPAM'
neg_tag = 'OK'
cm = BinaryConfusionMatrix(pos_tag, neg_tag)
cm.compute_from_dicts(truth_dict, pred_dict)
confusion_dict = cm.as_dict()
tp = confusion_dict['tp']
tn = confusion_dict['tn']
fp = confusion_dict['fp']
fn = confusion_dict['fn']
return quality_score(tp, tn, fp, fn)
def setUp(self):
# Prepare fixture
self.cm = BinaryConfusionMatrix(pos_tag=INI_SPAM_TAG,
neg_tag=INI_HAM_TAG)
class BinaryConfusionMatrixTest(unittest.TestCase):
def setUp(self):
# Prepare fixture
self.cm = BinaryConfusionMatrix(pos_tag=INI_SPAM_TAG,
neg_tag=INI_HAM_TAG)
def test_countersAreZero_afterCreation(self):
# Exercise the SUT
cmdict = self.cm.as_dict()
# Assert
self.assertDictEqual(cmdict, {'tp': 0, 'tn': 0, 'fp': 0, 'fn': 0})
def test_updatesTPcorrectly(self):
# Exercise the SUT
self.cm.update(SPAM_TAG, SPAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(), {
'tp': 1,
'tn': 0,
'fp': 0,
'fn': 0
})
def test_updatesTNcorrectly(self):
# Exercise the SUT
self.cm.update(HAM_TAG, HAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(), {
'tp': 0,
'tn': 1,
'fp': 0,
'fn': 0
})
def test_updatesFPcorrectly(self):
# Exercise the SUT
self.cm.update(HAM_TAG, SPAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(), {
'tp': 0,
'tn': 0,
'fp': 1,
'fn': 0
})
def test_updatesFNcorrectly(self):
# Exercise the SUT
self.cm.update(SPAM_TAG, HAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(), {
'tp': 0,
'tn': 0,
'fp': 0,
'fn': 1
})
def test_update_raisesValueError_forWrongTruthValue(self):
# This test may be ignored (deleted).
# It tests an additional feature of the BCF class.
# Assert and exercise the SUT
with self.assertRaises(ValueError):
self.cm.update('a bad value', SPAM_TAG)
def test_update_raisesValueError_forWrongPredictionValue(self):
# This test may be ignored (deleted).
# It tests an additional feature of the BCF class.
# Assert and exercise the SUT
with self.assertRaises(ValueError):
self.cm.update(SPAM_TAG, 'a bad value')
def test_computeFromDicts_allCasesOnce(self):
# Prepare fixture
truth = {1: SPAM_TAG, 2: SPAM_TAG, 3: HAM_TAG, 4: HAM_TAG}
prediction = {1: SPAM_TAG, 2: HAM_TAG, 3: SPAM_TAG, 4: HAM_TAG}
# Excercise the SUT
self.cm.compute_from_dicts(truth, prediction)
# Assert
self.assertDictEqual(self.cm.as_dict(), {
'tp': 1,
'tn': 1,
'fp': 1,
'fn': 1
})
def setUp(self):
# Prepare fixture
self.cm = BinaryConfusionMatrix(pos_tag=SPAM_TAG, neg_tag=HAM_TAG)
class BinaryConfusionMatrixTest(unittest.TestCase):
def setUp(self):
# Prepare fixture
self.cm = BinaryConfusionMatrix(pos_tag=SPAM_TAG, neg_tag=HAM_TAG)
def test_countersAreZero_afterCreation(self):
# Exercise the SUT
cmdict = self.cm.as_dict()
# Assert
self.assertDictEqual(cmdict, {'tp': 0, 'tn': 0, 'fp': 0, 'fn': 0})
def test_updatesTPcorrectly(self):
# Exercise the SUT
self.cm.update(SPAM_TAG, SPAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(),
{'tp': 1, 'tn': 0, 'fp': 0, 'fn': 0})
def test_updatesTNcorrectly(self):
# Exercise the SUT
self.cm.update(HAM_TAG, HAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(),
{'tp': 0, 'tn': 1, 'fp': 0, 'fn': 0})
def test_updatesFPcorrectly(self):
# Exercise the SUT
self.cm.update(HAM_TAG, SPAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(),
{'tp': 0, 'tn': 0, 'fp': 1, 'fn': 0})
def test_updatesFNcorrectly(self):
# Exercise the SUT
self.cm.update(SPAM_TAG, HAM_TAG)
# Assert
self.assertDictEqual(self.cm.as_dict(),
{'tp': 0, 'tn': 0, 'fp': 0, 'fn': 1})
#def test_update_raisesValueError_forWrongTruthValue(self):
# This test may be ignored (deleted).
# It tests an additional feature of the BCF class.
# Assert and exercise the SUT
# with self.assertRaises(ValueError):
# self.cm.update('a bad value', SPAM_TAG)
#def test_update_raisesValueError_forWrongPredictionValue(self):
# This test may be ignored (deleted).
# It tests an additional feature of the BCF class.
# Assert and exercise the SUT
# with self.assertRaises(ValueError):
# self.cm.update(SPAM_TAG, 'a bad value')
def test_computeFromDicts_allCasesOnce(self):
# Prepare fixture
truth = {1: SPAM_TAG,
2: SPAM_TAG,
3: HAM_TAG,
4: HAM_TAG}
prediction = {1: SPAM_TAG,
2: HAM_TAG,
3: SPAM_TAG,
4: HAM_TAG}
# Excercise the SUT
self.cm.compute_from_dicts(truth, prediction)
# Assert
self.assertDictEqual(self.cm.as_dict(),
{'tp': 1, 'tn': 1, 'fp': 1, 'fn': 1})
f.write(part + " " + key + " " +
str(self.classification[part][key]) + "\n")
def convert(file, out):
"""
I used this to convert the truth file for emails I found on internet, where the SPAM or HAM was before file name
:param file:
:param out:
"""
dic = {}
with open(file, 'r', encoding="utf-8") as f:
for line in f.readlines():
key, val = line.split()
dic[val] = key
with open(out, 'w', encoding="utf-8") as f:
for key in dic:
f.write(key + " " + dic[key] + "\n")
if __name__ == "__main__":
# used for testing and debugging
filter = MyFilter()
filter.test("SPAM-data/2/")
confusion_matrix = BinaryConfusionMatrix('SPAM', 'OK')
confusion_matrix.compute_from_dicts(
utils.read_classification_from_file("SPAM-data/2/"),
filter.predictions)
print("Quality: %.2f%%" % (confusion_matrix.quality_score() * 100))
utils.clean_up("SPAM-data/2/") # clean !truth
