if __name__ == '__main__':
opts = docopt(__doc__)
# load the model
print("\nLoading the model...")
filename = opts['-i']
f = open(filename, 'rb')
model = pickle.load(f)
f.close()
print("Model type: %s" % type(model))
# load the data
print("Loading corpus data...")
files = '3LB-CAST/.*\.tbf\.xml'
corpus = SimpleAncoraCorpusReader('corpus/ancora-2.0/', files)
sents = list(corpus.tagged_sents())
# compute statistics
print("Computing results...")
# Compute Accuracy
# Global accuracy of the model (percentage of right tagging)
acc, hits, total = 0.0, 0, 0
# Accuracy over known(k) and unknowns(u) words for the model
hits_k, total_k, hits_u, total_u = 0, 0, 0, 0
y_true, y_pred = [], []
# Data for Confusion Matrix
tagset = set()
for t_sent in sents:
for _, tag in t_sent:
tagset.add(tag)
if __name__ == '__main__':
opts = docopt(__doc__)
# load the model
filename = opts['-i']
f = open(filename, 'rb')
model = pickle.load(f)
f.close()
# Load the data
files = '3LB-CAST/.*\.tbf\.xml'
PATH = "./../../ancora-3.0.1es"
corpus = SimpleAncoraCorpusReader(PATH, files)
sents = list(corpus.tagged_sents())
# Tag
hits = 0
total = 0
# Hits Palabras conocidas
hits_known_word = 0
total_known_word = 0
# Hits Palabras desconocidas
hits_unknown_word = 0
total_unknown_word = 0
# Para Matriz de Confusion
tags_gold = [] # Tags correctos
def evaluate(model=None, matrix='n'):
'''
model -- The model trained that has been evaluated
matrix -- If you want to generate the confusion matrix ('y') or not ('n')
'''
start = time()
if model is None:
opts = docopt(__doc__)
matrix = opts['-m'] == 'y'
# load the model
filename = opts['-i']
filename = 'Models/' + filename
f = open(filename, 'rb')
model = pickle.load(f)
f.close()
# load the data
files = '3LB-CAST/.*\.tbf\.xml'
actual_dir = os.path.dirname(os.path.abspath(__file__))
corpus = SimpleAncoraCorpusReader(actual_dir + '/corpus/ancora/', files)
sents = list(corpus.tagged_sents())
n = len(sents)
# tag
hits, total = 0, 0
hits_known, hits_unknown = 0, 0
total_known, total_unknown = 0, 0
are_known = []
# confusion matrix
test = []
prediction = []
for i, sent in enumerate(sents):
word_sent, gold_tag_sent = zip(*sent)
model_tag_sent = model.tag(word_sent).tolist()
assert len(model_tag_sent) == len(gold_tag_sent), i
# For confusion matrix
test += list(gold_tag_sent)
prediction += model_tag_sent
# global score
hits_sent = [m == g for m, g in zip(model_tag_sent, gold_tag_sent)]
hits += sum(hits_sent)
total += len(sent)
total_acc = float(hits) / total
# known words score
for j in range(len(hits_sent)):
# using the Counter method, descripted later, we have to asign
# some values if are known or unknown and if are hit or not.
if not model.unknown(word_sent[j]):
are_known += [hits_sent[j] + 1]
else:
are_known += [hits_sent[j] - 2]
progress('{:3.1f}% (Total: {:2.2f}%)'.format(
float(i) * 100 / n, total_acc * 100))
# For eficiency we will use the Counter object from collections
# library.
# We redefine some things to look for them later
known = 2
fail_known = 1
unknown = -1
fail_unknown = -2
# Counter creates a dictionary whose keys are known, fail_known, unknown
# and fail_unknown.
counter = Counter(are_known)
# Now get the values that represent how many times does apears each one
hits_known += counter[known]
total_known += counter[known] + counter[fail_known]
hits_unknown += counter[unknown]
total_unknown += counter[unknown] + counter[fail_unknown]
# Compute accuracy
total_acc = float(hits) / total
known_acc = float(hits_known) / total_known
unknown_acc = float(hits_unknown) / total_unknown
finish = time() - start
print('')
print('Total accuracy: {:2.2f}%'.format(total_acc * 100))
print('Known accuracy: {:2.2f}%'.format(known_acc * 100))
print('Unknown accuracy: {:2.2f}%'.format(unknown_acc * 100))
print('Time running: {:2.2f}seconds'.format(finish))
if matrix:
matrix = confusion_matrix(test, prediction)
classes = list(set(test) | set(prediction))
classes.sort()
plot_confusion_matrix(matrix, classes, filename.split('.')[0] + '.png')
