import codecs
from plp import PLP
from stemmer import Stemmer
__author__ = 'maciej'
plp = PLP()
plp._init()
ile_poprawnych = 0
ile_wszystkich = 0
s = Stemmer(plp, filename='trie.bak', word_type=None)
f = codecs.open('test.txt', 'r', 'utf-8')
for line in f:
ile_wszystkich += 1
parts = line.split(',')
b_form = s.find_basic_form(parts[0])
if b_form.basic_form.strip() == parts[1].strip():
ile_poprawnych += 1
else:
print b_form.basic_form, ';', parts[1], ';', parts[0]
print 'Liczba poprawnie rozpoznanych: ', ile_poprawnych, '\nLiczba niepoprawnie rozpoznanych:', ile_wszystkich - ile_poprawnych
class Test:
"""
Class responsible for running test against cities retrieved by TestPreparer
"""
trie_files = ['trie.bak', 'trie_only_nouns.bak', 'trie_nouns_and_adjectives.bak', 'trie_nouns_and_numerals.bak',
'trie_nouns_adjectives_and_numerals.bak']
# trie_files = ['trie.bak']
def __init__(self):
self.plp = PLP()
self.plp._init()
print 'Initialized plp'
self.cities = TestPreparer().start()
# print 'Loaded cities: ', self.cities.__len__()
def test(self):
print 'Starting analysis'
for trie_name in self.trie_files:
print 'Starting', trie_name
correct_number = 0
all_number = 0
s = Stemmer(self.plp, filename=trie_name, word_type=None)
corrects_file = codecs.open('../wyniki/single_name/wies_miasto_kolonia_osada/success_' + trie_name.replace('bak', 'txt'), 'w', 'utf-8')
result_file = codecs.open('../wyniki/single_name/wies_miasto_kolonia_osada/' + trie_name.replace('bak', 'txt'), 'w', 'utf-8')
result_file.write(u'Dopełniacz;Mianownik;Wynik Stemmera\n')
corrects_file.write(u'Dopełniacz;Mianownik;Wynik Stemmera\n')
# for k, v in self.cities.iteritems():
cities = codecs.open('../data/cities_wies_miasto_kolonia_osada.csv', 'r', 'utf-8')
for city in cities:
k = city.split(';')[1].strip()
v = city.split(';')[0].strip()
all_number += 1
basic_form = ''
# word_labels = []
# if k.__contains__('-'):
# for city_parts in v.split('-'):
# b = s.find_basic_form(city_parts)
# basic_form += b.basic_form + '-'
# word_labels.append(b.word_labels)
# basic_form = basic_form[0:basic_form.__len__() - 1]
# else:
# for city_parts in v.split(' '):
# b = s.find_basic_form(city_parts)
# basic_form += b.basic_form + ' '
# word_labels.append(b.word_labels)
basic_form = s.find_basic_form(v).basic_form.strip()
if basic_form != k:
# if basic_form == k:
result_file.write(v + ';' + k + ';' + basic_form + ';')
# for w_label in word_labels:
# result_file.write(self.find_most_label(w_label) + ' ')
result_file.write('\n')
else:
# corrects_file.write(v + ';' + k + ';' + basic_form + ';')
# for label in s.find_labels(word_labels):
# corrects_file.write(label + ' ')
# corrects_file.write('\n')
correct_number += 1
result_file.write(u'Liczba miejscowości;Liczba niepoprawnie rozpoznanych;Liczba poprawnie rozpoznanych\n')
result_file.write(
str(all_number) + ';' + str(all_number - correct_number) + ';' + str(correct_number))
print 'Done', trie_name
def find_most_label(self, w_label):
max_labels = dict()
for word in w_label:
for id in self.plp.rec(word):
label = self.plp.label(id)
if label in max_labels:
max_labels[label] += 1
else:
max_labels[label] = 1
return max(max_labels.iteritems(), key=operator.itemgetter(1))[0]
def prepare_cities(self):
print 'Preparing cities'
res_file = codecs.open('../data/cities.csv', 'w', 'utf-8')
res_file.write(u'Dopełniacz;Mianownik\n')
for k, v in self.cities.iteritems():
res_file.write(v + ';' + k + '\n')
