Beispiel #1
0
def tag_eptmk(show_stats=True):
    eptmk_untagged = epistemonikos.corpus_load().sents()
    if not os.path.isfile('tagger.pickle'):
        print ("\ntagger.pickle file doesn't exist! Use save_tagger() function"
               " to create it.\n")
    else:
        with open('tagger.pickle', 'rb') as tagger_file:
            print '\nLoaded and unpickled POS tagger.\n'
            trained_tagger = cPickle.load(tagger_file)

        eptmk_tagged = trained_tagger.batch_tag(eptmk_untagged)
        if show_stats is True:
            print 'BACKOFF:'
            pprint_tag_stats(eptmk_tagged, none_tag=None)

        eptmk_tagged = dict_tagger(eptmk_tagged, EPTMK_DICT, none_tag=None)
        if show_stats is True:
            print '+ DICT:'
            pprint_tag_stats(eptmk_tagged, none_tag=None)

        eptmk_tagged = wordnet_tagger(eptmk_tagged, none_tag=None)
        if show_stats is True:
            print '+ WNET:'
            pprint_tag_stats(eptmk_tagged, none_tag=None)

        return eptmk_tagged
Beispiel #2
0
def tag_eptmk(show_stats=True):
    eptmk_untagged = epistemonikos.corpus_load().sents()
    if not os.path.isfile('tagger.pickle'):
        print(
            "\ntagger.pickle file doesn't exist! Use save_tagger() function"
            " to create it.\n")
    else:
        with open('tagger.pickle', 'rb') as tagger_file:
            print '\nLoaded and unpickled POS tagger.\n'
            trained_tagger = cPickle.load(tagger_file)

        eptmk_tagged = trained_tagger.batch_tag(eptmk_untagged)
        if show_stats is True:
            print 'BACKOFF:'
            pprint_tag_stats(eptmk_tagged, none_tag=None)

        eptmk_tagged = dict_tagger(eptmk_tagged, EPTMK_DICT, none_tag=None)
        if show_stats is True:
            print '+ DICT:'
            pprint_tag_stats(eptmk_tagged, none_tag=None)

        eptmk_tagged = wordnet_tagger(eptmk_tagged, none_tag=None)
        if show_stats is True:
            print '+ WNET:'
            pprint_tag_stats(eptmk_tagged, none_tag=None)

        return eptmk_tagged
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections import defaultdict
from corpora import epistemonikos
from nltk.corpus import stopwords

eptmk = epistemonikos.corpus_load().words()
words = [w.lower() for w in eptmk if w.isalpha()]

word_freq = defaultdict(int)
for w in words:
    word_freq[w] += 1
word_freq = [(v, k) for k, v in word_freq.items()]
word_freq = sorted(word_freq, reverse=True)
for w in word_freq[:50]: print w
raw_input('press ENTER')

stoplist = stopwords.words('english')
nonfunc_freq = defaultdict(int)
for w in [x for x in words if x not in stoplist]:
    nonfunc_freq[w] += 1
nonfunc_freq = [(v, k) for k, v in nonfunc_freq.items()]
nonfunc_freq = sorted(nonfunc_freq, reverse=True)
for w in nonfunc_freq[:50]: print w
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections import defaultdict
from corpora import epistemonikos
from nltk.corpus import stopwords

eptmk = epistemonikos.corpus_load().words()
words = [w.lower() for w in eptmk if w.isalpha()]

word_freq = defaultdict(int)
for w in words:
    word_freq[w] += 1
word_freq = [(v, k) for k, v in word_freq.items()]
word_freq = sorted(word_freq, reverse=True)
for w in word_freq[:50]:
    print w
raw_input('press ENTER')

stoplist = stopwords.words('english')
nonfunc_freq = defaultdict(int)
for w in [x for x in words if x not in stoplist]:
    nonfunc_freq[w] += 1
nonfunc_freq = [(v, k) for k, v in nonfunc_freq.items()]
nonfunc_freq = sorted(nonfunc_freq, reverse=True)
for w in nonfunc_freq[:50]:
    print w