def train_tagger():
"""
This function trains the tagger
"""
print("Training POS tagger...")
# https://github.com/japerk/nltk3-cookbook/blob/master/chapter4.py
tagged_sentences = treebank.tagged_sents()
size = int(len(tagged_sentences) * 0.9)
train_sents = tagged_sentences[:size]
test_sents = tagged_sentences[3000:]
default = DefaultTagger("NN")
tagger = ClassifierBasedPOSTagger(
train=train_sents, backoff=default, cutoff_prob=0.3
)
print(tagger.evaluate(test_sents)) # 0.9613641269156055
# save model to pickle file as binary
file_name = MODEL_PATH + "tag_model.pkl"
with open(file_name, "wb") as fout:
pickle.dump(tagger, fout)
print("model written to: " + file_name)
print("")
return tagger
def NER_HINDINBC():
reader = TaggedCorpusReader('/python27/POS_9/', r'.*\.pos')
f1 = reader.fileids()
print "The Files of Corpus are:", f1
sents = reader.tagged_sents()
sentn = reader.sents()
#words=sentn.split()
ls = len(sents)
#lw=len(words)
print "Length of Corpus Is:", ls
#print "The Words are:",lw
size1 = int(ls * 0.3)
test_sents = sents[:size1]
train_sents = sents[size1:]
nbc_tagger = ClassifierBasedPOSTagger(train=train_sents)
test = nbc_tagger.evaluate(test_sents)
print "The Test Result is:", test
#THE GIVEN INPUT
given_sent = "नीतीश कुमार द्वारा भाजपा के साथ हाथ मिलाने से वहां का पूरा राजनीतिक परिदृश्य ही बदल गया है मगर शरद यादव इससे खुश नहीं हैं".decode(
'utf-8')
gsw = given_sent.split()
tag_gs = nbc_tagger.tag(gsw)
print "GIVEN SENT TAG:", tag_gs
ftag_gs = " ".join(list(itertools.chain(*tag_gs)))
print "And its flattened Version is:", ftag_gs
def nltk_classifier_based_pos_tagger(input_dict):
"""
A sequential tagger that uses a classifier to choose the tag for
each token in a sentence. The featureset input for the classifier
is generated by a feature detector function::
feature_detector(tokens, index, history) -> featureset
Where tokens is the list of unlabeled tokens in the sentence;
index is the index of the token for which feature detection
should be performed; and history is list of the tags for all
tokens before index.
Construct a new classifier-based sequential tagger.
:param training_corpus: A tagged corpus consisting of a list of tagged
sentences, where each sentence is a list of (word, tag) tuples.
:param backoff_tagger: A backoff tagger, to be used by the new tagger
if it encounters an unknown context.
TODO: odloci se katerega se obdrzi od naslednjih dveh
:param classifier_builder: A function used to train a new
classifier based on the data in *train*. It should take
one argument, a list of labeled featuresets (i.e.,
(featureset, label) tuples).
:param classifier: The classifier that should be used by the
tagger. #This is only useful if you want to manually
construct the classifier; normally, you would use *train*
instead.
:param backoff_tagger: A backoff tagger, used if this tagger is
unable to determine a tag for a given token.
:param cutoff_prob: If specified, then this tagger will fall
back on its backoff tagger if the probability of the most
likely tag is less than *cutoff_prob*.
:returns pos_tagger: A python dictionary containing the POS tagger
object and its arguments.
"""
chunk = input_dict['training_corpus']['chunk']
corpus = input_dict['training_corpus']['corpus']
training_corpus=corpus_reader(corpus, chunk)
backoff_tagger=input_dict['backoff_tagger']['object'] if input_dict['backoff_tagger'] else DefaultTagger('-None-')
classifier=None #(input_dict['classifier'])
cutoff_prob=int(input_dict['cutoff_prob']) if input_dict['cutoff_prob'] else None
import nltk
tagger_object=ClassifierBasedPOSTagger(train=training_corpus, classifier=classifier,
backoff=backoff_tagger, cutoff_prob=cutoff_prob)
return {'pos_tagger': {
'function':'tag_sents',
'object': tagger_object
}
}
def myParse(sentence):
print("ClassifierBasedPOSTagger tag:")
brown_tagged_sents = brown.tagged_sents(categories='news')
train_sents = brown_tagged_sents[:500000]
tagger = ClassifierBasedPOSTagger(
train=train_sents) # , classifier_builder=MaxentClassifier.train)
mytagger = SQLPosTagger(tagger)
words = nltk.word_tokenize(sentence)
result = mytagger.tag(words)
print(result)
def nbc_tagger():
news_text = brown.tagged_sents(categories='news')
train_sents = news_text[:3230]
test_sents = news_text[3230:4600]
nbc_tagger = ClassifierBasedPOSTagger(train=train_sents)
test = nbc_tagger.evaluate(test_sents)
print "The Test Results Is:", test
sent3 = "Narendra Modi won Lok Sabha election with massive majority after long years"
sent_w = sent3.lower().split()
print sent_w
tag = nbc_tagger.tag(sent_w)
print "The Tag Is:", tag
def __init__(self, limit=300, debug=True):
'''Instance the TrainingSetAnalyzer
Keyword arguments:
@param: limit size of the tweets which need to be analyzed (300)
@param: debug flag for development process
'''
self.__debug = debug
self.__limit = limit
self.__speller = SpellChecker()
self.__splitter = Splitter("rtw")
self.__replacer = RegexpReplacer()
self.__ngramHandler = NgramHandler()
train_sents = treebank.tagged_sents()[:3000]
self.__tagger = ClassifierBasedPOSTagger(train=train_sents)
def wordTagger(self, wordlist,number):
train_sents = treebank.tagged_sents()[:3000]
if number==1:
taglist = nltk.pos_tag(wordlist)
elif number ==2:
tagger = DefaultTagger('NN')
taglist = tagger.tag(wordlist)
elif number ==3:
tagger = UnigramTagger(train_sents)
taglist = tagger.tag(wordlist)
elif number ==4:
tnt_tagger = tnt.TnT()
tnt_tagger.train(train_sents)
taglist = tnt_tagger.tag(wordlist)
elif number ==5:
tagger = ClassifierBasedPOSTagger(train=train_sents)
taglist = tagger.tag(wordlist)
return taglist
def parse():
tagger_classes = ([nltk.UnigramTagger, nltk.BigramTagger])
trained_sents, tagged_sents = trainer("WSJ_02-21.pos-chunk", "WSJ_23.pos")
#tagger = nltk.UnigramTagger(trained_sents)
print len(trained_sents)
tagger = ClassifierBasedPOSTagger(
train=trained_sents[:10000],
classifier_builder=lambda train_feats: MaxentClassifier.train(
train_feats, trace=0, max_iter=10))
f = open("WSJ_23.chunk", 'w')
#print sents
for sents in tagged_sents:
(words, tags) = sents[0], sents[1]
chunks = tagger.tag(tags)
#print words, chunks
wtc = zip(words, chunks)
for tup in wtc:
f.write("%s\t%s\n" % (tup[0], tup[1][1]))
f.write("\n")
def get_chunks(text_string):
# tokenization
print('Tokenising text...')
sentences = sent_tokenize(text_string)
tokenized_sentences = []
for s in sentences:
tokenized_sentences.append(word_tokenize(s))
# PoS tagging
train_sents = treebank.tagged_sents()
print('Training PoS tagger...')
tagger = ClassifierBasedPOSTagger(train=train_sents)
tagged_sentences = []
print('Tagging sentences...')
for s in tokenized_sentences:
tagged_sentences.append(tagger.tag(s))
# chunking
print('Getting trained chunk classifier...')
chunk_classifier = get_trained_classifier()
chunked_sentences = []
print('Chunking sentences...')
for s in tagged_sentences:
chunked_sentences.append(chunk_classifier.parse(s))
return chunked_sentences
print tt.evaluate(test_data)
print tt.tag(tokens)
def combined_tagger(train_data, taggers, backoff=None):
for tagger in taggers:
backoff = tagger(train_data, backoff=backoff)
return backoff
ct = combined_tagger(train_data=train_data,
taggers=[UnigramTagger, BigramTagger, TrigramTagger],
backoff=rt)
print ct.evaluate(test_data)
print ct.tag(tokens)
from nltk.classify import NaiveBayesClassifier, MaxentClassifier
from nltk.tag.sequential import ClassifierBasedPOSTagger
nbt = ClassifierBasedPOSTagger(train=train_data,
classifier_builder=NaiveBayesClassifier.train)
print nbt.evaluate(test_data)
print nbt.tag(tokens)
# try this out for fun!
met = ClassifierBasedPOSTagger(train=train_data,
classifier_builder=MaxentClassifier.train)
print met.evaluate(test_data)
print met.tag(tokens)
import nltk
from nltk.corpus import treebank
from nltk.tag import DefaultTagger
from nltk.tag.sequential import ClassifierBasedPOSTagger
default = DefaultTagger('NN')
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents,
backoff=default,
cutoff_prob=0.3)
tagger.evaluate(test_sents)
#token = nltk.word_tokenize(title) #title string tokenized
#removing all the punctuation marks
#punctuation = re.compile(r'[-.?,":;()`~!@#$%^*()_=+{}]')
#tword = [punctuation.sub("", word) for word in token]
#print(tword) #without punctuation
#removing all the MS smart quotes
# train
tic()
tnt_tagger = tnt.TnT()
tnt_tagger.train(train_sents)
tnt_eval['train_time'] = toc()
# test
tic()
tnt_eval['test_accuracy'] = tnt_tagger.evaluate(val_sents)
tnt_eval['test_time'] = toc()
# display results
display_training_metrics(tnt_eval)
""" 2. Naive Bayes classifier tagger """
nb_eval = dict()
# train
tic()
nb_tagger = ClassifierBasedPOSTagger(train=train_sents)
nb_eval['train_time'] = toc()
# test
tic()
nb_eval['test_accuracy'] = nb_tagger.evaluate(val_sents)
nb_eval['test_time'] = toc()
# display results
display_training_metrics(nb_eval)
""" 3. Naive Bayes classifier tagger with features """
nb_eval = dict()
# train
tic()
nb_tagger = ClassifierBasedTagger(train=train_sents,
feature_detector=add_features)
nb_eval['train_time'] = toc()
# test
# print( 'Training TnT...' )
# tnt_tagger = tnt.TnT()
# tnt_tagger.train(train_corpus)
# print( 'Testing...' )
# acc = tnt_tagger.evaluate(test_corpus)
# print( 'TnT accuracy={0}\n'.format(acc) )
#
# # ----------------------------------------------------------------------
#
# print( 'Training UnigramTagger...' )
# unigram_tagger = UnigramTagger(train_corpus)
# with open( 'unigram.pos_tagger.pickle', 'wb' ) as f:
# pickle.dump( unigram_tagger, f )
#
# print( 'Testing...' )
# acc = unigram_tagger.evaluate(test_corpus)
# print( 'UnigramTagger accuracy={0}\n'.format(acc) )
# ----------------------------------------------------------------------
print('Training ClassifierBasedPOSTagger...')
cbt = ClassifierBasedPOSTagger(train=train_corpus)
print('Testing...')
acc = cbt.evaluate(test_corpus)
print('accuracy={0}\n'.format(acc))
print('Storing...')
with open(os.path.join(model_folder, 'ClassifierBasedPOSTagger.pickle'),
'wb') as f:
pickle.dump(cbt, f)
def classify_based_tag_train(): train_sents = treebank.tagged_sents()[:5000] #train_sents = brown.tagged_sents(categories='learned', tagset='universal') bigram_tagger = BigramTagger(train_sents) cbtagger = ClassifierBasedPOSTagger(train=train_sents, backoff = bigram_tagger) pickle.dump(cbtagger, open( 'my_tagger.pkl', 'wb' ) )
