def NER_HINDINBC():
reader = TaggedCorpusReader('/python27/POS_9/', r'.*\.pos')
f1 = reader.fileids()
print "The Files of Corpus are:", f1
sents = reader.tagged_sents()
sentn = reader.sents()
#words=sentn.split()
ls = len(sents)
#lw=len(words)
print "Length of Corpus Is:", ls
#print "The Words are:",lw
size1 = int(ls * 0.3)
test_sents = sents[:size1]
train_sents = sents[size1:]
nbc_tagger = ClassifierBasedPOSTagger(train=train_sents)
test = nbc_tagger.evaluate(test_sents)
print "The Test Result is:", test
#THE GIVEN INPUT
given_sent = "नीतीश कुमार द्वारा भाजपा के साथ हाथ मिलाने से वहां का पूरा राजनीतिक परिदृश्य ही बदल गया है मगर शरद यादव इससे खुश नहीं हैं".decode(
'utf-8')
gsw = given_sent.split()
tag_gs = nbc_tagger.tag(gsw)
print "GIVEN SENT TAG:", tag_gs
ftag_gs = " ".join(list(itertools.chain(*tag_gs)))
print "And its flattened Version is:", ftag_gs
def nbc_tagger():
news_text = brown.tagged_sents(categories='news')
train_sents = news_text[:3230]
test_sents = news_text[3230:4600]
nbc_tagger = ClassifierBasedPOSTagger(train=train_sents)
test = nbc_tagger.evaluate(test_sents)
print "The Test Results Is:", test
sent3 = "Narendra Modi won Lok Sabha election with massive majority after long years"
sent_w = sent3.lower().split()
print sent_w
tag = nbc_tagger.tag(sent_w)
print "The Tag Is:", tag
def parse():
tagger_classes=([nltk.UnigramTagger, nltk.BigramTagger])
trained_sents, tagged_sents = trainer("WSJ_02-21.pos-chunk","WSJ_23.pos")
#tagger = nltk.UnigramTagger(trained_sents)
print len(trained_sents)
tagger = ClassifierBasedPOSTagger(train=trained_sents[:10000], classifier_builder=lambda train_feats:
MaxentClassifier.train(train_feats, trace = 0,max_iter=10))
f = open("WSJ_23.chunk",'w')
#print sents
for sents in tagged_sents:
(words,tags)=sents[0],sents[1]
chunks = tagger.tag(tags)
#print words, chunks
wtc = zip(words, chunks)
for tup in wtc:
f.write("%s\t%s\n" %(tup[0],tup[1][1]))
f.write("\n")
def parse():
tagger_classes = ([nltk.UnigramTagger, nltk.BigramTagger])
trained_sents, tagged_sents = trainer("WSJ_02-21.pos-chunk", "WSJ_23.pos")
#tagger = nltk.UnigramTagger(trained_sents)
print len(trained_sents)
tagger = ClassifierBasedPOSTagger(
train=trained_sents[:10000],
classifier_builder=lambda train_feats: MaxentClassifier.train(
train_feats, trace=0, max_iter=10))
f = open("WSJ_23.chunk", 'w')
#print sents
for sents in tagged_sents:
(words, tags) = sents[0], sents[1]
chunks = tagger.tag(tags)
#print words, chunks
wtc = zip(words, chunks)
for tup in wtc:
f.write("%s\t%s\n" % (tup[0], tup[1][1]))
f.write("\n")
def get_chunks(text_string):
# tokenization
print('Tokenising text...')
sentences = sent_tokenize(text_string)
tokenized_sentences = []
for s in sentences:
tokenized_sentences.append(word_tokenize(s))
# PoS tagging
train_sents = treebank.tagged_sents()
print('Training PoS tagger...')
tagger = ClassifierBasedPOSTagger(train=train_sents)
tagged_sentences = []
print('Tagging sentences...')
for s in tokenized_sentences:
tagged_sentences.append(tagger.tag(s))
# chunking
print('Getting trained chunk classifier...')
chunk_classifier = get_trained_classifier()
chunked_sentences = []
print('Chunking sentences...')
for s in tagged_sentences:
chunked_sentences.append(chunk_classifier.parse(s))
return chunked_sentences
default = DefaultTagger('NN')
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents,backoff=default, cutoff_prob = 0.3 )
#tagger.evaluate(test_sents)
#applying the tagger
rtag = tagger.tag(r)
print(rtag)
#extracting all the noun phrases from raw string
nlist = []
for word,tag in rtag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
#####
from nltk.tag.sequential import ClassifierBasedPOSTagger
print("started classified")
class_tagger = None
try:
with open('test_pickles/class.pickle', 'rb') as fa:
class_tagger = pickle.load(fa)
except FileNotFoundError as a:
# training data
print("dumping class")
class_tagger = ClassifierBasedPOSTagger(train=train)
with open('test_pickles/class.pickle', 'wb') as fb:
pickle.dump(class_tagger, fb)
#print(class_tagger.evaluate(test))
print(class_tagger.tag(tokenized_words))
####
#
# 4 TnT
#
####
print("started tnt")
from nltk.tag import tnt
tnt_tagger = None
try:
with open('test_pickles/tnt.pickle', 'rb') as fa:
tnt_tagger = pickle.load(fa)
except FileNotFoundError as a:
# training data
print("dumping tnt")
from nltk.tag.sequential import ClassifierBasedPOSTagger
default = DefaultTagger('NN')
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents,backoff=default, cutoff_prob = 0.3 )
#implementing it on the url names
ntag = tagger.tag(ntoken)
#extracting all the noun phrases from URL string
nlist = []
for word,tag in ntag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
value = "%s" % word
#punctuation = re.compile(r'[-.?,":;()`~!@#$%^*()_=+{}]')
#tword = [punctuation.sub("", word) for word in token]
#print(tword) #without punctuation
#removing all the MS smart quotes
#smart_quotes = re.compile(r'[\x80-\x9f]')
#words = [smart_quotes.sub("", i) for i in tword]
#print(words) #without the smart quotes
titletag = tagger.tag(ttoken) #tagging the list
print(titletag)
#extracting all the noun phrases from title string
nlist = []
for word, tag in titletag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
default = DefaultTagger('NN')
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents,backoff=default, cutoff_prob = 0.3 )
#tagger.evaluate(test_sents)
#applying the tagger
htag = tagger.tag(hd_tokens)
print(htag)
#extracting all the noun phrases from raw string
nlist = []
for word,tag in htag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
#tword = [punctuation.sub("", word) for word in token]
#print(tword) #without punctuation
#removing all the MS smart quotes
#smart_quotes = re.compile(r'[\x80-\x9f]')
#words = [smart_quotes.sub("", i) for i in tword]
#print(words) #without the smart quotes
titletag = tagger.tag(ttoken) #tagging the list
print(titletag)
#extracting all the noun phrases from title string
nlist = []
for word,tag in titletag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
from nltk.tag.sequential import ClassifierBasedPOSTagger
default = DefaultTagger('NN')
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents,
backoff=default,
cutoff_prob=0.3)
#applying the tagger
rawtag = tagger.tag(clean)
print rawtag
#extracting all the noun phrases from raw string
nlist = []
for word, tag in rawtag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
#%%
# combined tagger with a list of taggers and use a backoff tagger
def combined_tagger(train_data, taggers, backoff=None):
for tagger in taggers:
backoff = tagger(train_data, backoff=backoff)
return backoff
ct = combined_tagger(train_data=train_data,
taggers=[UnigramTagger, BigramTagger, TrigramTagger],
backoff=rt)
# evaluating the new combined tagger with backoff taggers
print(ct.evaluate(test_data))
print(ct.tag(nltk.word_tokenize(sentence)))
#%%
## Training using Supervised classification algorithm
from nltk.classify import NaiveBayesClassifier, MaxentClassifier
from nltk.tag.sequential import ClassifierBasedPOSTagger
nbt = ClassifierBasedPOSTagger(train=train_data,
classifier_builder=NaiveBayesClassifier.train)
# evaluate tagger on test data and sample sentences
print(nbt.evaluate(test_data))
print(nbt.tag(nltk.word_tokenize(sentence)))
class TrainingSetAnalyzer():
'''
This class handles the setting of the training set data
and provides support for features exctraction given a text'''
def __init__(self, limit=300, debug=True):
'''Instance the TrainingSetAnalyzer
Keyword arguments:
@param: limit size of the tweets which need to be analyzed (300)
@param: debug flag for development process
'''
self.__debug = debug
self.__limit = limit
self.__speller = SpellChecker()
self.__splitter = Splitter("rtw")
self.__replacer = RegexpReplacer()
self.__ngramHandler = NgramHandler()
train_sents = treebank.tagged_sents()[:3000]
self.__tagger = ClassifierBasedPOSTagger(train=train_sents)
def __analyzeSingleTweet(self, tweet):
'''
Helper function to get unigrams, emoticons, ngrams given a text
Keyword arguments:
@param: tweet the tweet to be analyzed
'''
chunks = self.__splitter.split(u'' + tweet)
raw_feature_list_neg = []
emot_list = []
ngrams = []
for subTweet in chunks:
try:
preprocessed_tweet = self.__replacer.preprocess(subTweet)
acr_expanded, tmp_emot_list = self.__replacer \
.acr_emot_exctractor(preprocessed_tweet)
emot_list += tmp_emot_list
enanched_txt = self.__speller.check_and_replace(acr_expanded)
tagged_sent = self.__tagger.tag(enanched_txt)
raw_feature_list_neg += self.__replacer \
.filter_raw_feature_list(
acr_expanded)
ngrams += self.__ngramHandler.exctract_ngrams(tagged_sent)
except Exception as e:
print "Sorry, something goes wrong: %s txt: %s" \
% (str(e), tweet)
return raw_feature_list_neg, emot_list, ngrams
def analyze(self):
'''
Analyzes a set of tweets
'''
print "Found %i elments for training" % self.__limit
n = 0
while n < 20:
qs = get_tweets_for_analyzing(skip=n)
for tweet in qs:
raw_feature_list_neg, emot, ngrams = self.__analyzeSingleTweet(
tweet.text)
if not self.__debug:
print "saving...."
tweet.set_features(raw_feature_list_neg, emot, ngrams)
n += 1
return
def extract_features_for_classification(self, text):
'''
Helper function to exctract features given a text
Keyword arguments:
@param: text the text whose the features will be exctracted
'''
raw_feature_list_neg, emot_list, ngrams = self.__analyzeSingleTweet(
text)
return raw_feature_list_neg, emot_list, ngrams, dict([
(word, True) for word in raw_feature_list_neg + emot_list + ngrams
])
def purge_useless_features(self):
'''Helper function to prune less frequent unigram features'''
tweets = get_tweets_for_pruning()
print "Pruning process for %i tweets" % tweets.count()
mrt = tweets.map_reduce(mapfunc_filter, reducefunc, "cn")
mrt = filter(lambda status: status.value > PURGE_TRESHOLD, mrt)
purged_qs = [item.key for item in mrt]
for tweet in tweets:
try:
tweet.features.filtered_unigram = [
item for item in purged_qs
if item in tweet.features.raw_feature_list_neg
]
tweet.save()
except Exception, e:
print e
print "Done!"
default = DefaultTagger("NN")
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents, backoff=default, cutoff_prob=0.3)
# tagger.evaluate(test_sents)
# applying the tagger
htag = tagger.tag(hd_tokens)
print(htag)
# extracting all the noun phrases from raw string
nlist = []
for word, tag in htag:
if tag == "NN":
value = "%s" % word
nlist.append(value)
if tag == "NNP":
from nltk.tag import DefaultTagger
from nltk.tag.sequential import ClassifierBasedPOSTagger
default = DefaultTagger('NN')
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents,backoff=default, cutoff_prob = 0.3 )
#applying the tagger
rawtag = tagger.tag(clean)
print rawtag
#extracting all the noun phrases from raw string
nlist = []
for word,tag in rawtag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
def combined_tagger(training_data, taggers, backoff=None):
for tagger in taggers:
backoff = tagger(training_data, backoff=backoff)
return backoff
ct = combined_tagger(training_data=train_data,
taggers=[UnigramTagger, BigramTagger, TrigramTagger],
backoff=rt)
# Evaluate the chained tagger with backoff. (Great accuracy! 90.91%)
print(ct.evaluate(test_data))
print("\n4-Chained tags:")
print(ct.tag(tokens))
# 5. TAGGER TRAINED BY SUPERVISED CLASSIFICATION ALTGORITHM (Naive Bayes Classifier)
# The feature_detector function forms the core of the training process. It generates features of the training data,
# such as word, previous word, tag, previous tag, case etc.
nbt = ClassifierBasedPOSTagger(train=train_data,
classifier_builder=NaiveBayesClassifier.train)
# Evaluate tagger on test data and sample sentence. (Awesome accuracy! 93.07%)
print(nbt.evaluate(test_data))
print("\n5-Naive Bayes Classifier-based tags:")
print(nbt.tag(tokens))
print tt.evaluate(test_data)
print tt.tag(tokens)
def combined_tagger(train_data, taggers, backoff=None):
for tagger in taggers:
backoff = tagger(train_data, backoff=backoff)
return backoff
ct = combined_tagger(train_data=train_data,
taggers=[UnigramTagger, BigramTagger, TrigramTagger],
backoff=rt)
print ct.evaluate(test_data)
print ct.tag(tokens)
from nltk.classify import NaiveBayesClassifier, MaxentClassifier
from nltk.tag.sequential import ClassifierBasedPOSTagger
nbt = ClassifierBasedPOSTagger(train=train_data,
classifier_builder=NaiveBayesClassifier.train)
print nbt.evaluate(test_data)
print nbt.tag(tokens)
# try this out for fun!
met = ClassifierBasedPOSTagger(train=train_data,
classifier_builder=MaxentClassifier.train)
print met.evaluate(test_data)
print met.tag(tokens)
from nltk.tag.sequential import ClassifierBasedPOSTagger
default = DefaultTagger('NN')
train_sents = treebank.tagged_sents()[:3000]
test_sents = treebank.tagged_sents()[3000:]
tagger = ClassifierBasedPOSTagger(train=train_sents,
backoff=default,
cutoff_prob=0.3)
#implementing it on the url names
ntag = tagger.tag(ntoken)
#extracting all the noun phrases from URL string
nlist = []
for word, tag in ntag:
if (tag == 'NN'):
value = "%s" % word
nlist.append(value)
if (tag == 'NNP'):
value = "%s" % word
nlist.append(value)
