def decide_actionable_tweet(doc_standard): actionable_tweet = [] from textblob.classifiers import NaiveBayesClassifier as NBC from textblob import TextBlob training_corpus = [ ('naredra modi is good politician','not_actionable'), ('how congress become good oppositor','actionable'), ('python is popular language','not_actionable'), ('here is new version of python available see it','actionable'), ('retweet why india is poor country','actionable'), ('Pro cubbadi startion on 1 august 2017 ','not_actionable'), ('book ticket for goa at reasonable cost','actinable')] test_corpus = [('here is new version of motorola see it','actionable'), ('hellow friends how are you','not_actionable')] model = NBC(training_corpus) print("model",model) try: for doc in doc_standard: # for testing use other list instead of doc_standard result = model.classify(doc) if result is 'actionable': actionable_tweet.append(doc) except: print("error in classify") print("actionable_tweet", actionable_tweet) return actionable_tweet
def investigate(self): data = self.scrape(self.base_urls) train_crud = CRUD("sqlite:///database.db", Ads, "ads") #getting dummy data from http://www.dummytextgenerator.com/#jump dummy_crud = CRUD("sqlite:///database.db", TrainData, "training_data") train = train_crud.get_all() dummy = dummy_crud.get_all() t_docs = [elem.text for elem in train_crud.get_all() ] #all documents with trafficking train = [(elem.text, "trafficking") for elem in train] + [(elem.text, "not trafficking") for elem in dummy] cls = [] #make use of tdf-idf here #add in this example: http://scikit-learn.org/0.11/auto_examples/document_classification_20newsgroups.html cls.append(NBC(train)) cls.append(DTC(train)) for datum in data: for cl in cls: if cl.classify(datum["text_body"]) == "trafficking": self.save_ads([datum]) #so I don't have to eye ball things if doc_comparison(datum["text_body"], t_docs) == "trafficking": self.save_ads([datum]) if self.doc_comparison(datum["text_body"], t_docs) == "trafficking": self.save_ads([datum]) time.sleep(700) # wait ~ 12 minutes self.investigate() #this is an infinite loop, which I am okay with.
def train_classifier(training_corpus): """Building Naive Bayes Classifier model. :param training_corpus: List of training tuples containing prhase and class. :returns: Navie Bayes Classifier model. """ print("Training model...") return NBC(training_corpus)
def nlp(corpus): from textblob.classifiers import NaiveBayesClassifier as NBC from textblob import TextBlob training_corpus = [ ('What is the status of covid in kerela?', 'Class_B'), ("Who is the latest celebrith to be tested positive?", 'Class_B'), ('When will vaccines be found?', 'Class_B'), ('When will colleges open in karnataka', 'Class_B'), ('What is the active case count in Goa?', 'Class_A'), ('Total deaths in bangalore', 'Class_A'), ('number of tested positive cases in the world', 'Class_A') ] test_corpus = [("Is joe tested positive?", 'Class_B'), ("increasing cases in kerela", 'Class_A'), ("chennais active cases", 'Class_A'), ("when will the vaccine be found", 'Class_B')] model = NBC(training_corpus) op = model.classify(corpus) #print(model.accuracy(test_corpus)) return op
def naive_bayes_classify(data): class_to_predict = 'type' # product importance all_data = [ tuple(x) for x in data[['text', class_to_predict]].to_records(index=False) ] text_counts = {} for item in all_data: for word in set(item[0].split()): if word in text_counts: text_counts[word] += 1 else: text_counts[word] = 1 for i in range(len(all_data)): new_text = '' for word in all_data[i][0].split(): if text_counts[word] >= 5: new_text += ' ' + word all_data[i] = (new_text, all_data[i][1]) print('Finished preprocessing!') test_corpus = all_data[3000:3600] training_corpus = all_data[:3000] model = NBC(training_corpus, verbose=True) print('Done training!') print('Accuracy: ' + str(model.accuracy(test_corpus))) y_pred = [] y_true = [] for test_item in test_corpus: y_pred.append(model.prob_classify(test_item[0]).max()) y_true.append(test_item[1]) print('F1 score: ' + str(f1_score(y_true, y_pred, average='weighted')))
return noise_free_text user_text = "" noicy_text = _remove_noise(user_text.lower()) noicy_ = lemmatizer.lemmatize(noicy_text, "v") training_corpus = [('I am exhausted of this work.', 'Class_B'), ("I can't cooperate with this", 'Class_B'), ('He is my badest enemy!', 'Class_B'), ('My management is poor.', 'Class_B'), ('I love this burger.', 'Class_A'), ('This is an brilliant place!', 'Class_A'), ('I feel very good about these dates.', 'Class_A'), ('This is my best work.', 'Class_A'), ("What an awesome view", 'Class_A'), ('I do not like this dish', 'Class_B')] test_corpus = [("I am not feeling well today.", 'Class_B'), ("I feel brilliant!", 'Class_A'), ('Gary is a friend of mine.', 'Class_A'), ("I can't believe I'm doing this.", 'Class_B'), ('The date was good.', 'Class_A'), ('I do not enjoy my job', 'Class_B')] model = NBC(training_corpus) print(model.classify(input())) print(noicy_)
def naive_bayes(train_data): """ cl.classify("some new text") #a label returned """ return NBC(train_data)
def scrape(self,links=[],auto_learn=False,long_running=False,translator=False): responses = [] values = {} data = [] if links == []: for base_url in self.base_urls: r = requests.get(base_url) text = unidecode(r.text) html = lxml.html.fromstring(text) links = html.xpath("//div[@class='cat']/a/@href") for link in links: if len(self.base_urls) > 1 or len(self.base_urls[0]) > 3: time.sleep(random.randint(1,2)) if long_running: time.sleep(random.randint(5,27)) try: responses.append(requests.get(link)) print link except requests.exceptions.ConnectionError: print "hitting connection error" continue else: for link in links: if len(self.base_urls) > 1 or len(self.base_urls[0]) > 3: time.sleep(random.randint(1,2)) if long_running: time.sleep(random.randint(5,17)) try: responses.append(requests.get(link)) print link except requests.exceptions.ConnectionError: print "hitting connection error" continue for r in responses: text = r.text html = lxml.html.fromstring(text) values["title"] = html.xpath("//div[@id='postingTitle']/a/h1")[0].text_content() values["link"] = unidecode(r.url) values["new_keywords"] = [] try: values["images"] = html.xpath("//img/@src") except IndexError: values["images"] = "weird index error" pre_decode_text = html.xpath("//div[@class='postingBody']")[0].text_content().replace("\n","").replace("\r","") values["text_body"] = pre_decode_text try: values["posted_at"] = html.xpath("//div[class='adInfo']")[0].text_content().replace("\n"," ").replace("\r","") except IndexError: values["posted_at"] = "not given" values["scraped_at"] = str(datetime.datetime.now()) body_blob = TextBlob(values["text_body"]) title_blob = TextBlob(values["title"]) values["language"] = body_blob.detect_language() #requires the internet - makes use of google translate api values["polarity"] = body_blob.polarity values["subjectivity"] = body_blob.sentiment[1] translated = translator or values["language"] == "es" if translated: values["translated_body"] = body_blob.translate(from_lang="es") values["translated_title"] = title_blob.translate(from_lang="es") else: values["translated_body"] = "none" values["translated_title"] = "none" text_body = values["text_body"] title = values["title"] if translated: text_body = values["translated_body"] title = values["translated_title"] if auto_learn: train = pickle.load(open("train.p","rb")) cls = [] cls.append(NBC(train)) cls.append(DTC(train)) #increase this number trk_count = 0 for cl in cls: if cl.classify(text_body) == "trafficking": trk_count += 1 if float(trk_count)/len(cls) > 0.5: train = pickle.load(open("train.p","rb")) train.append((values["text_body"],"trafficking") ) pickle.dump(train,open("train.p","wb")) values["trafficking"] = "found" else: values["trafficking"] = "not_found" #To do set up postmark here. #Documentation: https://devcenter.heroku.com/articles/postmark #even more docs: https://postmarkapp.com/servers/645009/get_started else: values["trafficking"] = "not_found" values["child_urls"] = [] for keyword in self.child_keywords: if keyword in text_body: values["child_urls"].append(values["link"]) elif keyword in title: values["child_urls"].append(values["link"]) values["trafficking_urls"] = [] for keyword in self.trafficking_keywords: if keyword in text_body: values["trafficking_urls"].append(values["link"]) elif keyword in title: values["trafficking_urls"].append(values["link"]) values["new_keywords"].append(self.pull_keywords(text_body)) values["new_keywords"].append(self.pull_keywords(title)) values = self.phone_number_parse(values) numbers = pickle.load(open("numbers.p","rb")) values["network"] = [] for network in numbers.keys(): if values["phone_number"] in numbers[network]: values["network"].append(network) data.append(values) self.save_ads(data) return data
nlp = spacy.load('en_core_web_sm') from textblob.classifiers import NaiveBayesClassifier as NBC #project_root = os.path.dirname(os.path.realpath(os.path.join(__file__, '..', '..'))) misswords = [ "Sorry I didn't get you", "I apologize i couldnt decifer you", "Sorry i am Dumb", "Can you repeat it please?", "Pardon?", "I am not a human so Plz cooporate sir", "I didn't get you" ] with open('C:\\Users\\Mayukh\\Desktop\\ChatBOT\\ChatTemplate\\file.json', 'r') as json2: data = json.load(json2) training = [] for i in data: for sentence in data[i]['train']: training.append((sentence, i)) model = NBC(training) app = Flask(__name__, static_url_path='', static_folder='static') @app.route('/') def chat(): return render_template('chat.html') @app.route('/chat') def index(): return render_template('index.html') @app.route('/about') def about():
#### Naive Bayes #### ##################### import pandas as pd import numpy as np from sklearn.model_selection import train_test_split from textblob.classifiers import NaiveBayesClassifier as NBC df = pd.DataFrame({"labels": trainLabels, "trainData": trainData}) train, test = train_test_split(df, test_size = 0.25) training = zip(train["trainData"].tolist() , train["labels"].tolist()) testing = zip(test["trainData"].tolist() , test["labels"].tolist()) ## training model %time model = NBC(training) %time print(model.accuracy(training)) ## getting accuracy of 90% ## Shows important features for detecting intent print model.show_informative_features() #Most Informative Features # contains(please) = True Yes : No = 10.0 : 1.0 # contains(ve) = True No : Yes = 9.9 : 1.0 # contains(verifi) = True Yes : No = 9.3 : 1.0 # contains(sale) = True No : Yes = 9.3 : 1.0 # contains(moment) = True Yes : No = 8.6 : 1.0 # contains(compani) = True No : Yes = 7.3 : 1.0 # contains(deliveri) = True No : Yes = 6.9 : 1.0 # contains(unsubscrib) = True No : Yes = 6.9 : 1.0
from textblob.classifiers import NaiveBayesClassifier as NBC from textblob import TextBlob with open('train_ticketdata.csv','r') as training_corpus: model = NBC(training_corpus, format='csv') with open('test_ticketdata.csv','r') as test_corpus: accuracy = model.accuracy(test_corpus) def apply_nlp(data): print("start nlp") print(data) print("end nlp") return model.classify(data) def get_accuracy(): return accuracy
from textblob.classifiers import NaiveBayesClassifier as NBC import glob import sys f = glob.glob("/home/aravind/AI-Nielit/NLP/Day3_Nov14/A4/Data/*/*") trainData = [] for i in f: s = i.split('/') cls = s[-2] f = open(i, "r") f = f.read().strip() tup = f, cls trainData.append(tup) print "Data Importing Completed" classifier = NBC(trainData) print "Training Complete" f = open('Test', "r") f1 = f.read() testData = [] testData.append(f1) print "Test Data\n-------------------------------------" print f1 print "-------------------------------------" c = classifier.classify(testData) print c
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2018/7/3 下午4:00 # @Author : Kaiyu # @Site : # @File : test.py from textblob.classifiers import NaiveBayesClassifier as NBC from textblob import TextBlob import json if __name__ == '__main__': with open('okoo-merged-labels.json', encoding='utf-8') as f: data = json.load(f)['all'] data = [(item['text'], str(item['merged_label'])) for item in data] train_data = data[:-1000] test_data = data[-1000:-1] model = NBC(train_data) for test_item in test_data: # label_ = model.classify(test_item[0]) print('True: {} predict: {}'.format(str(test_item[1]), label_)) print(model.accuracy(test_data))
from textblob.classifiers import NaiveBayesClassifier as NBC train = pickle.load(open("train.p", "rb")) cl = NBC(train)
from textblob.classifiers import NaiveBayesClassifier as NBC with open('train.json', 'r') as fp: cl = NBC(fp, format="json") with open('test.json', 'r') as fp: print(str(cl.accuracy(fp, format="json") * 100) + ' %') with open('file.txt', 'r') as fp: for i in fp.readlines(): print(i[:-4] + ': ' + cl.classify(i))