def classifier(something):
speech = something
train = []
test = []
with open("training.csv") as csvfile:
reader = csv.reader(csvfile) # change contents to floats
for row in reader: # each row is a list
train.append(row)
with open("test.csv") as csvfile:
reader = csv.reader(csvfile) # change contents to floats
for row in reader: # each row is a list
test.append(row)
cl = NaiveBayesClassifier(train)
cl.classify("This is an amazing library!")
prob_dist = cl.prob_classify("This one's a doozy.")
prob_dist.max()
round(prob_dist.prob("machine"), 2)
round(prob_dist.prob("no machine"), 2)
blob = TextBlob(speech, classifier=cl)
blob.classify()
for s in blob.sentences:
print("\n\n\n" + str(s))
print("\n" + str(s.classify()))
return (s.classify())
def evaluate_message(message,
dataset,
desired_emotions,
neg_limit,
partisan_limit,
censor_mode=False):
# full_message = TextBlob(message)
cl = NaiveBayesClassifier(dataset)
classified_message = TextBlob(message, classifier=cl)
if censor_mode == True:
final_message = []
for s in classified_message.sentences:
print(s)
print(s.sentiment)
print(s.classify())
if (s.classify()
in desired_emotions) and (s.sentiment[0] > neg_limit) and (
abs(s.sentiment[1]) < partisan_limit):
final_message.append(str(s))
return " ".join(final_message)
else:
print(classified_message)
print(classified_message.sentiment)
print(classified_message.classify())
if (classified_message.classify() in desired_emotions) and (
classified_message.sentiment[0] > neg_limit) and (abs(
classified_message.sentiment[1]) < partisan_limit):
return True
else:
return False
def compute_clickbait_metric(self):
"""
Computes clickbaityness metric for text set.
"""
train = [
("Whoa Trump Orders Congress to Go After Deep State Obama Holdovers",
"neg"),
("Marked for ‘De-escalation Syrian Towns Endure Surge of Attacks",
"pos"), ("Perfume E-Mail Raises a Stink.", "neg"),
("Woman Pricked by Hidden Needle", "neg"),
("Four Accused in Facebook Live Torture Case Plead Not Guilty.",
"neg"),
("Mayor Tries to Save Warren Buffett's Old Berkshire Hathaway Headquarters",
"pos"),
("Senate Passes Sweeping Republican Tax Overhaul Bill", "pos"),
("Colombian General Captured, Released by Rebels Resigns", "pos"),
("Bulldog Bites Pedophile’s Penis Off as He Tried to Rape Sleeping Children.",
"neg"),
("The Scallop Sees With Space-Age Eyes — Hundreds of Them", "pos")
]
for i in range(10):
temp = (train[i][0], train[i][1])
temp = (train[i][0].decode('utf-8'), train[i][1])
train[i] = temp
cl = NaiveBayesClassifier(train)
blob = TextBlob(self.head, classifier=cl)
if blob.classify() == "pos":
return 5
else:
return 0
def classificar(texto):
with open("trained.pickle", "rb") as f:
cl = pickle.load(f)
resultado = TextBlob(texto, classifier=cl)
sentimento = resultado.classify()
print(sentimento)
return sentimento
def classify(text: str) -> bool:
cl = load()
blob = TextBlob(text, classifier=cl)
out = blob.classify()
if out == "neg":
return False
return True
def testSentence(self, sentence, classifier):
filtered_word_list = [
word for word in sentence.lower().split()
if word not in stopwords.words('english')
]
refinedText = " ".join(filtered_word_list)
blob = TextBlob(refinedText, classifier=classifier)
return blob.classify()
def nbclassify(dftest, dftrain, postdf, control):
dftrain = dftrain[['body', 'sentiment']]
tuples = [tuple(x) for x in dftrain.to_numpy()]
cl = NaiveBayesClassifier(tuples)
#cl.show_informative_features(10)
print('base case')
for index in control.index:
blob = TextBlob(dftest.at[index, 'body'], classifier=cl)
prob = blob.classify()
control.at[index, 'sentiment'] = prob
controlstat(control)
print('\n\n')
print('just NB classifier')
for index in dftest.index:
blob = TextBlob(dftest.at[index, 'body'], classifier=cl)
prob = blob.classify()
dftest.at[index, 'sentiment'] = prob
stats(dftest)
controlstat(dftest)
print('\n\n')
print('with parent into consideration')
for index in dftest.index:
parentid = dftest.at[index, 'parent_id']
parsent = parentsent(parentid, dftest, postdf)
if 't3_' in parentid:
if prob == parsent:
prob = 'positive'
elif parsent != 'neutral':
prob = 'negative'
else:
if prob == 'positive' and parsent != 'neutral':
prob = parsent
elif prob == 'negative' and parsent != 'neutral':
if parsent == 'negative':
prob = 'positive'
else:
prob = 'negative'
if prob == 'neutral':
print('neutrality shows up here')
dftest.at[index, 'sentiment'] = prob
stats(dftest)
controlstat(dftest)
def top_level(line: str, stripped=False, fname=None):
"""
Choses what to send the line to
"""
global count_tabs, tabnum, order, model, transpile, variables
tabnum = len(order)
if fname is None:
line = synonyms(line)
if line.startswith(".dev;transpile"):
if transpile:
transpile = False
else:
transpile = True
transpiler.starter(variables)
return
if count_tabs is True and stripped is False:
order = []
count_tabs = False
blob = TextBlob(line, classifier=model)
# Every Statement that needs to be classified
if line.startswith("#"):
return "#ignore"
elif list(line) == []:
return None
if line.startswith("dump"):
dump()
elif line.startswith("say"):
return say(line)
elif line.startswith("if") or (blob.classify() == "if"
and not re.match(r"^[ ]*say", line)):
if_statement(line)
elif line.startswith("elif"):
return elseif_statement(line)
elif line.startswith("else"):
return else_statement(line)
elif line.startswith("while") or blob.classify() == "while":
while_loop(line)
elif re.match(r"^<-|}|:", line):
end_arrow()
elif line.startswith("\t") or line.startswith(" "):
return tab_dealer(line)
elif util.var_math_check(line) is True:
return var_math(line)
elif re.match(r"\w+ ?= ?.+", line) or blob.classify() == "var":
return set_variable(line)
def response(self, phrase):
"""
Retorna a rasposta da frase de acordo com o classificador criado
"""
logging.debug('Analisa a frase "{}"'.format(phrase))
blob = TextBlob(phrase, classifier=self.__cl)
result = blob.classify()
logging.debug('Resposta: "{}"'.format(result))
return result
def _sentiTrain(self):
print('Treinando classificador de sentimento...')
training = self._loadTrainData(os.path.join(dirname,
'data_senti.json'))
classifier = classifiers.NaiveBayesClassifier(training)
classifier.show_informative_features(1)
blob = TextBlob('', classifier=classifier)
print(blob.classify())
return classifier
def classify_comments(comments):
sentiments = []
for comment in comments:
blob = TextBlob(comment, classifier=cl)
sentiments.append(blob.classify())
polarity = get_sentiment_polarity(sentiments)
return polarity
def find_sentiment(string):
# train the classifier
cl = NaiveBayesClassifier(train)
# load the classifier
blob = TextBlob(string, classifier=cl)
print "\n\n\n\t\t\t{}".format(blob.classify())
return
def classificar1(treino, teste, frase):
texto = tratarTexto(frase)
cl = NaiveBayesClassifier(treino)
accuracy = cl.accuracy(teste)
blob = TextBlob(texto, classifier=cl)
frase = str(frase)
return frase + ' ({})'.format(
blob.classify()) + '\nacurácia: {}'.format(accuracy)
def analyze_tweets(cl):
file = 'consumer.json'
f = open(file, 'r')
data = f.read()
blob = TextBlob(data,classifier=cl)
a = blob.classify()
print 'The product have an overall ' + a + ' rating\n'
#print 'The majority of the tweets that were positive are ' + str(b) + '\n'
#print 'The percent of the tweets that were negative are ' + str(c) + '\n'
return 'The product have an overall ' + a + ' rating\n'
def Sent_Analysis(dic, shouldSaveData, candidate, classifier=train_data()):
#Perform Sentiment Analysis
for e in dic:
x = e.get('text')
twt_lst = tokenize_str(x) #tokenize tweet (account for contractions)
twt_lst = remove_sw(twt_lst) #remove stop words from tweet
twt_lst = normalize_data(twt_lst) #normalize/lemmatize
twt_lst = " ".join(twt_lst)
cleaned_str = TextBlob(twt_lst, classifier=classifier)
sent = cleaned_str.sentiment
# Add the polarity and sentiment to the tweet
e['polarity'] = sent[0]
e['sentiment'] = cleaned_str.classify()
# Check user wants to save data
if shouldSaveData:
# Open file of data with polarity and sentiment added
f = open(candidate + "sent.txt", "a", encoding="utf8")
f.write(str(e) + "\n")
f.close()
# Return the classified data set
return dic
def nbsentiment(dftest, dftrain):
dftrain = dftrain[['body', 'sentiment']]
tuples = [tuple(x) for x in dftrain.to_numpy()]
cl = NaiveBayesClassifier(tuples)
dftest.insert(len(dftest.columns) - 1, 'new sent', 'blank')
for index in dftest.index:
#prob = cl.classify(dftest.at[index, 'body'])
blob = TextBlob(dftest.at[index, 'body'], classifier=cl)
prob = blob.classify()
#pol = blob.sentiment.polarity
#if pol > .05:
# prob = 'positive'
#else:
# prob = 'negative'
dftest.at[index, 'new sent'] = prob
dftest.to_csv('sentimentizednaivebayes.csv', index=False)
#dftest = dftest[['body','sentiment']]
#test_tuples = [tuple(x) for x in dftest.to_numpy()]
#print('accuracy of {}'.format(cl.accuracy(test_tuples)))
evaluation(dftest, 'new sent')
def model_classifier(self):
df = pd.read_csv(self.file_name)
#print(df.head())
##open classifier
classifier_f = open("naivebayes.pickle", "rb")
classifier = pickle.load(classifier_f)
classifier_f.close()
for row in df[1:5].iterrows():
#print(row[1][1])
self.date_list.append(row[1][1])
#print(row[1][2])
self.name_list.append(row[1][2])
#print(row[1][3])
self.text_list.append(row[1][3])
#print(row[1][4])
self.num_of_death.append(row[1][4])
text = row[1][3]
#blob = TextBlob(text, classifier=classifier.prob_classify(text))
blob = TextBlob(text, classifier=classifier)
print(blob.polarity)
#print(blob.classify())
self.classify.append(blob.classify())
##After analyzing the text
self.After_analyzing_data_dict.update({
"Date": self.date_list,
"name": self.name_list,
"tweet": self.text_list,
"death": self.num_of_death,
"Classification": self.classify
})
new_data = pd.DataFrame(self.After_analyzing_data_dict)
print(new_data.head())
new_data.to_csv("A_Trump_D12.csv")
('Gary is a friend of mine.', 'pos'),
("I can't believe I'm doing this.", 'neg')]
from textblob.classifiers import NaiveBayesClassifier
cl = NaiveBayesClassifier(train)
# classify method
print(cl.classify("This is amazing!"))
# probablity of class
prob_dist = cl.prob_classify("This one's a doozy.")
prob_dist.max()
round(prob_dist.prob("pos"), 2)
round(prob_dist.prob("neg"), 2)
# classify text blob
blob = TextBlob("I have good spelling!", classifier=cl)
blob.classify()
cl.accuracy(test)
cl.show_informative_features(5)
new_data = [('She is my best friend.', 'pos'),
("I'm happy to have a new friend.", 'pos'),
("Stay thirsty, my friend.", 'pos'),
("He ain't from around here.", 'neg')]
cl.update(new_data)
cl.accuracy(test)
def analyse_text(texts):
for i in texts:
blob = TextBlob(i, classifier=classifier)
print(i+" : "+blob.classify())
#sample_text=state_union.raw("2006-GWBush.txt");
#custom_set_tokenizer=PunktSentenceTokenizer(train_text);#Here we are training punktse..er with data train_set then we are going to use the trained classifier for sample text
#tokenizer=custom_set_tokenizer.tokenize(sample_text);
#print(custom_set_tokenizer);
#MISSING TEST DATASET MAKE ONE AND TEST THE ACCURACY
cl = NaiveBayesClassifier(train)
myfile = open('F:/anubhav.txt') # Windows
mytxt = myfile.read()
print mytxt
blob=TextBlob(mytxt,classifier=cl)
print blob.classify()+"\n"
myfile=open('F:/result.txt',"w+")
myfile.write(blob.classify())
print "NOW,BREAKING THE WHOLE SENTENCE INTO PARTS AND CLASSIFYING \n"
for sent in blob.sentences:
print sent+" = "+sent.classify()
#print(sent.classify())
print "\n"
myfile.close()
#for sentence in blob.sentences:
#print(sentence)
#print(sentence.classify())
cl = NaiveBayesClassifier(train)
print cl.classify(tx_cl)
print cl.classify("El subte funciona bien")
prob_dist = cl.prob_classify(tx_prob)
print prob_dist.max()
print round(prob_dist.prob("pos"), 2)
print round(prob_dist.prob("neg"), 2)
print cl.accuracy(data_sets.en_test)
print cl.show_informative_features(5)
#Using TextBlob
blob = TextBlob("No funca por que hay obras para mejorar la cosa",
classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob("El subte funciona normal", classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob("Se realizan obras en el subte A", classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob("No funciona, anda averiguar por que. Quizas hay un accidente",
classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob(u"El subte funciona ok", classifier=cl)
train_csv=pd.DataFrame.to_csv(train, index= False)
var.write(train_csv)
var.close()
train = pd.concat([X_train['text'],y_train], axis=1)
with open('train.csv', 'r') as fp2:
cl2 = DecisionTreeClassifier(fp2, format="csv")
pred_train=[]
feature_train=[]
true_train= train.label
for instance in train.text:
feature_train.append(feats(instance))
blob = TextBlob(instance, classifier=cl2)
pred_train.append(int(float(blob.classify())))
count=0
for i in range(len(pred_train)):
if pred_train[i] == y_train[i]:
count= count+1
#print('train_accuracy',count/len(pred_train))
pred_train= pd.DataFrame(pred_train)
global pred_test
global true_test
test = pd.concat([X_test['text'],y_test], axis=1)
var= open("test.json","w")
test_csv=pd.DataFrame.to_csv(test, index= False)
def post(self, request, *args, **kwargs):
response = {}
response['status'] = 500
try:
data = request.data
userText = data["user-msg"]
booky_bot_user = BookyBotUser.objects.get(
username=request.user.username)
bookings = Booking.objects.filter(user=booky_bot_user)
#print(bookings)
blob = TextBlob(userText, classifier=classifier)
if (blob.classify() == "pos" and len(bookings) != 0
and booky_bot_user.v_destination == ""
and booky_bot_user.v_source == ""):
booking = bookings.all().last()
flight_details = json.loads(booking.flight_details)
booky_bot_user.v_source = flight_details["flight_details"][6]
booky_bot_user.v_destination = flight_details[
"flight_details"][7]
if booky_bot_user.v_date == "" and booky_bot_user.v_destination == "" and booky_bot_user.v_source == "":
arr_temp = extract_info.blob_search(userText)
booky_bot_user.v_destination = arr_temp[1]
booky_bot_user.v_source = arr_temp[0]
booky_bot_user.v_date = arr_temp[2]
if booky_bot_user.v_source == "":
#response['bot-msg']="Enter source city"
if source_dest_check(userText, df) == True:
booky_bot_user.v_source = userText
print(booky_bot_user.v_source)
else:
arr_1 = extract_info.blob_search(userText)
booky_bot_user.v_source = arr_1[0]
elif booky_bot_user.v_destination == "":
#response['bot-msg'] = "Enter destination city"
if source_dest_check(userText, df) == True:
booky_bot_user.v_destination = userText
else:
arr_1 = extract_info.blob_search(userText)
booky_bot_user.v_destination = arr_1[1]
elif booky_bot_user.v_date == "":
#response['bot-msg'] = "Enter date of travel"
arr_1 = extract_info.blob_search(userText)
booky_bot_user.v_date = arr_1[2]
if booky_bot_user.v_source == "":
response['bot-msg'] = "Enter source city"
elif booky_bot_user.v_destination == "":
response['bot-msg'] = "Enter destination city"
elif booky_bot_user.v_date == "":
response['bot-msg'] = "Enter travel date"
elif booky_bot_user.v_date != "" and booky_bot_user.v_destination != "" and booky_bot_user.v_source != "":
if booky_bot_user.v_source == booky_bot_user.v_destination:
booky_bot_user.v_source = ""
booky_bot_user.v_destination = ""
response[
'bot-msg'] = 'Source city and destination city are same. Please enter valid source city.'
else:
response['bot-msg'] = 'Fetching options'
# print(userText)
# blob_text = TextBlob(userText, classifier=classifier)
# booky_bot_user = BookyBotUser.objects.get(username=request.user.username)
# booky_bot_user.step_counter=booky_bot_user.step_counter+1
# #booky_bot_user.save()
# #print(booky_bot_user.step_counter)
# if (booky_bot_user.step_counter == 1):
# if(blob_text.classify() == "pos"):
# booky_bot_user.trail_flag=1
# response['bot-msg']="Do you want to book a ticket?"
# else:
# booky_bot_user.trail_flag=0
# response['bot-msg']='Okay. You can always say "Help Me" if you need my assistance.'
# if(booky_bot_user.trail_flag==1):
# if(booky_bot_user.step_counter==2):
# if (blob_text.classify() == "pos"):
# response['bot-msg']="Enter city name from where you want to travel."
# else:
# booky_bot_user.trail_flag = 0
# response['bot-msg']='Okay. You can always say "Help Me" if you need my assistance.'
# elif(booky_bot_user.step_counter==3):
# print(source_dest_check(userText,df))
# if source_dest_check(userText,df) == True:
# booky_bot_user.v_source = str(userText).lower()
# response['bot-msg']='Enter destination city where you want to go'
# else:
# booky_bot_user.step_counter = booky_bot_user.step_counter - 1
# response['bot-msg']='Enter a valid source city'
# elif(booky_bot_user.step_counter==4):
# if source_dest_check(userText,df) == True:
# booky_bot_user.v_destination = str(userText).lower()
# response['bot-msg']='Enter travel date'
# else:
# booky_bot_user.step_counter = booky_bot_user.step_counter - 1
# response['bot-msg']="Enter a valid destination City"
# elif(booky_bot_user.step_counter==5):
# booky_bot_user.v_date = userText
# #api_integration(source,destination,date)
# response['bot-msg']="Fetching options"
# elif(booky_bot_user.step_counter==6):
# if (blob_text.classify() == "neg"):
# response['bot-msg']="Thank You for using our service. Come next time."
# elif(booky_bot_user.trail_flag==0 and booky_bot_user.step_counter!=1):
# if (booky_bot_user.step_counter!=8 and booky_bot_user.step_counter!=9):
# print("First If",booky_bot_user.step_counter)
# arr_temp = extract_info.blob_search(userText)
# print(arr_temp)
# if all(''==s or s.isspace() for s in arr_temp):
# booky_bot_user.step_counter = 2
# booky_bot_user.trail_flag = 1
# response['bot-msg']='Enter city from where you want to travel.'
# elif(arr_temp[0]=='' and arr_temp[1]==''):
# booky_bot_user.step_counter = 3
# booky_bot_user.trail_flag = 1
# booky_bot_user.v_destination = arr_temp[0]
# response['bot-msg']='Enter city where you want to go'
# elif(arr_temp[1]=='' and arr_temp[2]==''):
# booky_bot_user.step_counter = 3
# booky_bot_user.trail_flag = 1
# booky_bot_user.v_destination = arr_temp[0]
# response['bot-msg']='Enter city where you want to go'
# elif(arr_temp[0]=='' and arr_temp[2]==''):
# booky_bot_user.step_counter = 3
# booky_bot_user.trail_flag = 1
# booky_bot_user.v_source = arr_temp[1]
# response['bot-msg']='Enter city from where you want to travel.'
# elif(arr_temp[0]==''):
# booky_bot_user.v_destination = arr_temp[1]
# booky_bot_user.v_date = arr_temp[2]
# booky_bot_user.step_counter = 7
# booky_bot_user.trail_flag = 0
# response['bot-msg']='Enter city from where you want to travel.'
# elif(arr_temp[1]==''):
# booky_bot_user.v_source = arr_temp[0]
# booky_bot_user.v_date = arr_temp[2]
# booky_bot_user.step_counter = 8
# booky_bot_user.trail_flag = 0
# response['bot-msg']='Enter city where you want to go.'
# elif(arr_temp[2]==''):
# booky_bot_user.v_source = arr_temp[0]
# booky_bot_user.v_destination = arr_temp[1]
# booky_bot_user.step_counter = 4
# booky_bot_user.trail_flag = 1
# response['bot-msg']='Enter travel date'
# elif not any(''==s or s.isspace() for s in arr_temp):
# booky_bot_user.v_source = arr_temp[0]
# booky_bot_user.v_destination = arr_temp[1]
# booky_bot_user.v_date = arr_temp[2]
# response['bot-msg']='Fetching options'
# else:
# print("Else",booky_bot_user.step_counter)
# if(booky_bot_user.step_counter==8):
# booky_bot_user.v_source = userText
# response['bot-msg']="Fetching options"
# if(booky_bot_user.step_counter==9):
# booky_bot_user.v_destination = userText
# response['bot-msg']="Fetching options"
booky_bot_user.save()
print(booky_bot_user.v_source)
print(booky_bot_user.v_destination)
print(booky_bot_user.v_date)
print(response['bot-msg'])
response['status'] = 200
except Exception as e:
print("Error GetResponseAPI", str(e))
return Response(data=response)
def is_fake_or_real(article):
blob = TextBlob(article, classifier=cl)
print blob.classify()
('The beer was good.', 'pos'),
('I do not enjoy my job', 'neg'),
("I ain't feeling dandy today.", 'neg'),
("I feel amazing!", 'pos'),
('Gary is a friend of mine.', 'pos'),
("I can't believe I'm doing this.", 'neg')
]
cl = NaiveBayesClassifier(train)
cl.classify("Their burgers are amazing") # "pos"
cl.classify("I don't like their pizza.") # "neg"
from textblob import TextBlob
blob = TextBlob("The beer was amazing. "
"But the hangover was horrible. My boss was not happy.",
classifier=cl)
blob.classify() # "neg"
for sentence in blob.sentences:
print(sentence)
print(sentence.classify())
# "pos", "neg", "neg"
cl.accuracy(test)
# 0.83
cl.show_informative_features(5)
# Most Informative Features
# contains(my) = True neg : pos = 1.7 : 1.0
# contains(an) = False neg : pos = 1.6 : 1.0
# contains(my) = False pos : neg = 1.3 : 1.0
# contains(place) = False neg : pos = 1.2 : 1.0
# contains(of) = False pos : neg = 1.2 : 1.0
('Ok then', 'neg'),
('Fine', 'neg'),
("I can't deal with this", 'neg'),
("Whatever.", "neg"),
("lol ok", "neg"),
('Yeah sure!', 'pos'),
('Yeehaw!', 'pos'),
('Yeah totally!', 'pos'),
('so excited', 'pos')
]
cl = NaiveBayesClassifier(train)
m1 = TextBlob("I have booked my first ever flight! So excited", classifier=cl)
print(m1.sentiment)
print(m1.classify())
h1 = TextBlob("West Palm?")
print(h1.sentiment)
m2 = TextBlob("Yeehaw!")
print(m2.sentiment)
print(cl.classify(m2))
h2 = TextBlob("ok then. You seemed like you'd changed your mind last time we spoke.")
print(h2.sentiment_assessments)
m3 = TextBlob("No I mean we got a good house its all good")
print(m3.sentiment_assessments)
h3 = TextBlob("Alrighty. I was concerned about spending a few days with that group. I figured Chris will thirst over you, and I'd get to know the convention bar. Am I missing anyone.")
cl = NaiveBayesClassifier(train)
print cl.classify(tx_cl)
print cl.classify("El subte funciona bien")
prob_dist = cl.prob_classify(tx_prob)
print prob_dist.max()
print round(prob_dist.prob("pos"), 2)
print round(prob_dist.prob("neg"), 2)
print cl.accuracy(data_sets.en_test)
print cl.show_informative_features(5)
#Using TextBlob
blob = TextBlob("No funca por que hay obras para mejorar la cosa", classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob("El subte funciona normal", classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob("Se realizan obras en el subte A", classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob("No funciona, anda averiguar por que. Quizas hay un accidente", classifier=cl)
print blob.sentiment
print blob.classify()
blob = TextBlob(u"El subte funciona ok", classifier=cl)
print blob.sentiment
('Fantastic Four should have never been made.', 'pos'),
('Wes Anderson is my favorite director!', 'neg'),
('Captain America 2 is pretty awesome.', 'neg'),
('Lets pretend "Batman and Robin" never happened..', 'pos'),
]
testing_set = [
('Superman was never an interesting character.', 'pos'),
('Fantastic Mr Fox is an awesome film!', 'neg'),
('Dragonball Evolution is simply terrible!!', 'pos')
]
from textblob import classifiers as tbclassifiers
nb_clf = tbclassifiers.NaiveBayesClassifier(training_set)
print(nb_clf.accuracy(testing_set)) # 1.0
nb_clf.show_informative_features(3)
test_clf_blob = TextBlob('the weather is super!', classifier=nb_clf)
print(test_clf_blob.classify()) # neg
# decision tree classifier is also available
dt_clf = tbclassifiers.DecisionTreeClassifier(training_set)
print(dt_clf.accuracy(testing_set)) # 0.(6)
#%% [markdown]
"""
Pros and Cons
Pros:
Since, it is built on the shoulders of NLTK and Pattern, therefore making it simple for beginners by providing an intuitive interface to NLTK.
It provides language translation and detection which is powered by Google Translate ( not provided with Spacy).
Cons:
It is little slower in the comparison to spacy but faster than NLTK. (Spacy > TextBlob > NLTK)
It does not provide features like dependency parsing, word vectors etc. which is provided by spacy.
"""
rich.print("[blue]Alter Command Line Interface v.ALPHA")
# Progress Info when running code
console = Console()
tasks = [
"Loaded Machine Learning Model",
"Re-Processed model",
"Verified model speed",
"Verified model accuracy",
]
with console.status("[bold green]Starting...") as status:
model = usemodel.load()
task = tasks.pop(0)
console.log(f"{task}")
blob = TextBlob("the value of x is 5", classifier=model)
blob.classify()
task = tasks.pop(0)
console.log(f"{task}")
ct = time.time()
blob = TextBlob("the value of x is 4", classifier=model)
blob.classify()
if time.time() - ct < 1.5:
pass
else:
print("[bold red]Model May Run Slow")
task = tasks.pop(0)
console.log(f"{task}")
blob = TextBlob("let x equal 4", classifier=model)
if blob.classify() == "var":
pass
else:
("Restaurants nearby me", "L"), ("Where I play games", "L")]
test = [("Where you live", "L"), ("What is your age", "L"),
("When do you go school", "N"), ("What is dog in animals", "D")]
#cl = NaiveBayesClassifier(train)
with open("classifier.pickle", "rb") as f:
cl = pickle.load(f)
# Classify some text
print(cl.classify("What is a burger")) # "pos"
print(cl.classify("Where can I get pizza")) # "neg"
# Classify a TextBlob
blob = TextBlob(
"The beer was amazing. But the hangover was horrible. "
"My boss was not pleased.",
classifier=cl)
print(blob)
print(blob.classify())
for sentence in blob.sentences:
print(sentence)
print(sentence.classify())
# Compute accuracy
print("Accuracy: {0}".format(cl.accuracy(test)))
pickle.dump(cl, open("classifier.pickle", "wb"))
# Show 5 most informative features
cl.show_informative_features(5)
def testSentence(self,sentence,classifier):
filtered_word_list = [word for word in sentence.lower().split() if word not in stopwords.words('english')]
refinedText = " ".join(filtered_word_list)
blob = TextBlob(refinedText, classifier=classifier)
return blob.classify()
def common_replies(user_input_array, conn):
question1 = user_input_array[0]
train = [('hi', 'pos'), ('hello', 'pos'),
('please improve your answers', 'pos'),
('can you give me this?', 'pos'), ('i love your answers.', 'pos'),
('this is an amazing answer!', 'pos'),
('a very Good Morning', 'pos'),
('this is your best work.', 'pos'),
("this is an awesome answer", 'pos'),
('i do not like this', 'neg'), ('sorry', 'neg'),
('i am tired of this stuff.', 'neg'),
("i can't deal with this", 'neg'), ('you have to improve', 'neg'),
('you are taking so much time to learn.', 'neg')]
cl = NaiveBayesClassifier(train)
for i in user_input_array:
ques = ' '.join(user_input_array)
blob = TextBlob(ques, classifier=cl)
sentiment = blob.classify()
print(sentiment)
log.writetofile("entering positive or negative checking")
common_questions = [
'request appointment', 'text books references download',
'sjsu main campus map', 'share cmpe273 greensheet',
'slack manual assistance error problem', 'help', 'improve',
'great day', 'bye tata cya see you later', 'thank you thanks',
'thanks Sara', 'please improve your answers',
'are you feeling good today?', 'amazing answers', 'bye'
]
for i in common_questions:
if (sentiment == 'pos' and question1 != "is" and question1 != "does"
and (ques not in ' '.join(common_questions))):
response = check_for_greeting(ques)
elif (sentiment == 'neg' and question1 != "is" and question1 != "does"
and (ques not in ' '.join(common_questions))):
response = check_for_complaints(ques)
elif (question1 == "is" or question1 == "does"):
keywords = lang_processor.removeUnwantedWords(user_input_array)
resp = database.isCorrectAnswer(keywords, conn)
if (resp == True):
response = "yes"
else:
response = "no"
elif (ques in "request appointment"):
response = sendemail.SendEmail()
elif (ques in "share cmpe273 greensheet"):
attachments = [{
"fallback":
"Greensheet Cmpe 273 spring 2017 semester - https://www.dropbox.com/sh/qrzvf659cw2k4uv/AACrelHpOwJTX88TDN-PQ8o9a?dl=0&preview=cmpe273-greensheet.docx",
"pretext": "Cmpe 273 Greensheet",
"title": "Cmpe 273 Spring 2017 Semester Greensheet",
"title_link":
"https://www.dropbox.com/sh/qrzvf659cw2k4uv/AACrelHpOwJTX88TDN-PQ8o9a?dl=0&preview=cmpe273-greensheet.docx",
"color": "#36a64f",
}]
response = "click on this link for greensheet"
sara.send_image(config.channel, attachments)
elif (ques in "thank you thanks"):
response = "You are welcome! :thumbsup:"
elif (ques in "sjsu main campus map"):
attachments = [{
"title": "Sjsu Main Campus",
"title_link": "http://www.sjsu.edu/map/docs/campus-map.pdf",
"color": "#36a64f"
}]
response = "Check out this link for main campus map!"
sara.send_image(config.channel, attachments)
elif (ques in "more clear"):
response = "sure :ok_hand:"
elif (ques in "text books references download"):
response = "Check these links to download text books! :thumbsup:"
attachments = [{
"text":
"Text Books/Readings -",
"fallback":
"text book 1 - https://books.google.com/books?id=CclkovBDqJkC&printsec=frontcover&source=gbs_ge_summary_r&cad=0#v=onepage&q&f=false",
"fields": [{
"title":
"Web Services, by Gustavo Alonso, Fabio Casati, Harumi Kuno and Vijay Machiraju (2003) ",
"value":
"<https://books.google.com/books?id=CclkovBDqJkC&printsec=frontcover&source=gbs_ge_summary_r&cad=0#v=onepage&q&f=false|Download here>",
"short": True
}, {
"title":
"Enterprise Integration Patterns, by Gregor Hohpe and Bobby Woolf (2003)",
"value":
"<http://ptgmedia.pearsoncmg.com/images/9780321200686/samplepages/0321200683.pdf|Download here>",
"short": True
}, {
"title":
"Restful Web Services, by Leonard Richardson, Sam Ruby and David Hansson (2007)",
"value":
"<https://www.crummy.com/writing/RESTful-Web-Services/RESTful_Web_Services.pdf|Download here>",
"short": True
}],
"color":
"#F35A00"
}]
sara.send_image(config.channel, attachments)
elif (ques in "great day"):
response = "Yes,Thanks. Wish you a Wonderful Day. :ok_hand:"
elif (ques in "please improve your answers"):
response = "Yes..sure, I will definitely improve them. :ok_hand:"
elif (ques in "are you feeling good today?"):
response = "Hi! I'm good..How are you feeling this week?"
elif (ques in "amazing answers"):
response = "Thanks! I know im so good at it!. :heart_eyes:"
elif (ques in "bye tata cya see you later"):
response = "Bye! Adios! Have a good day! :wave: "
elif (ques in "help :rolling_eyes:"):
response = "Sure! What help do u need? "
elif (ques in "slack manual assistance error problem"):
attachments = [{
"fallback":
"help with slack - https://get.slack.help/hc/en-us",
"pretext": "Slack help center",
"title": "Hi, How can we help?",
"title_link": "https://get.slack.help/hc/en-us",
"color": "#36a64f"
}]
response = "Check out this link for help center!"
sara.send_image(config.channel, attachments)
elif (ques in "improve"):
response = "Yes..sure, I will definitely improve them.:thumbsup: "
else:
response = "I'm sorry, I don't understand! Sometimes I have an easier time with a few simple keywords.\n "
return response
('My boss is horrible.', 'neg')]
test = [
('The beer was good.', 'pos'),
('I do not enjoy my job', 'neg'),
("I ain't feeling dandy today.", 'neg'),
("I feel amazing!", 'pos'),
('Gary is a friend of mine.', 'pos'),
("I can't believe I'm doing this.", 'neg')]
cl = NaiveBayesClassifier(train)
# Classify some text
# print(cl.classify("Their burgers are amazing.")) # "pos"
# print(cl.classify("I don't like their pizza.")) # "neg"
# Classify a TextBlob
blob = TextBlob("The beer was amazing. But the hangover was horrible. "
"My boss was not pleased.", classifier=cl)
print(blob)
print(blob.classify())
for sentence in blob.sentences:
print(sentence)
print(sentence.classify())
# Compute accuracy
print("Accuracy: {0}".format(cl.accuracy(test)))
# Show 5 most informative features
cl.show_informative_features(5)
def SA():
r = Rake()
# Opens file and reads in training data
# NB classifier trains using the read in data
with open("datasets/trainingData.csv", 'r') as trainingdata:
classifier = NaiveBayesClassifier(trainingdata, format="csv")
print("Training Data")
classifier.show_informative_features(15)
# Opens file and reads in testing data
# Prints testing data accuracy
# Not needed for final product
with open("datasets/testingData.csv", 'r') as testingdata:
print("Testing data accuracy", classifier.accuracy(testingdata))
# Asks for user input
userInput = input("Please provide a test input: ")
# Removes all non letter characters
regex = re.compile('[^a-zA-Z ]')
punctuationRemoved = regex.sub('', userInput)
print("Punctuation removed: ", punctuationRemoved)
# Defines stopwords
stop_words = set(stopwords.words('english'))
# Takes user input, removes stopwords
word_tokens = word_tokenize(punctuationRemoved)
# Creates list size based on number of words left after stop words are removed
filtered_sentence = [w for w in word_tokens if not w in stop_words]
# Initialize empty list
filtered_sentence = []
# Appends each word to end of list
# Runs for as many words are stored in word_tokens
for w in word_tokens:
# If word is not in stop_words, append to end of list
if w not in stop_words:
filtered_sentence.append(w)
# Prints list to see new sentence with stopwords removed
print("Stopwords removed: ", filtered_sentence)
# Converts the filtered stop word sentence to string
stringWithoutStopwords = ' '.join(
[str(elem) for elem in filtered_sentence])
# Extracts keywords from the filtered sentence
r.extract_keywords_from_text(stringWithoutStopwords)
# Ranks the keywords that have been extracted
ranked_phrases = r.get_ranked_phrases()
print("Keywords extracted: ", ranked_phrases)
# Converts extracted keywords list to string
listToStr = ' '.join([str(elem) for elem in ranked_phrases])
# Runs string through trained NB classifier
finalString = TextBlob(listToStr, classifier=classifier)
# Print string followed by classification
print("String followed by classification: ", finalString,
finalString.classify())
if finalString.classify() == ("pos"):
binaryClassify = 1
else:
binaryClassify = 0
print(binaryClassify)
def vk_clasterization():
users = UserSettings.objects.all()
vk_info_list = []
print('selecting users...')
with open('trainmusic.csv', 'r') as fp:
cl = NaiveBayesClassifier(fp, format="csv")
for u in users:
query_string = u.user.first_name + ' ' + u.user.last_name
print(query_string)
query_array = api.users.search(
q=query_string,
city=119,
age_from=u.age,
age_to=u.age,
count=1,
fields=['universities', 'occupation', 'military', 'music'])
#query_array = api.users.search(q=query_string, count=1, fields=['universities', 'occupation', 'military', 'movies', 'music', 'tv', 'personal'])
print(query_array)
time.sleep(2)
user_vector = []
has_university = 0
has_military = 0
which_music = 0
which_movies = ""
which_occupation = 0
how_many_followers = 0
try:
try:
info = query_array[1]
if 'universities' in info:
has_university = 1
if 'military' in info:
if len(info['military']) > 0:
has_military = 1
if 'occupation' in info:
if info['occupation']['type'] == 'work':
which_occupation = 1
elif info['occupation']['type'] == 'university':
which_occupation = 2
elif info['occupation']['type'] == 'school':
which_occupation = 3
followers_array = api.users.getFollowers(user_id=info['uid'])
if followers_array['count'] < 100:
how_many_followers = 0
elif followers_array['count'] >= 100 and followers_array[
'count'] < 500:
how_many_followers = 1
elif followers_array['count'] >= 500 and followers_array[
'count'] < 1000:
how_many_followers = 2
elif followers_array['count'] >= 1000 and followers_array[
'count'] < 10000:
how_many_followers = 3
elif followers_array['count'] >= 10000 and followers_array[
'count'] < 100000:
how_many_followers = 4
elif followers_array['count'] >= 100000 and followers_array[
'count'] < 500000:
how_many_followers = 5
elif followers_array['count'] >= 500000:
how_many_followers = 6
if 'music' in info:
blob = TextBlob(info['music'].lower(), classifier=cl)
music_answer = blob.classify()
print(query_string)
print(music_answer)
if music_answer == "classic":
which_music = 1
elif music_answer == "rock":
which_music = 2
elif music_answer == "ruspop":
which_music = 3
elif music_answer == "pop":
which_music = 4
except KeyError:
pass
user_vector = [
has_university, has_military, which_music, which_occupation,
how_many_followers
]
except IndexError:
user_vector = [
has_university, has_military, which_music, which_occupation,
how_many_followers
]
print(user_vector)
vk_info_list.append(user_vector)
print(vk_info_list)
return vk_info_list
