def GET(self):
query = web.ctx.get('query')
html = html_helper.HTMLHelper()
twitterData = get_twitter_data.TwitterData()
if query:
if(query[0] == '?'):
query = query[1:]
arr = query.split('&')
logging.warning(arr)
#default values
time = 'daily'
for item in arr:
if 'keyword' in item:
keyword = item.split('=')[1]
elif 'method' in item:
method = item.split('=')[1]
elif 'time' in item:
time = item.split('=')[1]
#end loop
if(method != 'baseline' and method != 'naivebayes' and method != 'maxentropy' and method != 'svm'):
return html.getDefaultHTML(error=2)
tweets = twitterData.getTwitterData(keyword, time)
if(tweets):
if(method == 'baseline'):
bc = baseline_classifier.BaselineClassifier(tweets, keyword, time)
bc.classify()
return bc.getHTML()
elif(method == 'naivebayes'):
trainingDataFile = 'data/training_neatfile_2.csv'
#classifierDumpFile = 'data/naivebayes_trained_model.pickle'
classifierDumpFile = 'data/test/naivebayes_test_model.pickle'
trainingRequired = 0
nb = naive_bayes_classifier.NaiveBayesClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
nb.classify()
return nb.getHTML()
elif(method == 'maxentropy'):
trainingDataFile = 'data/training_neatfile.csv'
classifierDumpFile = 'data/maxent_trained_model.pickle'
trainingRequired = 0
maxent = max_entropy_classifier.MaxEntClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
maxent.classify()
return maxent.getHTML()
elif(method == 'svm'):
trainingDataFile = 'data/training_neatfile.csv'
classifierDumpFile = 'data/svm_trained_model.pickle'
trainingRequired = 0
sc = libsvm_classifier.SVMClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
sc.classify()
return sc.getHTML()
else:
return html.getDefaultHTML(error=1)
else:
return html.getDefaultHTML()
def search():
keyword = request.args.get('keyword')
method = request.args.get('method')
time = 'daily'
time = request.args.get('time')
html = html_helper.HTMLHelper()
#print html
twitterData = get_twitter_data.TwitterData()
#print twitterData
if keyword:
if(method != 'baseline' and method != 'naivebayes' and method != 'maxentropy'):
return html.getDefaultHTML(error=2)
ACCESS_TOKEN = session['oauth_token']
ACCESS_TOKEN_SECRET = session['oauth_secret']
tweets = twitterData.getTwitterData(keyword, time,CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
print tweets,"-------------------------"
if(tweets):
if(method == 'baseline'):
bc = baseline_classifier.BaselineClassifier(tweets, keyword, time)
bc.classify()
return bc.getHTML()
elif(method == 'naivebayes'):
trainingDataFile = 'data/training_neatfile_2.csv'
classifierDumpFile = 'data/naivebayes_trained_model.pickle'
#classifierDumpFile = 'data/test/naivebayes_test_model.pickle'
trainingRequired = 0
nb = naive_bayes_classifier.NaiveBayesClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
nb.classify()
return nb.getHTML()
elif(method == 'maxentropy'):
trainingDataFile = 'data/training_neatfile.csv'
classifierDumpFile = 'data/maxent_trained_model.pickle'
trainingRequired = 0
maxent = max_entropy_classifier.MaxEntClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
maxent.classify()
return maxent.getHTML()
else:
return html.getDefaultHTML(error=1)
else:
return html.getDefaultHTML()
def submit(keyword,time):
print "Analysing "+keyword+ " for "+time+" : \n"
twitterData = get_twitter_data.TwitterData()
tweets = twitterData.getTwitterData(keyword, time)
print "The collected tweets are : \n"
trainingDataFile = 'data/training_neatfile.csv'
classifierDumpFile = 'data/maxent_trained_model.pickle'
trainingRequired = 0
maxent = max_entropy_classifier.MaxEntClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
maxent.classify()
val,val2,time,pos_count,neg_count,neut_count=maxent.print_value()
items=len(val2)-1
for i in range(0,items):
print val[i]+ " : " + val2[i]+ "\n"
print "The positive count be : "+str(pos_count)
print "The negative count be : "+str(neg_count)
print "The neutral count be : "+str(neut_count)
return 0
def submit():
try:
keyword=request.form['yourname']
time=request.form['options']
twitterData = get_twitter_data.TwitterData()
tweets = twitterData.getTwitterData(keyword, time)
classifier = request.form['c']
if classifier=="maxent":
print "Maxent chosen"
trainingDataFile = 'data/training_neatfile.csv'
classifierDumpFile = 'data/maxent_trained_model.pickle'
trainingRequired = 0
maxent = max_entropy_classifier.MaxEntClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
maxent.classify()
val,val2,time,pos_count,neg_count,neut_count=maxent.print_value()
pos_tweet,neg_tweet,neut_tweet=process(val,val2)
print "maxent finished"
else:
trainingDataFile = 'data/training_neatfile.csv'
classifierDumpFile = 'data/svm_trained_model.pickle'
trainingRequired = 0
sc = libsvm_classifier.SVMClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
sc.classify()
print "classified"
val,val2,time,pos_count,neg_count,neut_count=sc.print_value()
pos_tweet,neg_tweet,neut_tweet=process(val,val2)
res=str(pos_count)+" "+str(neut_count)+" "+str(neg_count)
count = dbcon.Searchresults.select().count()
if time == 'today':
dbcon.Searchresults.create(time = get_time('today'),search_id=count+1,search_keyword=keyword,search_result=res,classifier_used=str(classifier))
return render_template('form_action.html', name=keyword, option=get_time(time), pos_count=pos_count, neg_count=neg_count, neut_count=neut_count, pos_tweet=pos_tweet, neg_tweet=neg_tweet, neut_tweet=neut_tweet)
elif time == 'lastweek':
dbcon.Searchresults.create(time = get_time('week'),search_id=count+1,search_keyword=keyword,search_result=res,classifier_used=str(classifier))
return render_template('form_action_weekly.html', name=keyword, option=get_time(time), pos_count=pos_count, neg_count=neg_count, neut_count=neut_count, pos_tweet=pos_tweet, neg_tweet=neg_tweet, neut_tweet=neut_tweet)
else:
return render_template('form_submit.html',sorry="T")
except:
return render_template('form_submit.html',sorry="Y")
def GET(self):
query = web.ctx.get('query')
html = html_helper.HTMLHelper()
twitterData = get_twitter_data.TwitterData()
if query:
if (query[0] == '?'):
query = query[1:]
arr = query.split('&')
logging.warning(arr)
#default values
time = 'daily'
for item in arr:
if 'keyword' in item:
keyword = item.split('=')[1]
elif 'time' in item:
time = item.split('=')[1]
#end loop
tweets = twitterData.getTwitterData(keyword, time)
if (tweets):
trainingDataFile = 'fulltrainingdataset-csv.csv'
#classifierDumpFile = 'data/naivebayes_trained_model.pickle'
classifierDumpFile = 'my_final_classifier2.pickle'
trainingRequired = 0
nb = naive_bayes_classifier.NaiveBayesClassifier(tweets, keyword, time, \
trainingDataFile, classifierDumpFile, trainingRequired)
nb.classify()
#nb.main()#added for testing
#nb.accuracy()
return nb.getHTML()
else:
return html.getDefaultHTML(error=1)
else:
return html.getDefaultHTML()
print "-------------------------------------------------"
response = raw_input("Please enter Keyword: ")
while not response:
response = raw_input("Please enter Keyword: ")
# Get Tweets
keyword = '$'+response
# time = 'today'
time = 'lastweek'
print "Fetch twitter data for "+ response+" company keyword...."
twitterData = get_twitter_data.TwitterData('2016-04-18')
tweets = twitterData.getTwitterData(keyword, time)
print "Twitter data fetched \n"
print "Fetch yahoo finance data for "+response+" given company keyword.... "
keyword2 = response
yahooData = get_yahoo_data.YahooData('2016-04-01', "2016-04-17")
historical_data = yahooData.getYahooData(keyword2)
yahoo_open_price = {}
yahoo_close_price = {}
yahoo_high_price = {}
yahoo_low_price = {}
for i in range(len(historical_data)):
date = historical_data[i]['Date'].replace(",","")
yahoo_open_price.update({date: historical_data[i]['Open']})
#!/usr/bin/env python import get_twitter_data ## PLACE YOUR CREDENTIALS in config.json file or run this file with appropriate arguments from command line keyword = 'Donald Trump' time = 'today' twitterData = get_twitter_data.TwitterData() tweets = twitterData.getTwitterData(keyword, time) print tweets
print "-------------------------------------------------"
print "-------------------------------------------------"
response = raw_input("Please enter Keyword: ")
while not response:
response = raw_input("Please enter Keyword: ")
# Get Tweets
keyword = '$' + response
# time = 'today'
time = 'lastweek'
print "Fetch twitter data for " + response + " company keyword...."
twitterData = get_twitter_data.TwitterData('2017-05-23')
tweets = twitterData.getTwitterData(keyword, time)
print "Twitter data fetched \n"
print "Fetch yahoo finance data for " + response + " given company keyword.... "
keyword2 = response
quandl.ApiConfig.api_key = "nFXzo5mjYxKTdFvxmx1E"
# yahooData = get_yahoo_data.YahooData('2017-05-14', "2017-05-18")
# historical_data = yahooData.getYahooData(keyword2)
historical_data = quandl.get('WIKI/' + keyword2,
start_date="2017-05-10",
end_date="2017-05-23",
returns="numpy")
yahoo_open_price = {}
yahoo_close_price = {}
keyword = response
response = '$' + keyword
print(response)
print(keyword)
# Get Tweets
# keyword = '$'+response
# time = 'today'
# time = 'lastweek'
# time = 'month'
time = 'none'
print("Fetch Twitter data matching " + response)
twitterData = get_twitter_data.TwitterData('2019-02-20')
tweets = twitterData.getTwitterData(response, time)
print("tweets: ")
print("Twitter data fetched \n")
# print ("Fetch Yahoo finance data for "+response+" given company keyword.... ")
# keyword2 = response
# yahooData = get_yahoo_data.YahooData('2019-02-11', "2017-02-15")
# historical_data = yahooData.getYahooData(keyword2)
# yahoo_open_price = {}
# yahoo_close_price = {}
# yahoo_high_price = {}
# yahoo_low_price = {}
# for i in range(len(historical_data)):
# date = historical_data[i]['Date'].replace(",","")
print "-------------------------------------------------"
print "-------------------------------------------------"
response = raw_input("Please enter Keyword: ")
while not response:
response = raw_input("Please enter Keyword: ")
# Get Tweets
keyword = '$' + response
# time = 'today'
time = 'lastweek'
print "Fetch twitter data for " + response + " company keyword...."
twitterData = get_twitter_data.TwitterData('2017-08-26')
tweets = twitterData.getTwitterData(keyword, time)
print "Twitter data fetched \n"
#Get input from yahoo
print "Fetch yahoo finance data for " + response + " given company keyword.... "
keyword2 = '^IXIC'
nasdaq = pdr.get_data_yahoo(keyword2, start, end)
nasdaq.columns.values[-2] = 'AdjClose'
nasdaq.columns = nasdaq.columns + '_Nasdaq'
nasdaq['Return_Nasdaq'] = nasdaq['AdjClose_Nasdaq'].pct_change()
nasdaq.columns
print nasdaq
