def __init__(self):
self.root = tk.Tk()
#create a Frame for Text and Scrollbar
text_frame = tk.Frame(self.root,width=900,height=600)
text_frame.pack(fill="both",expand=True)
text_frame.grid_propagate(False)
text_frame.grid_rowconfigure(0,weight=1)
text_frame.grid_columnconfigure(0,weight=1)
#create a Text widget
self.text = tk.Text(text_frame,borderwidth=3,relief="sunken")
self.text.config(font=("consolas",12),undo = True,wrap='word')
self.text.grid(row=0,column=0,sticky="nsew",padx=2,pady=2)
cnt = 0
for tweet in get_tweets():
self.text.insert('1.0',tweet['text']+'\n')
cnt = cnt + 1
if cnt == 50 :
break
#create a Scrollbar and connect it with text
scrollbar = tk.Scrollbar(text_frame,command=self.text.yview)
scrollbar.grid(row=0,column=1,sticky='nsew')
self.text['yscrollcommand'] = scrollbar.set
def user_graph(directory):
'''
A function to build a graph based on retweets. Every vertex is a
different tweet and every edge connects a tweet with a retweet.
directory: the directory with all the tweets in it
Returns: a graph of the tweet-retweet relationship
'''
g = grapher.Graph() # the dictionary to return
# Get all the tweets in the directory
tweets = get_tweets(directory)
for tweet in tweets:
try:
# treat it first like a retweet
retweet = tweet["retweeted_status"]
# add the edge with id string of the tweets as the id of the
# vertices and the screen names as the data of the vertices
g.addEdge(retweet["id_str"],
tweet["id_str"],
f_data=retweet["user"]["screen_name"],
t_data=tweet["user"]["screen_name"])
except KeyError:
# not a retweet because the retweeted status not been found!
# add the tweet to the relationship dictionary
g.addVertex(tweet["id_str"], tweet["user"]["screen_name"])
return g
def build_documents_country(directory):
'''
A function to build the documents separated by country
to get a TDM from a directory
directory: a directory with Flume files
'''
documents = dict()
resolver = carmen.get_resolver()
resolver.load_locations()
raw_tweets = get_tweets(directory)
for tweet in raw_tweets:
country = str()
try:
country = resolver.resolve_tweet(tweet)[1].country
except TypeError:
continue
text = clean_text(tweet["text"])
try:
documents[country] += text
except KeyError:
documents[country] = text
return documents
def get_user_tweets(uid, limit=None, offset=0):
"""Return specified user tweets limited by limit and offset."""
if limit is None:
limit = config.DEFAULT_LIMIT
else:
limit = min(config.MAX_LIMIT, limit)
tids = conn.zrange(utils.get_user_tweets_key(uid),
offset, offset + limit - 1)
return tweets.get_tweets(tids)
def get_timeline(uid, limit=None, offset=0):
"""Return sepcified user timeline limited by limit and offset."""
if limit is None:
limit = config.DEFAULT_LIMIT
else:
limit = min(config.MAX_LIMIT, limit)
refresh_timeline(uid)
tids = conn.zrevrange(utils.get_timeline_key(uid),
offset, offset + limit - 1)
return tweets.get_tweets(tids)
def tweets_page():
q = request.args.get('q') or ''
tweet_list = []
if q:
tweet_list = tweets.get_tweets(q)
sent_list = [
sentiment.get_text_category(STATE, tweet) for tweet in tweet_list
]
pairs = zip(tweet_list, sent_list)
return render_template('tweets.html', q=q, pairs=pairs)
def tokenize_tweets(user):
"""Returns a list of all useful tokenized tweets."""
raw_text = ''
for tweet in get_tweets(user):
raw_text += tweet.text + " "
words = raw_text.split(" ")
words = [word for word in words if not LINKS_RT.search(word)]
words = " ".join(words)
words = [word for word in wordpunct_tokenize(words)]
words = [word for word in words if word not in stop_words()]
words = [word.lower() for word in words if len(word) > 4]
return words
def words_handler():
form = request.form.to_dict()
screen_name = form['screen_name']
recent_tweets = tweets.get_tweets(screen_name)
tokens = tweets.tokenize_tweets(recent_tweets)
result = []
for token, size in tweets.top_tokens(tokens):
data = {
'text': token.upper(),
'size': size
}
result.append(data)
return Response(json.dumps(result), mimetype='application/json', headers={'Cache-Control': 'no-cache'})
def build_documents_user(directory):
'''
A function to build the documents separated by user
to get a TDM from a directory
directory: a directory with Flume files
'''
documents = dict()
raw_tweets = get_tweets(directory)
for tweet in raw_tweets:
text = clean_text(tweet["text"])
try:
documents[tweet["user"]["screen_name"]] += text
except KeyError:
documents[tweet["user"]["screen_name"]] = text
return documents
def build_graph(directory):
'''
A function to build a table based on retweets
directory: the directory with all the tweets in it
Return: a dictionary with Tweet ID's for keys that map to a tuple of the
form (username, relationship)
relationship is an integer (1-2). 1 = author, 2 = retweeter
'''
g = nx.DiGraph() # the one graph to rule them all
# Get all the tweets in the directory
raw_tweets = get_tweets(directory)
# Append location data to tweets
tweets = add_positions(raw_tweets)
# Add all the users as nodes with the position data from the first tweet
# of theirs that is discovered
userlocs = list()
for tweet in tweets:
username = tweet["user"]["screen_name"]
if username not in userlocs:
userlocs.append(username)
g.add_node(username, pos=tweet["position"])
for tweet in tweets:
try:
# treat if first like a retweet
head = tweet["retweeted_status"]["user"]["screen_name"]
tail = tweet["user"]["screen_name"]
g.add_edge(head, tail)
g.edges[head, tail]['tweet'] = tweet["retweeted_status"]
except KeyError:
# it's not a retweet, so don't add any edges
pass
return g
def show_tweets(request):
scr_name= request.POST.get("screen_name")
twt_count = int(request.POST.get("tweet_count"))
stat_count = int(request.POST.get("stat_count"))
try:
user = User.objects.get(scr_name=scr_name)
tweets = [tweet.tweet_text for tweet in user.tweet_set.all()]
hash_tags, mentions = get_hash_mentions(tweets[:twt_count], stat_count)
tweets = tweets[:twt_count]
return result(request, tweets, hash_tags, mentions)
except User.DoesNotExist:
tweets = get_tweets(scr_name, MAX_TWEET)
if 'error' in tweets:
retry = '<br><a href="/timeline/get_input">Try another user!</a>'
return HttpResponse(tweets['error']+retry)
tweets = extract_tweets(tweets)
user = User(scr_name=scr_name)
user.save()
for tweet in tweets:
user.tweet_set.create(tweet_text=tweet)
hash_tags, mentions = get_hash_mentions(tweets[:twt_count], stat_count)
return result(request, tweets[:twt_count], hash_tags, mentions)
'''
Count
A script to count the english and non-english tweets from flume files
Author: Karsten Ladner
Date: 7/12/2018
'''
import sys
from os.path import join as jn
from tweets import get_tweets
from config import PROC, OUT, PATH
if __name__ == '__main__':
out = jn(OUT, sys.argv[2])
dirs = sys.argv[1].split(',')
with open(out, 'a') as fout:
for d in dirs:
tweets = get_tweets([d], prefix=PATH)
print("{0},{1},{2}".format(d, len(tweets[0]), tweets[1]))
fout.write("{0},{1},{2}".format(d, len(tweets[0]), tweets[1]))
fout.close()
sc = afinn.score(tweet['text'])
total += sc
if sc < _min:
_min = sc
if sc > _max:
_max = sc
if sc < 0:
neg += 1
elif sc > 0:
pos += 1
elif sc == 0:
neut += 1
return (total, _min, _max, neg, neut, pos)
if __name__ == '__main__':
out = jn(OUT, sys.argv[2])
dirs = sys.argv[1].split(',')
with open(out, 'w') as fout:
for d in dirs:
tweets = get_tweets([d], key='text', prefix=PATH)
sent, total, mn, mx, neg, neut, pos = get_sentiment(tweets[0])
print("Sentiment Calculated")
fout.write("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\n".format(
d, sent, mn, mx, neg, neut, pos, total, tweets[1]))
fout.close()
def _query():
keywords = request.args.get('keywords', "default", type=str)
tweets = get_tweets(keywords)
return jsonify(tweets=tweets)
'''
Usernames
A script to save the usernames of the users for each day
'''
import sys
from tweets import get_tweets, save_to_file as save
from config import PATH
if __name__ == '__main__':
dirs = sys.argv[1].split(',')
for d in dirs:
out = "./out/users-" + d.replace("/", "_") + ".dat"
tweets = get_tweets([d], key='names', prefix=PATH)
data = {'data': tweets[0], 'num_filtered': tweets[1]}
save(data, out)
from predict import predict
from tweets import get_tweets
import json
import os
import torch
import argparse
if __name__ == "__main__":
my_parser = argparse.ArgumentParser()
my_parser.version = '1.0'
my_parser.add_argument("search_term", help="The search to be used in order to retrieve tweets")
my_parser.add_argument('-n','--num_tweets', action='store',type=int,default=500,help='No of tweets to be retrieved, must be less than 1000')
args = my_parser.parse_args()
search_term = args.search_term
no_of_tweets = args.num_tweets
if not os.path.isfile('twitterAPIkeys.json'):
raise FileNotFoundError('Please make sure a \'twitterAPIkeys.json\' exists in the src folder!')
with open('twitterAPIkeys.json') as f:
APIkeys_dict = json.load(f)
tweet_df = get_tweets(search_term, no_of_tweets, APIkeys_dict, retweets=False)
inputs = list(tweet_df['tweet_text'].values)
outputs = predict(inputs)
outputs_2d = tsne_cpu(outputs)
# print(outputs_2d)
tweet_df['x_coord'] = outputs_2d[:,0]
tweet_df['y_coord'] = outputs_2d[:,1]
visualise(tweet_df)
def evaluatePositions(self):
openTrades = []
for trade in self.trades:
if (trade.status == "OPEN"):
openTrades.append(trade)
if (len(openTrades) < self.numSimulTrades):
if (self.btc_historical_total <= 100000):
bitcoin_query = 'BTC OR Bitcoin OR $BTC'
btc_historical_tweets, self.btc_sinceid = tweets.get_tweets(
50, self.btc_sinceID, bitcoin_query)
btc_total_score, btc_positive, btc_negative, btc_total = tweets.classify(
btc_historical_tweets)
self.btc_historical_positive = self.btc_historical_positive + btc_positive
self.btc_historical_negative = self.btc_historical_negative + btc_negative
self.btc_historical_score = self.btc_historical_score + btc_total_score
self.btc_historical_total = self.btc_historical_total + btc_total
self.btc_historical_percent = (self.btc_historical_positive /
self.btc_historical_total) * 100
ethereum_query = 'Ethereum OR ETH OR $ETH'
eth_historical_tweets, self.eth_sinceID = tweets.get_tweets(
50, self.eth_sinceID, ethereum_query)
eth_total_score, eth_positive, eth_negative, eth_total = tweets.classify(
eth_historical_tweets)
self.eth_historical_positive = self.eth_historical_positive + eth_positive
self.eth_historical_negative = self.eth_historical_negative + eth_negative
self.eth_historical_score = self.eth_historical_score + eth_total_score
self.eth_historical_total = self.eth_historical_total + eth_total
self.eth_historical_percent = (self.eth_historical_positive /
self.eth_historical_total) * 100
if self.btc_historical_total >= 100000:
print '\033[1m' + "Historical Tweets Analyzed"
print "historical btc percent: " + str(
self.btc_historical_percent)
print "historical eth percent: " + str(
self.eth_historical_percent)
print '\033[0m'
elif (self.btc_historical_total > 100000):
bitcoin_query = 'BTC OR Bitcoin OR $BTC'
btc_tweets, sinceid_recent = tweets.get_tweets(
50, 0, bitcoin_query)
btc_total_score2, btc_positive2, btc_negative2, btc_total2 = tweets.classify(
btc_tweets)
btc_percent = (btc_positive2 / btc_total2) * 100
ethereum_query = 'Ethereum OR ETH OR $ETH'
eth_tweets, sinceid_recent = tweets.get_tweets(
50, 0, ethereum_query)
eth_total_score2, eth_positive2, eth_negative2, eth_total2 = tweets.classify(
eth_tweets)
eth_percent = (eth_positive2 / eth_total2) * 100
if (eth_percent > 1.042 * self.eth_historical_percent
or btc_percent < 0.943 * self.btc_historical_percent):
if btc_percent < 0.943 * self.btc_historical_percent:
self.btc_sentiments = []
self.btc_trading_percent = btc_percent
self.type_of_trade = 'BTC'
self.trades.append(BotTrade(self.prices,
stopLoss=0.01))
elif (eth_percent > 1.042 * self.eth_historical_percent):
self.eth_sentiments = []
self.eth_trading_percent = eth_percent
self.type_of_trade = 'ETH'
self.trades.append(BotTrade(self.prices,
stopLoss=0.01))
else:
self.type_of_trade = ''
time.sleep(60 * 5)
for trade in openTrades:
if (self.type_of_trade == 'BTC'):
bitcoin_query = 'BTC OR Bitcoin OR $BTC'
btc_tweets, sinceid_recent = tweets.get_tweets(
10, 0, bitcoin_query)
btc_total_score2, btc_positive2, btc_negative2, btc_total2 = tweets.classify(
btc_tweets)
btc_percent = (btc_positive2 / btc_total2) * 100
self.btc_sentiments.append(btc_percent)
if (len(self.btc_sentiments) > 5):
mean_sentiment = np.mean(self.btc_sentiments)
std_sentiment = np.std(self.btc_sentiments)
if btc_percent >= mean_sentiment + (
(0.800 * std_sentiment) /
math.sqrt(len(self.btc_sentiments))):
price = self.api.getticker('BTC-ETH')
self.currentClose = price['Bid']
trade.close(self.currentClose)
else:
time.sleep(60 * 3)
elif (self.type_of_trade == 'ETH'):
ethereum_query = 'Ethereum OR ETH OR $ETH'
eth_tweets, sinceid_recent = tweets.get_tweets(
10, 0, ethereum_query)
eth_total_score2, eth_positive2, eth_negative2, eth_total2 = tweets.classify(
eth_tweets)
eth_percent = (eth_positive2 / eth_total2) * 100
self.eth_sentiments.append(eth_percent)
if (len(self.eth_sentiments) > 5):
mean_sentiment = np.mean(self.eth_sentiments)
std_sentiment = np.std(self.eth_sentiments)
if eth_percent <= mean_sentiment - (
(0.674 * std_sentiment) /
math.sqrt(len(self.eth_sentiments))):
price = self.api.getticker('BTC-ETH')
self.currentClose = price['Bid']
trade.close(self.currentClose)
else:
time.sleep(60 * 3)
else:
price = self.api.getticker('BTC-ETH')
self.currentClose = price['Bid']
trade.close(self.currentClose)
def testNumberRequest(self):
from tweets import get_tweets
t = get_tweets(name='JennyJarv', count=7)
self.assertTrue(len(t) == 7)
def testGetAnythingFromTwitter(self):
alstweets = tweets.get_tweets(name='allieehenry')
self.assertTrue(len(alstweets) >= 1)
def testNoName(self):
t = tweets.get_tweets()
self.assertTrue(t is None)
import extract
import tags
import tweets
if __name__ == "__main__":
print("Don't forget to chcp 65001") # this is a reminder to me when I run this program from command line
print('Enter your query')
query = input()
list_of_tags = tags.get_tags(query)
list_of_tweets = tweets.get_tweets(list_of_tags)
result = extract.get_codes(list_of_tweets)
for code in result:
print (code)
r = extract.get_likely_tweets(list_of_tweets)
for t in r:
print(t)
for col in row:
out += str(col) + "\t"
out.rstrip("\t")
out += "\n"
fout.write(out)
fout.close()
if __name__ == '__main__':
if sys.argv[1] == "save":
g = build_graph(sys.argv[2])
parts = sys.argv[2].strip(".").split("/")
fileparts = [p + "-" for p in parts[1:] if p is not "crisis"]
filename = "".join(fileparts) + "data"
save_graph(g, filename)
elif sys.argv[1] == "show":
g = load_graph(sys.argv[2])
display_graph(g)
elif sys.argv[1] == "print":
tweets = get_tweets(sys.argv[2].split(','))
g = user_graph(tweets)
print_user_graph(g)
elif sys.argv[1] == "mout":
tweets = get_tweets_from_dirs(sys.argv[2].split(','),
prefix='./crisis/crisis/2018/06/')
g = user_graph(tweets)
write_file(g, sys.argv[3])
else:
g = build_graph("./data/25crisis")
print("Please enter either 'save','show', or 'print'")
def tweets(username, count=2):
return get_tweets(username, count)
# print the first row (which is all the usernames)
# get the list of users and sort the keys alphabetically
for name in users:
# fancy print all the screen_names of the users
# truncate the name to 10 characters and right align at 15 chars
print("{0:>15.10}".format(name), end="")
for tw in graph.getRoots():
# fancy print the screen_name of the user who tweeted
# truncate it at 10 characters and right align it at 10 chars
print("\n{0:>10.10}".format(tw.getData()), end=" " * 5)
for value in matrix[tweet_indices[tw.getId()]]:
# fancy print every value in the list associated with this
print("{0:<15}".format(value), end="")
if __name__ == '__main__':
if sys.argv[1] == "save":
dirs = [pjoin(PATH, d) for d in sys.argv[2].split(',')]
tweets = get_tweets(dirs)
g = build_graph(tweets[0])
save_graph(g, sys.argv[3])
elif sys.argv[1] == "show":
g = load_graph(sys.argv[2])
display_graph(g)
elif sys.argv[1] == "print":
g = user_graph(sys.argv[2])
print_user_graph(g)
else:
print("Please enter either 'save','show', or 'print'")
def tweets(username, count=2):
return get_tweets(username, count)
from PIL import Image, ImageFont, ImageDraw
import tweets
tweets = tweets.get_tweets()
for i, tweet in enumerate(tweets):
base = Image.open("boat.jpg").convert('RGBA')
fnt = ImageFont.truetype('Impact.ttf', 30)
txt = Image.new('RGBA', base.size, (255, 255, 255, 0))
# The greatest line wrapping function
tweetlines = [tweet[:32], tweet[32:64], tweet[64:96], tweet[96:]]
d = ImageDraw.Draw(txt)
d.text((10, 32), tweetlines[0], font=fnt, fill=(255, 255, 255, 255))
d.text((10, 64), tweetlines[1], font=fnt, fill=(255, 255, 255, 255))
d.text((10, 128), tweetlines[2], font=fnt, fill=(255, 255, 255, 255))
d.text((10, 256), tweetlines[3], font=fnt, fill=(255, 255, 255, 255))
out = Image.alpha_composite(base, txt)
out.save('output/{}'.format(i), 'JPEG')
