def open_link(self, view, frame, req, nav, pol, data=None):
print "Opening %s" % req.get_uri()
if req.get_uri() == "wasabi://compose/tweet":
self.full_content = open(tweet.template_dir+'/compose.html', 'r').read()
self.view.load_string(self.full_content, "text/html", "UTF-8", "")
return True
elif req.get_uri().startswith("wasabi://compose/reply"):
in_reply_to = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query)['reply_id'][0]
reply_text = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query)['reply_text'][0]
page = Template(open(tweet.template_dir+'/reply.html', 'r').read())
self.full_content = page.render(reply_id=in_reply_to, reply_text=reply_text)
print self.full_content
self.view.load_string(self.full_content, "text/html", "UTF-8", "")
return True
elif req.get_uri().startswith("wasabi://compose/send"):
params = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query)
text = params['tweet[text]'][0]
if 'tweet[in_reply_to]' in params:
in_reply_to = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query)['tweet[in_reply_to]'][0]
twt = tweet.Tweet(self.rest.send_tweet(status=text, in_reply_to_status_id=in_reply_to))
else:
twt = tweet.Tweet(self.rest.send_tweet(status=text))
if twt:
self.tweets[twt.data['id']] = twt
self.update_content()
self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home")
return False
elif req.get_uri() == "wasabi://home":
print view.get_uri()
#if view.get_uri() != "wasabi://home":
# self.update_content()
# self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home")
return False
else:
return self.open_external_link(view, frame, req, None, None)
def test_is_snub(self):
grats_tweets = [
tweet.Tweet("kevin spacey got snubbed!"),
tweet.Tweet("boyhood got robbed!"),
tweet.Tweet("it's a shame boyhood lost")
]
not_grats_tweets = [
tweet.Tweet("congrats to kevin spacey!")
]
for t in grats_tweets:
self.assertTrue(t.is_snub())
for t in not_grats_tweets:
self.assertFalse(t.is_snub())
def read_tweets(user, num_tweets):
tweets = []
url = "http://api.twitter.com/1/statuses/user_timeline.json?\
screen_name=%s&count=%s&include_rts=true" % (user, num_tweets)
file = urllib2.urlopen(url)
content = file.read()
json = simplejson.loads(content)
for js_tweet in json:
t = tweet.Tweet()
t.id = js_tweet['id']
t.username = js_tweet['user']['screen_name']
try:
t.retweet_user = js_tweet['retweeted_status']['user']['screen_name']
t.retweeted = True
except:
t.retweeted = False
t.set_date(js_tweet['created_at'])
t.set_tweet_url()
t.set_text(js_tweet['text'])
t.set_profile_url()
tweets.append(t)
return tweets
def test_nonames(self):
self.tweetdata['text'] = 'This is some text with no names'
tw = tweet.Tweet(self.tweetdata)
self.assertEqual(tw.parse_tweet(), [('This', 'DT'), ('is', 'VBZ'),
('some', 'DT'), ('text', 'NN'),
('with', 'IN'), ('no', 'DT'),
('names', 'NNS')])
def fetch_tweets(self, view, sw):
#print self.tweets
#print self.tweet_queue.qsize()
while True:
try:
twt = self.tweet_queue.get(False)
except Queue.Empty:
#print "%d - No tweets." % (time.time())
break
print "Got tweet."
print twt
twt = tweet.Tweet(twt)
if twt.validate():
if twt.data['id'] in self.to_delete:
self.to_delete.remove(twt.data['id'])
else:
self.tweets[twt.data['id']] = twt
self.update_content()
if view.get_uri() == "wasabi://home":
self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home")
elif 'delete' in twt.data and 'status' in twt.data['delete']:
if twt.data['delete']['status']['id'] in self.tweets:
self.tweets[twt.data['delete']['status']['id']].time_str = "[DEL]"
self.update_content()
if view.get_uri() == "wasabi://home":
self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home")
else:
self.to_delete.append(twt.data['delete']['status']['id'])
return True
def test_start_with_name(self):
self.tweetdata['text'] = 'Argo was nominated in 2013.'
tw = tweet.Tweet(self.tweetdata)
self.assertEqual(tw.parse_tweet(), [([('Argo', 'NNP')], 'NNP-phrase'),
('was', 'VBD'),
('nominated', 'VBN'), ('in', 'IN'),
('2013', 'CD'), ('.', '.')])
def test_name_with_of(self):
self.tweetdata['text'] = 'The show House of Cards was nominated.'
tw = tweet.Tweet(self.tweetdata)
self.assertEqual(tw.parse_tweet(), [('The', 'DT'), ('show', 'NN'),
([('House', 'NNP'), ('of', 'IN'),
('Cards', 'NNP')], 'NNP-phrase'),
('was', 'VBD'),
('nominated', 'VBN'), ('.', '.')])
def test_name_with_the(self):
self.tweetdata['text'] = 'I watched The Affair the other day.'
tw = tweet.Tweet(self.tweetdata)
self.assertEqual(tw.parse_tweet(),
[('I', 'PRP'), ('watched', 'VBD'),
([('The', 'DT'), ('Affair', 'NNP')], 'NNP-phrase'),
('the', 'DT'), ('other', 'JJ'), ('day', 'NN'),
('.', '.')])
def test_name_with_ofthe(self):
self.tweetdata['text'] = 'Maybe Lord of the Flies will win.'
tw = tweet.Tweet(self.tweetdata)
self.assertEqual(tw.parse_tweet(),
[('Maybe', 'RB'),
([('Lord', 'NNP'), ('of', 'IN'), ('the', 'DT'),
('Flies', 'NNPS')], 'NNP-phrase'), ('will', 'MD'),
('win', 'VB'), ('.', '.')])
def getTweetClient(config):
payload = {
'apiKey': config.TWITTER_API_KEY,
'apiSecret': config.TWITTER_API_SECRET,
'bearerToken': config.TWITTER_BEARER_TOKEN,
'proxy': config.PROXY,
'screenName': config.SCREEN_NAME,
'interval': config.INTERVAL
}
tweetClient = tweet.Tweet(**payload)
return tweetClient
def test_double_token_name(self):
self.tweetdata[
'text'] = 'Ironically, Amal Clooney may be the get of the #GoldenGlobes even though she\'s not nominated for anything. And supposedly,'
tw = tweet.Tweet(self.tweetdata)
self.assertEqual(tw.parse_tweet(),
[('Ironically', 'RB'), (',', ','),
([('Amal', 'NNP'),
('Clooney', 'NNP')], 'NNP-phrase'), ('may', 'MD'),
('be', 'VB'), ('the', 'DT'), ('get', 'NN'),
('of', 'IN'), ('the', 'DT'), ('#GoldenGlobes', 'HT'),
('even', 'RB'), ('though', 'IN'), ('she', 'PRP'),
("'s", 'VBZ'), ('not', 'RB'), ('nominated', 'VBN'),
('for', 'IN'), ('anything', 'NN'), ('.', '.'),
('And', 'CC'), ('supposedly', 'RB'), (',', ',')])
def test_multinames_with_conjunction(self):
self.tweetdata[
'text'] = '\"The Affair\" or \"House of Cards\"? Our #GoldenGlobes TV predictions: http://t.co/Wc4BXiThed http://t.co/WYp278s9F0'
tw = tweet.Tweet(self.tweetdata)
self.assertEqual(tw.parse_tweet(),
[('``', '``'),
([('The', 'DT'), ('Affair', 'NNP')], 'NNP-phrase'),
("''", "''"), ('or', 'CC'), ('``', '``'),
([('House', 'NNP'), ('of', 'IN'),
('Cards', 'NNPS')], 'NNP-phrase'), ("''", "''"),
('?', '.'), ('Our', 'PRP$'), ('#GoldenGlobes', 'HT'),
('TV', 'NN'), ('predictions', 'NNS'), (':', ':'),
('http', 'NN'), (':', ':'),
('//t.co/Wc4BXiThed', 'VBN'), ('http', 'NN'),
(':', ':'), ('//t.co/WYp278s9F0', 'NN')])
def read_clusters_sorted_by_time(file_name, named_entities_file_name):
tweets = []
clusters = {}
clusters_timeline = []
named_entities = {}
file_to_read = open(file_name, "r")
file_entities = open(named_entities_file_name, 'r')
for line in file_entities.readlines():
data = line.split(",", 1)
named_entities[data[0]] = data[1]
for line in file_to_read.readlines():
data = line.split(",")
tokens = [
en.SimpleEntity(entity)
if not (entity in named_entities.get(data[2])) else
en.NamedEntity(entity) for entity in data[5].split(" ")
]
tweet = tw.Tweet(clst_id=data[0],
id=data[2],
timestamp_ms=data[3],
user_id=data[4],
tokens=tokens,
content=data[6])
tweets.append(tweet)
if not clusters.get(data[0]):
new_cl = cluster.Cluster(clst_id=data[0],
clst_name=data[1],
created_time=data[3])
clusters[data[0]] = new_cl
oldest_valid_time = new_cl.get_created_time(
) - ed.Constants.EPOCH * 8
for past_cluster in clusters_timeline[::-1]:
if past_cluster.get_created_time() >= oldest_valid_time:
new_cl.add_past_neighbour(past_cluster)
else:
break
clusters_timeline.append(new_cl)
clusters[data[0]].append(tweet)
for clst_id in clusters:
clusters[clst_id].aggregate_entities()
return tweets, clusters
def get_data_pool(path):
tweet_pool = []
with open(path, 'r') as json_data:
res = json_data.readline()
# Do loop until reaching the end of json file
while res != "":
res_json_object = json.loads(res)
if res_json_object["coordinates"] is not None:
tweet_pool.append(
tweet.Tweet(res_json_object["id_str"],
res_json_object["user"]["id"],
res_json_object["user"]["name"],
res_json_object["text"],
res_json_object["geo"]["coordinates"][0],
res_json_object["geo"]["coordinates"][1],
res_json_object["timestamp_ ms"]))
res = json_data.readline()
print "Data length of object: " + str(len(tweet_pool)) + "!"
return tweet_pool
def __init__(self, tweet_source, rest_api):
# Default window size
self.default_width = 340
self.default_height = 700
# Minimum window size
self.min_width = 200
self.min_height = 400
# Non-mobile photo workaround
self.photo_workaround = True
self.config_dir_path = '.'
self.config_path = os.path.join(self.config_dir_path, "twitter.conf")
self.config_save_interval = 120
self.refresh_time = 5
self.last_save = time.time()
self.window_geometry = {'x': 0, 'y': 0, 'w': self.default_width, 'h': self.default_height}
self.save_scheduled = False
self.refresh_count = 0
self.tweet_source = tweet_source.iter_tweets()
self.tweet_queue = Queue.Queue()
self.tweets = {}
self.to_delete = []
self.rest = rest_api
preloaded_tweets = self.rest.get_home_timeline(count=50)
for twt in preloaded_tweets:
self.tweets[twt['id']] = tweet.Tweet(twt)
for t in sorted(self.tweets.items())[-5:-1]:
print t[-1].data
GObject.threads_init()
self.tweet_thread = threading.Thread(target=self.retrieve_tweets)
self.tweet_thread.daemon = True
self.tweet_thread.start()
self.update_content()
self.init_view()
def massage(data, bag_of_words={}):
tweets = []
tknzr = TweetTokenizer()
for lines in data:
#split record into tab separated values
body = lines.split("\t")[-1]
if (body == "Not Available\n"):
continue
body = re.sub("(http|https):\/\/t\..*?\s", "httpstco", body)
#body = re.sub("([a-zA-Z])\1{2,}", body)
tokens = tknzr.tokenize(body)
#Conditional for the differnt formatting between task A data and data
#for other 4 tasks
if (len(lines.split("\t")) == 3):
sentiment = lines.split("\t")[1]
twit_id = lines.split("\t")[0]
subject = ""
else:
sentiment = lines.split("\t")[2]
twit_id = lines.split("\t")[0]
subject = lines.split("\t")[1]
# create feature vectors
w_tokens = create_and_annotate_words(tokens)
# print('printing w_tokens: ', w_tokens)
# update bag of words from tokens.
if bag_of_words is not None:
for token in w_tokens:
key = token.word + "_" + token.pos
bag_of_words[key] = bag_of_words.get(key, 0) + 1
tweet_obj = tweet.Tweet(w_tokens, twit_id, body, sentiment, subject)
#add the analyzed tweet into the final array for the neural net
tweets.append(tweet_obj)
return tweets
def test_congratulatory(self):
grats_tweets = [
tweet.Tweet("congratulations to kevin spacey!"),
tweet.Tweet("Congrats! you rock"),
tweet.Tweet("kevin WINS!")
]
not_grats_tweets = [
tweet.Tweet("boo to kevin spacey!"),
tweet.Tweet("kevin spacey got robbed!"),
tweet.Tweet("wonderful that kevin lost!")
]
for t in grats_tweets:
self.assertTrue(t.is_congratulatory())
for t in not_grats_tweets:
self.assertFalse(t.is_congratulatory())
def tweet(self, record):
t = tweet.Tweet(record)
if settings.DEBUG == "true":
print("debug mode")
return
res = self.client.post(self.endpoint, params=t.content)
res_dict = json.loads(res.text)
print(json.dumps(res_dict, indent=2))
created_at = res_dict['created_at']
print(created_at)
st = time.strptime(created_at, config.TWITTER_FORMAT)
utc_time = datetime.datetime(st.tm_year, st.tm_mon,st.tm_mday, \
st.tm_hour,st.tm_min,st.tm_sec, tzinfo=datetime.timezone.utc)
jst_time = utc_time.astimezone(pytz.timezone(config.TIMEZONE))
jst_str = jst_time.strftime(config.FORMAT)
return rec.Record(num=record.num,
word=record.word,
category=record.category,
mean=record.mean,
supplement=record.supplement,
created_at=jst_str)
def showDialog(self):
fname = QtGui.QFileDialog.getOpenFileName(self, 'Open file', '/home')
if fname == '':
return
tweets = io_geojson.read_tweets(fname)
tweets_data = []
for i in tweets:
tweets_data.append(tweet.Tweet(i))
average_lat = 0
average_lon = 0
count_tweets = 0
random.seed(1234)
for i in tweets_data:
lat, lon = i.gen_point_in_bounds()
average_lat += lat
average_lon += lon
count_tweets += 1
average_lon /= len(tweets_data)
average_lat /= len(tweets_data)
map_1 = folium.Map(location=[average_lat, average_lon])
countttz = 0
for i in tweets_data:
lat, lon = i.gen_point_in_bounds()
if countttz < 400:
#print(i.username)
folium.Marker([lat, lon], popup="Test").add_to(map_1)
countttz += 1
map_1.save(r"./map.html")
self.webView.setHtml(open("./map.html").read())
self.webView.show()
def showDialog(self):
global currentPoints
global positiveList
global negativeList
global neutralList
global activeList
global map_1
global average_lon
global average_lat
currentPoints = []
positiveList = []
negativeList = []
neutralList = []
fname = QtGui.QFileDialog.getOpenFileName(self, 'Open file', '/home')
if fname == '':
return
tweets = io_geojson.read_tweets(fname)
tweets_data = []
# count = 0
for i in tweets:
tweets_data.append(tweet.Tweet(i))
# count = count+1
#if count==30:
# break
average_lat = 0
average_lon = 0
count_tweets = 0
random.seed(1234)
for i in tweets_data:
lat, lon = i.gen_point_in_bounds()
currentPoints.append(point.Point(lat, lon, i.sentiment))
if i.sentiment == 'Positive':
positiveList.append(point.Point(lat, lon, i.sentiment))
elif i.sentiment == 'Negative':
negativeList.append(point.Point(lat, lon, i.sentiment))
elif i.sentiment == 'Neutral':
neutralList.append(point.Point(lat, lon, i.sentiment))
average_lat += lat
average_lon += lon
count_tweets += 1
average_lon /= len(tweets_data)
average_lat /= len(tweets_data)
map_1 = folium.Map(location=[average_lat, average_lon])
#activeList = currentPoints
#countttz = 0
for i in currentPoints:
#lat, lon = i.gen_point_in_bounds()
lat = i.x
lon = i.y
sent = i.mark
folium.Marker([lat, lon], popup=sent).add_to(map_1)
#if countttz < 500:
#print('Sup')
#print(type(i.username))
# print("hello")
#print(i.sentiment)
#print(type("Test"))
#folium.Marker([lat, lon], popup = i.username.decode('utf-8')).add_to(map_1)
#countttz+=1
map_1.save(r"./map.html")
self.webView.setHtml(open("./map.html").read())
self.webView.show()
def run(self):
t = tweet.Tweet()
t.post('hi')
def __init__(self, json_user_data, tweets=None):
self.user_data = json_user_data
self.tweets = [tweet.Tweet(t) for t in tweets]
return response
if __name__ == "__main__":
args = docopt(__doc__, version='Twitter Crawler Version:1.0')
client = Crawler(args)
oauth = client.connect_twitter()
client.connect_db()
response = client.twitter_stream(oauth)
for status in response.iter_lines(chunk_size=10000):
if status:
try:
status = json.loads(status)
tweet = twtr.Tweet(status)
if tweet.retweeted_status or tweet.text == "":
pass
else:
frozen = jsonpickle.encode(tweet.__dict__)
thawed = jsonpickle.decode(frozen)
util.logger.info(f"{tweet.text}")
mydb = client.pool.get_connection()
tweet.save_tweet(mydb)
tweet.save_to_graph(tweet, mydb, client.search_term)
mydb.close()
except Exception as error:
if status["limit"]:
util.logger.warning(f"{status['limit']['track']}")
else:
print(json.dumps(status, indent=4, sort_keys=True))
def main():
#tweetlist = []
file_name = "tweets.dat"
try:
load_file = open(file_name,'rb')
tweetlist = pickle.load(load_file)
load_file.close()
except:
tweetlist = []
while (True):
ch = display_menu()
if (ch==1):
author = input("\nWhat is your name? ")
text = input("What would you like to tweet? ")
while (len(text) > 140):
print("\nTweets can only be 140 characters!")
text = input("\nWhat would you like to tweet? ")
tweets = tweet.Tweet(author, text)
tweetlist.append(tweets)
try:
outputfile = open(file_name, 'wb')
pickle.dump(tweetlist, outputfile)
outputfile.close()
print(author, ", your tweet has been saved.\n",sep = "")
except:
print("Your tweets could not be saved!")
elif(ch==2):
tweetlist.reverse()
#if(tweetlist == []):
#load_file = open(file_name,'rb')
#tweetlist = pickle.load(load_file)
#load_file.close()
print()
print("Recent Tweets")
print("-----------")
if len(tweetlist) == 0:
print("There are no recent tweets. \n")
else:
for tweets in tweetlist[:5]:
print(tweets.get_author(), "-", tweets.get_age())
print(tweets.get_text(),"\n")
tweetlist.reverse()
elif(ch==3):
match = 0
tweetlist.reverse()
if tweetlist == []:
print()
print("There are no tweets to search.\n")
continue
else:
search = input("What would you like to search for? ")
print ()
print ("Search Results")
print("----------")
match = 0
for tweets in tweetlist:
if search in tweets.get_text():
matched_tweets = []
match = 1
matched_tweets.append(tweets) # append all the matches to an array
#for tweets in matched_tweets:
print(tweets.get_author(), "-", tweets.get_age())
print(tweets.get_text(), "\n")
if match == 0:
print("No tweets contained,", search)
print()
tweetlist.reverse()
elif(ch==4):
print("Thank you for using the Tweet Manager!")
print()
#load_file = open(file_name, 'wb')
#pickle.dump(tweetlist, load_file)
#load_file.close()
exit()
def next(self):
line = self.fp.readline()
if not line:
return None
else:
return tweet.Tweet(line)
import tweepy
import tweet
import tweet_listener
from keys import keys
# Authentication details
CONSUMER_KEY = keys['consumer_key']
CONSUMER_SECRET = keys['consumer_secret']
ACCESS_TOKEN = keys['access_token']
ACCESS_TOKEN_SECRET = keys['access_token_secret']
if __name__ == '__main__':
# Create authentication token using our details
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
# Get API handler
api = tweepy.API(auth)
tweet = tweet.Tweet(api)
t_list = tweet_listener.TweetListener(tweet)
stream = tweepy.Stream(auth, t_list)
stream.filter(track=['#avispabaleares'])
def test_tokens(self):
t = tweet.Tweet("COOL a new! WaY! to #GOTOSCHOOl")
self.assertEqual(t.tokens(), ['cool', 'a', 'new', 'way', 'to', 'gotoschool'])
def test_mentions(self):
self.assertListEqual(tweet.Tweet("@danaKStew @50ShadesWorldcm @ScarletteDrake Also Red Carpet um 12").mentions(),
['@danaKStew','@50ShadesWorldcm', '@ScarletteDrake'])
self.assertListEqual(tweet.Tweet("going to bed @ 2 o'clock").mentions(), [])
def setUp(self):
self.tweet = tweet.Tweet("foobar")
