def open_link(self, view, frame, req, nav, pol, data=None): print "Opening %s" % req.get_uri() if req.get_uri() == "wasabi://compose/tweet": self.full_content = open(tweet.template_dir+'/compose.html', 'r').read() self.view.load_string(self.full_content, "text/html", "UTF-8", "") return True elif req.get_uri().startswith("wasabi://compose/reply"): in_reply_to = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query)['reply_id'][0] reply_text = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query)['reply_text'][0] page = Template(open(tweet.template_dir+'/reply.html', 'r').read()) self.full_content = page.render(reply_id=in_reply_to, reply_text=reply_text) print self.full_content self.view.load_string(self.full_content, "text/html", "UTF-8", "") return True elif req.get_uri().startswith("wasabi://compose/send"): params = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query) text = params['tweet[text]'][0] if 'tweet[in_reply_to]' in params: in_reply_to = urlparse.parse_qs(urlparse.urlparse(req.get_uri()).query)['tweet[in_reply_to]'][0] twt = tweet.Tweet(self.rest.send_tweet(status=text, in_reply_to_status_id=in_reply_to)) else: twt = tweet.Tweet(self.rest.send_tweet(status=text)) if twt: self.tweets[twt.data['id']] = twt self.update_content() self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home") return False elif req.get_uri() == "wasabi://home": print view.get_uri() #if view.get_uri() != "wasabi://home": # self.update_content() # self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home") return False else: return self.open_external_link(view, frame, req, None, None)
def test_is_snub(self): grats_tweets = [ tweet.Tweet("kevin spacey got snubbed!"), tweet.Tweet("boyhood got robbed!"), tweet.Tweet("it's a shame boyhood lost") ] not_grats_tweets = [ tweet.Tweet("congrats to kevin spacey!") ] for t in grats_tweets: self.assertTrue(t.is_snub()) for t in not_grats_tweets: self.assertFalse(t.is_snub())
def read_tweets(user, num_tweets): tweets = [] url = "http://api.twitter.com/1/statuses/user_timeline.json?\ screen_name=%s&count=%s&include_rts=true" % (user, num_tweets) file = urllib2.urlopen(url) content = file.read() json = simplejson.loads(content) for js_tweet in json: t = tweet.Tweet() t.id = js_tweet['id'] t.username = js_tweet['user']['screen_name'] try: t.retweet_user = js_tweet['retweeted_status']['user']['screen_name'] t.retweeted = True except: t.retweeted = False t.set_date(js_tweet['created_at']) t.set_tweet_url() t.set_text(js_tweet['text']) t.set_profile_url() tweets.append(t) return tweets
def test_nonames(self): self.tweetdata['text'] = 'This is some text with no names' tw = tweet.Tweet(self.tweetdata) self.assertEqual(tw.parse_tweet(), [('This', 'DT'), ('is', 'VBZ'), ('some', 'DT'), ('text', 'NN'), ('with', 'IN'), ('no', 'DT'), ('names', 'NNS')])
def fetch_tweets(self, view, sw): #print self.tweets #print self.tweet_queue.qsize() while True: try: twt = self.tweet_queue.get(False) except Queue.Empty: #print "%d - No tweets." % (time.time()) break print "Got tweet." print twt twt = tweet.Tweet(twt) if twt.validate(): if twt.data['id'] in self.to_delete: self.to_delete.remove(twt.data['id']) else: self.tweets[twt.data['id']] = twt self.update_content() if view.get_uri() == "wasabi://home": self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home") elif 'delete' in twt.data and 'status' in twt.data['delete']: if twt.data['delete']['status']['id'] in self.tweets: self.tweets[twt.data['delete']['status']['id']].time_str = "[DEL]" self.update_content() if view.get_uri() == "wasabi://home": self.view.load_string(self.full_content, "text/html", "UTF-8", "wasabi://home") else: self.to_delete.append(twt.data['delete']['status']['id']) return True
def test_start_with_name(self): self.tweetdata['text'] = 'Argo was nominated in 2013.' tw = tweet.Tweet(self.tweetdata) self.assertEqual(tw.parse_tweet(), [([('Argo', 'NNP')], 'NNP-phrase'), ('was', 'VBD'), ('nominated', 'VBN'), ('in', 'IN'), ('2013', 'CD'), ('.', '.')])
def test_name_with_of(self): self.tweetdata['text'] = 'The show House of Cards was nominated.' tw = tweet.Tweet(self.tweetdata) self.assertEqual(tw.parse_tweet(), [('The', 'DT'), ('show', 'NN'), ([('House', 'NNP'), ('of', 'IN'), ('Cards', 'NNP')], 'NNP-phrase'), ('was', 'VBD'), ('nominated', 'VBN'), ('.', '.')])
def test_name_with_the(self): self.tweetdata['text'] = 'I watched The Affair the other day.' tw = tweet.Tweet(self.tweetdata) self.assertEqual(tw.parse_tweet(), [('I', 'PRP'), ('watched', 'VBD'), ([('The', 'DT'), ('Affair', 'NNP')], 'NNP-phrase'), ('the', 'DT'), ('other', 'JJ'), ('day', 'NN'), ('.', '.')])
def test_name_with_ofthe(self): self.tweetdata['text'] = 'Maybe Lord of the Flies will win.' tw = tweet.Tweet(self.tweetdata) self.assertEqual(tw.parse_tweet(), [('Maybe', 'RB'), ([('Lord', 'NNP'), ('of', 'IN'), ('the', 'DT'), ('Flies', 'NNPS')], 'NNP-phrase'), ('will', 'MD'), ('win', 'VB'), ('.', '.')])
def getTweetClient(config): payload = { 'apiKey': config.TWITTER_API_KEY, 'apiSecret': config.TWITTER_API_SECRET, 'bearerToken': config.TWITTER_BEARER_TOKEN, 'proxy': config.PROXY, 'screenName': config.SCREEN_NAME, 'interval': config.INTERVAL } tweetClient = tweet.Tweet(**payload) return tweetClient
def test_double_token_name(self): self.tweetdata[ 'text'] = 'Ironically, Amal Clooney may be the get of the #GoldenGlobes even though she\'s not nominated for anything. And supposedly,' tw = tweet.Tweet(self.tweetdata) self.assertEqual(tw.parse_tweet(), [('Ironically', 'RB'), (',', ','), ([('Amal', 'NNP'), ('Clooney', 'NNP')], 'NNP-phrase'), ('may', 'MD'), ('be', 'VB'), ('the', 'DT'), ('get', 'NN'), ('of', 'IN'), ('the', 'DT'), ('#GoldenGlobes', 'HT'), ('even', 'RB'), ('though', 'IN'), ('she', 'PRP'), ("'s", 'VBZ'), ('not', 'RB'), ('nominated', 'VBN'), ('for', 'IN'), ('anything', 'NN'), ('.', '.'), ('And', 'CC'), ('supposedly', 'RB'), (',', ',')])
def test_multinames_with_conjunction(self): self.tweetdata[ 'text'] = '\"The Affair\" or \"House of Cards\"? Our #GoldenGlobes TV predictions: http://t.co/Wc4BXiThed http://t.co/WYp278s9F0' tw = tweet.Tweet(self.tweetdata) self.assertEqual(tw.parse_tweet(), [('``', '``'), ([('The', 'DT'), ('Affair', 'NNP')], 'NNP-phrase'), ("''", "''"), ('or', 'CC'), ('``', '``'), ([('House', 'NNP'), ('of', 'IN'), ('Cards', 'NNPS')], 'NNP-phrase'), ("''", "''"), ('?', '.'), ('Our', 'PRP$'), ('#GoldenGlobes', 'HT'), ('TV', 'NN'), ('predictions', 'NNS'), (':', ':'), ('http', 'NN'), (':', ':'), ('//t.co/Wc4BXiThed', 'VBN'), ('http', 'NN'), (':', ':'), ('//t.co/WYp278s9F0', 'NN')])
def read_clusters_sorted_by_time(file_name, named_entities_file_name): tweets = [] clusters = {} clusters_timeline = [] named_entities = {} file_to_read = open(file_name, "r") file_entities = open(named_entities_file_name, 'r') for line in file_entities.readlines(): data = line.split(",", 1) named_entities[data[0]] = data[1] for line in file_to_read.readlines(): data = line.split(",") tokens = [ en.SimpleEntity(entity) if not (entity in named_entities.get(data[2])) else en.NamedEntity(entity) for entity in data[5].split(" ") ] tweet = tw.Tweet(clst_id=data[0], id=data[2], timestamp_ms=data[3], user_id=data[4], tokens=tokens, content=data[6]) tweets.append(tweet) if not clusters.get(data[0]): new_cl = cluster.Cluster(clst_id=data[0], clst_name=data[1], created_time=data[3]) clusters[data[0]] = new_cl oldest_valid_time = new_cl.get_created_time( ) - ed.Constants.EPOCH * 8 for past_cluster in clusters_timeline[::-1]: if past_cluster.get_created_time() >= oldest_valid_time: new_cl.add_past_neighbour(past_cluster) else: break clusters_timeline.append(new_cl) clusters[data[0]].append(tweet) for clst_id in clusters: clusters[clst_id].aggregate_entities() return tweets, clusters
def get_data_pool(path): tweet_pool = [] with open(path, 'r') as json_data: res = json_data.readline() # Do loop until reaching the end of json file while res != "": res_json_object = json.loads(res) if res_json_object["coordinates"] is not None: tweet_pool.append( tweet.Tweet(res_json_object["id_str"], res_json_object["user"]["id"], res_json_object["user"]["name"], res_json_object["text"], res_json_object["geo"]["coordinates"][0], res_json_object["geo"]["coordinates"][1], res_json_object["timestamp_ ms"])) res = json_data.readline() print "Data length of object: " + str(len(tweet_pool)) + "!" return tweet_pool
def __init__(self, tweet_source, rest_api): # Default window size self.default_width = 340 self.default_height = 700 # Minimum window size self.min_width = 200 self.min_height = 400 # Non-mobile photo workaround self.photo_workaround = True self.config_dir_path = '.' self.config_path = os.path.join(self.config_dir_path, "twitter.conf") self.config_save_interval = 120 self.refresh_time = 5 self.last_save = time.time() self.window_geometry = {'x': 0, 'y': 0, 'w': self.default_width, 'h': self.default_height} self.save_scheduled = False self.refresh_count = 0 self.tweet_source = tweet_source.iter_tweets() self.tweet_queue = Queue.Queue() self.tweets = {} self.to_delete = [] self.rest = rest_api preloaded_tweets = self.rest.get_home_timeline(count=50) for twt in preloaded_tweets: self.tweets[twt['id']] = tweet.Tweet(twt) for t in sorted(self.tweets.items())[-5:-1]: print t[-1].data GObject.threads_init() self.tweet_thread = threading.Thread(target=self.retrieve_tweets) self.tweet_thread.daemon = True self.tweet_thread.start() self.update_content() self.init_view()
def massage(data, bag_of_words={}): tweets = [] tknzr = TweetTokenizer() for lines in data: #split record into tab separated values body = lines.split("\t")[-1] if (body == "Not Available\n"): continue body = re.sub("(http|https):\/\/t\..*?\s", "httpstco", body) #body = re.sub("([a-zA-Z])\1{2,}", body) tokens = tknzr.tokenize(body) #Conditional for the differnt formatting between task A data and data #for other 4 tasks if (len(lines.split("\t")) == 3): sentiment = lines.split("\t")[1] twit_id = lines.split("\t")[0] subject = "" else: sentiment = lines.split("\t")[2] twit_id = lines.split("\t")[0] subject = lines.split("\t")[1] # create feature vectors w_tokens = create_and_annotate_words(tokens) # print('printing w_tokens: ', w_tokens) # update bag of words from tokens. if bag_of_words is not None: for token in w_tokens: key = token.word + "_" + token.pos bag_of_words[key] = bag_of_words.get(key, 0) + 1 tweet_obj = tweet.Tweet(w_tokens, twit_id, body, sentiment, subject) #add the analyzed tweet into the final array for the neural net tweets.append(tweet_obj) return tweets
def test_congratulatory(self): grats_tweets = [ tweet.Tweet("congratulations to kevin spacey!"), tweet.Tweet("Congrats! you rock"), tweet.Tweet("kevin WINS!") ] not_grats_tweets = [ tweet.Tweet("boo to kevin spacey!"), tweet.Tweet("kevin spacey got robbed!"), tweet.Tweet("wonderful that kevin lost!") ] for t in grats_tweets: self.assertTrue(t.is_congratulatory()) for t in not_grats_tweets: self.assertFalse(t.is_congratulatory())
def tweet(self, record): t = tweet.Tweet(record) if settings.DEBUG == "true": print("debug mode") return res = self.client.post(self.endpoint, params=t.content) res_dict = json.loads(res.text) print(json.dumps(res_dict, indent=2)) created_at = res_dict['created_at'] print(created_at) st = time.strptime(created_at, config.TWITTER_FORMAT) utc_time = datetime.datetime(st.tm_year, st.tm_mon,st.tm_mday, \ st.tm_hour,st.tm_min,st.tm_sec, tzinfo=datetime.timezone.utc) jst_time = utc_time.astimezone(pytz.timezone(config.TIMEZONE)) jst_str = jst_time.strftime(config.FORMAT) return rec.Record(num=record.num, word=record.word, category=record.category, mean=record.mean, supplement=record.supplement, created_at=jst_str)
def showDialog(self): fname = QtGui.QFileDialog.getOpenFileName(self, 'Open file', '/home') if fname == '': return tweets = io_geojson.read_tweets(fname) tweets_data = [] for i in tweets: tweets_data.append(tweet.Tweet(i)) average_lat = 0 average_lon = 0 count_tweets = 0 random.seed(1234) for i in tweets_data: lat, lon = i.gen_point_in_bounds() average_lat += lat average_lon += lon count_tweets += 1 average_lon /= len(tweets_data) average_lat /= len(tweets_data) map_1 = folium.Map(location=[average_lat, average_lon]) countttz = 0 for i in tweets_data: lat, lon = i.gen_point_in_bounds() if countttz < 400: #print(i.username) folium.Marker([lat, lon], popup="Test").add_to(map_1) countttz += 1 map_1.save(r"./map.html") self.webView.setHtml(open("./map.html").read()) self.webView.show()
def showDialog(self): global currentPoints global positiveList global negativeList global neutralList global activeList global map_1 global average_lon global average_lat currentPoints = [] positiveList = [] negativeList = [] neutralList = [] fname = QtGui.QFileDialog.getOpenFileName(self, 'Open file', '/home') if fname == '': return tweets = io_geojson.read_tweets(fname) tweets_data = [] # count = 0 for i in tweets: tweets_data.append(tweet.Tweet(i)) # count = count+1 #if count==30: # break average_lat = 0 average_lon = 0 count_tweets = 0 random.seed(1234) for i in tweets_data: lat, lon = i.gen_point_in_bounds() currentPoints.append(point.Point(lat, lon, i.sentiment)) if i.sentiment == 'Positive': positiveList.append(point.Point(lat, lon, i.sentiment)) elif i.sentiment == 'Negative': negativeList.append(point.Point(lat, lon, i.sentiment)) elif i.sentiment == 'Neutral': neutralList.append(point.Point(lat, lon, i.sentiment)) average_lat += lat average_lon += lon count_tweets += 1 average_lon /= len(tweets_data) average_lat /= len(tweets_data) map_1 = folium.Map(location=[average_lat, average_lon]) #activeList = currentPoints #countttz = 0 for i in currentPoints: #lat, lon = i.gen_point_in_bounds() lat = i.x lon = i.y sent = i.mark folium.Marker([lat, lon], popup=sent).add_to(map_1) #if countttz < 500: #print('Sup') #print(type(i.username)) # print("hello") #print(i.sentiment) #print(type("Test")) #folium.Marker([lat, lon], popup = i.username.decode('utf-8')).add_to(map_1) #countttz+=1 map_1.save(r"./map.html") self.webView.setHtml(open("./map.html").read()) self.webView.show()
def run(self): t = tweet.Tweet() t.post('hi')
def __init__(self, json_user_data, tweets=None): self.user_data = json_user_data self.tweets = [tweet.Tweet(t) for t in tweets]
return response if __name__ == "__main__": args = docopt(__doc__, version='Twitter Crawler Version:1.0') client = Crawler(args) oauth = client.connect_twitter() client.connect_db() response = client.twitter_stream(oauth) for status in response.iter_lines(chunk_size=10000): if status: try: status = json.loads(status) tweet = twtr.Tweet(status) if tweet.retweeted_status or tweet.text == "": pass else: frozen = jsonpickle.encode(tweet.__dict__) thawed = jsonpickle.decode(frozen) util.logger.info(f"{tweet.text}") mydb = client.pool.get_connection() tweet.save_tweet(mydb) tweet.save_to_graph(tweet, mydb, client.search_term) mydb.close() except Exception as error: if status["limit"]: util.logger.warning(f"{status['limit']['track']}") else: print(json.dumps(status, indent=4, sort_keys=True))
def main(): #tweetlist = [] file_name = "tweets.dat" try: load_file = open(file_name,'rb') tweetlist = pickle.load(load_file) load_file.close() except: tweetlist = [] while (True): ch = display_menu() if (ch==1): author = input("\nWhat is your name? ") text = input("What would you like to tweet? ") while (len(text) > 140): print("\nTweets can only be 140 characters!") text = input("\nWhat would you like to tweet? ") tweets = tweet.Tweet(author, text) tweetlist.append(tweets) try: outputfile = open(file_name, 'wb') pickle.dump(tweetlist, outputfile) outputfile.close() print(author, ", your tweet has been saved.\n",sep = "") except: print("Your tweets could not be saved!") elif(ch==2): tweetlist.reverse() #if(tweetlist == []): #load_file = open(file_name,'rb') #tweetlist = pickle.load(load_file) #load_file.close() print() print("Recent Tweets") print("-----------") if len(tweetlist) == 0: print("There are no recent tweets. \n") else: for tweets in tweetlist[:5]: print(tweets.get_author(), "-", tweets.get_age()) print(tweets.get_text(),"\n") tweetlist.reverse() elif(ch==3): match = 0 tweetlist.reverse() if tweetlist == []: print() print("There are no tweets to search.\n") continue else: search = input("What would you like to search for? ") print () print ("Search Results") print("----------") match = 0 for tweets in tweetlist: if search in tweets.get_text(): matched_tweets = [] match = 1 matched_tweets.append(tweets) # append all the matches to an array #for tweets in matched_tweets: print(tweets.get_author(), "-", tweets.get_age()) print(tweets.get_text(), "\n") if match == 0: print("No tweets contained,", search) print() tweetlist.reverse() elif(ch==4): print("Thank you for using the Tweet Manager!") print() #load_file = open(file_name, 'wb') #pickle.dump(tweetlist, load_file) #load_file.close() exit()
def next(self): line = self.fp.readline() if not line: return None else: return tweet.Tweet(line)
import tweepy import tweet import tweet_listener from keys import keys # Authentication details CONSUMER_KEY = keys['consumer_key'] CONSUMER_SECRET = keys['consumer_secret'] ACCESS_TOKEN = keys['access_token'] ACCESS_TOKEN_SECRET = keys['access_token_secret'] if __name__ == '__main__': # Create authentication token using our details auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) # Get API handler api = tweepy.API(auth) tweet = tweet.Tweet(api) t_list = tweet_listener.TweetListener(tweet) stream = tweepy.Stream(auth, t_list) stream.filter(track=['#avispabaleares'])
def test_tokens(self): t = tweet.Tweet("COOL a new! WaY! to #GOTOSCHOOl") self.assertEqual(t.tokens(), ['cool', 'a', 'new', 'way', 'to', 'gotoschool'])
def test_mentions(self): self.assertListEqual(tweet.Tweet("@danaKStew @50ShadesWorldcm @ScarletteDrake Also Red Carpet um 12").mentions(), ['@danaKStew','@50ShadesWorldcm', '@ScarletteDrake']) self.assertListEqual(tweet.Tweet("going to bed @ 2 o'clock").mentions(), [])
def setUp(self): self.tweet = tweet.Tweet("foobar")