def main(
twitter_api,
keyword,
):
twitter_stream = twitter.TwitterStream(auth=twitter_api.auth)
q = f"{keyword}" # lang:en until:{until} place_country:23424977
results = twitter_stream.statuses.filter(track=q,
language='en',
place_country=23424977)
print('+' * 10)
for i in results:
# print(json.dumps(i))
try:
yield {
'user_id':
i['user']['id'],
'username':
i['user']['name'],
'created_at':
i['created_at'],
'full_text':
i['extended_tweet']['full_text']
if i.get('extended_tweet') is not None else
i['retweeted_status']['extended_tweet']['full_text']
}
except KeyError:
continue
def main():
conn, cursor = get_SQL_connection()
# cursor.execute('DROP TABLE twitter')
try:
cursor.execute("CREATE TABLE twitter (id INT UNSIGNED NOT NULL AUTO_INCREMENT, at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, user_id CHAR(20), state CHAR(2), city VARCHAR(30), tweet VARCHAR(140), PRIMARY KEY (id))")
except:
print("Table already exists")
token_dict = getApiKeys(sys.argv[1])
tw_stream = twitter.TwitterStream(auth=twitter.OAuth(
token_dict['access_token_key'],
token_dict['access_token_secret'],
token_dict['consumer_key'],
token_dict['consumer_secret']))
tw_iterator = tw_stream.statuses.sample()
insert_string = "INSERT INTO twitter (user_id,state,city,tweet) VALUES (%s, %s, %s, %s)"
i = 1
for tw in tw_iterator:
tw_userid, tw_text, tw_lang, tw_country, tw_state, tw_city = '', '', '', '', '', ''
try:
tw_userid = tw['id_str']
tw_text = tw['text']
tw_lang = tw['lang']
tw_place = tw['place']
if tw_text and tw_place and tw_lang == 'en':
tw_country = tw_place['country']
if tw_country == 'United States':
tw_fullname = tw_place['full_name']
tw_location = tw_fullname.split()
for s in tw_location:
if s in States:
tw_state = s
tw_city = tw_place['name']
if tw_city:
data = (tw_userid, tw_state, tw_city, tw_text)
print (i, tw_state, tw_city, tw_text)
cursor.execute(insert_string, data)
i = i + 1
break
except:
continue
if i%10 == 0:
cursor.close()
conn.commit()
conn.close()
print("Total size of Twitter table: {0}".format(len(get_SQL_table())))
if i==20:
break
else:
conn, cursor = get_SQL_connection()
def oauth_twitter_streaming():
oauth_token, oauth_token_secret = OAUTH_TOKEN, OAUTH_TOKEN_SECRET
auth = twitter.oauth.OAuth(oauth_token, oauth_token_secret, CONSUMER_KEY,
CONSUMER_SECRET)
return twitter.TwitterStream(auth=auth)
def scrape_twitter():
auth = twitter.OAuth(OAUTH_TOKEN, OAUTH_SECRET, CONSUMER_KEY,
CONSUMER_SECRET)
stream = twitter.TwitterStream(auth=auth)
samples = stream.statuses.sample(filter_level='medium')
client = pymongo.MongoClient()
db = client.medisas_challenge
rt_count = db.rt_count
tweets = db.tweets
for tweet in samples:
try:
if tweet['retweeted_status']:
original_tweet = tweet['retweeted_status']
created_at = datetime.datetime.strptime(
tweet['created_at'], '%a %b %d %X +0000 %Y')
original_created_at = datetime.datetime.strptime(
original_tweet['created_at'], '%a %b %d %X +0000 %Y')
original_text = original_tweet['text']
tweet_id = original_tweet['id']
data = {'tweet_id':tweet_id, 'text': original_text,\
'created_at':original_created_at }
db.tweets.update({'tweet_id': tweet_id}, data, upsert=True)
db.rt_count.insert({
'tweet_id': tweet_id,
'created_at': created_at
})
except KeyError:
# Given tweet isn't a RT, ignore it
pass
def run(self):
""" Check Twitter user stream for updates and send these to the
TG chat via .send_message().
This code runs in a separate thread until .run is set to
False.
Caveat: Terminating the thread is difficult, since it can
only check the .run flag when receiving new tweets or
messages from the stream.
"""
twitter_userstream = twitter.TwitterStream(
auth=self.oauth,
domain='userstream.twitter.com')
for msg in twitter_userstream.user():
if not self.run:
break
print ('Incoming Twitter stream message:')
print ('-' * 72)
pprint.pprint(msg)
print ('-' * 72)
if 'text' not in msg:
# Not a status update, so skip this...
continue
self.send_message(u'_Received tweet from @%s:_\n%s' % (
msg['user']['screen_name'],
msg['text']),
parse_mode='Markdown')
def _twitter_reconnect(self): """Logs in to Twitter, using the stored OAuth. This function is intended for internal use, and should ONLY be called after twitter_login has been called. """ # Report the reconnection attempt. self._message(u'_twitter_reconnect', \ u"Attempting to reconnect to Twitter.") # Raise an Exception if the twitter library wasn't imported if not IMPTWITTER: self._error(u'_twitter_reconnect', u"The 'twitter' library could not be imported. Check whether it is installed correctly.") # Log in to a Twitter account self._t = twitter.Twitter(auth=self._oauth) self._ts = twitter.TwitterStream(auth=self._oauth) self._loggedin = True # Get the bot's own user credentials self._credentials = self._t.account.verify_credentials() # Report the reconnection success. self._message(u'_twitter_reconnect', \ u"Successfully reconnected to Twitter!")
def connect_twitter(): twitter_stream = twitter.TwitterStream(auth=twitter.OAuth( token = "get_your_own_credentials", token_secret = "get_your_own_credentials", consumer_key = "get_your_own_credentials", consumer_secret = "get_your_own_credentials")) return twitter_stream
def streamAPI():
twitter_stream = twitter.TwitterStream(auth=twitter.oauth.OAuth(
access_token, access_token_secret, consumer_key, consumer_secret))
res = twitter_stream.statuses.filter(track='singapore')
reslist = []
tweetfields = set()
for r in res:
if len(reslist) < 10:
print len(reslist)
reslist.append(r)
tweetfields = tweetfields.union(r.keys())
else:
break
print tweetfields
print "==========="
print reslist
fname = 'out/example_tweets.csv'
f = open(fname, 'w')
dw = csv.DictWriter(f, fieldnames=list(tweetfields))
dw.writeheader()
for r in reslist:
dw.writerow({
k: v.encode('utf8') if isinstance(v, unicode) else v
for k, v in r.items()
})
f.close()
def rt_hashtag_tweets(hashtag,limit,list_of_ignore=[]):
count = 0
twitter_stream = twitter.TwitterStream(auth=twitter_api.auth)
stream = twitter_stream.statuses.filter(track=hashtag)
doNotTweet = False
for tweet in stream:
if count == limit: #change back to 1000 or 2000 after testing
break
doNotTweet = False
print 'Tweet found\n'
print tweet
if 'lang' in tweet.keys():
if tweet['lang'] != 'en':
print '1'
doNotTweet = True
if 'user' in tweet.keys():
if tweet['user']['screen_name'] == 'jennifer_s_life':
print '2'
doNotTweet = True
#if 'retweeted' in tweet.keys():
#if tweet['retweeted'] == True:
#doNotTweet = True
#retweeted_status']['retweeted
if 'retweeted_status' in tweet.keys():
if 'retweeted' in tweet['retweeted_status'].keys():
if tweet['retweeted_status']['retweeted'] == True:
print '3'
doNotTweet = True
if 'retweeted_status' in tweet.keys():
if 'user' in tweet['retweeted_status'].keys():
if 'screen_name' in tweet['retweeted_status']['user'].keys():
if tweet['retweeted_status']['user']['screen_name'] == 'jennifer_s_life':
print '4'
doNotTweet = True
if 'id' not in tweet.keys():
doNotTweet = True
print '5'
if doNotTweet == False:
x = 60*randint(10,30)
print 'Retweeting in ' + str(x) + '\n'
sleep(randint(5,20))
twitter_api.statuses.retweet(id=tweet['id'])
print 'Retweeted'
count = count + 1
sleep(x)
def _get_twit(*args, **kwargs):
"""Instantiates a Twitter object based on streaming or RESTful."""
settings = kwargs.get("settings", {})
_twit = None
def _lookup_uid(twit):
"""Updates settings, adding the `uid` key."""
uid = twit.users.lookup(screen_name=settings["user"])[0]["id"]
settings.update({"uid": uid})
if not "twit" in kwargs or kwargs["twit"] is None:
oauth = get_oauth()
if settings.get("stream") and (settings.get("search")
or settings.get("user")):
if settings.get("user"):
# the screen_name->uid lookup is only available RESTfully
twit_rest = twitter.Twitter(auth=oauth)
_lookup_uid(twit_rest)
twit = twitter.TwitterStream(auth=oauth)
else:
twit = twitter.Twitter(auth=oauth)
settings["stream"] = False
if settings.get("user"):
_lookup_uid(twit)
_twit = twit
else:
_twit = kwargs["twit"]
kwargs.update({"twit": _twit, "settings": settings})
return func(*args, **kwargs)
def runner():
q = 'UFC' #Comma-separated list of terms
print >> sys.stderr, 'Filtering the public timeline for track="%s"' % (q, )
# Returns an instance of twitter.Twitter
twitter_api = oauth_login()
# Reference the self.auth parameter
twitter_stream = twitter.TwitterStream(auth=twitter_api.auth)
# See https://dev.twitter.com/docs/streaming-apis
stream = twitter_stream.statuses.filter(track=q)
# For illustrative purposes, when all else fails, search for Justin Bieber
# and something is sure to turn up (at least, on Twitter)
for tweetings in stream:
#if tweet.hangup==True:
# print 'hangup'
try:
print tweetings["text"]
producer.send_messages("test", str(tweetings)) #["text"]
except:
continue
def __init__(self):
self.counter = 0
self.fprefix = ""
self.output = None
self.twitter_rest = twitter__login.login()
self.twitter_stream = twitter.TwitterStream(auth=twitter.oauth.OAuth(
access_token, access_token_secret, consumer_key, consumer_secret))
def twitter_login(self, cons_key, cons_secret, access_token, \ access_token_secret): """Logs in to Twitter, using the provided access keys. You can get these for your own Twitter account at apps.twitter.com Arguments cons_key - String of your Consumer Key (API Key) cons_secret - String of your Consumer Secret (API Secret) access_token - String of your Access Token access_token_secret - String of your Access Token Secret """ # Raise an Exception if the twitter library wasn't imported if not IMPTWITTER: self._error(u'twitter_login', u"The 'twitter' library could not be imported. Check whether it is installed correctly.") # Log in to a Twitter account self._oauth = twitter.OAuth(access_token, access_token_secret, \ cons_key, cons_secret) self._t = twitter.Twitter(auth=self._oauth) self._ts = twitter.TwitterStream(auth=self._oauth) self._loggedin = True # Get the bot's own user credentials self._credentials = self._t.account.verify_credentials()
def main():
twitter_api = oauth_login()
filename = "" + time.strftime("%Y%m%d-%H%M%S")
twitter_stream = twitter.TwitterStream(auth=twitter_api.auth)
twitter_tweets = twitter_stream.statuses.filter(track=TRACK,
locations=LOCATIONS)
for tweet in twitter_tweets:
save_json(filename, tweet)
def connect_to_twitter(self):
oauth_token, oauth_secret = twitter.read_token_file('account_credentials')
consumer_key, consumer_secret = twitter.read_token_file('app_credentials')
authpoop = twitter.OAuth(oauth_token, oauth_secret,
consumer_key, consumer_secret)
self.rest_t = twitter.Twitter(auth=authpoop )
self.stream_t = twitter.TwitterStream(domain='userstream.twitter.com',auth=authpoop,timeout=30)
def main(window):
refresh_window(window, [])
auth = twitter.OAuth(context.token, context.token_key, context.con_secret,
context.con_secret_key)
stream = twitter.TwitterStream(auth=auth)
iterator = stream.statuses.sample()
for tweet in iterator:
process_tweet(tweet, window)
def connect_twitter():
twitter_stream = twitter.TwitterStream(auth=twitter.OAuth(
token="739682825863995393-ecE6XipKWPtuTwC0m7vd5JRr2hNwBYX",
token_secret="jGUKOHW6AdbICJsLcCbO3YFb86dBUHAp5P3IbueJ1O51a",
consumer_key="zrG3CPkCa6x0NQmGKci7GVLBL",
consumer_secret="RIuXUc8qwh8EywXVSUqNj48nkZYLtUsQGxc7pK6XxJQvCCDOeN"))
return twitter_stream
def show_tl_stream():
import twitter
auth = twitter.OAuth(AT, AS, CK, CS)
t_stream = twitter.TwitterStream(auth=auth,
domain="userstream.twitter.com")
for msg in t_stream.user():
if "text" in msg:
print(msg["user"]["screen_name"] + ":" + msg["text"] + "\n")
def search_stream(word="進捗"):
import twitter
auth = twitter.OAuth(AT, AS, CK, CS)
t_stream = twitter.TwitterStream(auth=auth, domain="stream.twitter.com")
for msg in t_stream.statuses.filter(track=word):
if "text" in msg:
print("@" + msg["user"]["screen_name"] + ":\n" + msg["text"] +
"\n")
def stream_data(self,
track=None,
follow=None,
locations=None,
save_to_db=False,
collection_name='stream'):
"""
https://dev.twitter.com/streaming/reference/post/statuses/filter
The default access level allows up to 400 track keywords, 5,000 follow userids and 25 0.1-360 degree location boxes.
:param track: str ;
:param follow:list str ;
:param locations: str ;
:param save_to_db:
:param collection_name:
:return: None
"""
def location_bounding_box(_locations):
t = _locations.split(',')
res = ''
for i in xrange(0, len(t), 2):
x, y = str(float(t[i]) + 1), str(float(t[i + 1]) + 1)
res += t[i] + ',' + t[i + 1] + ',' + x + ',' + y + ','
return res
kwg = {'language': 'en'}
if not track and not follow and not locations:
kwg['track'] = 'twitter'
if track:
kwg['track'] = track
if follow:
kwg['follow'] = follow
if locations:
kwg['locations'] = location_bounding_box(locations)
print kwg
twitter_stream = twitter.TwitterStream(auth=self.twitter_api.auth)
stream = twitter_stream.statuses.filter(**kwg)
for i, tweet in enumerate(stream):
if not i % 200 and 'text' in tweet:
print i, datetime.datetime.now(), ' ', tweet["text"]
tweet = dict(tweet)
if 'id' in tweet:
self.tweets.append(tweet)
if self.get_data:
self.get_data = False
self.conn.send(self.tweets)
self.tweets = []
if save_to_db:
self.save_tweets_to_mongodb(tweet, colname=collection_name)
def twitter_streaming(self, query, location_api):
"""
:Date: 2017-06-18
:Version: 0.2
:Author: CAOBA - Pontificia Universidad Javeriana.
This method downloads tweets from Twitter and save them into the semi structured database and queue, according
to the filters received. The streaming is done by location or by a particular word.
See https://dev.twitter.com/docs/streaming-apis
:param query: filter words, comma-separated list of terms. For example: 'Colombia,Peru,Ecuador'
:type query: str
:param location_api: Coordinates of a squared located on the particular geographic zone from which the tweets
will be taken. Must contain two points with latitude and longitude. For example:
'-81.728111,-4.2304,-66.869827,13.39029'
:type location_api: str
"""
reconnect = 0
try:
while self.threadOn:
try:
Logging.write_success_message('Filtering query: ' + query +
' and location: ' +
location_api)
twitter_stream = twitter.TwitterStream(
auth=self._API_TWITTER.auth)
if query == '':
stream = twitter_stream.statuses.filter(
locations=location_api)
else:
stream = twitter_stream.statuses.filter(
track=query, locations=location_api)
for tweet in stream:
if tweet is not None:
# Call food detector service
self.food_detector.detect_food(tweet)
else:
reconnect += 1
Logging.write_specific_error(
"Tweet is None. Reconnect TwitterStreaming #{0}"
.format(reconnect))
self._twitter_api(True)
time.sleep(10)
break
except:
Logging.write_standard_error(sys.exc_info())
finally:
reconnect += 1
Logging.write_specific_error(
'Trying new connection #{0}'.format(reconnect))
time.sleep(10)
self._twitter_api(True)
except:
Logging.write_standard_error(sys.exc_info())
def get_stream(self,key="fun"): # Reference the self.auth parameter twitter_stream = twitter.TwitterStream(auth=self.api_manager.auth) # See https://dev.twitter.com/docs/streaming-apis stream = twitter_stream.statuses.filter(track=key,language='en') # For illustrative purposes, when all else fails, search for Justin Bieber # and something is sure to turn up (at least, on Twitter) for tweet in stream: if 'text' in tweet: print tweet['text']
def login_twitter1():
CONSUMER_KEY = 'QlZUBAbMB9suzDKI1PeVsX4Hk'
CONSUMER_SECRET = 'qbclbrBtJyP3v22JMZY2plyO2PzyMIJXIqbs1NUjVrFV4HOiO5'
OAUTH_TOKEN = '781110054284103681-DLx1WVyCzSLCzH0zLYK25cU0A7vLQow'
OAUTH_TOKEN_SECRET = 'qN0EpBgFNKrpHYmhf47qhODi1vP6g8nVLs7OTAYlkJLq2'
auth1 = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,
CONSUMER_KEY, CONSUMER_SECRET)
twitter_api = twitter.TwitterStream(auth=auth1)
return twitter_api
def __init__(self):
CONSUMER_KEY = 'J13MHa67p9NthjiIC6qqaFAf5'
CONSUMER_SECRET = 'EvWGQPRsILvqkUZ9Vtua2CsPGoNuqKxBGWIvfIeYbBkxeQLdbJ'
OAUTH_TOKEN = '267913941-Ao06Y0QZ8YSb2Pj2bPgQXivhoHbvQ48G211JgQVz'
OAUTH_TOKEN_SECRET = 'depMgzGtSt6MwAZ1t6kcDr1dO57SaWkUdJL8nw4IdLlVf'
self.auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,
CONSUMER_KEY, CONSUMER_SECRET)
self.twitter_stream = twitter.TwitterStream(auth=self.auth)
def twitter_login(self, cons_key, cons_secret, access_token,
access_token_secret):
self._oauth = twitter.OAuth(access_token, access_token_secret,
cons_key, cons_secret)
self._t = twitter.Twitter(auth=self._oauth)
self._ts = twitter.TwitterStream(auth=self._oauth)
self._loggedin = True
self._credentials = self._t.account.verify_credentials()
print(u"login!")
def tweet_stream(twitter_api, q):
print >> sys.stderr, 'Filtering the public timeline for track="%s"' % (q, )
twitter_stream = twitter.TwitterStream(auth=twitter_api.auth)
stream = twitter_stream.statuses.filter(track=q)
for tweet in stream:
output_tweet = tweet['text']
print output_tweet
return output_tweet
def connect_twitter():
access_token = "934581358344790016-yq3GrSkXlZoTM0Yskv9b2DoTNBz5YUW"
access_secret = "yaG9Qpjm5fmSyE2L5hU8uM54pvLbY5eC0iAzWms9Xd2CB"
consumer_key = "lMHMMbmNxtlLHcZA9Bqxh8h6w"
consumer_secret = "KAT8bFoZknI1TYnx19Gx6p4cQqMbTtFAUOoBf1ndmHQ7eskgEX"
auth = twitter.OAuth(token=access_token,
token_secret=access_secret,
consumer_key=consumer_key,
consumer_secret=consumer_secret)
return twitter.TwitterStream(auth=auth)
def connect_twitter():
"""
Twitter API config
"""
twitter_stream = twitter.TwitterStream(auth=twitter.OAuth(
token = add_token_here,
token_secret = add_token_secret_here,
consumer_key = add_consumer_key_here),
consumer_secret = add_consumer_secret_here))
return twitter_stream
def obtainTweetsFromStream(twitter_api, q, lang, emotion, max_results):
kw = {}
kw['track'] = q
kw['language'] = lang
twitter_stream = twitter.TwitterStream(auth=twitter_api.auth)
tweets = make_twitter_request(twitter_stream.statuses.filter, **kw)
#max_results = 200#can be modified
date = time.strftime("%d%b%Y%H%M")
file_name = date + "_" + lang + "_" + emotion + "_from_stream.txt" #this text file should be moved to another directory
out_file_path = "/home/muga/twitter/tweets_from_stream/training/"
cf.validate_directory(out_file_path, True)
output = open(out_file_path + file_name, 'w')
start_time = datetime.now()
count = 0
for tweet in tweets:
if 'text' in tweet:
txt = tweet['text']
else:
break #goes outside of this for loop
if validateTweet(txt, emotion):
s = json.dumps(tweet['text'], indent=1)
#print s + ', ' + tweet['created_at'].encode('utf-8')
output.write(s + ', ' + tweet['created_at'].encode('utf-8') + '\n')
count += 1
if count % 100 == 0:
print txt + ' : ' + str(count) + ' out of ' + str(max_results)
if count > max_results:
cf.write_exec_time(start_time, output)
return
print 'goes into the while loop'
#while len(tweets) > 0 and count < max_results:
while count < max_results:
tweets = make_twitter_request(twitter_stream.statuses.filter, **kw)
for tweet in tweets:
if 'text' in tweet:
txt = tweet['text']
else:
break #goes outside of this for loop
if validateTweet(txt, emotion):
s = json.dumps(tweet['text'], indent=1)
output.write(s + ', ' + tweet['created_at'].encode('utf-8') +
'\n')
count += 1
if count % 100 == 0:
print txt + ' : ' + str(count) + ' out of ' + str(
max_results)
print 'goes out from the while loop'
output.close()
print 'Extracting ' + emotion + ' tweets of ' + lang + ' has done.'
cf.write_exec_time(start_time, output)
return
def authTWStream():
CONSUMER_KEY = 'TvCZMTQiFqfdQvTBD9pjcP7er'
CONSUMER_SECRET = 'jIsY3uRqcUjL7OaUgGq5ZDzmbDH89BmRdLoXuQxCPF2GwERZXT'
OAUTH_TOKEN = '831305604694163456-yv1JNqaALtufztgZNvwh6Y2sWh7DReo'
OAUTH_TOKEN_SECRET = '2ZAhYp7MxvXUe40pLxq5xDF377ninAJPDUzeBdejvcYHi'
auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET, CONSUMER_KEY,
CONSUMER_SECRET)
twitter_stream_api = twitter.TwitterStream(auth=auth)
return twitter_stream_api