class TweetAnalyser:
"""
Wrapper for Tweepy API and vaderSentiment.vaderSentiment
TweetAnalyser builds an abstraction between tweepy api and the twitter sentiment app
giving the developer freedom to integrate tweepy api and vaderSentiment.vaderSentiment
with a minimal change to the app code.
"""
def __init__(self, access_token, access_token_secret, consumer_key,
consumer_secret):
self.analyser = SentimentIntensityAnalyzer()
self.auth_handler = OAuthHandler(consumer_key, consumer_secret)
self.auth_handler.set_access_token(access_token, access_token_secret)
self.api = API(self.auth_handler,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
def search_tweets(self, q: str, lang='en', rpp=100, items=12):
"""
Return iterator for items in each page
"""
return Cursor(self.api.search, q, lang, rpp).items(items)
def get_polarity_scores(self, text: str):
"""
Return a float for sentiment strength based on the input text.
Positive values are positive valence, negative value are negative
valence.
:param text: the input text
:return: a float value sentiment strength
"""
return self.analyser.polarity_scores(text)
def send_tweets(c_socket):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
twitter_stream = Stream(auth, TweetsListener(c_socket))
twitter_stream.filter(track=['Iran'
]) # este es el tema que nos interesa buscar
def trends(channel='assignment1', scheduled=True):
#consumer_key = environ.get('consumer_key', None)
#consumer_secret = environ.get('consumer_secret', None)
#access_token = environ.get('access_token', None)
#access_token_secret = environ.get('access_token_secret', None)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = API(auth)
# Where On Earth ID for Philippines is 1187115.
# Where On Earth ID for Worldwide 1.
WOE_ID = 23424934
trends = api.trends_place(WOE_ID)
trends = json.loads(json.dumps(trends, indent=1))
trendy = []
for trend in trends[0]["trends"]:
trendy.append((trend["name"]))
trending = ', \n'.join(trendy[:10])
if scheduled:
slack_client.api_call("chat.postMessage",
channel=channel,
text=trending)
return trending
def request(self, method, route, params=None, json=None, user_auth=False):
host = "https://api.twitter.com"
headers = {"User-Agent": self.user_agent}
auth = None
# TODO: Ability to choose app or user auth
if user_auth:
auth = OAuthHandler(self.consumer_key, self.consumer_secret)
auth.set_access_token(self.access_token, self.access_token_secret)
auth = auth.apply_auth()
else:
headers["Authorization"] = f"Bearer {self.bearer_token}"
# TODO: log.debug
with self.session.request(
method, host + route, params=params, json=json, headers=headers,
auth=auth
) as response:
# TODO: log.debug
if response.status_code in (400, 403):
print(response.status_code)
# TODO: Handle
elif response.status_code != 200:
print(response.status_code)
# TODO: Handle
# TODO: Handle rate limits
return response.json()
def send_tweets(c_socket):
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
twitter_stream = Stream(auth, TweetsListener(c_socket))
twitter_stream.filter(track=['football'
]) # this is the topic we are interested in
def trends(channel='assignment1', scheduled=True):
consumer_key = environ.get('consumer_key', None)
consumer_secret = environ.get('consumer_secret', None)
access_token = environ.get('access_token', None)
access_token_secret = environ.get('access_token_secret', None)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = API(auth)
WOE_ID = 1
trends = api.trends_place(WOE_ID)
trends = json.loads(json.dumps(trends, indent=1))
trendy = []
for trend in trends[0]["trends"]:
trendy.append((trend["name"]))
trending = ', \n'.join(trendy[:10])
if scheduled:
slack_client.api_call("chat.postMessage",
channel=channel,
text=trending)
else:
return trending
def startStream(topicDict, keywords, users, locationDict):
print('Starting stream using', keywords)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
print('Authorization successful: ', auth.oauth.verify)
while True:
twitter_stream = Stream(auth,
TweetsListener(topicDict, locationDict, users))
try:
twitter_stream.filter(languages=['en'],
track=keywords,
follow=users) # start the stream
except KeyboardInterrupt:
print('Keyboard Interrupt')
return True
except Exception as e:
print(e)
print("IM HERE")
continue
finally:
# we do this because tweepy streams never actually clear their queued tweets when they stop/crash
# this way if something goes wrong we start fresh. We might lose some data but at least it'll keep running
del twitter_stream
def get_oauth():
conf = ConfigParser.SafeConfigParser()
conf.read("twitter.ini")
auth = OAuthHandler(conf.get("Twitter", "CK"), conf.get("Twitter", "CS"))
auth.set_access_token(conf.get("Twitter", "AT"), conf.get("Twitter", "AS"))
return auth
class TweepyStreamBackoffTests(unittest.TestCase):
def setUp(self):
#bad auth causes twitter to return 401 errors
self.auth = OAuthHandler("bad-key", "bad-secret")
self.auth.set_access_token("bad-token", "bad-token-secret")
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener)
def tearDown(self):
self.stream.disconnect()
def test_exp_backoff(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=100.0)
self.stream.sample()
# 1 retry, should be 4x the retry_time
self.assertEqual(self.stream.retry_time, 4.0)
def test_exp_backoff_cap(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=3.0)
self.stream.sample()
# 1 retry, but 4x the retry_time exceeds the cap, so should be capped
self.assertEqual(self.stream.retry_time, 3.0)
mock_resp = MagicMock()
mock_resp.return_value.status_code = 420
@patch('requests.Session.request', mock_resp)
def test_420(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0,
retry_time=1.0, retry_420=1.5, retry_time_cap=20.0)
self.stream.sample()
# no retries, but error 420, should be double the retry_420, not double the retry_time
self.assertEqual(self.stream.retry_time, 3.0)
def get_tweets():
access_token = settings.TWITTER_ACCESS_TOKEN
access_token_secret = settings.TWITTER_ACCESS_SECRET
auth = OAuthHandler(
settings.TWITTER_OAUTH_CONSUMER_KEY,
settings.TWITTER_OAUTH_CONSUMER_SECRET,
)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
twitter_users = [
settings.TWITTER_DEPT_USER,
settings.TWITTER_PERM_SEC_USER,
]
tweets = []
for twitter_user in twitter_users:
if not twitter_user:
continue
for status in tweepy.Cursor(
api.user_timeline,
screen_name=twitter_user,
include_rts=False,
exclude_replies=True,
).items(5):
tweets.append(status)
return tweets
def getTweetsByUser(username, amountOfTweets):
'''
Get the last 3 hours tweets contents and do data analysis with it.
'''
#authroization
auth = OAuthHandler(consumer_key, consumer_secret)
#access to user's key
auth.set_access_token(access_token, access_token_secret)
#calling API
api = tweepy.API(auth)
#extracted desired amount of tweet
tweets = api.user_timeline(
screen_name=username,
count=amountOfTweets,
tweet_mode='extended',
)
result = []
for info in tweets:
temp = {}
temp['id'] = str(info.id)
temp['created_at'] = info.created_at.strftime("%m/%d/%Y at %H o'clock")
temp['full_text'] = info.full_text
result.append(temp)
return result
#=================test script=================
# print getTweetsByUser("cnn", 5)
def send_tweets(c_socket):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
twitter_stream = Stream(auth, TweetsListener(c_socket))
twitter_stream.filter(track=['football'
]) # this is the topic we are interested in
def __init__(self, api_key, api_key_secret, access_token, token_secret):
'''
Initialize the twitter API client.
'''
auth_handler = OAuthHandler(api_key, api_key_secret)
auth_handler.set_access_token(access_token, token_secret)
self.twitter_client = API(auth_handler)
class TwitterClient():
def __init__(self, twitter_user=None):
self.auth = OAuthHandler(credentials.CONSUMER_KEY,
credentials.CONSUMER_SECRET)
self.auth.set_access_token(credentials.ACCESS_TOKEN,
credentials.ACCESS_TOKEN_SECRET)
#self.auth.Authenticate_Twitter_app()
self.twitter_client = tweepy.API(self.auth)
self.twitter_user = twitter_user
def getTwitterClientAPI(self):
return self.twitter_client
def getUserTimelineTweets(self, numTweets):
return [
tweet
for tweet in tweepy.Cursor(self.twitter_client.user_timeline,
id=self.twitter_user).items(numTweets)
]
def getFriendList(self, numFriends):
return [
friend
for friend in tweepy.Cursor(self.twitter_client.friends,
id=self.twitter_user).items(numFriends)
]
def getHomeTimeline(self, numTweets):
return [
tweet
for tweet in tweepy.Cursor(self.twitter_client.home_timeline,
id=self.twitter_user).items(numTweets)
]
def search_for_hashtags(consumer_key, consumer_secret, access_token,
access_token_secret, hashtag_phrase):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
#initialize Tweepy API
api = tweepy.API(auth)
#get the name of the spreadsheet we will write to
fname = '_'.join(re.findall(r"#(\w+)", hashtag_phrase))
#open the spreadsheet we will write to
with open('%s.csv' % (fname), 'w') as file:
w = csv.writer(file)
#write header row to spreadsheet
w.writerow([
'timestamp', 'tweet_text', 'username', 'all_hashtags',
'followers_count'
])
#for each tweet matching our hashtags, write relevant info to the spreadsheet
for tweet in tweepy.Cursor(api.search, q=hashtag_phrase+' -filter:retweets', \
lang="en", tweet_mode='extended').items(100):
w.writerow([
tweet.created_at,
tweet.full_text.replace('\n', ' ').encode('utf-8'),
tweet.user.screen_name.encode('utf-8'),
[e['text'] for e in tweet._json['entities']['hashtags']],
tweet.user.followers_count
])
def filter_track(follow):
auth = OAuthHandler(consumer_key2, consumer_secret2)
auth.set_access_token(access_token2, access_token_secret2)
stream = Stream(auth, MyStreamListener())
api = API(auth)
#print 'start filter track ', ','.join(track)
stream.filter(track=follow)
def get_oauth():
conf = ConfigParser.SafeConfigParser()
conf.read("twitter.ini")
auth = OAuthHandler(conf.get("Twitter", "CK"), conf.get("Twitter", "CS"))
auth.set_access_token(conf.get("Twitter", "AT"), conf.get("Twitter", "AS"))
return auth
class TweepyStreamBackoffTests(unittest.TestCase):
def setUp(self):
#bad auth causes twitter to return 401 errors
self.auth = OAuthHandler("bad-key", "bad-secret")
self.auth.set_access_token("bad-token", "bad-token-secret")
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener)
def tearDown(self):
self.stream.disconnect()
def test_exp_backoff(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=100.0)
self.stream.sample()
# 1 retry, should be 4x the retry_time
self.assertEqual(self.stream.retry_time, 4.0)
def test_exp_backoff_cap(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=3.0)
self.stream.sample()
# 1 retry, but 4x the retry_time exceeds the cap, so should be capped
self.assertEqual(self.stream.retry_time, 3.0)
mock_resp = MagicMock()
mock_resp.return_value.status = 420
@patch(getresponse_location, mock_resp)
def test_420(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0,
retry_time=1.0, retry_420=1.5, retry_time_cap=20.0)
self.stream.sample()
# no retries, but error 420, should be double the retry_420, not double the retry_time
self.assertEqual(self.stream.retry_time, 3.0)
def send_tweets(c_socket):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
twitter_stream = Stream(auth, TweetStreamListener(c_socket))
twitter_stream.filter(
languages=["nl"],
track=['de', 'het', 'een', 'is', 'ben', 'zijn', 'word', 'werd'])
def get_oauth():
consumer_key = consumer['key']
consumer_secret = consumer['secret']
access_token = token['token']
access_secret = token['secret']
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
return auth
def get_oauth():
consumer_key = 'TWITTER_CONSUMER_KEY'
consumer_secret = 'TWITTER_CONSUMER_SECRET'
access_key = 'TWITTER_ACCESS_KEY'
access_secret = 'TWITTER_ACCESS_SECRET'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def get_oauth():
consumer_key = "cJXMUBcenymJSgUPfnyHYVBqd"
consumer_secret = "JZbjCP6MzUnWLOoU6K2VouCpacmCwvYLmyXMY6wZzR6MyepvsN"
access_key = "46786366-3BrGV7f5vOXlLMMhrPG32Y3tIQBJ14qiG5rnNVDPo"
access_secret = "CMOZ61hlhuvDsddbeiYC9Qi34vXkrvyJhZa1v2AFrD3UK"
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def setup_listener(MEDIOS):
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
# Set up the listener. The 'wait_on_rate_limit=True' is needed to help with Twitter API rate limiting.
listener = StreamListener(api=tweepy.API(wait_on_rate_limit=True))
streamer = tweepy.Stream(auth=auth, listener=listener)
print("Tracking: " + str(MEDIOS))
streamer.filter(track=MEDIOS)
def get_oauth(): consumer_key = '' consumer_secret = '' access_key = '' access_secret = '' auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def _get_rate_limit_status(self, key, secret):
"""
Get rate limit status for specified access token key.
"""
auth = OAuthHandler(self.consumer_key, self.consumer_secret)
auth.set_access_token(key, secret)
api = API(auth)
return api.rate_limit_status()
def get_oauth():
consumer_key = 'TWITTER_CONSUMER_KEY'
consumer_secret = 'TWITTER_CONSUMER_SECRET'
access_key = 'TWITTER_ACCESS_KEY'
access_secret = 'TWITTER_ACCESS_SECRET'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def __init__(self):
auth = OAuthHandler(API_KEY, API_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
self.api = tweepy.API(auth)
self.username = self.api.me().screen_name
self.max_tweet_len = 270 # the limit is 280 but twitter counts some characters as 2
self.max_reply_len = self.max_tweet_len - len(self.username) - 2
def get_oauth():
consumer_key = config.get(CONFIG_SECTION, 'TWITTER_CONSUMER_KEY')
consumer_secret = config.get(CONFIG_SECTION, 'TWITTER_CONSUMER_SECRET')
access_key = config.get(CONFIG_SECTION, 'TWITTER_ACCESS_KEY')
access_secret = config.get(CONFIG_SECTION, 'TWITTER_ACCESS_SECRET')
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def send_tweets(c_socket):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
twitter_stream = Stream(auth, TweetsListener(c_socket))
twitter_stream.filter(
locations=[-180, -90, 180,
90]) # this is the topic we are interested in
def oauth():
consumer_key = ""
consumer_secret = ""
access_token = ""
access_token_secret = ""
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
return auth
def get_oauth():
consumer_key = keys.twKeys['consumer_key'].encode('utf-8')
consumer_secret = keys.twKeys['consumer_secret'].encode('utf-8')
access_key = keys.twKeys['access_token'].encode('utf-8')
access_secret = keys.twKeys['access_secret'].encode('utf-8')
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def get_oauth():
consumer_key = ''
consumer_secret = ''
access_key = ''
access_secret = ''
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def get_oauth():
consumer_key = 'dTFlSvzNIkROO6BfAkTsDQ'
consumer_secret = 'UnWZeo7rxdx38aVnGhbZSdmiraNgSUnwGuTxYdrvts'
access_key = '105981624-84OpytBltuseQ1XWx2hTAlhQO9oO8IpSiSnxgL0e'
access_secret = 'Kh3xAsVLyBiEESfRVpqp1WE4KVZH2w46JlKUU5wOI'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def get_oauth(self,):
consumer_key = self.consumer['key']
consumer_secret = self.consumer['secret']
access_token = self.token['token']
access_secret = self.token['secret']
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
return auth
def _get_rate_limit_status(self, key, secret):
"""
Get rate limit status for specified access token key.
"""
auth = OAuthHandler(self.consumer_key, self.consumer_secret)
auth.set_access_token(key, secret)
api = API(auth)
return api.rate_limit_status()
def get_oauth(init):
consumer_key = init['consumer_key']
consumer_secret = init['consumer_secret']
access_key = init['access_key']
access_secret = init['access_secret']
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def _setup_apis(self):
auth = OAuthHandler(self._consumer_key, self._consumer_secret)
auth.set_access_token(self._access_token, self._access_token_secret)
if self._raw_json:
self._api = TwitterAPI(auth, parser=RawParser())
else:
self._api = TwitterAPI(auth)
def get_oauth():
# 以下4つのキー等は適宜取得して置き換えてください。
consumer_key = ''
consumer_secret = ''
access_key = ''
access_secret = ''
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def getOauth():
CONSUMER_KEY=secret.CONSUMER_KEY
CONSUMER_SECRET=secret.CONSUMER_SECRET
ACCESS_TOKEN_KEY=secret.ACCESS_TOKEN_KEY
ACCESS_TOKEN_SECRET=secret.ACCESS_TOKEN_SECRET
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
return auth
def main_logger(basename=None):
l = RotatingLogListener(basename=basename)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
# This fetches ANY geotagged tweet:
# https://dev.twitter.com/docs/streaming-apis/parameters#locations
stream.filter(locations=[-180,-90,180,90])
def get_oauth():
# 以下4つのキー等は適宜取得して置き換えてください。
CONSUMER_KEY = '*********************'
CONSUMER_SECRET = '******************************************'
ACCESS_KEY = '**************************************************'
ACCESS_SECRET = '*******************************************'
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
return auth
def get_oauth():
# 以下4つのキー等は適宜取得して置き換えてください。
consumer_key = "E7PLfevrHNVI9ozuQlF6xXDgG"
consumer_secret = "CfobRZOWqXvPytSbn3zUDklsS81bUWXJjd3RIEYcUKnOnd1kQx"
access_key = "721661385977909249-wtx82HRTTSzm1mJgOnW0vbjR13tWCOr"
access_secret = "uFeckhWHjU5JHBOGIvPLEHQgSbfRk3SjrxUBMyLNRNh7X"
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def filter_track():
q = Scrapers.all().filter('', '')
follow = [s.uid for s in q]
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, MyStreamListener())
api = API(auth)
#print 'start filter track ', ','.join(track)
stream.filter(follow=follow)
def getOauth():
CONSUMER_KEY = secret.CONSUMER_KEY
CONSUMER_SECRET = secret.CONSUMER_SECRET
ACCESS_TOKEN_KEY = secret.ACCESS_TOKEN_KEY
ACCESS_TOKEN_SECRET = secret.ACCESS_TOKEN_SECRET
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
return auth
def get_oauth():
consumer_key = 'JpU9Lb1PEpFtLgC7XOlkw'
consumer_secret = 'k6TsCWbybimpcBMhi8o5simuuoX3XjoMGtG5Ij9U'
access_key = '132108032-vQ58uSC6HE0IOOMtmZcRjjko4gbykAOmDF4Vg3tI'
access_secret = 'Q2uMS4NvwPhAs6BthJ19ImdYCixXmxfUos1B76D4'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def filter_track():
q = Scrapers.all().filter('','')
follow=[s.uid for s in q]
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, MyStreamListener())
api = API(auth)
#print 'start filter track ', ','.join(track)
stream.filter(follow=follow)
def sendData(c_socket):
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
twitter_stream = Stream(auth, TweetsListener(c_socket))
twitter_stream.filter(track=[
'donald', 'Trump', 'US2020', 'USelection', '2020Election', 'Biden',
'JoeBiden', 'DonaldTrump'
])
def getOauth(): consumerKey = 'jCLoeEVXxq4ygfPsJma33jUTN' consumerSecret = 'yPSDOl17fqBqaSKfDa8rDlaHaHBZ6llyClcLRO1G3kV6cIWnWz' accessKey = '495476602-OalwRLSU9poetl3HuQiX9AZ8tcd9QQuhfHLjQ8xf' accessSecret = 'bQhekPGvSn87VmxSuPtP8ODCXFzZKJ1zoZnRvMThVLNof' auth = OAuthHandler(consumerKey, consumerSecret) auth.set_access_token(accessKey, accessSecret) return auth
def oauth():
# Fill in your keys here:
consumer_key = 'XXXXXXXXXXXXXXXXXXXX'
consumer_secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
access_key = 'XXXXXXXX-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
access_secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def tweet(answer):
CONSUMER_KEY = config.get("auth", "CONSUMER_KEY")
CONSUMER_SECRET = config.get("auth", "CONSUMER_SECRET")
ACCESS_TOKEN = config.get("auth", "ACCESS_TOKEN")
ACCESS_TOKEN_SECRET = config.get("auth", "ACCESS_TOKEN_SECRET")
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
api = API(auth)
result = api.update_status(status=answer)
def get_stream(self, listener=None):
if self._stream is not None:
self._stream.disconnect()
if listener is None:
listener = self._stream.listener
auth = OAuthHandler(self._consumer_key, self._consumer_secret)
auth.set_access_token(self._access_token, self._access_token_secret)
self._stream = ExStream(auth, listener)
return self._stream
def get_oauth():
inifile = ConfigParser.SafeConfigParser()
inifile.read("config.ini")
consumer_key = inifile.get("CONSUMER", "KEY")
consumer_secret = inifile.get("CONSUMER", "SECRET")
access_key = inifile.get("ACCESS","KEY")
access_secret = inifile.get("ACCESS","SECRET")
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
return auth
def build_stream(self):
if self.stream != None:
self.stop_query()
time.sleep(.01) # make sure old stream has time to disconnect
oauth = OAuthHandler(self.consumer_key, self.consumer_secret)
oauth.set_access_token(self.access_token, self.access_token_secret)
self.stream = Stream(oauth, # do OAuthentication for stream
self, # this object implements StreamListener
timeout = 600, # reconnect if no messages in 600s
retry_count = 20, # try reconnecting 20 times
retry_time = 10.0, # wait 10s if no HTTP 200
snooze_time = 1.0) # wait 1s if timeout in 600s
class NewsMeDigestTweeter(object):
def __init__(self,debug=True):
self._oauth = None
self._oauth_api = None
self._oauth_init()
self._debug = debug
self.tweet_counter = 0
self._model_queries = NewsMeModelQueries()
def _oauth_init(self):
self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET)
self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET)
self._oauth_api = API(self._oauth)
def follow_digestion_user(self,digestion_user):
pass
#try:
# if self._oauth_api.exists_friendship(digestion_user, 'twixmit') == False:
# friend = self._oauth_api.create_friendship(digestion_user)
#except TweepError, e:
# logging.error("TweepError: %s", e)
#logging.info("following: %s" % digestion_user)
def tweet_from_digestion(self,digest_articles, digestion_user):
for k, v in digest_articles.iteritems():
status_text = "%s %s" % (v,k)
model_check = self._model_queries.check_model_for_tweet(digestion_user,k)
logging.info("model check for user and link is %s" % model_check)
if not self._debug and not model_check:
try:
self._oauth_api.update_status(status=status_text,source="twixmit")
self.tweet_counter = self.tweet_counter + 1
except TweepError, e:
logging.error("TweepError: %s", e)
if not model_check:
self.save_tweet_to_model(digestion_user,k,v)
else:
logging.warn("link was already tweeted: %s" % k)
class StreamsTestsPlain(object):
def __init__(self):
self._stream = None
self._oauth = None
self._oauth_api = None
self._stream_init()
self._oauth_init()
def _oauth_init(self):
self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET)
self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET)
self._oauth_api = API(self._oauth)
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers)
def sample(self):
self._stream.sample()
def filter_follow(self):
follow_names = ['hnfirehose','StockTwits','YahooFinance','Street_Insider','TheStreet','SquawkCNBC','CNBC','AP_PersonalFin','themotleyfool','MarketWatch','Reuters_Biz']
follow_usr_objs = self._oauth_api.lookup_users(screen_names=follow_names)
follow_ids = []
for follow_usr in follow_usr_objs:
follow_ids.append(follow_usr.id)
print follow_ids
self._stream.filter(follow=follow_ids)
def filter(self):
self._stream.filter(track=["@newsdotme","@twixmit","@app_engine"])
def scrapeThread(index):
auth = OAuthHandler(consumerKeys[index], consumerSecrets[index])
auth.set_access_token(accessTokens[index], accessSecrets[index])
api = API(auth)
try:
api.verify_credentials()
except TweepError:
print "Failed to authenticate - most likely reached rate limit/incorrect credentials!"
return
else:
print "You have successfully logged on as: " + api.me().screen_name
for i in range(0, numDays):
for query in queries[index]:
count = 0
cursor = Cursor(api.search,
q=quote(query.encode('utf-8')),
lang=langs[index],
since=sinces[i],
until=untils[i],
include_entities=True).items()
while True:
try:
tweet = cursor.next()
utc = datetime.now().strftime('%Y%m%dT%H%M%S%f')
outPath = path.join(outDir, sinces[i], langs[index], utc + '.json')
with open(outPath, 'w') as output:
output.write(dumps(tweet._json, ensure_ascii=False).encode('utf8'))
count += 1
if count == int(perDay / len(queries[index])):
break
except TweepError:
print langs[index] + " - rate limit reached! Pausing thread for 15 minutes."
sleep(60 * 15)
continue
except StopIteration:
break
print str(count) + " tweets stored in " + outPath
def main():
try:
parser = ArgumentParser()
parser.add_argument("oauth_keys_file",
help="The location of a file containing the application " +
"oath consumer key, consumer secret, " +
"access token, and access token secret, " +
"each on its own line, in that order. " +
"See the tweepy example on oauth to figure " +
"out how to get these keys."
)
parser.add_argument("mail_settings_file", help="The automail settings file for sending emails.")
args = parser.parse_args()
(consumer_key, consumer_secret, access_token, access_token_secret) = \
parse_oauth_file(args.oauth_keys_file)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
pl = PaxListener(args.mail_settings_file, auth)
#open a stream, with ourself as the listener
stream = Stream(auth, pl)
#stream.filter(track=["the"])
pax_user_id = '26281970' #follow requires userid, found at mytwitterid.com
daemon_user_id = '1954653840'
stream.filter(follow=[pax_user_id]) #track ignores follow, pulls from firehose regardless (this is testing acct)
except BaseException as e:
subject = "Exception from paxtix"
exc_type, exc_value, exc_traceback = sys.exc_info()
message = '\n'.join(["Pax tix listener has hit an exception!",] +
traceback.format_exception(exc_type, exc_value, exc_traceback),
)
send_email(parse_settings(args.mail_settings_file), subject, message)
traceback.print_exception(exc_type, exc_value, exc_traceback)
def authenticate():
auth = OAuthHandler(consumer_key=API_KEY, consumer_secret=API_SECRET)
auth.set_access_token(key=ACCESS_TOKEN, secret=ACCESS_SECRET)
return auth
def create_auth():
auth = OAuthHandler(oauth_consumer_key, oauth_consumer_secret)
auth.set_access_token(oauth_token, oauth_token_secret)
return auth
class TweepyLib:
CONFIG_FILENAME = "twitterConfig.ini" # name of the config file defining e.g. the Twitter API key
CONFIG_SECTION_KEYS = "keys" # name of the section of the config file holding the various Twitter API keys/tokens/etc.
def __init__(self):
config = RawConfigParser()
configFilePath = os.path.join(os.path.dirname(__file__), self.__class__.CONFIG_FILENAME)
logger.info("HELLO WORLD")
logger.info(configFilePath)
config.read(configFilePath)
# Read API keys/tokens from config file
self.consumer_key = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_key")
self.consumer_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_secret")
self.access_token = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token")
self.access_token_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token_secret")
self.auth = OAuthHandler(self.consumer_key, self.consumer_secret)
self.isStreaming=False
self.connect()
self.tweetsQueue = Queue.Queue()
self.filter=[]
def connect(self):
#todo: recheck needed auth
self.auth.set_access_token(self.access_token, self.access_token_secret)
logger.info("Connected with Twitter")
def doStreaming(self):
class StdOutListener(StreamListener):
def __init__(self, twitterObject):
super(StdOutListener, self).__init__()
self.twitterObject = twitterObject
def on_status(self, status):
logger.info("adds new tweeter to queue with id %s: %s" % (status.id, status.text))
self.twitterObject.tweetsQueue.put(status)
return True
def on_error(self, status):
logger.warning("Error %s while reading twitter" % status)
listener = StdOutListener(self)
self.stream = Stream(self.auth, listener)
self.stream.filter(track=self.filter, async=True)
def startStreamer(self):
if(not self.filter):
logger.warning("Trying to start stream, but the filter is empty")
return
self.doStreaming()
logger.info("Streamer was started")
self.isStreaming = True
def stopStreamer(self):
if(self.isStreaming):
self.isStreaming = False
self.stream.disconnect()
logger.info("Streamer was stopped")
def restartStreamer(self):
self.stopStreamer()
self.startStreamer()
def addHashtagToFilter(self, newHashtag):
if(newHashtag[:1] != '#'):
newHashtag = '#' + newHashtag
if(newHashtag in self.filter):
return
self.filter.append(newHashtag.encode())
logger.info("%s added to the filter" % newHashtag)
self.restartStreamer()
def removeHashtagFromFilter(self, existingHashtag):
if(existingHashtag[:1] != '#'):
existingHashtag = '#' + existingHashtag
if(existingHashtag in self.filter):
return
self.filter.remove(existingHashtag)
logger.info("%s removed from the filter" % existingHashtag)
self.restartStreamer()
def removeAllHashtagsFromFilter(self):
changes = 0
for i in self.filter:
# we only want to rmeove hashtags, not other filters
if(i[:1] == '#'):
changes += 1
self.filter.remove(i)
logger.info("All hashtags removed from the filter")
if(changes != 0):
self.restartStreamer()
