class TweetAnalyser: """ Wrapper for Tweepy API and vaderSentiment.vaderSentiment TweetAnalyser builds an abstraction between tweepy api and the twitter sentiment app giving the developer freedom to integrate tweepy api and vaderSentiment.vaderSentiment with a minimal change to the app code. """ def __init__(self, access_token, access_token_secret, consumer_key, consumer_secret): self.analyser = SentimentIntensityAnalyzer() self.auth_handler = OAuthHandler(consumer_key, consumer_secret) self.auth_handler.set_access_token(access_token, access_token_secret) self.api = API(self.auth_handler, wait_on_rate_limit=True, wait_on_rate_limit_notify=True) def search_tweets(self, q: str, lang='en', rpp=100, items=12): """ Return iterator for items in each page """ return Cursor(self.api.search, q, lang, rpp).items(items) def get_polarity_scores(self, text: str): """ Return a float for sentiment strength based on the input text. Positive values are positive valence, negative value are negative valence. :param text: the input text :return: a float value sentiment strength """ return self.analyser.polarity_scores(text)
def send_tweets(c_socket): auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) twitter_stream = Stream(auth, TweetsListener(c_socket)) twitter_stream.filter(track=['Iran' ]) # este es el tema que nos interesa buscar
def trends(channel='assignment1', scheduled=True): #consumer_key = environ.get('consumer_key', None) #consumer_secret = environ.get('consumer_secret', None) #access_token = environ.get('access_token', None) #access_token_secret = environ.get('access_token_secret', None) auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) api = API(auth) # Where On Earth ID for Philippines is 1187115. # Where On Earth ID for Worldwide 1. WOE_ID = 23424934 trends = api.trends_place(WOE_ID) trends = json.loads(json.dumps(trends, indent=1)) trendy = [] for trend in trends[0]["trends"]: trendy.append((trend["name"])) trending = ', \n'.join(trendy[:10]) if scheduled: slack_client.api_call("chat.postMessage", channel=channel, text=trending) return trending
def request(self, method, route, params=None, json=None, user_auth=False): host = "https://api.twitter.com" headers = {"User-Agent": self.user_agent} auth = None # TODO: Ability to choose app or user auth if user_auth: auth = OAuthHandler(self.consumer_key, self.consumer_secret) auth.set_access_token(self.access_token, self.access_token_secret) auth = auth.apply_auth() else: headers["Authorization"] = f"Bearer {self.bearer_token}" # TODO: log.debug with self.session.request( method, host + route, params=params, json=json, headers=headers, auth=auth ) as response: # TODO: log.debug if response.status_code in (400, 403): print(response.status_code) # TODO: Handle elif response.status_code != 200: print(response.status_code) # TODO: Handle # TODO: Handle rate limits return response.json()
def send_tweets(c_socket): auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET) twitter_stream = Stream(auth, TweetsListener(c_socket)) twitter_stream.filter(track=['football' ]) # this is the topic we are interested in
def trends(channel='assignment1', scheduled=True): consumer_key = environ.get('consumer_key', None) consumer_secret = environ.get('consumer_secret', None) access_token = environ.get('access_token', None) access_token_secret = environ.get('access_token_secret', None) auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) api = API(auth) WOE_ID = 1 trends = api.trends_place(WOE_ID) trends = json.loads(json.dumps(trends, indent=1)) trendy = [] for trend in trends[0]["trends"]: trendy.append((trend["name"])) trending = ', \n'.join(trendy[:10]) if scheduled: slack_client.api_call("chat.postMessage", channel=channel, text=trending) else: return trending
def startStream(topicDict, keywords, users, locationDict): print('Starting stream using', keywords) auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) print('Authorization successful: ', auth.oauth.verify) while True: twitter_stream = Stream(auth, TweetsListener(topicDict, locationDict, users)) try: twitter_stream.filter(languages=['en'], track=keywords, follow=users) # start the stream except KeyboardInterrupt: print('Keyboard Interrupt') return True except Exception as e: print(e) print("IM HERE") continue finally: # we do this because tweepy streams never actually clear their queued tweets when they stop/crash # this way if something goes wrong we start fresh. We might lose some data but at least it'll keep running del twitter_stream
def get_oauth(): conf = ConfigParser.SafeConfigParser() conf.read("twitter.ini") auth = OAuthHandler(conf.get("Twitter", "CK"), conf.get("Twitter", "CS")) auth.set_access_token(conf.get("Twitter", "AT"), conf.get("Twitter", "AS")) return auth
class TweepyStreamBackoffTests(unittest.TestCase): def setUp(self): #bad auth causes twitter to return 401 errors self.auth = OAuthHandler("bad-key", "bad-secret") self.auth.set_access_token("bad-token", "bad-token-secret") self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener) def tearDown(self): self.stream.disconnect() def test_exp_backoff(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=100.0) self.stream.sample() # 1 retry, should be 4x the retry_time self.assertEqual(self.stream.retry_time, 4.0) def test_exp_backoff_cap(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=3.0) self.stream.sample() # 1 retry, but 4x the retry_time exceeds the cap, so should be capped self.assertEqual(self.stream.retry_time, 3.0) mock_resp = MagicMock() mock_resp.return_value.status_code = 420 @patch('requests.Session.request', mock_resp) def test_420(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0, retry_time=1.0, retry_420=1.5, retry_time_cap=20.0) self.stream.sample() # no retries, but error 420, should be double the retry_420, not double the retry_time self.assertEqual(self.stream.retry_time, 3.0)
def get_tweets(): access_token = settings.TWITTER_ACCESS_TOKEN access_token_secret = settings.TWITTER_ACCESS_SECRET auth = OAuthHandler( settings.TWITTER_OAUTH_CONSUMER_KEY, settings.TWITTER_OAUTH_CONSUMER_SECRET, ) auth.set_access_token(access_token, access_token_secret) api = tweepy.API(auth) twitter_users = [ settings.TWITTER_DEPT_USER, settings.TWITTER_PERM_SEC_USER, ] tweets = [] for twitter_user in twitter_users: if not twitter_user: continue for status in tweepy.Cursor( api.user_timeline, screen_name=twitter_user, include_rts=False, exclude_replies=True, ).items(5): tweets.append(status) return tweets
def getTweetsByUser(username, amountOfTweets): ''' Get the last 3 hours tweets contents and do data analysis with it. ''' #authroization auth = OAuthHandler(consumer_key, consumer_secret) #access to user's key auth.set_access_token(access_token, access_token_secret) #calling API api = tweepy.API(auth) #extracted desired amount of tweet tweets = api.user_timeline( screen_name=username, count=amountOfTweets, tweet_mode='extended', ) result = [] for info in tweets: temp = {} temp['id'] = str(info.id) temp['created_at'] = info.created_at.strftime("%m/%d/%Y at %H o'clock") temp['full_text'] = info.full_text result.append(temp) return result #=================test script================= # print getTweetsByUser("cnn", 5)
def send_tweets(c_socket): auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) twitter_stream = Stream(auth, TweetsListener(c_socket)) twitter_stream.filter(track=['football' ]) # this is the topic we are interested in
def __init__(self, api_key, api_key_secret, access_token, token_secret): ''' Initialize the twitter API client. ''' auth_handler = OAuthHandler(api_key, api_key_secret) auth_handler.set_access_token(access_token, token_secret) self.twitter_client = API(auth_handler)
class TwitterClient(): def __init__(self, twitter_user=None): self.auth = OAuthHandler(credentials.CONSUMER_KEY, credentials.CONSUMER_SECRET) self.auth.set_access_token(credentials.ACCESS_TOKEN, credentials.ACCESS_TOKEN_SECRET) #self.auth.Authenticate_Twitter_app() self.twitter_client = tweepy.API(self.auth) self.twitter_user = twitter_user def getTwitterClientAPI(self): return self.twitter_client def getUserTimelineTweets(self, numTweets): return [ tweet for tweet in tweepy.Cursor(self.twitter_client.user_timeline, id=self.twitter_user).items(numTweets) ] def getFriendList(self, numFriends): return [ friend for friend in tweepy.Cursor(self.twitter_client.friends, id=self.twitter_user).items(numFriends) ] def getHomeTimeline(self, numTweets): return [ tweet for tweet in tweepy.Cursor(self.twitter_client.home_timeline, id=self.twitter_user).items(numTweets) ]
def search_for_hashtags(consumer_key, consumer_secret, access_token, access_token_secret, hashtag_phrase): auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) #initialize Tweepy API api = tweepy.API(auth) #get the name of the spreadsheet we will write to fname = '_'.join(re.findall(r"#(\w+)", hashtag_phrase)) #open the spreadsheet we will write to with open('%s.csv' % (fname), 'w') as file: w = csv.writer(file) #write header row to spreadsheet w.writerow([ 'timestamp', 'tweet_text', 'username', 'all_hashtags', 'followers_count' ]) #for each tweet matching our hashtags, write relevant info to the spreadsheet for tweet in tweepy.Cursor(api.search, q=hashtag_phrase+' -filter:retweets', \ lang="en", tweet_mode='extended').items(100): w.writerow([ tweet.created_at, tweet.full_text.replace('\n', ' ').encode('utf-8'), tweet.user.screen_name.encode('utf-8'), [e['text'] for e in tweet._json['entities']['hashtags']], tweet.user.followers_count ])
def filter_track(follow): auth = OAuthHandler(consumer_key2, consumer_secret2) auth.set_access_token(access_token2, access_token_secret2) stream = Stream(auth, MyStreamListener()) api = API(auth) #print 'start filter track ', ','.join(track) stream.filter(track=follow)
class TweepyStreamBackoffTests(unittest.TestCase): def setUp(self): #bad auth causes twitter to return 401 errors self.auth = OAuthHandler("bad-key", "bad-secret") self.auth.set_access_token("bad-token", "bad-token-secret") self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener) def tearDown(self): self.stream.disconnect() def test_exp_backoff(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=100.0) self.stream.sample() # 1 retry, should be 4x the retry_time self.assertEqual(self.stream.retry_time, 4.0) def test_exp_backoff_cap(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=3.0) self.stream.sample() # 1 retry, but 4x the retry_time exceeds the cap, so should be capped self.assertEqual(self.stream.retry_time, 3.0) mock_resp = MagicMock() mock_resp.return_value.status = 420 @patch(getresponse_location, mock_resp) def test_420(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0, retry_time=1.0, retry_420=1.5, retry_time_cap=20.0) self.stream.sample() # no retries, but error 420, should be double the retry_420, not double the retry_time self.assertEqual(self.stream.retry_time, 3.0)
def send_tweets(c_socket): auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) twitter_stream = Stream(auth, TweetStreamListener(c_socket)) twitter_stream.filter( languages=["nl"], track=['de', 'het', 'een', 'is', 'ben', 'zijn', 'word', 'werd'])
def get_oauth(): consumer_key = consumer['key'] consumer_secret = consumer['secret'] access_token = token['token'] access_secret = token['secret'] auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) return auth
def get_oauth(): consumer_key = 'TWITTER_CONSUMER_KEY' consumer_secret = 'TWITTER_CONSUMER_SECRET' access_key = 'TWITTER_ACCESS_KEY' access_secret = 'TWITTER_ACCESS_SECRET' auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def get_oauth(): consumer_key = "cJXMUBcenymJSgUPfnyHYVBqd" consumer_secret = "JZbjCP6MzUnWLOoU6K2VouCpacmCwvYLmyXMY6wZzR6MyepvsN" access_key = "46786366-3BrGV7f5vOXlLMMhrPG32Y3tIQBJ14qiG5rnNVDPo" access_secret = "CMOZ61hlhuvDsddbeiYC9Qi34vXkrvyJhZa1v2AFrD3UK" auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def setup_listener(MEDIOS): auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) # Set up the listener. The 'wait_on_rate_limit=True' is needed to help with Twitter API rate limiting. listener = StreamListener(api=tweepy.API(wait_on_rate_limit=True)) streamer = tweepy.Stream(auth=auth, listener=listener) print("Tracking: " + str(MEDIOS)) streamer.filter(track=MEDIOS)
def get_oauth(): consumer_key = '' consumer_secret = '' access_key = '' access_secret = '' auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def _get_rate_limit_status(self, key, secret): """ Get rate limit status for specified access token key. """ auth = OAuthHandler(self.consumer_key, self.consumer_secret) auth.set_access_token(key, secret) api = API(auth) return api.rate_limit_status()
def __init__(self): auth = OAuthHandler(API_KEY, API_SECRET) auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) self.api = tweepy.API(auth) self.username = self.api.me().screen_name self.max_tweet_len = 270 # the limit is 280 but twitter counts some characters as 2 self.max_reply_len = self.max_tweet_len - len(self.username) - 2
def get_oauth(): consumer_key = config.get(CONFIG_SECTION, 'TWITTER_CONSUMER_KEY') consumer_secret = config.get(CONFIG_SECTION, 'TWITTER_CONSUMER_SECRET') access_key = config.get(CONFIG_SECTION, 'TWITTER_ACCESS_KEY') access_secret = config.get(CONFIG_SECTION, 'TWITTER_ACCESS_SECRET') auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def send_tweets(c_socket): auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) twitter_stream = Stream(auth, TweetsListener(c_socket)) twitter_stream.filter( locations=[-180, -90, 180, 90]) # this is the topic we are interested in
def oauth(): consumer_key = "" consumer_secret = "" access_token = "" access_token_secret = "" auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) return auth
def get_oauth(): consumer_key = keys.twKeys['consumer_key'].encode('utf-8') consumer_secret = keys.twKeys['consumer_secret'].encode('utf-8') access_key = keys.twKeys['access_token'].encode('utf-8') access_secret = keys.twKeys['access_secret'].encode('utf-8') auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def get_oauth(): consumer_key = 'dTFlSvzNIkROO6BfAkTsDQ' consumer_secret = 'UnWZeo7rxdx38aVnGhbZSdmiraNgSUnwGuTxYdrvts' access_key = '105981624-84OpytBltuseQ1XWx2hTAlhQO9oO8IpSiSnxgL0e' access_secret = 'Kh3xAsVLyBiEESfRVpqp1WE4KVZH2w46JlKUU5wOI' auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def get_oauth(self,): consumer_key = self.consumer['key'] consumer_secret = self.consumer['secret'] access_token = self.token['token'] access_secret = self.token['secret'] auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) return auth
def get_oauth(init): consumer_key = init['consumer_key'] consumer_secret = init['consumer_secret'] access_key = init['access_key'] access_secret = init['access_secret'] auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def _setup_apis(self): auth = OAuthHandler(self._consumer_key, self._consumer_secret) auth.set_access_token(self._access_token, self._access_token_secret) if self._raw_json: self._api = TwitterAPI(auth, parser=RawParser()) else: self._api = TwitterAPI(auth)
def get_oauth(): # 以下4つのキー等は適宜取得して置き換えてください。 consumer_key = '' consumer_secret = '' access_key = '' access_secret = '' auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def getOauth(): CONSUMER_KEY=secret.CONSUMER_KEY CONSUMER_SECRET=secret.CONSUMER_SECRET ACCESS_TOKEN_KEY=secret.ACCESS_TOKEN_KEY ACCESS_TOKEN_SECRET=secret.ACCESS_TOKEN_SECRET auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET) return auth
def main_logger(basename=None): l = RotatingLogListener(basename=basename) auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) stream = Stream(auth, l) # This fetches ANY geotagged tweet: # https://dev.twitter.com/docs/streaming-apis/parameters#locations stream.filter(locations=[-180,-90,180,90])
def get_oauth(): # 以下4つのキー等は適宜取得して置き換えてください。 CONSUMER_KEY = '*********************' CONSUMER_SECRET = '******************************************' ACCESS_KEY = '**************************************************' ACCESS_SECRET = '*******************************************' auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_KEY, ACCESS_SECRET) return auth
def get_oauth(): # 以下4つのキー等は適宜取得して置き換えてください。 consumer_key = "E7PLfevrHNVI9ozuQlF6xXDgG" consumer_secret = "CfobRZOWqXvPytSbn3zUDklsS81bUWXJjd3RIEYcUKnOnd1kQx" access_key = "721661385977909249-wtx82HRTTSzm1mJgOnW0vbjR13tWCOr" access_secret = "uFeckhWHjU5JHBOGIvPLEHQgSbfRk3SjrxUBMyLNRNh7X" auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def filter_track(): q = Scrapers.all().filter('', '') follow = [s.uid for s in q] auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) stream = Stream(auth, MyStreamListener()) api = API(auth) #print 'start filter track ', ','.join(track) stream.filter(follow=follow)
def getOauth(): CONSUMER_KEY = secret.CONSUMER_KEY CONSUMER_SECRET = secret.CONSUMER_SECRET ACCESS_TOKEN_KEY = secret.ACCESS_TOKEN_KEY ACCESS_TOKEN_SECRET = secret.ACCESS_TOKEN_SECRET auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET) return auth
def get_oauth(): consumer_key = 'JpU9Lb1PEpFtLgC7XOlkw' consumer_secret = 'k6TsCWbybimpcBMhi8o5simuuoX3XjoMGtG5Ij9U' access_key = '132108032-vQ58uSC6HE0IOOMtmZcRjjko4gbykAOmDF4Vg3tI' access_secret = 'Q2uMS4NvwPhAs6BthJ19ImdYCixXmxfUos1B76D4' auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def filter_track(): q = Scrapers.all().filter('','') follow=[s.uid for s in q] auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) stream = Stream(auth, MyStreamListener()) api = API(auth) #print 'start filter track ', ','.join(track) stream.filter(follow=follow)
def sendData(c_socket): auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) twitter_stream = Stream(auth, TweetsListener(c_socket)) twitter_stream.filter(track=[ 'donald', 'Trump', 'US2020', 'USelection', '2020Election', 'Biden', 'JoeBiden', 'DonaldTrump' ])
def getOauth(): consumerKey = 'jCLoeEVXxq4ygfPsJma33jUTN' consumerSecret = 'yPSDOl17fqBqaSKfDa8rDlaHaHBZ6llyClcLRO1G3kV6cIWnWz' accessKey = '495476602-OalwRLSU9poetl3HuQiX9AZ8tcd9QQuhfHLjQ8xf' accessSecret = 'bQhekPGvSn87VmxSuPtP8ODCXFzZKJ1zoZnRvMThVLNof' auth = OAuthHandler(consumerKey, consumerSecret) auth.set_access_token(accessKey, accessSecret) return auth
def oauth(): # Fill in your keys here: consumer_key = 'XXXXXXXXXXXXXXXXXXXX' consumer_secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' access_key = 'XXXXXXXX-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' access_secret = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def tweet(answer): CONSUMER_KEY = config.get("auth", "CONSUMER_KEY") CONSUMER_SECRET = config.get("auth", "CONSUMER_SECRET") ACCESS_TOKEN = config.get("auth", "ACCESS_TOKEN") ACCESS_TOKEN_SECRET = config.get("auth", "ACCESS_TOKEN_SECRET") auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) api = API(auth) result = api.update_status(status=answer)
def get_stream(self, listener=None): if self._stream is not None: self._stream.disconnect() if listener is None: listener = self._stream.listener auth = OAuthHandler(self._consumer_key, self._consumer_secret) auth.set_access_token(self._access_token, self._access_token_secret) self._stream = ExStream(auth, listener) return self._stream
def get_oauth(): inifile = ConfigParser.SafeConfigParser() inifile.read("config.ini") consumer_key = inifile.get("CONSUMER", "KEY") consumer_secret = inifile.get("CONSUMER", "SECRET") access_key = inifile.get("ACCESS","KEY") access_secret = inifile.get("ACCESS","SECRET") auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) return auth
def build_stream(self): if self.stream != None: self.stop_query() time.sleep(.01) # make sure old stream has time to disconnect oauth = OAuthHandler(self.consumer_key, self.consumer_secret) oauth.set_access_token(self.access_token, self.access_token_secret) self.stream = Stream(oauth, # do OAuthentication for stream self, # this object implements StreamListener timeout = 600, # reconnect if no messages in 600s retry_count = 20, # try reconnecting 20 times retry_time = 10.0, # wait 10s if no HTTP 200 snooze_time = 1.0) # wait 1s if timeout in 600s
class NewsMeDigestTweeter(object): def __init__(self,debug=True): self._oauth = None self._oauth_api = None self._oauth_init() self._debug = debug self.tweet_counter = 0 self._model_queries = NewsMeModelQueries() def _oauth_init(self): self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET) self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET) self._oauth_api = API(self._oauth) def follow_digestion_user(self,digestion_user): pass #try: # if self._oauth_api.exists_friendship(digestion_user, 'twixmit') == False: # friend = self._oauth_api.create_friendship(digestion_user) #except TweepError, e: # logging.error("TweepError: %s", e) #logging.info("following: %s" % digestion_user) def tweet_from_digestion(self,digest_articles, digestion_user): for k, v in digest_articles.iteritems(): status_text = "%s %s" % (v,k) model_check = self._model_queries.check_model_for_tweet(digestion_user,k) logging.info("model check for user and link is %s" % model_check) if not self._debug and not model_check: try: self._oauth_api.update_status(status=status_text,source="twixmit") self.tweet_counter = self.tweet_counter + 1 except TweepError, e: logging.error("TweepError: %s", e) if not model_check: self.save_tweet_to_model(digestion_user,k,v) else: logging.warn("link was already tweeted: %s" % k)
class StreamsTestsPlain(object): def __init__(self): self._stream = None self._oauth = None self._oauth_api = None self._stream_init() self._oauth_init() def _oauth_init(self): self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET) self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET) self._oauth_api = API(self._oauth) def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers) def sample(self): self._stream.sample() def filter_follow(self): follow_names = ['hnfirehose','StockTwits','YahooFinance','Street_Insider','TheStreet','SquawkCNBC','CNBC','AP_PersonalFin','themotleyfool','MarketWatch','Reuters_Biz'] follow_usr_objs = self._oauth_api.lookup_users(screen_names=follow_names) follow_ids = [] for follow_usr in follow_usr_objs: follow_ids.append(follow_usr.id) print follow_ids self._stream.filter(follow=follow_ids) def filter(self): self._stream.filter(track=["@newsdotme","@twixmit","@app_engine"])
def scrapeThread(index): auth = OAuthHandler(consumerKeys[index], consumerSecrets[index]) auth.set_access_token(accessTokens[index], accessSecrets[index]) api = API(auth) try: api.verify_credentials() except TweepError: print "Failed to authenticate - most likely reached rate limit/incorrect credentials!" return else: print "You have successfully logged on as: " + api.me().screen_name for i in range(0, numDays): for query in queries[index]: count = 0 cursor = Cursor(api.search, q=quote(query.encode('utf-8')), lang=langs[index], since=sinces[i], until=untils[i], include_entities=True).items() while True: try: tweet = cursor.next() utc = datetime.now().strftime('%Y%m%dT%H%M%S%f') outPath = path.join(outDir, sinces[i], langs[index], utc + '.json') with open(outPath, 'w') as output: output.write(dumps(tweet._json, ensure_ascii=False).encode('utf8')) count += 1 if count == int(perDay / len(queries[index])): break except TweepError: print langs[index] + " - rate limit reached! Pausing thread for 15 minutes." sleep(60 * 15) continue except StopIteration: break print str(count) + " tweets stored in " + outPath
def main(): try: parser = ArgumentParser() parser.add_argument("oauth_keys_file", help="The location of a file containing the application " + "oath consumer key, consumer secret, " + "access token, and access token secret, " + "each on its own line, in that order. " + "See the tweepy example on oauth to figure " + "out how to get these keys." ) parser.add_argument("mail_settings_file", help="The automail settings file for sending emails.") args = parser.parse_args() (consumer_key, consumer_secret, access_token, access_token_secret) = \ parse_oauth_file(args.oauth_keys_file) auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) pl = PaxListener(args.mail_settings_file, auth) #open a stream, with ourself as the listener stream = Stream(auth, pl) #stream.filter(track=["the"]) pax_user_id = '26281970' #follow requires userid, found at mytwitterid.com daemon_user_id = '1954653840' stream.filter(follow=[pax_user_id]) #track ignores follow, pulls from firehose regardless (this is testing acct) except BaseException as e: subject = "Exception from paxtix" exc_type, exc_value, exc_traceback = sys.exc_info() message = '\n'.join(["Pax tix listener has hit an exception!",] + traceback.format_exception(exc_type, exc_value, exc_traceback), ) send_email(parse_settings(args.mail_settings_file), subject, message) traceback.print_exception(exc_type, exc_value, exc_traceback)
def authenticate(): auth = OAuthHandler(consumer_key=API_KEY, consumer_secret=API_SECRET) auth.set_access_token(key=ACCESS_TOKEN, secret=ACCESS_SECRET) return auth
def create_auth(): auth = OAuthHandler(oauth_consumer_key, oauth_consumer_secret) auth.set_access_token(oauth_token, oauth_token_secret) return auth
class TweepyLib: CONFIG_FILENAME = "twitterConfig.ini" # name of the config file defining e.g. the Twitter API key CONFIG_SECTION_KEYS = "keys" # name of the section of the config file holding the various Twitter API keys/tokens/etc. def __init__(self): config = RawConfigParser() configFilePath = os.path.join(os.path.dirname(__file__), self.__class__.CONFIG_FILENAME) logger.info("HELLO WORLD") logger.info(configFilePath) config.read(configFilePath) # Read API keys/tokens from config file self.consumer_key = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_key") self.consumer_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_secret") self.access_token = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token") self.access_token_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token_secret") self.auth = OAuthHandler(self.consumer_key, self.consumer_secret) self.isStreaming=False self.connect() self.tweetsQueue = Queue.Queue() self.filter=[] def connect(self): #todo: recheck needed auth self.auth.set_access_token(self.access_token, self.access_token_secret) logger.info("Connected with Twitter") def doStreaming(self): class StdOutListener(StreamListener): def __init__(self, twitterObject): super(StdOutListener, self).__init__() self.twitterObject = twitterObject def on_status(self, status): logger.info("adds new tweeter to queue with id %s: %s" % (status.id, status.text)) self.twitterObject.tweetsQueue.put(status) return True def on_error(self, status): logger.warning("Error %s while reading twitter" % status) listener = StdOutListener(self) self.stream = Stream(self.auth, listener) self.stream.filter(track=self.filter, async=True) def startStreamer(self): if(not self.filter): logger.warning("Trying to start stream, but the filter is empty") return self.doStreaming() logger.info("Streamer was started") self.isStreaming = True def stopStreamer(self): if(self.isStreaming): self.isStreaming = False self.stream.disconnect() logger.info("Streamer was stopped") def restartStreamer(self): self.stopStreamer() self.startStreamer() def addHashtagToFilter(self, newHashtag): if(newHashtag[:1] != '#'): newHashtag = '#' + newHashtag if(newHashtag in self.filter): return self.filter.append(newHashtag.encode()) logger.info("%s added to the filter" % newHashtag) self.restartStreamer() def removeHashtagFromFilter(self, existingHashtag): if(existingHashtag[:1] != '#'): existingHashtag = '#' + existingHashtag if(existingHashtag in self.filter): return self.filter.remove(existingHashtag) logger.info("%s removed from the filter" % existingHashtag) self.restartStreamer() def removeAllHashtagsFromFilter(self): changes = 0 for i in self.filter: # we only want to rmeove hashtags, not other filters if(i[:1] == '#'): changes += 1 self.filter.remove(i) logger.info("All hashtags removed from the filter") if(changes != 0): self.restartStreamer()