def filter_track(follow): auth = OAuthHandler(consumer_key2, consumer_secret2) auth.set_access_token(access_token2, access_token_secret2) stream = Stream(auth, MyStreamListener()) api = API(auth) #print 'start filter track ', ','.join(track) stream.filter(track=follow)
def test_follow_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(follow=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['follow'])
def main(): logging.info("bot starting") bot = Bot() logging.info("markov data loading") if load_csv == True: markov = MarkovGenerator() logging.info("load_csv == True") csv_tweets = Bot.load_csv_tweets("./tweets.csv") #newest_tweet_idを保存 newest_tweet = csv_tweets.next() save_data(newest_tweet[0], "newest_tweet_id") markov.add_from_sentence(newest_tweet[7]) #読み込み for tweet in csv_tweets: markov.add_from_sentence(tweet) markov.save_dictionary() else: markov = MarkovGenerator(markov_dictionary=load_data("markov_dictionary"), reversed_markov_dictionary=load_data("reversed_markov_dictionary")) # 20分ごとの定期ツイート #multiprocess間での値のやり取り用変数 logging.info("regular tweet starting") p = Process(target=regular_tweet, args=(bot, markov)) p.start() # userstreamによるTL監視(リプライ・ふぁぼ・パクツイ・RT) logging.info("start timeline watching") stream = Stream(bot.auth, AbstractedlyListener(), secure=True) user_stream = stream.userstream() user_stream.start()
def test_track_encoding(self): s = Stream(None, None) s._start = lambda is_async: None s.filter(track=['Caf\xe9']) # Should be UTF-8 encoded self.assertEqual('Caf\xe9'.encode('utf8'), s.body['track'])
def test_track_encoding(self): s = Stream(None, None) s._start = lambda is_async: None s.filter(track=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.body['track'])
class KeywordStreamer: ''' Streams in tweets matching a set of terms/from a set of user streams. ''' def __init__(self, kw_list): self.kws = kw_list self.api = get_api() self.stream = Stream(auth=self.api.auth, listener=MyStreamListener()) def streamTweets(self): goForIt = True while goForIt: try: goForIt = False self.stream.filter(track=self.kws) except IncompleteRead as ex: try: tb = traceback.format_exc() except: tb = '' print("%s: %s\n%s\n\n" % (getTimeString(), str(sys.exc_info()) + "\nStill ticking!", tb)) time.sleep(15 * 60) goForIt = True except: try: tb = traceback.format_exc() except: tb = '' print('%s: %s\n%s\n\n' % (getTimeString(), str(sys.exc_info()), tb)) time.sleep(15 * 60) goForIt = True
def startBot(): print("") print("Initializing....") getBanAccounts() print("getting banned accounts....check") getBanWords() print("getting banned words....check") getTrackWords() print("getting track words....check") getFollowAccounts() print("getting follow accounts....check") getWhiteListAccounts() print("getting whitelist accounts....check") print("") print("""\ _____ _ _ _ _____ _____ _____ |_ _|_ _ _|_| |_| |_ ___ ___| __ | |_ _| | | | | | | | _| _| -_| _| __ -| | | | | |_| |_____|_|_| |_| |___|_| |_____|_____| |_| created by vishwenga Running.... :) """) try: twt = Stream(auths, listener()) twt.filter(track=track_words) except Exception as e: print(str(e)) pass
def init_stream(self): self.q = Queue() self.keywords = [] self.listener = TwitterStreamListener(self.keywords, self.q) auth = OAuthHandler(config.con_secret, config.con_secret_key) auth.set_access_token(config.token, config.token_key) self.stream = Stream(auth, self.listener)
def test_follow_encoding(self): s = Stream(None, None) s._start = lambda is_async: None s.filter(follow=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.body['follow'])
def test_track_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(track=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['track'])
class IRCPlugin(BasePlugin): name = "twitter_stream" enabled = True def _validate(self, event): return False def stop(self): if self._stream: self._stream.disconnect() def run(self): try: auth = tweepy.OAuthHandler(config.TWITTER_CONSUMER_KEY, config.TWITTER_CONSUMER_SECRET) auth.set_access_token(config.TWITTER_ACCESS_KEY, config.TWITTER_ACCESS_SECRET) self._stream = Stream(auth, StreamerToIrc(self)) LOG.info("Twitter stream successfully initializing") except Exception as e: LOG.error("Twitter stream authorization failed: %s" % e) return followers = [] api = API(auth) for f in config.TWITTER_FOLLOW_IDS: if isinstance(f, (str, unicode)): try: user_id = api.get_user(f).id followers.append(str(user_id)) except Exception: LOG.debug("Can't get ID for %s" % user_id) continue else: followers.append(str(f)) self._stream.filter(followers)
def main(api: tweepy.API, main_user): """ Authentication and calling the helper function """ listner = MyListner(api) stream = Stream(auth, listner) ans = stream.filter(follow=[f"{main_user}"], is_async=True)
def main(): inifile = configparser.ConfigParser() inifile.read('./user.conf') init = { 'consumer_key': inifile.get('twitter', 'consumer_key'), 'consumer_secret': inifile.get('twitter', 'consumer_secret'), 'access_key': inifile.get('twitter', 'access_key'), 'access_secret': inifile.get('twitter', 'access_secret'), } github_user = { 'repo_owner': inifile.get('github', 'repo_owner'), 'repo_name': inifile.get('github', 'repo_name'), 'username': inifile.get('github', 'username'), 'password': inifile.get('github', 'password'), } auth = get_oauth(init) stream = Stream(auth, AbstractedlyListener(github_user=github_user), secure=True) error = 0 while True: try: stream.userstream() except: error += 1 print(error, traceback.format_exc())
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_userstream_with_params(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream(_with='user', replies='all', stall_warnings=True) self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_track_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(track=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['track']) def test_follow_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(follow=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['follow'])
def filter_track(): track = ["python"] stream_auth = BasicAuthHandler('<USERNAME>', '<PASSWORD>') api = API() stream = Stream(stream_auth, MyStreamListener(api)) print 'start filter track ', ','.join(track) stream.filter(track=track)
def get_tweets(keyword): init_tweepy() auth = tweepy.OAuthHandler('QxpECDyg9eM6inD0jo7Q','a0A8OE2ZN7imCg6WR5ygkrGvKG6tNtoZIChQXQ8NIf4') auth.set_access_token('18752311-FXmc9zaPGcszH1bdDNJQa0MY2XRYfYzT3nBRnMqgB','tzXURgYPAbsD1VgmchkoKH9QOJ80qGgSSL13K5A3rY') api = tweepy.API(auth) listener = Listener() stream = Stream(auth, listener) stream.filter(track=[keyword])
def filter_track(): q = Scrapers.all().filter('','') follow=[s.uid for s in q] auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) stream = Stream(auth, MyStreamListener()) api = API(auth) #print 'start filter track ', ','.join(track) stream.filter(follow=follow)
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_userstream_with_params(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream(_with='user', replies='all', stall_warnings=True) self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_track_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(track=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['track']) def test_follow_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(follow=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['follow'])
def main_logger(basename=None): l = RotatingLogListener(basename=basename) auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) stream = Stream(auth, l) # This fetches ANY geotagged tweet: # https://dev.twitter.com/docs/streaming-apis/parameters#locations stream.filter(locations=[-180,-90,180,90])
def my_main(): args = get_parser().parse_args() auth = OAuthHandler("HqQlPfDvsu6oOVKcbSaC3dwy6", "Nr801nZ9gJoq5D6x1cQewFJeHDBnyI2IqUsYVePv7sQLLLjwxU") auth.set_access_token("2344102761-3D84t8gRyGEE2N9tBM99n8oTjGMdDJb12lIAWJ9", "MEZtBsCwbFjrku0JOnggvlGmqB2O8x8gBC7s65wkjkJGx") print("Streaming...") twitter_stream = Stream(auth, Streamer(args.data_dir)) twitter_stream.filter(track=[search_word])
def filter_track(): q = Scrapers.all().filter('', '') follow = [s.uid for s in q] auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) stream = Stream(auth, MyStreamListener()) api = API(auth) #print 'start filter track ', ','.join(track) stream.filter(follow=follow)
def test_exp_backoff_cap(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=3.0) self.stream.sample() # 1 retry, but 4x the retry_time exceeds the cap, so should be capped self.assertEqual(self.stream.retry_time, 3.0)
def test_exp_backoff(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=100.0) self.stream.sample() # 1 retry, should be 4x the retry_time self.assertEqual(self.stream.retry_time, 4.0)
def test_420(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0, retry_time=1.0, retry_420=1.5, retry_time_cap=20.0) self.stream.sample() # no retries, but error 420, should be double the retry_420, not double the retry_time self.assertEqual(self.stream.retry_time, 3.0)
def __init__(self, termPath): try: os.mkdir(OUT_DIR) except: pass kwFile = open(termPath, 'r') self.kws = [line.strip() for line in kwFile if line.strip()] kwFile.close() self.stream = Stream(getKeyAndAuth(), TermListener(NUM_TO_CACHE))
def __init__(self): listener = TweetStreamListener() auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) stream = Stream(auth, listener) stream.filter(track=[ "soccer transfer", "transfer news", "premier league", "liverpool" ], is_async=True, languages=["en"]) logging.basicConfig(filename="tweet_stream.log", level=logging.DEBUG) logging.info("Streaming Soccer Related Tweets\n")
def train_tweets(): tokyo = LocationFence("Tokyo", 138.946381, 35.523285, 139.953232, 35.906849) hokkaido = LocationFence("Hokkaido", 139.546509, 41.393294, 145.742798, 45.729191) kyusyu = LocationFence("Kyusyu", 129.538879, 31.147006, 131.856995, 33.934245) locations = [tokyo, hokkaido, kyusyu] request_coordinates = [] for l in locations: request_coordinates += l.get_coordinates() stream = Stream(oauth(), Trainer(locations), secure=True) stream.filter(locations=request_coordinates)
def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U, social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth, listener=l, secure=True, headers=headers)
def run(): # Authenticates with Twitter print('Auth') auth = authenticate() api = connect_to_api(auth) print('Handling') listener = TweepyListener(api) listener.add_handler(handler=PrintTweetHandler()) print('Streaming') stream = Stream(auth=auth, listener=listener) stream.sample()
class TweepyStreamBackoffTests(unittest.TestCase): def setUp(self): #bad auth causes twitter to return 401 errors self.auth = OAuthHandler("bad-key", "bad-secret") self.auth.set_access_token("bad-token", "bad-token-secret") self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener) def tearDown(self): self.stream.disconnect() def test_exp_backoff(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=100.0) self.stream.sample() # 1 retry, should be 4x the retry_time self.assertEqual(self.stream.retry_time, 4.0) def test_exp_backoff_cap(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=3.0) self.stream.sample() # 1 retry, but 4x the retry_time exceeds the cap, so should be capped self.assertEqual(self.stream.retry_time, 3.0) mock_resp = MagicMock() mock_resp.return_value.status_code = 420 @patch('requests.Session.request', mock_resp) def test_420(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0, retry_time=1.0, retry_420=1.5, retry_time_cap=20.0) self.stream.sample() # no retries, but error 420, should be double the retry_420, not double the retry_time self.assertEqual(self.stream.retry_time, 3.0)
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count)
class TweepyStreamBackoffTests(unittest.TestCase): def setUp(self): #bad auth causes twitter to return 401 errors self.auth = OAuthHandler("bad-key", "bad-secret") self.auth.set_access_token("bad-token", "bad-token-secret") self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener) def tearDown(self): self.stream.disconnect() def test_exp_backoff(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=100.0) self.stream.sample() # 1 retry, should be 4x the retry_time self.assertEqual(self.stream.retry_time, 4.0) def test_exp_backoff_cap(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=3.0) self.stream.sample() # 1 retry, but 4x the retry_time exceeds the cap, so should be capped self.assertEqual(self.stream.retry_time, 3.0) mock_resp = MagicMock() mock_resp.return_value.status = 420 @patch(getresponse_location, mock_resp) def test_420(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0, retry_time=1.0, retry_420=1.5, retry_time_cap=20.0) self.stream.sample() # no retries, but error 420, should be double the retry_420, not double the retry_time self.assertEqual(self.stream.retry_time, 3.0)
def main(): try: config = get_twitter_config(CONFIG_FILE, TWITTER_SCREEN_NAME) auth = tweepy.OAuthHandler(config['consumer_key'], config['consumer_secret']) auth.set_access_token(config['access_key'], config['access_secret']) # auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) # auth.set_access_token(ACCESS_KEY, ACCESS_SECRET) listener = Listener() stream = Stream(auth, listener, timeout=None,secure=1) #stream = Stream(USERNAME, PASSWORD, listener, timeout=None) # sample returns all tweets stream.filter(track = KEYWORDS) except KeyboardInterrupt: print '\nGoodbye!'
def do_stream(self): listener = twitter_listen() auth = tweepy.OAuthHandler(credentials.CONSUMER_KEY, credentials.CONSUMER_SECRET) api = tweepy.API(auth) auth.set_access_token(credentials.ACCESS_TOKEN, credentials.ACCESS_TOKEN_SECRET) stream = Stream(auth, listener) #filter tweets stream.filter(languages=["en"], track=[ 'Storm', 'Winter', 'Canada', 'Temprature', 'Flu', 'Snow', 'Indoor', 'Safety' ])
def _start_stream(self): self.listener = Streamer(self) self._auth() self.stream = Stream(self.auth, self.listener) self.stream.userstream()
def __init__(self, stream_ref, auth_params, stream_manager, db=None, bot_user_agent=None): """ :param username: twitter username :param auth: tuple of auth credentials representing twitter user :param mstream: multi stream container owning this stream `mstream` should implement listener protocol to handle user events """ # self.me = self.username = stream_id self.stream_ref = stream_ref self.stream_id = stream_ref.id self.auth_params = auth_params self.auth = TwitterApiWrapper.auth_from_settings(*auth_params) self.api = TwitterApiWrapper.make_api(self.auth) self.listener = self.ListenerCls()(self, stream_manager, db=db) headers = {'X-User-Agent': bot_user_agent} if bot_user_agent else {} self.stream = Stream(self.auth, self.listener, headers=headers) self.listener.log_event('initialized') if on_test(): self.test_stream = TestStream(self.listener)
def __listen(self): listener = TweetStreamListener(self.api, self.sentiment, self.error) stream = Stream(self.auth, listener) print 'Starting stream...' try: stream.filter(track=self.track) except: print 'Encountered error!' print 'Exiting application' item = { 'status' : 'stream down', 'timestamp' : datetime.utcnow() } self.error.save(item) stream.disconnect()
def stream(self): self.botname = self.api.me().screen_name print "Bot started: @" + self.botname self.taghandler = TagHandler(self.api, self.botname) self.customhandler = CustomHandler(self.api) self.listener = Listener(self.botname, self.taghandler, self.customhandler) self.stream = Stream(self.auth, self.listener) self.stream.filter(track=(self.botname, ))
class KeywordStreamer: ''' Streams in tweets matching a set of terms/from a set of user streams. ''' def __init__(self, termPath): try: os.mkdir(OUT_DIR) except: pass kwFile = open(termPath, 'r') self.kws = [line.strip() for line in kwFile if line.strip()] kwFile.close() self.stream = Stream(getKeyAndAuth(), TermListener(NUM_TO_CACHE)) def streamTweets(self, searchForTerms=True): goForIt = True while goForIt: try: goForIt = False if searchForTerms: self.stream.filter(track=self.kws, locations=LOCATIONS) else: self.stream.filter(follow=[int(k) for k in self.kws], locations=LOCATIONS) #except IncompleteRead as ex: # try: # tb = traceback.format_exc() # except: # tb = '' # emailAlert("%s: %s\n%s\n\n" % (getTimeString(), # str(sys.exc_info()) + "\nStill ticking!", tb)) # zzz(SNOOZE) # goForIt = True except: try: tb = traceback.format_exc() except: tb = '' emailAlert('%s: %s\n%s\n\n' % (getTimeString(), str(sys.exc_info()), tb)) zzz(SNOOZE) goForIt = True
def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers)
class Crawler(): ''' The global crawler manager ''' def __init__(self, name): self.name = name self.regexp = re.compile(r'@%s\s+' % (self.name), re.IGNORECASE) self._start_stream() def _auth(self): self.auth = OAuthHandler(Ukey, Usecret) self.auth.set_access_token(Akey, Asecret) self.api = API(self.auth) def _start_stream(self): self.listener = Streamer(self) self._auth() self.stream = Stream(self.auth, self.listener) self.stream.userstream() def on_request(self, text, author): text = re.sub(self.regexp, '', text) text = re.sub(r'\s+$', '', text) lang = get_language(text, key=u'startword') if lang is not None: core = Core(lang) self.send_message(core.send_message(), author) def send_message(self, text, target=None): if(target is not None): text = u'@%s %s' % (target, text) print(text) print '>>>', len(text) self.api.update_status(text)
def __init__(self, settings, publishers): """Constructor.""" if "consumer_key" not in settings: raise ValueError("consumer_key must not be None") if "consumer_secret" not in settings: raise ValueError("consumer_secret must not be None") if "access_token" not in settings: raise ValueError("access_token must not be None") if "access_secret" not in settings: raise ValueError("access_secret must not be None") if "fields" not in settings: raise ValueError("access_secret must not be None") self.auth = OAuthHandler(settings["consumer_key"], settings["consumer_secret"]) self.auth.set_access_token(settings["access_token"], settings["access_secret"]) twitter_stream = Stream(self.auth, StdOutListener(settings["fields"], publishers)) twitter_stream.filter(track=settings["post_strings"])
def main(): """ Main run function """ set_logging() logger = logging.getLogger(f"{__name__}.main") auth = tweepy.OAuthHandler(os.getenv("CONSUMER_KEY"), os.getenv("CONSUMER_KEY_SECRET")) auth.set_access_token(os.getenv("ACCESS_TOKEN"), os.getenv("ACCESS_TOKEN_SECRET")) api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True) following = get_following_users(api) following.extend(get_default_users(api)) tracks = get_following_searches() logger.info(f"Following {len(following)} users and {len(tracks)} searches") while True: listener = Listener() stream = Stream(auth=api.auth, listener=listener) try: logger.info("Started streaming") stream.filter(follow=following, track=tracks) except KeyboardInterrupt: logger.info("Stopped") break except ReadTimeoutError as exc: logger.error("Handled exception:", str(exc), exc_info=True) finally: logger.info("Done") stream.disconnect()
def main(): try: parser = ArgumentParser() parser.add_argument("oauth_keys_file", help="The location of a file containing the application " + "oath consumer key, consumer secret, " + "access token, and access token secret, " + "each on its own line, in that order. " + "See the tweepy example on oauth to figure " + "out how to get these keys." ) parser.add_argument("mail_settings_file", help="The automail settings file for sending emails.") args = parser.parse_args() (consumer_key, consumer_secret, access_token, access_token_secret) = \ parse_oauth_file(args.oauth_keys_file) auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) pl = PaxListener(args.mail_settings_file, auth) #open a stream, with ourself as the listener stream = Stream(auth, pl) #stream.filter(track=["the"]) pax_user_id = '26281970' #follow requires userid, found at mytwitterid.com daemon_user_id = '1954653840' stream.filter(follow=[pax_user_id]) #track ignores follow, pulls from firehose regardless (this is testing acct) except BaseException as e: subject = "Exception from paxtix" exc_type, exc_value, exc_traceback = sys.exc_info() message = '\n'.join(["Pax tix listener has hit an exception!",] + traceback.format_exception(exc_type, exc_value, exc_traceback), ) send_email(parse_settings(args.mail_settings_file), subject, message) traceback.print_exception(exc_type, exc_value, exc_traceback)
class Bot: consumer_key='FSWJUlCHsAaiLBDY1MMEA' consumer_secret='aCFv6Tslb038cdUAIv7m4QDplBn076wvLd4YKmC3yU' key = "135845084-FONlvU4wGXyl9o4RkfyPdv4grclbqr4Pi2UH2w3j" secret = "DC6I1xp2uONTIYOL2DtEIljhl46LEkUwlPB5NUEBds" def __init__(self): self.auth = tweepy.OAuthHandler(consumer_key = self.consumer_key, consumer_secret = self.consumer_secret) self.auth.set_access_token(self.key, self.secret) self.api = tweepy.API(self.auth) def auth(self): try: redirect_url = self.auth.get_authorization_url() except tweepy.TweepError: print 'Error! Failed to get request token.' verifier = raw_input('Verifier:') try: self.auth.get_access_token(verifier) except tweepy.TweepError: print 'Error! Failed to get access token.' self.key, self.secret = self.auth.access_token.key, self.auth.access_token.secret print self.key, self.secret def stream(self): self.botname = self.api.me().screen_name print "Bot started: @" + self.botname self.taghandler = TagHandler(self.api, self.botname) self.customhandler = CustomHandler(self.api) self.listener = Listener(self.botname, self.taghandler, self.customhandler) self.stream = Stream(self.auth, self.listener) self.stream.filter(track=(self.botname, ))
def handle(self, *args, **options): if not args or len(args) < 3: self.stdout.write('No hashtags and/or options provided.\n') else: # Fetch info from DB hashtag_name = args.pop(0) hashtag = HashTag.objects.get(name=hashtag_name) options = VoteOption.objects.filter(hashtag=hashtag) self.stdout.write('Following: %s\n' % hashtag_name) self.stdout.write('Options: %s\n' % ', '.join(options)) self.stdout.flush() # OAuth connect consumer_key = settings.TWITTERVOTER['consumer_key'] consumer_secret = settings.TWITTERVOTER['consumer_secret'] access_token = settings.TWITTERVOTER['access_token'] access_token_secret = settings.TWITTERVOTER['access_token_secret'] auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) # Connect too the streaming API. mylisten = MyStreamListener(self.stdout) mystream = Stream(auth, mylisten, timeout=30) mystream.filter(track=args)
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_on_data(self): test_wrong_data = [ '{"disc', # this is actual data read from twitter '600', # this is actual data read from twitter '41\n', # this is actual data read from twitter 'obviously non-json', '"json but not dict"', '{"json dict":"but not a twitter message"}' ] for raw_data in test_wrong_data: # should log errors but not raise / not return False self.assertEquals(self.listener.on_data(raw_data), None) self.assertEquals(self.listener.status_count, 0)
def doStreaming(self): class StdOutListener(StreamListener): def __init__(self, twitterObject): super(StdOutListener, self).__init__() self.twitterObject = twitterObject def on_status(self, status): logger.info("adds new tweeter to queue with id %s: %s" % (status.id, status.text)) self.twitterObject.tweetsQueue.put(status) return True def on_error(self, status): logger.warning("Error %s while reading twitter" % status) listener = StdOutListener(self) self.stream = Stream(self.auth, listener) self.stream.filter(track=self.filter, async=True)
class StreamsTestsPlain(object): def __init__(self): self._stream = None self._oauth = None self._oauth_api = None self._stream_init() self._oauth_init() def _oauth_init(self): self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET) self._oauth.set_access_token( social_keys.TWITTER_APP_ACCESS_TOKEN, social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET) self._oauth_api = API(self._oauth) def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U, social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth, listener=l, secure=True, headers=headers) def sample(self): self._stream.sample() def filter_follow(self): follow_names = [ 'hnfirehose', 'StockTwits', 'YahooFinance', 'Street_Insider', 'TheStreet', 'SquawkCNBC', 'CNBC', 'AP_PersonalFin', 'themotleyfool', 'MarketWatch', 'Reuters_Biz' ] follow_usr_objs = self._oauth_api.lookup_users( screen_names=follow_names) follow_ids = [] for follow_usr in follow_usr_objs: follow_ids.append(follow_usr.id) print follow_ids self._stream.filter(follow=follow_ids) def filter(self): self._stream.filter(track=["@newsdotme", "@twixmit", "@app_engine"])
def run(self): try: auth = tweepy.OAuthHandler(config.TWITTER_CONSUMER_KEY, config.TWITTER_CONSUMER_SECRET) auth.set_access_token(config.TWITTER_ACCESS_KEY, config.TWITTER_ACCESS_SECRET) self._stream = Stream(auth, StreamerToIrc(self)) LOG.info("Twitter stream successfully initializing") except Exception as e: LOG.error("Twitter stream authorization failed: %s" % e) return followers = [] api = API(auth) for f in config.TWITTER_FOLLOW_IDS: if isinstance(f, (str, unicode)): try: user_id = api.get_user(f).id followers.append(user_id) except Exception: LOG.debug("Can't get ID for %s" % user_id) continue else: followers.append(f) self._stream.filter(followers)
class StreamsTestsPlain(object): def __init__(self): self._stream = None self._oauth = None self._oauth_api = None self._stream_init() self._oauth_init() def _oauth_init(self): self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET) self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET) self._oauth_api = API(self._oauth) def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers) def sample(self): self._stream.sample() def filter_follow(self): follow_names = ['hnfirehose','StockTwits','YahooFinance','Street_Insider','TheStreet','SquawkCNBC','CNBC','AP_PersonalFin','themotleyfool','MarketWatch','Reuters_Biz'] follow_usr_objs = self._oauth_api.lookup_users(screen_names=follow_names) follow_ids = [] for follow_usr in follow_usr_objs: follow_ids.append(follow_usr.id) print follow_ids self._stream.filter(follow=follow_ids) def filter(self): self._stream.filter(track=["@newsdotme","@twixmit","@app_engine"])
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def on_connect(self): API(self.auth).update_status(mock_tweet()) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEqual(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEqual(self.listener.status_count, self.listener.status_stop_count) def test_track_encoding(self): s = Stream(None, None) s._start = lambda is_async: None s.filter(track=['Caf\xe9']) # Should be UTF-8 encoded self.assertEqual('Caf\xe9'.encode('utf8'), s.body['track']) def test_follow_encoding(self): s = Stream(None, None) s._start = lambda is_async: None s.filter(follow=['Caf\xe9']) # Should be UTF-8 encoded self.assertEqual('Caf\xe9'.encode('utf8'), s.body['follow'])
def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0)
def retweet_by_all(status): for auth in get_client_auths(): api = tweepy.API(auth) api.retweet(status["id"]) print "i will retweet", status["id"], status["text"] class StdOutListener(StreamListener): # listens to stream and will start retweet_by_all if condition def on_data(self, data): status = json.loads(data) if condition(status): retweet_by_all(status) def on_error(self, status): print status if __name__ == "__main__": # starts main loop auth = get_main_identity_auth() api = tweepy.API(auth) stream = Stream(auth, StdOutListener(StreamListener)) stream.filter(config["listen"])
self.output_filenames = output_filenames self.OpenOutputFiles() def ReloadOutputFiles(self): self.CloseOutputFiles() self.OpenOutputFiles() """ Let's stare abstractedly at the User Streams ! """ def on_status(self, status): dump_text = json.dumps(status._json) + "\n" if self.output_files is None: print dump_text sys.stdout.flush() return for file in self.output_files: file.write(dump_text) file.flush() listener = Listener() def SigHupHandler(signum, frame): listener.ReloadOutputFiles() if __name__ == '__main__': signal.signal(signal.SIGHUP, SigHupHandler) auth = get_oauth() listener.SetOutputFilenames(sys.argv[1:]) stream = Stream(auth, listener, secure=True) daemon.daemonize(PID_FILE_NAME) stream.userstream()
class TweepyLib: CONFIG_FILENAME = "twitterConfig.ini" # name of the config file defining e.g. the Twitter API key CONFIG_SECTION_KEYS = "keys" # name of the section of the config file holding the various Twitter API keys/tokens/etc. def __init__(self): config = RawConfigParser() configFilePath = os.path.join(os.path.dirname(__file__), self.__class__.CONFIG_FILENAME) logger.info("HELLO WORLD") logger.info(configFilePath) config.read(configFilePath) # Read API keys/tokens from config file self.consumer_key = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_key") self.consumer_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_secret") self.access_token = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token") self.access_token_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token_secret") self.auth = OAuthHandler(self.consumer_key, self.consumer_secret) self.isStreaming=False self.connect() self.tweetsQueue = Queue.Queue() self.filter=[] def connect(self): #todo: recheck needed auth self.auth.set_access_token(self.access_token, self.access_token_secret) logger.info("Connected with Twitter") def doStreaming(self): class StdOutListener(StreamListener): def __init__(self, twitterObject): super(StdOutListener, self).__init__() self.twitterObject = twitterObject def on_status(self, status): logger.info("adds new tweeter to queue with id %s: %s" % (status.id, status.text)) self.twitterObject.tweetsQueue.put(status) return True def on_error(self, status): logger.warning("Error %s while reading twitter" % status) listener = StdOutListener(self) self.stream = Stream(self.auth, listener) self.stream.filter(track=self.filter, async=True) def startStreamer(self): if(not self.filter): logger.warning("Trying to start stream, but the filter is empty") return self.doStreaming() logger.info("Streamer was started") self.isStreaming = True def stopStreamer(self): if(self.isStreaming): self.isStreaming = False self.stream.disconnect() logger.info("Streamer was stopped") def restartStreamer(self): self.stopStreamer() self.startStreamer() def addHashtagToFilter(self, newHashtag): if(newHashtag[:1] != '#'): newHashtag = '#' + newHashtag if(newHashtag in self.filter): return self.filter.append(newHashtag.encode()) logger.info("%s added to the filter" % newHashtag) self.restartStreamer() def removeHashtagFromFilter(self, existingHashtag): if(existingHashtag[:1] != '#'): existingHashtag = '#' + existingHashtag if(existingHashtag in self.filter): return self.filter.remove(existingHashtag) logger.info("%s removed from the filter" % existingHashtag) self.restartStreamer() def removeAllHashtagsFromFilter(self): changes = 0 for i in self.filter: # we only want to rmeove hashtags, not other filters if(i[:1] == '#'): changes += 1 self.filter.remove(i) logger.info("All hashtags removed from the filter") if(changes != 0): self.restartStreamer()