class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count)
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count)
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_userstream_with_params(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream(_with='user', replies='all', stall_warnings=True) self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_track_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(track=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['track']) def test_follow_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(follow=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['follow'])
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_userstream_with_params(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream(_with='user', replies='all', stall_warnings=True) self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_track_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(track=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['track']) def test_follow_encoding(self): s = Stream(None, None) s._start = lambda async: None s.filter(follow=[u'Caf\xe9']) # Should be UTF-8 encoded self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['follow'])
class StreamsTestsPlain(object): def __init__(self): self._stream = None self._oauth = None self._oauth_api = None self._stream_init() self._oauth_init() def _oauth_init(self): self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET) self._oauth.set_access_token( social_keys.TWITTER_APP_ACCESS_TOKEN, social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET) self._oauth_api = API(self._oauth) def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U, social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth, listener=l, secure=True, headers=headers) def sample(self): self._stream.sample() def filter_follow(self): follow_names = [ 'hnfirehose', 'StockTwits', 'YahooFinance', 'Street_Insider', 'TheStreet', 'SquawkCNBC', 'CNBC', 'AP_PersonalFin', 'themotleyfool', 'MarketWatch', 'Reuters_Biz' ] follow_usr_objs = self._oauth_api.lookup_users( screen_names=follow_names) follow_ids = [] for follow_usr in follow_usr_objs: follow_ids.append(follow_usr.id) print follow_ids self._stream.filter(follow=follow_ids) def filter(self): self._stream.filter(track=["@newsdotme", "@twixmit", "@app_engine"])
def run(): # Authenticates with Twitter print('Auth') auth = authenticate() api = connect_to_api(auth) print('Handling') listener = TweepyListener(api) listener.add_handler(handler=PrintTweetHandler()) print('Streaming') stream = Stream(auth=auth, listener=listener) stream.sample()
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def test_userstream(self): # Generate random tweet which should show up in the stream. def on_connect(): API(self.auth).update_status(mock_tweet()) self.listener.connect_cb = on_connect self.listener.status_stop_count = 1 self.stream.userstream() self.assertEqual(self.listener.status_count, 1) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEquals(self.listener.status_count, self.listener.status_stop_count) def test_on_data(self): test_wrong_data = [ '{"disc', # this is actual data read from twitter '600', # this is actual data read from twitter '41\n', # this is actual data read from twitter 'obviously non-json', '"json but not dict"', '{"json dict":"but not a twitter message"}' ] for raw_data in test_wrong_data: # should log errors but not raise / not return False self.assertEquals(self.listener.on_data(raw_data), None) self.assertEquals(self.listener.status_count, 0)
class StreamsTestsPlain(object): def __init__(self): self._stream = None self._oauth = None self._oauth_api = None self._stream_init() self._oauth_init() def _oauth_init(self): self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET) self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET) self._oauth_api = API(self._oauth) def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers) def sample(self): self._stream.sample() def filter_follow(self): follow_names = ['hnfirehose','StockTwits','YahooFinance','Street_Insider','TheStreet','SquawkCNBC','CNBC','AP_PersonalFin','themotleyfool','MarketWatch','Reuters_Biz'] follow_usr_objs = self._oauth_api.lookup_users(screen_names=follow_names) follow_ids = [] for follow_usr in follow_usr_objs: follow_ids.append(follow_usr.id) print follow_ids self._stream.filter(follow=follow_ids) def filter(self): self._stream.filter(track=["@newsdotme","@twixmit","@app_engine"])
class TweepyStreamBackoffTests(unittest.TestCase): def setUp(self): #bad auth causes twitter to return 401 errors self.auth = OAuthHandler("bad-key", "bad-secret") self.auth.set_access_token("bad-token", "bad-token-secret") self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener) def tearDown(self): self.stream.disconnect() def test_exp_backoff(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=100.0) self.stream.sample() # 1 retry, should be 4x the retry_time self.assertEqual(self.stream.retry_time, 4.0) def test_exp_backoff_cap(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=3.0) self.stream.sample() # 1 retry, but 4x the retry_time exceeds the cap, so should be capped self.assertEqual(self.stream.retry_time, 3.0) mock_resp = MagicMock() mock_resp.return_value.status_code = 420 @patch('requests.Session.request', mock_resp) def test_420(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0, retry_time=1.0, retry_420=1.5, retry_time_cap=20.0) self.stream.sample() # no retries, but error 420, should be double the retry_420, not double the retry_time self.assertEqual(self.stream.retry_time, 3.0)
class TweepyStreamTests(unittest.TestCase): def setUp(self): self.auth = create_auth() self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener, timeout=3.0) def tearDown(self): self.stream.disconnect() def on_connect(self): API(self.auth).update_status(mock_tweet()) def test_sample(self): self.listener.status_stop_count = 10 self.stream.sample() self.assertEqual(self.listener.status_count, self.listener.status_stop_count) def test_filter_track(self): self.listener.status_stop_count = 5 phrases = ['twitter'] self.stream.filter(track=phrases) self.assertEqual(self.listener.status_count, self.listener.status_stop_count) def test_track_encoding(self): s = Stream(None, None) s._start = lambda is_async: None s.filter(track=['Caf\xe9']) # Should be UTF-8 encoded self.assertEqual('Caf\xe9'.encode('utf8'), s.body['track']) def test_follow_encoding(self): s = Stream(None, None) s._start = lambda is_async: None s.filter(follow=['Caf\xe9']) # Should be UTF-8 encoded self.assertEqual('Caf\xe9'.encode('utf8'), s.body['follow'])
class TweepyStreamBackoffTests(unittest.TestCase): def setUp(self): #bad auth causes twitter to return 401 errors self.auth = OAuthHandler("bad-key", "bad-secret") self.auth.set_access_token("bad-token", "bad-token-secret") self.listener = MockStreamListener(self) self.stream = Stream(self.auth, self.listener) def tearDown(self): self.stream.disconnect() def test_exp_backoff(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=100.0) self.stream.sample() # 1 retry, should be 4x the retry_time self.assertEqual(self.stream.retry_time, 4.0) def test_exp_backoff_cap(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=1, retry_time=1.0, retry_time_cap=3.0) self.stream.sample() # 1 retry, but 4x the retry_time exceeds the cap, so should be capped self.assertEqual(self.stream.retry_time, 3.0) mock_resp = MagicMock() mock_resp.return_value.status = 420 @patch(getresponse_location, mock_resp) def test_420(self): self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0, retry_time=1.0, retry_420=1.5, retry_time_cap=20.0) self.stream.sample() # no retries, but error 420, should be double the retry_420, not double the retry_time self.assertEqual(self.stream.retry_time, 3.0)
def classify_tweets(label): stream = Stream(oauth(), TweetAnalyzer(label), secure=True) stream.sample()
def print_to_terminal(): listener = PrintListener() stream = Stream(auth, listener) languages = ('en', ) # Only stream.sample(languages=languages)
def classify_tweets(label): stream = Stream(oauth(), TweetAnalyzer(label), secure=True) stream.sample()
def on_error(self, status_code): print >> sys.stderr, 'Encountered error with status code:', status_code return True # Don't kill the stream def on_timeout(self): print >> sys.stderr, 'Timeout...' return True # Don't kill the stream if __name__ == '__main__': #con = Connection('localhost') #db = con.tweettest #col = db.foo auth = get_oauth() stream = Stream(auth, AbstractedlyListener(), secure=True) stream.timeout = None # stream.filter(track=["twitter"]) # stream.filter(languages=['ja'], track=['ja']) while True: try: stream.sample() except Exception: time.sleep(20) print "sleep 20 sec" stream = Stream(auth, AbstractedlyListener(), secure=True)