class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_userstream_with_params(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream(_with='user',
replies='all',
stall_warnings=True)
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(track=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['track'])
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(follow=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['follow'])
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_userstream_with_params(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream(_with='user', replies='all', stall_warnings=True)
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(track=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['track'])
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(follow=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['follow'])
class StreamsTestsPlain(object):
def __init__(self):
self._stream = None
self._oauth = None
self._oauth_api = None
self._stream_init()
self._oauth_init()
def _oauth_init(self):
self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY,
social_keys.TWITTER_CONSUMER_SECRET)
self._oauth.set_access_token(
social_keys.TWITTER_APP_ACCESS_TOKEN,
social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET)
self._oauth_api = API(self._oauth)
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,
social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,
listener=l,
secure=True,
headers=headers)
def sample(self):
self._stream.sample()
def filter_follow(self):
follow_names = [
'hnfirehose', 'StockTwits', 'YahooFinance', 'Street_Insider',
'TheStreet', 'SquawkCNBC', 'CNBC', 'AP_PersonalFin',
'themotleyfool', 'MarketWatch', 'Reuters_Biz'
]
follow_usr_objs = self._oauth_api.lookup_users(
screen_names=follow_names)
follow_ids = []
for follow_usr in follow_usr_objs:
follow_ids.append(follow_usr.id)
print follow_ids
self._stream.filter(follow=follow_ids)
def filter(self):
self._stream.filter(track=["@newsdotme", "@twixmit", "@app_engine"])
def run():
# Authenticates with Twitter
print('Auth')
auth = authenticate()
api = connect_to_api(auth)
print('Handling')
listener = TweepyListener(api)
listener.add_handler(handler=PrintTweetHandler())
print('Streaming')
stream = Stream(auth=auth, listener=listener)
stream.sample()
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_on_data(self):
test_wrong_data = [
'{"disc', # this is actual data read from twitter
'600', # this is actual data read from twitter
'41\n', # this is actual data read from twitter
'obviously non-json',
'"json but not dict"',
'{"json dict":"but not a twitter message"}'
]
for raw_data in test_wrong_data:
# should log errors but not raise / not return False
self.assertEquals(self.listener.on_data(raw_data), None)
self.assertEquals(self.listener.status_count, 0)
class StreamsTestsPlain(object):
def __init__(self):
self._stream = None
self._oauth = None
self._oauth_api = None
self._stream_init()
self._oauth_init()
def _oauth_init(self):
self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET)
self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET)
self._oauth_api = API(self._oauth)
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers)
def sample(self):
self._stream.sample()
def filter_follow(self):
follow_names = ['hnfirehose','StockTwits','YahooFinance','Street_Insider','TheStreet','SquawkCNBC','CNBC','AP_PersonalFin','themotleyfool','MarketWatch','Reuters_Biz']
follow_usr_objs = self._oauth_api.lookup_users(screen_names=follow_names)
follow_ids = []
for follow_usr in follow_usr_objs:
follow_ids.append(follow_usr.id)
print follow_ids
self._stream.filter(follow=follow_ids)
def filter(self):
self._stream.filter(track=["@newsdotme","@twixmit","@app_engine"])
class TweepyStreamBackoffTests(unittest.TestCase):
def setUp(self):
#bad auth causes twitter to return 401 errors
self.auth = OAuthHandler("bad-key", "bad-secret")
self.auth.set_access_token("bad-token", "bad-token-secret")
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener)
def tearDown(self):
self.stream.disconnect()
def test_exp_backoff(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=100.0)
self.stream.sample()
# 1 retry, should be 4x the retry_time
self.assertEqual(self.stream.retry_time, 4.0)
def test_exp_backoff_cap(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=3.0)
self.stream.sample()
# 1 retry, but 4x the retry_time exceeds the cap, so should be capped
self.assertEqual(self.stream.retry_time, 3.0)
mock_resp = MagicMock()
mock_resp.return_value.status_code = 420
@patch('requests.Session.request', mock_resp)
def test_420(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0,
retry_time=1.0, retry_420=1.5, retry_time_cap=20.0)
self.stream.sample()
# no retries, but error 420, should be double the retry_420, not double the retry_time
self.assertEqual(self.stream.retry_time, 3.0)
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def on_connect(self):
API(self.auth).update_status(mock_tweet())
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEqual(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEqual(self.listener.status_count,
self.listener.status_stop_count)
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda is_async: None
s.filter(track=['Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual('Caf\xe9'.encode('utf8'), s.body['track'])
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda is_async: None
s.filter(follow=['Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual('Caf\xe9'.encode('utf8'), s.body['follow'])
class TweepyStreamBackoffTests(unittest.TestCase):
def setUp(self):
#bad auth causes twitter to return 401 errors
self.auth = OAuthHandler("bad-key", "bad-secret")
self.auth.set_access_token("bad-token", "bad-token-secret")
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener)
def tearDown(self):
self.stream.disconnect()
def test_exp_backoff(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=100.0)
self.stream.sample()
# 1 retry, should be 4x the retry_time
self.assertEqual(self.stream.retry_time, 4.0)
def test_exp_backoff_cap(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=3.0)
self.stream.sample()
# 1 retry, but 4x the retry_time exceeds the cap, so should be capped
self.assertEqual(self.stream.retry_time, 3.0)
mock_resp = MagicMock()
mock_resp.return_value.status = 420
@patch(getresponse_location, mock_resp)
def test_420(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0,
retry_time=1.0, retry_420=1.5, retry_time_cap=20.0)
self.stream.sample()
# no retries, but error 420, should be double the retry_420, not double the retry_time
self.assertEqual(self.stream.retry_time, 3.0)
def classify_tweets(label):
stream = Stream(oauth(), TweetAnalyzer(label), secure=True)
stream.sample()
def print_to_terminal():
listener = PrintListener()
stream = Stream(auth, listener)
languages = ('en', ) # Only
stream.sample(languages=languages)
def classify_tweets(label):
stream = Stream(oauth(), TweetAnalyzer(label), secure=True)
stream.sample()
def on_error(self, status_code):
print >> sys.stderr, 'Encountered error with status code:', status_code
return True # Don't kill the stream
def on_timeout(self):
print >> sys.stderr, 'Timeout...'
return True # Don't kill the stream
if __name__ == '__main__':
#con = Connection('localhost')
#db = con.tweettest
#col = db.foo
auth = get_oauth()
stream = Stream(auth, AbstractedlyListener(), secure=True)
stream.timeout = None
# stream.filter(track=["twitter"])
# stream.filter(languages=['ja'], track=['ja'])
while True:
try:
stream.sample()
except Exception:
time.sleep(20)
print "sleep 20 sec"
stream = Stream(auth, AbstractedlyListener(), secure=True)
