def setUp(self):
# Mock tweepy.Stream
self.stream_patcher = mock.patch('tweepy.Stream')
self.MockTweepyStream = self.stream_patcher.start()
self.tweepy_stream_instance = mock.Mock()
self.MockTweepyStream.return_value = self.tweepy_stream_instance
# Mock the supporting objects
self.auth = mock.Mock()
self.listener = mock.Mock()
self.checker = mock.Mock()
# For controlling the tracking terms
self.term_list = []
self.checker.tracking_terms.return_value = self.term_list
self.stop_timeout = DynamicTwitterStream.STOP_TIMEOUT
DynamicTwitterStream.STOP_TIMEOUT = 0
self.retry_count = 7
# Create a dynamic twitter stream
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count)
def test_start_unfiltered_no_terms(self):
# Create an unfiltered stream instead
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True)
# Start the stream without a term
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# But it should a stream even without any terms
self.MockTweepyStream.assert_called_once_with(
self.auth,
self.listener,
stall_warnings=True,
timeout=90,
retry_count=self.retry_count)
# It should be using the sample endpoint
self.tweepy_stream_instance.sample.assert_called_once_with(
is_async=True, languages=None)
def setUp(self):
# Mock tweepy.Stream
self.stream_patcher = mock.patch('tweepy.Stream')
self.MockTweepyStream = self.stream_patcher.start()
self.tweepy_stream_instance = mock.Mock()
self.MockTweepyStream.return_value = self.tweepy_stream_instance
# Mock the supporting objects
self.auth = mock.Mock()
self.listener = mock.Mock()
self.checker = mock.Mock()
# For controlling the tracking terms
self.term_list = []
self.checker.tracking_terms.return_value = self.term_list
self.stop_timeout = DynamicTwitterStream.STOP_TIMEOUT
DynamicTwitterStream.STOP_TIMEOUT = 0
self.retry_count = 7
# Create a dynamic twitter stream
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count)
def test_start_with_languages(self):
# Create a stream with languages
languages = ['en', 'fr']
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True,
languages=languages)
# Start the stream without a term
self.stream.start_stream()
# It should be using the sample endpoint with languages
self.tweepy_stream_instance.sample.assert_called_once_with(
is_async=True, languages=languages)
def test_start_with_languages(self):
# Create a stream with languages
languages = ['en', 'fr']
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True,
languages=languages)
# Start the stream without a term
self.stream.start_stream()
# It should be using the sample endpoint with languages
self.tweepy_stream_instance.sample.assert_called_once_with(async=True, languages=languages)
def test_start_unfiltered_no_terms(self):
# Create an unfiltered stream instead
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True)
# Start the stream without a term
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# But it should a stream even without any terms
self.MockTweepyStream.assert_called_once_with(self.auth, self.listener,
stall_warnings=True,
timeout=90,
retry_count=self.retry_count)
# It should be using the sample endpoint
self.tweepy_stream_instance.sample.assert_called_once_with(async=True, languages=None)
class TestDynamicTwitterStream(TestCase):
def setUp(self):
# Mock tweepy.Stream
self.stream_patcher = mock.patch('tweepy.Stream')
self.MockTweepyStream = self.stream_patcher.start()
self.tweepy_stream_instance = mock.Mock()
self.MockTweepyStream.return_value = self.tweepy_stream_instance
# Mock the supporting objects
self.auth = mock.Mock()
self.listener = mock.Mock()
self.checker = mock.Mock()
# For controlling the tracking terms
self.term_list = []
self.checker.tracking_terms.return_value = self.term_list
self.stop_timeout = DynamicTwitterStream.STOP_TIMEOUT
DynamicTwitterStream.STOP_TIMEOUT = 0
self.retry_count = 7
# Create a dynamic twitter stream
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count)
def tearDown(self):
self.stream_patcher.stop()
# Restore the normal stop timeout
DynamicTwitterStream.STOP_TIMEOUT = self.stop_timeout
def test_start_stream_no_terms(self):
# Start the stream without a term
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# A stream should NOT have been created, because no terms yet
self.MockTweepyStream.assert_has_calls([])
def test_start_unfiltered_no_terms(self):
# Create an unfiltered stream instead
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True)
# Start the stream without a term
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# But it should a stream even without any terms
self.MockTweepyStream.assert_called_once_with(self.auth, self.listener,
stall_warnings=True,
timeout=90,
retry_count=self.retry_count)
# It should be using the sample endpoint
self.tweepy_stream_instance.sample.assert_called_once_with(async=True, languages=None)
def test_start_with_languages(self):
# Create a stream with languages
languages = ['en', 'fr']
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True,
languages=languages)
# Start the stream without a term
self.stream.start_stream()
# It should be using the sample endpoint with languages
self.tweepy_stream_instance.sample.assert_called_once_with(async=True, languages=languages)
def test_start_stream_with_terms(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# Should create a Stream instance
self.MockTweepyStream.assert_called_once_with(self.auth, self.listener,
stall_warnings=True,
timeout=90,
retry_count=self.retry_count)
# Should start the filter with the terms
self.tweepy_stream_instance.filter.assert_called_once_with(track=self.term_list, async=True, languages=None)
def test_stop_stream_not_started(self):
self.stream.stop_stream()
# No attempt to disconnect stream that isn't started
self.tweepy_stream_instance.disconnect.assert_has_calls([])
def test_stop_stream_started(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
self.stream.stop_stream()
# Should try to disconnect tweepy stream
self.tweepy_stream_instance.disconnect.assert_called_once_with()
def test_update_stream_terms_unchanged(self):
self.checker.check.return_value = False
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
self.stream.update_stream()
# Should have checked if terms changed
self.checker.check.assert_called_once_with()
# Should NOT have stopped the old stream
self.assertEqual(self.stream.stop_stream.call_count, 0)
# Should NOT have started a new stream
self.assertEqual(self.stream.start_stream.call_count, 0)
def test_update_stream_terms_changed(self):
self.checker.check.return_value = True
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
self.stream.update_stream()
# Should have checked if terms changed
self.checker.check.assert_called_once_with()
# Should have stopped the old stream
self.stream.stop_stream.assert_called_once_with()
# Should have started a new stream
self.stream.start_stream.assert_called_once_with()
def test_update_stream_after_error(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
# Simulate a dead stream
self.tweepy_stream_instance.running = False
self.listener.error = 500
self.stream.update_stream()
# Should have stopped the old stream
self.stream.stop_stream.assert_called_once_with()
# Should have started a new stream
self.stream.start_stream.assert_called_once_with()
# Should have turned off the error
self.assertFalse(self.listener.error)
def test_update_stream_after_exception(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
# Simulate an exception inside Tweepy
self.tweepy_stream_instance.running = False
self.listener.streaming_exception = Exception("testing")
self.stream.update_stream()
# Should have stopped the old stream
self.stream.stop_stream.assert_called_once_with()
# Should have started a new stream
self.stream.start_stream.assert_called_once_with()
# Should have turned off the exception
self.assertFalse(self.listener.streaming_exception)
def test_handle_exceptions(self):
self.listener.streaming_exception = None
try:
self.stream.handle_exceptions()
except Exception:
self.fail("Raised exception when no streaming exception set")
self.listener.streaming_exception = Exception("testing")
self.assertRaises(Exception, self.stream.handle_exceptions)
def test_clears_exceptions(self):
self.listener.streaming_exception = Exception("testing")
try:
self.stream.handle_exceptions()
except:
pass
self.assertEquals(self.listener.streaming_exception, None)
def test_stop_polling(self):
# Mock some supporting methods we test separately
self.stream.update_stream = mock.Mock()
self.stream.handle_exceptions = mock.Mock()
# This should poll every 1 second
thread = threading.Thread(target=self.stream.start_polling, args=[1])
thread.start()
waits = 0
# Wait for a maximum of 3 seconds (ish), or until the loop has run at least once
while self.stream.update_stream.call_count < 2 and waits < 12:
print "Waiting..."
time.sleep(0.25)
waits += 1
self.assertTrue(self.stream.polling)
# Try to stop the thread
self.stream.stop_polling()
self.assertFalse(self.stream.polling)
# Wait for a maximimum of 2 seconds
thread.join(timeout=2)
self.assertTrue(self.stream.update_stream.call_count >= 1, "Checked for stream/term updates")
self.assertTrue(self.stream.handle_exceptions.call_count >= 1, "Checked for stream exceptions")
class TestDynamicTwitterStream(TestCase):
def setUp(self):
# Mock tweepy.Stream
self.stream_patcher = mock.patch('tweepy.Stream')
self.MockTweepyStream = self.stream_patcher.start()
self.tweepy_stream_instance = mock.Mock()
self.MockTweepyStream.return_value = self.tweepy_stream_instance
# Mock the supporting objects
self.auth = mock.Mock()
self.listener = mock.Mock()
self.checker = mock.Mock()
# For controlling the tracking terms
self.term_list = []
self.checker.tracking_terms.return_value = self.term_list
self.stop_timeout = DynamicTwitterStream.STOP_TIMEOUT
DynamicTwitterStream.STOP_TIMEOUT = 0
self.retry_count = 7
# Create a dynamic twitter stream
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count)
def tearDown(self):
self.stream_patcher.stop()
# Restore the normal stop timeout
DynamicTwitterStream.STOP_TIMEOUT = self.stop_timeout
def test_start_stream_no_terms(self):
# Start the stream without a term
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# A stream should NOT have been created, because no terms yet
self.MockTweepyStream.assert_has_calls([])
def test_start_unfiltered_no_terms(self):
# Create an unfiltered stream instead
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True)
# Start the stream without a term
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# But it should a stream even without any terms
self.MockTweepyStream.assert_called_once_with(
self.auth,
self.listener,
stall_warnings=True,
timeout=90,
retry_count=self.retry_count)
# It should be using the sample endpoint
self.tweepy_stream_instance.sample.assert_called_once_with(
is_async=True, languages=None)
def test_start_with_languages(self):
# Create a stream with languages
languages = ['en', 'fr']
self.stream = DynamicTwitterStream(auth=self.auth,
listener=self.listener,
term_checker=self.checker,
retry_count=self.retry_count,
unfiltered=True,
languages=languages)
# Start the stream without a term
self.stream.start_stream()
# It should be using the sample endpoint with languages
self.tweepy_stream_instance.sample.assert_called_once_with(
is_async=True, languages=languages)
def test_start_stream_with_terms(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
# Should check the list of terms
self.checker.tracking_terms.assert_called_once_with()
# Should create a Stream instance
self.MockTweepyStream.assert_called_once_with(
self.auth,
self.listener,
stall_warnings=True,
timeout=90,
retry_count=self.retry_count)
# Should start the filter with the terms
self.tweepy_stream_instance.filter.assert_called_once_with(
track=self.term_list, is_async=True, languages=None)
def test_stop_stream_not_started(self):
self.stream.stop_stream()
# No attempt to disconnect stream that isn't started
self.tweepy_stream_instance.disconnect.assert_has_calls([])
def test_stop_stream_started(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
self.stream.stop_stream()
# Should try to disconnect tweepy stream
self.tweepy_stream_instance.disconnect.assert_called_once_with()
def test_update_stream_terms_unchanged(self):
self.checker.check.return_value = False
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
self.stream.update_stream()
# Should have checked if terms changed
self.checker.check.assert_called_once_with()
# Should NOT have stopped the old stream
self.assertEqual(self.stream.stop_stream.call_count, 0)
# Should NOT have started a new stream
self.assertEqual(self.stream.start_stream.call_count, 0)
def test_update_stream_terms_changed(self):
self.checker.check.return_value = True
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
self.stream.update_stream()
# Should have checked if terms changed
self.checker.check.assert_called_once_with()
# Should have stopped the old stream
self.stream.stop_stream.assert_called_once_with()
# Should have started a new stream
self.stream.start_stream.assert_called_once_with()
def test_update_stream_after_error(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
# Simulate a dead stream
self.tweepy_stream_instance.running = False
self.listener.error = 500
self.stream.update_stream()
# Should have stopped the old stream
self.stream.stop_stream.assert_called_once_with()
# Should have started a new stream
self.stream.start_stream.assert_called_once_with()
# Should have turned off the error
self.assertFalse(self.listener.error)
def test_update_stream_after_exception(self):
# Start the stream with a term
self.term_list.append("hello")
self.stream.start_stream()
self.stream.start_stream = mock.Mock()
self.stream.stop_stream = mock.Mock()
# Simulate an exception inside Tweepy
self.tweepy_stream_instance.running = False
self.listener.streaming_exception = Exception("testing")
self.stream.update_stream()
# Should have stopped the old stream
self.stream.stop_stream.assert_called_once_with()
# Should have started a new stream
self.stream.start_stream.assert_called_once_with()
# Should have turned off the exception
self.assertFalse(self.listener.streaming_exception)
def test_handle_exceptions(self):
self.listener.streaming_exception = None
try:
self.stream.handle_exceptions()
except Exception:
self.fail("Raised exception when no streaming exception set")
self.listener.streaming_exception = Exception("testing")
self.assertRaises(Exception, self.stream.handle_exceptions)
def test_clears_exceptions(self):
self.listener.streaming_exception = Exception("testing")
try:
self.stream.handle_exceptions()
except:
pass
self.assertEqual(self.listener.streaming_exception, None)
def test_stop_polling(self):
# Mock some supporting methods we test separately
self.stream.update_stream = mock.Mock()
self.stream.handle_exceptions = mock.Mock()
# This should poll every 1 second
thread = threading.Thread(target=self.stream.start_polling, args=[1])
thread.start()
waits = 0
# Wait for a maximum of 3 seconds (ish), or until the loop has run at least once
while self.stream.update_stream.call_count < 2 and waits < 12:
print("Waiting...")
time.sleep(0.25)
waits += 1
self.assertTrue(self.stream.polling)
# Try to stop the thread
self.stream.stop_polling()
self.assertFalse(self.stream.polling)
# Wait for a maximimum of 2 seconds
thread.join(timeout=2)
self.assertTrue(self.stream.update_stream.call_count >= 1,
"Checked for stream/term updates")
self.assertTrue(self.stream.handle_exceptions.call_count >= 1,
"Checked for stream exceptions")
def handle(self, credentials_name=None, *args, **options):
# The suggested time between hearbeats
poll_interval = options.get('poll_interval', 10)
# First expire any old stream process records that have failed
# to report in for a while
timeout_seconds = 3 * poll_interval
StreamProcess.expire_timed_out()
stream_process = StreamProcess.create(timeout_seconds=3 *
poll_interval)
def stop(signum, frame):
"""
Handle sudden stops
"""
logger.debug("Stopping because of signal")
if stream_process:
stream_process.status = StreamProcess.STREAM_STATUS_STOPPED
stream_process.heartbeat()
raise SystemExit()
def install_signal_handlers():
"""
Installs signal handlers for handling SIGINT and SIGTERM
gracefully.
"""
signal.signal(signal.SIGINT, stop)
signal.signal(signal.SIGTERM, stop)
install_signal_handlers()
try:
credentials = get_credentials(credentials_name)
logger.info("Using credentials for %s", credentials.name)
stream_process.credentials = credentials
stream_process.save()
auth = tweepy.OAuthHandler(credentials.api_key,
credentials.api_secret)
auth.set_access_token(credentials.access_token,
credentials.access_token_secret)
listener = QueueStreamListener()
checker = FeelsTermChecker(queue_listener=listener,
stream_process=stream_process)
# Start and maintain the streaming connection...
stream = DynamicTwitterStream(auth, listener, checker)
while checker.ok():
try:
stream.start(poll_interval)
except Exception as e:
checker.error(e)
time.sleep(3) # to avoid craziness
logger.error("Stopping because term checker not ok")
stream_process.status = StreamProcess.STREAM_STATUS_STOPPED
stream_process.heartbeat()
except Exception as e:
logger.error(e)
finally:
stop(None, None)
def handle(self, credentials_name=None, *args, **options):
# The suggested time between hearbeats
poll_interval = options.get('poll_interval', 10)
# First expire any old stream process records that have failed
# to report in for a while
timeout_seconds = 3 * poll_interval
StreamProcess.expire_timed_out()
stream_process = StreamProcess.create(
timeout_seconds=3 * poll_interval
)
def stop(signum, frame):
"""
Handle sudden stops
"""
logger.debug("Stopping because of signal")
if stream_process:
stream_process.status = StreamProcess.STREAM_STATUS_STOPPED
stream_process.heartbeat()
raise SystemExit()
def install_signal_handlers():
"""
Installs signal handlers for handling SIGINT and SIGTERM
gracefully.
"""
signal.signal(signal.SIGINT, stop)
signal.signal(signal.SIGTERM, stop)
install_signal_handlers()
try:
credentials = get_credentials(credentials_name)
logger.info("Using credentials for %s", credentials.name)
stream_process.credentials = credentials
stream_process.save()
auth = tweepy.OAuthHandler(credentials.api_key, credentials.api_secret)
auth.set_access_token(credentials.access_token, credentials.access_token_secret)
listener = QueueStreamListener()
checker = FeelsTermChecker(queue_listener=listener,
stream_process=stream_process)
# Start and maintain the streaming connection...
stream = DynamicTwitterStream(auth, listener, checker)
while checker.ok():
try:
stream.start(poll_interval)
except Exception as e:
checker.error(e)
time.sleep(3) # to avoid craziness
logger.error("Stopping because term checker not ok")
stream_process.status = StreamProcess.STREAM_STATUS_STOPPED
stream_process.heartbeat()
except Exception as e:
logger.error(e)
finally:
stop(None, None)
