def setUp(self): # Mock tweepy.Stream self.stream_patcher = mock.patch('tweepy.Stream') self.MockTweepyStream = self.stream_patcher.start() self.tweepy_stream_instance = mock.Mock() self.MockTweepyStream.return_value = self.tweepy_stream_instance # Mock the supporting objects self.auth = mock.Mock() self.listener = mock.Mock() self.checker = mock.Mock() # For controlling the tracking terms self.term_list = [] self.checker.tracking_terms.return_value = self.term_list self.stop_timeout = DynamicTwitterStream.STOP_TIMEOUT DynamicTwitterStream.STOP_TIMEOUT = 0 self.retry_count = 7 # Create a dynamic twitter stream self.stream = DynamicTwitterStream(auth=self.auth, listener=self.listener, term_checker=self.checker, retry_count=self.retry_count)
def test_start_unfiltered_no_terms(self): # Create an unfiltered stream instead self.stream = DynamicTwitterStream(auth=self.auth, listener=self.listener, term_checker=self.checker, retry_count=self.retry_count, unfiltered=True) # Start the stream without a term self.stream.start_stream() # Should check the list of terms self.checker.tracking_terms.assert_called_once_with() # But it should a stream even without any terms self.MockTweepyStream.assert_called_once_with( self.auth, self.listener, stall_warnings=True, timeout=90, retry_count=self.retry_count) # It should be using the sample endpoint self.tweepy_stream_instance.sample.assert_called_once_with( is_async=True, languages=None)
def test_start_with_languages(self): # Create a stream with languages languages = ['en', 'fr'] self.stream = DynamicTwitterStream(auth=self.auth, listener=self.listener, term_checker=self.checker, retry_count=self.retry_count, unfiltered=True, languages=languages) # Start the stream without a term self.stream.start_stream() # It should be using the sample endpoint with languages self.tweepy_stream_instance.sample.assert_called_once_with( is_async=True, languages=languages)
def handle(self, credentials_name=None, *args, **options): # The suggested time between hearbeats poll_interval = options.get('poll_interval', 10) # First expire any old stream process records that have failed # to report in for a while timeout_seconds = 3 * poll_interval StreamProcess.expire_timed_out() stream_process = StreamProcess.create(timeout_seconds=3 * poll_interval) def stop(signum, frame): """ Handle sudden stops """ logger.debug("Stopping because of signal") if stream_process: stream_process.status = StreamProcess.STREAM_STATUS_STOPPED stream_process.heartbeat() raise SystemExit() def install_signal_handlers(): """ Installs signal handlers for handling SIGINT and SIGTERM gracefully. """ signal.signal(signal.SIGINT, stop) signal.signal(signal.SIGTERM, stop) install_signal_handlers() try: credentials = get_credentials(credentials_name) logger.info("Using credentials for %s", credentials.name) stream_process.credentials = credentials stream_process.save() auth = tweepy.OAuthHandler(credentials.api_key, credentials.api_secret) auth.set_access_token(credentials.access_token, credentials.access_token_secret) listener = QueueStreamListener() checker = FeelsTermChecker(queue_listener=listener, stream_process=stream_process) # Start and maintain the streaming connection... stream = DynamicTwitterStream(auth, listener, checker) while checker.ok(): try: stream.start(poll_interval) except Exception as e: checker.error(e) time.sleep(3) # to avoid craziness logger.error("Stopping because term checker not ok") stream_process.status = StreamProcess.STREAM_STATUS_STOPPED stream_process.heartbeat() except Exception as e: logger.error(e) finally: stop(None, None)