def _read_loop(self, resp): charset = resp.headers.get('content-type', default='') enc_search = re.search('charset=(?P<enc>\S*)', charset) if enc_search is not None: encoding = enc_search.group('enc') else: encoding = 'utf-8' buf = ReadBuffer(resp.raw, self.chunk_size, encoding=encoding) while self.running and not resp.raw.closed: length = 0 try: while not resp.raw.closed: line = buf.read_line() or '' stripped_line = line.strip() if not stripped_line: # keep-alive new lines are expected self.listener.keep_alive() elif stripped_line.isdigit(): length = int(stripped_line) break else: raise TweepError('Expecting length, unexpected value found') next_status_obj = buf.read_len(length) except Exception as error: TwiLogger.exception('Unable to process response: \n') continue if self.running and next_status_obj: self._data(next_status_obj) if resp.raw.closed: self.on_closed(resp)
def test_read_empty_buffer(self): """ Requests can be closed by twitter. The ReadBuffer should not loop infinitely when this happens. Instead it should return and let the outer _read_loop handle it. """ # If the test fails, we are in danger of an infinite loop # so we need to do some work to block that from happening class InfiniteLoopException(Exception): pass self.called_count = 0 call_limit = 5 def on_read(chunk_size): self.called_count += 1 if self.called_count > call_limit: # we have failed raise InfiniteLoopException( "Oops, read() was called a bunch of times") return "" # Create a fake stream stream = six.StringIO('') # Mock it's read function so it can't be called too many times mock_read = MagicMock(side_effect=on_read) try: with patch.multiple(stream, create=True, read=mock_read, closed=True): # Now the stream can't call 'read' more than call_limit times # and it looks like a requests stream that is closed buf = ReadBuffer(stream, 50) buf.read_line("\n") except InfiniteLoopException: self.fail("ReadBuffer.read_line tried to loop infinitely.") # The mocked function not have been called at all since the stream looks closed self.assertEqual(mock_read.call_count, 0)
def test_read_tweet(self): for length in [1, 2, 5, 10, 20, 50]: buf = ReadBuffer(six.BytesIO(self.stream), length) self.assertEqual('11\n', buf.read_line()) self.assertEqual('{id:12345}\n', buf.read_len(11)) self.assertEqual('\n', buf.read_line()) self.assertEqual('24\n', buf.read_line()) self.assertEqual('{id:23456, test:"blah"}\n', buf.read_len(24))
def test_read_unicode_tweet(self): stream = six.b('11\n{id:12345}\n\n23\n{id:23456, test:"\xe3\x81\x93"}\n\n') for length in [1, 2, 5, 10, 20, 50]: buf = ReadBuffer(six.BytesIO(stream), length) self.assertEqual('11\n', buf.read_line()) self.assertEqual('{id:12345}\n', buf.read_len(11)) self.assertEqual('\n', buf.read_line()) self.assertEqual('23\n', buf.read_line()) self.assertEqual(u'{id:23456, test:"\u3053"}\n', buf.read_len(23))
def test_read_empty_buffer(self): """ Requests can be closed by twitter. The ReadBuffer should not loop infinitely when this happens. Instead it should return and let the outer _read_loop handle it. """ # If the test fails, we are in danger of an infinite loop # so we need to do some work to block that from happening class InfiniteLoopException(Exception): pass self.called_count = 0 call_limit = 5 def on_read(chunk_size): self.called_count += 1 if self.called_count > call_limit: # we have failed raise InfiniteLoopException("Oops, read() was called a bunch of times") return "" # Create a fake stream stream = six.BytesIO(six.b('')) # Mock it's read function so it can't be called too many times mock_read = MagicMock(side_effect=on_read) try: stream.close() with patch.multiple(stream, create=True, read=mock_read): # Now the stream can't call 'read' more than call_limit times # and it looks like a requests stream that is closed buf = ReadBuffer(stream, 50) buf.read_line("\n") except InfiniteLoopException: self.fail("ReadBuffer.read_line tried to loop infinitely.") # The mocked function not have been called at all since the stream looks closed self.assertEqual(mock_read.call_count, 0)
def test_read_tweet(self): for length in [1, 2, 5, 10, 20, 50]: buf = ReadBuffer(six.StringIO(self.stream), length) self.assertEqual('11\n', buf.read_line()) self.assertEqual('{id:12345}\n', buf.read_len(11)) self.assertEqual('\n', buf.read_line()) self.assertEqual('24\n', buf.read_line()) self.assertEqual('{id:23456, test:"blah"}\n', buf.read_len(24))
def test_read_incomplete_buffer(self): from six.moves import http_client as httplib for length in [1, 2, 5, 10, 20, 50]: with self.assertRaises(httplib.IncompleteRead): msg = "11\n".replace('\n', '') buf = ReadBuffer(six.BytesIO(six.b(msg)), length) buf.read_line() with self.assertRaises(httplib.IncompleteRead): msg = '{id:12345}' buf = ReadBuffer(six.BytesIO(six.b(msg)), length) buf.read_len(len(msg) - 1) with self.assertRaises(httplib.IncompleteRead): msg = '{id:23456, test:"blah"}\n' buf = ReadBuffer(six.BytesIO(six.b(msg)), length) buf.read_len(len(msg) - 2)