def _read_loop(self, resp):
charset = resp.headers.get('content-type', default='')
enc_search = re.search('charset=(?P<enc>\S*)', charset)
if enc_search is not None:
encoding = enc_search.group('enc')
else:
encoding = 'utf-8'
buf = ReadBuffer(resp.raw, self.chunk_size, encoding=encoding)
while self.running and not resp.raw.closed:
length = 0
try:
while not resp.raw.closed:
line = buf.read_line() or ''
stripped_line = line.strip()
if not stripped_line:
# keep-alive new lines are expected
self.listener.keep_alive()
elif stripped_line.isdigit():
length = int(stripped_line)
break
else:
raise TweepError('Expecting length, unexpected value found')
next_status_obj = buf.read_len(length)
except Exception as error:
TwiLogger.exception('Unable to process response: \n')
continue
if self.running and next_status_obj:
self._data(next_status_obj)
if resp.raw.closed:
self.on_closed(resp)
def test_read_empty_buffer(self):
"""
Requests can be closed by twitter.
The ReadBuffer should not loop infinitely when this happens.
Instead it should return and let the outer _read_loop handle it.
"""
# If the test fails, we are in danger of an infinite loop
# so we need to do some work to block that from happening
class InfiniteLoopException(Exception):
pass
self.called_count = 0
call_limit = 5
def on_read(chunk_size):
self.called_count += 1
if self.called_count > call_limit:
# we have failed
raise InfiniteLoopException(
"Oops, read() was called a bunch of times")
return ""
# Create a fake stream
stream = six.StringIO('')
# Mock it's read function so it can't be called too many times
mock_read = MagicMock(side_effect=on_read)
try:
with patch.multiple(stream,
create=True,
read=mock_read,
closed=True):
# Now the stream can't call 'read' more than call_limit times
# and it looks like a requests stream that is closed
buf = ReadBuffer(stream, 50)
buf.read_line("\n")
except InfiniteLoopException:
self.fail("ReadBuffer.read_line tried to loop infinitely.")
# The mocked function not have been called at all since the stream looks closed
self.assertEqual(mock_read.call_count, 0)
def test_read_tweet(self):
for length in [1, 2, 5, 10, 20, 50]:
buf = ReadBuffer(six.BytesIO(self.stream), length)
self.assertEqual('11\n', buf.read_line())
self.assertEqual('{id:12345}\n', buf.read_len(11))
self.assertEqual('\n', buf.read_line())
self.assertEqual('24\n', buf.read_line())
self.assertEqual('{id:23456, test:"blah"}\n', buf.read_len(24))
def test_read_unicode_tweet(self):
stream = six.b('11\n{id:12345}\n\n23\n{id:23456, test:"\xe3\x81\x93"}\n\n')
for length in [1, 2, 5, 10, 20, 50]:
buf = ReadBuffer(six.BytesIO(stream), length)
self.assertEqual('11\n', buf.read_line())
self.assertEqual('{id:12345}\n', buf.read_len(11))
self.assertEqual('\n', buf.read_line())
self.assertEqual('23\n', buf.read_line())
self.assertEqual(u'{id:23456, test:"\u3053"}\n', buf.read_len(23))
def test_read_empty_buffer(self):
"""
Requests can be closed by twitter.
The ReadBuffer should not loop infinitely when this happens.
Instead it should return and let the outer _read_loop handle it.
"""
# If the test fails, we are in danger of an infinite loop
# so we need to do some work to block that from happening
class InfiniteLoopException(Exception):
pass
self.called_count = 0
call_limit = 5
def on_read(chunk_size):
self.called_count += 1
if self.called_count > call_limit:
# we have failed
raise InfiniteLoopException("Oops, read() was called a bunch of times")
return ""
# Create a fake stream
stream = six.BytesIO(six.b(''))
# Mock it's read function so it can't be called too many times
mock_read = MagicMock(side_effect=on_read)
try:
stream.close()
with patch.multiple(stream, create=True, read=mock_read):
# Now the stream can't call 'read' more than call_limit times
# and it looks like a requests stream that is closed
buf = ReadBuffer(stream, 50)
buf.read_line("\n")
except InfiniteLoopException:
self.fail("ReadBuffer.read_line tried to loop infinitely.")
# The mocked function not have been called at all since the stream looks closed
self.assertEqual(mock_read.call_count, 0)
def test_read_tweet(self):
for length in [1, 2, 5, 10, 20, 50]:
buf = ReadBuffer(six.StringIO(self.stream), length)
self.assertEqual('11\n', buf.read_line())
self.assertEqual('{id:12345}\n', buf.read_len(11))
self.assertEqual('\n', buf.read_line())
self.assertEqual('24\n', buf.read_line())
self.assertEqual('{id:23456, test:"blah"}\n', buf.read_len(24))
def test_read_unicode_tweet(self):
stream = six.b('11\n{id:12345}\n\n23\n{id:23456, test:"\xe3\x81\x93"}\n\n')
for length in [1, 2, 5, 10, 20, 50]:
buf = ReadBuffer(six.BytesIO(stream), length)
self.assertEqual('11\n', buf.read_line())
self.assertEqual('{id:12345}\n', buf.read_len(11))
self.assertEqual('\n', buf.read_line())
self.assertEqual('23\n', buf.read_line())
self.assertEqual(u'{id:23456, test:"\u3053"}\n', buf.read_len(23))
def test_read_incomplete_buffer(self):
from six.moves import http_client as httplib
for length in [1, 2, 5, 10, 20, 50]:
with self.assertRaises(httplib.IncompleteRead):
msg = "11\n".replace('\n', '')
buf = ReadBuffer(six.BytesIO(six.b(msg)), length)
buf.read_line()
with self.assertRaises(httplib.IncompleteRead):
msg = '{id:12345}'
buf = ReadBuffer(six.BytesIO(six.b(msg)), length)
buf.read_len(len(msg) - 1)
with self.assertRaises(httplib.IncompleteRead):
msg = '{id:23456, test:"blah"}\n'
buf = ReadBuffer(six.BytesIO(six.b(msg)), length)
buf.read_len(len(msg) - 2)
