def test_field_array(self):
# easier to mock the behavior here
r = Reader('')
expect( r.read_long ).returns( 42 )
expect( r._read_field ).returns( 3.14 ).side_effect( lambda: setattr(r, '_pos', 20) )
expect( r._read_field ).returns( 'pi' ).side_effect( lambda: setattr(r, '_pos', 42) )
assert_equals( [3.14,'pi'], r._field_array() )
def test_field_array(self):
# easier to mock the behavior here
r = Reader('')
expect(r.read_long).returns(42)
expect(r._read_field).returns(3.14).side_effect(
lambda: setattr(r, '_pos', 20))
expect(r._read_field).returns('pi').side_effect(
lambda: setattr(r, '_pos', 42))
assert_equals([3.14, 'pi'], r._field_array())
def test_read_table(self):
# mock everything to keep this simple
r = Reader('')
expect( r.read_long ).returns( 42 )
expect( r._check_underflow ).args( 42 )
expect( r._field_shortstr ).returns( 'a' )
expect( r._read_field ).returns( 3.14 ).side_effect( lambda: setattr(r, '_pos', 20) )
expect( r._field_shortstr ).returns( 'b' )
expect( r._read_field ).returns( 'pi' ).side_effect( lambda: setattr(r, '_pos', 42) )
assert_equals( {'a':3.14,'b':'pi'}, r.read_table() )
def read_frames(self):
'''
Read frames from the transport and process them. Some transports may
choose to do this in the background, in several threads, and so on.
'''
# It's possible in a concurrent environment that our transport handle
# has gone away, so handle that cleanly.
# TODO: Consider moving this block into Translator base class. In many
# ways it belongs there. One of the problems though is that this is
# essentially the read loop. Each Transport has different rules for
# how to kick this off, and in the case of gevent, this is how a
# blocking call to read from the socket is kicked off.
if self._transport is None:
return
# Send a heartbeat (if needed)
self._channels[0].send_heartbeat()
data = self._transport.read(self._heartbeat)
if data is None:
return
reader = Reader(data)
p_channels = set()
try:
for frame in Frame.read_frames(reader):
if self._debug > 1:
self.logger.debug("READ: %s", frame)
self._frames_read += 1
ch = self.channel(frame.channel_id)
ch.buffer_frame(frame)
p_channels.add(ch)
except Frame.FrameError as e:
# Frame error in the peer, disconnect
self.close(reply_code=501,
reply_text='frame error from %s : %s' %
(self._host, str(e)),
class_id=0,
method_id=0,
disconnect=True)
raise ConnectionClosed("connection is closed: %s : %s" %
(self._close_info['reply_code'],
self._close_info['reply_text']))
self._transport.process_channels(p_channels)
# HACK: read the buffer contents and re-buffer. Would prefer to pass
# buffer back, but there's no good way of asking the total size of the
# buffer, comparing to tell(), and then re-buffering. There's also no
# ability to clear the buffer up to the current position. It would be
# awesome if we could free that memory without a new allocation.
if reader.tell() < len(data):
self._transport.buffer(data[reader.tell():])
def read_frames(self):
'''
Read frames from the transport and process them. Some transports may
choose to do this in the background, in several threads, and so on.
'''
# It's possible in a concurrent environment that our transport handle
# has gone away, so handle that cleanly.
# TODO: Consider moving this block into Translator base class. In many
# ways it belongs there. One of the problems though is that this is
# essentially the read loop. Each Transport has different rules for
# how to kick this off, and in the case of gevent, this is how a
# blocking call to read from the socket is kicked off.
if self._transport is None:
return
# Send a heartbeat (if needed)
self._channels[0].send_heartbeat()
data = self._transport.read(self._heartbeat)
if data is None:
return
reader = Reader(data)
p_channels = set()
try:
for frame in Frame.read_frames(reader):
if self._debug > 1:
self.logger.debug("READ: %s", frame)
self._frames_read += 1
ch = self.channel(frame.channel_id)
ch.buffer_frame(frame)
p_channels.add(ch)
except Frame.FrameError as e:
# Frame error in the peer, disconnect
self.close(reply_code=501,
reply_text='frame error from %s : %s' % (
self._host, str(e)),
class_id=0, method_id=0, disconnect=True)
raise ConnectionClosed("connection is closed: %s : %s" %
(self._close_info['reply_code'],
self._close_info['reply_text']))
self._transport.process_channels(p_channels)
# HACK: read the buffer contents and re-buffer. Would prefer to pass
# buffer back, but there's no good way of asking the total size of the
# buffer, comparing to tell(), and then re-buffering. There's also no
# ability to clear the buffer up to the current position. It would be
# awesome if we could free that memory without a new allocation.
if reader.tell() < len(data):
self._transport.buffer(data[reader.tell():])
def test_read_table(self):
# mock everything to keep this simple
r = Reader('')
expect(r.read_long).returns(42)
expect(r._check_underflow).args(42)
expect(r._field_shortstr).returns('a')
expect(r._read_field).returns(3.14).side_effect(
lambda: setattr(r, '_pos', 20))
expect(r._field_shortstr).returns('b')
expect(r._read_field).returns('pi').side_effect(
lambda: setattr(r, '_pos', 42))
assert_equals({'a': 3.14, 'b': 'pi'}, r.read_table())
def _read_frame(cls, reader):
'''
Read a single frame from a Reader. Will return None if there is an
incomplete frame in the stream.
Raise MissingFooter if there's a problem reading the footer byte.
'''
frame_type = reader.read_octet()
channel_id = reader.read_short()
size = reader.read_long()
payload = Reader(reader, reader.tell(), size)
# Seek to end of payload
reader.seek(size, 1)
ch = reader.read_octet() # footer
if ch != 0xce:
raise Frame.FormatError(
'Framing error, unexpected byte: %x. frame type %x. channel %d, payload size %d',
ch, frame_type, channel_id, size)
frame_class = cls._frame_type_map.get(frame_type)
if not frame_class:
raise Frame.InvalidFrameType("Unknown frame type %x", frame_type)
return frame_class.parse(channel_id, payload)
def test_read(self):
b = Reader('foo')
assert_equals('foo', b.read(3))
b = Reader('foo')
assert_equals('fo', b.read(2))
b = Reader('foo')
assert_raises(Reader.BufferUnderflow, b.read, 4)
def test_str(self):
frame = MethodFrame(42, 5, 6, Reader(bytearray('hello')))
assert_equals(
'MethodFrame[channel: 42, class_id: 5, method_id: 6, args: \\x68\\x65\\x6c\\x6c\\x6f]', str(frame))
frame = MethodFrame(42, 5, 6)
assert_equals(
'MethodFrame[channel: 42, class_id: 5, method_id: 6, args: None]', str(frame))
def test_read_frames_handles_reader_errors(self):
reader = mock()
self.mock( Frame, '_read_frame' )
expect( reader.tell ).returns( 0 )
expect( Frame._read_frame ).args(reader).raises( Reader.ReaderError("bad!") )
assertRaises( Frame.FormatError, Frame.read_frames, reader )
def test_read_shortstr(self):
b = Reader('\x05hello')
assert_equals('hello', b.read_shortstr())
assert_raises(Reader.BufferUnderflow, b.read_shortstr)
b = Reader('\x0bD\xc3\xbcsseldorf')
assert_equals('D\xc3\xbcsseldorf', b.read_shortstr())
b = Reader('\x05hell')
assert_raises(Reader.BufferUnderflow, b.read_shortstr)
def test_write_frame(self):
args = mock()
expect(args.buffer).returns('hello')
frame = MethodFrame(42, 5, 6, args)
buf = bytearray()
frame.write_frame(buf)
reader = Reader(buf)
assert_equals(1, reader.read_octet())
assert_equals(42, reader.read_short())
size = reader.read_long()
start_pos = reader.tell()
assert_equals(5, reader.read_short())
assert_equals(6, reader.read_short())
args_pos = reader.tell()
assert_equals('hello', reader.read(size - (args_pos - start_pos)))
assert_equals(0xce, reader.read_octet())
def _read_frames(self):
'''
Read frames from the socket.
'''
# Because of the timer callback to dataRead when we re-buffered, there's a
# chance that in between we've lost the socket. If that's the case, just
# silently return as some code elsewhere would have already notified us.
# That bug could be fixed by improving the message reading so that we consume
# all possible messages and ensure that only a partial message was rebuffered,
# so that we can rely on the next read event to read the subsequent message.
if self._sock is None:
return
data = self._sock.read()
reader = Reader( data )
p_channels = set()
try:
frames = Frame.read_frames( reader )
except Frame.FrameError as e:
self.logger.exception( "Framing error", exc_info=True )
for frame in frames:
if self._debug > 1:
self.logger.debug( "READ: %s", frame )
self._frames_read += 1
ch = self.channel( frame.channel_id )
ch.buffer_frame( frame )
p_channels.add( ch )
# Still not clear on what's the best approach here. It seems there's a
# slight speedup by calling this directly rather than delaying, but the
# delay allows for pending IO with higher priority to execute.
self._process_channels( p_channels )
#event.timeout(0, self._process_channels, p_channels)
# HACK: read the buffer contents and re-buffer. Would prefer to pass
# buffer back, but there's no good way of asking the total size of the
# buffer, comparing to tell(), and then re-buffering. There's also no
# ability to clear the buffer up to the current position.
# NOTE: This will be cleared up once eventsocket supports the
# uber-awesome buffering scheme that will utilize mmap.
if reader.tell() < len(data):
self._sock.buffer( data[reader.tell():] )
def _read_frames(self):
'''
Read frames from the socket.
'''
# Because of the timer callback to dataRead when we re-buffered, there's a
# chance that in between we've lost the socket. If that's the case, just
# silently return as some code elsewhere would have already notified us.
# That bug could be fixed by improving the message reading so that we consume
# all possible messages and ensure that only a partial message was rebuffered,
# so that we can rely on the next read event to read the subsequent message.
if self._sock is None:
return
data = self._sock.read()
reader = Reader(data)
p_channels = set()
for frame in Frame.read_frames(reader):
if self._debug > 1:
self.logger.debug("READ: %s", frame)
self._frames_read += 1
ch = self.channel(frame.channel_id)
ch.buffer_frame(frame)
p_channels.add(ch)
# Still not clear on what's the best approach here. It seems there's a
# slight speedup by calling this directly rather than delaying, but the
# delay allows for pending IO with higher priority to execute.
self._process_channels(p_channels)
#event.timeout(0, self._process_channels, p_channels)
# HACK: read the buffer contents and re-buffer. Would prefer to pass
# buffer back, but there's no good way of asking the total size of the
# buffer, comparing to tell(), and then re-buffering. There's also no
# ability to clear the buffer up to the current position.
# NOTE: This will be cleared up once eventsocket supports the
# uber-awesome buffering scheme that will utilize mmap.
if reader.tell() < len(data):
self._sock.buffer(data[reader.tell():])
def test_parse_slow_for_standard_properties(self):
HeaderFrame.DEFAULT_PROPERTIES = False
bit_writer = Writer()
val_writer = Writer()
# strip ms because amqp doesn't include it
now = datetime.utcfromtimestamp(
long(time.mktime(datetime.now().timetuple())))
bit_field = 0
for pname, ptype, reader, writer, mask in HeaderFrame.PROPERTIES:
bit_field = (bit_field << 1) | 1
if ptype == 'shortstr':
val_writer.write_shortstr(pname)
elif ptype == 'octet':
val_writer.write_octet(42)
elif ptype == 'timestamp':
val_writer.write_timestamp(now)
elif ptype == 'table':
val_writer.write_table({'foo': 'bar'})
bit_field <<= (16 - len(HeaderFrame.PROPERTIES))
bit_writer.write_short(bit_field)
header_writer = Writer()
header_writer.write_short(5)
header_writer.write_short(6)
header_writer.write_longlong(7)
payload = header_writer.buffer()
payload += bit_writer.buffer()
payload += val_writer.buffer()
reader = Reader(payload)
frame = HeaderFrame.parse(4, reader)
HeaderFrame.DEFAULT_PROPERTIES = True
for pname, ptype, reader, writer, mask in HeaderFrame.PROPERTIES:
if ptype == 'shortstr':
self.assertEquals(pname, frame.properties[pname])
elif ptype == 'octet':
self.assertEquals(42, frame.properties[pname])
elif ptype == 'timestamp':
self.assertEquals(now, frame.properties[pname])
elif ptype == 'table':
self.assertEquals({'foo': 'bar'}, frame.properties[pname])
assert_equals(4, frame.channel_id)
assert_equals(5, frame._class_id)
assert_equals(6, frame._weight)
assert_equals(7, frame._size)
def test_field_decimal(self):
r = Reader(struct.pack('>Bi', 2, 5))
assert_equals(Decimal('0.05'), r._field_decimal())
assert_equals(5, r._pos)
r = Reader(struct.pack('>Bi', 2, -5))
assert_equals(Decimal('-0.05'), r._field_decimal())
assert_equals(5, r._pos)
def test_read_bit(self):
b = Reader('\x01')
assert_true(b.read_bit())
b = Reader('\x00')
assert_false(b.read_bit())
b = Reader('\x02')
assert_false(b.read_bit())
b = Reader('')
assert_raises(Reader.BufferUnderflow, b.read_bit)
def test_init(self):
ba = Reader(bytearray('foo'))
assert_true(isinstance(ba._input, buffer))
assert_equals('foo', str(ba._input))
assert_equals(0, ba._start_pos)
assert_equals(0, ba._pos)
assert_equals(3, ba._end_pos)
s = Reader('foo')
assert_true(isinstance(s._input, buffer))
assert_equals('foo', str(s._input))
u = Reader(u'D\xfcsseldorf')
assert_true(isinstance(u._input, buffer))
assert_equals('D\xc3\xbcsseldorf', str(u._input))
b = BytesIO('foo')
i = Reader(b)
assert_true(isinstance(i._input, buffer))
assert_equals('foo', str(i._input))
src = Reader('foo')
r = Reader(src)
assert_true(isinstance(r._input, buffer))
assert_equals(id(src._input), id(r._input))
assert_equals(0, r._start_pos)
assert_equals(3, r._end_pos)
src = Reader('hello world')
r = Reader(src, 3, 5)
assert_true(isinstance(r._input, buffer))
assert_equals(id(src._input), id(r._input))
assert_equals(3, r._start_pos)
assert_equals(8, r._end_pos)
assert_equals(3, r._pos)
assert_raises(ValueError, Reader, 1)
def test_field_longstr(self):
r = Reader('\x00\x00\x01\x00' + ('a' * 256))
assert_equals('a' * 256, r._field_longstr())
assert_equals(260, r._pos)
def test_field_shortstr(self):
r = Reader('\x05hello')
assert_equals('hello', r._field_shortstr())
assert_equals(6, r._pos)
def test_field_none(self):
b = Reader('')
assert_equals(None, b._field_none())
def test_seek_whence_one(self):
r = Reader('')
r._pos = 2
r.seek(5, 1)
assert_equals(7, r._pos)
def test_tell(self):
r = Reader('')
r._pos = 'foo'
assert_equals('foo', r.tell())
def test_field_long_int(self):
r = Reader(struct.pack('>ii', 2 ** 16, -2 ** 16))
assert_equals(2 ** 16, r._field_long_int())
assert_equals(-2 ** 16, r._field_long_int())
assert_equals(8, r._pos)
def test_field_short_uint(self):
r = Reader(struct.pack('>HH', 256, 512))
assert_equals(256, r._field_short_uint())
assert_equals(512, r._field_short_uint())
assert_equals(4, r._pos)
def test_field_short_int(self):
r = Reader(struct.pack('>hh', 256, -256))
assert_equals(256, r._field_short_int())
assert_equals(-256, r._field_short_int())
assert_equals(4, r._pos)
def test_field_short_short_uint(self):
r = Reader(struct.pack('BB', 5, 255))
assert_equals(5, r._field_short_short_uint())
assert_equals(255, r._field_short_short_uint())
assert_equals(2, r._pos)
def test_field_short_short_int(self):
r = Reader(struct.pack('bb', 5, -5))
assert_equals(5, r._field_short_short_int())
assert_equals(-5, r._field_short_short_int())
assert_equals(2, r._pos)
def test_field_timestamp(self):
b = Reader('\x00\x00\x00\x00\x4d\x34\xc4\x71')
d = datetime(2011, 1, 17, 22, 36, 33)
assert_equals(d, b._field_timestamp())
def test_field_bytearray(self):
b = Reader('\x00\x00\x00\x03\x04\x05\x06')
assert_equals(bytearray('\x04\x05\x06'), b._field_bytearray())
def test_check_underflow(self):
r = Reader('')
r._pos = 0
r._end_pos = 5
r._check_underflow(3)
assert_raises(Reader.BufferUnderflow, r._check_underflow, 8)
def test_str(self):
assert_equals('\\x66\\x6f\\x6f', str(Reader('foo')))
def test_buffer(self):
r = Reader('hello world', 3, 5)
self.assert_equals(buffer('lo wo'), r.buffer())
def test_seek_whence_zero(self):
r = Reader('', 3)
assert_equals(3, r._pos)
r.seek(5)
assert_equals(8, r._pos)
def test_field_long_long_int(self):
r = Reader(struct.pack('>qq', 2 ** 32, -2 ** 32))
assert_equals(2 ** 32, r._field_long_long_int())
assert_equals(-2 ** 32, r._field_long_long_int())
assert_equals(16, r._pos)
def test_seek_whence_two(self):
r = Reader('foo bar')
r.seek(-3, 2)
assert_equals(3, r._pos)
def test_field_long_long_uint(self):
r = Reader(struct.pack('>Q', 2 ** 64 - 1))
assert_equals(2 ** 64 - 1, r._field_long_long_uint())
assert_equals(8, r._pos)
def test_check_len(self):
r = Reader('foo bar')
self.assert_equals(7, len(r))
r = Reader('foo bar', 3)
self.assert_equals(4, len(r))
def test_field_float(self):
r = Reader(struct.pack('>f', 3.1421))
assert_almost_equals(3.1421, r._field_float(), 4)
assert_equals(4, r._pos)
def test_field_long_uint(self):
r = Reader(struct.pack('>I', 2 ** 32 - 1))
assert_equals(2 ** 32 - 1, r._field_long_uint())
assert_equals(4, r._pos)
def test_field_double(self):
r = Reader(struct.pack('>d', 8675309.1138))
assert_almost_equals(8675309.1138, r._field_double(), 4)
assert_equals(8, r._pos)