def test_field_decimal(self): r = Reader(struct.pack('>Bi', 2, 5)) assert_equals(Decimal('0.05'), r._field_decimal()) assert_equals(5, r._pos) r = Reader(struct.pack('>Bi', 2, -5)) assert_equals(Decimal('-0.05'), r._field_decimal()) assert_equals(5, r._pos)
def test_read(self): b = Reader('foo') assert_equals('foo', b.read(3)) b = Reader('foo') assert_equals('fo', b.read(2)) b = Reader('foo') assert_raises(Reader.BufferUnderflow, b.read, 4)
def test_read_shortstr(self): b = Reader('\x05hello') assert_equals('hello', b.read_shortstr()) assert_raises(Reader.BufferUnderflow, b.read_shortstr) b = Reader('\x0bD\xc3\xbcsseldorf') assert_equals('D\xc3\xbcsseldorf', b.read_shortstr()) b = Reader('\x05hell') assert_raises(Reader.BufferUnderflow, b.read_shortstr)
def test_read_bit(self): b = Reader('\x01') assert_true(b.read_bit()) b = Reader('\x00') assert_false(b.read_bit()) b = Reader('\x02') assert_false(b.read_bit()) b = Reader('') assert_raises(Reader.BufferUnderflow, b.read_bit)
def _read_frame(cls, reader): ''' Read a single frame from a Reader. Will return None if there is an incomplete frame in the stream. Raise MissingFooter if there's a problem reading the footer byte. ''' frame_type = reader.read_octet() channel_id = reader.read_short() size = reader.read_long() payload = Reader(reader, reader.tell(), size) # Seek to end of payload reader.seek(size, 1) ch = reader.read_octet() # footer if ch != 0xce: raise Frame.FormatError( 'Framing error, unexpected byte: %x. frame type %x. channel %d, payload size %d', ch, frame_type, channel_id, size) frame_class = cls._frame_type_map.get(frame_type) if not frame_class: raise Frame.InvalidFrameType("Unknown frame type %x", frame_type) return frame_class.parse(channel_id, payload)
def test_str(self): frame = MethodFrame(42, 5, 6, Reader(bytearray('hello'))) assert_equals( 'MethodFrame[channel: 42, class_id: 5, method_id: 6, args: \\x68\\x65\\x6c\\x6c\\x6f]', str(frame)) frame = MethodFrame(42, 5, 6) assert_equals( 'MethodFrame[channel: 42, class_id: 5, method_id: 6, args: None]', str(frame))
def test_field_array(self): # easier to mock the behavior here r = Reader('') expect(r.read_long).returns(42) expect(r._read_field).returns(3.14).side_effect( lambda: setattr(r, '_pos', 20)) expect(r._read_field).returns('pi').side_effect( lambda: setattr(r, '_pos', 42)) assert_equals([3.14, 'pi'], r._field_array())
def read_frames(self): ''' Read frames from the transport and process them. Some transports may choose to do this in the background, in several threads, and so on. ''' # It's possible in a concurrent environment that our transport handle # has gone away, so handle that cleanly. # TODO: Consider moving this block into Translator base class. In many # ways it belongs there. One of the problems though is that this is # essentially the read loop. Each Transport has different rules for # how to kick this off, and in the case of gevent, this is how a # blocking call to read from the socket is kicked off. if self._transport is None: return # Send a heartbeat (if needed) self._channels[0].send_heartbeat() data = self._transport.read(self._heartbeat) if data is None: return reader = Reader(data) p_channels = set() try: for frame in Frame.read_frames(reader): if self._debug > 1: self.logger.debug("READ: %s", frame) self._frames_read += 1 ch = self.channel(frame.channel_id) ch.buffer_frame(frame) p_channels.add(ch) except Frame.FrameError as e: # Frame error in the peer, disconnect self.close(reply_code=501, reply_text='frame error from %s : %s' % (self._host, str(e)), class_id=0, method_id=0, disconnect=True) raise ConnectionClosed("connection is closed: %s : %s" % (self._close_info['reply_code'], self._close_info['reply_text'])) self._transport.process_channels(p_channels) # HACK: read the buffer contents and re-buffer. Would prefer to pass # buffer back, but there's no good way of asking the total size of the # buffer, comparing to tell(), and then re-buffering. There's also no # ability to clear the buffer up to the current position. It would be # awesome if we could free that memory without a new allocation. if reader.tell() < len(data): self._transport.buffer(data[reader.tell():])
def test_read_table(self): # mock everything to keep this simple r = Reader('') expect(r.read_long).returns(42) expect(r._check_underflow).args(42) expect(r._field_shortstr).returns('a') expect(r._read_field).returns(3.14).side_effect( lambda: setattr(r, '_pos', 20)) expect(r._field_shortstr).returns('b') expect(r._read_field).returns('pi').side_effect( lambda: setattr(r, '_pos', 42)) assert_equals({'a': 3.14, 'b': 'pi'}, r.read_table())
def test_parse_slow_for_standard_properties(self): HeaderFrame.DEFAULT_PROPERTIES = False bit_writer = Writer() val_writer = Writer() # strip ms because amqp doesn't include it now = datetime.utcfromtimestamp( long(time.mktime(datetime.now().timetuple()))) bit_field = 0 for pname, ptype, reader, writer, mask in HeaderFrame.PROPERTIES: bit_field = (bit_field << 1) | 1 if ptype == 'shortstr': val_writer.write_shortstr(pname) elif ptype == 'octet': val_writer.write_octet(42) elif ptype == 'timestamp': val_writer.write_timestamp(now) elif ptype == 'table': val_writer.write_table({'foo': 'bar'}) bit_field <<= (16 - len(HeaderFrame.PROPERTIES)) bit_writer.write_short(bit_field) header_writer = Writer() header_writer.write_short(5) header_writer.write_short(6) header_writer.write_longlong(7) payload = header_writer.buffer() payload += bit_writer.buffer() payload += val_writer.buffer() reader = Reader(payload) frame = HeaderFrame.parse(4, reader) HeaderFrame.DEFAULT_PROPERTIES = True for pname, ptype, reader, writer, mask in HeaderFrame.PROPERTIES: if ptype == 'shortstr': self.assertEquals(pname, frame.properties[pname]) elif ptype == 'octet': self.assertEquals(42, frame.properties[pname]) elif ptype == 'timestamp': self.assertEquals(now, frame.properties[pname]) elif ptype == 'table': self.assertEquals({'foo': 'bar'}, frame.properties[pname]) assert_equals(4, frame.channel_id) assert_equals(5, frame._class_id) assert_equals(6, frame._weight) assert_equals(7, frame._size)
def test_write_frame_slow_for_standard_properties(self): HeaderFrame.DEFAULT_PROPERTIES = False bit_field = 0 properties = {} now = datetime.utcfromtimestamp( long(time.mktime(datetime.now().timetuple()))) for pname, ptype, reader, writer, mask in HeaderFrame.PROPERTIES: bit_field |= mask if ptype == 'shortstr': properties[pname] = pname elif ptype == 'octet': properties[pname] = 42 elif ptype == 'timestamp': properties[pname] = now elif ptype == 'table': properties[pname] = {'foo': 'bar'} frame = HeaderFrame(42, 5, 6, 7, properties) buf = bytearray() frame.write_frame(buf) HeaderFrame.DEFAULT_PROPERTIES = True reader = Reader(buf) assert_equals(2, reader.read_octet()) assert_equals(42, reader.read_short()) size = reader.read_long() start_pos = reader.tell() assert_equals(5, reader.read_short()) assert_equals(6, reader.read_short()) assert_equals(7, reader.read_longlong()) assert_equals(0b1111111111111100, reader.read_short()) for pname, ptype, rfunc, wfunc, mask in HeaderFrame.PROPERTIES: if ptype == 'shortstr': assertEquals(pname, reader.read_shortstr()) elif ptype == 'octet': assertEquals(42, reader.read_octet()) elif ptype == 'timestamp': assertEquals(now, reader.read_timestamp()) elif ptype == 'table': assertEquals({'foo': 'bar'}, reader.read_table()) end_pos = reader.tell() assert_equals(size, end_pos - start_pos) assert_equals(0xce, reader.read_octet())
def test_write_frame(self): args = mock() expect(args.buffer).returns('hello') frame = MethodFrame(42, 5, 6, args) buf = bytearray() frame.write_frame(buf) reader = Reader(buf) assert_equals(1, reader.read_octet()) assert_equals(42, reader.read_short()) size = reader.read_long() start_pos = reader.tell() assert_equals(5, reader.read_short()) assert_equals(6, reader.read_short()) args_pos = reader.tell() assert_equals('hello', reader.read(size - (args_pos - start_pos))) assert_equals(0xce, reader.read_octet())
def test_init(self): ba = Reader(bytearray('foo')) assert_true(isinstance(ba._input, buffer)) assert_equals('foo', str(ba._input)) assert_equals(0, ba._start_pos) assert_equals(0, ba._pos) assert_equals(3, ba._end_pos) s = Reader('foo') assert_true(isinstance(s._input, buffer)) assert_equals('foo', str(s._input)) u = Reader(u'D\xfcsseldorf') assert_true(isinstance(u._input, buffer)) assert_equals('D\xc3\xbcsseldorf', str(u._input)) b = BytesIO('foo') i = Reader(b) assert_true(isinstance(i._input, buffer)) assert_equals('foo', str(i._input)) src = Reader('foo') r = Reader(src) assert_true(isinstance(r._input, buffer)) assert_equals(id(src._input), id(r._input)) assert_equals(0, r._start_pos) assert_equals(3, r._end_pos) src = Reader('hello world') r = Reader(src, 3, 5) assert_true(isinstance(r._input, buffer)) assert_equals(id(src._input), id(r._input)) assert_equals(3, r._start_pos) assert_equals(8, r._end_pos) assert_equals(3, r._pos) assert_raises(ValueError, Reader, 1)
def _read_frames(self): ''' Read frames from the socket. ''' # Because of the timer callback to dataRead when we re-buffered, there's a # chance that in between we've lost the socket. If that's the case, just # silently return as some code elsewhere would have already notified us. # That bug could be fixed by improving the message reading so that we consume # all possible messages and ensure that only a partial message was rebuffered, # so that we can rely on the next read event to read the subsequent message. if self._sock is None: return data = self._sock.read() reader = Reader(data) p_channels = set() for frame in Frame.read_frames(reader): if self._debug > 1: self.logger.debug("READ: %s", frame) self._frames_read += 1 ch = self.channel(frame.channel_id) ch.buffer_frame(frame) p_channels.add(ch) # Still not clear on what's the best approach here. It seems there's a # slight speedup by calling this directly rather than delaying, but the # delay allows for pending IO with higher priority to execute. self._process_channels(p_channels) #event.timeout(0, self._process_channels, p_channels) # HACK: read the buffer contents and re-buffer. Would prefer to pass # buffer back, but there's no good way of asking the total size of the # buffer, comparing to tell(), and then re-buffering. There's also no # ability to clear the buffer up to the current position. # NOTE: This will be cleared up once eventsocket supports the # uber-awesome buffering scheme that will utilize mmap. if reader.tell() < len(data): self._sock.buffer(data[reader.tell():])
def test_read_bits(self): b = Reader('\x01') assert_equals([True], b.read_bits(1)) b = Reader('\x00') assert_equals([False], b.read_bits(1)) b = Reader('\x02') assert_equals([False, True], b.read_bits(2)) b = Reader('\x02') assert_equals( [False, True, False, False, False, False, False, False], b.read_bits(8)) b = Reader('\x00') assert_raises(ValueError, b.read_bits, 9) assert_raises(ValueError, b.read_bits, -1) assert_equals([], b.read_bits(0)) b = Reader('') assert_raises(Reader.BufferUnderflow, b.read_bits, 2)
def test_seek_whence_one(self): r = Reader('') r._pos = 2 r.seek(5, 1) assert_equals(7, r._pos)
def test_seek_whence_two(self): r = Reader('foo bar') r.seek(-3, 2) assert_equals(3, r._pos)
def test_tell(self): r = Reader('') r._pos = 'foo' assert_equals('foo', r.tell())
def test_seek_whence_zero(self): r = Reader('', 3) assert_equals(3, r._pos) r.seek(5) assert_equals(8, r._pos)
def test_field_none(self): b = Reader('') assert_equals(None, b._field_none())
def test_str(self): assert_equals('\\x66\\x6f\\x6f', str(Reader('foo')))
def test_field_float(self): r = Reader(struct.pack('>f', 3.1421)) assert_almost_equals(3.1421, r._field_float(), 4) assert_equals(4, r._pos)
def test_buffer(self): r = Reader('hello world', 3, 5) self.assert_equals(buffer('lo wo'), r.buffer())
def test_check_underflow(self): r = Reader('') r._pos = 0 r._end_pos = 5 r._check_underflow(3) assert_raises(Reader.BufferUnderflow, r._check_underflow, 8)
def test_field_longstr(self): r = Reader('\x00\x00\x01\x00' + ('a' * 256)) assert_equals('a' * 256, r._field_longstr()) assert_equals(260, r._pos)
def test_field_double(self): r = Reader(struct.pack('>d', 8675309.1138)) assert_almost_equals(8675309.1138, r._field_double(), 4) assert_equals(8, r._pos)
def test_check_len(self): r = Reader('foo bar') self.assert_equals(7, len(r)) r = Reader('foo bar', 3) self.assert_equals(4, len(r))
def test_field_timestamp(self): b = Reader('\x00\x00\x00\x00\x4d\x34\xc4\x71') d = datetime(2011, 1, 17, 22, 36, 33) assert_equals(d, b._field_timestamp())
def test_field_shortstr(self): r = Reader('\x05hello') assert_equals('hello', r._field_shortstr()) assert_equals(6, r._pos)
def test_field_bytearray(self): b = Reader('\x00\x00\x00\x03\x04\x05\x06') assert_equals(bytearray('\x04\x05\x06'), b._field_bytearray())