Пример #1
0
    def read_frames(self):
        '''
        Read frames from the transport and process them. Some transports may
        choose to do this in the background, in several threads, and so on.
        '''
        # It's possible in a concurrent environment that our transport handle
        # has gone away, so handle that cleanly.
        # TODO: Consider moving this block into Translator base class. In many
        # ways it belongs there. One of the problems though is that this is
        # essentially the read loop. Each Transport has different rules for
        # how to kick this off, and in the case of gevent, this is how a
        # blocking call to read from the socket is kicked off.
        if self._transport is None:
            return

        # Send a heartbeat (if needed)
        self._channels[0].send_heartbeat()

        data = self._transport.read(self._heartbeat)
        if data is None:
            return

        reader = Reader(data)
        p_channels = set()

        try:
            for frame in Frame.read_frames(reader):
                if self._debug > 1:
                    self.logger.debug("READ: %s", frame)
                self._frames_read += 1
                ch = self.channel(frame.channel_id)
                ch.buffer_frame(frame)
                p_channels.add(ch)
        except Frame.FrameError as e:
            # Frame error in the peer, disconnect
            self.close(reply_code=501,
                       reply_text='frame error from %s : %s' %
                       (self._host, str(e)),
                       class_id=0,
                       method_id=0,
                       disconnect=True)
            raise ConnectionClosed("connection is closed: %s : %s" %
                                   (self._close_info['reply_code'],
                                    self._close_info['reply_text']))

        self._transport.process_channels(p_channels)

        # HACK: read the buffer contents and re-buffer.  Would prefer to pass
        # buffer back, but there's no good way of asking the total size of the
        # buffer, comparing to tell(), and then re-buffering.  There's also no
        # ability to clear the buffer up to the current position. It would be
        # awesome if we could free that memory without a new allocation.
        if reader.tell() < len(data):
            self._transport.buffer(data[reader.tell():])
Пример #2
0
    def read_frames(self):
        '''
        Read frames from the transport and process them. Some transports may
        choose to do this in the background, in several threads, and so on.
        '''
        # It's possible in a concurrent environment that our transport handle
        # has gone away, so handle that cleanly.
        # TODO: Consider moving this block into Translator base class. In many
        # ways it belongs there. One of the problems though is that this is
        # essentially the read loop. Each Transport has different rules for
        # how to kick this off, and in the case of gevent, this is how a
        # blocking call to read from the socket is kicked off.
        if self._transport is None:
            return

        # Send a heartbeat (if needed)
        self._channels[0].send_heartbeat()

        data = self._transport.read(self._heartbeat)
        if data is None:
            return

        reader = Reader(data)
        p_channels = set()

        try:
            for frame in Frame.read_frames(reader):
                if self._debug > 1:
                    self.logger.debug("READ: %s", frame)
                self._frames_read += 1
                ch = self.channel(frame.channel_id)
                ch.buffer_frame(frame)
                p_channels.add(ch)
        except Frame.FrameError as e:
            # Frame error in the peer, disconnect
            self.close(reply_code=501,
                       reply_text='frame error from %s : %s' % (
                           self._host, str(e)),
                       class_id=0, method_id=0, disconnect=True)
            raise ConnectionClosed("connection is closed: %s : %s" %
                                   (self._close_info['reply_code'],
                                    self._close_info['reply_text']))

        self._transport.process_channels(p_channels)

        # HACK: read the buffer contents and re-buffer.  Would prefer to pass
        # buffer back, but there's no good way of asking the total size of the
        # buffer, comparing to tell(), and then re-buffering.  There's also no
        # ability to clear the buffer up to the current position. It would be
        # awesome if we could free that memory without a new allocation.
        if reader.tell() < len(data):
            self._transport.buffer(data[reader.tell():])
Пример #3
0
    def test_write_frame_slow_for_standard_properties(self):
        HeaderFrame.DEFAULT_PROPERTIES = False
        bit_field = 0
        properties = {}
        now = datetime.utcfromtimestamp(
            long(time.mktime(datetime.now().timetuple())))
        for pname, ptype, reader, writer, mask in HeaderFrame.PROPERTIES:
            bit_field |= mask

            if ptype == 'shortstr':
                properties[pname] = pname
            elif ptype == 'octet':
                properties[pname] = 42
            elif ptype == 'timestamp':
                properties[pname] = now
            elif ptype == 'table':
                properties[pname] = {'foo': 'bar'}

        frame = HeaderFrame(42, 5, 6, 7, properties)
        buf = bytearray()
        frame.write_frame(buf)
        HeaderFrame.DEFAULT_PROPERTIES = True

        reader = Reader(buf)
        assert_equals(2, reader.read_octet())
        assert_equals(42, reader.read_short())
        size = reader.read_long()
        start_pos = reader.tell()
        assert_equals(5, reader.read_short())
        assert_equals(6, reader.read_short())
        assert_equals(7, reader.read_longlong())
        assert_equals(0b1111111111111100, reader.read_short())

        for pname, ptype, rfunc, wfunc, mask in HeaderFrame.PROPERTIES:
            if ptype == 'shortstr':
                assertEquals(pname, reader.read_shortstr())
            elif ptype == 'octet':
                assertEquals(42, reader.read_octet())
            elif ptype == 'timestamp':
                assertEquals(now, reader.read_timestamp())
            elif ptype == 'table':
                assertEquals({'foo': 'bar'}, reader.read_table())

        end_pos = reader.tell()
        assert_equals(size, end_pos - start_pos)
        assert_equals(0xce, reader.read_octet())
Пример #4
0
    def test_write_frame_slow_for_standard_properties(self):
        HeaderFrame.DEFAULT_PROPERTIES = False
        bit_field = 0
        properties = {}
        now = datetime.utcfromtimestamp(
            long(time.mktime(datetime.now().timetuple())))
        for pname, ptype, reader, writer, mask in HeaderFrame.PROPERTIES:
            bit_field |= mask

            if ptype == 'shortstr':
                properties[pname] = pname
            elif ptype == 'octet':
                properties[pname] = 42
            elif ptype == 'timestamp':
                properties[pname] = now
            elif ptype == 'table':
                properties[pname] = {'foo': 'bar'}

        frame = HeaderFrame(42, 5, 6, 7, properties)
        buf = bytearray()
        frame.write_frame(buf)
        HeaderFrame.DEFAULT_PROPERTIES = True

        reader = Reader(buf)
        assert_equals(2, reader.read_octet())
        assert_equals(42, reader.read_short())
        size = reader.read_long()
        start_pos = reader.tell()
        assert_equals(5, reader.read_short())
        assert_equals(6, reader.read_short())
        assert_equals(7, reader.read_longlong())
        assert_equals(0b1111111111111100, reader.read_short())

        for pname, ptype, rfunc, wfunc, mask in HeaderFrame.PROPERTIES:
            if ptype == 'shortstr':
                assertEquals(pname, reader.read_shortstr())
            elif ptype == 'octet':
                assertEquals(42, reader.read_octet())
            elif ptype == 'timestamp':
                assertEquals(now, reader.read_timestamp())
            elif ptype == 'table':
                assertEquals({'foo': 'bar'}, reader.read_table())

        end_pos = reader.tell()
        assert_equals(size, end_pos - start_pos)
        assert_equals(0xce, reader.read_octet())
Пример #5
0
    def test_write_frame(self):
        args = mock()
        expect(args.buffer).returns('hello')

        frame = MethodFrame(42, 5, 6, args)
        buf = bytearray()
        frame.write_frame(buf)

        reader = Reader(buf)
        assert_equals(1, reader.read_octet())
        assert_equals(42, reader.read_short())
        size = reader.read_long()
        start_pos = reader.tell()
        assert_equals(5, reader.read_short())
        assert_equals(6, reader.read_short())
        args_pos = reader.tell()
        assert_equals('hello', reader.read(size - (args_pos - start_pos)))
        assert_equals(0xce, reader.read_octet())
Пример #6
0
  def _read_frames(self):
    '''
    Read frames from the socket.
    '''
    # Because of the timer callback to dataRead when we re-buffered, there's a
    # chance that in between we've lost the socket.  If that's the case, just
    # silently return as some code elsewhere would have already notified us.
    # That bug could be fixed by improving the message reading so that we consume
    # all possible messages and ensure that only a partial message was rebuffered,
    # so that we can rely on the next read event to read the subsequent message.
    if self._sock is None:
      return
    
    data = self._sock.read()
    reader = Reader( data )
    p_channels = set()
    
    try:
      frames = Frame.read_frames( reader )
    except Frame.FrameError as e:
      self.logger.exception( "Framing error", exc_info=True )
      
    for frame in frames:
      if self._debug > 1:
        self.logger.debug( "READ: %s", frame )
      self._frames_read += 1
      ch = self.channel( frame.channel_id )
      ch.buffer_frame( frame )
      p_channels.add( ch )

    # Still not clear on what's the best approach here. It seems there's a
    # slight speedup by calling this directly rather than delaying, but the
    # delay allows for pending IO with higher priority to execute.
    self._process_channels( p_channels )
    #event.timeout(0, self._process_channels, p_channels)

    # HACK: read the buffer contents and re-buffer.  Would prefer to pass
    # buffer back, but there's no good way of asking the total size of the
    # buffer, comparing to tell(), and then re-buffering.  There's also no
    # ability to clear the buffer up to the current position.
    # NOTE: This will be cleared up once eventsocket supports the 
    # uber-awesome buffering scheme that will utilize mmap.
    if reader.tell() < len(data):
      self._sock.buffer( data[reader.tell():] )
Пример #7
0
    def _read_frames(self):
        '''
    Read frames from the socket.
    '''
        # Because of the timer callback to dataRead when we re-buffered, there's a
        # chance that in between we've lost the socket.  If that's the case, just
        # silently return as some code elsewhere would have already notified us.
        # That bug could be fixed by improving the message reading so that we consume
        # all possible messages and ensure that only a partial message was rebuffered,
        # so that we can rely on the next read event to read the subsequent message.
        if self._sock is None:
            return

        data = self._sock.read()
        reader = Reader(data)
        p_channels = set()

        for frame in Frame.read_frames(reader):
            if self._debug > 1:
                self.logger.debug("READ: %s", frame)
            self._frames_read += 1
            ch = self.channel(frame.channel_id)
            ch.buffer_frame(frame)
            p_channels.add(ch)

        # Still not clear on what's the best approach here. It seems there's a
        # slight speedup by calling this directly rather than delaying, but the
        # delay allows for pending IO with higher priority to execute.
        self._process_channels(p_channels)
        #event.timeout(0, self._process_channels, p_channels)

        # HACK: read the buffer contents and re-buffer.  Would prefer to pass
        # buffer back, but there's no good way of asking the total size of the
        # buffer, comparing to tell(), and then re-buffering.  There's also no
        # ability to clear the buffer up to the current position.
        # NOTE: This will be cleared up once eventsocket supports the
        # uber-awesome buffering scheme that will utilize mmap.
        if reader.tell() < len(data):
            self._sock.buffer(data[reader.tell():])
Пример #8
0
 def test_tell(self):
     r = Reader('')
     r._pos = 'foo'
     assert_equals('foo', r.tell())
Пример #9
0
    def read_frames(self):
        '''
        Read frames from the transport and process them. Some transports may
        choose to do this in the background, in several threads, and so on.
        '''
        # It's possible in a concurrent environment that our transport handle
        # has gone away, so handle that cleanly.
        # TODO: Consider moving this block into Translator base class. In many
        # ways it belongs there. One of the problems though is that this is
        # essentially the read loop. Each Transport has different rules for
        # how to kick this off, and in the case of gevent, this is how a
        # blocking call to read from the socket is kicked off.
        if self._transport is None:
            return

        # Send a heartbeat (if needed)
        self._channels[0].send_heartbeat()

        data = self._transport.read(self._heartbeat)
        current_time = time.time()

        if data is None:
            # Wait for 2 heartbeat intervals before giving up. See AMQP 4.2.7:
            # "If a peer detects no incoming traffic (i.e. received octets) for two heartbeat intervals or longer,
            # it should close the connection"
            if self._heartbeat and (current_time-self._last_octet_time > 2*self._heartbeat):
                msg = 'Heartbeats not received from %s for %d seconds' % (self._host, 2*self._heartbeat)
                self.transport_closed(msg=msg)
                raise ConnectionClosed('Connection is closed: ' + msg)
            return
        self._last_octet_time = current_time
        reader = Reader(data)
        p_channels = set()

        try:
            for frame in Frame.read_frames(reader):
                if self._debug > 1:
                    self.logger.debug("READ: %s", frame)
                self._frames_read += 1
                ch = self.channel(frame.channel_id)
                ch.buffer_frame(frame)
                p_channels.add(ch)
        except Frame.FrameError as e:
            # Frame error in the peer, disconnect
            self.close(reply_code=501,
                       reply_text='frame error from %s : %s' % (
                           self._host, str(e)),
                       class_id=0, method_id=0, disconnect=True)
            raise ConnectionClosed("connection is closed: %s : %s" %
                                   (self._close_info['reply_code'],
                                    self._close_info['reply_text']))

        # NOTE: we process channels after buffering unused data in order to
        # preserve the integrity of the input stream in case a channel needs to
        # read input, such as when a channel framing error necessitates the use
        # of the synchronous channel.close method. See `Channel.process_frames`.
        #
        # HACK: read the buffer contents and re-buffer.  Would prefer to pass
        # buffer back, but there's no good way of asking the total size of the
        # buffer, comparing to tell(), and then re-buffering.  There's also no
        # ability to clear the buffer up to the current position. It would be
        # awesome if we could free that memory without a new allocation.
        if reader.tell() < len(data):
            self._transport.buffer(data[reader.tell():])

        self._transport.process_channels(p_channels)
Пример #10
0
    def read_frames(self):
        '''
        Read frames from the transport and process them. Some transports may
        choose to do this in the background, in several threads, and so on.
        '''
        # It's possible in a concurrent environment that our transport handle
        # has gone away, so handle that cleanly.
        # TODO: Consider moving this block into Translator base class. In many
        # ways it belongs there. One of the problems though is that this is
        # essentially the read loop. Each Transport has different rules for
        # how to kick this off, and in the case of gevent, this is how a
        # blocking call to read from the socket is kicked off.
        if self._transport is None:
            return

        # Send a heartbeat (if needed)
        self._channels[0].send_heartbeat()

        data = self._transport.read(self._heartbeat)
        current_time = time.time()

        if data is None:
            # Wait for 2 heartbeat intervals before giving up. See AMQP 4.2.7:
            # "If a peer detects no incoming traffic (i.e. received octets) for two heartbeat intervals or longer,
            # it should close the connection"
            if self._heartbeat and (current_time-self._last_octet_time > 2*self._heartbeat):
                msg = 'Heartbeats not received from %s for %d seconds' % (self._host, 2*self._heartbeat)
                self.transport_closed(msg=msg)
                raise ConnectionClosed('Connection is closed: ' + msg)
            return
        self._last_octet_time = current_time
        reader = Reader(data)
        p_channels = set()

        try:
            for frame in Frame.read_frames(reader):
                if self._debug > 1:
                    self.logger.debug("READ: %s", frame)
                self._frames_read += 1
                ch = self.channel(frame.channel_id)
                ch.buffer_frame(frame)
                p_channels.add(ch)
        except Frame.FrameError as e:
            # Frame error in the peer, disconnect
            self.close(reply_code=501,
                       reply_text='frame error from %s : %s' % (
                           self._host, str(e)),
                       class_id=0, method_id=0, disconnect=True)
            raise ConnectionClosed("connection is closed: %s : %s" %
                                   (self._close_info['reply_code'],
                                    self._close_info['reply_text']))

        # NOTE: we process channels after buffering unused data in order to
        # preserve the integrity of the input stream in case a channel needs to
        # read input, such as when a channel framing error necessitates the use
        # of the synchronous channel.close method. See `Channel.process_frames`.
        #
        # HACK: read the buffer contents and re-buffer.  Would prefer to pass
        # buffer back, but there's no good way of asking the total size of the
        # buffer, comparing to tell(), and then re-buffering.  There's also no
        # ability to clear the buffer up to the current position. It would be
        # awesome if we could free that memory without a new allocation.
        if reader.tell() < len(data):
            self._transport.buffer(data[reader.tell():])

        self._transport.process_channels(p_channels)