示例#1
0
    def __init__(self, connection, channel_number, transport=None):

        # We need to do this before the channel is invoked and send_method is
        # called
        CallbackManager.instance().add(channel_number,
                                       spec.Channel.OpenOk,
                                       transport._on_rpc_complete)
        Channel.__init__(self, connection, channel_number, None, transport)
        self.basic_get_ = Channel.basic_get
示例#2
0
    def __init__(self, parameters=None, on_open_callback=None, reconnection_strategy=None):
        """
        Connection initialization expects a ConnectionParameters object and
        a callback function to notify when we have successfully connected
        to the AMQP Broker.

        A reconnection_strategy of None will use the NullReconnectionStrategy
        """
        self._buffer = ""

        # Define our callback dictionary
        self.callbacks = CallbackManager.instance()

        # On connection callback
        if on_open_callback:
            self.add_on_open_callback(on_open_callback)

        # Set our configuration options
        self.parameters = parameters or ConnectionParameters()

        # If we did not pass in a reconnection_strategy, setup the default
        self.reconnection = reconnection_strategy or NullReconnectionStrategy()

        # Add our callback for if we close by being disconnected
        self.add_on_close_callback(self.reconnection.on_connection_closed)

        # Set all of our default connection state values
        self._init_connection_state()

        # Connect to the AMQP Broker
        self._connect()
示例#3
0
    def __init__(self,
                 parameters=None,
                 on_open_callback=None,
                 reconnection_strategy=None):
        """
        Connection initialization expects a ConnectionParameters object and
        a callback function to notify when we have successfully connected
        to the AMQP Broker.

        A reconnection_strategy of None will use the NullReconnectionStrategy
        """
        self._buffer = ''

        # Define our callback dictionary
        self.callbacks = CallbackManager()

        # On connection callback
        if on_open_callback:
            self.add_on_open_callback(on_open_callback)

        # Set our configuration options
        self.parameters = parameters or ConnectionParameters()

        # If we did not pass in a reconnection_strategy, setup the default
        self.reconnection = reconnection_strategy or NullReconnectionStrategy()

        # Validate that we don't have erase_credentials enabled with a non
        # Null reconnection strategy
        if self.parameters.credentials.erase_on_connect and \
            not isinstance(self.reconnection, NullReconnectionStrategy):
            # Warn the developer
            warn(
                "%s was initialized to erase credentials but you have \
specified a %s. Reconnections will fail.",
                self.parameters.credentials.__class__.__name__,
                self.reconnection.__class__.__name__)

        # Add our callback for if we close by being disconnected
        if not isinstance(self.reconnection, NullReconnectionStrategy):
            self.add_on_close_callback(self.reconnection.on_connection_closed)

        # Set all of our default connection state values
        self._init_connection_state()

        # Connect to the AMQP Broker
        self._connect()
示例#4
0
文件: connection.py 项目: majnet/pika
    def __init__(self, parameters=None,
                 on_open_callback=None,
                 reconnection_strategy=None):
        """
        Connection initialization expects a ConnectionParameters object and
        a callback function to notify when we have successfully connected
        to the AMQP Broker.

        A reconnection_strategy of None will use the NullReconnectionStrategy
        """
        self._buffer = ''

        # Define our callback dictionary
        self.callbacks = CallbackManager()

        # On connection callback
        if on_open_callback:
            self.add_on_open_callback(on_open_callback)

        # Set our configuration options
        self.parameters = parameters or ConnectionParameters()

        # If we did not pass in a reconnection_strategy, setup the default
        self.reconnection = reconnection_strategy or NullReconnectionStrategy()

        # Validate that we don't have erase_credentials enabled with a non
        # Null reconnection strategy
        if self.parameters.credentials.erase_on_connect and \
            not isinstance(self.reconnection, NullReconnectionStrategy):
            # Warn the developer
            warn("%s was initialized to erase credentials but you have \
specified a %s. Reconnections will fail.",
                 self.parameters.credentials.__class__.__name__,
                 self.reconnection.__class__.__name__)

        # Add our callback for if we close by being disconnected
        if not isinstance(self.reconnection, NullReconnectionStrategy) and \
            hasattr(self.reconnection, 'on_connection_closed'):
            self.add_on_close_callback(self.reconnection.on_connection_closed)

        # Set all of our default connection state values
        self._init_connection_state()

        # Connect to the AMQP Broker
        self._connect()
示例#5
0
    def __init__(self,
                 connection_id=None,
                 hostname=None,
                 port=None,
                 virtual_host=None,
                 username=None,
                 password=None,
                 heartbeat=None,
                 prefetch_count=None,
                 tx_select=None):

        # Allow class variables to provide defaults for:

        # connection_id
        if self.connection_id is None and connection_id is None:
            connection_id =\
                getattr(self, 'grokcore.component.directive.name', 'default')
        if connection_id is not None:
            self.connection_id = connection_id

        # hostname
        if hostname is not None:
            self.hostname = hostname
        assert self.hostname is not None,\
            u"Connection configuration is missing hostname."

        # port
        if port is not None:
            self.port = port
        assert self.port is not None,\
            u"Connection configuration is missing port."

        # virtual_host
        if virtual_host is not None:
            self.virtual_host = virtual_host
        assert self.virtual_host is not None,\
            u"Connection configuration is missing virtual_host."

        # username
        if username is not None:
            self.username = username
        assert self.username is not None,\
            u"Connection configuration is missing username."

        # password
        if password is not None:
            self.password = password
        assert self.password is not None,\
            u"Connection configuration is missing password."

        # heartbeat
        if heartbeat is not None:
            self.heartbeat = heartbeat

        # prefetch_count
        if prefetch_count is not None:
            self.prefetch_count = prefetch_count

        # tx_select
        if tx_select is not None:
            self.tx_select = tx_select

        self._callbacks = CallbackManager()  # callbacks are NOT thread-safe
        self._reconnection_time = time.time()
        self._reconnection_delay = 1.0

        # BBB for affinitic.zamqp
        if getattr(self, 'userid', None):
            from zope.deprecation import deprecated
            self.username = self.userid
            self.userid =\
                deprecated(self.userid,
                           ('Connection.userid is no more. '
                            'Please, use Connection.username instead.'))

        logger.default(
            u"AMQP Broker connection '%s' created. "
            u"hostname: '%s', "
            u"port: '%s', "
            u"virtual_host: '%s', "
            u"username: '******', "
            u"heartbeat: '%s', "
            u"prefetch_count: '%s', "
            u"tx_select: '%s'", self.connection_id, self.hostname, self.port,
            self.virtual_host, self.username, self.heartbeat,
            self.prefetch_count, self.tx_select)
示例#6
0
class BrokerConnection(grok.GlobalUtility):
    """Connection utility base class"""

    grok.implements(IBrokerConnection)
    grok.baseclass()

    connection_id = None

    hostname = 'localhost'
    port = 5672
    virtual_host = '/'

    username = '******'
    password = '******'

    heartbeat = 0

    prefetch_count = 0
    tx_select = False  # Be aware, that rollback for transactional channel

    # is safe to use (and tx_select useful) only on
    # dedicated single-threaded AMQP-consuming ZEO-clients.

    def __init__(self,
                 connection_id=None,
                 hostname=None,
                 port=None,
                 virtual_host=None,
                 username=None,
                 password=None,
                 heartbeat=None,
                 prefetch_count=None,
                 tx_select=None):

        # Allow class variables to provide defaults for:

        # connection_id
        if self.connection_id is None and connection_id is None:
            connection_id =\
                getattr(self, 'grokcore.component.directive.name', 'default')
        if connection_id is not None:
            self.connection_id = connection_id

        # hostname
        if hostname is not None:
            self.hostname = hostname
        assert self.hostname is not None,\
            u"Connection configuration is missing hostname."

        # port
        if port is not None:
            self.port = port
        assert self.port is not None,\
            u"Connection configuration is missing port."

        # virtual_host
        if virtual_host is not None:
            self.virtual_host = virtual_host
        assert self.virtual_host is not None,\
            u"Connection configuration is missing virtual_host."

        # username
        if username is not None:
            self.username = username
        assert self.username is not None,\
            u"Connection configuration is missing username."

        # password
        if password is not None:
            self.password = password
        assert self.password is not None,\
            u"Connection configuration is missing password."

        # heartbeat
        if heartbeat is not None:
            self.heartbeat = heartbeat

        # prefetch_count
        if prefetch_count is not None:
            self.prefetch_count = prefetch_count

        # tx_select
        if tx_select is not None:
            self.tx_select = tx_select

        self._callbacks = CallbackManager()  # callbacks are NOT thread-safe
        self._reconnection_time = time.time()
        self._reconnection_delay = 1.0

        # BBB for affinitic.zamqp
        if getattr(self, 'userid', None):
            from zope.deprecation import deprecated
            self.username = self.userid
            self.userid =\
                deprecated(self.userid,
                           ('Connection.userid is no more. '
                            'Please, use Connection.username instead.'))

        logger.default(
            u"AMQP Broker connection '%s' created. "
            u"hostname: '%s', "
            u"port: '%s', "
            u"virtual_host: '%s', "
            u"username: '******', "
            u"heartbeat: '%s', "
            u"prefetch_count: '%s', "
            u"tx_select: '%s'", self.connection_id, self.hostname, self.port,
            self.virtual_host, self.username, self.heartbeat,
            self.prefetch_count, self.tx_select)

    def connect(self):
        logger.default(u"Connection '%s' connecting", self.connection_id)
        credentials = PlainCredentials(self.username,
                                       self.password,
                                       erase_on_connect=False)
        parameters = ConnectionParameters(self.hostname,
                                          self.port,
                                          self.virtual_host,
                                          credentials=credentials,
                                          heartbeat=self.heartbeat and True
                                          or False)
        # AMQP-heartbeat timeout must be set manually due to bug in pika 0.9.5:
        if parameters.heartbeat:
            parameters.heartbeat = int(self.heartbeat)
        self._connection = AsyncoreConnection(parameters=parameters,
                                              on_open_callback=self.on_connect)
        self._reconnection_timeout = None
        self._connection.add_on_close_callback(self.reconnect)

    def reconnect(self, conn=None):
        if time.time() > self._reconnection_time + 60:
            self._reconnection_delay = 1.0
        if not getattr(self, '_reconnection_timeout', None):
            logger.info(u"Trying to reconnect connection '%s' in %s seconds",
                        self.connection_id, self._reconnection_delay)
            self._reconnection_timeout =\
                AsyncoreScheduling(self.connect, self._reconnection_delay)
            self._reconnection_delay *= (random.random() * 0.5) + 1.0
            self._reconnection_delay = min(self._reconnection_delay, 60.0)

    @property
    def is_open(self):
        return getattr(self._connection, 'is_open', False)

    def add_on_channel_open_callback(self, callback):
        self._callbacks.add(0, '_on_channel_open', callback, False)

    def on_connect(self, connection):
        logger.default(u"Connection '%s' connected", self.connection_id)
        self._connection = connection
        self._connection.channel(self.on_channel_open)
        self._reconnection_time = time.time()

    def on_channel_open(self, channel):
        logger.default(u"Channel for connection '%s' opened",
                       self.connection_id)
        self._channel = channel
        self._channel.add_on_close_callback(self.on_channel_closed)

        if self.prefetch_count:
            from pika import spec
            self._channel.transport.rpc(
                spec.Basic.Qos(0, self.prefetch_count, False), self.on_qos_ok,
                [spec.Basic.QosOk])
        else:
            self.on_qos_ok(self._channel)

    def on_qos_ok(self, channel):
        if self.tx_select:
            channel.tx_select(self.on_channel_tx_select)
        else:
            self._callbacks.process(0, '_on_channel_open', self, self._channel)

    def on_channel_closed(self, code, text):
        logger.warning(u"Channel closed with reason '%s %s'", code, text)
        # With ZAMQP channel should only be closed when
        # 1) connection is closed due ot any reason
        # 2) permission or binding error from producer or consumer
        self._connection.close(code, text)  # safe to call also during closing

        # Enforce connection close because _connection.close() does not
        # close socket when connection has been closed by RabbitMQ or network
        self._connection._adapter_disconnect()

    def on_channel_tx_select(self, frame):
        self._callbacks.process(0, '_on_channel_open', self, self._channel)
示例#7
0
class Connection(object):

    """
    Pika Connection Class.

    This is the core class that implements communication with RabbitMQ. This
    class should not be invoked directly but rather through the use of an
    adapter such as SelectConnection or BlockingConnection.
    """
    def __init__(self, parameters=None,
                 on_open_callback=None,
                 reconnection_strategy=None):
        """
        Connection initialization expects a ConnectionParameters object and
        a callback function to notify when we have successfully connected
        to the AMQP Broker.

        A reconnection_strategy of None will use the NullReconnectionStrategy.
        """
        # Define our callback dictionary
        self.callbacks = CallbackManager()

        # On connection callback
        if on_open_callback:
            self.add_on_open_callback(on_open_callback)

        # Set our configuration options
        self.parameters = parameters or ConnectionParameters()

        # If we did not pass in a reconnection_strategy, setup the default
        self.reconnection = reconnection_strategy or NullReconnectionStrategy()

        # Validate that we don't have erase_credentials enabled with a non
        # Null reconnection strategy
        if self.parameters.credentials.erase_on_connect and \
            not isinstance(self.reconnection, NullReconnectionStrategy):
            # Warn the developer
            warn(("%s was initialized to erase credentials but you have "
                  "specified a %s. Reconnections will fail."),
                 self.parameters.credentials.__class__.__name__,
                 self.reconnection.__class__.__name__)

        # Add our callback for if we close by being disconnected
        self.add_on_open_callback(self.reconnection.on_connection_open)
        self.add_on_close_callback(self.reconnection.on_connection_closed)

        # Set all of our default connection state values
        self._init_connection_state()

        # Connect to the AMQP Broker
        self._connect()

    def _init_connection_state(self):
        """
        Initialize or reset all of our internal state variables for a given
        connection. If we disconnect and reconnect, all of our state needs to
        be wiped.
        """
        # Outbound buffer for buffering writes until we're able to send them
        self.outbound_buffer = simplebuffer.SimpleBuffer()

        # Inbound buffer for decoding frames
        self._frame_buffer = ''

        # Connection state, server properties and channels all change on
        # each connection
        self.server_properties = None
        self._channels = dict()

        # Data used for Heartbeat checking and back-pressure detection
        self.bytes_sent = 0
        self.bytes_received = 0
        self.frames_sent = 0
        self.frames_received = 0
        self.heartbeat = None

        # Default back-pressure multiplier value
        self._backpressure = 10

        # Connection state
        self.connection_state = CONNECTION_CLOSED

        # When closing, hold reason why
        self.closing = 0, None

        # Our starting point once connected, first frame received
        self.callbacks.add(0, spec.Connection.Start, self._on_connection_start)

    def _adapter_connect(self, host, port):
        """
        Subclasses should override to set up the outbound socket connection.
        """
        raise NotImplementedError('%s needs to implement this function' %\
                                  self.__class__.__name__)

    def _adapter_disconnect(self):
        """
        Subclasses should override this to cause the underlying
        transport (socket) to close.
        """
        raise NotImplementedError('%s needs to implement this function' %\
                                  self.__class__.__name__)

    def _connect(self):
        """
        Call the Adapter's connect method after letting the.
        ReconnectionStrategy know that we're going to do so.
        """
        # Let our RS know what we're up to
        self.reconnection.on_connect_attempt(self)

        # Set our connection state
        self.connection_state = CONNECTION_INIT

        # Try and connect
        self._adapter_connect()

    def force_reconnect(self):
        # We're not closing and we're not open, so reconnect
        self._on_connection_closed(None)
        self._connect()

    def _reconnect(self, conn=None):
        """
        Called by the Reconnection Strategy classes or Adapters to disconnect
        and reconnect to the broker.
        """
        # We're already closing but it may not be from reconnect, so first
        # Add a callback that won't be duplicated
        if self.connection_state == CONNECTION_CLOSING:
            self.add_on_close_callback(self._reconnect)
            return

        # If we're open, we want to close normally if we can, then actually
        # reconnect via callback that can't be added more than once
        if self.connection_state == CONNECTION_OPEN:
            self.add_on_close_callback(self._reconnect)
            self._ensure_closed()
            return

        # We're not closing and we're not open, so reconnect
        self._init_connection_state()
        self._connect()

    def _on_connected(self):
        """
        This is called by our connection Adapter to let us know that we've
        connected and we can notify our connection strategy.
        """
        # Set our connection state
        self.connection_state = CONNECTION_PROTOCOL

        # Start the communication with the RabbitMQ Broker
        self._send_frame(pika.frame.ProtocolHeader())

        # Let our reconnection_strategy know we're connected
        self.reconnection.on_transport_connected(self)

    def _on_connection_open(self, frame):
        """
        This is called once we have tuned the connection with the server and
        called the Connection.Open on the server and it has replied with
        Connection.Ok.
        """
        self.known_hosts = frame.method.known_hosts

        # Add a callback handler for the Broker telling us to disconnect
        self.callbacks.add(0, spec.Connection.Close, self._on_remote_close)

        # We're now connected at the AMQP level
        self.connection_state = CONNECTION_OPEN

        # Call our initial callback that we're open
        self.callbacks.process(0, '_on_connection_open', self, self)

    def _on_connection_start(self, frame):
        """
        This is called as a callback once we have received a Connection.Start
        from the server.
        """
        # We're now connected to the broker
        self.connection_state = CONNECTION_START

        # We didn't expect a FrameProtocolHeader, did we get one?
        if isinstance(frame, pika.frame.ProtocolHeader):
            raise ProtocolVersionMismatch(pika.frame.ProtocolHeader, frame)

        # Make sure that the major and minor version matches our spec version
        if (frame.method.version_major,
            frame.method.version_minor) != spec.PROTOCOL_VERSION[0:2]:
            raise ProtocolVersionMismatch(pika.frame.ProtocolHeader(), frame)

        # Get our server properties and split out capabilities
        self.server_properties = frame.method.server_properties
        self.server_capabilities = self.server_properties.get('capabilities',
                                                              dict())
        if hasattr(self.server_properties, 'capabilities'):
            del self.server_properties['capabilities']

        # Build our StartOk authentication response from the credentials obj
        authentication_type, response = \
            self.parameters.credentials.response_for(frame.method)

        # Server asked for credentials for a method we don't support so raise
        # an exception to let the implementing app know
        if not authentication_type:
            raise AuthenticationError("No %s support for the credentials" %\
                                      self.parameters.credentials.TYPE)

        # Erase the credentials if the credentials object wants to
        self.parameters.credentials.erase_credentials()

        # Add our callback for our Connection Tune event
        self.callbacks.add(0, spec.Connection.Tune, self._on_connection_tune)

        # Specify our client properties
        properties = {'product': PRODUCT,
                      'platform': 'Python %s' % python_version(),
                      'capabilities': {'basic.nack': True,
                                            'consumer_cancel_notify': True,
                                            'publisher_confirms': True},
                      'information': 'See http://pika.github.com',
                      'version': __version__}

        # Send our Connection.StartOk
        method = spec.Connection.StartOk(client_properties=properties,
                                         mechanism=authentication_type,
                                         response=response)
        self._send_method(0, method)

    def _combine(self, a, b):
        """
        Pass in two values, if a is 0, return b otherwise if b is 0, return a.
        If neither case matches return the smallest value.
        """
        return min(a, b) or (a or b)

    def _on_connection_tune(self, frame):
        """
        Once the Broker sends back a Connection.Tune, we will set our tuning
        variables that have been returned to us and kick off the Heartbeat
        monitor if required, send our TuneOk and then the Connection. Open rpc
        call on channel 0.
        """
        # Set our connection state
        self.connection_state = CONNECTION_TUNE

        # Get our max channels, frames and heartbeat interval
        cmax = self._combine(self.parameters.channel_max,
                             frame.method.channel_max)
        fmax = self._combine(self.parameters.frame_max,
                             frame.method.frame_max)
        hint = self._combine(self.parameters.heartbeat,
                             frame.method.heartbeat)

        # If we have a heartbeat interval, create a heartbeat checker
        if hint:
            self.heartbeat = HeartbeatChecker(self, hint)

        # Update our connection state with our tuned values
        self.parameters.channel_max = cmax
        self.parameters.frame_max = fmax

        # Send the TuneOk response with what we've agreed upon
        self._send_method(0, spec.Connection.TuneOk(channel_max=cmax,
                                                    frame_max=fmax,
                                                    heartbeat=hint))

        # Send the Connection.Open RPC call for the vhost
        cmd = spec.Connection.Open(virtual_host=self.parameters.virtual_host,
                                   insist=True)
        self._rpc(0, cmd, self._on_connection_open, [spec.Connection.OpenOk])

    def close(self, code=200, text='Normal shutdown'):
        """
        Disconnect from RabbitMQ. If there are any open channels, it will
        attempt to close them prior to fully disconnecting. Channels which
        have active consumers will attempt to send a Basic.Cancel to RabbitMQ
        to cleanly stop the delivery of messages prior to closing the channel.
        """
        if self.is_closing or self.is_closed:
            warn("%s.Close invoked while closing or closed" % \
                 self.__class__.__name__)
            return

        # Set our connection state
        self.connection_state = CONNECTION_CLOSING

        # Carry our code and text around with us
        pika.log.info("Closing connection: %i - %s", code, text)
        self.closing = code, text

        # Disable reconnection strategy on clean shutdown
        if code == 200:
            self.reconnection.set_active(False)

        if self._channels:
            # If we're not already closed
            for channel_number in self._channels.keys():
                self._channels[channel_number].close(code, text)
        else:
            # If we don't have any channels, close
            self._on_close_ready()

    def _on_close_ready(self):
        """
        On a clean shutdown we'll call this once all of our channels are closed
        Let the Broker know we want to close.
        """
        if self.is_closed:
            warn("%s.on_close_ready invoked while closed" %\
                 self.__class__.__name__)
            return

        self._rpc(0, spec.Connection.Close(self.closing[0],
                                           self.closing[1], 0, 0),
                  self._on_connection_closed,
                  [spec.Connection.CloseOk])

    def _on_connection_closed(self, frame, from_adapter=False):
        """
        Let both our RS and Event object know we closed.
        """
        # Set that we're actually closed
        self.connection_state = CONNECTION_CLOSED

        # Call any callbacks registered for this
        self.callbacks.process(0, '_on_connection_closed', self, self)

        pika.log.info("Disconnected from RabbitMQ at %s:%i",
                      self.parameters.host, self.parameters.port)

        # Disconnect our transport if it didn't call on_disconnected
        if not from_adapter:
            self._adapter_disconnect()

        # Cleanup lingering callbacks
        if self._channels:
            # Cleanup channel state
            for channel_number in self._channels.keys():
                self._channels[channel_number].cleanup()
        # Cleanup connection state
        self.callbacks.remove(0, spec.Connection.Close)
        self.callbacks.remove(0, spec.Connection.Start)
        self.callbacks.remove(0, spec.Connection.Open)

    def _on_remote_close(self, frame):
        """
        We've received a remote close from the server.
        """
        self.close(frame.method.reply_code, frame.method.reply_text)

    def _ensure_closed(self):
        """
        If we're not already closed, make sure we're closed.
        """
        # We carry the connection state and so we want to close if we know
        if self.is_open:
            self.close()

    @property
    def is_closed(self):
        """
        Returns a boolean reporting the current connection state.
        """
        return self.connection_state == CONNECTION_CLOSED

    @property
    def is_closing(self):
        """
        Returns a boolean reporting the current connection state.
        """
        return self.connection_state == CONNECTION_CLOSING

    @property
    def is_open(self):
        """
        Returns a boolean reporting the current connection state.
        """
        return self.connection_state == CONNECTION_OPEN

    def add_on_close_callback(self, callback):
        """
        Add a callback notification when the connection has closed.
        """
        self.callbacks.add(0, '_on_connection_closed', callback, False)

    def add_on_open_callback(self, callback):
        """
        Add a callback notification when the connection has opened.
        """
        self.callbacks.add(0, '_on_connection_open', callback, False)

    def add_backpressure_callback(self, callback):
        """
        Add a callback notification when we think backpressue is being applied
        due to the size of the output buffer being exceeded.
        """
        self.callbacks.add(0, 'backpressure', callback, False)

    def set_backpressure_multiplier(self, value=10):
        """
        Alter the backpressure multiplier value. We set this to 10 by default.
        This value is used to raise warnings and trigger the backpressure
        callback.
        """
        self._backpressure = value

    def add_timeout(self, deadline, callback):
        """
        Adapters should override to call the callback after the
        specified number of seconds have elapsed, using a timer, or a
        thread, or similar.
        """
        raise NotImplementedError('%s needs to implement this function ' %\
                                  self.__class__.__name__)

    def remove_timeout(self, callback):
        """
        Adapters should override to call the callback after the
        specified number of seconds have elapsed, using a timer, or a
        thread, or similar.
        """
        raise NotImplementedError('%s needs to implement this function' %\
                                  self.__class__.__name__)

    # Channel related functionality
    def channel(self, on_open_callback, channel_number=None):
        """
        Create a new channel with the next available channel number or pass in
        a channel number to use. Must be non-zero if you would like to specify
        but it is recommended that you let Pika manage the channel numbers.
        """
        # If the user didn't specify a channel_number get the next avail
        if not channel_number:
            channel_number = self._next_channel_number()

        # Open the channel passing the users callback
        self._channels[channel_number] = channel.Channel(self,
                                                         channel_number,
                                                         on_open_callback)

        # Add the callback for our Channel.Close event in case the Broker
        # wants to close us for some reason
        self.callbacks.add(channel_number,
                           spec.Channel.Close,
                           self._on_channel_close)

        # Add the channel spec.Channel.CloseOk callback for _on_channel_close
        self.callbacks.add(channel_number,
                           spec.Channel.CloseOk,
                           self._on_channel_close)

        # Open the channel
        self._channels[channel_number].open()

    def _next_channel_number(self):
        """
        Return the next available channel number or raise on exception.
        """
        # Our limit is the the Codec's Channel Max or MAX_CHANNELS if it's None
        limit = self.parameters.channel_max or channel.MAX_CHANNELS

        # We've used all of our channels
        if len(self._channels) == limit:
            raise NoFreeChannels()

        # Return channel # 1 if we don't have any channels
        if not self._channels:
            return 1

        # Else return our max channel # + 1
        else:
            return max(self._channels.keys()) + 1

    def _on_channel_close(self, frame):
        """
        RPC Response from when a channel closes itself, remove from our stack.
        """
        channel_number = frame.channel_number
        # If we have this channel number in our channels:
        if channel_number in self._channels:
            # If our server called Channel.Close on us remotely
            if isinstance(frame.method, spec.Channel.Close):
                # Call the channel.close() function letting it know it came
                # From the server.
                self._channels[channel_number].close(frame.method.reply_code,
                                                     frame.method.reply_text,
                                                     True)  # Forced close
                # Send a Channel.CloseOk frame
                self._rpc(channel_number, spec.Channel.CloseOk())
                # Remove the CloseOk callback
                self.callbacks.remove(channel_number,
                                      spec.Channel.CloseOk)

            # Remove the channel from our dict
            del(self._channels[channel_number])

        # If we're closing and don't have any channels, go to the next step
        if self.is_closing and not self._channels:
            self._on_close_ready()

    # Data packet and frame handling functions
    def _on_data_available(self, data_in):
        """
        This is called by our Adapter, passing in the data from the socket.
        As long as we have buffer try and map out frame data.
        """
        # Append the data
        self._frame_buffer += data_in

        # Loop while we have a buffer and are getting frames from it
        while self._frame_buffer:

            # Try and build a frame
            consumed_count, frame = pika.frame.decode_frame(self._frame_buffer)

            # Remove the frame we just consumed from our data
            self._frame_buffer = self._frame_buffer[consumed_count:]

            # If we don't have a frame, exit
            if not frame:
                break

            # Increment our bytes received buffer for heartbeat checking
            self.bytes_received += consumed_count

            # Will receive a frame type of -1 if protocol version mismatch
            if frame.frame_type < 0:
                continue

            # Keep track of how many frames have been read
            self.frames_received += 1

            # If we have a Method Frame and have callbacks for it
            if isinstance(frame, pika.frame.Method) and \
                self.callbacks.pending(frame.channel_number, frame.method):

                # Process the callbacks for it
                self.callbacks.process(frame.channel_number,  # Prefix
                                       frame.method,          # Key
                                       self,                  # Caller
                                       frame)                 # Args

            # We don't check for heartbeat frames because we can not count
            # atomic frames reliably due to different message behaviors
            # such as large content frames being transferred slowly
            elif isinstance(frame, pika.frame.Heartbeat):
                continue

            elif frame.channel_number > 0:
                # Call our Channel Handler with the frame
                if frame.channel_number in self._channels:
                    self._channels[\
                        frame.channel_number].transport.deliver(frame)
                else:
                    pika.log.error("Received %s for non-existing channel %i",
                                   frame.method.NAME, frame.channel_number)

    def _rpc(self, channel_number, method,
             callback=None, acceptable_replies=None):
        """
        Make an RPC call for the given callback, channel number and method.
        acceptable_replies lists out what responses we'll process from the
        server with the specified callback.
        """
        # Validate that acceptable_replies is a list or None
        if acceptable_replies and not isinstance(acceptable_replies, list):
            raise TypeError("acceptable_replies should be list or None")

        # Validate the callback is callable
        if callback and not is_callable(callback):
            raise TypeError("callback should be None, a function or method.")

        # If we were passed a callback, add it to our stack
        if callback:
            for reply in acceptable_replies:
                self.callbacks.add(channel_number, reply, callback)

        # Send the rpc call to RabbitMQ
        self._send_method(channel_number, method)

    def _send_frame(self, frame):
        """
        This appends the fully generated frame to send to the broker to the
        output buffer which will be then sent via the connection adapter.
        """
        marshalled_frame = frame.marshal()
        self.bytes_sent += len(marshalled_frame)
        self.frames_sent += 1

        #pika.frame.log_frame(frame.name, marshalled_frame)
        self.outbound_buffer.write(marshalled_frame)
        #self._flush_outbound()
        self._detect_backpressure()

    def _detect_backpressure(self):
        """
        Attempt to calculate if TCP backpressure is being applied due to
        our outbound buffer being larger than the average frame size over
        a window of frames.
        """
        avg_frame_size = self.bytes_sent / self.frames_sent
        if self.outbound_buffer.size > (avg_frame_size * self._backpressure):
            est_frames_behind = self.outbound_buffer.size / avg_frame_size
            message = "Pika: Write buffer exceeded warning threshold" + \
                      " at %i bytes and an estimated %i frames behind"
            warn(message % (self.outbound_buffer.size, est_frames_behind))
            self.callbacks.process(0, 'backpressure', self)

    def _flush_outbound(self):
        """
        Adapters should override to flush the contents of
        outbound_buffer out along the socket.
        """
        raise NotImplementedError('%s needs to implement this function ' %\
                                  self.__class__.__name__)

    def _send_method(self, channel_number, method, content=None):
        """
        Constructs a RPC method frame and then sends it to the broker.
        """
        self._send_frame(pika.frame.Method(channel_number, method))

        if isinstance(content, tuple):
            props = content[0]
            body = content[1]
        else:
            props = None
            body = content

        if props:
            length = 0
            if body:
                length = len(body)
            self._send_frame(pika.frame.Header(channel_number, length, props))

        if body:
            max_piece = (self.parameters.frame_max - \
                         spec.FRAME_HEADER_SIZE - \
                         spec.FRAME_END_SIZE)
            body_buf = simplebuffer.SimpleBuffer(body)
            frames_sent = 0
            frames = len(body) / spec.FRAME_MAX_SIZE
            while body_buf:
                piece_len = min(len(body_buf), max_piece)
                piece = body_buf.read_and_consume(piece_len)
                self._send_frame(pika.frame.Body(channel_number, piece))
                frames_sent += 1

    @property
    def _suggested_buffer_size(self):
        """
        Return the suggested buffer size from the connection state/tune or the
        default if that is None.
        """
        return self.parameters.frame_max or spec.FRAME_MAX_SIZE

    @property
    def basic_nack(self):
        """
        Defines if the active connection has the ability to use basic.nack.
        """
        return self.server_capabilities.get('basic.nack', False)

    @property
    def consumer_cancel_notify(self):
        """
        Specifies if our active connection has the ability to use
        consumer cancel notification.
        """
        return self.server_capabilities.get('consumer_cancel_notify', False)

    @property
    def exchange_exchange_bindings(self):
        """
        Specifies if the active connection can use exchange to exchange
        bindings.
        """
        return self.server_capabilities.get('exchange_exchange_bindings',
                                            False)

    @property
    def publisher_confirms(self):
        """
        Specifies if the active connection can use publisher confirmations.
        """
        return self.server_capabilities.get('publisher_confirms', False)
示例#8
0
    def __init__(self, connection_id=None, hostname=None, port=None,
                 virtual_host=None, username=None, password=None,
                 heartbeat=None, prefetch_count=None, tx_select=None):

        # Allow class variables to provide defaults for:

        # connection_id
        if self.connection_id is None and connection_id is None:
            connection_id =\
                getattr(self, 'grokcore.component.directive.name', 'default')
        if connection_id is not None:
            self.connection_id = connection_id

        # hostname
        if hostname is not None:
            self.hostname = hostname
        assert self.hostname is not None,\
            u"Connection configuration is missing hostname."

        # port
        if port is not None:
            self.port = port
        assert self.port is not None,\
            u"Connection configuration is missing port."

        # virtual_host
        if virtual_host is not None:
            self.virtual_host = virtual_host
        assert self.virtual_host is not None,\
            u"Connection configuration is missing virtual_host."

        # username
        if username is not None:
            self.username = username
        assert self.username is not None,\
            u"Connection configuration is missing username."

        # password
        if password is not None:
            self.password = password
        assert self.password is not None,\
            u"Connection configuration is missing password."

        # heartbeat
        if heartbeat is not None:
            self.heartbeat = heartbeat

        # prefetch_count
        if prefetch_count is not None:
            self.prefetch_count = prefetch_count

        # tx_select
        if tx_select is not None:
            self.tx_select = tx_select

        self._callbacks = CallbackManager()  # callbacks are NOT thread-safe
        self._reconnection_delay = 1.0

        # BBB for affinitic.zamqp
        if getattr(self, 'userid', None):
            from zope.deprecation import deprecated
            self.username = self.userid
            self.userid =\
                deprecated(self.userid,
                           ('Connection.userid is no more. '
                            'Please, use Connection.username instead.'))

        logger.default(u"AMQP Broker connection '%s' created. "
                       u"hostname: '%s', "
                       u"port: '%s', "
                       u"virtual_host: '%s', "
                       u"username: '******', "
                       u"heartbeat: '%s', "
                       u"prefetch_count: '%s', "
                       u"tx_select: '%s'",
                       self.connection_id,
                       self.hostname,
                       self.port,
                       self.virtual_host,
                       self.username,
                       self.heartbeat,
                       self.prefetch_count,
                       self.tx_select)
示例#9
0
class BrokerConnection(grok.GlobalUtility):
    """Connection utility base class"""

    grok.implements(IBrokerConnection)
    grok.baseclass()

    connection_id = None

    hostname = 'localhost'
    port = 5672
    virtual_host = '/'

    username = '******'
    password = '******'

    heartbeat = 0

    prefetch_count = 0
    tx_select = False  # Be aware, that rollback for transactional channel
                       # is safe to use (and tx_select useful) only on
                       # dedicated single-threaded AMQP-consuming ZEO-clients.

    def __init__(self, connection_id=None, hostname=None, port=None,
                 virtual_host=None, username=None, password=None,
                 heartbeat=None, prefetch_count=None, tx_select=None):

        # Allow class variables to provide defaults for:

        # connection_id
        if self.connection_id is None and connection_id is None:
            connection_id =\
                getattr(self, 'grokcore.component.directive.name', 'default')
        if connection_id is not None:
            self.connection_id = connection_id

        # hostname
        if hostname is not None:
            self.hostname = hostname
        assert self.hostname is not None,\
            u"Connection configuration is missing hostname."

        # port
        if port is not None:
            self.port = port
        assert self.port is not None,\
            u"Connection configuration is missing port."

        # virtual_host
        if virtual_host is not None:
            self.virtual_host = virtual_host
        assert self.virtual_host is not None,\
            u"Connection configuration is missing virtual_host."

        # username
        if username is not None:
            self.username = username
        assert self.username is not None,\
            u"Connection configuration is missing username."

        # password
        if password is not None:
            self.password = password
        assert self.password is not None,\
            u"Connection configuration is missing password."

        # heartbeat
        if heartbeat is not None:
            self.heartbeat = heartbeat

        # prefetch_count
        if prefetch_count is not None:
            self.prefetch_count = prefetch_count

        # tx_select
        if tx_select is not None:
            self.tx_select = tx_select

        self._callbacks = CallbackManager()  # callbacks are NOT thread-safe
        self._reconnection_delay = 1.0

        # BBB for affinitic.zamqp
        if getattr(self, 'userid', None):
            from zope.deprecation import deprecated
            self.username = self.userid
            self.userid =\
                deprecated(self.userid,
                           ('Connection.userid is no more. '
                            'Please, use Connection.username instead.'))

        logger.default(u"AMQP Broker connection '%s' created. "
                       u"hostname: '%s', "
                       u"port: '%s', "
                       u"virtual_host: '%s', "
                       u"username: '******', "
                       u"heartbeat: '%s', "
                       u"prefetch_count: '%s', "
                       u"tx_select: '%s'",
                       self.connection_id,
                       self.hostname,
                       self.port,
                       self.virtual_host,
                       self.username,
                       self.heartbeat,
                       self.prefetch_count,
                       self.tx_select)


    def connect(self):
        logger.default(u"Connection '%s' connecting",
                       self.connection_id)
        credentials = PlainCredentials(
            self.username, self.password, erase_on_connect=False)
        parameters = ConnectionParameters(
            self.hostname, self.port, self.virtual_host,
            credentials=credentials,
            heartbeat=self.heartbeat and True or False)
        # AMQP-heartbeat timeout must be set manually due to bug in pika 0.9.5:
        if parameters.heartbeat:
            parameters.heartbeat = int(self.heartbeat)
        self._connection = AsyncoreConnection(
            parameters=parameters,
            on_open_callback=self.on_connect)
        self._reconnection_timeout = None
        self._connection.add_on_close_callback(self.reconnect)

    def reconnect(self, conn=None):
        if not getattr(self, '_reconnection_timeout', None):
            logger.info(u"Trying to reconnect connection '%s' in %s seconds",
                        self.connection_id, self._reconnection_delay)
            self._reconnection_timeout =\
                AsyncoreScheduling(self.connect, self._reconnection_delay)
            self._reconnection_delay *= (random.random() * 0.5) + 1.0
            self._reconnection_delay = min(self._reconnection_delay, 60.0)

    @property
    def is_open(self):
        return getattr(self._connection, 'is_open', False)

    def add_on_channel_open_callback(self, callback):
        self._callbacks.add(0, '_on_channel_open', callback, False)

    def on_connect(self, connection):
        logger.default(u"Connection '%s' connected",
                       self.connection_id)
        self._connection = connection
        self._connection.channel(self.on_channel_open)
        self._reconnection_delay = 1.0

    def on_channel_open(self, channel):
        logger.default(u"Channel for connection '%s' opened",
                       self.connection_id)
        self._channel = channel
        self._channel.add_on_close_callback(self.on_channel_closed)

        if self.prefetch_count:
            from pika import spec
            self._channel.transport.rpc(
                spec.Basic.Qos(0, self.prefetch_count, False),
                self.on_qos_ok,
                [spec.Basic.QosOk]
            )
        else:
            self.on_qos_ok(self._channel)

    def on_qos_ok(self, channel):
        if self.tx_select:
            channel.tx_select(self.on_channel_tx_select)
        else:
            self._callbacks.process(0, '_on_channel_open', self, self._channel)

    def on_channel_closed(self, code, text):
        logger.warning(u"Channel closed with reason '%s %s'",
                       code, text)
        self._connection.close(code, text)
        self.reconnect()

    def on_channel_tx_select(self, frame):
        self._callbacks.process(0, '_on_channel_open', self, self._channel)
示例#10
0
class Connection(object):
    """
    Pika Connection Class.

    This is the core class that implements communication with RabbitMQ. This
    class should not be invoked directly but rather through the use of an
    adapter such as SelectConnection or BlockingConnection.
    """
    def __init__(self,
                 parameters=None,
                 on_open_callback=None,
                 reconnection_strategy=None):
        """
        Connection initialization expects a ConnectionParameters object and
        a callback function to notify when we have successfully connected
        to the AMQP Broker.

        A reconnection_strategy of None will use the NullReconnectionStrategy.
        """
        # Define our callback dictionary
        self.callbacks = CallbackManager()

        # On connection callback
        if on_open_callback:
            self.add_on_open_callback(on_open_callback)

        # Set our configuration options
        self.parameters = parameters or ConnectionParameters()

        # If we did not pass in a reconnection_strategy, setup the default
        self.reconnection = reconnection_strategy or NullReconnectionStrategy()

        # Validate that we don't have erase_credentials enabled with a non
        # Null reconnection strategy
        if self.parameters.credentials.erase_on_connect and \
            not isinstance(self.reconnection, NullReconnectionStrategy):
            # Warn the developer
            warn(("%s was initialized to erase credentials but you have "
                  "specified a %s. Reconnections will fail."),
                 self.parameters.credentials.__class__.__name__,
                 self.reconnection.__class__.__name__)

        # Add our callback for if we close by being disconnected
        self.add_on_open_callback(self.reconnection.on_connection_open)
        self.add_on_close_callback(self.reconnection.on_connection_closed)

        # Set all of our default connection state values
        self._init_connection_state()

        # Connect to the AMQP Broker
        self._connect()

    def _init_connection_state(self):
        """
        Initialize or reset all of our internal state variables for a given
        connection. If we disconnect and reconnect, all of our state needs to
        be wiped.
        """
        # Outbound buffer for buffering writes until we're able to send them
        self.outbound_buffer = simplebuffer.SimpleBuffer()

        # Inbound buffer for decoding frames
        self._frame_buffer = ''

        # Connection state, server properties and channels all change on
        # each connection
        self.server_properties = None
        self._channels = dict()

        # Data used for Heartbeat checking and back-pressure detection
        self.bytes_sent = 0
        self.bytes_received = 0
        self.frames_sent = 0
        self.frames_received = 0
        self.heartbeat = None

        # Default back-pressure multiplier value
        self._backpressure = 10

        # Connection state
        self.connection_state = CONNECTION_CLOSED

        # When closing, hold reason why
        self.closing = 0, None

        # Our starting point once connected, first frame received
        self.callbacks.add(0, spec.Connection.Start, self._on_connection_start)

    def _adapter_connect(self, host, port):
        """
        Subclasses should override to set up the outbound socket connection.
        """
        raise NotImplementedError('%s needs to implement this function' %\
                                  self.__class__.__name__)

    def _adapter_disconnect(self):
        """
        Subclasses should override this to cause the underlying
        transport (socket) to close.
        """
        raise NotImplementedError('%s needs to implement this function' %\
                                  self.__class__.__name__)

    def _connect(self):
        """
        Call the Adapter's connect method after letting the.
        ReconnectionStrategy know that we're going to do so.
        """
        # Let our RS know what we're up to
        self.reconnection.on_connect_attempt(self)

        # Set our connection state
        self.connection_state = CONNECTION_INIT

        # Try and connect
        self._adapter_connect()

    def force_reconnect(self):
        # We're not closing and we're not open, so reconnect
        self._on_connection_closed(None)
        self._connect()

    def _reconnect(self, conn=None):
        """
        Called by the Reconnection Strategy classes or Adapters to disconnect
        and reconnect to the broker.
        """
        # We're already closing but it may not be from reconnect, so first
        # Add a callback that won't be duplicated
        if self.connection_state == CONNECTION_CLOSING:
            self.add_on_close_callback(self._reconnect)
            return

        # If we're open, we want to close normally if we can, then actually
        # reconnect via callback that can't be added more than once
        if self.connection_state == CONNECTION_OPEN:
            self.add_on_close_callback(self._reconnect)
            self._ensure_closed()
            return

        # We're not closing and we're not open, so reconnect
        self._init_connection_state()
        self._connect()

    def _on_connected(self):
        """
        This is called by our connection Adapter to let us know that we've
        connected and we can notify our connection strategy.
        """
        # Set our connection state
        self.connection_state = CONNECTION_PROTOCOL

        # Start the communication with the RabbitMQ Broker
        self._send_frame(pika.frame.ProtocolHeader())

        # Let our reconnection_strategy know we're connected
        self.reconnection.on_transport_connected(self)

    def _on_connection_open(self, frame):
        """
        This is called once we have tuned the connection with the server and
        called the Connection.Open on the server and it has replied with
        Connection.Ok.
        """
        self.known_hosts = frame.method.known_hosts

        # Add a callback handler for the Broker telling us to disconnect
        self.callbacks.add(0, spec.Connection.Close, self._on_remote_close)

        # We're now connected at the AMQP level
        self.connection_state = CONNECTION_OPEN

        # Call our initial callback that we're open
        self.callbacks.process(0, '_on_connection_open', self, self)

    def _on_connection_start(self, frame):
        """
        This is called as a callback once we have received a Connection.Start
        from the server.
        """
        # We're now connected to the broker
        self.connection_state = CONNECTION_START

        # We didn't expect a FrameProtocolHeader, did we get one?
        if isinstance(frame, pika.frame.ProtocolHeader):
            raise ProtocolVersionMismatch(pika.frame.ProtocolHeader, frame)

        # Make sure that the major and minor version matches our spec version
        if (frame.method.version_major,
                frame.method.version_minor) != spec.PROTOCOL_VERSION[0:2]:
            raise ProtocolVersionMismatch(pika.frame.ProtocolHeader(), frame)

        # Get our server properties and split out capabilities
        self.server_properties = frame.method.server_properties
        self.server_capabilities = self.server_properties.get(
            'capabilities', dict())
        if hasattr(self.server_properties, 'capabilities'):
            del self.server_properties['capabilities']

        # Build our StartOk authentication response from the credentials obj
        authentication_type, response = \
            self.parameters.credentials.response_for(frame.method)

        # Server asked for credentials for a method we don't support so raise
        # an exception to let the implementing app know
        if not authentication_type:
            raise AuthenticationError("No %s support for the credentials" %\
                                      self.parameters.credentials.TYPE)

        # Erase the credentials if the credentials object wants to
        self.parameters.credentials.erase_credentials()

        # Add our callback for our Connection Tune event
        self.callbacks.add(0, spec.Connection.Tune, self._on_connection_tune)

        # Specify our client properties
        properties = {
            'product': PRODUCT,
            'platform': 'Python %s' % python_version(),
            'capabilities': {
                'basic.nack': True,
                'consumer_cancel_notify': True,
                'publisher_confirms': True
            },
            'information': 'See http://pika.github.com',
            'version': __version__
        }

        # Send our Connection.StartOk
        method = spec.Connection.StartOk(client_properties=properties,
                                         mechanism=authentication_type,
                                         response=response)
        self._send_method(0, method)

    def _combine(self, a, b):
        """
        Pass in two values, if a is 0, return b otherwise if b is 0, return a.
        If neither case matches return the smallest value.
        """
        return min(a, b) or (a or b)

    def _on_connection_tune(self, frame):
        """
        Once the Broker sends back a Connection.Tune, we will set our tuning
        variables that have been returned to us and kick off the Heartbeat
        monitor if required, send our TuneOk and then the Connection. Open rpc
        call on channel 0.
        """
        # Set our connection state
        self.connection_state = CONNECTION_TUNE

        # Get our max channels, frames and heartbeat interval
        cmax = self._combine(self.parameters.channel_max,
                             frame.method.channel_max)
        fmax = self._combine(self.parameters.frame_max, frame.method.frame_max)
        hint = self._combine(self.parameters.heartbeat, frame.method.heartbeat)

        # If we have a heartbeat interval, create a heartbeat checker
        if hint:
            self.heartbeat = HeartbeatChecker(self, hint)

        # Update our connection state with our tuned values
        self.parameters.channel_max = cmax
        self.parameters.frame_max = fmax

        # Send the TuneOk response with what we've agreed upon
        self._send_method(
            0,
            spec.Connection.TuneOk(channel_max=cmax,
                                   frame_max=fmax,
                                   heartbeat=hint))

        # Send the Connection.Open RPC call for the vhost
        cmd = spec.Connection.Open(virtual_host=self.parameters.virtual_host,
                                   insist=True)
        self._rpc(0, cmd, self._on_connection_open, [spec.Connection.OpenOk])

    def close(self, code=200, text='Normal shutdown'):
        """
        Disconnect from RabbitMQ. If there are any open channels, it will
        attempt to close them prior to fully disconnecting. Channels which
        have active consumers will attempt to send a Basic.Cancel to RabbitMQ
        to cleanly stop the delivery of messages prior to closing the channel.
        """
        if self.is_closing or self.is_closed:
            warn("%s.Close invoked while closing or closed" % \
                 self.__class__.__name__)
            return

        # Set our connection state
        self.connection_state = CONNECTION_CLOSING

        # Carry our code and text around with us
        pika.log.info("Closing connection: %i - %s", code, text)
        self.closing = code, text

        # Disable reconnection strategy on clean shutdown
        if code == 200:
            self.reconnection.set_active(False)

        if self._channels:
            # If we're not already closed
            for channel_number in self._channels.keys():
                self._channels[channel_number].close(code, text)
        else:
            # If we don't have any channels, close
            self._on_close_ready()

    def _on_close_ready(self):
        """
        On a clean shutdown we'll call this once all of our channels are closed
        Let the Broker know we want to close.
        """
        if self.is_closed:
            warn("%s.on_close_ready invoked while closed" %\
                 self.__class__.__name__)
            return

        self._rpc(
            0, spec.Connection.Close(self.closing[0], self.closing[1], 0, 0),
            self._on_connection_closed, [spec.Connection.CloseOk])

    def _on_connection_closed(self, frame, from_adapter=False):
        """
        Let both our RS and Event object know we closed.
        """
        # Set that we're actually closed
        self.connection_state = CONNECTION_CLOSED

        # Call any callbacks registered for this
        self.callbacks.process(0, '_on_connection_closed', self, self)

        pika.log.info("Disconnected from RabbitMQ at %s:%i",
                      self.parameters.host, self.parameters.port)

        # Disconnect our transport if it didn't call on_disconnected
        if not from_adapter:
            self._adapter_disconnect()

        # Cleanup lingering callbacks
        if self._channels:
            # Cleanup channel state
            for channel_number in self._channels.keys():
                self._channels[channel_number].cleanup()
        # Cleanup connection state
        self.callbacks.remove(0, spec.Connection.Close)
        self.callbacks.remove(0, spec.Connection.Start)
        self.callbacks.remove(0, spec.Connection.Open)

    def _on_remote_close(self, frame):
        """
        We've received a remote close from the server.
        """
        self.close(frame.method.reply_code, frame.method.reply_text)

    def _ensure_closed(self):
        """
        If we're not already closed, make sure we're closed.
        """
        # We carry the connection state and so we want to close if we know
        if self.is_open:
            self.close()

    @property
    def is_closed(self):
        """
        Returns a boolean reporting the current connection state.
        """
        return self.connection_state == CONNECTION_CLOSED

    @property
    def is_closing(self):
        """
        Returns a boolean reporting the current connection state.
        """
        return self.connection_state == CONNECTION_CLOSING

    @property
    def is_open(self):
        """
        Returns a boolean reporting the current connection state.
        """
        return self.connection_state == CONNECTION_OPEN

    def add_on_close_callback(self, callback):
        """
        Add a callback notification when the connection has closed.
        """
        self.callbacks.add(0, '_on_connection_closed', callback, False)

    def add_on_open_callback(self, callback):
        """
        Add a callback notification when the connection has opened.
        """
        self.callbacks.add(0, '_on_connection_open', callback, False)

    def add_backpressure_callback(self, callback):
        """
        Add a callback notification when we think backpressure is being applied
        due to the size of the output buffer being exceeded.
        """
        self.callbacks.add(0, 'backpressure', callback, False)

    def set_backpressure_multiplier(self, value=10):
        """
        Alter the backpressure multiplier value. We set this to 10 by default.
        This value is used to raise warnings and trigger the backpressure
        callback.
        """
        self._backpressure = value

    def add_timeout(self, deadline, callback):
        """
        Adapters should override to call the callback after the
        specified number of seconds have elapsed, using a timer, or a
        thread, or similar.
        """
        raise NotImplementedError('%s needs to implement this function ' %\
                                  self.__class__.__name__)

    def remove_timeout(self, callback):
        """
        Adapters should override to call the callback after the
        specified number of seconds have elapsed, using a timer, or a
        thread, or similar.
        """
        raise NotImplementedError('%s needs to implement this function' %\
                                  self.__class__.__name__)

    # Channel related functionality
    def channel(self, on_open_callback, channel_number=None):
        """
        Create a new channel with the next available channel number or pass in
        a channel number to use. Must be non-zero if you would like to specify
        but it is recommended that you let Pika manage the channel numbers.
        """
        # If the user didn't specify a channel_number get the next avail
        if not channel_number:
            channel_number = self._next_channel_number()

        # Open the channel passing the users callback
        self._channels[channel_number] = channel.Channel(
            self, channel_number, on_open_callback)

        # Add the callback for our Channel.Close event in case the Broker
        # wants to close us for some reason
        self.callbacks.add(channel_number, spec.Channel.Close,
                           self._on_channel_close)

        # Add the channel spec.Channel.CloseOk callback for _on_channel_close
        self.callbacks.add(channel_number, spec.Channel.CloseOk,
                           self._on_channel_close)

        # Open the channel
        self._channels[channel_number].open()

    def _next_channel_number(self):
        """
        Return the next available channel number or raise on exception.
        """
        # Our limit is the the Codec's Channel Max or MAX_CHANNELS if it's None
        limit = self.parameters.channel_max or channel.MAX_CHANNELS

        # We've used all of our channels
        if len(self._channels) == limit:
            raise NoFreeChannels()

        # Return channel # 1 if we don't have any channels
        if not self._channels:
            return 1

        # Else return our max channel # + 1
        else:
            return max(self._channels.keys()) + 1

    def _on_channel_close(self, frame):
        """
        RPC Response from when a channel closes itself, remove from our stack.
        """
        channel_number = frame.channel_number
        # If we have this channel number in our channels:
        if channel_number in self._channels:
            # If our server called Channel.Close on us remotely
            if isinstance(frame.method, spec.Channel.Close):
                # Call the channel.close() function letting it know it came
                # From the server.
                self._channels[channel_number].close(frame.method.reply_code,
                                                     frame.method.reply_text,
                                                     True)  # Forced close
                # Send a Channel.CloseOk frame
                self._rpc(channel_number, spec.Channel.CloseOk())
                # Remove the CloseOk callback
                self.callbacks.remove(channel_number, spec.Channel.CloseOk)

            # Remove the channel from our dict
            del (self._channels[channel_number])

        # If we're closing and don't have any channels, go to the next step
        if self.is_closing and not self._channels:
            self._on_close_ready()

    # Data packet and frame handling functions
    def _on_data_available(self, data_in):
        """
        This is called by our Adapter, passing in the data from the socket.
        As long as we have buffer try and map out frame data.
        """
        # Append the data
        self._frame_buffer += data_in

        # Loop while we have a buffer and are getting frames from it
        while self._frame_buffer:

            # Try and build a frame
            consumed_count, frame = pika.frame.decode_frame(self._frame_buffer)

            # Remove the frame we just consumed from our data
            self._frame_buffer = self._frame_buffer[consumed_count:]

            # If we don't have a frame, exit
            if not frame:
                break

            # Increment our bytes received buffer for heartbeat checking
            self.bytes_received += consumed_count

            # Will receive a frame type of -1 if protocol version mismatch
            if frame.frame_type < 0:
                continue

            # Keep track of how many frames have been read
            self.frames_received += 1

            # If we have a Method Frame and have callbacks for it
            if isinstance(frame, pika.frame.Method) and \
                self.callbacks.pending(frame.channel_number, frame.method):

                # Process the callbacks for it
                self.callbacks.process(
                    frame.channel_number,  # Prefix
                    frame.method,  # Key
                    self,  # Caller
                    frame)  # Args

            # We don't check for heartbeat frames because we can not count
            # atomic frames reliably due to different message behaviors
            # such as large content frames being transferred slowly
            elif isinstance(frame, pika.frame.Heartbeat):
                continue

            elif frame.channel_number > 0:
                # Call our Channel Handler with the frame
                if frame.channel_number in self._channels:
                    self._channels[\
                        frame.channel_number].transport.deliver(frame)
                else:
                    pika.log.error("Received %s for non-existing channel %i",
                                   frame.method.NAME, frame.channel_number)

    def _rpc(self,
             channel_number,
             method,
             callback=None,
             acceptable_replies=None):
        """
        Make an RPC call for the given callback, channel number and method.
        acceptable_replies lists out what responses we'll process from the
        server with the specified callback.
        """
        # Validate that acceptable_replies is a list or None
        if acceptable_replies and not isinstance(acceptable_replies, list):
            raise TypeError("acceptable_replies should be list or None")

        # Validate the callback is callable
        if callback and not is_callable(callback):
            raise TypeError("callback should be None, a function or method.")

        # If we were passed a callback, add it to our stack
        if callback:
            for reply in acceptable_replies:
                self.callbacks.add(channel_number, reply, callback)

        # Send the rpc call to RabbitMQ
        self._send_method(channel_number, method)

    def _send_frame(self, frame):
        """
        This appends the fully generated frame to send to the broker to the
        output buffer which will be then sent via the connection adapter.
        """
        marshalled_frame = frame.marshal()
        self.bytes_sent += len(marshalled_frame)
        self.frames_sent += 1

        #pika.frame.log_frame(frame.name, marshalled_frame)
        self.outbound_buffer.write(marshalled_frame)
        #self._flush_outbound()
        self._detect_backpressure()

    def _detect_backpressure(self):
        """
        Attempt to calculate if TCP backpressure is being applied due to
        our outbound buffer being larger than the average frame size over
        a window of frames.
        """
        avg_frame_size = self.bytes_sent / self.frames_sent
        if self.outbound_buffer.size > (avg_frame_size * self._backpressure):
            est_frames_behind = self.outbound_buffer.size / avg_frame_size
            message = "Pika: Write buffer exceeded warning threshold" + \
                      " at %i bytes and an estimated %i frames behind"
            warn(message % (self.outbound_buffer.size, est_frames_behind))
            self.callbacks.process(0, 'backpressure', self)

    def _flush_outbound(self):
        """
        Adapters should override to flush the contents of
        outbound_buffer out along the socket.
        """
        raise NotImplementedError('%s needs to implement this function ' %\
                                  self.__class__.__name__)

    def _send_method(self, channel_number, method, content=None):
        """
        Constructs a RPC method frame and then sends it to the broker.
        """
        self._send_frame(pika.frame.Method(channel_number, method))

        if isinstance(content, tuple):
            props = content[0]
            body = content[1]
        else:
            props = None
            body = content

        if props:
            length = 0
            if body:
                length = len(body)
            self._send_frame(pika.frame.Header(channel_number, length, props))

        if body:
            max_piece = (self.parameters.frame_max - \
                         spec.FRAME_HEADER_SIZE - \
                         spec.FRAME_END_SIZE)
            body_buf = simplebuffer.SimpleBuffer(body)
            frames_sent = 0
            frames = len(body) / spec.FRAME_MAX_SIZE
            while body_buf:
                piece_len = min(len(body_buf), max_piece)
                piece = body_buf.read_and_consume(piece_len)
                self._send_frame(pika.frame.Body(channel_number, piece))
                frames_sent += 1

    @property
    def _suggested_buffer_size(self):
        """
        Return the suggested buffer size from the connection state/tune or the
        default if that is None.
        """
        return self.parameters.frame_max or spec.FRAME_MAX_SIZE

    @property
    def basic_nack(self):
        """
        Defines if the active connection has the ability to use basic.nack.
        """
        return self.server_capabilities.get('basic.nack', False)

    @property
    def consumer_cancel_notify(self):
        """
        Specifies if our active connection has the ability to use
        consumer cancel notification.
        """
        return self.server_capabilities.get('consumer_cancel_notify', False)

    @property
    def exchange_exchange_bindings(self):
        """
        Specifies if the active connection can use exchange to exchange
        bindings.
        """
        return self.server_capabilities.get('exchange_exchange_bindings',
                                            False)

    @property
    def publisher_confirms(self):
        """
        Specifies if the active connection can use publisher confirmations.
        """
        return self.server_capabilities.get('publisher_confirms', False)
示例#11
0
    def __init__(self, connection_id=None, exchange=None, routing_key=None,
                 durable=None, exchange_type=None, exchange_durable=None,
                 exchange_auto_delete=None, exchange_auto_declare=None,
                 queue=None, queue_durable=None, queue_auto_delete=None,
                 queue_exclusive=None, queue_arguments=None,
                 queue_auto_declare=None, auto_declare=None, auto_delete=None,
                 reply_to=None, serializer=None):

        self._connection = None
        self._queue = None  # will default to self.queue
        self._lock = threading.Lock()
        self._threadlocal = threading.local()  # to thread-safe some attributes

        # Allow class variables to provide defaults for:

        # connection_id
        if connection_id is not None:
            self.connection_id = connection_id
        assert self.connection_id is not None,\
            u"Producer configuration is missing connection_id."

        # exchange
        if exchange is not None:
            self.exchange = exchange
        assert self.exchange is not None,\
            u"Producer configuration is missing exchange."

        # durable (and the default for exchange/queue_durable)
        if durable is not None:
            self.durable = durable

        # auto_delete (and the default for exchange/queue_auto_delete)
        if auto_delete is not None:
            self.auto_delete = auto_delete
        elif self.auto_delete is None:
            self.auto_delete = not self.durable

        # exchange_type
        if exchange_type is not None:
            self.exchange_type = exchange_type
        # exchange_durable
        if exchange_durable is not None:
            self.exchange_durable = exchange_durable
        elif self.exchange_durable is None:
            self.exchange_durable = self.durable
        # exchange_auto_delete
        if exchange_auto_delete is not None:
            self.exchange_auto_delete = exchange_auto_delete
        elif self.exchange_auto_delete is None:
            self.exchange_auto_delete = self.auto_delete

        # queue
        if queue is not None:
            self._queue = self.queue = queue
        # queue_durable
        if queue_durable is not None:
            self.queue_durable = queue_durable
        elif self.queue_durable is None:
            self.queue_durable = self.durable
        # queue_auto_delete
        if queue_auto_delete is not None:
            self.queue_auto_delete = queue_auto_delete
        elif self.queue_auto_delete is None:
            self.queue_auto_delete = self.auto_delete
        # queue_exclusive
        if queue_exclusive is not None:
            self.queue_exclusive = queue_exclusive
        if self.queue_exclusive is True:
            self.queue_durable = False
            self.queue_auto_delete = True
        # queue_arguments
        if queue_arguments is not None:
            self.queue_arguments = queue_arguments

        # routing_key
        if self.routing_key is None and routing_key is None:
            routing_key =\
                getattr(self, 'grokcore.component.directive.name', None)
        if routing_key is not None:
            self.routing_key = routing_key
        elif self.routing_key is None:
            if self.queue is not None:
                self.routing_key = self.queue
            elif self.exchange_type == 'fanout':
                self.routing_key = '*'
        assert self.routing_key is not None,\
            u"Producer configuration is missing routing_key."

        # auto_declare
        if auto_declare is not None:
            self.auto_declare = auto_declare
        if exchange_auto_declare is not None:
            self.exchange_auto_declare = exchange_auto_declare
        elif self.exchange_auto_declare is None:
            self.exchange_auto_declare = self.auto_declare
        if queue_auto_declare is not None:
            self.queue_auto_declare = queue_auto_declare
        elif self.queue_auto_declare is None:
            self.queue_auto_declare = self.auto_declare

        # reply_to
        if reply_to is not None:
            self.reply_to = reply_to

        # serializer
        if serializer is not None:
            self.serializer = serializer

        # initialize callbacks
        self._callbacks = CallbackManager()  # callbacks are NOT thread-safe

        # subscribe to the connect initialization event
        provideHandler(self.on_before_broker_connect,
                       [IBeforeBrokerConnectEvent])
示例#12
0
class Producer(grok.GlobalUtility, VTM):
    """Producer utility base class"""

    grok.baseclass()
    grok.implements(IProducer)

    connection_id = None

    exchange = ''
    routing_key = None
    durable = True

    exchange_type = 'direct'
    exchange_durable = None
    exchange_auto_delete = None
    exchange_auto_declare = None

    queue = None
    queue_durable = None
    queue_auto_delete = None
    queue_exclusive = False
    queue_arguments = {}
    queue_auto_declare = None

    auto_declare = True
    auto_delete = None

    reply_to = None
    serializer = 'pickle'

    def __init__(self, connection_id=None, exchange=None, routing_key=None,
                 durable=None, exchange_type=None, exchange_durable=None,
                 exchange_auto_delete=None, exchange_auto_declare=None,
                 queue=None, queue_durable=None, queue_auto_delete=None,
                 queue_exclusive=None, queue_arguments=None,
                 queue_auto_declare=None, auto_declare=None, auto_delete=None,
                 reply_to=None, serializer=None):

        self._connection = None
        self._queue = None  # will default to self.queue
        self._lock = threading.Lock()
        self._threadlocal = threading.local()  # to thread-safe some attributes

        # Allow class variables to provide defaults for:

        # connection_id
        if connection_id is not None:
            self.connection_id = connection_id
        assert self.connection_id is not None,\
            u"Producer configuration is missing connection_id."

        # exchange
        if exchange is not None:
            self.exchange = exchange
        assert self.exchange is not None,\
            u"Producer configuration is missing exchange."

        # durable (and the default for exchange/queue_durable)
        if durable is not None:
            self.durable = durable

        # auto_delete (and the default for exchange/queue_auto_delete)
        if auto_delete is not None:
            self.auto_delete = auto_delete
        elif self.auto_delete is None:
            self.auto_delete = not self.durable

        # exchange_type
        if exchange_type is not None:
            self.exchange_type = exchange_type
        # exchange_durable
        if exchange_durable is not None:
            self.exchange_durable = exchange_durable
        elif self.exchange_durable is None:
            self.exchange_durable = self.durable
        # exchange_auto_delete
        if exchange_auto_delete is not None:
            self.exchange_auto_delete = exchange_auto_delete
        elif self.exchange_auto_delete is None:
            self.exchange_auto_delete = self.auto_delete

        # queue
        if queue is not None:
            self._queue = self.queue = queue
        # queue_durable
        if queue_durable is not None:
            self.queue_durable = queue_durable
        elif self.queue_durable is None:
            self.queue_durable = self.durable
        # queue_auto_delete
        if queue_auto_delete is not None:
            self.queue_auto_delete = queue_auto_delete
        elif self.queue_auto_delete is None:
            self.queue_auto_delete = self.auto_delete
        # queue_exclusive
        if queue_exclusive is not None:
            self.queue_exclusive = queue_exclusive
        if self.queue_exclusive is True:
            self.queue_durable = False
            self.queue_auto_delete = True
        # queue_arguments
        if queue_arguments is not None:
            self.queue_arguments = queue_arguments

        # routing_key
        if self.routing_key is None and routing_key is None:
            routing_key =\
                getattr(self, 'grokcore.component.directive.name', None)
        if routing_key is not None:
            self.routing_key = routing_key
        elif self.routing_key is None:
            if self.queue is not None:
                self.routing_key = self.queue
            elif self.exchange_type == 'fanout':
                self.routing_key = '*'
        assert self.routing_key is not None,\
            u"Producer configuration is missing routing_key."

        # auto_declare
        if auto_declare is not None:
            self.auto_declare = auto_declare
        if exchange_auto_declare is not None:
            self.exchange_auto_declare = exchange_auto_declare
        elif self.exchange_auto_declare is None:
            self.exchange_auto_declare = self.auto_declare
        if queue_auto_declare is not None:
            self.queue_auto_declare = queue_auto_declare
        elif self.queue_auto_declare is None:
            self.queue_auto_declare = self.auto_declare

        # reply_to
        if reply_to is not None:
            self.reply_to = reply_to

        # serializer
        if serializer is not None:
            self.serializer = serializer

        # initialize callbacks
        self._callbacks = CallbackManager()  # callbacks are NOT thread-safe

        # subscribe to the connect initialization event
        provideHandler(self.on_before_broker_connect,
                       [IBeforeBrokerConnectEvent])

    def on_before_broker_connect(self, event=None):
        self._connection = queryUtility(IBrokerConnection,
                                        name=self.connection_id)
        if self._connection:
            self._connection.add_on_channel_open_callback(self.on_channel_open)
        else:
            logger.warning(u"Connection '%s' was not registered. "
                           u"Producer '%s' cannot be connected.",
                           self.connection_id, self.routing_key)

    def on_channel_open(self, channel):
        self._channel = channel

        if self.exchange_auto_declare and self.exchange\
                and not self.exchange.startswith('amq.'):
            self.declare_exchange()
        elif self.queue_auto_declare and self.queue is not None\
                and not self.queue.startswith('amq.'):
            self.declare_queue()
        else:
            self.on_ready_to_publish()

    def declare_exchange(self):
        self._channel.exchange_declare(exchange=self.exchange,
                                       type=self.exchange_type,
                                       durable=self.exchange_durable,
                                       auto_delete=self.exchange_auto_delete,
                                       callback=self.on_exchange_declared)

    def on_exchange_declared(self, frame):
        logger.default(u"Producer declared exchange '%s' on connection '%s'",
                       self.exchange, self.connection_id)
        if self.queue_auto_declare and self.queue is not None\
                and not self.queue.startswith('amq.'):
            self.declare_queue()
        else:
            self.on_ready_to_publish()

    def declare_queue(self):
        self._channel.queue_declare(queue=self.queue,
                                    durable=self.queue_durable,
                                    exclusive=self.queue_exclusive,
                                    auto_delete=self.queue_auto_delete,
                                    arguments=self.queue_arguments,
                                    callback=self.on_queue_declared)

    def on_queue_declared(self, frame):
        self._queue = frame.method.queue  # get the real queue name
        logger.default(u"Producer declared queue '%s' on connection '%s'",
                       self._queue, self.connection_id)
        if self.auto_declare and self.exchange:
            self.bind_queue()
        else:
            self.on_ready_to_publish()

    def bind_queue(self):
        self._channel.queue_bind(exchange=self.exchange, queue=self._queue,
                                 routing_key=self.routing_key or self._queue,
                                 callback=self.on_queue_bound)

    def on_queue_bound(self, frame):
        logger.default(u"Producer bound queue '%s' to exchange '%s' "
                       u"on connection '%s'",
                       self._queue, self.exchange, self.connection_id)
        self.on_ready_to_publish()

    def on_ready_to_publish(self):
        logger.default(u"Producer ready to publish to exchange '%s' "
                       u"with routing key '%s' on connection '%s'",
                       self.exchange, self.routing_key, self.connection_id)
        self._callbacks.process(0, "_on_ready_to_publish", self)

    @property
    def is_connected(self):
        if getattr(self._connection, "is_open", False)\
                and getattr(self, '_channel', None):
            return True
        else:
            return False

    def register(self):
        self._register()

    def publish(self, message, exchange=None, routing_key=None,
                mandatory=False, immediate=False,
                content_type=None, content_encoding=None,
                headers=None, delivery_mode=None, priority=None,
                correlation_id=None, reply_to=None, expiration=None,
                message_id=None, timestamp=None, type=None, user_id=None,
                app_id=None, cluster_id=None, serializer=None):

        exchange = exchange or self.exchange
        routing_key = routing_key or self.routing_key
        reply_to = reply_to or self.reply_to
        serializer = serializer or self.serializer

        if correlation_id is not None:
            correlation_id = str(correlation_id)

        if serializer and not content_type:
            util = getUtility(ISerializer, name=serializer)
            content_type = util.content_type
            message = util.serialize(message)
        elif not content_type:
            content_type = 'text/plain'

        if delivery_mode is None:
            if not self.durable:
                delivery_mode = 1  # message is transient
            else:
                delivery_mode = 2  # message is persistent

        properties = BasicProperties(
            content_type=content_type, content_encoding=content_encoding,
            headers=headers, delivery_mode=delivery_mode, priority=priority,
            correlation_id=correlation_id, reply_to=reply_to,
            expiration=expiration, message_id=message_id, timestamp=timestamp,
            type=type, user_id=user_id, app_id=app_id, cluster_id=cluster_id)

        msg = {
            'exchange': exchange,
            'routing_key': routing_key,
            'body': message,
            'properties': properties,
        }

        if self.registered():
            self._pending_messages.insert(0, msg)
        elif self._basic_publish(**msg) and self._connection.tx_select:
            self._tx_commit()  # minimal support for transactional channel

    def __len__(self):
        """Return message count of the target queue"""
        # XXX: Producer knows it target queue only, if it's explicitly
        # set in its definition. Otherwise self._queue is None
        assert self._queue, "Sorry, producer doesn't know its target queue."
        with BlockingChannel(self.connection_id) as channel:
            frame = channel.queue_declare(queue=self._queue,
                                          durable=self.queue_durable,
                                          exclusive=self.queue_exclusive,
                                          auto_delete=self.queue_auto_delete,
                                          arguments=self.queue_arguments)
            return frame.method.message_count

    def _basic_publish(self, **kwargs):
        retry_constructor = lambda func, kwargs: lambda: func(**kwargs)

        if getattr(self._connection, "is_open", False)\
                and getattr(self, '_channel', None):
            self._channel.basic_publish(**kwargs)
            return True

        elif getattr(kwargs.get("properties"), "delivery_mode", 1) == 2:
            logger.warning(u"No connection. Durable message was left into "
                           u"volatile memory to wait for a new connection "
                           u"'%s'", kwargs)
            retry_callback = retry_constructor(self._basic_publish, kwargs)
            with self._lock:
                self._callbacks.add(0, '_on_ready_to_publish', retry_callback)
            # XXX: ^^^ When connection is down, durable messages should be
            # stored in ZODB to prevent losing them, e.g. because of restart.
            return False

    def _tx_commit(self):
        if getattr(self._connection, "is_open", False)\
                and getattr(self, '_channel', None):
            self._channel.tx_commit()
        else:
            logger.warning(u'No connection. Tx.Commit was not sent')

    def _begin(self):
        self._pending_messages = []

    def _abort(self):
        self._pending_messages = None

    def _finish(self):
        while self._pending_messages:
            self._basic_publish(**self._pending_messages.pop())
        if getattr(self._connection, "tx_select", False):
            self._tx_commit()  # minimal support for transactional channel

    # Define threadlocal VTM._v_registered:

    def _get_v_registered(self):
        return getattr(self._threadlocal, "_v_registered", 0)

    def _set_v_registered(self, value):
        self._threadlocal._v_registered = value

    _v_registered = property(_get_v_registered, _set_v_registered)

    # Define threadlocal VTM._v_finalize:

    def _get_v_finalize(self):
        return getattr(self._threadlocal, "_v_finalize", 0)

    def _set_v_finalize(self, value):
        self._threadlocal._v_finalize = value

    _v_finalize = property(_get_v_finalize, _set_v_finalize)

    # Define threadlocal self._pending_messages:

    def _get_pending_messages(self):
        return getattr(self._threadlocal, "_pending_messages", None)

    def _set_pending_messages(self, value):
        self._threadlocal._pending_messages = value

    _pending_messages = property(_get_pending_messages, _set_pending_messages)
示例#13
0
class Producer(grok.GlobalUtility, VTM):
    """Producer utility base class"""

    grok.baseclass()
    grok.implements(IProducer)

    connection_id = None

    exchange = ''
    routing_key = None
    durable = True

    exchange_type = 'direct'
    exchange_durable = None

    queue = None
    queue_durable = None
    queue_exclusive = False
    queue_arguments = {}

    auto_declare = True

    reply_to = None
    serializer = 'pickle'

    def __init__(self, connection_id=None, exchange=None, routing_key=None,
                 durable=None, exchange_type=None, exchange_durable=None,
                 queue=None, queue_durable=None, queue_exclusive = None,
                 queue_arguments=None, auto_declare=None, reply_to=None,
                 serializer=None):

        self._connection = None

        # Allow class variables to provide defaults for:

        # connection_id
        if connection_id is not None:
            self.connection_id = connection_id
        assert self.connection_id is not None,\
               u"Producer configuration is missing connection_id."

        # exchange
        if exchange is not None:
            self.exchange = exchange
        assert self.exchange is not None,\
               u"Producer configuration is missing exchange."

        # routing_key
        if self.routing_key is None and routing_key is None:
            routing_key =\
                getattr(self, 'grokcore.component.directive.name', None)
        if routing_key is not None:
            self.routing_key = routing_key
        assert self.routing_key is not None,\
               u"Producer configuration is missing routing_key."

        # durable (and the default for exchange_durable)
        if durable is not None:
            self.durable = durable

        # exchange_type
        if exchange_type is not None:
            self.exchange_type = exchange_type
        # exchange_durable
        if exchange_durable is not None:
            self.exchange_durable = exchange_durable
        elif self.exchange_durable is None:
            self.exchange_durable = self.durable

        # queue
        if queue:
            self.queue = queue
        # queue_durable
        if queue_durable is not None:
            self.queue_durable = queue_durable
        elif self.queue_durable is None:
            self.queue_durable = self.durable
        # queue_exclusive
        if queue_exclusive is not None:
            self.queue_exclusive = queue_exclusive
        # queue_arguments
        if queue_arguments is not None:
            self.queue_arguments = queue_arguments

        # auto_declare
        if auto_declare is not None:
            self.auto_declare = auto_declare

        # reply_to
        if reply_to is not None:
            self.reply_to = reply_to

        # serializer
        if serializer is not None:
            self.serializer = serializer

        # initialize callbacks
        self._callbacks = CallbackManager()  # callbacks are NOT thread-safe

        # subscribe to the connect initialization event
        provideHandler(self.on_before_broker_connect,
                       [IBeforeBrokerConnectEvent])

    def on_before_broker_connect(self, event=None):
        self._connection = queryUtility(IBrokerConnection,
                                        name=self.connection_id)
        if self._connection:
            self._connection.add_on_channel_open_callback(self.on_channel_open)
        else:
            logger.warning(("Connection '%s' was not registered. "
                            "Producer '%s' cannot be connected."),
                            self.connection_id, self.routing_key)

    def on_channel_open(self, channel):
        self._channel = channel

        if self.auto_declare and self.exchange:
            self.declare_exchange()
        elif self.auto_declare and self.queue:
            self.declare_queue()
        else:
            self.on_ready_to_publish()

    def declare_exchange(self):
        self._channel.exchange_declare(exchange=self.exchange,
                                       type=self.exchange_type,
                                       durable=self.exchange_durable,
                                       auto_delete=not self.exchange_durable,
                                       callback=self.on_exchange_declared)

    def on_exchange_declared(self, frame):
        logger.info("Producer declared exchange '%s'", self.exchange)
        if self.auto_declare and self.queue:
            self.declare_queue()
        else:
            self.on_ready_to_publish()

    def declare_queue(self):
        self._channel.queue_declare(queue=self.queue,
                                    durable=self.queue_durable,
                                    exclusive=self.queue_exclusive,
                                    auto_delete=not self.queue_durable,
                                    arguments=self.queue_arguments,
                                    callback=self.on_queue_declared)

    def on_queue_declared(self, frame):
        logger.info("Producer declared queue '%s'", self.queue)
        if self.auto_declare and self.exchange:
            self.bind_queue()
        else:
            self.on_ready_to_publish()

    def bind_queue(self):
        self._channel.queue_bind(exchange=self.exchange, queue=self.queue,
                                 routing_key=self.routing_key or self.queue,
                                 callback=self.on_queue_bound)

    def on_queue_bound(self, frame):
        logger.info("Producer bound queue '%s' to exchange '%s'",
                    self.queue, self.exchange)
        self.on_ready_to_publish()

    def on_ready_to_publish(self):
        logger.info(("Producer ready to publish to exchange '%s' "
                     "with routing key '%s'"),
                    self.exchange, self.routing_key)
        self._callbacks.process(0, "_on_ready_to_publish", self)

    def publish(self, message, exchange=None, routing_key=None,
                mandatory=False, immediate=False,
                content_type=None, content_encoding=None,
                headers=None, delivery_mode=None, priority=None,
                correlation_id=None, reply_to=None, expiration=None,
                message_id=None, timestamp=None, type=None, user_id=None,
                app_id=None, cluster_id=None, serializer=None):

        exchange = exchange or self.exchange
        routing_key = routing_key or self.routing_key
        reply_to = reply_to or self.reply_to
        serializer = serializer or self.serializer

        if serializer and not content_type:
            util = getUtility(ISerializer, name=serializer)
            content_type = util.content_type
            message = util.serialize(message)
        elif not content_type:
            content_type = 'text/plain'

        if delivery_mode is None:
            if not self.durable:
                delivery_mode = 1  # message is transient
            else:
                delivery_mode = 2  # message is persistent

        properties = BasicProperties(
            content_type=content_type, content_encoding=content_encoding,
            headers=headers, delivery_mode=delivery_mode, priority=priority,
            correlation_id=str(correlation_id), reply_to=reply_to,
            expiration=expiration, message_id=message_id, timestamp=timestamp,
            type=type, user_id=user_id, app_id=app_id, cluster_id=cluster_id)

        msg = {
            'exchange': exchange,
            'routing_key': routing_key,
            'body': message,
            'properties': properties,
        }

        if self.registered():
            self._pending_messages.insert(0, msg)
        elif self._basic_publish(**msg) and self._connection.tx_select:
            self._tx_commit()  # minimal support for transactional channel

    def _basic_publish(self, **kwargs):
        retry_constructor = lambda func, kwargs: lambda: func(**kwargs)

        if getattr(self._connection, "is_open", False)\
            and getattr(self, '_channel', None):
            self._channel.basic_publish(**kwargs)
            return True

        elif self.durable:
            logger.warning(('No connection. Durable message will be left to '
                            'wait for the new connection: %s'), kwargs)
            retry_callback = retry_constructor(self._basic_publish, kwargs)
            self._callbacks.add(0, '_on_ready_to_publish', retry_callback)
            return False

    def _tx_commit(self):
        if getattr(self._connection, "is_open", False)\
            and getattr(self, '_channel', None):
            self._channel.tx_commit()
        else:
            logger.warning('No connection. Tx.Commit could not be sent.')

    def _begin(self):
        self._pending_messages = []

    def _abort(self):
        self._pending_messages = None

    def _finish(self):
        while self._pending_messages:
            self._basic_publish(**self._pending_messages.pop())
        if getattr(self._connection, "tx_select", False):
            self._tx_commit()  # minimal support for transactional channel

    # Define thread-safe VTM._v_registered:

    def _get_v_registered(self):
        if hasattr(threadlocal, "collective_zamqp_v_registered"):
            return threadlocal.collective_zamqp_v_registered.get(str(self))
        else:
            return 0

    def _set_v_registered(self, value):
        if not hasattr(threadlocal, "collective_zamqp_v_registered"):
            threadlocal.collective_zamqp_v_registered = {}
        threadlocal.collective_zamqp_v_registered[str(self)] = value

    _v_registered = property(_get_v_registered, _set_v_registered)

    # Define thread-safe VMT._v_finalize:

    def _get_v_finalize(self):
        if hasattr(threadlocal, "collective_zamqp_v_finalize"):
            return threadlocal.collective_zamqp_v_finalize.get(str(self))
        else:
            return 0

    def _set_v_finalize(self, value):
        if not hasattr(threadlocal, "collective_zamqp_v_finalize"):
            threadlocal.collective_zamqp_v_finalize = {}
        threadlocal.collective_zamqp_v_finalize[str(self)] = value

    _v_finalize = property(_get_v_finalize, _set_v_finalize)

    # Define thread-safe self._pending_messages:

    def _get_pending_messages(self):
        if hasattr(threadlocal, "collective_zamqp_pending_messages"):
            return threadlocal.collective_zamqp_pending_messages.get(str(self))
        else:
            return None

    def _set_pending_messages(self, value):
        if not hasattr(threadlocal, "collective_zamqp_pending_messages"):
            threadlocal.collective_zamqp_pending_messages = {}
        threadlocal.collective_zamqp_pending_messages[str(self)] = value

    _pending_messages = property(_get_pending_messages, _set_pending_messages)