def __init__(self, connection, channel_number, transport=None):
# We need to do this before the channel is invoked and send_method is
# called
CallbackManager.instance().add(channel_number,
spec.Channel.OpenOk,
transport._on_rpc_complete)
Channel.__init__(self, connection, channel_number, None, transport)
self.basic_get_ = Channel.basic_get
def __init__(self, parameters=None, on_open_callback=None, reconnection_strategy=None):
"""
Connection initialization expects a ConnectionParameters object and
a callback function to notify when we have successfully connected
to the AMQP Broker.
A reconnection_strategy of None will use the NullReconnectionStrategy
"""
self._buffer = ""
# Define our callback dictionary
self.callbacks = CallbackManager.instance()
# On connection callback
if on_open_callback:
self.add_on_open_callback(on_open_callback)
# Set our configuration options
self.parameters = parameters or ConnectionParameters()
# If we did not pass in a reconnection_strategy, setup the default
self.reconnection = reconnection_strategy or NullReconnectionStrategy()
# Add our callback for if we close by being disconnected
self.add_on_close_callback(self.reconnection.on_connection_closed)
# Set all of our default connection state values
self._init_connection_state()
# Connect to the AMQP Broker
self._connect()
def __init__(self,
parameters=None,
on_open_callback=None,
reconnection_strategy=None):
"""
Connection initialization expects a ConnectionParameters object and
a callback function to notify when we have successfully connected
to the AMQP Broker.
A reconnection_strategy of None will use the NullReconnectionStrategy
"""
self._buffer = ''
# Define our callback dictionary
self.callbacks = CallbackManager()
# On connection callback
if on_open_callback:
self.add_on_open_callback(on_open_callback)
# Set our configuration options
self.parameters = parameters or ConnectionParameters()
# If we did not pass in a reconnection_strategy, setup the default
self.reconnection = reconnection_strategy or NullReconnectionStrategy()
# Validate that we don't have erase_credentials enabled with a non
# Null reconnection strategy
if self.parameters.credentials.erase_on_connect and \
not isinstance(self.reconnection, NullReconnectionStrategy):
# Warn the developer
warn(
"%s was initialized to erase credentials but you have \
specified a %s. Reconnections will fail.",
self.parameters.credentials.__class__.__name__,
self.reconnection.__class__.__name__)
# Add our callback for if we close by being disconnected
if not isinstance(self.reconnection, NullReconnectionStrategy):
self.add_on_close_callback(self.reconnection.on_connection_closed)
# Set all of our default connection state values
self._init_connection_state()
# Connect to the AMQP Broker
self._connect()
def __init__(self, parameters=None,
on_open_callback=None,
reconnection_strategy=None):
"""
Connection initialization expects a ConnectionParameters object and
a callback function to notify when we have successfully connected
to the AMQP Broker.
A reconnection_strategy of None will use the NullReconnectionStrategy
"""
self._buffer = ''
# Define our callback dictionary
self.callbacks = CallbackManager()
# On connection callback
if on_open_callback:
self.add_on_open_callback(on_open_callback)
# Set our configuration options
self.parameters = parameters or ConnectionParameters()
# If we did not pass in a reconnection_strategy, setup the default
self.reconnection = reconnection_strategy or NullReconnectionStrategy()
# Validate that we don't have erase_credentials enabled with a non
# Null reconnection strategy
if self.parameters.credentials.erase_on_connect and \
not isinstance(self.reconnection, NullReconnectionStrategy):
# Warn the developer
warn("%s was initialized to erase credentials but you have \
specified a %s. Reconnections will fail.",
self.parameters.credentials.__class__.__name__,
self.reconnection.__class__.__name__)
# Add our callback for if we close by being disconnected
if not isinstance(self.reconnection, NullReconnectionStrategy) and \
hasattr(self.reconnection, 'on_connection_closed'):
self.add_on_close_callback(self.reconnection.on_connection_closed)
# Set all of our default connection state values
self._init_connection_state()
# Connect to the AMQP Broker
self._connect()
def __init__(self,
connection_id=None,
hostname=None,
port=None,
virtual_host=None,
username=None,
password=None,
heartbeat=None,
prefetch_count=None,
tx_select=None):
# Allow class variables to provide defaults for:
# connection_id
if self.connection_id is None and connection_id is None:
connection_id =\
getattr(self, 'grokcore.component.directive.name', 'default')
if connection_id is not None:
self.connection_id = connection_id
# hostname
if hostname is not None:
self.hostname = hostname
assert self.hostname is not None,\
u"Connection configuration is missing hostname."
# port
if port is not None:
self.port = port
assert self.port is not None,\
u"Connection configuration is missing port."
# virtual_host
if virtual_host is not None:
self.virtual_host = virtual_host
assert self.virtual_host is not None,\
u"Connection configuration is missing virtual_host."
# username
if username is not None:
self.username = username
assert self.username is not None,\
u"Connection configuration is missing username."
# password
if password is not None:
self.password = password
assert self.password is not None,\
u"Connection configuration is missing password."
# heartbeat
if heartbeat is not None:
self.heartbeat = heartbeat
# prefetch_count
if prefetch_count is not None:
self.prefetch_count = prefetch_count
# tx_select
if tx_select is not None:
self.tx_select = tx_select
self._callbacks = CallbackManager() # callbacks are NOT thread-safe
self._reconnection_time = time.time()
self._reconnection_delay = 1.0
# BBB for affinitic.zamqp
if getattr(self, 'userid', None):
from zope.deprecation import deprecated
self.username = self.userid
self.userid =\
deprecated(self.userid,
('Connection.userid is no more. '
'Please, use Connection.username instead.'))
logger.default(
u"AMQP Broker connection '%s' created. "
u"hostname: '%s', "
u"port: '%s', "
u"virtual_host: '%s', "
u"username: '******', "
u"heartbeat: '%s', "
u"prefetch_count: '%s', "
u"tx_select: '%s'", self.connection_id, self.hostname, self.port,
self.virtual_host, self.username, self.heartbeat,
self.prefetch_count, self.tx_select)
class BrokerConnection(grok.GlobalUtility):
"""Connection utility base class"""
grok.implements(IBrokerConnection)
grok.baseclass()
connection_id = None
hostname = 'localhost'
port = 5672
virtual_host = '/'
username = '******'
password = '******'
heartbeat = 0
prefetch_count = 0
tx_select = False # Be aware, that rollback for transactional channel
# is safe to use (and tx_select useful) only on
# dedicated single-threaded AMQP-consuming ZEO-clients.
def __init__(self,
connection_id=None,
hostname=None,
port=None,
virtual_host=None,
username=None,
password=None,
heartbeat=None,
prefetch_count=None,
tx_select=None):
# Allow class variables to provide defaults for:
# connection_id
if self.connection_id is None and connection_id is None:
connection_id =\
getattr(self, 'grokcore.component.directive.name', 'default')
if connection_id is not None:
self.connection_id = connection_id
# hostname
if hostname is not None:
self.hostname = hostname
assert self.hostname is not None,\
u"Connection configuration is missing hostname."
# port
if port is not None:
self.port = port
assert self.port is not None,\
u"Connection configuration is missing port."
# virtual_host
if virtual_host is not None:
self.virtual_host = virtual_host
assert self.virtual_host is not None,\
u"Connection configuration is missing virtual_host."
# username
if username is not None:
self.username = username
assert self.username is not None,\
u"Connection configuration is missing username."
# password
if password is not None:
self.password = password
assert self.password is not None,\
u"Connection configuration is missing password."
# heartbeat
if heartbeat is not None:
self.heartbeat = heartbeat
# prefetch_count
if prefetch_count is not None:
self.prefetch_count = prefetch_count
# tx_select
if tx_select is not None:
self.tx_select = tx_select
self._callbacks = CallbackManager() # callbacks are NOT thread-safe
self._reconnection_time = time.time()
self._reconnection_delay = 1.0
# BBB for affinitic.zamqp
if getattr(self, 'userid', None):
from zope.deprecation import deprecated
self.username = self.userid
self.userid =\
deprecated(self.userid,
('Connection.userid is no more. '
'Please, use Connection.username instead.'))
logger.default(
u"AMQP Broker connection '%s' created. "
u"hostname: '%s', "
u"port: '%s', "
u"virtual_host: '%s', "
u"username: '******', "
u"heartbeat: '%s', "
u"prefetch_count: '%s', "
u"tx_select: '%s'", self.connection_id, self.hostname, self.port,
self.virtual_host, self.username, self.heartbeat,
self.prefetch_count, self.tx_select)
def connect(self):
logger.default(u"Connection '%s' connecting", self.connection_id)
credentials = PlainCredentials(self.username,
self.password,
erase_on_connect=False)
parameters = ConnectionParameters(self.hostname,
self.port,
self.virtual_host,
credentials=credentials,
heartbeat=self.heartbeat and True
or False)
# AMQP-heartbeat timeout must be set manually due to bug in pika 0.9.5:
if parameters.heartbeat:
parameters.heartbeat = int(self.heartbeat)
self._connection = AsyncoreConnection(parameters=parameters,
on_open_callback=self.on_connect)
self._reconnection_timeout = None
self._connection.add_on_close_callback(self.reconnect)
def reconnect(self, conn=None):
if time.time() > self._reconnection_time + 60:
self._reconnection_delay = 1.0
if not getattr(self, '_reconnection_timeout', None):
logger.info(u"Trying to reconnect connection '%s' in %s seconds",
self.connection_id, self._reconnection_delay)
self._reconnection_timeout =\
AsyncoreScheduling(self.connect, self._reconnection_delay)
self._reconnection_delay *= (random.random() * 0.5) + 1.0
self._reconnection_delay = min(self._reconnection_delay, 60.0)
@property
def is_open(self):
return getattr(self._connection, 'is_open', False)
def add_on_channel_open_callback(self, callback):
self._callbacks.add(0, '_on_channel_open', callback, False)
def on_connect(self, connection):
logger.default(u"Connection '%s' connected", self.connection_id)
self._connection = connection
self._connection.channel(self.on_channel_open)
self._reconnection_time = time.time()
def on_channel_open(self, channel):
logger.default(u"Channel for connection '%s' opened",
self.connection_id)
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_closed)
if self.prefetch_count:
from pika import spec
self._channel.transport.rpc(
spec.Basic.Qos(0, self.prefetch_count, False), self.on_qos_ok,
[spec.Basic.QosOk])
else:
self.on_qos_ok(self._channel)
def on_qos_ok(self, channel):
if self.tx_select:
channel.tx_select(self.on_channel_tx_select)
else:
self._callbacks.process(0, '_on_channel_open', self, self._channel)
def on_channel_closed(self, code, text):
logger.warning(u"Channel closed with reason '%s %s'", code, text)
# With ZAMQP channel should only be closed when
# 1) connection is closed due ot any reason
# 2) permission or binding error from producer or consumer
self._connection.close(code, text) # safe to call also during closing
# Enforce connection close because _connection.close() does not
# close socket when connection has been closed by RabbitMQ or network
self._connection._adapter_disconnect()
def on_channel_tx_select(self, frame):
self._callbacks.process(0, '_on_channel_open', self, self._channel)
class Connection(object):
"""
Pika Connection Class.
This is the core class that implements communication with RabbitMQ. This
class should not be invoked directly but rather through the use of an
adapter such as SelectConnection or BlockingConnection.
"""
def __init__(self, parameters=None,
on_open_callback=None,
reconnection_strategy=None):
"""
Connection initialization expects a ConnectionParameters object and
a callback function to notify when we have successfully connected
to the AMQP Broker.
A reconnection_strategy of None will use the NullReconnectionStrategy.
"""
# Define our callback dictionary
self.callbacks = CallbackManager()
# On connection callback
if on_open_callback:
self.add_on_open_callback(on_open_callback)
# Set our configuration options
self.parameters = parameters or ConnectionParameters()
# If we did not pass in a reconnection_strategy, setup the default
self.reconnection = reconnection_strategy or NullReconnectionStrategy()
# Validate that we don't have erase_credentials enabled with a non
# Null reconnection strategy
if self.parameters.credentials.erase_on_connect and \
not isinstance(self.reconnection, NullReconnectionStrategy):
# Warn the developer
warn(("%s was initialized to erase credentials but you have "
"specified a %s. Reconnections will fail."),
self.parameters.credentials.__class__.__name__,
self.reconnection.__class__.__name__)
# Add our callback for if we close by being disconnected
self.add_on_open_callback(self.reconnection.on_connection_open)
self.add_on_close_callback(self.reconnection.on_connection_closed)
# Set all of our default connection state values
self._init_connection_state()
# Connect to the AMQP Broker
self._connect()
def _init_connection_state(self):
"""
Initialize or reset all of our internal state variables for a given
connection. If we disconnect and reconnect, all of our state needs to
be wiped.
"""
# Outbound buffer for buffering writes until we're able to send them
self.outbound_buffer = simplebuffer.SimpleBuffer()
# Inbound buffer for decoding frames
self._frame_buffer = ''
# Connection state, server properties and channels all change on
# each connection
self.server_properties = None
self._channels = dict()
# Data used for Heartbeat checking and back-pressure detection
self.bytes_sent = 0
self.bytes_received = 0
self.frames_sent = 0
self.frames_received = 0
self.heartbeat = None
# Default back-pressure multiplier value
self._backpressure = 10
# Connection state
self.connection_state = CONNECTION_CLOSED
# When closing, hold reason why
self.closing = 0, None
# Our starting point once connected, first frame received
self.callbacks.add(0, spec.Connection.Start, self._on_connection_start)
def _adapter_connect(self, host, port):
"""
Subclasses should override to set up the outbound socket connection.
"""
raise NotImplementedError('%s needs to implement this function' %\
self.__class__.__name__)
def _adapter_disconnect(self):
"""
Subclasses should override this to cause the underlying
transport (socket) to close.
"""
raise NotImplementedError('%s needs to implement this function' %\
self.__class__.__name__)
def _connect(self):
"""
Call the Adapter's connect method after letting the.
ReconnectionStrategy know that we're going to do so.
"""
# Let our RS know what we're up to
self.reconnection.on_connect_attempt(self)
# Set our connection state
self.connection_state = CONNECTION_INIT
# Try and connect
self._adapter_connect()
def force_reconnect(self):
# We're not closing and we're not open, so reconnect
self._on_connection_closed(None)
self._connect()
def _reconnect(self, conn=None):
"""
Called by the Reconnection Strategy classes or Adapters to disconnect
and reconnect to the broker.
"""
# We're already closing but it may not be from reconnect, so first
# Add a callback that won't be duplicated
if self.connection_state == CONNECTION_CLOSING:
self.add_on_close_callback(self._reconnect)
return
# If we're open, we want to close normally if we can, then actually
# reconnect via callback that can't be added more than once
if self.connection_state == CONNECTION_OPEN:
self.add_on_close_callback(self._reconnect)
self._ensure_closed()
return
# We're not closing and we're not open, so reconnect
self._init_connection_state()
self._connect()
def _on_connected(self):
"""
This is called by our connection Adapter to let us know that we've
connected and we can notify our connection strategy.
"""
# Set our connection state
self.connection_state = CONNECTION_PROTOCOL
# Start the communication with the RabbitMQ Broker
self._send_frame(pika.frame.ProtocolHeader())
# Let our reconnection_strategy know we're connected
self.reconnection.on_transport_connected(self)
def _on_connection_open(self, frame):
"""
This is called once we have tuned the connection with the server and
called the Connection.Open on the server and it has replied with
Connection.Ok.
"""
self.known_hosts = frame.method.known_hosts
# Add a callback handler for the Broker telling us to disconnect
self.callbacks.add(0, spec.Connection.Close, self._on_remote_close)
# We're now connected at the AMQP level
self.connection_state = CONNECTION_OPEN
# Call our initial callback that we're open
self.callbacks.process(0, '_on_connection_open', self, self)
def _on_connection_start(self, frame):
"""
This is called as a callback once we have received a Connection.Start
from the server.
"""
# We're now connected to the broker
self.connection_state = CONNECTION_START
# We didn't expect a FrameProtocolHeader, did we get one?
if isinstance(frame, pika.frame.ProtocolHeader):
raise ProtocolVersionMismatch(pika.frame.ProtocolHeader, frame)
# Make sure that the major and minor version matches our spec version
if (frame.method.version_major,
frame.method.version_minor) != spec.PROTOCOL_VERSION[0:2]:
raise ProtocolVersionMismatch(pika.frame.ProtocolHeader(), frame)
# Get our server properties and split out capabilities
self.server_properties = frame.method.server_properties
self.server_capabilities = self.server_properties.get('capabilities',
dict())
if hasattr(self.server_properties, 'capabilities'):
del self.server_properties['capabilities']
# Build our StartOk authentication response from the credentials obj
authentication_type, response = \
self.parameters.credentials.response_for(frame.method)
# Server asked for credentials for a method we don't support so raise
# an exception to let the implementing app know
if not authentication_type:
raise AuthenticationError("No %s support for the credentials" %\
self.parameters.credentials.TYPE)
# Erase the credentials if the credentials object wants to
self.parameters.credentials.erase_credentials()
# Add our callback for our Connection Tune event
self.callbacks.add(0, spec.Connection.Tune, self._on_connection_tune)
# Specify our client properties
properties = {'product': PRODUCT,
'platform': 'Python %s' % python_version(),
'capabilities': {'basic.nack': True,
'consumer_cancel_notify': True,
'publisher_confirms': True},
'information': 'See http://pika.github.com',
'version': __version__}
# Send our Connection.StartOk
method = spec.Connection.StartOk(client_properties=properties,
mechanism=authentication_type,
response=response)
self._send_method(0, method)
def _combine(self, a, b):
"""
Pass in two values, if a is 0, return b otherwise if b is 0, return a.
If neither case matches return the smallest value.
"""
return min(a, b) or (a or b)
def _on_connection_tune(self, frame):
"""
Once the Broker sends back a Connection.Tune, we will set our tuning
variables that have been returned to us and kick off the Heartbeat
monitor if required, send our TuneOk and then the Connection. Open rpc
call on channel 0.
"""
# Set our connection state
self.connection_state = CONNECTION_TUNE
# Get our max channels, frames and heartbeat interval
cmax = self._combine(self.parameters.channel_max,
frame.method.channel_max)
fmax = self._combine(self.parameters.frame_max,
frame.method.frame_max)
hint = self._combine(self.parameters.heartbeat,
frame.method.heartbeat)
# If we have a heartbeat interval, create a heartbeat checker
if hint:
self.heartbeat = HeartbeatChecker(self, hint)
# Update our connection state with our tuned values
self.parameters.channel_max = cmax
self.parameters.frame_max = fmax
# Send the TuneOk response with what we've agreed upon
self._send_method(0, spec.Connection.TuneOk(channel_max=cmax,
frame_max=fmax,
heartbeat=hint))
# Send the Connection.Open RPC call for the vhost
cmd = spec.Connection.Open(virtual_host=self.parameters.virtual_host,
insist=True)
self._rpc(0, cmd, self._on_connection_open, [spec.Connection.OpenOk])
def close(self, code=200, text='Normal shutdown'):
"""
Disconnect from RabbitMQ. If there are any open channels, it will
attempt to close them prior to fully disconnecting. Channels which
have active consumers will attempt to send a Basic.Cancel to RabbitMQ
to cleanly stop the delivery of messages prior to closing the channel.
"""
if self.is_closing or self.is_closed:
warn("%s.Close invoked while closing or closed" % \
self.__class__.__name__)
return
# Set our connection state
self.connection_state = CONNECTION_CLOSING
# Carry our code and text around with us
pika.log.info("Closing connection: %i - %s", code, text)
self.closing = code, text
# Disable reconnection strategy on clean shutdown
if code == 200:
self.reconnection.set_active(False)
if self._channels:
# If we're not already closed
for channel_number in self._channels.keys():
self._channels[channel_number].close(code, text)
else:
# If we don't have any channels, close
self._on_close_ready()
def _on_close_ready(self):
"""
On a clean shutdown we'll call this once all of our channels are closed
Let the Broker know we want to close.
"""
if self.is_closed:
warn("%s.on_close_ready invoked while closed" %\
self.__class__.__name__)
return
self._rpc(0, spec.Connection.Close(self.closing[0],
self.closing[1], 0, 0),
self._on_connection_closed,
[spec.Connection.CloseOk])
def _on_connection_closed(self, frame, from_adapter=False):
"""
Let both our RS and Event object know we closed.
"""
# Set that we're actually closed
self.connection_state = CONNECTION_CLOSED
# Call any callbacks registered for this
self.callbacks.process(0, '_on_connection_closed', self, self)
pika.log.info("Disconnected from RabbitMQ at %s:%i",
self.parameters.host, self.parameters.port)
# Disconnect our transport if it didn't call on_disconnected
if not from_adapter:
self._adapter_disconnect()
# Cleanup lingering callbacks
if self._channels:
# Cleanup channel state
for channel_number in self._channels.keys():
self._channels[channel_number].cleanup()
# Cleanup connection state
self.callbacks.remove(0, spec.Connection.Close)
self.callbacks.remove(0, spec.Connection.Start)
self.callbacks.remove(0, spec.Connection.Open)
def _on_remote_close(self, frame):
"""
We've received a remote close from the server.
"""
self.close(frame.method.reply_code, frame.method.reply_text)
def _ensure_closed(self):
"""
If we're not already closed, make sure we're closed.
"""
# We carry the connection state and so we want to close if we know
if self.is_open:
self.close()
@property
def is_closed(self):
"""
Returns a boolean reporting the current connection state.
"""
return self.connection_state == CONNECTION_CLOSED
@property
def is_closing(self):
"""
Returns a boolean reporting the current connection state.
"""
return self.connection_state == CONNECTION_CLOSING
@property
def is_open(self):
"""
Returns a boolean reporting the current connection state.
"""
return self.connection_state == CONNECTION_OPEN
def add_on_close_callback(self, callback):
"""
Add a callback notification when the connection has closed.
"""
self.callbacks.add(0, '_on_connection_closed', callback, False)
def add_on_open_callback(self, callback):
"""
Add a callback notification when the connection has opened.
"""
self.callbacks.add(0, '_on_connection_open', callback, False)
def add_backpressure_callback(self, callback):
"""
Add a callback notification when we think backpressue is being applied
due to the size of the output buffer being exceeded.
"""
self.callbacks.add(0, 'backpressure', callback, False)
def set_backpressure_multiplier(self, value=10):
"""
Alter the backpressure multiplier value. We set this to 10 by default.
This value is used to raise warnings and trigger the backpressure
callback.
"""
self._backpressure = value
def add_timeout(self, deadline, callback):
"""
Adapters should override to call the callback after the
specified number of seconds have elapsed, using a timer, or a
thread, or similar.
"""
raise NotImplementedError('%s needs to implement this function ' %\
self.__class__.__name__)
def remove_timeout(self, callback):
"""
Adapters should override to call the callback after the
specified number of seconds have elapsed, using a timer, or a
thread, or similar.
"""
raise NotImplementedError('%s needs to implement this function' %\
self.__class__.__name__)
# Channel related functionality
def channel(self, on_open_callback, channel_number=None):
"""
Create a new channel with the next available channel number or pass in
a channel number to use. Must be non-zero if you would like to specify
but it is recommended that you let Pika manage the channel numbers.
"""
# If the user didn't specify a channel_number get the next avail
if not channel_number:
channel_number = self._next_channel_number()
# Open the channel passing the users callback
self._channels[channel_number] = channel.Channel(self,
channel_number,
on_open_callback)
# Add the callback for our Channel.Close event in case the Broker
# wants to close us for some reason
self.callbacks.add(channel_number,
spec.Channel.Close,
self._on_channel_close)
# Add the channel spec.Channel.CloseOk callback for _on_channel_close
self.callbacks.add(channel_number,
spec.Channel.CloseOk,
self._on_channel_close)
# Open the channel
self._channels[channel_number].open()
def _next_channel_number(self):
"""
Return the next available channel number or raise on exception.
"""
# Our limit is the the Codec's Channel Max or MAX_CHANNELS if it's None
limit = self.parameters.channel_max or channel.MAX_CHANNELS
# We've used all of our channels
if len(self._channels) == limit:
raise NoFreeChannels()
# Return channel # 1 if we don't have any channels
if not self._channels:
return 1
# Else return our max channel # + 1
else:
return max(self._channels.keys()) + 1
def _on_channel_close(self, frame):
"""
RPC Response from when a channel closes itself, remove from our stack.
"""
channel_number = frame.channel_number
# If we have this channel number in our channels:
if channel_number in self._channels:
# If our server called Channel.Close on us remotely
if isinstance(frame.method, spec.Channel.Close):
# Call the channel.close() function letting it know it came
# From the server.
self._channels[channel_number].close(frame.method.reply_code,
frame.method.reply_text,
True) # Forced close
# Send a Channel.CloseOk frame
self._rpc(channel_number, spec.Channel.CloseOk())
# Remove the CloseOk callback
self.callbacks.remove(channel_number,
spec.Channel.CloseOk)
# Remove the channel from our dict
del(self._channels[channel_number])
# If we're closing and don't have any channels, go to the next step
if self.is_closing and not self._channels:
self._on_close_ready()
# Data packet and frame handling functions
def _on_data_available(self, data_in):
"""
This is called by our Adapter, passing in the data from the socket.
As long as we have buffer try and map out frame data.
"""
# Append the data
self._frame_buffer += data_in
# Loop while we have a buffer and are getting frames from it
while self._frame_buffer:
# Try and build a frame
consumed_count, frame = pika.frame.decode_frame(self._frame_buffer)
# Remove the frame we just consumed from our data
self._frame_buffer = self._frame_buffer[consumed_count:]
# If we don't have a frame, exit
if not frame:
break
# Increment our bytes received buffer for heartbeat checking
self.bytes_received += consumed_count
# Will receive a frame type of -1 if protocol version mismatch
if frame.frame_type < 0:
continue
# Keep track of how many frames have been read
self.frames_received += 1
# If we have a Method Frame and have callbacks for it
if isinstance(frame, pika.frame.Method) and \
self.callbacks.pending(frame.channel_number, frame.method):
# Process the callbacks for it
self.callbacks.process(frame.channel_number, # Prefix
frame.method, # Key
self, # Caller
frame) # Args
# We don't check for heartbeat frames because we can not count
# atomic frames reliably due to different message behaviors
# such as large content frames being transferred slowly
elif isinstance(frame, pika.frame.Heartbeat):
continue
elif frame.channel_number > 0:
# Call our Channel Handler with the frame
if frame.channel_number in self._channels:
self._channels[\
frame.channel_number].transport.deliver(frame)
else:
pika.log.error("Received %s for non-existing channel %i",
frame.method.NAME, frame.channel_number)
def _rpc(self, channel_number, method,
callback=None, acceptable_replies=None):
"""
Make an RPC call for the given callback, channel number and method.
acceptable_replies lists out what responses we'll process from the
server with the specified callback.
"""
# Validate that acceptable_replies is a list or None
if acceptable_replies and not isinstance(acceptable_replies, list):
raise TypeError("acceptable_replies should be list or None")
# Validate the callback is callable
if callback and not is_callable(callback):
raise TypeError("callback should be None, a function or method.")
# If we were passed a callback, add it to our stack
if callback:
for reply in acceptable_replies:
self.callbacks.add(channel_number, reply, callback)
# Send the rpc call to RabbitMQ
self._send_method(channel_number, method)
def _send_frame(self, frame):
"""
This appends the fully generated frame to send to the broker to the
output buffer which will be then sent via the connection adapter.
"""
marshalled_frame = frame.marshal()
self.bytes_sent += len(marshalled_frame)
self.frames_sent += 1
#pika.frame.log_frame(frame.name, marshalled_frame)
self.outbound_buffer.write(marshalled_frame)
#self._flush_outbound()
self._detect_backpressure()
def _detect_backpressure(self):
"""
Attempt to calculate if TCP backpressure is being applied due to
our outbound buffer being larger than the average frame size over
a window of frames.
"""
avg_frame_size = self.bytes_sent / self.frames_sent
if self.outbound_buffer.size > (avg_frame_size * self._backpressure):
est_frames_behind = self.outbound_buffer.size / avg_frame_size
message = "Pika: Write buffer exceeded warning threshold" + \
" at %i bytes and an estimated %i frames behind"
warn(message % (self.outbound_buffer.size, est_frames_behind))
self.callbacks.process(0, 'backpressure', self)
def _flush_outbound(self):
"""
Adapters should override to flush the contents of
outbound_buffer out along the socket.
"""
raise NotImplementedError('%s needs to implement this function ' %\
self.__class__.__name__)
def _send_method(self, channel_number, method, content=None):
"""
Constructs a RPC method frame and then sends it to the broker.
"""
self._send_frame(pika.frame.Method(channel_number, method))
if isinstance(content, tuple):
props = content[0]
body = content[1]
else:
props = None
body = content
if props:
length = 0
if body:
length = len(body)
self._send_frame(pika.frame.Header(channel_number, length, props))
if body:
max_piece = (self.parameters.frame_max - \
spec.FRAME_HEADER_SIZE - \
spec.FRAME_END_SIZE)
body_buf = simplebuffer.SimpleBuffer(body)
frames_sent = 0
frames = len(body) / spec.FRAME_MAX_SIZE
while body_buf:
piece_len = min(len(body_buf), max_piece)
piece = body_buf.read_and_consume(piece_len)
self._send_frame(pika.frame.Body(channel_number, piece))
frames_sent += 1
@property
def _suggested_buffer_size(self):
"""
Return the suggested buffer size from the connection state/tune or the
default if that is None.
"""
return self.parameters.frame_max or spec.FRAME_MAX_SIZE
@property
def basic_nack(self):
"""
Defines if the active connection has the ability to use basic.nack.
"""
return self.server_capabilities.get('basic.nack', False)
@property
def consumer_cancel_notify(self):
"""
Specifies if our active connection has the ability to use
consumer cancel notification.
"""
return self.server_capabilities.get('consumer_cancel_notify', False)
@property
def exchange_exchange_bindings(self):
"""
Specifies if the active connection can use exchange to exchange
bindings.
"""
return self.server_capabilities.get('exchange_exchange_bindings',
False)
@property
def publisher_confirms(self):
"""
Specifies if the active connection can use publisher confirmations.
"""
return self.server_capabilities.get('publisher_confirms', False)
def __init__(self, connection_id=None, hostname=None, port=None,
virtual_host=None, username=None, password=None,
heartbeat=None, prefetch_count=None, tx_select=None):
# Allow class variables to provide defaults for:
# connection_id
if self.connection_id is None and connection_id is None:
connection_id =\
getattr(self, 'grokcore.component.directive.name', 'default')
if connection_id is not None:
self.connection_id = connection_id
# hostname
if hostname is not None:
self.hostname = hostname
assert self.hostname is not None,\
u"Connection configuration is missing hostname."
# port
if port is not None:
self.port = port
assert self.port is not None,\
u"Connection configuration is missing port."
# virtual_host
if virtual_host is not None:
self.virtual_host = virtual_host
assert self.virtual_host is not None,\
u"Connection configuration is missing virtual_host."
# username
if username is not None:
self.username = username
assert self.username is not None,\
u"Connection configuration is missing username."
# password
if password is not None:
self.password = password
assert self.password is not None,\
u"Connection configuration is missing password."
# heartbeat
if heartbeat is not None:
self.heartbeat = heartbeat
# prefetch_count
if prefetch_count is not None:
self.prefetch_count = prefetch_count
# tx_select
if tx_select is not None:
self.tx_select = tx_select
self._callbacks = CallbackManager() # callbacks are NOT thread-safe
self._reconnection_delay = 1.0
# BBB for affinitic.zamqp
if getattr(self, 'userid', None):
from zope.deprecation import deprecated
self.username = self.userid
self.userid =\
deprecated(self.userid,
('Connection.userid is no more. '
'Please, use Connection.username instead.'))
logger.default(u"AMQP Broker connection '%s' created. "
u"hostname: '%s', "
u"port: '%s', "
u"virtual_host: '%s', "
u"username: '******', "
u"heartbeat: '%s', "
u"prefetch_count: '%s', "
u"tx_select: '%s'",
self.connection_id,
self.hostname,
self.port,
self.virtual_host,
self.username,
self.heartbeat,
self.prefetch_count,
self.tx_select)
class BrokerConnection(grok.GlobalUtility):
"""Connection utility base class"""
grok.implements(IBrokerConnection)
grok.baseclass()
connection_id = None
hostname = 'localhost'
port = 5672
virtual_host = '/'
username = '******'
password = '******'
heartbeat = 0
prefetch_count = 0
tx_select = False # Be aware, that rollback for transactional channel
# is safe to use (and tx_select useful) only on
# dedicated single-threaded AMQP-consuming ZEO-clients.
def __init__(self, connection_id=None, hostname=None, port=None,
virtual_host=None, username=None, password=None,
heartbeat=None, prefetch_count=None, tx_select=None):
# Allow class variables to provide defaults for:
# connection_id
if self.connection_id is None and connection_id is None:
connection_id =\
getattr(self, 'grokcore.component.directive.name', 'default')
if connection_id is not None:
self.connection_id = connection_id
# hostname
if hostname is not None:
self.hostname = hostname
assert self.hostname is not None,\
u"Connection configuration is missing hostname."
# port
if port is not None:
self.port = port
assert self.port is not None,\
u"Connection configuration is missing port."
# virtual_host
if virtual_host is not None:
self.virtual_host = virtual_host
assert self.virtual_host is not None,\
u"Connection configuration is missing virtual_host."
# username
if username is not None:
self.username = username
assert self.username is not None,\
u"Connection configuration is missing username."
# password
if password is not None:
self.password = password
assert self.password is not None,\
u"Connection configuration is missing password."
# heartbeat
if heartbeat is not None:
self.heartbeat = heartbeat
# prefetch_count
if prefetch_count is not None:
self.prefetch_count = prefetch_count
# tx_select
if tx_select is not None:
self.tx_select = tx_select
self._callbacks = CallbackManager() # callbacks are NOT thread-safe
self._reconnection_delay = 1.0
# BBB for affinitic.zamqp
if getattr(self, 'userid', None):
from zope.deprecation import deprecated
self.username = self.userid
self.userid =\
deprecated(self.userid,
('Connection.userid is no more. '
'Please, use Connection.username instead.'))
logger.default(u"AMQP Broker connection '%s' created. "
u"hostname: '%s', "
u"port: '%s', "
u"virtual_host: '%s', "
u"username: '******', "
u"heartbeat: '%s', "
u"prefetch_count: '%s', "
u"tx_select: '%s'",
self.connection_id,
self.hostname,
self.port,
self.virtual_host,
self.username,
self.heartbeat,
self.prefetch_count,
self.tx_select)
def connect(self):
logger.default(u"Connection '%s' connecting",
self.connection_id)
credentials = PlainCredentials(
self.username, self.password, erase_on_connect=False)
parameters = ConnectionParameters(
self.hostname, self.port, self.virtual_host,
credentials=credentials,
heartbeat=self.heartbeat and True or False)
# AMQP-heartbeat timeout must be set manually due to bug in pika 0.9.5:
if parameters.heartbeat:
parameters.heartbeat = int(self.heartbeat)
self._connection = AsyncoreConnection(
parameters=parameters,
on_open_callback=self.on_connect)
self._reconnection_timeout = None
self._connection.add_on_close_callback(self.reconnect)
def reconnect(self, conn=None):
if not getattr(self, '_reconnection_timeout', None):
logger.info(u"Trying to reconnect connection '%s' in %s seconds",
self.connection_id, self._reconnection_delay)
self._reconnection_timeout =\
AsyncoreScheduling(self.connect, self._reconnection_delay)
self._reconnection_delay *= (random.random() * 0.5) + 1.0
self._reconnection_delay = min(self._reconnection_delay, 60.0)
@property
def is_open(self):
return getattr(self._connection, 'is_open', False)
def add_on_channel_open_callback(self, callback):
self._callbacks.add(0, '_on_channel_open', callback, False)
def on_connect(self, connection):
logger.default(u"Connection '%s' connected",
self.connection_id)
self._connection = connection
self._connection.channel(self.on_channel_open)
self._reconnection_delay = 1.0
def on_channel_open(self, channel):
logger.default(u"Channel for connection '%s' opened",
self.connection_id)
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_closed)
if self.prefetch_count:
from pika import spec
self._channel.transport.rpc(
spec.Basic.Qos(0, self.prefetch_count, False),
self.on_qos_ok,
[spec.Basic.QosOk]
)
else:
self.on_qos_ok(self._channel)
def on_qos_ok(self, channel):
if self.tx_select:
channel.tx_select(self.on_channel_tx_select)
else:
self._callbacks.process(0, '_on_channel_open', self, self._channel)
def on_channel_closed(self, code, text):
logger.warning(u"Channel closed with reason '%s %s'",
code, text)
self._connection.close(code, text)
self.reconnect()
def on_channel_tx_select(self, frame):
self._callbacks.process(0, '_on_channel_open', self, self._channel)
class Connection(object):
"""
Pika Connection Class.
This is the core class that implements communication with RabbitMQ. This
class should not be invoked directly but rather through the use of an
adapter such as SelectConnection or BlockingConnection.
"""
def __init__(self,
parameters=None,
on_open_callback=None,
reconnection_strategy=None):
"""
Connection initialization expects a ConnectionParameters object and
a callback function to notify when we have successfully connected
to the AMQP Broker.
A reconnection_strategy of None will use the NullReconnectionStrategy.
"""
# Define our callback dictionary
self.callbacks = CallbackManager()
# On connection callback
if on_open_callback:
self.add_on_open_callback(on_open_callback)
# Set our configuration options
self.parameters = parameters or ConnectionParameters()
# If we did not pass in a reconnection_strategy, setup the default
self.reconnection = reconnection_strategy or NullReconnectionStrategy()
# Validate that we don't have erase_credentials enabled with a non
# Null reconnection strategy
if self.parameters.credentials.erase_on_connect and \
not isinstance(self.reconnection, NullReconnectionStrategy):
# Warn the developer
warn(("%s was initialized to erase credentials but you have "
"specified a %s. Reconnections will fail."),
self.parameters.credentials.__class__.__name__,
self.reconnection.__class__.__name__)
# Add our callback for if we close by being disconnected
self.add_on_open_callback(self.reconnection.on_connection_open)
self.add_on_close_callback(self.reconnection.on_connection_closed)
# Set all of our default connection state values
self._init_connection_state()
# Connect to the AMQP Broker
self._connect()
def _init_connection_state(self):
"""
Initialize or reset all of our internal state variables for a given
connection. If we disconnect and reconnect, all of our state needs to
be wiped.
"""
# Outbound buffer for buffering writes until we're able to send them
self.outbound_buffer = simplebuffer.SimpleBuffer()
# Inbound buffer for decoding frames
self._frame_buffer = ''
# Connection state, server properties and channels all change on
# each connection
self.server_properties = None
self._channels = dict()
# Data used for Heartbeat checking and back-pressure detection
self.bytes_sent = 0
self.bytes_received = 0
self.frames_sent = 0
self.frames_received = 0
self.heartbeat = None
# Default back-pressure multiplier value
self._backpressure = 10
# Connection state
self.connection_state = CONNECTION_CLOSED
# When closing, hold reason why
self.closing = 0, None
# Our starting point once connected, first frame received
self.callbacks.add(0, spec.Connection.Start, self._on_connection_start)
def _adapter_connect(self, host, port):
"""
Subclasses should override to set up the outbound socket connection.
"""
raise NotImplementedError('%s needs to implement this function' %\
self.__class__.__name__)
def _adapter_disconnect(self):
"""
Subclasses should override this to cause the underlying
transport (socket) to close.
"""
raise NotImplementedError('%s needs to implement this function' %\
self.__class__.__name__)
def _connect(self):
"""
Call the Adapter's connect method after letting the.
ReconnectionStrategy know that we're going to do so.
"""
# Let our RS know what we're up to
self.reconnection.on_connect_attempt(self)
# Set our connection state
self.connection_state = CONNECTION_INIT
# Try and connect
self._adapter_connect()
def force_reconnect(self):
# We're not closing and we're not open, so reconnect
self._on_connection_closed(None)
self._connect()
def _reconnect(self, conn=None):
"""
Called by the Reconnection Strategy classes or Adapters to disconnect
and reconnect to the broker.
"""
# We're already closing but it may not be from reconnect, so first
# Add a callback that won't be duplicated
if self.connection_state == CONNECTION_CLOSING:
self.add_on_close_callback(self._reconnect)
return
# If we're open, we want to close normally if we can, then actually
# reconnect via callback that can't be added more than once
if self.connection_state == CONNECTION_OPEN:
self.add_on_close_callback(self._reconnect)
self._ensure_closed()
return
# We're not closing and we're not open, so reconnect
self._init_connection_state()
self._connect()
def _on_connected(self):
"""
This is called by our connection Adapter to let us know that we've
connected and we can notify our connection strategy.
"""
# Set our connection state
self.connection_state = CONNECTION_PROTOCOL
# Start the communication with the RabbitMQ Broker
self._send_frame(pika.frame.ProtocolHeader())
# Let our reconnection_strategy know we're connected
self.reconnection.on_transport_connected(self)
def _on_connection_open(self, frame):
"""
This is called once we have tuned the connection with the server and
called the Connection.Open on the server and it has replied with
Connection.Ok.
"""
self.known_hosts = frame.method.known_hosts
# Add a callback handler for the Broker telling us to disconnect
self.callbacks.add(0, spec.Connection.Close, self._on_remote_close)
# We're now connected at the AMQP level
self.connection_state = CONNECTION_OPEN
# Call our initial callback that we're open
self.callbacks.process(0, '_on_connection_open', self, self)
def _on_connection_start(self, frame):
"""
This is called as a callback once we have received a Connection.Start
from the server.
"""
# We're now connected to the broker
self.connection_state = CONNECTION_START
# We didn't expect a FrameProtocolHeader, did we get one?
if isinstance(frame, pika.frame.ProtocolHeader):
raise ProtocolVersionMismatch(pika.frame.ProtocolHeader, frame)
# Make sure that the major and minor version matches our spec version
if (frame.method.version_major,
frame.method.version_minor) != spec.PROTOCOL_VERSION[0:2]:
raise ProtocolVersionMismatch(pika.frame.ProtocolHeader(), frame)
# Get our server properties and split out capabilities
self.server_properties = frame.method.server_properties
self.server_capabilities = self.server_properties.get(
'capabilities', dict())
if hasattr(self.server_properties, 'capabilities'):
del self.server_properties['capabilities']
# Build our StartOk authentication response from the credentials obj
authentication_type, response = \
self.parameters.credentials.response_for(frame.method)
# Server asked for credentials for a method we don't support so raise
# an exception to let the implementing app know
if not authentication_type:
raise AuthenticationError("No %s support for the credentials" %\
self.parameters.credentials.TYPE)
# Erase the credentials if the credentials object wants to
self.parameters.credentials.erase_credentials()
# Add our callback for our Connection Tune event
self.callbacks.add(0, spec.Connection.Tune, self._on_connection_tune)
# Specify our client properties
properties = {
'product': PRODUCT,
'platform': 'Python %s' % python_version(),
'capabilities': {
'basic.nack': True,
'consumer_cancel_notify': True,
'publisher_confirms': True
},
'information': 'See http://pika.github.com',
'version': __version__
}
# Send our Connection.StartOk
method = spec.Connection.StartOk(client_properties=properties,
mechanism=authentication_type,
response=response)
self._send_method(0, method)
def _combine(self, a, b):
"""
Pass in two values, if a is 0, return b otherwise if b is 0, return a.
If neither case matches return the smallest value.
"""
return min(a, b) or (a or b)
def _on_connection_tune(self, frame):
"""
Once the Broker sends back a Connection.Tune, we will set our tuning
variables that have been returned to us and kick off the Heartbeat
monitor if required, send our TuneOk and then the Connection. Open rpc
call on channel 0.
"""
# Set our connection state
self.connection_state = CONNECTION_TUNE
# Get our max channels, frames and heartbeat interval
cmax = self._combine(self.parameters.channel_max,
frame.method.channel_max)
fmax = self._combine(self.parameters.frame_max, frame.method.frame_max)
hint = self._combine(self.parameters.heartbeat, frame.method.heartbeat)
# If we have a heartbeat interval, create a heartbeat checker
if hint:
self.heartbeat = HeartbeatChecker(self, hint)
# Update our connection state with our tuned values
self.parameters.channel_max = cmax
self.parameters.frame_max = fmax
# Send the TuneOk response with what we've agreed upon
self._send_method(
0,
spec.Connection.TuneOk(channel_max=cmax,
frame_max=fmax,
heartbeat=hint))
# Send the Connection.Open RPC call for the vhost
cmd = spec.Connection.Open(virtual_host=self.parameters.virtual_host,
insist=True)
self._rpc(0, cmd, self._on_connection_open, [spec.Connection.OpenOk])
def close(self, code=200, text='Normal shutdown'):
"""
Disconnect from RabbitMQ. If there are any open channels, it will
attempt to close them prior to fully disconnecting. Channels which
have active consumers will attempt to send a Basic.Cancel to RabbitMQ
to cleanly stop the delivery of messages prior to closing the channel.
"""
if self.is_closing or self.is_closed:
warn("%s.Close invoked while closing or closed" % \
self.__class__.__name__)
return
# Set our connection state
self.connection_state = CONNECTION_CLOSING
# Carry our code and text around with us
pika.log.info("Closing connection: %i - %s", code, text)
self.closing = code, text
# Disable reconnection strategy on clean shutdown
if code == 200:
self.reconnection.set_active(False)
if self._channels:
# If we're not already closed
for channel_number in self._channels.keys():
self._channels[channel_number].close(code, text)
else:
# If we don't have any channels, close
self._on_close_ready()
def _on_close_ready(self):
"""
On a clean shutdown we'll call this once all of our channels are closed
Let the Broker know we want to close.
"""
if self.is_closed:
warn("%s.on_close_ready invoked while closed" %\
self.__class__.__name__)
return
self._rpc(
0, spec.Connection.Close(self.closing[0], self.closing[1], 0, 0),
self._on_connection_closed, [spec.Connection.CloseOk])
def _on_connection_closed(self, frame, from_adapter=False):
"""
Let both our RS and Event object know we closed.
"""
# Set that we're actually closed
self.connection_state = CONNECTION_CLOSED
# Call any callbacks registered for this
self.callbacks.process(0, '_on_connection_closed', self, self)
pika.log.info("Disconnected from RabbitMQ at %s:%i",
self.parameters.host, self.parameters.port)
# Disconnect our transport if it didn't call on_disconnected
if not from_adapter:
self._adapter_disconnect()
# Cleanup lingering callbacks
if self._channels:
# Cleanup channel state
for channel_number in self._channels.keys():
self._channels[channel_number].cleanup()
# Cleanup connection state
self.callbacks.remove(0, spec.Connection.Close)
self.callbacks.remove(0, spec.Connection.Start)
self.callbacks.remove(0, spec.Connection.Open)
def _on_remote_close(self, frame):
"""
We've received a remote close from the server.
"""
self.close(frame.method.reply_code, frame.method.reply_text)
def _ensure_closed(self):
"""
If we're not already closed, make sure we're closed.
"""
# We carry the connection state and so we want to close if we know
if self.is_open:
self.close()
@property
def is_closed(self):
"""
Returns a boolean reporting the current connection state.
"""
return self.connection_state == CONNECTION_CLOSED
@property
def is_closing(self):
"""
Returns a boolean reporting the current connection state.
"""
return self.connection_state == CONNECTION_CLOSING
@property
def is_open(self):
"""
Returns a boolean reporting the current connection state.
"""
return self.connection_state == CONNECTION_OPEN
def add_on_close_callback(self, callback):
"""
Add a callback notification when the connection has closed.
"""
self.callbacks.add(0, '_on_connection_closed', callback, False)
def add_on_open_callback(self, callback):
"""
Add a callback notification when the connection has opened.
"""
self.callbacks.add(0, '_on_connection_open', callback, False)
def add_backpressure_callback(self, callback):
"""
Add a callback notification when we think backpressure is being applied
due to the size of the output buffer being exceeded.
"""
self.callbacks.add(0, 'backpressure', callback, False)
def set_backpressure_multiplier(self, value=10):
"""
Alter the backpressure multiplier value. We set this to 10 by default.
This value is used to raise warnings and trigger the backpressure
callback.
"""
self._backpressure = value
def add_timeout(self, deadline, callback):
"""
Adapters should override to call the callback after the
specified number of seconds have elapsed, using a timer, or a
thread, or similar.
"""
raise NotImplementedError('%s needs to implement this function ' %\
self.__class__.__name__)
def remove_timeout(self, callback):
"""
Adapters should override to call the callback after the
specified number of seconds have elapsed, using a timer, or a
thread, or similar.
"""
raise NotImplementedError('%s needs to implement this function' %\
self.__class__.__name__)
# Channel related functionality
def channel(self, on_open_callback, channel_number=None):
"""
Create a new channel with the next available channel number or pass in
a channel number to use. Must be non-zero if you would like to specify
but it is recommended that you let Pika manage the channel numbers.
"""
# If the user didn't specify a channel_number get the next avail
if not channel_number:
channel_number = self._next_channel_number()
# Open the channel passing the users callback
self._channels[channel_number] = channel.Channel(
self, channel_number, on_open_callback)
# Add the callback for our Channel.Close event in case the Broker
# wants to close us for some reason
self.callbacks.add(channel_number, spec.Channel.Close,
self._on_channel_close)
# Add the channel spec.Channel.CloseOk callback for _on_channel_close
self.callbacks.add(channel_number, spec.Channel.CloseOk,
self._on_channel_close)
# Open the channel
self._channels[channel_number].open()
def _next_channel_number(self):
"""
Return the next available channel number or raise on exception.
"""
# Our limit is the the Codec's Channel Max or MAX_CHANNELS if it's None
limit = self.parameters.channel_max or channel.MAX_CHANNELS
# We've used all of our channels
if len(self._channels) == limit:
raise NoFreeChannels()
# Return channel # 1 if we don't have any channels
if not self._channels:
return 1
# Else return our max channel # + 1
else:
return max(self._channels.keys()) + 1
def _on_channel_close(self, frame):
"""
RPC Response from when a channel closes itself, remove from our stack.
"""
channel_number = frame.channel_number
# If we have this channel number in our channels:
if channel_number in self._channels:
# If our server called Channel.Close on us remotely
if isinstance(frame.method, spec.Channel.Close):
# Call the channel.close() function letting it know it came
# From the server.
self._channels[channel_number].close(frame.method.reply_code,
frame.method.reply_text,
True) # Forced close
# Send a Channel.CloseOk frame
self._rpc(channel_number, spec.Channel.CloseOk())
# Remove the CloseOk callback
self.callbacks.remove(channel_number, spec.Channel.CloseOk)
# Remove the channel from our dict
del (self._channels[channel_number])
# If we're closing and don't have any channels, go to the next step
if self.is_closing and not self._channels:
self._on_close_ready()
# Data packet and frame handling functions
def _on_data_available(self, data_in):
"""
This is called by our Adapter, passing in the data from the socket.
As long as we have buffer try and map out frame data.
"""
# Append the data
self._frame_buffer += data_in
# Loop while we have a buffer and are getting frames from it
while self._frame_buffer:
# Try and build a frame
consumed_count, frame = pika.frame.decode_frame(self._frame_buffer)
# Remove the frame we just consumed from our data
self._frame_buffer = self._frame_buffer[consumed_count:]
# If we don't have a frame, exit
if not frame:
break
# Increment our bytes received buffer for heartbeat checking
self.bytes_received += consumed_count
# Will receive a frame type of -1 if protocol version mismatch
if frame.frame_type < 0:
continue
# Keep track of how many frames have been read
self.frames_received += 1
# If we have a Method Frame and have callbacks for it
if isinstance(frame, pika.frame.Method) and \
self.callbacks.pending(frame.channel_number, frame.method):
# Process the callbacks for it
self.callbacks.process(
frame.channel_number, # Prefix
frame.method, # Key
self, # Caller
frame) # Args
# We don't check for heartbeat frames because we can not count
# atomic frames reliably due to different message behaviors
# such as large content frames being transferred slowly
elif isinstance(frame, pika.frame.Heartbeat):
continue
elif frame.channel_number > 0:
# Call our Channel Handler with the frame
if frame.channel_number in self._channels:
self._channels[\
frame.channel_number].transport.deliver(frame)
else:
pika.log.error("Received %s for non-existing channel %i",
frame.method.NAME, frame.channel_number)
def _rpc(self,
channel_number,
method,
callback=None,
acceptable_replies=None):
"""
Make an RPC call for the given callback, channel number and method.
acceptable_replies lists out what responses we'll process from the
server with the specified callback.
"""
# Validate that acceptable_replies is a list or None
if acceptable_replies and not isinstance(acceptable_replies, list):
raise TypeError("acceptable_replies should be list or None")
# Validate the callback is callable
if callback and not is_callable(callback):
raise TypeError("callback should be None, a function or method.")
# If we were passed a callback, add it to our stack
if callback:
for reply in acceptable_replies:
self.callbacks.add(channel_number, reply, callback)
# Send the rpc call to RabbitMQ
self._send_method(channel_number, method)
def _send_frame(self, frame):
"""
This appends the fully generated frame to send to the broker to the
output buffer which will be then sent via the connection adapter.
"""
marshalled_frame = frame.marshal()
self.bytes_sent += len(marshalled_frame)
self.frames_sent += 1
#pika.frame.log_frame(frame.name, marshalled_frame)
self.outbound_buffer.write(marshalled_frame)
#self._flush_outbound()
self._detect_backpressure()
def _detect_backpressure(self):
"""
Attempt to calculate if TCP backpressure is being applied due to
our outbound buffer being larger than the average frame size over
a window of frames.
"""
avg_frame_size = self.bytes_sent / self.frames_sent
if self.outbound_buffer.size > (avg_frame_size * self._backpressure):
est_frames_behind = self.outbound_buffer.size / avg_frame_size
message = "Pika: Write buffer exceeded warning threshold" + \
" at %i bytes and an estimated %i frames behind"
warn(message % (self.outbound_buffer.size, est_frames_behind))
self.callbacks.process(0, 'backpressure', self)
def _flush_outbound(self):
"""
Adapters should override to flush the contents of
outbound_buffer out along the socket.
"""
raise NotImplementedError('%s needs to implement this function ' %\
self.__class__.__name__)
def _send_method(self, channel_number, method, content=None):
"""
Constructs a RPC method frame and then sends it to the broker.
"""
self._send_frame(pika.frame.Method(channel_number, method))
if isinstance(content, tuple):
props = content[0]
body = content[1]
else:
props = None
body = content
if props:
length = 0
if body:
length = len(body)
self._send_frame(pika.frame.Header(channel_number, length, props))
if body:
max_piece = (self.parameters.frame_max - \
spec.FRAME_HEADER_SIZE - \
spec.FRAME_END_SIZE)
body_buf = simplebuffer.SimpleBuffer(body)
frames_sent = 0
frames = len(body) / spec.FRAME_MAX_SIZE
while body_buf:
piece_len = min(len(body_buf), max_piece)
piece = body_buf.read_and_consume(piece_len)
self._send_frame(pika.frame.Body(channel_number, piece))
frames_sent += 1
@property
def _suggested_buffer_size(self):
"""
Return the suggested buffer size from the connection state/tune or the
default if that is None.
"""
return self.parameters.frame_max or spec.FRAME_MAX_SIZE
@property
def basic_nack(self):
"""
Defines if the active connection has the ability to use basic.nack.
"""
return self.server_capabilities.get('basic.nack', False)
@property
def consumer_cancel_notify(self):
"""
Specifies if our active connection has the ability to use
consumer cancel notification.
"""
return self.server_capabilities.get('consumer_cancel_notify', False)
@property
def exchange_exchange_bindings(self):
"""
Specifies if the active connection can use exchange to exchange
bindings.
"""
return self.server_capabilities.get('exchange_exchange_bindings',
False)
@property
def publisher_confirms(self):
"""
Specifies if the active connection can use publisher confirmations.
"""
return self.server_capabilities.get('publisher_confirms', False)
def __init__(self, connection_id=None, exchange=None, routing_key=None,
durable=None, exchange_type=None, exchange_durable=None,
exchange_auto_delete=None, exchange_auto_declare=None,
queue=None, queue_durable=None, queue_auto_delete=None,
queue_exclusive=None, queue_arguments=None,
queue_auto_declare=None, auto_declare=None, auto_delete=None,
reply_to=None, serializer=None):
self._connection = None
self._queue = None # will default to self.queue
self._lock = threading.Lock()
self._threadlocal = threading.local() # to thread-safe some attributes
# Allow class variables to provide defaults for:
# connection_id
if connection_id is not None:
self.connection_id = connection_id
assert self.connection_id is not None,\
u"Producer configuration is missing connection_id."
# exchange
if exchange is not None:
self.exchange = exchange
assert self.exchange is not None,\
u"Producer configuration is missing exchange."
# durable (and the default for exchange/queue_durable)
if durable is not None:
self.durable = durable
# auto_delete (and the default for exchange/queue_auto_delete)
if auto_delete is not None:
self.auto_delete = auto_delete
elif self.auto_delete is None:
self.auto_delete = not self.durable
# exchange_type
if exchange_type is not None:
self.exchange_type = exchange_type
# exchange_durable
if exchange_durable is not None:
self.exchange_durable = exchange_durable
elif self.exchange_durable is None:
self.exchange_durable = self.durable
# exchange_auto_delete
if exchange_auto_delete is not None:
self.exchange_auto_delete = exchange_auto_delete
elif self.exchange_auto_delete is None:
self.exchange_auto_delete = self.auto_delete
# queue
if queue is not None:
self._queue = self.queue = queue
# queue_durable
if queue_durable is not None:
self.queue_durable = queue_durable
elif self.queue_durable is None:
self.queue_durable = self.durable
# queue_auto_delete
if queue_auto_delete is not None:
self.queue_auto_delete = queue_auto_delete
elif self.queue_auto_delete is None:
self.queue_auto_delete = self.auto_delete
# queue_exclusive
if queue_exclusive is not None:
self.queue_exclusive = queue_exclusive
if self.queue_exclusive is True:
self.queue_durable = False
self.queue_auto_delete = True
# queue_arguments
if queue_arguments is not None:
self.queue_arguments = queue_arguments
# routing_key
if self.routing_key is None and routing_key is None:
routing_key =\
getattr(self, 'grokcore.component.directive.name', None)
if routing_key is not None:
self.routing_key = routing_key
elif self.routing_key is None:
if self.queue is not None:
self.routing_key = self.queue
elif self.exchange_type == 'fanout':
self.routing_key = '*'
assert self.routing_key is not None,\
u"Producer configuration is missing routing_key."
# auto_declare
if auto_declare is not None:
self.auto_declare = auto_declare
if exchange_auto_declare is not None:
self.exchange_auto_declare = exchange_auto_declare
elif self.exchange_auto_declare is None:
self.exchange_auto_declare = self.auto_declare
if queue_auto_declare is not None:
self.queue_auto_declare = queue_auto_declare
elif self.queue_auto_declare is None:
self.queue_auto_declare = self.auto_declare
# reply_to
if reply_to is not None:
self.reply_to = reply_to
# serializer
if serializer is not None:
self.serializer = serializer
# initialize callbacks
self._callbacks = CallbackManager() # callbacks are NOT thread-safe
# subscribe to the connect initialization event
provideHandler(self.on_before_broker_connect,
[IBeforeBrokerConnectEvent])
class Producer(grok.GlobalUtility, VTM):
"""Producer utility base class"""
grok.baseclass()
grok.implements(IProducer)
connection_id = None
exchange = ''
routing_key = None
durable = True
exchange_type = 'direct'
exchange_durable = None
exchange_auto_delete = None
exchange_auto_declare = None
queue = None
queue_durable = None
queue_auto_delete = None
queue_exclusive = False
queue_arguments = {}
queue_auto_declare = None
auto_declare = True
auto_delete = None
reply_to = None
serializer = 'pickle'
def __init__(self, connection_id=None, exchange=None, routing_key=None,
durable=None, exchange_type=None, exchange_durable=None,
exchange_auto_delete=None, exchange_auto_declare=None,
queue=None, queue_durable=None, queue_auto_delete=None,
queue_exclusive=None, queue_arguments=None,
queue_auto_declare=None, auto_declare=None, auto_delete=None,
reply_to=None, serializer=None):
self._connection = None
self._queue = None # will default to self.queue
self._lock = threading.Lock()
self._threadlocal = threading.local() # to thread-safe some attributes
# Allow class variables to provide defaults for:
# connection_id
if connection_id is not None:
self.connection_id = connection_id
assert self.connection_id is not None,\
u"Producer configuration is missing connection_id."
# exchange
if exchange is not None:
self.exchange = exchange
assert self.exchange is not None,\
u"Producer configuration is missing exchange."
# durable (and the default for exchange/queue_durable)
if durable is not None:
self.durable = durable
# auto_delete (and the default for exchange/queue_auto_delete)
if auto_delete is not None:
self.auto_delete = auto_delete
elif self.auto_delete is None:
self.auto_delete = not self.durable
# exchange_type
if exchange_type is not None:
self.exchange_type = exchange_type
# exchange_durable
if exchange_durable is not None:
self.exchange_durable = exchange_durable
elif self.exchange_durable is None:
self.exchange_durable = self.durable
# exchange_auto_delete
if exchange_auto_delete is not None:
self.exchange_auto_delete = exchange_auto_delete
elif self.exchange_auto_delete is None:
self.exchange_auto_delete = self.auto_delete
# queue
if queue is not None:
self._queue = self.queue = queue
# queue_durable
if queue_durable is not None:
self.queue_durable = queue_durable
elif self.queue_durable is None:
self.queue_durable = self.durable
# queue_auto_delete
if queue_auto_delete is not None:
self.queue_auto_delete = queue_auto_delete
elif self.queue_auto_delete is None:
self.queue_auto_delete = self.auto_delete
# queue_exclusive
if queue_exclusive is not None:
self.queue_exclusive = queue_exclusive
if self.queue_exclusive is True:
self.queue_durable = False
self.queue_auto_delete = True
# queue_arguments
if queue_arguments is not None:
self.queue_arguments = queue_arguments
# routing_key
if self.routing_key is None and routing_key is None:
routing_key =\
getattr(self, 'grokcore.component.directive.name', None)
if routing_key is not None:
self.routing_key = routing_key
elif self.routing_key is None:
if self.queue is not None:
self.routing_key = self.queue
elif self.exchange_type == 'fanout':
self.routing_key = '*'
assert self.routing_key is not None,\
u"Producer configuration is missing routing_key."
# auto_declare
if auto_declare is not None:
self.auto_declare = auto_declare
if exchange_auto_declare is not None:
self.exchange_auto_declare = exchange_auto_declare
elif self.exchange_auto_declare is None:
self.exchange_auto_declare = self.auto_declare
if queue_auto_declare is not None:
self.queue_auto_declare = queue_auto_declare
elif self.queue_auto_declare is None:
self.queue_auto_declare = self.auto_declare
# reply_to
if reply_to is not None:
self.reply_to = reply_to
# serializer
if serializer is not None:
self.serializer = serializer
# initialize callbacks
self._callbacks = CallbackManager() # callbacks are NOT thread-safe
# subscribe to the connect initialization event
provideHandler(self.on_before_broker_connect,
[IBeforeBrokerConnectEvent])
def on_before_broker_connect(self, event=None):
self._connection = queryUtility(IBrokerConnection,
name=self.connection_id)
if self._connection:
self._connection.add_on_channel_open_callback(self.on_channel_open)
else:
logger.warning(u"Connection '%s' was not registered. "
u"Producer '%s' cannot be connected.",
self.connection_id, self.routing_key)
def on_channel_open(self, channel):
self._channel = channel
if self.exchange_auto_declare and self.exchange\
and not self.exchange.startswith('amq.'):
self.declare_exchange()
elif self.queue_auto_declare and self.queue is not None\
and not self.queue.startswith('amq.'):
self.declare_queue()
else:
self.on_ready_to_publish()
def declare_exchange(self):
self._channel.exchange_declare(exchange=self.exchange,
type=self.exchange_type,
durable=self.exchange_durable,
auto_delete=self.exchange_auto_delete,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
logger.default(u"Producer declared exchange '%s' on connection '%s'",
self.exchange, self.connection_id)
if self.queue_auto_declare and self.queue is not None\
and not self.queue.startswith('amq.'):
self.declare_queue()
else:
self.on_ready_to_publish()
def declare_queue(self):
self._channel.queue_declare(queue=self.queue,
durable=self.queue_durable,
exclusive=self.queue_exclusive,
auto_delete=self.queue_auto_delete,
arguments=self.queue_arguments,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self._queue = frame.method.queue # get the real queue name
logger.default(u"Producer declared queue '%s' on connection '%s'",
self._queue, self.connection_id)
if self.auto_declare and self.exchange:
self.bind_queue()
else:
self.on_ready_to_publish()
def bind_queue(self):
self._channel.queue_bind(exchange=self.exchange, queue=self._queue,
routing_key=self.routing_key or self._queue,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
logger.default(u"Producer bound queue '%s' to exchange '%s' "
u"on connection '%s'",
self._queue, self.exchange, self.connection_id)
self.on_ready_to_publish()
def on_ready_to_publish(self):
logger.default(u"Producer ready to publish to exchange '%s' "
u"with routing key '%s' on connection '%s'",
self.exchange, self.routing_key, self.connection_id)
self._callbacks.process(0, "_on_ready_to_publish", self)
@property
def is_connected(self):
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
return True
else:
return False
def register(self):
self._register()
def publish(self, message, exchange=None, routing_key=None,
mandatory=False, immediate=False,
content_type=None, content_encoding=None,
headers=None, delivery_mode=None, priority=None,
correlation_id=None, reply_to=None, expiration=None,
message_id=None, timestamp=None, type=None, user_id=None,
app_id=None, cluster_id=None, serializer=None):
exchange = exchange or self.exchange
routing_key = routing_key or self.routing_key
reply_to = reply_to or self.reply_to
serializer = serializer or self.serializer
if correlation_id is not None:
correlation_id = str(correlation_id)
if serializer and not content_type:
util = getUtility(ISerializer, name=serializer)
content_type = util.content_type
message = util.serialize(message)
elif not content_type:
content_type = 'text/plain'
if delivery_mode is None:
if not self.durable:
delivery_mode = 1 # message is transient
else:
delivery_mode = 2 # message is persistent
properties = BasicProperties(
content_type=content_type, content_encoding=content_encoding,
headers=headers, delivery_mode=delivery_mode, priority=priority,
correlation_id=correlation_id, reply_to=reply_to,
expiration=expiration, message_id=message_id, timestamp=timestamp,
type=type, user_id=user_id, app_id=app_id, cluster_id=cluster_id)
msg = {
'exchange': exchange,
'routing_key': routing_key,
'body': message,
'properties': properties,
}
if self.registered():
self._pending_messages.insert(0, msg)
elif self._basic_publish(**msg) and self._connection.tx_select:
self._tx_commit() # minimal support for transactional channel
def __len__(self):
"""Return message count of the target queue"""
# XXX: Producer knows it target queue only, if it's explicitly
# set in its definition. Otherwise self._queue is None
assert self._queue, "Sorry, producer doesn't know its target queue."
with BlockingChannel(self.connection_id) as channel:
frame = channel.queue_declare(queue=self._queue,
durable=self.queue_durable,
exclusive=self.queue_exclusive,
auto_delete=self.queue_auto_delete,
arguments=self.queue_arguments)
return frame.method.message_count
def _basic_publish(self, **kwargs):
retry_constructor = lambda func, kwargs: lambda: func(**kwargs)
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.basic_publish(**kwargs)
return True
elif getattr(kwargs.get("properties"), "delivery_mode", 1) == 2:
logger.warning(u"No connection. Durable message was left into "
u"volatile memory to wait for a new connection "
u"'%s'", kwargs)
retry_callback = retry_constructor(self._basic_publish, kwargs)
with self._lock:
self._callbacks.add(0, '_on_ready_to_publish', retry_callback)
# XXX: ^^^ When connection is down, durable messages should be
# stored in ZODB to prevent losing them, e.g. because of restart.
return False
def _tx_commit(self):
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.tx_commit()
else:
logger.warning(u'No connection. Tx.Commit was not sent')
def _begin(self):
self._pending_messages = []
def _abort(self):
self._pending_messages = None
def _finish(self):
while self._pending_messages:
self._basic_publish(**self._pending_messages.pop())
if getattr(self._connection, "tx_select", False):
self._tx_commit() # minimal support for transactional channel
# Define threadlocal VTM._v_registered:
def _get_v_registered(self):
return getattr(self._threadlocal, "_v_registered", 0)
def _set_v_registered(self, value):
self._threadlocal._v_registered = value
_v_registered = property(_get_v_registered, _set_v_registered)
# Define threadlocal VTM._v_finalize:
def _get_v_finalize(self):
return getattr(self._threadlocal, "_v_finalize", 0)
def _set_v_finalize(self, value):
self._threadlocal._v_finalize = value
_v_finalize = property(_get_v_finalize, _set_v_finalize)
# Define threadlocal self._pending_messages:
def _get_pending_messages(self):
return getattr(self._threadlocal, "_pending_messages", None)
def _set_pending_messages(self, value):
self._threadlocal._pending_messages = value
_pending_messages = property(_get_pending_messages, _set_pending_messages)
class Producer(grok.GlobalUtility, VTM):
"""Producer utility base class"""
grok.baseclass()
grok.implements(IProducer)
connection_id = None
exchange = ''
routing_key = None
durable = True
exchange_type = 'direct'
exchange_durable = None
queue = None
queue_durable = None
queue_exclusive = False
queue_arguments = {}
auto_declare = True
reply_to = None
serializer = 'pickle'
def __init__(self, connection_id=None, exchange=None, routing_key=None,
durable=None, exchange_type=None, exchange_durable=None,
queue=None, queue_durable=None, queue_exclusive = None,
queue_arguments=None, auto_declare=None, reply_to=None,
serializer=None):
self._connection = None
# Allow class variables to provide defaults for:
# connection_id
if connection_id is not None:
self.connection_id = connection_id
assert self.connection_id is not None,\
u"Producer configuration is missing connection_id."
# exchange
if exchange is not None:
self.exchange = exchange
assert self.exchange is not None,\
u"Producer configuration is missing exchange."
# routing_key
if self.routing_key is None and routing_key is None:
routing_key =\
getattr(self, 'grokcore.component.directive.name', None)
if routing_key is not None:
self.routing_key = routing_key
assert self.routing_key is not None,\
u"Producer configuration is missing routing_key."
# durable (and the default for exchange_durable)
if durable is not None:
self.durable = durable
# exchange_type
if exchange_type is not None:
self.exchange_type = exchange_type
# exchange_durable
if exchange_durable is not None:
self.exchange_durable = exchange_durable
elif self.exchange_durable is None:
self.exchange_durable = self.durable
# queue
if queue:
self.queue = queue
# queue_durable
if queue_durable is not None:
self.queue_durable = queue_durable
elif self.queue_durable is None:
self.queue_durable = self.durable
# queue_exclusive
if queue_exclusive is not None:
self.queue_exclusive = queue_exclusive
# queue_arguments
if queue_arguments is not None:
self.queue_arguments = queue_arguments
# auto_declare
if auto_declare is not None:
self.auto_declare = auto_declare
# reply_to
if reply_to is not None:
self.reply_to = reply_to
# serializer
if serializer is not None:
self.serializer = serializer
# initialize callbacks
self._callbacks = CallbackManager() # callbacks are NOT thread-safe
# subscribe to the connect initialization event
provideHandler(self.on_before_broker_connect,
[IBeforeBrokerConnectEvent])
def on_before_broker_connect(self, event=None):
self._connection = queryUtility(IBrokerConnection,
name=self.connection_id)
if self._connection:
self._connection.add_on_channel_open_callback(self.on_channel_open)
else:
logger.warning(("Connection '%s' was not registered. "
"Producer '%s' cannot be connected."),
self.connection_id, self.routing_key)
def on_channel_open(self, channel):
self._channel = channel
if self.auto_declare and self.exchange:
self.declare_exchange()
elif self.auto_declare and self.queue:
self.declare_queue()
else:
self.on_ready_to_publish()
def declare_exchange(self):
self._channel.exchange_declare(exchange=self.exchange,
type=self.exchange_type,
durable=self.exchange_durable,
auto_delete=not self.exchange_durable,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
logger.info("Producer declared exchange '%s'", self.exchange)
if self.auto_declare and self.queue:
self.declare_queue()
else:
self.on_ready_to_publish()
def declare_queue(self):
self._channel.queue_declare(queue=self.queue,
durable=self.queue_durable,
exclusive=self.queue_exclusive,
auto_delete=not self.queue_durable,
arguments=self.queue_arguments,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
logger.info("Producer declared queue '%s'", self.queue)
if self.auto_declare and self.exchange:
self.bind_queue()
else:
self.on_ready_to_publish()
def bind_queue(self):
self._channel.queue_bind(exchange=self.exchange, queue=self.queue,
routing_key=self.routing_key or self.queue,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
logger.info("Producer bound queue '%s' to exchange '%s'",
self.queue, self.exchange)
self.on_ready_to_publish()
def on_ready_to_publish(self):
logger.info(("Producer ready to publish to exchange '%s' "
"with routing key '%s'"),
self.exchange, self.routing_key)
self._callbacks.process(0, "_on_ready_to_publish", self)
def publish(self, message, exchange=None, routing_key=None,
mandatory=False, immediate=False,
content_type=None, content_encoding=None,
headers=None, delivery_mode=None, priority=None,
correlation_id=None, reply_to=None, expiration=None,
message_id=None, timestamp=None, type=None, user_id=None,
app_id=None, cluster_id=None, serializer=None):
exchange = exchange or self.exchange
routing_key = routing_key or self.routing_key
reply_to = reply_to or self.reply_to
serializer = serializer or self.serializer
if serializer and not content_type:
util = getUtility(ISerializer, name=serializer)
content_type = util.content_type
message = util.serialize(message)
elif not content_type:
content_type = 'text/plain'
if delivery_mode is None:
if not self.durable:
delivery_mode = 1 # message is transient
else:
delivery_mode = 2 # message is persistent
properties = BasicProperties(
content_type=content_type, content_encoding=content_encoding,
headers=headers, delivery_mode=delivery_mode, priority=priority,
correlation_id=str(correlation_id), reply_to=reply_to,
expiration=expiration, message_id=message_id, timestamp=timestamp,
type=type, user_id=user_id, app_id=app_id, cluster_id=cluster_id)
msg = {
'exchange': exchange,
'routing_key': routing_key,
'body': message,
'properties': properties,
}
if self.registered():
self._pending_messages.insert(0, msg)
elif self._basic_publish(**msg) and self._connection.tx_select:
self._tx_commit() # minimal support for transactional channel
def _basic_publish(self, **kwargs):
retry_constructor = lambda func, kwargs: lambda: func(**kwargs)
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.basic_publish(**kwargs)
return True
elif self.durable:
logger.warning(('No connection. Durable message will be left to '
'wait for the new connection: %s'), kwargs)
retry_callback = retry_constructor(self._basic_publish, kwargs)
self._callbacks.add(0, '_on_ready_to_publish', retry_callback)
return False
def _tx_commit(self):
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.tx_commit()
else:
logger.warning('No connection. Tx.Commit could not be sent.')
def _begin(self):
self._pending_messages = []
def _abort(self):
self._pending_messages = None
def _finish(self):
while self._pending_messages:
self._basic_publish(**self._pending_messages.pop())
if getattr(self._connection, "tx_select", False):
self._tx_commit() # minimal support for transactional channel
# Define thread-safe VTM._v_registered:
def _get_v_registered(self):
if hasattr(threadlocal, "collective_zamqp_v_registered"):
return threadlocal.collective_zamqp_v_registered.get(str(self))
else:
return 0
def _set_v_registered(self, value):
if not hasattr(threadlocal, "collective_zamqp_v_registered"):
threadlocal.collective_zamqp_v_registered = {}
threadlocal.collective_zamqp_v_registered[str(self)] = value
_v_registered = property(_get_v_registered, _set_v_registered)
# Define thread-safe VMT._v_finalize:
def _get_v_finalize(self):
if hasattr(threadlocal, "collective_zamqp_v_finalize"):
return threadlocal.collective_zamqp_v_finalize.get(str(self))
else:
return 0
def _set_v_finalize(self, value):
if not hasattr(threadlocal, "collective_zamqp_v_finalize"):
threadlocal.collective_zamqp_v_finalize = {}
threadlocal.collective_zamqp_v_finalize[str(self)] = value
_v_finalize = property(_get_v_finalize, _set_v_finalize)
# Define thread-safe self._pending_messages:
def _get_pending_messages(self):
if hasattr(threadlocal, "collective_zamqp_pending_messages"):
return threadlocal.collective_zamqp_pending_messages.get(str(self))
else:
return None
def _set_pending_messages(self, value):
if not hasattr(threadlocal, "collective_zamqp_pending_messages"):
threadlocal.collective_zamqp_pending_messages = {}
threadlocal.collective_zamqp_pending_messages[str(self)] = value
_pending_messages = property(_get_pending_messages, _set_pending_messages)