def _create_connection(self):
"Tries to create a connection, returns True on success"
self._connection = None
self._last_confirmed_message = None
host = self._get_next_host()
self._logger.debug('Trying to connect to {}'.format(host))
try:
self._connection = RabbitConnection(
host=host,
port=self._connection_port,
user=self._connection_user,
password=self._connection_password,
vhost=self._connection_path,
close_cb=self._close_cb,
heartbeat=self._connection_heartbeat,
client_properties={
'connection_name': self._connection_name,
},
)
except socket.error as exc:
self._logger.error('Error connecting to rabbitmq {}'.format(exc))
return False
self._channel = self._connection.channel()
if self._confirm:
self._channel.confirm.select(nowait=False)
self._channel.basic.set_ack_listener(self._ack)
self._logger.debug('Connected to {}'.format(host))
return True
class Client(object):
"""The RPC Client
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
result = self._channel.queue.declare(exclusive = True)
self._callbackQueue = result[0]
self._channel.basic.consume(self._callbackQueue, self.onResponse, no_ack = True)
self._response = None
def onResponse(self, message):
"""On response
"""
correlationID = message.properties.get('correlation_id')
print 'Receive server response [%s] correlationID [%s]' % (message.body, correlationID)
self._response = int(message.body)
def call(self, number):
"""The call method
"""
self._response = None
corrID = str(uuid4())
print 'Call server with request [%s] correlationID [%s]' % (number, corrID)
self._channel.basic.publish(Message(str(number), reply_to = self._callbackQueue, correlation_id = corrID), '', 'test_rpc')
while self._response is None:
self._conn.read_frames()
# Done
return self._response
class Server(object):
"""The Meta Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_meta', auto_delete=True)
self._channel.basic.consume('test_meta', self.callback, no_ack=False)
def run(self):
"""Waiting for response
"""
while True:
self._conn.read_frames()
def callback(self, message):
"""The callee method
"""
body = message.body
deliveryInfo = message.delivery_info
properties = message.properties
# Print the meta
print 'Receive message body [%s] deliveryInfo [%s] properties [%s]' % (
body, deliveryInfo, properties)
# ACK
deliveryTag = deliveryInfo.get('delivery_tag')
self._channel.basic.ack(deliveryTag)
class Inspector(object):
"""The Meta Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_event', auto_delete = True)
self._channel.queue.bind('test_event', exchange = 'amq.rabbitmq.event', routing_key = '#')
self._channel.basic.consume('test_event', self.callback, no_ack = False)
def run(self):
"""Waiting for response
"""
while True:
self._conn.read_frames()
def callback(self, message):
"""The callee method
"""
body = message.body
eventType = message.delivery_info.get('routing_key')
deliveryTag = message.delivery_info.get('delivery_tag')
# Print the meta
print 'Event [%s] message body [%s] headers [%s] deliveryInfo [%s]' % (eventType, body, message.properties, message.delivery_info)
# ACK
self._channel.basic.ack(deliveryTag)
class Server(object):
"""The RPC Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host, port=port, vhost=vhost, user=user, password=password)
self._channel = self._conn.channel()
self._channel.queue.declare("test_rpc", auto_delete=True)
self._channel.basic.consume("test_rpc", self.callee, no_ack=False)
def run(self):
"""Waiting for response
"""
while True:
self._conn.read_frames()
def callee(self, message):
"""The callee method
"""
num = int(message.body)
replyTo = message.properties.get("reply_to")
correlationID = message.properties.get("correlation_id")
deliveryTag = message.delivery_info.get("delivery_tag")
# Print the meta
print "Receive message body [%s] replyTo [%s] correlationID [%s]" % (num, replyTo, correlationID)
# Return add 1
self._channel.basic.publish(Message(str(num + 1), correlation_id=correlationID), "", replyTo)
# ACK
self._channel.basic.ack(deliveryTag)
class Client(object):
"""The RPC Client
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport='gevent',
host=host,
port=port,
vhost=vhost,
user=user,
password=password)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
def loop(self):
"""The loop
"""
while self._conn:
self._conn.read_frames()
gevent.sleep()
def call(self):
"""The call method
"""
self._channel.basic.publish(Message('A test body'), '', 'test_gevent')
class Inspector(object):
"""The Meta Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_dead_channel', auto_delete = True)
self._channel.queue.bind('test_dead_channel', exchange = 'amq.topic', routing_key = 'test.dead_channel')
self._channel.basic.consume('test_dead_channel', self.callback, no_ack = False)
def run(self):
"""Waiting for response
"""
while True:
self._conn.read_frames()
def callback(self, message):
"""The callee method
"""
body = message.body
deliveryInfo = message.delivery_info
# Print body
print 'Receive dead message [%s]' % body
# ACK
deliveryTag = deliveryInfo.get('delivery_tag')
self._channel.basic.ack(deliveryTag)
class Inspector(object):
"""The Meta Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_event', auto_delete=True)
self._channel.queue.bind('test_event',
exchange='amq.rabbitmq.event',
routing_key='#')
self._channel.basic.consume('test_event', self.callback, no_ack=False)
def run(self):
"""Waiting for response
"""
while True:
self._conn.read_frames()
def callback(self, message):
"""The callee method
"""
body = message.body
eventType = message.delivery_info.get('routing_key')
deliveryTag = message.delivery_info.get('delivery_tag')
# Print the meta
print 'Event [%s] message body [%s] headers [%s] deliveryInfo [%s]' % (
eventType, body, message.properties, message.delivery_info)
# ACK
self._channel.basic.ack(deliveryTag)
class Server(object):
"""The Meta Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_meta', auto_delete = True)
self._channel.basic.consume('test_meta', self.callback, no_ack = False)
def run(self):
"""Waiting for response
"""
while True:
self._conn.read_frames()
def callback(self, message):
"""The callee method
"""
body = message.body
deliveryInfo = message.delivery_info
properties = message.properties
# Print the meta
print 'Receive message body [%s] deliveryInfo [%s] properties [%s]' % (body, deliveryInfo, properties)
# ACK
deliveryTag = deliveryInfo.get('delivery_tag')
self._channel.basic.ack(deliveryTag)
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_dead_channel', auto_delete = True)
self._channel.queue.bind('test_dead_channel', exchange = 'amq.topic', routing_key = 'test.dead_channel')
self._channel.basic.consume('test_dead_channel', self.callback, no_ack = False)
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
result = self._channel.queue.declare(arguments = { 'x-dead-letter-exchange': 'amq.topic', 'x-dead-letter-routing-key': 'test.dead_channel' })
self._deadQueue = result[0]
# Send a message
self._channel.basic.publish(Message('OMG! I\'m dead!'), '', self._deadQueue)
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport = 'gevent', host = host, port = port, vhost = vhost, user = user, password = password, open_cb = self.onConnected)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
self._channel.basic.qos(prefetch_count = 10)
self._channel.queue.declare('test_confirm', auto_delete = True)
self._channel.basic.consume('test_confirm', self.callback, no_ack = False)
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport = 'gevent', host = host, port = port, vhost = vhost, user = user, password = password)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
self._channel.confirm.select()
self._channel.basic.set_return_listener(self.onBasicReturn)
self._channel.basic.set_ack_listener(self.onDeliverAck)
self._channel.basic.set_nack_listener(self.onDeliverNAck)
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_meta', auto_delete=True)
self._channel.basic.consume('test_meta', self.callback, no_ack=False)
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport='gevent',
host=host,
port=port,
vhost=vhost,
user=user,
password=password)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
result = self._channel.queue.declare(exclusive=True)
self._callbackQueue = result[0]
self._channel.basic.consume(self._callbackQueue,
self.onResponse,
no_ack=True)
self._response = None
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_meta', auto_delete = True)
self._channel.basic.consume('test_meta', self.callback, no_ack = False)
def _create_connection(self):
"Tries to create a connection, returns True on success"
self._connection = None
self._last_confirmed_message = None
host = self._get_next_host()
self._logger.debug('Trying to connect to {}'.format(host))
try:
self._connection = RabbitConnection(
host=host,
port=self._connection_port,
user=self._connection_user,
password=self._connection_password,
vhost=self._connection_path,
close_cb=self._close_cb,
heartbeat=self._connection_heartbeat,
client_properties={
'connection_name': self._connection_name,
},
)
except socket.error as exc:
self._logger.error('Error connecting to rabbitmq {}'.format(exc))
return False
self._channel = self._connection.channel()
if self._confirm:
self._channel.confirm.select(nowait=False)
self._channel.basic.set_ack_listener(self._ack)
self._logger.debug('Connected to {}'.format(host))
return True
def __init__(self, connectionParams=None, channelConfigCb=None):
"""
NOTE: Connection establishment may be performed in the scope of the
constructor or on demand, depending on the underlying implementation
:param nta.utils.amqp.connection.ConnectionParams connectionParams:
parameters for connecting to AMQP broker;
[default=default params for RabbitMQ broker on localhost]
:param channelConfigCb: An optional callback function that will be
called whenever a new AMQP Channel is being brought up
:type channelConfigCb: None or callable with the signature
channelConfigCb(SynchronousAmqpClient)
"""
self._channelConfigCb = channelConfigCb
# Holds _ChannelContext when channel is created; we create the channel on
# demand. The implementation accesses this member via the
# `_liveChannelContext` property getter when it's desirable to bring up the
# channel on-on demand. When it's undesirable to bring up the channel, the
# implementation interacts directly with this member, which will be None
# when we don't have a channel.
self._channelContextInstance = None
# Set to True when user calls close, so we know to not raise an exception
# from our _on*Closed callback methods
self._userInitiatedClosing = False
# Instantiate underlying connection object
params = (connectionParams if connectionParams is not None else
amqp_connection.ConnectionParams())
# NOTE: we could get a `close_cb` call from RabbitConnection constructor, so
# prepare for it by initializing `self._connection`
self._connection = None
self._connection = RabbitConnection(
transport="socket",
sock_opts={(socket.IPPROTO_TCP, socket.TCP_NODELAY): 1},
synchronous=True,
close_cb=self._onConnectionClosed,
user=params.credentials.username,
password=params.credentials.password,
vhost=params.vhost,
host=params.host,
port=params.port,
heartbeat=self._DEFAULT_HEARTBEAT_TIMEOUT_SEC,
logger=g_log)
class Server(object):
"""The Meta Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport='gevent',
host=host,
port=port,
vhost=vhost,
user=user,
password=password)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
self._channel.basic.qos(prefetch_count=10)
self._channel.queue.declare('test_gevent', auto_delete=True)
self._channel.basic.consume('test_gevent', self.callback, no_ack=False)
def loop(self):
"""Waiting for response
"""
while self._conn:
self._conn.read_frames()
gevent.sleep()
def callback(self, message):
"""The callee method
"""
# NOTE:
# Here, we have to spawn the actually message processing method in order to process the message parallely
gevent.spawn(self.process, message)
def process(self, message):
"""Process the message
"""
body = message.body
deliveryInfo = message.delivery_info
deliveryTag = deliveryInfo.get('delivery_tag')
# Print the meta
print 'Receive message body [%s] deliveryTag [%s], will sleep 10s' % (
body, deliveryTag)
gevent.sleep(10)
# ACK
print 'Wake up, ACK the message'
self._channel.basic.ack(deliveryTag)
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
result = self._channel.queue.declare(exclusive = True)
self._callbackQueue = result[0]
self._channel.basic.consume(self._callbackQueue, self.onResponse, no_ack = True)
self._response = None
class EventSender(object):
"""The event sender
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new EventSender
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
self._channel.queue.declare('_aprefix/%s/webservice/%s' % (socket.gethostname(), os.getpid()), exclusive = True)
class Server(object):
"""The Meta Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport = 'gevent', host = host, port = port, vhost = vhost, user = user, password = password, open_cb = self.onConnected)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
self._channel.basic.qos(prefetch_count = 10)
self._channel.queue.declare('test_confirm', auto_delete = True)
self._channel.basic.consume('test_confirm', self.callback, no_ack = False)
def onConnected(self):
"""On connected
"""
print 'Connected'
def loop(self):
"""Waiting for response
"""
while self._conn:
self._conn.read_frames()
gevent.sleep()
def callback(self, message):
"""The callee method
"""
# NOTE:
# Here, we have to spawn the actually message processing method in order to process the message parallely
gevent.spawn(self.process, message)
def process(self, message):
"""Process the message
"""
body = message.body
deliveryInfo = message.delivery_info
deliveryTag = deliveryInfo.get('delivery_tag')
# Print the meta
print 'Receive message body [%s] deliveryTag [%s], will sleep 10s' % (body, deliveryTag)
gevent.sleep(10)
# ACK
print 'Wake up, ACK the message'
self._channel.basic.ack(deliveryTag)
def plain_rabbit_connection_to_hosts(hosts, **kwargs):
for host in hosts:
logger.info('Trying to connect to host: {0}'.format(host))
try:
conn = RabbitConnection(host=host, **kwargs)
logger.info("...success")
return conn
except socket.error:
logger.info('Error connecting to {0}'.format(host))
logger.error('Could not connect to any hosts')
class Client(object):
"""The RPC Client
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
result = self._channel.queue.declare(exclusive=True)
self._callbackQueue = result[0]
self._channel.basic.consume(self._callbackQueue,
self.onResponse,
no_ack=True)
self._response = None
def onResponse(self, message):
"""On response
"""
correlationID = message.properties.get('correlation_id')
print 'Receive server response [%s] correlationID [%s]' % (
message.body, correlationID)
self._response = int(message.body)
def call(self, number):
"""The call method
"""
self._response = None
corrID = str(uuid4())
print 'Call server with request [%s] correlationID [%s]' % (number,
corrID)
self._channel.basic.publish(
Message(str(number),
reply_to=self._callbackQueue,
correlation_id=corrID), '', 'test_rpc')
while self._response is None:
self._conn.read_frames()
# Done
return self._response
class Client(object):
"""The RPC Client
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport = 'gevent', host = host, port = port, vhost = vhost, user = user, password = password)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
def loop(self):
"""The loop
"""
while self._conn:
self._conn.read_frames()
gevent.sleep()
def call(self):
"""The call method
"""
self._channel.basic.publish(Message('A test body'), '', 'test_gevent')
class Client(object):
"""The RPC Client
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport = 'gevent', host = host, port = port, vhost = vhost, user = user, password = password)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
self._channel.confirm.select()
self._channel.basic.set_return_listener(self.onBasicReturn)
self._channel.basic.set_ack_listener(self.onDeliverAck)
self._channel.basic.set_nack_listener(self.onDeliverNAck)
def loop(self):
"""The loop
"""
while self._conn:
self._conn.read_frames()
gevent.sleep()
def onBasicReturn(self, message):
"""On basic return
"""
print 'Basic return message [%s]' % message
def onDeliverAck(self, messageID):
"""On deliver ACK
"""
print 'Deliver ACK [%s]' % messageID
def onDeliverNAck(self, messageID, requeue):
"""On deliver nack
"""
print 'Deliver NACK [%s] Requeue [%s]' % (messageID, requeue)
def call(self, content, queue):
"""The call method
"""
return self._channel.basic.publish(Message(content), '', queue)
class Client(object):
"""The RPC Client
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
def call(self):
"""The call method
"""
self._channel.basic.publish(Message('A test body', correlation_id = '123', application_headers = { 'custom_header': 'value', 'custom_header1': 1 }), '', 'test_meta')
def _connect_to_broker(self):
''' Connect to broker and regisiter cleanup action to disconnect
:returns: connection instance
:rtype: `haigha.connections.rabbit_connection.Connection`
'''
sock_opts = {
(socket.IPPROTO_TCP, socket.TCP_NODELAY): 1,
}
connection = RabbitConnection(logger=_LOG,
debug=_OPTIONS.debug,
user=_OPTIONS.user,
password=_OPTIONS.password,
vhost=_OPTIONS.vhost,
host=_OPTIONS.host,
heartbeat=None,
sock_opts=sock_opts,
transport='socket')
self.addCleanup(lambda: connection.close(disconnect=True)
if not connection.closed else None)
return connection
def _connect_to_broker(self):
''' Connect to broker and regisiter cleanup action to disconnect
:returns: connection instance
:rtype: `haigha.connections.rabbit_connection.Connection`
'''
sock_opts = {
(socket.IPPROTO_TCP, socket.TCP_NODELAY) : 1,
}
connection = RabbitConnection(
logger=_LOG,
debug=_OPTIONS.debug,
user=_OPTIONS.user,
password=_OPTIONS.password,
vhost=_OPTIONS.vhost,
host=_OPTIONS.host,
heartbeat=None,
sock_opts=sock_opts,
transport='socket')
self.addCleanup(lambda: connection.close(disconnect=True)
if not connection.closed else None)
return connection
class Server(object):
"""The RPC Server
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
self._channel.queue.declare('test_rpc', auto_delete=True)
self._channel.basic.consume('test_rpc', self.callee, no_ack=False)
def run(self):
"""Waiting for response
"""
while True:
self._conn.read_frames()
def callee(self, message):
"""The callee method
"""
num = int(message.body)
replyTo = message.properties.get('reply_to')
correlationID = message.properties.get('correlation_id')
deliveryTag = message.delivery_info.get('delivery_tag')
# Print the meta
print 'Receive message body [%s] replyTo [%s] correlationID [%s]' % (
num, replyTo, correlationID)
# Return add 1
self._channel.basic.publish(
Message(str(num + 1), correlation_id=correlationID), '', replyTo)
# ACK
self._channel.basic.ack(deliveryTag)
def __init__(self, connectionParams=None, channelConfigCb=None):
"""
NOTE: Connection establishment may be performed in the scope of the
constructor or on demand, depending on the underlying implementation
:param nta.utils.amqp.connection.ConnectionParams connectionParams:
parameters for connecting to AMQP broker;
[default=default params for RabbitMQ broker on localhost]
:param channelConfigCb: An optional callback function that will be
called whenever a new AMQP Channel is being brought up
:type channelConfigCb: None or callable with the signature
channelConfigCb(SynchronousAmqpClient)
"""
self._channelConfigCb = channelConfigCb
# Holds _ChannelContext when channel is created; we create the channel on
# demand. The implementation accesses this member via the
# `_liveChannelContext` property getter when it's desirable to bring up the
# channel on-on demand. When it's undesirable to bring up the channel, the
# implementation interacts directly with this member, which will be None
# when we don't have a channel.
self._channelContextInstance = None
# Set to True when user calls close, so we know to not raise an exception
# from our _on*Closed callback methods
self._userInitiatedClosing = False
# Instantiate underlying connection object
params = connectionParams if connectionParams is not None else amqp_connection.ConnectionParams()
# NOTE: we could get a `close_cb` call from RabbitConnection constructor, so
# prepare for it by initializing `self._connection`
self._connection = None
self._connection = RabbitConnection(
transport="socket",
sock_opts={(socket.IPPROTO_TCP, socket.TCP_NODELAY): 1},
synchronous=True,
close_cb=self._onConnectionClosed,
user=params.credentials.username,
password=params.credentials.password,
vhost=params.vhost,
host=params.host,
port=params.port,
heartbeat=self._DEFAULT_HEARTBEAT_TIMEOUT_SEC,
logger=g_log,
)
def connect(self, url):
"""
Connect to rabbitmq and create a channel
"""
parts = re.search(r'amqp://(\w+):(\w+)@([^\:]+)\:(\d+)\/(.*)\/?', url).groups()
self.user, self.password, self.host, self.port, self.vhost_url = parts
self.vhost = self.vhost_url.replace('%2F', '/')
params = dict(
user=self.user, password=self.password,
vhost=self.vhost, host=self.host, transport=self.transport
)
if self.transport == 'gevent':
params['close_cb'] = self._connection_closed_cb,
self.connection = RabbitConnection(**params)
self.ch = self.connection.channel()
if self.transport == 'gevent':
self._message_pump_greenlet = gevent.spawn(self._message_pump_greenthread)
self.ch.add_close_listener(self._channel_closed_cb)
def get_connection(amqp_url):
global connection
if connection and not connection.closed:
return connection
parse = urlparse.urlparse(amqp_url)
sock_opts = {
(socket.IPPROTO_TCP, socket.TCP_NODELAY): 1
}
connection = RabbitConnection(logger=logger, debug=logging.WARN, user=parse.username, password=parse.password,
vhost=parse.path, host=parse.hostname, hearbeat=None, port=parse.port,
close_cb=connection_closed, sock_opts=sock_opts, transport="gevent")
global connection_task
connection_task = gevent.spawn(frame_loop)
global connection_consumers
if connection_consumers:
for c in connection_consumers.itervalues():
c._reconnect()
c.start_consuming()
return connection
class Client(object):
"""The RPC Client
"""
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host=host,
port=port,
vhost=vhost,
user=user,
password=password)
self._channel = self._conn.channel()
def call(self):
"""The call method
"""
self._channel.basic.publish(
Message('A test body',
correlation_id='123',
application_headers={
'custom_header': 'value',
'custom_header1': 1
}), '', 'test_meta')
class SynchronousAmqpClient(object):
"""Synchronous (blocking) AMQP client abstraction that exposes AMQP
functionality presently needed by nta.utils and products. New AMQP
functionality will be exposed as the need arises.
This class provides a consistent AMQP client API to the rest of the products
regardless of the underlying implementation. This helps avoid/minimize changes
to the higher-level code when we need to swap out the underlying client
implementation.
This class is NOT the place for higher-level constructs: see
message_bus_connector module for an example of higher-level functionality
built on top of this class.
NOTE: this implementation is completely synchronous and MUST NOT expose
callbacks (callbaks lead to complexities, such as an opportunity for
unintended/unsupported recursion)
NOTE: The `no*` parameters, such as `noLocal` and `noAck`, are artificats
of the AMQP protocol specification. The author of SynchronousAmqpClient chose
to preserve those semantics to facilitate better correlation with AMQP
documentation.
"""
# Correlations between names of BasicProperties attributes and Haigha's
# message property names.
#
# The table consists of the tollowing columns
# BasicProperty attribute name
# Corresponding Haigha property name
# Value conversion function from BasicProperty to Haigha
# Value conversion function from Haigha to BasicProperty
#
_asIs = lambda x: x
_PROPERTY_CORRELATIONS = (
("contentType", "content_type", _asIs, _asIs),
("contentEncoding", "content_encoding", _asIs, _asIs),
("headers", "application_headers", _asIs, _asIs),
("deliveryMode", "delivery_mode", _asIs, _asIs),
("priority", "priority", _asIs, _asIs),
("correlationId", "correlation_id", _asIs, _asIs),
("replyTo", "reply_to", _asIs, _asIs),
("expiration", "expiration", _asIs, _asIs),
("messageId", "message_id", _asIs, _asIs),
("timestamp", "timestamp", datetime.utcfromtimestamp, epochFromNaiveUTCDatetime),
("messageType", "type", _asIs, _asIs),
("userId", "user_id", _asIs, _asIs),
("appId", "app_id", _asIs, _asIs),
("clusterId", "cluster_id", _asIs, _asIs),
)
# haigha returns body as bytearray, but our current users of the interface
# assume they are getting str or bytes
_decodeMessageBody = str
# NOTE: RabbitMQ release 3.5.5, changed the server's default heartbeat timeout
# from 580 to 60, causing frequent dropped connections on our blocking
# transport. So, we restore the longer timeout by passing a bigger value
# to the broker during connection tuning.
_DEFAULT_HEARTBEAT_TIMEOUT_SEC = 600
def __init__(self, connectionParams=None, channelConfigCb=None):
"""
NOTE: Connection establishment may be performed in the scope of the
constructor or on demand, depending on the underlying implementation
:param nta.utils.amqp.connection.ConnectionParams connectionParams:
parameters for connecting to AMQP broker;
[default=default params for RabbitMQ broker on localhost]
:param channelConfigCb: An optional callback function that will be
called whenever a new AMQP Channel is being brought up
:type channelConfigCb: None or callable with the signature
channelConfigCb(SynchronousAmqpClient)
"""
self._channelConfigCb = channelConfigCb
# Holds _ChannelContext when channel is created; we create the channel on
# demand. The implementation accesses this member via the
# `_liveChannelContext` property getter when it's desirable to bring up the
# channel on-on demand. When it's undesirable to bring up the channel, the
# implementation interacts directly with this member, which will be None
# when we don't have a channel.
self._channelContextInstance = None
# Set to True when user calls close, so we know to not raise an exception
# from our _on*Closed callback methods
self._userInitiatedClosing = False
# Instantiate underlying connection object
params = connectionParams if connectionParams is not None else amqp_connection.ConnectionParams()
# NOTE: we could get a `close_cb` call from RabbitConnection constructor, so
# prepare for it by initializing `self._connection`
self._connection = None
self._connection = RabbitConnection(
transport="socket",
sock_opts={(socket.IPPROTO_TCP, socket.TCP_NODELAY): 1},
synchronous=True,
close_cb=self._onConnectionClosed,
user=params.credentials.username,
password=params.credentials.password,
vhost=params.vhost,
host=params.host,
port=params.port,
heartbeat=self._DEFAULT_HEARTBEAT_TIMEOUT_SEC,
logger=g_log,
)
def __repr__(self):
# NOTE: we don't use the _liveChannelContext property getter in order to
# avoid creation of channel here
return "%s(channelContext=%r)" % (self.__class__.__name__, self._channelContextInstance)
@property
def _liveChannelContext(self):
"""NOTE: Creates channel on demand"""
if self._channelContextInstance is None:
self._channelContextInstance = _ChannelContext(self._connection.channel(synchronous=True))
try:
self._channelContextInstance.channel.add_close_listener(self._onChannelClosed)
self._channelContextInstance.channel.basic.set_return_listener(self._onMessageReturn)
if self._channelConfigCb is not None:
self._channelConfigCb(self)
except Exception: # pylint: disable=W0703
g_log.exception("Channel configuration failed")
try:
# Preserve the original exception
raise
finally:
# Close channel and reset channel context
try:
if self._channelContextInstance is not None:
self._channelContextInstance.channel.close(disconnect=True)
except Exception: # pylint: disable=W0703
# Suppress the secondary exception from cleanup
g_log.exception("Channel closing Failed following configuration failure")
finally:
self._channelContextInstance.reset()
self._channelContextInstance = None
return self._channelContextInstance
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def isOpen(self):
return self._connection is not None and not self._connection.closed
def close(self):
"""Gracefully close client"""
self._userInitiatedClosing = True
try:
# NOTE: we check _channelContextInstance directly to avoid creating a
# channel if one doesn't exist
if self._channelContextInstance is not None:
channelContext = self._channelContextInstance
try:
channelContext.channel.close()
except Exception: # pylint: disable=W0703
g_log.exception("Channel close failed")
if self._connection is not None:
try:
self._connection.close(disconnect=True)
except Exception: # pylint: disable=W0703
g_log.exception("Connection close failed")
finally:
self._connection = None
if self._channelContextInstance is not None:
self._channelContextInstance.reset()
self._channelContextInstance = None
def enablePublisherAcks(self):
"""Enable RabbitMQ publisher acknowledgments
:raises nta.utils.amqp.exceptions.UnroutableError: raised when messages
that were sent in non-publisher-acknowledgments mode are returned by
the time the Confirm.Select-Ok is received
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
channelContext = self._liveChannelContext
if channelContext.pubacksSelected:
g_log.warning("enablePublisherAcks: already enabled")
else:
channelContext.channel.confirm.select(nowait=False)
channelContext.pubacksSelected = True
# NOTE: Unroutable messages returned after this will be in the context of
# publisher acknowledgments
self._raiseAndClearIfReturnedMessages()
def publish(self, message, exchange, routingKey, mandatory=False):
""" Publish a message
:param nta.utils.amqp.messages.Message message:
:param str exchange: destination exchange name; "" for default exchange
:param str routingKey: Message routing key
:param bool mandatory: This flag tells the server how to react if the
message cannot be routed to a queue. If this flag is True, the server will
return an unroutable message with a Return method. If this flag is False
the server silently drops the message.
:raises nta.utils.amqp.exceptions.UnroutableError: when in
non-publisher-acknowledgments mode, raised before attempting to publish
given message if unroutable messages had been returned. In
publisher-acknowledgments mode, raised if the given
message is returned as unroutable.
:raises nta.utils.amqp.exceptions.NackError: when the given message is
NACKed by broker while channel is in RabbitMQ publisher-acknowledgments
mode
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
message = HaighaMessage(body=message.body, **self._makeHaighaPropertiesDict(message.properties))
channelContext = self._liveChannelContext
if channelContext.pubacksSelected:
# In publisher-acknowledgments mode
assert not channelContext.returnedMessages, (
len(channelContext.returnedMessages),
channelContext.returnedMessages,
)
pubackState = _PubackState()
channelContext.channel.basic.set_ack_listener(pubackState.handleAck)
channelContext.channel.basic.set_nack_listener(pubackState.handleNack)
deliveryTag = channelContext.channel.basic.publish(
message, exchange=exchange, routing_key=routingKey, mandatory=mandatory
)
# Wait for ACK or NACK
while not pubackState.ready:
self._connection.read_frames()
try:
((how, responseTag),) = pubackState.values
except ValueError:
g_log.exception("Error unpacking values=%r", pubackState.values)
raise
assert responseTag == deliveryTag, ((how, responseTag), deliveryTag)
if how == _PubackState.NACK:
# Raise NackError with returned message
returnedMessages = channelContext.returnedMessages
channelContext.returnedMessages = []
raise amqp_exceptions.NackError(returnedMessages)
# It was Acked
assert how == _PubackState.ACK, how
# Raise if this message was returned as unroutable
self._raiseAndClearIfReturnedMessages()
else:
# Not in publisher-acknowledgments mode
# Raise if some prior messages were returned as unroutable
self._raiseAndClearIfReturnedMessages()
channelContext.channel.basic.publish(
message, exchange=exchange, routing_key=routingKey, mandatory=mandatory
)
def requestQoS(self, prefetchSize=0, prefetchCount=0, entireConnection=False):
"""This method requests a specific quality of service. The QoS can be
specified for the current channel or for all channels on the connection. The
particular properties and semantics of a qos method always depend on the
content class semantics
:param int prefetchSize: The client can request that messages be sent in
advance so that when the client finishes processing a message, the
following message is already held locally, rather than needing to be sent
down the channel. Prefetching gives a performance improvement. This field
specifies the prefetch window size in octets. The server will send a
message in advance if it is equal to or smaller in size than the available
prefetch size (and also falls into other prefetch limits). May be set to
zero, meaning "no specific limit", although other prefetch limits may
still apply. The prefetchsize is ignored if the no-ack option is set.
:param int prefetchCount: Specifies a prefetch window in terms of whole
messages. This field may be used in combination with the prefetch-size
field; a message will only be sent in advance if both prefetch windows
(and those at the channel and connection level) allow it. The
prefetch-count is ignored if the no-ack option is set.
:param bool entireConnection: By default the QoS settings apply to the
current channel only. If this field is set, they are applied to the entire
connection.
:raises AmqpChannelError:
"""
self._liveChannelContext.channel.basic.qos(
prefetch_size=prefetchSize, prefetch_count=prefetchCount, is_global=entireConnection
)
def createConsumer(self, queue, noLocal=False, noAck=False, exclusive=False):
"""This method asks the server to start a "consumer", which is a transient
request for messages from a specific queue. Consumers last as long as the
channel they were declared on, or until the client or broker cancels them.
See `Consumer.cancel()`
Use `getNextEvent()` to retrieve consumed messages and other events.
:param str queue: name of the queue to consume from
:param bool noLocal: If true, the server will not send messages to the
connection that published them.
:param bool noAck: if true, the broker will not expect messages to be ACKed
:param bool exclusive: Request exclusive consumer access, meaning only this
consumer can access the queue
:returns: consumer context
:rtype: nta.utils.amqp.consumer.Consumer
:raises nta.utils.amqp.exceptions.AmqpChannelError:
:raises nta.utils.amqp.exceptions.AmqpConnectionError:
"""
consumerTag = self._makeConsumerTag()
channelContext = self._liveChannelContext
channelContext.channel.basic.consume(
queue,
consumer=self._onMessageDelivery,
consumer_tag=consumerTag,
no_local=noLocal,
no_ack=noAck,
exclusive=exclusive,
cancel_cb=self._onConsumerCancelled,
nowait=False,
)
channelContext.consumerSet.add(consumerTag)
consumer = amqp_consumer.Consumer(tag=consumerTag, queue=queue, cancelImpl=channelContext.cancelConsumer)
g_log.info(
"Created consumer=%r; queue=%r, noLocal=%r, noAck=%r, exclusive=%r",
consumer,
queue,
noLocal,
noAck,
exclusive,
)
return consumer
def hasEvent(self):
"""Check if there are events ready for consumption. See `getNextEvent()`.
:returns: True if there is at least one event ready for consumption, in
which case `getNextEvent()` may be called once without blocking.
:rtype: bool
"""
channelContext = self._channelContextInstance
return bool(channelContext is not None and channelContext.pendingEvents)
def getNextEvent(self):
"""Get next event, blocking if there isn't one yet. See `hasEvent()`. You
MUST have an active consumer (`createConsumer`) or other event source before
calling this method.
An event may be an object of one of the following classes:
nta.utils.amqp.messages.ConsumerMessage
nta.utils.amqp.consumer.ConsumerCancellation
:returns: the next event when it becomes available
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
# We expect the context to be set up already
channelContext = self._channelContextInstance
while not channelContext.pendingEvents:
self._connection.read_frames()
return channelContext.pendingEvents.popleft()
def readEvents(self):
"""Generator that yields results of `getNextEvent()`"""
while True:
yield self.getNextEvent()
def getOneMessage(self, queue, noAck=False):
"""This is the polling, less-performant method of getting a message. This
method provides a direct access to the messages in a queue using a
synchronous dialogue that is designed for specific types of application
where synchronous functionality is more important than performance.
:param str queue: name of the queue to get a message from
:param bool noAck: if true, the broker will not be expecting an Ack
:returns: A PolledMessage object if there was a message in queue; None if
there was no message in queue.
:rtype: PolledMessage or None
:raises AmqpChannelError:
"""
channelContext = self._liveChannelContext
consumer = _CallbackSink()
channelContext.channel.basic.get(queue, consumer=consumer, no_ack=noAck)
while not consumer.ready:
self._connection.read_frames()
try:
((message,),) = consumer.values
except ValueError:
g_log.exception("Error unpacking values=%r", consumer.values)
raise
if message is not None:
ackImpl = channelContext.ack if not noAck else None
nackImpl = channelContext.nack if not noAck else None
message = self._makePolledMessage(message, ackImpl, nackImpl)
return message
@classmethod
def _makePolledMessage(cls, haighaMessage, ackImpl, nackImpl):
"""Make PolledMessage from haigha message retrieved via Basic.Get
:param haigha.message.Message haighaMessage: haigha message retrieved via
Basic.Get
:param ackImpl: callable for acking the message that has the following
signature: ackImpl(deliveryTag, multiple=False); or None
:param nackImpl: callable for nacking the message that has the following
signature: nackImpl(deliveryTag, requeue); or None
:rtype: nta.utils.amqp.messages.PolledMessage
"""
info = haighaMessage.delivery_info
methodInfo = amqp_messages.MessageGetInfo(
deliveryTag=info["delivery_tag"],
redelivered=bool(info["redelivered"]),
exchange=info["exchange"],
routingKey=info["routing_key"],
messageCount=info["message_count"],
)
return amqp_messages.PolledMessage(
body=cls._decodeMessageBody(haighaMessage.body),
properties=cls._makeBasicProperties(haighaMessage.properties),
methodInfo=methodInfo,
ackImpl=ackImpl,
nackImpl=nackImpl,
)
def recover(self, requeue=False):
"""This method asks the server to redeliver all unacknowledged messages on a
specified channel. Zero or more messages may be redelivered.
NOTE: RabbitMQ does not currently support recovering with requeue=False
:param bool requeue: If false, the message will be redelivered to the
original recipient. If true, the server will attempt to requeue the
message, potentially then delivering it to an alternative subscriber.
"""
self._liveChannelContext.channel.basic.recover(requeue=requeue)
def ackAll(self):
"""Acknowledge all unacknowledged messages on current channel instance.
Acknowledgemets related to specific messages are performed via the message's
own `ack()` method.
NOTE: messages received prior to the AmqpChannelError exception cannot be
acknowledged since that exception occurs after the AMQP channel is closed
"""
self._channelContextInstance.ack(deliveryTag=0, multiple=True)
def nackAll(self, requeue=False):
"""Reject all outstanding (unacknowledged) messages on current channel
instance.
Rejections related to specific messages are performed via the message's
own `nack()` method.
NOTE: has no impact on messages received prior to the AmqpChannelError
exception since AmqpChannelError is raised after the AMQP channel is closed
:param bool requeue: If requeue is true, the server will attempt to requeue
the messages. If requeue is false or the requeue attempt fails the
messages are discarded or dead-lettered
"""
self._channelContextInstance.nack(deliveryTag=0, multiple=True, requeue=requeue)
def declareExchange(self, exchange, exchangeType, passive=False, durable=False, autoDelete=False, arguments=None):
"""Declare an exchange
:param str exchange: name of the exchange
:param str exchangeType: type of the exchange
:param bool passive: If True, the server will reply with Declare-Ok if the
exchange already exists with the same name, and raise an error if not. The
client can use this to check whether an exchange exists without modifying
the server state. When set, all other method fields except name are
ignored. Arguments are compared for semantic equivalence.
:param bool durable: If True when creating a new exchange, the exchange will
be marked as durable. Durable exchanges remain active when a server
restarts. Non-durable exchanges (transient exchanges) are purged if/when a
server restarts.
:param bool autoDelete: If true, the exchange is deleted when all queues
have finished using it (RabbitMQ-specific).
:param dict arguments: custom key/value pairs for the exchange
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.exchange.declare(
exchange,
exchangeType,
passive=passive,
durable=durable,
auto_delete=autoDelete,
arguments=arguments or dict(),
nowait=False,
)
def deleteExchange(self, exchange, ifUnused=False):
"""Delete an exchange
:param str exchange: exchange to be deleted
:param bool ifUnused: If True, the server will only delete the exchange if
it has no queue bindings. If the exchange has queue bindings the server
does not delete it but raises a channel exception instead.
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.exchange.delete(exchange, if_unused=ifUnused, nowait=False)
def declareQueue(self, queue, passive=False, durable=False, exclusive=False, autoDelete=False, arguments=None):
"""Declare a queue
:param str queue: name of queue
:param bool passive: If True, the server will reply with Declare-Ok if the
queue already exists with the same name, and raise an error if not. The
client can use this to check whether a queue exists without modifying the
server state. When set, all other method fields except name are ignored.
Arguments are compared for semantic equivalence.
:param bool durable: If true when creating a new queue, the queue will be
marked as durable. Durable queues remain active when a server restarts.
Non-durable queues (transient queues) are purged if/when a server
restarts. Note that durable queues do not necessarily hold persistent
messages, although it does not make sense to send persistent messages to a
transient queue.
:param bool exclusive: Exclusive queues may only be accessed by the current
connection, and are deleted when that connection closes. Passive
declaration of an exclusive queue by other connections are not allowed.
:param bool autoDelete: If true, the queue is deleted when all consumers
have finished using it. The last consumer can be cancelled either
explicitly or because its channel is closed. If there was no consumer ever
on the queue, it won't be deleted. Applications can explicitly delete
auto-delete queues using the Delete method as normal. (RabbitMQ-specific)
:param dict arguments: A set of key/value pairs for the declaration. The
syntax and semantics of these arguments depends on the server
implementation.
:rtype: nta.utils.amqp.queue.QueueDeclarationResult
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
queue, messageCount, consumerCount = self._liveChannelContext.channel.queue.declare(
queue,
passive=passive,
durable=durable,
exclusive=exclusive,
auto_delete=autoDelete,
arguments=arguments or dict(),
nowait=False,
)
return amqp_queue.QueueDeclarationResult(queue, messageCount, consumerCount)
def deleteQueue(self, queue, ifUnused=False, ifEmpty=False):
"""Delete a queue. When a queue is deleted any pending messages are sent to
a dead-letter queue if this is defined in the server configuration, and all
consumers on the queue are cancelled
:param str queue: name of the queue to delete
:param bool ifUnused: If true, the server will only delete the queue if it
has no consumers. If the queue has consumers the server does does not
delete it but raises a channel exception instead.
:param bool ifEmpty: If true, the server will only delete the queue if it
has no messages
:returns: number of messages deleted
:rtype: int
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
return self._liveChannelContext.channel.queue.delete(queue, if_unused=ifUnused, if_empty=ifEmpty, nowait=False)
def purgeQueue(self, queue):
"""Remove all messages from a queue which are not awaiting acknowledgment.
:param str queue: name of the queue to purge
:returns: number of messages purged
:rtype: int
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
return self._liveChannelContext.channel.queue.purge(queue, nowait=False)
def bindQueue(self, queue, exchange, routingKey, arguments=None):
"""Bind a queue to an exchange
:param str queue: name of the queue to bind
:param str exchange: name of the exchange to bind to
:param str routingKey: Specifies the routing key for the binding. The
routing key is used for routing messages depending on the exchange
configuration. Not all exchanges use a routing key refer to the
specific exchange documentation. If the queue name is empty, the server
uses the last queue declared on the channel. If the routing key is also
empty, the server uses this queue name for the routing key as well. If the
queue name is provided but the routing key is empty, the server does the
binding with that empty routing key. The meaning of empty routing keys
depends on the exchange implementation.
:param dict arguments: A set of key/value pairs for the binding. The syntax
and semantics of these arguments depends on the exchange class and server
implemenetation.
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.queue.bind(
queue=queue, exchange=exchange, routing_key=routingKey, arguments=arguments or dict(), nowait=False
)
def unbindQueue(self, queue, exchange, routingKey, arguments=None):
"""Unbind a queue fro an exchange
:param str queue: name of the queue to unbind
:param str exchange: name of the exchange to unbind from
:param str routingKey: the routing key of the binding to unbind
:param dict arguments: Specifies the arguments of the binding to unbind
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.queue.unbind(
queue=queue, exchange=exchange, routing_key=routingKey, arguments=arguments or dict()
)
def _raiseAndClearIfReturnedMessages(self):
"""If returned messages are present, raise UnroutableError and clear
returned messages holding buffer
:raises nta.utils.amqp.exceptions.UnroutableError: if returned messages
are present
"""
channelContext = self._channelContextInstance
if channelContext and channelContext.returnedMessages:
messages = channelContext.returnedMessages
channelContext.returnedMessages = []
raise amqp_exceptions.UnroutableError(messages)
@classmethod
def _makeHaighaPropertiesDict(cls, BasicProperties):
"""Marshal BasicProperties into the haigha properties dict
:param nta.utils.amqp.messages.BasicProperties BasicProperties:
:returns: dict of Properties expected by Haigha's publish method
:rtype: dict
"""
props = dict()
for attrName, propName, clientToHaigha, _ in cls._PROPERTY_CORRELATIONS:
value = getattr(BasicProperties, attrName)
# Add only those properties that have concrete values
if value is not None:
props[propName] = clientToHaigha(value)
return props
@classmethod
def _makeBasicProperties(cls, haighaProperties):
attributes = dict()
for attrName, propName, _, haighaToClient in cls._PROPERTY_CORRELATIONS:
value = haighaProperties.get(propName)
if value is not None:
value = haighaToClient(value)
attributes[attrName] = value
return amqp_messages.BasicProperties(**attributes)
def _makeConsumerTag(self):
channelContext = self._liveChannelContext
tag = "channel-%d-%d" % (channelContext.channel.channel_id, channelContext.nextConsumerTag)
channelContext.nextConsumerTag += 1
return tag
def _onMessageDelivery(self, message):
"""Handle consumer message from Basic.Deliver
:param haigha.message.Message message:
"""
channelContext = self._channelContextInstance
g_log.debug("Consumer message received: %.255s", message)
channelContext.pendingEvents.append(self._makeConsumerMessage(message, channelContext.ack, channelContext.nack))
@classmethod
def _makeConsumerMessage(cls, haighaMessage, ackImpl, nackImpl):
"""Make ConsumerMessage from haigha message received via Basic.Deliver
:param haigha.message.Message haighaMessage: haigha message received via
Basic.Deliver
:param ackImpl: callable for acking the message that has the following
signature: ackImpl(deliveryTag, multiple=False); or None
:param nackImpl: callable for nacking the message that has the following
signature: nackImpl(deliveryTag, requeue); or None
:rtype: ConsumerMessage
"""
info = haighaMessage.delivery_info
methodInfo = amqp_messages.MessageDeliveryInfo(
consumerTag=info["consumer_tag"],
deliveryTag=info["delivery_tag"],
redelivered=bool(info["redelivered"]),
exchange=info["exchange"],
routingKey=info["routing_key"],
)
return amqp_messages.ConsumerMessage(
body=cls._decodeMessageBody(haighaMessage.body),
properties=cls._makeBasicProperties(haighaMessage.properties),
methodInfo=methodInfo,
ackImpl=ackImpl,
nackImpl=nackImpl,
)
def _onConsumerCancelled(self, consumerTag):
"""Handle notification of Basic.Cancel from broker
:param str consumerTag: tag of consumer cancelled by broker
"""
channelContext = self._channelContextInstance
channelContext.pendingEvents.append(amqp_consumer.ConsumerCancellation(consumerTag))
channelContext.consumerSet.discard(consumerTag)
def _onMessageReturn(self, message):
"""Handle unroutable message returned by broker
NOTE: this may happen regardless of RabbitMQ-specific puback mode; however,
if it's going to happen in puback mode, RabbitMQ guarantees that it will
take palce *before* Basic.Ack
:param haigha.message.Message message:
"""
g_log.warning("Message returned: %.255s", message)
self._channelContextInstance.returnedMessages.append(self._makeReturnedMessage(message))
@classmethod
def _makeReturnedMessage(cls, haighaMessage):
"""
:param haigha.message.Message haighaMessage: haigha message returned via
Basic.Return
:rtype: nta.utils.amqp.messages.ReturnedMessage
"""
info = haighaMessage.return_info
methodInfo = amqp_messages.MessageReturnInfo(
replyCode=info["reply_code"],
replyText=info["reply_text"],
exchange=info["exchange"],
routingKey=info["routing_key"],
)
return amqp_messages.ReturnedMessage(
body=cls._decodeMessageBody(haighaMessage.body),
properties=cls._makeBasicProperties(haighaMessage.properties),
methodInfo=methodInfo,
)
@staticmethod
def _amqpErrorArgsFromCloseInfo(closeInfo):
"""
:param dict closeInfo: channel or connection close_info from Haigha
:returns: a dict with property names compatible with _AmqpErrorBase
constructor args
"""
return dict(
code=closeInfo["reply_code"],
text=closeInfo["reply_text"],
classId=closeInfo["class_id"],
methodId=closeInfo["method_id"],
)
def _onConnectionClosed(self):
try:
if self._connection is None:
# Failure during connection setup
raise amqp_exceptions.AmqpConnectionError(code=0, text="connection setup failed", classId=0, methodId=0)
closeInfo = self._connection.close_info
if self._userInitiatedClosing:
if closeInfo["reply_code"] != 0:
raise amqp_exceptions.AmqpConnectionError(**self._amqpErrorArgsFromCloseInfo(closeInfo))
else:
raise amqp_exceptions.AmqpConnectionError(**self._amqpErrorArgsFromCloseInfo(closeInfo))
finally:
self._connection = None
if self._channelContextInstance is not None:
self._channelContextInstance.reset()
self._channelContextInstance = None
def _onChannelClosed(self, channel):
try:
closeInfo = channel.close_info
if self._userInitiatedClosing:
if closeInfo["reply_code"] != 0:
raise amqp_exceptions.AmqpChannelError(**self._amqpErrorArgsFromCloseInfo(closeInfo))
else:
raise amqp_exceptions.AmqpChannelError(**self._amqpErrorArgsFromCloseInfo(closeInfo))
finally:
if self._channelContextInstance is not None:
self._channelContextInstance.reset()
self._channelContextInstance = None
import json
import os
import time
from haigha.connections.rabbit_connection import RabbitConnection
from haigha.message import Message
connection = RabbitConnection(
user=os.getenv('RABBITMQ_USER'),
password=os.getenv('RABBITMQ_PASS'),
vhost='/',
host=os.getenv('RABBITMQ_HOSTS', 'localhost').split(',')[0],
heartbeat=None,
debug=True)
ch = connection.channel()
ch.exchange.declare('cronq', 'direct')
ch.queue.declare('cronq_jobs', auto_delete=False)
ch.queue.declare('cronq_results', auto_delete=False)
ch.queue.bind('cronq_jobs', 'cronq', 'cronq_jobs')
ch.queue.bind('cronq_results', 'cronq', 'cronq_results')
while True:
print 'publish'
cmd = {
"cmd": "sleep 1",
"job_id": 1024,
"name": "[TEST] A test job",
"run_id": "1234"
}
ch.basic.publish(
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(transport = 'gevent', host = host, port = port, vhost = vhost, user = user, password = password)
gevent.spawn(self.loop)
self._channel = self._conn.channel()
class RabbitClient(object):
"""
Wraps features defined in the rabbit max architecture
resources specs defines rules to check exchanges and queues
in order to declare them or delete on cleanups
* Internal queues definitions are marked as native, to avoid deletion
* Exchanges and queues that need to be created always are marked as global
"""
__client_properties__ = {
'library': 'haigha',
'library-version': pkg_resources.require('haigha')[0].version
}
resource_specs = {
'exchanges': [
{'name': 'conversations', 'spec': 'conversations', 'type': 'topic'},
{'name': 'twitter', 'spec': 'twitter', 'type': 'fanout'},
{'name': 'syncacl', 'spec': 'syncacl', 'type': 'fanout'},
{'name': 'activity', 'spec': 'activity', 'type': 'topic'},
{'name': 'user_subscribe', 'spec': '.*?.subscribe', 'type': 'fanout', 'global': False},
{'name': 'user_publish', 'spec': '.*?.publish', 'type': 'topic', 'global': False},
{'name': 'default', 'spec': '^$', 'type': 'direct', 'native': True},
{'name': 'internal', 'spec': 'amq\..*?', 'type': '.*', 'native': True},
],
'queues': [
{'name': 'dynamic', 'spec': 'amq\..*?', 'native': True, 'durable': False, 'auto_delete': True},
{'name': 'messages', 'spec': 'messages', 'bindings': [
{'exchange': 'conversations', 'routing_key': '*.messages'}
]},
{'name': 'push', 'spec': 'push', 'bindings': [
{'exchange': 'conversations', 'routing_key': '*.notifications'},
{'exchange': 'activity', 'routing_key': '#'}
]},
{'name': 'twitter', 'spec': 'twitter', 'bindings': [
{'exchange': 'twitter'}
]},
{'name': 'syncacl', 'spec': 'syncacl', 'bindings': [
{'exchange': 'syncacl'}
]},
{'name': 'tweety_restart', 'spec': 'tweety_restart'},
{'name': 'twitter', 'spec': 'twitter'},
]
}
def __init__(self, url, declare=False, user=None, client_properties={}, transport='socket'):
self.__client_properties__.update(client_properties)
self.transport = transport
# Fallback to socket transport if gevent not available
if self.transport == 'gevent' and not gevent_available:
self.transport = 'socket'
self.connect(url)
if declare:
self.declare()
self.exchange_specs_by_name = {spec['name']: spec for spec in self.resource_specs['exchanges']}
self.queue_specs_by_name = {spec['name']: spec for spec in self.resource_specs['queues']}
# Wrapper to interact with conversations
self.conversations = RabbitConversations(self)
self.activity = RabbitActivity(self)
self.management = RabbitManagement(self, 'http://{}:15672/api'.format(self.host), self.vhost_url, self.user, self.password)
if user is not None:
self.bind(user)
def spec_by_name(self, type, name):
for spec in self.resource_specs[type]:
if spec['name'] == name:
return spec
return None
def connect(self, url):
"""
Connect to rabbitmq and create a channel
"""
parts = re.search(r'amqp://(\w+):(\w+)@([^\:]+)\:(\d+)\/(.*)\/?', url).groups()
self.user, self.password, self.host, self.port, self.vhost_url = parts
self.vhost = self.vhost_url.replace('%2F', '/')
params = dict(
user=self.user, password=self.password,
vhost=self.vhost, host=self.host, transport=self.transport
)
if self.transport == 'gevent':
params['close_cb'] = self._connection_closed_cb,
self.connection = RabbitConnection(**params)
self.ch = self.connection.channel()
if self.transport == 'gevent':
self._message_pump_greenlet = gevent.spawn(self._message_pump_greenthread)
self.ch.add_close_listener(self._channel_closed_cb)
def _message_pump_greenthread(self):
try:
while self.connection is not None:
# Pump
self.connection.read_frames()
# Yield to other greenlets so they don't starve
gevent.sleep()
finally:
return
def _channel_closed_cb(self, ch):
self.ch = None
self.connection.close()
return
def _connection_closed_cb(self):
self.connection = None
def disconnect(self):
"""
Disconnecto from rabbitmq
"""
self.connection.close()
def declare(self):
"""
Create all defined exchanges and queues on rabbit.
Ignore those marked as internal or not global
"""
for exchange in self.resource_specs['exchanges']:
if exchange.get('global', True) and not exchange.get('native', False):
self.ch.exchange.declare(
exchange=exchange['name'],
type=exchange['type'],
durable=True,
auto_delete=False
)
for queue in self.resource_specs['queues']:
if queue.get('global', True) and not queue.get('native', False):
self.ch.queue.declare(
queue=queue['name'],
durable=True,
auto_delete=False
)
for binding in queue.get('bindings', []):
self.ch.queue.bind(
queue=queue['name'],
exchange=binding['exchange'],
routing_key=binding.get('routing_key', ''),
)
def bind(self, username):
"""
Declare a dynamic queue to consume user messages
and set active user
"""
self.user = username
self.queue, mc, cc = self.ch.queue.declare(exclusive=True)
self.ch.queue.bind(self.queue, self.user_subscribe_exchange(username))
def create_users(self, usernames):
"""
Batch create user exchanges
"""
for username in usernames:
self.create_user(username)
def create_user(self, username, create_exchanges=True):
"""
Creates user exchanges and internal binding
"""
if create_exchanges:
self.ch.exchange.declare(
exchange=self.user_publish_exchange(username),
type=self.exchange_specs_by_name['user_publish']['type'],
durable=True,
auto_delete=False
)
self.ch.exchange.declare(
exchange=self.user_subscribe_exchange(username),
type=self.exchange_specs_by_name['user_subscribe']['type'],
durable=True,
auto_delete=False
)
self.ch.exchange.bind(
exchange=self.user_subscribe_exchange(username),
source=self.user_publish_exchange(username),
routing_key='internal',
)
def delete_user(self, username):
self.ch.exchange.delete(self.user_publish_exchange(username))
self.ch.exchange.delete(self.user_subscribe_exchange(username))
def user_publish_exchange(self, username):
"""
Returns name of exchange used to send messages to rabbit
"""
return '{}.publish'.format(username)
def user_subscribe_exchange(self, username):
"""
Returns name of exchange used to broadcast messages for this user to all
the consumers identified with this username
"""
return '{}.subscribe'.format(username)
def send(self, exchange, message, routing_key=''):
body = message if isinstance(message, basestring) else json.dumps(message)
message = Message(body)
self.ch.publish(message, exchange, routing_key=routing_key)
def send_internal(self, message):
self.send(self.user_publish_exchange(self.user), message, routing_key='internal')
def get_all(self, queue=None, retry=False):
queue_name = self.queue if queue is None else queue
messages = []
message_obj = True
tries = 1 if not retry else -1
while not(tries == 0 or messages != []):
while message_obj is not None:
message_obj = self.get(queue_name)
if message_obj is not None:
tries = 1
try:
message = (json.loads(str(message_obj.body)), message_obj)
except ValueError:
message = (message_obj.body, message_obj)
messages.append(message)
tries -= 1
message_obj = True
return messages
def get(self, queue_name):
return self.ch.basic.get(queue_name)
class SynchronousAmqpClient(object):
"""Synchronous (blocking) AMQP client abstraction that exposes AMQP
functionality presently needed by nta.utils and products. New AMQP
functionality will be exposed as the need arises.
This class provides a consistent AMQP client API to the rest of the products
regardless of the underlying implementation. This helps avoid/minimize changes
to the higher-level code when we need to swap out the underlying client
implementation.
This class is NOT the place for higher-level constructs: see
message_bus_connector module for an example of higher-level functionality
built on top of this class.
NOTE: this implementation is completely synchronous and MUST NOT expose
callbacks (callbaks lead to complexities, such as an opportunity for
unintended/unsupported recursion)
NOTE: The `no*` parameters, such as `noLocal` and `noAck`, are artificats
of the AMQP protocol specification. The author of SynchronousAmqpClient chose
to preserve those semantics to facilitate better correlation with AMQP
documentation.
"""
# Correlations between names of BasicProperties attributes and Haigha's
# message property names.
#
# The table consists of the tollowing columns
# BasicProperty attribute name
# Corresponding Haigha property name
# Value conversion function from BasicProperty to Haigha
# Value conversion function from Haigha to BasicProperty
#
_asIs = lambda x: x
_PROPERTY_CORRELATIONS = (
("contentType", "content_type", _asIs, _asIs),
("contentEncoding", "content_encoding", _asIs, _asIs),
("headers", "application_headers", _asIs, _asIs),
("deliveryMode", "delivery_mode", _asIs, _asIs),
("priority", "priority", _asIs, _asIs),
("correlationId", "correlation_id", _asIs, _asIs),
("replyTo", "reply_to", _asIs, _asIs),
("expiration", "expiration", _asIs, _asIs),
("messageId", "message_id", _asIs, _asIs),
("timestamp", "timestamp", datetime.utcfromtimestamp,
epochFromNaiveUTCDatetime),
("messageType", "type", _asIs, _asIs),
("userId", "user_id", _asIs, _asIs),
("appId", "app_id", _asIs, _asIs),
("clusterId", "cluster_id", _asIs, _asIs),
)
# haigha returns body as bytearray, but our current users of the interface
# assume they are getting str or bytes
_decodeMessageBody = str
# NOTE: RabbitMQ release 3.5.5, changed the server's default heartbeat timeout
# from 580 to 60, causing frequent dropped connections on our blocking
# transport. So, we restore the longer timeout by passing a bigger value
# to the broker during connection tuning.
_DEFAULT_HEARTBEAT_TIMEOUT_SEC = 600
def __init__(self, connectionParams=None, channelConfigCb=None):
"""
NOTE: Connection establishment may be performed in the scope of the
constructor or on demand, depending on the underlying implementation
:param nta.utils.amqp.connection.ConnectionParams connectionParams:
parameters for connecting to AMQP broker;
[default=default params for RabbitMQ broker on localhost]
:param channelConfigCb: An optional callback function that will be
called whenever a new AMQP Channel is being brought up
:type channelConfigCb: None or callable with the signature
channelConfigCb(SynchronousAmqpClient)
"""
self._channelConfigCb = channelConfigCb
# Holds _ChannelContext when channel is created; we create the channel on
# demand. The implementation accesses this member via the
# `_liveChannelContext` property getter when it's desirable to bring up the
# channel on-on demand. When it's undesirable to bring up the channel, the
# implementation interacts directly with this member, which will be None
# when we don't have a channel.
self._channelContextInstance = None
# Set to True when user calls close, so we know to not raise an exception
# from our _on*Closed callback methods
self._userInitiatedClosing = False
# Instantiate underlying connection object
params = (connectionParams if connectionParams is not None else
amqp_connection.ConnectionParams())
# NOTE: we could get a `close_cb` call from RabbitConnection constructor, so
# prepare for it by initializing `self._connection`
self._connection = None
self._connection = RabbitConnection(
transport="socket",
sock_opts={(socket.IPPROTO_TCP, socket.TCP_NODELAY): 1},
synchronous=True,
close_cb=self._onConnectionClosed,
user=params.credentials.username,
password=params.credentials.password,
vhost=params.vhost,
host=params.host,
port=params.port,
heartbeat=self._DEFAULT_HEARTBEAT_TIMEOUT_SEC,
logger=g_log)
def __repr__(self):
# NOTE: we don't use the _liveChannelContext property getter in order to
# avoid creation of channel here
return "%s(channelContext=%r)" % (self.__class__.__name__,
self._channelContextInstance)
@property
def _liveChannelContext(self):
"""NOTE: Creates channel on demand"""
if self._channelContextInstance is None:
self._channelContextInstance = _ChannelContext(
self._connection.channel(synchronous=True))
try:
self._channelContextInstance.channel.add_close_listener(
self._onChannelClosed)
self._channelContextInstance.channel.basic.set_return_listener(
self._onMessageReturn)
if self._channelConfigCb is not None:
self._channelConfigCb(self)
except Exception: # pylint: disable=W0703
g_log.exception("Channel configuration failed")
try:
# Preserve the original exception
raise
finally:
# Close channel and reset channel context
try:
if self._channelContextInstance is not None:
self._channelContextInstance.channel.close(
disconnect=True)
except Exception: # pylint: disable=W0703
# Suppress the secondary exception from cleanup
g_log.exception(
"Channel closing Failed following configuration failure"
)
finally:
self._channelContextInstance.reset()
self._channelContextInstance = None
return self._channelContextInstance
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def isOpen(self):
return self._connection is not None and not self._connection.closed
def close(self):
"""Gracefully close client"""
self._userInitiatedClosing = True
try:
# NOTE: we check _channelContextInstance directly to avoid creating a
# channel if one doesn't exist
if self._channelContextInstance is not None:
channelContext = self._channelContextInstance
try:
channelContext.channel.close()
except Exception: # pylint: disable=W0703
g_log.exception("Channel close failed")
if self._connection is not None:
try:
self._connection.close(disconnect=True)
except Exception: # pylint: disable=W0703
g_log.exception("Connection close failed")
finally:
self._connection = None
if self._channelContextInstance is not None:
self._channelContextInstance.reset()
self._channelContextInstance = None
def enablePublisherAcks(self):
"""Enable RabbitMQ publisher acknowledgments
:raises nta.utils.amqp.exceptions.UnroutableError: raised when messages
that were sent in non-publisher-acknowledgments mode are returned by
the time the Confirm.Select-Ok is received
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
channelContext = self._liveChannelContext
if channelContext.pubacksSelected:
g_log.warning("enablePublisherAcks: already enabled")
else:
channelContext.channel.confirm.select(nowait=False)
channelContext.pubacksSelected = True
# NOTE: Unroutable messages returned after this will be in the context of
# publisher acknowledgments
self._raiseAndClearIfReturnedMessages()
def publish(self, message, exchange, routingKey, mandatory=False):
""" Publish a message
:param nta.utils.amqp.messages.Message message:
:param str exchange: destination exchange name; "" for default exchange
:param str routingKey: Message routing key
:param bool mandatory: This flag tells the server how to react if the
message cannot be routed to a queue. If this flag is True, the server will
return an unroutable message with a Return method. If this flag is False
the server silently drops the message.
:raises nta.utils.amqp.exceptions.UnroutableError: when in
non-publisher-acknowledgments mode, raised before attempting to publish
given message if unroutable messages had been returned. In
publisher-acknowledgments mode, raised if the given
message is returned as unroutable.
:raises nta.utils.amqp.exceptions.NackError: when the given message is
NACKed by broker while channel is in RabbitMQ publisher-acknowledgments
mode
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
message = HaighaMessage(body=message.body,
**self._makeHaighaPropertiesDict(
message.properties))
channelContext = self._liveChannelContext
if channelContext.pubacksSelected:
# In publisher-acknowledgments mode
assert not channelContext.returnedMessages, (
len(channelContext.returnedMessages),
channelContext.returnedMessages)
pubackState = _PubackState()
channelContext.channel.basic.set_ack_listener(
pubackState.handleAck)
channelContext.channel.basic.set_nack_listener(
pubackState.handleNack)
deliveryTag = channelContext.channel.basic.publish(
message,
exchange=exchange,
routing_key=routingKey,
mandatory=mandatory)
# Wait for ACK or NACK
while not pubackState.ready:
self._connection.read_frames()
try:
((how, responseTag), ) = pubackState.values
except ValueError:
g_log.exception("Error unpacking values=%r",
pubackState.values)
raise
assert responseTag == deliveryTag, ((how, responseTag),
deliveryTag)
if how == _PubackState.NACK:
# Raise NackError with returned message
returnedMessages = channelContext.returnedMessages
channelContext.returnedMessages = []
raise amqp_exceptions.NackError(returnedMessages)
# It was Acked
assert how == _PubackState.ACK, how
# Raise if this message was returned as unroutable
self._raiseAndClearIfReturnedMessages()
else:
# Not in publisher-acknowledgments mode
# Raise if some prior messages were returned as unroutable
self._raiseAndClearIfReturnedMessages()
channelContext.channel.basic.publish(message,
exchange=exchange,
routing_key=routingKey,
mandatory=mandatory)
def requestQoS(self,
prefetchSize=0,
prefetchCount=0,
entireConnection=False):
"""This method requests a specific quality of service. The QoS can be
specified for the current channel or for all channels on the connection. The
particular properties and semantics of a qos method always depend on the
content class semantics
:param int prefetchSize: The client can request that messages be sent in
advance so that when the client finishes processing a message, the
following message is already held locally, rather than needing to be sent
down the channel. Prefetching gives a performance improvement. This field
specifies the prefetch window size in octets. The server will send a
message in advance if it is equal to or smaller in size than the available
prefetch size (and also falls into other prefetch limits). May be set to
zero, meaning "no specific limit", although other prefetch limits may
still apply. The prefetchsize is ignored if the no-ack option is set.
:param int prefetchCount: Specifies a prefetch window in terms of whole
messages. This field may be used in combination with the prefetch-size
field; a message will only be sent in advance if both prefetch windows
(and those at the channel and connection level) allow it. The
prefetch-count is ignored if the no-ack option is set.
:param bool entireConnection: By default the QoS settings apply to the
current channel only. If this field is set, they are applied to the entire
connection.
:raises AmqpChannelError:
"""
self._liveChannelContext.channel.basic.qos(
prefetch_size=prefetchSize,
prefetch_count=prefetchCount,
is_global=entireConnection)
def createConsumer(self,
queue,
noLocal=False,
noAck=False,
exclusive=False):
"""This method asks the server to start a "consumer", which is a transient
request for messages from a specific queue. Consumers last as long as the
channel they were declared on, or until the client or broker cancels them.
See `Consumer.cancel()`
Use `getNextEvent()` to retrieve consumed messages and other events.
:param str queue: name of the queue to consume from
:param bool noLocal: If true, the server will not send messages to the
connection that published them.
:param bool noAck: if true, the broker will not expect messages to be ACKed
:param bool exclusive: Request exclusive consumer access, meaning only this
consumer can access the queue
:returns: consumer context
:rtype: nta.utils.amqp.consumer.Consumer
:raises nta.utils.amqp.exceptions.AmqpChannelError:
:raises nta.utils.amqp.exceptions.AmqpConnectionError:
"""
consumerTag = self._makeConsumerTag()
channelContext = self._liveChannelContext
channelContext.channel.basic.consume(
queue,
consumer=self._onMessageDelivery,
consumer_tag=consumerTag,
no_local=noLocal,
no_ack=noAck,
exclusive=exclusive,
cancel_cb=self._onConsumerCancelled,
nowait=False)
channelContext.consumerSet.add(consumerTag)
consumer = amqp_consumer.Consumer(
tag=consumerTag,
queue=queue,
cancelImpl=channelContext.cancelConsumer)
g_log.info(
"Created consumer=%r; queue=%r, noLocal=%r, noAck=%r, exclusive=%r",
consumer, queue, noLocal, noAck, exclusive)
return consumer
def hasEvent(self):
"""Check if there are events ready for consumption. See `getNextEvent()`.
:returns: True if there is at least one event ready for consumption, in
which case `getNextEvent()` may be called once without blocking.
:rtype: bool
"""
channelContext = self._channelContextInstance
return bool(channelContext is not None
and channelContext.pendingEvents)
def getNextEvent(self):
"""Get next event, blocking if there isn't one yet. See `hasEvent()`. You
MUST have an active consumer (`createConsumer`) or other event source before
calling this method.
An event may be an object of one of the following classes:
nta.utils.amqp.messages.ConsumerMessage
nta.utils.amqp.consumer.ConsumerCancellation
:returns: the next event when it becomes available
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
# We expect the context to be set up already
channelContext = self._channelContextInstance
while not channelContext.pendingEvents:
self._connection.read_frames()
return channelContext.pendingEvents.popleft()
def readEvents(self):
"""Generator that yields results of `getNextEvent()`"""
while True:
yield self.getNextEvent()
def getOneMessage(self, queue, noAck=False):
"""This is the polling, less-performant method of getting a message. This
method provides a direct access to the messages in a queue using a
synchronous dialogue that is designed for specific types of application
where synchronous functionality is more important than performance.
:param str queue: name of the queue to get a message from
:param bool noAck: if true, the broker will not be expecting an Ack
:returns: A PolledMessage object if there was a message in queue; None if
there was no message in queue.
:rtype: PolledMessage or None
:raises AmqpChannelError:
"""
channelContext = self._liveChannelContext
consumer = _CallbackSink()
channelContext.channel.basic.get(queue,
consumer=consumer,
no_ack=noAck)
while not consumer.ready:
self._connection.read_frames()
try:
((message, ), ) = consumer.values
except ValueError:
g_log.exception("Error unpacking values=%r", consumer.values)
raise
if message is not None:
ackImpl = channelContext.ack if not noAck else None
nackImpl = channelContext.nack if not noAck else None
message = self._makePolledMessage(message, ackImpl, nackImpl)
return message
@classmethod
def _makePolledMessage(cls, haighaMessage, ackImpl, nackImpl):
"""Make PolledMessage from haigha message retrieved via Basic.Get
:param haigha.message.Message haighaMessage: haigha message retrieved via
Basic.Get
:param ackImpl: callable for acking the message that has the following
signature: ackImpl(deliveryTag, multiple=False); or None
:param nackImpl: callable for nacking the message that has the following
signature: nackImpl(deliveryTag, requeue); or None
:rtype: nta.utils.amqp.messages.PolledMessage
"""
info = haighaMessage.delivery_info
methodInfo = amqp_messages.MessageGetInfo(
deliveryTag=info["delivery_tag"],
redelivered=bool(info["redelivered"]),
exchange=info["exchange"],
routingKey=info["routing_key"],
messageCount=info["message_count"])
return amqp_messages.PolledMessage(
body=cls._decodeMessageBody(haighaMessage.body),
properties=cls._makeBasicProperties(haighaMessage.properties),
methodInfo=methodInfo,
ackImpl=ackImpl,
nackImpl=nackImpl)
def recover(self, requeue=False):
"""This method asks the server to redeliver all unacknowledged messages on a
specified channel. Zero or more messages may be redelivered.
NOTE: RabbitMQ does not currently support recovering with requeue=False
:param bool requeue: If false, the message will be redelivered to the
original recipient. If true, the server will attempt to requeue the
message, potentially then delivering it to an alternative subscriber.
"""
self._liveChannelContext.channel.basic.recover(requeue=requeue)
def ackAll(self):
"""Acknowledge all unacknowledged messages on current channel instance.
Acknowledgemets related to specific messages are performed via the message's
own `ack()` method.
NOTE: messages received prior to the AmqpChannelError exception cannot be
acknowledged since that exception occurs after the AMQP channel is closed
"""
self._channelContextInstance.ack(deliveryTag=0, multiple=True)
def nackAll(self, requeue=False):
"""Reject all outstanding (unacknowledged) messages on current channel
instance.
Rejections related to specific messages are performed via the message's
own `nack()` method.
NOTE: has no impact on messages received prior to the AmqpChannelError
exception since AmqpChannelError is raised after the AMQP channel is closed
:param bool requeue: If requeue is true, the server will attempt to requeue
the messages. If requeue is false or the requeue attempt fails the
messages are discarded or dead-lettered
"""
self._channelContextInstance.nack(deliveryTag=0,
multiple=True,
requeue=requeue)
def declareExchange(self,
exchange,
exchangeType,
passive=False,
durable=False,
autoDelete=False,
arguments=None):
"""Declare an exchange
:param str exchange: name of the exchange
:param str exchangeType: type of the exchange
:param bool passive: If True, the server will reply with Declare-Ok if the
exchange already exists with the same name, and raise an error if not. The
client can use this to check whether an exchange exists without modifying
the server state. When set, all other method fields except name are
ignored. Arguments are compared for semantic equivalence.
:param bool durable: If True when creating a new exchange, the exchange will
be marked as durable. Durable exchanges remain active when a server
restarts. Non-durable exchanges (transient exchanges) are purged if/when a
server restarts.
:param bool autoDelete: If true, the exchange is deleted when all queues
have finished using it (RabbitMQ-specific).
:param dict arguments: custom key/value pairs for the exchange
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.exchange.declare(
exchange,
exchangeType,
passive=passive,
durable=durable,
auto_delete=autoDelete,
arguments=arguments or dict(),
nowait=False)
def deleteExchange(self, exchange, ifUnused=False):
"""Delete an exchange
:param str exchange: exchange to be deleted
:param bool ifUnused: If True, the server will only delete the exchange if
it has no queue bindings. If the exchange has queue bindings the server
does not delete it but raises a channel exception instead.
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.exchange.delete(exchange,
if_unused=ifUnused,
nowait=False)
def declareQueue(self,
queue,
passive=False,
durable=False,
exclusive=False,
autoDelete=False,
arguments=None):
"""Declare a queue
:param str queue: name of queue
:param bool passive: If True, the server will reply with Declare-Ok if the
queue already exists with the same name, and raise an error if not. The
client can use this to check whether a queue exists without modifying the
server state. When set, all other method fields except name are ignored.
Arguments are compared for semantic equivalence.
:param bool durable: If true when creating a new queue, the queue will be
marked as durable. Durable queues remain active when a server restarts.
Non-durable queues (transient queues) are purged if/when a server
restarts. Note that durable queues do not necessarily hold persistent
messages, although it does not make sense to send persistent messages to a
transient queue.
:param bool exclusive: Exclusive queues may only be accessed by the current
connection, and are deleted when that connection closes. Passive
declaration of an exclusive queue by other connections are not allowed.
:param bool autoDelete: If true, the queue is deleted when all consumers
have finished using it. The last consumer can be cancelled either
explicitly or because its channel is closed. If there was no consumer ever
on the queue, it won't be deleted. Applications can explicitly delete
auto-delete queues using the Delete method as normal. (RabbitMQ-specific)
:param dict arguments: A set of key/value pairs for the declaration. The
syntax and semantics of these arguments depends on the server
implementation.
:rtype: nta.utils.amqp.queue.QueueDeclarationResult
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
queue, messageCount, consumerCount = (
self._liveChannelContext.channel.queue.declare(
queue,
passive=passive,
durable=durable,
exclusive=exclusive,
auto_delete=autoDelete,
arguments=arguments or dict(),
nowait=False))
return amqp_queue.QueueDeclarationResult(queue, messageCount,
consumerCount)
def deleteQueue(self, queue, ifUnused=False, ifEmpty=False):
"""Delete a queue. When a queue is deleted any pending messages are sent to
a dead-letter queue if this is defined in the server configuration, and all
consumers on the queue are cancelled
:param str queue: name of the queue to delete
:param bool ifUnused: If true, the server will only delete the queue if it
has no consumers. If the queue has consumers the server does does not
delete it but raises a channel exception instead.
:param bool ifEmpty: If true, the server will only delete the queue if it
has no messages
:returns: number of messages deleted
:rtype: int
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
return self._liveChannelContext.channel.queue.delete(
queue, if_unused=ifUnused, if_empty=ifEmpty, nowait=False)
def purgeQueue(self, queue):
"""Remove all messages from a queue which are not awaiting acknowledgment.
:param str queue: name of the queue to purge
:returns: number of messages purged
:rtype: int
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
return self._liveChannelContext.channel.queue.purge(queue,
nowait=False)
def bindQueue(self, queue, exchange, routingKey, arguments=None):
"""Bind a queue to an exchange
:param str queue: name of the queue to bind
:param str exchange: name of the exchange to bind to
:param str routingKey: Specifies the routing key for the binding. The
routing key is used for routing messages depending on the exchange
configuration. Not all exchanges use a routing key refer to the
specific exchange documentation. If the queue name is empty, the server
uses the last queue declared on the channel. If the routing key is also
empty, the server uses this queue name for the routing key as well. If the
queue name is provided but the routing key is empty, the server does the
binding with that empty routing key. The meaning of empty routing keys
depends on the exchange implementation.
:param dict arguments: A set of key/value pairs for the binding. The syntax
and semantics of these arguments depends on the exchange class and server
implemenetation.
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.queue.bind(queue=queue,
exchange=exchange,
routing_key=routingKey,
arguments=arguments
or dict(),
nowait=False)
def unbindQueue(self, queue, exchange, routingKey, arguments=None):
"""Unbind a queue fro an exchange
:param str queue: name of the queue to unbind
:param str exchange: name of the exchange to unbind from
:param str routingKey: the routing key of the binding to unbind
:param dict arguments: Specifies the arguments of the binding to unbind
:raises nta.utils.amqp.exceptions.AmqpChannelError:
"""
self._liveChannelContext.channel.queue.unbind(queue=queue,
exchange=exchange,
routing_key=routingKey,
arguments=arguments
or dict())
def _raiseAndClearIfReturnedMessages(self):
"""If returned messages are present, raise UnroutableError and clear
returned messages holding buffer
:raises nta.utils.amqp.exceptions.UnroutableError: if returned messages
are present
"""
channelContext = self._channelContextInstance
if channelContext and channelContext.returnedMessages:
messages = channelContext.returnedMessages
channelContext.returnedMessages = []
raise amqp_exceptions.UnroutableError(messages)
@classmethod
def _makeHaighaPropertiesDict(cls, BasicProperties):
"""Marshal BasicProperties into the haigha properties dict
:param nta.utils.amqp.messages.BasicProperties BasicProperties:
:returns: dict of Properties expected by Haigha's publish method
:rtype: dict
"""
props = dict()
for attrName, propName, clientToHaigha, _ in cls._PROPERTY_CORRELATIONS:
value = getattr(BasicProperties, attrName)
# Add only those properties that have concrete values
if value is not None:
props[propName] = clientToHaigha(value)
return props
@classmethod
def _makeBasicProperties(cls, haighaProperties):
attributes = dict()
for attrName, propName, _, haighaToClient in cls._PROPERTY_CORRELATIONS:
value = haighaProperties.get(propName)
if value is not None:
value = haighaToClient(value)
attributes[attrName] = value
return amqp_messages.BasicProperties(**attributes)
def _makeConsumerTag(self):
channelContext = self._liveChannelContext
tag = "channel-%d-%d" % (channelContext.channel.channel_id,
channelContext.nextConsumerTag)
channelContext.nextConsumerTag += 1
return tag
def _onMessageDelivery(self, message):
"""Handle consumer message from Basic.Deliver
:param haigha.message.Message message:
"""
channelContext = self._channelContextInstance
g_log.debug("Consumer message received: %.255s", message)
channelContext.pendingEvents.append(
self._makeConsumerMessage(message, channelContext.ack,
channelContext.nack))
@classmethod
def _makeConsumerMessage(cls, haighaMessage, ackImpl, nackImpl):
"""Make ConsumerMessage from haigha message received via Basic.Deliver
:param haigha.message.Message haighaMessage: haigha message received via
Basic.Deliver
:param ackImpl: callable for acking the message that has the following
signature: ackImpl(deliveryTag, multiple=False); or None
:param nackImpl: callable for nacking the message that has the following
signature: nackImpl(deliveryTag, requeue); or None
:rtype: ConsumerMessage
"""
info = haighaMessage.delivery_info
methodInfo = amqp_messages.MessageDeliveryInfo(
consumerTag=info["consumer_tag"],
deliveryTag=info["delivery_tag"],
redelivered=bool(info["redelivered"]),
exchange=info["exchange"],
routingKey=info["routing_key"])
return amqp_messages.ConsumerMessage(
body=cls._decodeMessageBody(haighaMessage.body),
properties=cls._makeBasicProperties(haighaMessage.properties),
methodInfo=methodInfo,
ackImpl=ackImpl,
nackImpl=nackImpl)
def _onConsumerCancelled(self, consumerTag):
"""Handle notification of Basic.Cancel from broker
:param str consumerTag: tag of consumer cancelled by broker
"""
channelContext = self._channelContextInstance
channelContext.pendingEvents.append(
amqp_consumer.ConsumerCancellation(consumerTag))
channelContext.consumerSet.discard(consumerTag)
def _onMessageReturn(self, message):
"""Handle unroutable message returned by broker
NOTE: this may happen regardless of RabbitMQ-specific puback mode; however,
if it's going to happen in puback mode, RabbitMQ guarantees that it will
take palce *before* Basic.Ack
:param haigha.message.Message message:
"""
g_log.warning("Message returned: %.255s", message)
self._channelContextInstance.returnedMessages.append(
self._makeReturnedMessage(message))
@classmethod
def _makeReturnedMessage(cls, haighaMessage):
"""
:param haigha.message.Message haighaMessage: haigha message returned via
Basic.Return
:rtype: nta.utils.amqp.messages.ReturnedMessage
"""
info = haighaMessage.return_info
methodInfo = amqp_messages.MessageReturnInfo(
replyCode=info["reply_code"],
replyText=info["reply_text"],
exchange=info["exchange"],
routingKey=info["routing_key"])
return amqp_messages.ReturnedMessage(
body=cls._decodeMessageBody(haighaMessage.body),
properties=cls._makeBasicProperties(haighaMessage.properties),
methodInfo=methodInfo)
@staticmethod
def _amqpErrorArgsFromCloseInfo(closeInfo):
"""
:param dict closeInfo: channel or connection close_info from Haigha
:returns: a dict with property names compatible with _AmqpErrorBase
constructor args
"""
return dict(code=closeInfo["reply_code"],
text=closeInfo["reply_text"],
classId=closeInfo["class_id"],
methodId=closeInfo["method_id"])
def _onConnectionClosed(self):
try:
if self._connection is None:
# Failure during connection setup
raise amqp_exceptions.AmqpConnectionError(
code=0,
text="connection setup failed",
classId=0,
methodId=0)
closeInfo = self._connection.close_info
if self._userInitiatedClosing:
if closeInfo["reply_code"] != 0:
raise amqp_exceptions.AmqpConnectionError(
**self._amqpErrorArgsFromCloseInfo(closeInfo))
else:
raise amqp_exceptions.AmqpConnectionError(
**self._amqpErrorArgsFromCloseInfo(closeInfo))
finally:
self._connection = None
if self._channelContextInstance is not None:
self._channelContextInstance.reset()
self._channelContextInstance = None
def _onChannelClosed(self, channel):
try:
closeInfo = channel.close_info
if self._userInitiatedClosing:
if closeInfo["reply_code"] != 0:
raise amqp_exceptions.AmqpChannelError(
**self._amqpErrorArgsFromCloseInfo(closeInfo))
else:
raise amqp_exceptions.AmqpChannelError(
**self._amqpErrorArgsFromCloseInfo(closeInfo))
finally:
if self._channelContextInstance is not None:
self._channelContextInstance.reset()
self._channelContextInstance = None
import json
import os
import time
from haigha.connections.rabbit_connection import RabbitConnection
from haigha.message import Message
connection = RabbitConnection(user=os.getenv('RABBITMQ_USER'),
password=os.getenv('RABBITMQ_PASS'),
vhost='/',
host=os.getenv('RABBITMQ_HOSTS',
'localhost').split(',')[0],
heartbeat=None,
debug=True)
ch = connection.channel()
ch.exchange.declare('cronq', 'direct')
ch.queue.declare('cronq_jobs', auto_delete=False)
ch.queue.declare('cronq_results', auto_delete=False)
ch.queue.bind('cronq_jobs', 'cronq', 'cronq_jobs')
ch.queue.bind('cronq_results', 'cronq', 'cronq_results')
while True:
print 'publish'
cmd = {
"cmd": "sleep 1",
"job_id": 1024,
"name": "[TEST] A test job",
"run_id": "1234"
}
ch.basic.publish(Message(json.dumps(cmd),
class QueueConnection(object):
"""A wrapper around an AMQP connection for ease of publishing
Simple instantiation:
RABBITMQ_URL = os.getenv('RABBITMQ_URL', 'amqp://guest@localhost')
queueconnection = QueueConnection(RABBITMQ_URL)
Publishing requires knowing the exchange, routing_key, (headers) and
the string body:
# We don't really use headers
# look up AMQP stuff if you are interested
queueconnection.publish(
exchange='exchange',
routing_key='routing_key',
headers={},
body='String Body'
)
Usually we want to publish JSON so there is a method that will serialize
a dict for you:
# We don't really use headers
# look up AMQP stuff if you are interested
queueconnection.publish_json(
exchange='exchange',
routing_key='routing_key',
headers={},
body={'key': 'value'}
)
All of these are asynchronous publishes to the server, if you want to make
sure that these at least make it to the server you can turn on publisher
confirmations. Sync calls take longer so they aren't on by default. Turning
them on is done by passing `confirm=True` to the constructor.
RABBITMQ_URL = os.getenv('RABBITMQ_URL', 'amqp://guest@localhost')
queueconnection = QueueConnection(RABBITMQ_URL, confirm=True)
Now publishes will return a bool of whether the publish succeeded.
# We don't really use headers
# look up AMQP stuff if you are interested
succeeded = queueconnection.publish(
exchange='exchange',
routing_key='routing_key',
headers={},
body='String Body'
)
Without `confirm=True` the return value of publish will only indicate if
the message was written to a connection successfully.
"""
def __init__(self, url=None, confirm=False, **kwargs):
if url is None:
url = Config.RABBITMQ_URL
hosts, user, password, vhost, port, heartbeat = parse_url(url)
if heartbeat is None:
heartbeat = kwargs.get('heartbeat', None)
self._connection_hosts = hosts
self._connection_user = user
self._connection_password = password
self._connection_path = vhost
self._connection_port = port
self._connection_heartbeat = heartbeat
self._connection_name = self._generate_connection_name()
self._connection_params = urlparse.urlparse(url)
self._connect_attempt_delay = 0.1
self._get_next_host = create_host_factory(self._connection_hosts)
self._confirm = confirm
self._logger = logger
self._create_connection()
self._acked = False
self._last_confirmed_message = None
def _create_connection(self):
"Tries to create a connection, returns True on success"
self._connection = None
self._last_confirmed_message = None
host = self._get_next_host()
self._logger.debug('Trying to connect to {}'.format(host))
try:
self._connection = RabbitConnection(
host=host,
port=self._connection_port,
user=self._connection_user,
password=self._connection_password,
vhost=self._connection_path,
close_cb=self._close_cb,
heartbeat=self._connection_heartbeat,
client_properties={
'connection_name': self._connection_name,
},
)
except socket.error as exc:
self._logger.error('Error connecting to rabbitmq {}'.format(exc))
return False
self._channel = self._connection.channel()
if self._confirm:
self._channel.confirm.select(nowait=False)
self._channel.basic.set_ack_listener(self._ack)
self._logger.debug('Connected to {}'.format(host))
return True
def _generate_connection_name(self):
random_generator = random.SystemRandom()
random_string = ''.join([
random_generator.choice(string.ascii_lowercase) for i in xrange(10)
])
return '{0}-{1}-{2}'.format(
socket.gethostname(),
os.getpid(),
random_string,
)
def _try_to_connect(self, attempts=3):
"""Try to connect handling retries"""
for _ in range(attempts):
if self.is_connected():
return True
else:
self._logger.debug("attempt to create connection")
self._create_connection()
time.sleep(self._connect_attempt_delay)
def _ack(self, message_id):
self._last_confirmed_message = message_id
def is_connected(self):
if self._connection is None or self._channel is None:
return False
return True
def _close_cb(self):
if self._connection is None:
reason = 'unknown'
else:
reason = self._connection.close_info['reply_text']
self._logger.info('Disconnected because: {}'.format(reason))
self._connection = None
self._channel = None
def publish(self,
exchange,
routing_key,
headers,
body,
connect_attempts=3):
"""Publish a messages to AMQP
Returns a bool about the success of the publish. If `confirm=True` True
means it reached the AMQP server. If `confirm=False` it means that it
was able to be written to a connection but makes no guarantee about the
message making it to the server.
"""
if not self.is_connected():
self._try_to_connect(attempts=connect_attempts)
if self._connection is None or self._channel is None:
self._logger.error('Tried to publish without an AMQP connection')
return False
msg_number = self._channel.basic.publish(Message(
body, application_headers=headers),
exchange=exchange,
routing_key=routing_key)
if self._confirm:
if self.is_connected():
self._connection.read_frames()
return self._last_confirmed_message == msg_number
else:
return False
return True
def publish_json(self, exchange, routing_key, headers, body):
data = json.dumps(body)
return self.publish(exchange, routing_key, headers, data)
def close(self):
self._connection.close()
def __init__(self, host, port, vhost, user, password):
"""Create a new Server
"""
self._conn = RabbitConnection(host = host, port = port, vhost = vhost, user = user, password = password)
self._channel = self._conn.channel()
def run(args):
host = os.getenv('AMQP_HOST', 'localhost')
user = os.getenv('AMQP_USER', 'guest')
password = os.getenv('AMQP_PASS', 'guest')
vhost = os.getenv('AMQP_VHOST', '/')
connection = RabbitConnection(
user=user, password=password,
vhost=vhost, host=host,
heartbeat=None, debug=True)
config = get_config(args.config)
ch = connection.channel()
for exchange in config.get('exchanges'):
print 'Declaring exchange:'
print '\t', exchange
try:
ch.exchange.declare(
exchange['name'],
exchange['type'],
durable=exchange['durable'],
auto_delete=exchange['auto_delete'],
arguments=exchange.get('arguments', {}),
)
except AttributeError as ae:
print ae
print 'Declare conflict! This must be fixed manually'
sys.exit(1)
for queue in config.get('queues'):
print 'Declaring queue:'
print '\t', queue
try:
ch.queue.declare(
queue['name'],
auto_delete=queue['auto_delete'],
durable=queue['durable'],
arguments=queue.get('arguments', {}),
)
except AttributeError as ae:
print ae
print 'Declare conflict! This must be fixed manually'
sys.exit(1)
for binding in queue['bindings']:
print 'Binding queue:'
print '\t', binding
try:
if binding.get('binding_key'):
ch.queue.bind(
queue['name'],
binding['exchange'],
binding['binding_key'],
)
else:
ch.queue.bind(
queue['name'],
binding['exchange']
)
except AttributeError:
print 'Declare conflict! This must be fixed manually'
sys.exit(1)
connection.close()
class QueueConnection(object):
"""A wrapper around an AMQP connection for ease of publishing
Simple instantiation:
RABBITMQ_URL = os.getenv('RABBITMQ_URL', 'amqp://guest@localhost')
queueconnection = QueueConnection(RABBITMQ_URL)
Publishing requires knowing the exchange, routing_key, (headers) and
the string body:
# We don't really use headers
# look up AMQP stuff if you are interested
queueconnection.publish(
exchange='exchange',
routing_key='routing_key',
headers={},
body='String Body'
)
Usually we want to publish JSON so there is a method that will serialize
a dict for you:
# We don't really use headers
# look up AMQP stuff if you are interested
queueconnection.publish_json(
exchange='exchange',
routing_key='routing_key',
headers={},
body={'key': 'value'}
)
All of these are asynchronous publishes to the server, if you want to make
sure that these at least make it to the server you can turn on publisher
confirmations. Sync calls take longer so they aren't on by default. Turning
them on is done by passing `confirm=True` to the constructor.
RABBITMQ_URL = os.getenv('RABBITMQ_URL', 'amqp://guest@localhost')
queueconnection = QueueConnection(RABBITMQ_URL, confirm=True)
Now publishes will return a bool of whether the publish succeeded.
# We don't really use headers
# look up AMQP stuff if you are interested
succeeded = queueconnection.publish(
exchange='exchange',
routing_key='routing_key',
headers={},
body='String Body'
)
Without `confirm=True` the return value of publish will only indicate if
the message was written to a connection successfully.
"""
def __init__(self, url=None, confirm=False, **kwargs):
if url is None:
url = Config.RABBITMQ_URL
hosts, user, password, vhost, port, heartbeat = parse_url(url)
if heartbeat is None:
heartbeat = kwargs.get('heartbeat', None)
self._connection_hosts = hosts
self._connection_user = user
self._connection_password = password
self._connection_path = vhost
self._connection_port = port
self._connection_heartbeat = heartbeat
self._connection_name = self._generate_connection_name()
self._connection_params = urlparse.urlparse(url)
self._connect_attempt_delay = 0.1
self._get_next_host = create_host_factory(self._connection_hosts)
self._confirm = confirm
self._logger = logger
self._create_connection()
self._acked = False
self._last_confirmed_message = None
def _create_connection(self):
"Tries to create a connection, returns True on success"
self._connection = None
self._last_confirmed_message = None
host = self._get_next_host()
self._logger.debug('Trying to connect to {}'.format(host))
try:
self._connection = RabbitConnection(
host=host,
port=self._connection_port,
user=self._connection_user,
password=self._connection_password,
vhost=self._connection_path,
close_cb=self._close_cb,
heartbeat=self._connection_heartbeat,
client_properties={
'connection_name': self._connection_name,
},
)
except socket.error as exc:
self._logger.error('Error connecting to rabbitmq {}'.format(exc))
return False
self._channel = self._connection.channel()
if self._confirm:
self._channel.confirm.select(nowait=False)
self._channel.basic.set_ack_listener(self._ack)
self._logger.debug('Connected to {}'.format(host))
return True
def _generate_connection_name(self):
random_generator = random.SystemRandom()
random_string = ''.join([random_generator.choice(string.ascii_lowercase)
for i in xrange(10)])
return '{0}-{1}-{2}'.format(
socket.gethostname(),
os.getpid(),
random_string,
)
def _try_to_connect(self, attempts=3):
"""Try to connect handling retries"""
for _ in range(attempts):
if self.is_connected():
return True
else:
self._logger.debug("attempt to create connection")
self._create_connection()
time.sleep(self._connect_attempt_delay)
def _ack(self, message_id):
self._last_confirmed_message = message_id
def is_connected(self):
if self._connection is None or self._channel is None:
return False
return True
def _close_cb(self):
if self._connection is None:
reason = 'unknown'
else:
reason = self._connection.close_info['reply_text']
self._logger.info('Disconnected because: {}'.format(reason))
self._connection = None
self._channel = None
def publish(self,
exchange,
routing_key,
headers,
body,
connect_attempts=3):
"""Publish a messages to AMQP
Returns a bool about the success of the publish. If `confirm=True` True
means it reached the AMQP server. If `confirm=False` it means that it
was able to be written to a connection but makes no guarantee about the
message making it to the server.
"""
if not self.is_connected():
self._try_to_connect(attempts=connect_attempts)
if self._connection is None or self._channel is None:
self._logger.error('Tried to publish without an AMQP connection')
return False
msg_number = self._channel.basic.publish(
Message(body, application_headers=headers),
exchange=exchange,
routing_key=routing_key
)
if self._confirm:
if self.is_connected():
self._connection.read_frames()
return self._last_confirmed_message == msg_number
else:
return False
return True
def publish_json(self, exchange, routing_key, headers, body):
data = json.dumps(body)
return self.publish(exchange, routing_key, headers, data)
def close(self):
self._connection.close()
