def connect(self):
"""Initialize a connection with the AMQP server. If it fails, retry at
most <max_retries> times.
"""
# Patch gevent
monkey.patch_all()
retries = 0
self.connection = None
# Try to connect at most <max_retries> times
while self.max_retries == 0 or retries < self.max_retries:
retries += 1
# Pick up a host
host = self.hosts.pop()
self.hosts.insert(0, host)
try:
self.connection = RabbitConnection(logger=logger,
debug=True,
user='******',
password='******',
vhost='/',
host=host,
heartbeat=None,
sock_opts=sock_opts,
transport='gevent')
break
except socket.error as e:
self.connection = None
logger.error('Cannot connect to host %s: %s', host, e)
if retries > 2 * len(self.hosts):
sleep(1)
if not self.connection:
raise AMQPConnectionError(
'Aborting after %d connection failures!' % self.max_retries)
logger.info('Successfully connected to host: %s', host)
logger.info('Creating channel')
self.channel = self.connection.channel()
if self.confirms:
self._confirm_select()
if self.unacked:
self._resend_unacked_messages()
if self.consumers:
for queue, callback in self.consumers.items():
self.basic_consume(queue, callback)
def _connect(self,):
self.logger.msg('Connecting to RabbitMQ')
#: Connetion to Rabbit MQ Server
try:
self.connection = RabbitConnection(transport='gevent',
close_cb=self.connection_shutdown,
logger=self.logger)
except socket.error:
self.logger.msg('RabbitMQ Connection Failed with socket.error')
return connect_fail_event
# except Exception as e:
# self.logger.msg('RabbitMQ Connection Failed', error=str(e))
# return
#: Start Rabbit MQ Message Pump
gevent.spawn(self.message_pump)
#: log and signal
self.logger.msg(connect_event)
connect_signal.send(self)
#: Load Services
if self.services_callback is None:
self.logger.msg('Unable to Load Warren Services')
else:
s = self.services_callback()
self.load_services(s)
def connect(self, retries=0):
if retries > self.max_retries:
logger.error("Aborting after %s retries", retries - 1)
raise AMQPConnectionError(
'Aborting after %d connection failures.' % (retries - 1))
return
# Pick up a host
host = self.hosts.pop()
self.hosts.insert(0, host)
#Patch gevent
monkey.patch_all()
try:
self.connection = \
RabbitConnection(logger=logger, debug=True,
user='******', password='******',
vhost='/', host=host,
heartbeat=None,
sock_opts=sock_opts,
transport='gevent')
except socket.error as e:
logger.error('Cannot connect to host %s: %s', host, e)
if retries > 2 * len(self.hosts):
sleep(1)
return self.connect(retries + 1)
logger.info('Successfully connected to host: %s', host)
logger.info('Creating channel')
self.channel = self.connection.channel()
if self.confirms:
self._confirm_select()
if self.unacked:
self._resend_unacked_messages()
if self.consumers:
for queue, callback in self.consumers.items():
self.basic_consume(queue, callback)
def connect(self):
"""Initialize a connection with the AMQP server. If it fails, retry at
most <max_retries> times.
"""
# Patch gevent
monkey.patch_all()
retries = 0
self.connection = None
# Try to connect at most <max_retries> times
while self.max_retries == 0 or retries < self.max_retries:
retries += 1
# Pick up a host
host = self.hosts.pop()
self.hosts.insert(0, host)
try:
self.connection = RabbitConnection(logger=logger, debug=True,
user='******',
password='******',
vhost='/', host=host,
heartbeat=None,
sock_opts=sock_opts,
transport='gevent')
break
except socket.error as e:
self.connection = None
logger.error('Cannot connect to host %s: %s', host, e)
if retries > 2 * len(self.hosts):
sleep(1)
if not self.connection:
raise AMQPConnectionError('Aborting after %d connection failures!'
% self.max_retries)
logger.info('Successfully connected to host: %s', host)
logger.info('Creating channel')
self.channel = self.connection.channel()
if self.confirms:
self._confirm_select()
if self.unacked:
self._resend_unacked_messages()
if self.consumers:
for queue, callback in self.consumers.items():
self.basic_consume(queue, callback)
def connect(self, retries=0):
if retries > self.max_retries:
logger.error("Aborting after %s retries", retries - 1)
raise AMQPConnectionError('Aborting after %d connection failures.'
% (retries - 1))
return
# Pick up a host
host = self.hosts.pop()
self.hosts.insert(0, host)
#Patch gevent
monkey.patch_all()
try:
self.connection = \
RabbitConnection(logger=logger, debug=True,
user='******', password='******',
vhost='/', host=host,
heartbeat=None,
sock_opts=sock_opts,
transport='gevent')
except socket.error as e:
logger.error('Cannot connect to host %s: %s', host, e)
if retries > 2 * len(self.hosts):
sleep(1)
return self.connect(retries + 1)
logger.info('Successfully connected to host: %s', host)
logger.info('Creating channel')
self.channel = self.connection.channel()
if self.confirms:
self._confirm_select()
if self.unacked:
self._resend_unacked_messages()
if self.consumers:
for queue, callback in self.consumers.items():
self.basic_consume(queue, callback)
class AMQPHaighaClient():
def __init__(self,
hosts=settings.AMQP_HOSTS,
max_retries=30,
confirms=True,
confirm_buffer=200):
self.hosts = hosts
shuffle(self.hosts)
self.max_retries = max_retries
self.confirms = confirms
self.confirm_buffer = confirm_buffer
self.connection = None
self.channel = None
self.consumers = {}
self.unacked = OrderedDict()
def connect(self):
"""Initialize a connection with the AMQP server. If it fails, retry at
most <max_retries> times.
"""
# Patch gevent
monkey.patch_all()
retries = 0
self.connection = None
# Try to connect at most <max_retries> times
while self.max_retries == 0 or retries < self.max_retries:
retries += 1
# Pick up a host
host = self.hosts.pop()
self.hosts.insert(0, host)
try:
self.connection = RabbitConnection(logger=logger,
debug=True,
user='******',
password='******',
vhost='/',
host=host,
heartbeat=None,
sock_opts=sock_opts,
transport='gevent')
break
except socket.error as e:
self.connection = None
logger.error('Cannot connect to host %s: %s', host, e)
if retries > 2 * len(self.hosts):
sleep(1)
if not self.connection:
raise AMQPConnectionError(
'Aborting after %d connection failures!' % self.max_retries)
logger.info('Successfully connected to host: %s', host)
logger.info('Creating channel')
self.channel = self.connection.channel()
if self.confirms:
self._confirm_select()
if self.unacked:
self._resend_unacked_messages()
if self.consumers:
for queue, callback in self.consumers.items():
self.basic_consume(queue, callback)
def exchange_declare(self, exchange, type='direct'):
"""Declare an exchange
@type exchange_name: string
@param exchange_name: name of the exchange
@type exchange_type: string
@param exhange_type: one of 'direct', 'topic', 'fanout'
"""
logger.info('Declaring %s exchange: %s', type, exchange)
self.channel.exchange.declare(exchange,
type,
auto_delete=False,
durable=True)
def queue_declare(self,
queue,
exclusive=False,
mirrored=True,
mirrored_nodes='all',
ttl=None):
"""Declare a queue
@type queue: string
@param queue: name of the queue
@param mirrored: whether the queue will be mirrored to other brokers
@param mirrored_nodes: the policy for the mirrored queue.
Available policies:
- 'all': The queue is mirrored to all nodes and the
master node is the one to which the client is
connected
- a list of nodes. The queue will be mirrored only to
the specified nodes, and the master will be the
first node in the list. Node names must be provided
and not host IP. example: [node1@rabbit,node2@rabbit]
@type ttl: int
@param tll: Queue TTL in seconds
"""
logger.info('Declaring queue: %s', queue)
if mirrored:
if mirrored_nodes == 'all':
arguments = {'x-ha-policy': 'all'}
elif isinstance(mirrored_nodes, list):
arguments = {
'x-ha-policy': 'nodes',
'x-ha-policy-params': mirrored_nodes
}
else:
raise AttributeError
else:
arguments = {}
if ttl is not None:
arguments['x-expires'] = ttl * 1000
self.channel.queue.declare(queue,
durable=True,
exclusive=exclusive,
auto_delete=False,
arguments=arguments)
def queue_bind(self, queue, exchange, routing_key):
logger.info('Binding queue %s to exchange %s with key %s', queue,
exchange, routing_key)
self.channel.queue.bind(queue=queue,
exchange=exchange,
routing_key=routing_key)
def _confirm_select(self):
logger.info('Setting channel to confirm mode')
self.channel.confirm.select()
self.channel.basic.set_ack_listener(self._ack_received)
self.channel.basic.set_nack_listener(self._nack_received)
@reconnect_decorator
def basic_publish(self, exchange, routing_key, body):
msg = Message(body, delivery_mode=2)
mid = self.channel.basic.publish(msg, exchange, routing_key)
if self.confirms:
self.unacked[mid] = (exchange, routing_key, body)
if len(self.unacked) > self.confirm_buffer:
self.get_confirms()
logger.debug('Published message %s with id %s', body, mid)
@reconnect_decorator
def get_confirms(self):
self.connection.read_frames()
@reconnect_decorator
def _resend_unacked_messages(self):
msgs = self.unacked.values()
self.unacked.clear()
for exchange, routing_key, body in msgs:
logger.debug('Resending message %s', body)
self.basic_publish(exchange, routing_key, body)
@reconnect_decorator
def _ack_received(self, mid):
print mid
logger.debug('Received ACK for message with id %s', mid)
self.unacked.pop(mid)
@reconnect_decorator
def _nack_received(self, mid):
logger.error('Received NACK for message with id %s. Retrying.', mid)
(exchange, routing_key, body) = self.unacked[mid]
self.basic_publish(exchange, routing_key, body)
def basic_consume(self, queue, callback, no_ack=False, exclusive=False):
"""Consume from a queue.
@type queue: string or list of strings
@param queue: the name or list of names from the queues to consume
@type callback: function
@param callback: the callback function to run when a message arrives
"""
self.consumers[queue] = callback
self.channel.basic.consume(queue,
consumer=callback,
no_ack=no_ack,
exclusive=exclusive)
@reconnect_decorator
def basic_wait(self):
"""Wait for messages from the queues declared by basic_consume.
This function will block until a message arrives from the queues that
have been declared with basic_consume. If the optional arguments
'promise' is given, only messages for this promise will be delivered.
"""
self.connection.read_frames()
gevent.sleep(0)
@reconnect_decorator
def basic_get(self, queue, no_ack=False):
self.channel.basic.get(queue, no_ack=no_ack)
@reconnect_decorator
def basic_ack(self, message):
delivery_tag = message.delivery_info['delivery_tag']
self.channel.basic.ack(delivery_tag)
@reconnect_decorator
def basic_nack(self, message):
delivery_tag = message.delivery_info['delivery_tag']
self.channel.basic.ack(delivery_tag)
def close(self):
try:
if self.confirms:
while self.unacked:
print self.unacked
self.get_confirms()
self.channel.close()
close_info = self.channel.close_info
logger.info('Successfully closed channel. Info: %s', close_info)
self.connection.close()
except socket.error as e:
logger.error('Connection closed while closing connection:%s', e)
def queue_delete(self, queue, if_unused=True, if_empty=True):
self.channel.queue.delete(queue, if_unused, if_empty)
def exchange_delete(self, exchange, if_unused=True):
self.channel.exchange.delete(exchange, if_unused)
def basic_class(self):
pass
class AMQPHaighaClient():
def __init__(self, hosts=settings.AMQP_HOSTS, max_retries=30,
confirms=True, confirm_buffer=200):
self.hosts = hosts
shuffle(self.hosts)
self.max_retries = max_retries
self.confirms = confirms
self.confirm_buffer = confirm_buffer
self.connection = None
self.channel = None
self.consumers = {}
self.unacked = OrderedDict()
def connect(self):
"""Initialize a connection with the AMQP server. If it fails, retry at
most <max_retries> times.
"""
# Patch gevent
monkey.patch_all()
retries = 0
self.connection = None
# Try to connect at most <max_retries> times
while self.max_retries == 0 or retries < self.max_retries:
retries += 1
# Pick up a host
host = self.hosts.pop()
self.hosts.insert(0, host)
try:
self.connection = RabbitConnection(logger=logger, debug=True,
user='******',
password='******',
vhost='/', host=host,
heartbeat=None,
sock_opts=sock_opts,
transport='gevent')
break
except socket.error as e:
self.connection = None
logger.error('Cannot connect to host %s: %s', host, e)
if retries > 2 * len(self.hosts):
sleep(1)
if not self.connection:
raise AMQPConnectionError('Aborting after %d connection failures!'
% self.max_retries)
logger.info('Successfully connected to host: %s', host)
logger.info('Creating channel')
self.channel = self.connection.channel()
if self.confirms:
self._confirm_select()
if self.unacked:
self._resend_unacked_messages()
if self.consumers:
for queue, callback in self.consumers.items():
self.basic_consume(queue, callback)
def exchange_declare(self, exchange, type='direct'):
"""Declare an exchange
@type exchange_name: string
@param exchange_name: name of the exchange
@type exchange_type: string
@param exhange_type: one of 'direct', 'topic', 'fanout'
"""
logger.info('Declaring %s exchange: %s', type, exchange)
self.channel.exchange.declare(exchange, type,
auto_delete=False, durable=True)
def queue_declare(self, queue, exclusive=False, mirrored=True,
mirrored_nodes='all', ttl=None):
"""Declare a queue
@type queue: string
@param queue: name of the queue
@param mirrored: whether the queue will be mirrored to other brokers
@param mirrored_nodes: the policy for the mirrored queue.
Available policies:
- 'all': The queue is mirrored to all nodes and the
master node is the one to which the client is
connected
- a list of nodes. The queue will be mirrored only to
the specified nodes, and the master will be the
first node in the list. Node names must be provided
and not host IP. example: [node1@rabbit,node2@rabbit]
@type ttl: int
@param tll: Queue TTL in seconds
"""
logger.info('Declaring queue: %s', queue)
if mirrored:
if mirrored_nodes == 'all':
arguments = {'x-ha-policy': 'all'}
elif isinstance(mirrored_nodes, list):
arguments = {'x-ha-policy': 'nodes',
'x-ha-policy-params': mirrored_nodes}
else:
raise AttributeError
else:
arguments = {}
if ttl is not None:
arguments['x-expires'] = ttl * 1000
self.channel.queue.declare(queue, durable=True, exclusive=exclusive,
auto_delete=False, arguments=arguments)
def queue_bind(self, queue, exchange, routing_key):
logger.info('Binding queue %s to exchange %s with key %s', queue,
exchange, routing_key)
self.channel.queue.bind(queue=queue, exchange=exchange,
routing_key=routing_key)
def _confirm_select(self):
logger.info('Setting channel to confirm mode')
self.channel.confirm.select()
self.channel.basic.set_ack_listener(self._ack_received)
self.channel.basic.set_nack_listener(self._nack_received)
@reconnect_decorator
def basic_publish(self, exchange, routing_key, body):
msg = Message(body, delivery_mode=2)
mid = self.channel.basic.publish(msg, exchange, routing_key)
if self.confirms:
self.unacked[mid] = (exchange, routing_key, body)
if len(self.unacked) > self.confirm_buffer:
self.get_confirms()
logger.debug('Published message %s with id %s', body, mid)
@reconnect_decorator
def get_confirms(self):
self.connection.read_frames()
@reconnect_decorator
def _resend_unacked_messages(self):
msgs = self.unacked.values()
self.unacked.clear()
for exchange, routing_key, body in msgs:
logger.debug('Resending message %s', body)
self.basic_publish(exchange, routing_key, body)
@reconnect_decorator
def _ack_received(self, mid):
print mid
logger.debug('Received ACK for message with id %s', mid)
self.unacked.pop(mid)
@reconnect_decorator
def _nack_received(self, mid):
logger.error('Received NACK for message with id %s. Retrying.', mid)
(exchange, routing_key, body) = self.unacked[mid]
self.basic_publish(exchange, routing_key, body)
def basic_consume(self, queue, callback, no_ack=False, exclusive=False):
"""Consume from a queue.
@type queue: string or list of strings
@param queue: the name or list of names from the queues to consume
@type callback: function
@param callback: the callback function to run when a message arrives
"""
self.consumers[queue] = callback
self.channel.basic.consume(queue, consumer=callback, no_ack=no_ack,
exclusive=exclusive)
@reconnect_decorator
def basic_wait(self):
"""Wait for messages from the queues declared by basic_consume.
This function will block until a message arrives from the queues that
have been declared with basic_consume. If the optional arguments
'promise' is given, only messages for this promise will be delivered.
"""
self.connection.read_frames()
gevent.sleep(0)
@reconnect_decorator
def basic_get(self, queue, no_ack=False):
self.channel.basic.get(queue, no_ack=no_ack)
@reconnect_decorator
def basic_ack(self, message):
delivery_tag = message.delivery_info['delivery_tag']
self.channel.basic.ack(delivery_tag)
@reconnect_decorator
def basic_nack(self, message):
delivery_tag = message.delivery_info['delivery_tag']
self.channel.basic.ack(delivery_tag)
def close(self):
try:
if self.confirms:
while self.unacked:
print self.unacked
self.get_confirms()
self.channel.close()
close_info = self.channel.close_info
logger.info('Successfully closed channel. Info: %s', close_info)
self.connection.close()
except socket.error as e:
logger.error('Connection closed while closing connection:%s', e)
def queue_delete(self, queue, if_unused=True, if_empty=True):
self.channel.queue.delete(queue, if_unused, if_empty)
def exchange_delete(self, exchange, if_unused=True):
self.channel.exchange.delete(exchange, if_unused)
def basic_class(self):
pass
class WRabbit(object):
def __init__(self, **options):
#: logging facility
self.logger = get_logger()
#: keep track of retry attempts
self.retry_attempt = 0
#: retry limit
self.retry_limit = options.get('retry_limit', 30)
#: retry interval
self.retry_interval = options.get('retry_interval', 3)
#: rabbit mq connection
self.connection = None
#: rabbit mq channnels
self.channels = dict()
#: services callback
self.services_callback = None
#: keep track of whether or not we are retrying
self.retry_mode = False
def connect(self,):
self.retry_mode = True
while self.retry_attempt <= self.retry_limit:
#: log and signal
if self.retry_attempt > 0:
self.logger.msg(retry_event,
retry_attempt=self.retry_attempt
)
retry_signal.send(self)
#: attemp reconnect
status = self._connect()
#: if unsuccessful, sleep for retry interval
if status == connect_fail_event or self.connection is None:
self.retry_attempt += 1
gevent.sleep(self.retry_interval)
else:
self.retry_attempt = 0
break
self.retry_mode = False
def _connect(self,):
self.logger.msg('Connecting to RabbitMQ')
#: Connetion to Rabbit MQ Server
try:
self.connection = RabbitConnection(transport='gevent',
close_cb=self.connection_shutdown,
logger=self.logger)
except socket.error:
self.logger.msg('RabbitMQ Connection Failed with socket.error')
return connect_fail_event
# except Exception as e:
# self.logger.msg('RabbitMQ Connection Failed', error=str(e))
# return
#: Start Rabbit MQ Message Pump
gevent.spawn(self.message_pump)
#: log and signal
self.logger.msg(connect_event)
connect_signal.send(self)
#: Load Services
if self.services_callback is None:
self.logger.msg('Unable to Load Warren Services')
else:
s = self.services_callback()
self.load_services(s)
def services_function(self, callback):
'''doc
'''
self.services_callback = callback
return callback
def load_services(self, services):
'''doc
'''
#: x
for service_id, service_name in services.iteritems():
try:
exchange, routing_key = service_name.split(':')
except ValueError:
exchange = service_name
routing_key = 'default'
self.add_channel('service',
service_id=service_id,
exchange=exchange,
routing_key=routing_key,
service_name=service_name,
)
#: log and signal
self.logger.msg(services_event)
services_signal.send(self)
def message_wrap(self, m):
message = Message(m, application_headers={
#: None at this time
})
return message
def message_pump(self):
self.logger.msg('RabbitMQ Message Pump Starting')
try:
while self.connection is not None:
# Pump
self.connection.read_frames()
# Yield to other greenlets so they don't starve
gevent.sleep()
finally:
self.connection_shutdown()
return
def add_channel(self, channel_type, **options):
'''doc
'''
#: Extract Options
service_id = options.get('service_id')
exchange = options.get('exchange')
routing_key = options.get('routing_key')
#: Setup logging
log = self.logger.bind(
channel_type=channel_type,
service_id=service_id,
exchange=exchange,
routing_key=routing_key,
)
#: Create a channel
try:
channel = self.connection.channel()
except AttributeError:
log.error('Failed RabbitMQ Channel Creation')
return None
#: add channel close call back
channel.add_close_listener(self.channel_shutdown)
#: Configure Channel
channel.exchange.declare(exchange, 'direct',
durable=True,
auto_delete=False,
)
#: Add Channel to Dict of Service Channels
self.channels[service_id] = (channel,
exchange, routing_key)
#: Log and signal Channel Creation
log.info(channel_creation_event, channel_id=channel.channel_id)
channel_creation_signal.send(self)
def channel_shutdown(self, ch):
'''
'''
chdir = str(dir(ch))
#: Log and signal
self.logger.msg(channel_shutdown_event, chdir=chdir)
channel_shutdown_signal.send(self)
def connection_shutdown(self,):
'''Handle RabbitMQ Shutdown
'''
#: reset connection and channels
self.connection = None
self.channels = dict()
#: Log and signal
self.logger.msg('RabbitMQ Connection Shutdown')
shutdown_signal.send(self)
#: retry connection
if self.retry_mode == False:
gevent.spawn(self.connect)
