def __setup_rackhd_style_amqp(self):
"""
Need to make exchanges and named queus to make this
look like a RackHD instance amqp.
"""
# A freshly spun up on-demand docker likes to say it's there, but will
# then reset the connection. So, catch that scenario w/ a few retries.
con = None
done_time = time.time() + 30.0
while con is None:
con = Connection(hostname=self.host, port=self.ssl_port, ssl=False)
try:
con.connect()
except Exception as ex:
if time.time() > done_time:
raise ex
con = None
if con is None:
time.sleep(0.1)
on_task = self.__assure_exchange(con, 'on.task', 'topic')
self.__assure_named_queue(con, on_task, 'ipmi.command.sel.result')
self.__assure_named_queue(con, on_task, 'ipmi.command.sdr.result')
self.__assure_named_queue(con, on_task, 'ipmi.command.chassis.result')
on_events = self.__assure_exchange(con, 'on.events', 'topic')
self.__assure_named_queue(con, on_events, 'graph.finished')
self.__assure_named_queue(con, on_events, 'polleralert.sel.updated',
'#')
def init_adversary():
global __username, __my_type, __connection, __global_exchange, __producer, __backchannel, __backch_producer, __adv_exchange, __adv_queue #, __my_queue
__username = '******'
__my_type = 'adversary'
#adversary has broadcast-only access to the regular exchange
__connection = Connection('amqp://')
__connection.connect()
__global_exchange = Exchange('broadcast',
type='fanout',
durable=False,
delivery_mode=1)
__global_exchange.maybe_bind(__connection)
__producer = __connection.Producer(__connection)
__backchannel = Connection('amqp://')
__backchannel.connect()
__adv_exchange = Exchange('adversary', durable=False, delivery_mode=1)
__adv_exchange.maybe_bind(__backchannel)
__backch_producer = __backchannel.Producer(__backchannel)
__adv_queue = Queue('adversary',
exchange=__adv_exchange,
routing_key='adversary',
durable=False)
__adv_queue = __adv_queue(__backchannel)
__adv_queue.declare()
def _connect(self):
result = None
if not self.connected():
self.logger.info(
"Connect to RabbitMQ Broker %s" % (self.uri))
uri = self.uri
parsed_url = urlparse(uri)
uri = 'amqp://%s/%s' % (parsed_url.netloc, parsed_url.path)
result = Connection(uri)
try:
result.connect()
self.logger.info("Connected to AMQP Broker.")
except Exception as e:
result.release()
self.logger.error(
"Connection failure to %s: %s" % (uri, e))
else:
self.logger.debug("Allready connected")
return result
def __setup_rackhd_style_amqp(self):
"""
Need to make exchanges and named queus to make this
look like a RackHD instance amqp.
"""
# A freshly spun up on-demand docker likes to say it's there, but will
# then reset the connection. So, catch that scenario w/ a few retries.
con = None
done_time = time.time() + 30.0
while con is None:
con = Connection(hostname=self.host, port=self.ssl_port, ssl=False)
try:
con.connect()
except Exception as ex:
if time.time() > done_time:
raise ex
con = None
if con is None:
time.sleep(0.1)
on_task = self.__assure_exchange(con, 'on.task', 'topic')
self.__assure_named_queue(con, on_task, 'ipmi.command.sel.result')
self.__assure_named_queue(con, on_task, 'ipmi.command.sdr.result')
self.__assure_named_queue(con, on_task, 'ipmi.command.chassis.result')
on_events = self.__assure_exchange(con, 'on.events', 'topic')
self.__assure_named_queue(con, on_events, 'graph.finished')
self.__assure_named_queue(con, on_events, 'polleralert.sel.updated', '#')
class AmqpBundle(object):
def __init__(self):
self.logger = logging.getLogger("amqp_bundle")
self.config_mapping = {
"amqp": {
"uri": None,
"autodisconnect": False,
"heartbeat": 0,
}
}
self.autodisconnect = None
self.event_listeners = [
(ConfigurationReadyEvent, self.configuration_ready),
(KernelShutdownEvent, self.kernel_shutdown),
]
self.connection = Connection()
self.injection_bindings = {Connection: self.connection}
def kernel_shutdown(self, event):
if self.autodisconnect:
self.logger.info("Disconnecting...")
self.connection.release()
self.logger.info("Disconnected")
def configuration_ready(self, event):
c = event.configuration.amqp
self.autodisconnect = c.autodisconnect
# First connect the topic manager to avoid lose messages
self.connection.__init__(c.uri, heartbeat=c.heartbeat)
self.connection.connect()
self.logger.info("Connected")
class KombuSocket(object):
def __init__(self, host, port, username, password, virtual_host, exchange,
routing_key, durable, passive, exchange_type):
# create connection
self.connection = Connection(hostname=host,
port=port,
userid=username,
password=password,
virtual_host=virtual_host)
# create exchange
self.exchange = Exchange(exchange, type=exchange_type, durable=durable)
self.exchange.passive = passive
# other publishing params
self.routing_key = routing_key
def sendall(self, data):
with producers[self.connection].acquire(block=True) as producer:
producer.publish(data,
routing_key=self.routing_key,
exchange=self.exchange,
declare=[self.exchange])
def connect(self):
self.connection.connect()
def close(self):
try:
self.connection.close()
except Exception:
pass
def run(rabbit_url):
print rabbit_url
conn = Connection(rabbit_url)
conn.ensure_connection()
conn.connect()
channel = conn.channel()
exchange = Exchange(config.EXCHANGE_NAME, type='direct')
producer = Producer(exchange=exchange,
channel=channel,
routing_key=config.ROUTING_KEY)
queue = Queue(name=config.QUEUE_NAME,
exchange=exchange,
routing_key=config.ROUTING_KEY)
queue.maybe_bind(conn)
queue.declare()
index = 0
while True:
try:
time.sleep(1)
print 'producer'
index += 1
producer.publish("send message -- %s" % index)
except socket.timeout:
pass
def test_missing_master_name_transport_option(self):
connection = Connection('sentinel://localhost:65534/', )
with patch('redis.sentinel.Sentinel'), \
pytest.raises(ValueError) as excinfo:
connection.connect()
expected = "'master_name' transport option must be specified."
assert expected == excinfo.value.args[0]
class RabbitMQPublisher(object):
def __init__(self, source_id, network_id, model_key=None, run_key=None):
broker_url = get_broker_url(model_key)
self.conn = Connection(broker_url)
self.conn.connect()
self.channel = 'oa-{source_id}-{network_id}-{model_key}'.format(
source_id=source_id,
network_id=network_id,
model_key=model_key or environ.get(constants.MODEL_KEY)
)
if run_key is not None:
self.channel += '-{}'.format(run_key)
self.producer = self.conn.Producer(serializer='json')
# set up the Exchange, Queue, and Producer
media_exchange = Exchange('media', 'direct', durable=True)
video_queue = Queue('video', exchange=media_exchange, routing_key='video')
# publish updates
def publish(self, payload):
self.producer.publish(self.channel, json.dumps(payload))
def close(self):
self.conn.release()
class Test(unittest.TestCase):
def setUp(self):
self.__conn = Connection(transport=Transport)
self.__conn.connect()
self.__exchange = Exchange('test_exchange')
self.__routing_key = 'test.routing'
self.__queue = Queue('test_queue',
self.__exchange,
routing_key=self.__routing_key)
def test_register_player(self):
message, exchange, routing_key = register_player(
111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_FIND_PLAYER,
body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_pgpool_disconnected(self):
message, exchange, routing_key = acknowledgement_pgpool_disconnected(
111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_DISCONNECT_PGPOOL,
body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_pgpool_connected(self):
message, exchange, routing_key = acknowledgement_pgpool_connected(
111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_CONNECT_PGPOOL,
body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_master_disconnected(self):
message, exchange, routing_key = acknowledgement_master_disconnected(
111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_STOP_REPLICATION,
body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_master_connected(self):
message, exchange, routing_key = acknowledgement_master_connected(
111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_START_REPLICATION,
body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def inner(*args, **kwargs):
connection = Connection(settings.BROKER_URL)
connection.connect()
queue = connection.SimpleQueue(settings.QUEUE_NAME)
res = func(*args, **kwargs, queue=queue)
connection.release()
return res
def test_simple_queueing(self):
conn = Connection('sqlalchemy+sqlite:///:memory:')
conn.connect()
channel = conn.channel()
self.assertEqual(channel.queue_cls.__table__.name, 'kombu_queue')
self.assertEqual(channel.message_cls.__table__.name, 'kombu_message')
channel._put('celery', 'DATA')
assert channel._get('celery') == 'DATA'
def test_amqp_connect(config):
connect_url = 'amqp://{user}:{pwd}@{host}:{port}//' \
.format(user=config.get("user", ''),
pwd=config.get('pwd', ''),
host=config.get('host', ''),
port=int(config.get('port', 5671)))
conn = Connection(connect_url)
conn.connect()
class _AMQPServerWrapper(object):
def __init__(self, amqp_url, logs):
self.__logs = logs
self.__amqp_url = amqp_url
self.__monitors = {}
self.__connection = Connection(self.__amqp_url)
self.__connection.connect()
self.__running = True
self.__consumer_gl = gevent.spawn(self.__consumer_greenlet_main)
self.__consumer_gl.greenlet_name = 'amqp-consumer-gl' # allowing flogging to print a nice name
gevent.sleep(0.0)
def __consumer_greenlet_main(self):
gevent.sleep(0)
while self.__running:
try:
self.__connection.drain_events(timeout=0.5)
except Exception as ex: # NOQA: assigned but not used (left in for super-duper-low-level-debug)
# print("was woken because {}".format(ex))
pass
gevent.sleep(0.1) # make -sure- to yield cpu...
# print("---loop")
def stop_greenlet(self):
self.__running = False
@property
def connected(self):
return self.__connection.connected
def create_add_tracker(self, exchange, routing_key, event_cb, queue_name=None):
self.__logs.irl.debug("AMQPServerWrapper: create_add_tracker ex=%s, rk=%s, event_cb=%s",
exchange, routing_key, event_cb)
mon = _KeyedConsumerHandler.get_keyed_consumer(
self.__logs, self.__connection, exchange, routing_key, queue_name, event_cb)
return mon.exchange
def inject(self, exchange, routing_key, payload):
self.__logs.irl.debug("Injecting a test AMQP message: ex=%s, rk=%s, payload=%s", exchange, routing_key, payload)
if not isinstance(exchange, Exchange):
exchange = Exchange(exchange, 'topic')
prod = Producer(self.__connection, exchange=exchange, routing_key=routing_key)
prod.publish(payload)
def test_helper_sync_send_msg(self, exchange, ex_rk, send_rk, payload):
ex = Exchange(exchange, 'topic')
queue = Queue(exchange=ex, routing_key=ex_rk + '.*', exclusive=True, channel=self.__connection)
queue.declare()
prod = Producer(self.__connection, exchange=ex, routing_key=send_rk)
prod.publish(payload)
return queue
def test_helper_sync_recv_msg(self, queue):
for tick in range(10):
msg = queue.get()
if msg is not None:
break
return msg
def health_check_rabbitmq(broker_url):
try:
from kombu import Connection
conn = Connection(broker_url)
conn.connect()
conn.release()
return True, 'Rabbitmq is OK.', ''
except Exception as e:
return False, str(e), traceback.format_exc()
class InteractiveExchange:
'''
A context manager that provides the interactive exchange for publishing
messages.
'''
def __init__(self, task):
'''
Accessing an Interactive Exchange need a task only for the generic info in it,
and so this can be an instantiaton of Interactive()
:param task: A task that provides the exchange name and some logging formatters
'''
self.task = task
def __enter__(self):
# We have a task that we need only for the exchnage name and some logging formatters.
task = self.task
log.debug(f'Connecting to: {current_app.conf.broker_url}')
self.connection = Connection(current_app.conf.broker_url)
# Connection is lazy. Force a connection now.
self.connection.connect()
# Get the connection object for channel construction
#
# This syntax is odd, but we can blame kombu for that.
#
# self.connection is of type: kombu.connection.Connection
#
# and it has an attribute:
#
# self.connection.connection which is of type: kombu.transport.pyamqp.Connection
#
# So sname names, different things. The convoluted life of the
# word "connection". But we do need a kombu.transport.pyamqp.Connection
# object to create a channel. We can't do that on its parent.
c = self.connection.connection
log.debug(f'\tConnection: {task.connection_names(c)[1]}')
# Create a channel on the connection and log it in the RabbitMQ webmonitor format
ch = c.channel()
self.channel = ch
log.debug(f'\tChannel: {task.channel_names(ch)[1]}')
x = Exchange(task.exchange_name, channel=ch)
self.exchange = x
log.debug(f'\tExchange: {x.name}')
return x
def __exit__(self, type, value, traceback): # @ReservedAssignment
self.connection.release()
def test_simple_queueing(self):
conn = Connection('sqlalchemy+sqlite:///:memory:')
conn.connect()
try:
channel = conn.channel()
assert channel.queue_cls.__table__.name == 'kombu_queue'
assert channel.message_cls.__table__.name == 'kombu_message'
channel._put('celery', 'DATA_SIMPLE_QUEUEING')
assert channel._get('celery') == 'DATA_SIMPLE_QUEUEING'
finally:
conn.release()
class CeleryQueue(QueueBase):
""" At some point this will adapt `django-signalqueue` for use
with popular `(dj)celery` platform (but not today).
When this class is done, I will discuss it here. """
def __init__(self, *args, **kwargs):
super(CeleryQueue, self).__init__(*args, **kwargs)
self.celery_queue_name = self.queue_options.pop('celery_queue_name', 'inactive')
self.serializer = self.queue_options.pop('serializer', 'json')
self.compression = self.queue_options.pop('compression', None)
self.kc = Connection(**self.queue_options)
self.kc.connect()
self.qc = self.kc.SimpleQueue(name=self.celery_queue_name)
def ping(self):
return self.kc.connected and not self.qc.channel.closed
def push(self, value):
self.qc.put(value,
compression=self.compression, serializer=None)
def pop(self):
virtual_message = self.qc.get(block=False, timeout=1)
return virtual_message.payload
def count(self):
try:
return self.qc.qsize()
except kombu.exceptions.StdChannelError:
self.qc.queue.declare()
return 0
def clear(self):
self.qc.clear()
def values(self, **kwargs):
return []
def __getitem__(self, idx):
#return self.values().__getitem__(idx)
return ''
def dispatch(self, signal, sender=None, **kwargs):
task_name = "%s:%s" % (signal.regkey, signal.name)
try:
result = tasks[task_name].delay(sender=sender, **kwargs)
except tasks.NotRegistered:
pass
else:
return result
class CeleryQueue(QueueBase):
""" At some point this will adapt `django-signalqueue` for use
with popular `(dj)celery` platform (but not today).
When this class is done, I will discuss it here. """
def __init__(self, *args, **kwargs):
super(CeleryQueue, self).__init__(*args, **kwargs)
self.celery_queue_name = self.queue_options.pop(
'celery_queue_name', 'inactive')
self.serializer = self.queue_options.pop('serializer', 'json')
self.compression = self.queue_options.pop('compression', None)
self.kc = Connection(**self.queue_options)
self.kc.connect()
self.qc = self.kc.SimpleQueue(name=self.celery_queue_name)
def ping(self):
return self.kc.connected and not self.qc.channel.closed
def push(self, value):
self.qc.put(value, compression=self.compression, serializer=None)
def pop(self):
virtual_message = self.qc.get(block=False, timeout=1)
return virtual_message.payload
def count(self):
try:
return self.qc.qsize()
except kombu.exceptions.StdChannelError:
self.qc.queue.declare()
return 0
def clear(self):
self.qc.clear()
def values(self, **kwargs):
return []
def __getitem__(self, idx):
#return self.values().__getitem__(idx)
return ''
def dispatch(self, signal, sender=None, **kwargs):
task_name = "%s:%s" % (signal.regkey, signal.name)
try:
result = tasks[task_name].delay(sender=sender, **kwargs)
except tasks.NotRegistered:
pass
else:
return result
def test_custom_table_names(self):
conn = Connection('sqlalchemy+sqlite:///:memory:',
transport_options={
'queue_tablename': 'my_custom_queue',
'message_tablename': 'my_custom_message'
})
conn.connect()
channel = conn.channel()
self.assertEqual(channel.queue_cls.__table__.name, 'my_custom_queue')
self.assertEqual(channel.message_cls.__table__.name,
'my_custom_message')
channel._put('celery', 'DATA')
assert channel._get('celery') == 'DATA'
class Publisher(object):
def __init__(self, connection_string, exchange, is_active=True):
self._is_active = is_active
self.is_connected = True
if not is_active:
self.is_connected = False
return
self._connection = Connection(connection_string)
self._connections = set([self._connection]) # set of connection for the heartbeat
self._exchange = Exchange(exchange, durable=True, delivry_mode=2, type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
def _get_producer(self):
producer = producers[self._connection].acquire(block=True, timeout=2)
self._connections.add(producer.connection)
return producer
def publish(self, item, contributor):
if not self._is_active:
logging.getLogger(__name__).info('RabbitMQ is not enabled')
return True
with self._get_producer() as producer:
try:
self.is_connected = True
publish = producer.connection.ensure(producer, producer.publish, max_retries=3)
publish(item, exchange=self._exchange, routing_key=contributor, declare=[self._exchange])
logging.getLogger(__name__).info('Publishing message on exchange %s', self._exchange.name)
except:
self.is_connected = False
logging.exception("Impossible to publish message to rabbitmq")
finally:
return self.is_connected
def info(self):
result = {
"is_active": self._is_active,
"is_connected": self.is_connected
}
if not self._is_active:
return result
with self._get_producer() as producer:
res = producer.connection.info()
if 'password' in res:
del res['password']
for key, value in res.items():
result[key] = value
return result
def test_close_survives_connerror(self):
class _CustomError(Exception):
pass
class MyTransport(Transport):
connection_errors = (_CustomError, )
def close_connection(self, connection):
raise _CustomError('foo')
conn = Connection(transport=MyTransport)
conn.connect()
conn.close()
self.assertTrue(conn._closed)
def test_close_survives_connerror(self):
class _CustomError(Exception):
pass
class MyTransport(Transport):
connection_errors = (_CustomError,)
def close_connection(self, connection):
raise _CustomError("foo")
conn = Connection(transport=MyTransport)
conn.connect()
conn.close()
self.assertTrue(conn._closed)
def test_simple_queueing(self):
conn = Connection('sqlalchemy+sqlite:///:memory:')
conn.connect()
channel = conn.channel()
self.assertEqual(
channel.queue_cls.__table__.name,
'kombu_queue'
)
self.assertEqual(
channel.message_cls.__table__.name,
'kombu_message'
)
channel._put('celery', 'DATA')
assert channel._get('celery') == 'DATA'
def connect_to_amqp(sysconfig):
"""
Connect to an AMQP Server, and return the connection and Exchange.
:param sysconfig: The slickqaweb.model.systemConfiguration.amqpSystemConfiguration.AMQPSystemConfiguration instance
to use as the source of information of how to connect.
:return: (connection, exchange) on success, exception on error
"""
assert isinstance(sysconfig, AMQPSystemConfiguration)
configuration = dict()
configuration['AMQP'] = dict()
if hasattr(sysconfig, 'hostname') and sysconfig.hostname is not None:
configuration['AMQP']['hostname'] = sysconfig.hostname
else:
raise AMQPConnectionError(message="No hostname defined for AMQP connection.")
if hasattr(sysconfig, 'port') and sysconfig.port is not None:
configuration['AMQP']['port'] = sysconfig.port
if hasattr(sysconfig, 'username') and sysconfig.username is not None:
configuration['AMQP']['username'] = sysconfig.username
if hasattr(sysconfig, 'password') and sysconfig.password is not None:
configuration['AMQP']['password'] = sysconfig.password
if hasattr(sysconfig, 'virtualHost') and sysconfig.virtualHost is not None:
configuration['AMQP']['virtual host'] = sysconfig.virtualHost
if hasattr(sysconfig, 'exchangeName') and sysconfig.exchangeName is not None:
configuration['AMQP']['exchange'] = sysconfig.exchangeName
else:
raise AMQPConnectionError(message="No exchange defined for AMQP connection.")
logger = logging.getLogger("slickqaweb.amqpcon.connect_to_amqp")
url = str.format("amqp://{hostname}:{port}", **dict(list(configuration['AMQP'].items())))
if 'virtual host' in configuration['AMQP'] and configuration['AMQP']['virtual host'] != '':
url = str.format("{}/{}", url, configuration['AMQP']['virtual host'])
logger.debug("AMQPConnection configured with url %s", url)
exchange = Exchange(configuration['AMQP'].get('exchange', "amqp.topic"), type='topic', durable=True)
logger.debug("AMQPConnection is using exchange %s", exchange)
connection = None
if 'username' in configuration['AMQP'] and 'password' in configuration['AMQP']:
username = configuration['AMQP']['username']
password = configuration['AMQP']['password']
logger.debug("Using username %s and password %s to connect to AMQP Broker", username, password)
connection = Connection(url, userid=username, password=password)
else:
connection = Connection(url)
# typically connection connect on demand, but we want to flush out errors before proceeding
connection.connect()
exchange = exchange(connection)
exchange.declare()
return (connection, exchange)
def check_rabbit_mq():
conn = Connection(settings.AMQP_URL)
for attempt in range(1, 11):
timeout = attempt * 2
try:
conn.connect()
# Check connection
conn.release()
return True
except OSError as e:
logging.warning(
"Cannot connect to AMQP at {}. Retrying in {} secs...".format(
settings.AMQP_URL, timeout))
time.sleep(timeout)
return False
class Test(unittest.TestCase):
def setUp(self):
self.__conn = Connection(transport=Transport)
self.__conn.connect()
self.__exchange = Exchange('test_exchange')
self.__routing_key = 'test.routing'
self.__queue = Queue('test_queue', self.__exchange, routing_key=self.__routing_key)
def test_register_player(self):
message, exchange, routing_key = register_player(111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_FIND_PLAYER, body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_pgpool_disconnected(self):
message, exchange, routing_key = acknowledgement_pgpool_disconnected(111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_DISCONNECT_PGPOOL, body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_pgpool_connected(self):
message, exchange, routing_key = acknowledgement_pgpool_connected(111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_CONNECT_PGPOOL, body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_master_disconnected(self):
message, exchange, routing_key = acknowledgement_master_disconnected(111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_STOP_REPLICATION, body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
def test_acknowledgement_master_connected(self):
message, exchange, routing_key = acknowledgement_master_connected(111, self.__conn, self.__exchange, self.__routing_key)
body = json.loads(message['body'])
self.assertEqual(Constants.ACK_COMMAND_START_REPLICATION, body[Constants.MESSAGE_ACK_COMMAND])
self.assertEqual(111, body[Constants.MESSAGE_NODE_ID])
self.assertEqual(self.__exchange.name, exchange)
self.assertEqual(self.__routing_key, routing_key)
class Publisher(object):
def __init__(self, connection_string, exchange, is_active=True):
self._is_active = is_active
self.is_connected = True
if not is_active:
self.is_connected = False
return
self._connection = Connection(connection_string)
self._connections = set([self._connection])#set of connection for the heartbeat
self._exchange = Exchange(exchange, durable=True, delivry_mode=2, type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
def _get_producer(self):
producer = producers[self._connection].acquire(block=True, timeout=2)
self._connections.add(producer.connection)
return producer
def publish(self, item, contributor):
if not self._is_active:
return
with self._get_producer() as producer:
try:
publish = producer.connection.ensure(producer, producer.publish, max_retries=3)
publish(item, exchange=self._exchange, routing_key=contributor, declare=[self._exchange])
self.is_connected = True
except socket.error:
self.is_connected = False
logging.getLogger(__name__).debug('Impossible to publish message !')
raise
def info(self):
result = {
"is_active": self._is_active,
"is_connected": self.is_connected
}
if not self._is_active:
return result
with self._get_producer() as producer:
res = producer.connection.info()
if 'password' in res:
del res['password']
for key, value in res.items():
result[key] = value
return result
def consumer_from_queue():
connection = Connection(settings.BROKER_URL, heartbeat=5)
connection.connect()
exchange = Exchange(ADDRESS_EXCHANGE, type='direct')
queue = Queue(
name=ADDRESS_CUSTOMER_CONSUMER_QUEUE,
routing_key=ADDRESS_ROUTING_KEY,
exchange=exchange,
)
def process_message(body, message):
logger.info('Message arrived with the following body {}'.format(body))
get_longitude_latitude(body)
message.ack()
consumer = Consumer(connection, queues=queue, callbacks=[process_message], accept=['json', 'pickle', 'msgpack'])
consumer.consume()
def establish_connection():
revived_connection = connection.clone()
revived_connection.ensure_connection(max_retries=3)
channel = revived_connection.channel()
consumer.revive(channel)
consumer.consume()
return revived_connection
def consume():
new_connection = establish_connection()
while True:
try:
new_connection.drain_events(timeout=2)
except socket.timeout:
new_connection.heartbeat_check()
def run():
logger.info('Starting consumer from address queue')
while True:
try:
consume()
except connection.connection_errors:
logger.error('Something went wrong')
finally:
connection.close()
run()
def test_custom_table_names(self):
conn = Connection('sqlalchemy+sqlite:///:memory:', transport_options={
'queue_tablename': 'my_custom_queue',
'message_tablename': 'my_custom_message'
})
conn.connect()
channel = conn.channel()
self.assertEqual(
channel.queue_cls.__table__.name,
'my_custom_queue'
)
self.assertEqual(
channel.message_cls.__table__.name,
'my_custom_message'
)
channel._put('celery', 'DATA')
assert channel._get('celery') == 'DATA'
class Publisher(object):
def __init__(self, connection_string, exchange, is_active=True):
self._is_active = is_active
self.is_connected = True
if not is_active:
self.is_connected = False
return
self._connection = Connection(connection_string)
self._connections = set([self._connection
]) # set of connection for the heartbeat
self._exchange = Exchange(exchange,
durable=True,
delivry_mode=2,
type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
def _get_producer(self):
producer = producers[self._connection].acquire(block=True, timeout=2)
self._connections.add(producer.connection)
return producer
def publish(self, item, contributor):
if not self._is_active:
logging.getLogger(__name__).info('RabbitMQ is not enabled')
return True
with self._get_producer() as producer:
try:
self.is_connected = True
publish = producer.connection.ensure(producer,
producer.publish,
max_retries=3)
publish(item,
exchange=self._exchange,
routing_key=contributor,
declare=[self._exchange])
logging.getLogger(__name__).info(
'Publishing message on exchange %s', self._exchange.name)
except BaseException, e:
self.is_connected = False
logging.exception("Impossible to publish message to rabbitmq")
logging.exception(repr(e))
finally:
def handle(self, command):
command_body = {
"command": command.command,
"argument": command.argument,
"user_id": command.user_id,
"chatroom_id": command.chatroom_id,
}
connection = Connection(settings.AMQP_ADDRESS)
connection.connect()
producer = connection.Producer()
producer.publish(
command_body,
retry=True,
exchange=settings.EXCHANGE,
routing_key=settings.SENDER_ROUTING_KEY,
declare=[settings.BOT_QUEUE],
)
connection.release()
def check_rabbit_connections(self):
if not self._controllers:
self.fail('There are no online controllers')
pwd = self.get_conf_values().strip()
for host in self._controllers:
try:
conn = Connection(host, userid='nova',
password=pwd,
virtual_host='/', port=5673)
conn.connect()
channel = conn.channel()
self.connections.append((channel, host))
LOG.debug('connections is {0}'.format(self.connections))
except Exception:
LOG.debug(traceback.format_exc())
self.fail("Failed to connect to "
"5673 port on host {0}".format(host))
def test_rabbitmq_on_startup(retry: int = 3, wait: float = 10):
"""
Tests connection to RabbitMQ on application start up.
:param retry: amount of connection retries
:param wait: seconds to wait until connection retry
"""
logging.info("Trying to connect to RabbitMQ ...")
rabbitmq_connection = Kombu(rabbitmq_url)
for i in range(0, retry):
try:
rabbitmq_connection.connect()
logging.info("Successfully connected to RabbitMQ!")
rabbitmq_connection.release()
return True
except KombuConnectionError or ConnectionRefusedError:
logging.warning("RabbitMQ is not yet ready!")
time.sleep(wait)
raise Exception("Cannot connect to RabbitMQ!")
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url, exchange_name=None):
self.url = url
self.connect()
self.exchange_name = exchange_name if exchange_name else celery_queue(
"socket_notification")
self.channel = self.connection.channel()
self.socket_exchange = Exchange(self.exchange_name,
type="fanout",
channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""Test if connection is open.
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info("Connecting to broker {}".format(self.url))
self.connection = Connection(self.url, heartbeat=WS_HEART_BEAT)
self.connection.connect()
logger.info("Connected to broker {}".format(self.url))
def _close(self):
if hasattr(self, "connection") and self.connection:
logger.info("Closing connecting to broker {}".format(self.url))
self.connection.release()
self.connection = None
logger.info("Connection closed to broker {}".format(self.url))
def close(self):
self._close()
def __call__(self):
"""
Broker URL must start with "amqp://" else return None
Return True if can establish connection or not.
"""
if self.broker_url.startswith("amqp://"):
try:
connection = Connection(self.broker_url, connection_timeout=1)
try:
connection.connect()
status = connection.connected
finally:
connection.release()
except Exception as exc:
logger.exception(exc)
status = False
return status
else:
return None
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url):
self.url = url
self.connect()
self.channel = self.connection.channel()
self.socket_exchange = Exchange(exchange_name,
type='fanout',
channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""Test if connection is open.
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info('Connecting to broker {}'.format(self.url))
self.connection = Connection(self.url)
self.connection.connect()
logger.info('Connected to broker {}'.format(self.url))
def _close(self):
if hasattr(self, 'connection') and self.connection:
logger.info('Closing connecting to broker {}'.format(self.url))
self.connection.release()
self.connection = None
logger.info('Connection closed to broker {}'.format(self.url))
def close(self):
self._close()
def run(rabbit_url):
print rabbit_url
conn = Connection(rabbit_url)
conn.ensure_connection()
conn.connect()
exchange = Exchange(config.EXCHANGE_NAME, type='direct')
queue = Queue(name=config.QUEUE_NAME,
exchange=exchange,
routing_key=config.ROUTING_KEY)
consumer = Consumer(conn,
queues=queue,
callbacks=[process_message],
accept=['text/plain'])
consumer.consume()
while True:
try:
print 'drain_events'
conn.drain_events(timeout=2) # timeout
except socket.timeout:
pass
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url, exchange_name=None):
self.url = url
self.connect()
self.exchange_name = exchange_name if exchange_name else celery_queue('socket_notification')
self.channel = self.connection.channel()
self.socket_exchange = Exchange(self.exchange_name, type='fanout', channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""Test if connection is open.
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info('Connecting to broker {}'.format(self.url))
self.connection = Connection(self.url, heartbeat=WS_HEART_BEAT)
self.connection.connect()
logger.info('Connected to broker {}'.format(self.url))
def _close(self):
if hasattr(self, 'connection') and self.connection:
logger.info('Closing connecting to broker {}'.format(self.url))
self.connection.release()
self.connection = None
logger.info('Connection closed to broker {}'.format(self.url))
def close(self):
self._close()
def init(username, msgtype):
global __username, __my_type, __connection, __global_exchange, __producer, __my_queue, __backchannel, __backch_producer, __adv_exchange, __adv_queue
__username = username
__my_type = msgtype
print "Username: "******"Q:"
#print __my_queue
#__consumer = __connection.Consumer(queues=__my_queue) #we're not using consumer 'cause we're using queue get
#__recv_loop = drain_consumer(__consumer, timeout=1)
__backchannel = Connection('amqp://')
__backchannel.connect()
__adv_exchange = Exchange('adversary', durable=False, delivery_mode=1)
__adv_exchange.maybe_bind(__backchannel)
__backch_producer = __backchannel.Producer(__backchannel)
__adv_queue = Queue(username + '-adv',
exchange=__adv_exchange,
routing_key=username + '-adv',
durable=False)
__adv_queue = __adv_queue(__backchannel)
__adv_queue.declare()
class Publisher(object):
def __init__(self, connection_string, exchange, is_active=True):
self._is_active = is_active
self.is_connected = True
if not is_active:
self.is_connected = False
return
self._connection = Connection(connection_string)
self._connections = set([self._connection]) # set of connection for the heartbeat
self._exchange = Exchange(exchange, durable=True, delivry_mode=2, type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
def _get_producer(self):
producer = producers[self._connection].acquire(block=True, timeout=2)
self._connections.add(producer.connection)
return producer
def errback(self, exc, interval):
logging.getLogger(__name__).info('Error: %r', exc, exc_info=1)
logging.getLogger(__name__).info('Retry in %s seconds.', interval)
def publish(self, item, contributor):
if not self._is_active:
logging.getLogger(__name__).info('RabbitMQ is not enabled')
return True
with self._get_producer() as producer:
try:
self.is_connected = True
publish = producer.connection.ensure(producer, producer.publish, errback=self.errback, max_retries=3)
publish(item, exchange=self._exchange, routing_key=contributor, declare=[self._exchange])
logging.getLogger(__name__).info('Publishing message on exchange %s', self._exchange.name)
except Exception, e:
self.is_connected = False
logging.getLogger(__name__).exception("Impossible to publish message to rabbitmq")
logging.getLogger(__name__).exception(repr(e))
finally:
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url):
self.url = url
self.connect()
self.channel = self.connection.channel()
self.socket_exchange = Exchange(exchange_name, type="fanout", channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info("Connecting to broker {}".format(self.url))
self.connection = Connection(self.url)
self.connection.connect()
logger.info("Connected to broker {}".format(self.url))
def _close(self):
if hasattr(self, "connection") and self.connection:
logger.info("Closing connecting to broker {}".format(self.url))
self.connection.release()
self.connection = None
logger.info("Connection closed to broker {}".format(self.url))
def close(self):
self._close()
def check_server_access(sconf):
servers = ["broker server", "results server"]
if sconf.keys():
print("\nChecking server connections:")
print("-" * 28)
excpts = {}
for s in servers:
if s in sconf:
try:
conn = Connection(sconf[s])
conn.connect()
conn.release()
print(f"{s} connection: OK")
except Exception as e:
print(f"{s} connection: Error")
excpts[s] = e
if excpts:
print("\nExceptions:")
for k, v in excpts.items():
print(f"{k}: {v}")
class rpc(object):
def __init__(self):
self.exchange = None
self.connection = None
self.connection_pool = None
self.queue = {}
self.consumer = {}
self.producer = {}
self.callbacks = {}
def init_exchange(self, name = 'AllChat', type = 'direct', channel = None, durable = True, delivery_mode = 2):
if self.exchange is None:
self.exchange = Exchange(name, type, channel = channel, durable = durable, delivery_mode = delivery_mode)
return self.exchange
def init_connection(self,url,ssl = False):
if self.connection is None:
self.connection = Connection(url, ssl = ssl)
try:
self.connection.connect()
except Exception as e:
raise e
else:
self.connection.close()
return self.connection
def get_exchange(self):
if self.exchange is None:
raise Exception("No exchange. Please invoke init_exchange firstly")
else:
return self.exchange
def create_connection(self):
if self.connection_pool is None:
self.connection_pool = self.connection.Pool(allchat.app.config['RPC_POOL_NUM'])
return self.connection_pool.acquire()
def release_connection(self, conn):
try:
if conn is not None:
conn.release()
except Exception as e:
raise e
def close_connection(self, conn):
try:
if conn is not None:
conn.close()
except Exception as e:
raise e
# def create_channel(self, conn):
# if conn is not None:
# return conn.channel()
# else:
# return None
#
# def release_channel(self, channel):
# try:
# if channel is not None:
# channel.close()
# except Exception as e:
# raise e
# def create_consumer(self, name, channel , queues = None, callbacks = None):
# if name in self.consumer:
# self.consumer[name].revive(channel)
# else:
# if not any([queues, callbacks]):
# raise Exception("queues and callbacks can't be None")
# if isinstance(callbacks, list):
# self.consumer[name] = Consumer(channel, queues, callbacks)
# else:
# self.consumer[name] = Consumer(channel, queues)
# self.consumer[name].register_callback(callbacks)
# self.consumer[name].consume()
# return self.consumer[name]
def create_consumer(self, name, channel, queues = None):
if name in self.consumer:
try:
if self.callbacks[name] != self.consumer[name].callbacks:
self.consumer[name].callbacks = self.callbacks[name]
except KeyError as e:
raise Exception("Please invoke register_callbacks before")
self.consumer[name].revive(channel)
else:
if not queues:
queues = self.create_queue(name, name)
try:
self.callbacks[name]
except KeyError as e:
self.register_callbacks(name, [rpc_callbacks()])
#raise Exception("Please invoke register_callbacks before")
finally:
self.consumer[name] = Consumer(channel, queues, callbacks = self.callbacks[name])
#self.consumer[name].consume()
return self.consumer[name]
def release_consumer(self, name):
try:
if name in self.consumer:
self.consumer[name].cancel()
except Exception as e:
raise e
def create_producer(self, name, channel):
if name in self.producer:
self.producer[name].revive(channel)
else:
self.producer[name] = Producer(channel, self.get_exchange())
return self.producer[name]
def release_producer(self, name):
try:
if name in self.producer:
self.producer[name].close()
except Exception as e:
raise e
def create_queue(self, name, routing_key, durable = True):
if name not in self.queue:
self.queue[name] = Queue(name, self.get_exchange(), routing_key, durable = durable)
return self.queue[name]
def del_queue(self, name):
try:
if name in self.callbacks:
del self.callbacks[name]
if name in self.consumer:
try:
self.consumer[name].close()
except Exception as e:
pass
del self.consumer[name]
if name in self.producer:
try:
self.producer[name].close()
except Exception as e:
pass
del self.producer[name]
if name in self.queue:
tmp = self.create_connection() #Kombu中删除Queue，必须保证Queue是绑定在一个channel上的，否则删除失败
self.queue[name].maybe_bind(tmp)
self.queue[name].delete()
self.close_connection(tmp)
del self.queue[name]
except Exception as e:
raise e
def register_callbacks(self, name, callbacks):
if isinstance(callbacks, list):
self.callbacks[name] = callbacks
else:
raise Exception("The parameter callbacks should be a list")
def extend_callbacks(self, name, callbacks):
if isinstance(callbacks, list):
if name in self.callbacks:
self.callbacks[name].extend(callbacks)
else:
raise Exception("The {account} callbacks don't exist".format(account = name))
else:
raise Exception("The parameter callbacks should be a list")
class test_Consumer(TestCase):
def setUp(self):
self.connection = Connection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange('foo', 'direct')
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
self.assertFalse(q.declare.call_count)
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
self.assertIsNone(consumer.consume())
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue('a'), Queue('b')]
self.assertFalse(consumer.consuming_from(Queue('c')))
self.assertFalse(consumer.consuming_from('c'))
self.assertTrue(consumer.consuming_from(Queue('a')))
self.assertTrue(consumer.consuming_from(Queue('b')))
self.assertTrue(consumer.consuming_from('b'))
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, 'message_to_python')
channel.message_to_python = None
try:
message = Mock()
message.decode.return_value = 'Hello'
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with('Hello', message)
finally:
channel.message_to_python = m2p
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
callbacks = [lambda x, y: x,
lambda x, y: x]
consumer = Consumer(channel, queue, auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare',
'queue_declare',
'queue_bind',
'basic_consume'):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count('basic_consume'), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare',
'queue_declare',
'basic_consume'):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ('exchange_declare',
'queue_declare',
'queue_bind'):
self.assertIn(meth, channel)
self.assertNotIn('basic_consume', channel)
consumer.consume()
self.assertIn('basic_consume', channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn('flow', channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn('basic_qos', channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count('queue_purge'), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count('exchange_declare'), 4)
self.assertEqual(channel.called.count('queue_declare'), 4)
self.assertEqual(channel.called.count('queue_bind'), 4)
self.assertEqual(channel.called.count('basic_consume'), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count('basic_cancel'), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({u'foo': u'bar'})
self.assertIn('basic_ack', channel)
self.assertIn('message_to_python', channel)
self.assertEqual(received[0], {u'foo': u'bar'})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
with self.assertRaises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({'foo': 'bar'})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({'foo': 'bar'})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.loads(m), {'foo': 'bar'})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn('basic_recover', channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
self.assertTrue(repr(Consumer(channel, [b1])))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
self.assertIsNone(p.connection)
class test_Producer(TestCase):
def setUp(self):
self.exchange = Exchange('foo', 'direct')
self.connection = Connection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.assertFalse(self.exchange.is_bound)
@patch('kombu.common.maybe_declare')
def test_maybe_declare(self, maybe_declare):
p = self.connection.Producer()
q = Queue('foo')
p.maybe_declare(q)
maybe_declare.assert_called_with(q, p.channel, False)
@patch('kombu.common.maybe_declare')
def test_maybe_declare_when_entity_false(self, maybe_declare):
p = self.connection.Producer()
p.maybe_declare(None)
self.assertFalse(maybe_declare.called)
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
self.assertIsNot(p.exchange, self.exchange,
'creates Exchange clone at bind')
self.assertTrue(p.exchange.is_bound)
self.assertIn('exchange_declare', channel,
'auto_declare declares exchange')
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn('exchange_declare', channel,
'auto_declare=False does not declare exchange')
p.declare()
self.assertIn('exchange_declare', channel,
'p.declare() declares exchange')
def test_prepare(self):
message = {u'the quick brown fox': u'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, headers={})
self.assertDictEqual(message, anyjson.loads(m))
self.assertEqual(ctype, 'application/json')
self.assertEqual(cencoding, 'utf-8')
def test_prepare_compression(self):
message = {u'the quick brown fox': u'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
headers = {}
m, ctype, cencoding = p._prepare(message, compression='zlib',
headers=headers)
self.assertEqual(ctype, 'application/json')
self.assertEqual(cencoding, 'utf-8')
self.assertEqual(headers['compression'], 'application/x-gzip')
import zlib
self.assertEqual(anyjson.loads(
zlib.decompress(m).decode('utf-8')), message)
def test_prepare_custom_content_type(self):
message = 'the quick brown fox'.encode('utf-8')
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='custom')
self.assertEqual(m, message)
self.assertEqual(ctype, 'custom')
self.assertEqual(cencoding, 'binary')
m, ctype, cencoding = p._prepare(message, content_type='custom',
content_encoding='alien')
self.assertEqual(m, message)
self.assertEqual(ctype, 'custom')
self.assertEqual(cencoding, 'alien')
def test_prepare_is_already_unicode(self):
message = u'the quick brown fox'
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='text/plain')
self.assertEqual(m, message.encode('utf-8'))
self.assertEqual(ctype, 'text/plain')
self.assertEqual(cencoding, 'utf-8')
m, ctype, cencoding = p._prepare(message, content_type='text/plain',
content_encoding='utf-8')
self.assertEqual(m, message.encode('utf-8'))
self.assertEqual(ctype, 'text/plain')
self.assertEqual(cencoding, 'utf-8')
def test_publish_with_Exchange_instance(self):
p = self.connection.Producer()
p.channel = Mock()
p.publish('hello', exchange=Exchange('foo'))
self.assertEqual(
p._channel.basic_publish.call_args[1]['exchange'], 'foo',
)
def test_publish_retry_with_declare(self):
p = self.connection.Producer()
p.maybe_declare = Mock()
p.connection.ensure = Mock()
ex = Exchange('foo')
p._publish('hello', 0, '', '', {}, {}, 'rk', 0, 0, ex, declare=[ex])
p.maybe_declare.assert_called_with(ex)
def test_revive_when_channel_is_connection(self):
p = self.connection.Producer()
p.exchange = Mock()
new_conn = Connection('memory://')
defchan = new_conn.default_channel
p.revive(new_conn)
self.assertIs(p.channel, defchan)
p.exchange.revive.assert_called_with(defchan)
def test_enter_exit(self):
p = self.connection.Producer()
p.release = Mock()
self.assertIs(p.__enter__(), p)
p.__exit__()
p.release.assert_called_with()
def test_connection_property_handles_AttributeError(self):
p = self.connection.Producer()
p.channel = object()
p.__connection__ = None
self.assertIsNone(p.connection)
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
message = {u'the quick brown fox': u'jumps over the lazy dog'}
ret = p.publish(message, routing_key='process')
self.assertIn('prepare_message', channel)
self.assertIn('basic_publish', channel)
m, exc, rkey = ret
self.assertDictEqual(message, anyjson.loads(m['body']))
self.assertDictContainsSubset({'content_type': 'application/json',
'content_encoding': 'utf-8',
'priority': 0}, m)
self.assertDictContainsSubset({'delivery_mode': 2}, m['properties'])
self.assertEqual(exc, p.exchange.name)
self.assertEqual(rkey, 'process')
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
self.assertFalse(p.exchange.name)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
self.assertTrue(on_return in chan.events['basic_return'])
self.assertTrue(p.on_return)
class Amqp(Thread):
def __init__(
self,
host="localhost",
port=5672,
userid="guest",
password="******",
virtual_host="canopsis",
exchange_name="canopsis",
logging_name="Amqp",
logging_level=INFO,
read_config_file=True,
auto_connect=True,
on_ready=None,
max_retries=5
):
super(Amqp, self).__init__()
self.logger = getLogger(logging_name)
self.host = host
self.port = port
self.userid = userid
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.logging_level = logging_level
if read_config_file:
self.read_config("amqp")
self.amqp_uri = "amqp://{0}:{1}@{2}:{3}/{4}".format(
self.userid, self.password, self.host, self.port,
self.virtual_host)
self.logger.setLevel(logging_level)
# Event sent try count before event drop in case of connection problem
if max_retries != 5:
self.logger.info('Custom retries value : {} {}'.format(
max_retries,
type(max_retries)
))
self.max_retries = max_retries
self.exchange_name_events = exchange_name + ".events"
self.exchange_name_alerts = exchange_name + ".alerts"
self.exchange_name_incidents = exchange_name + ".incidents"
self.chan = None
self.conn = None
self.connected = False
self.on_ready = on_ready
self.RUN = True
self.exchanges = {}
self.queues = {}
self.producers = None
self.paused = False
self.connection_errors = (
ConnectionError,
error,
IOError,
OSError)
# Create exchange
self.logger.debug("Create exchanges object")
for exchange_name in [
self.exchange_name, self.exchange_name_events,
self.exchange_name_alerts, self.exchange_name_incidents]:
self.logger.debug(' + {}'.format(exchange_name))
self.get_exchange(exchange_name)
if auto_connect:
self.connect()
self.logger.debug("Object canamqp initialized")
def run(self):
"""
main thread loop
- connects to rabbit
- calls drain_events
- disconnects on error
"""
self.logger.debug("Start thread ...")
reconnect = False
while self.RUN:
self.connect()
if self.connected:
self.init_queue(reconnect=reconnect)
self.logger.debug("Drain events ...")
while self.RUN:
try:
if not self.paused:
self.conn.drain_events(timeout=0.5)
else:
sleep(0.5)
except KombuSerializationError as exc:
self.logger.error(
"Kombu serialization error: invalid message "
"received: {}".format(exc))
except timeout:
pass
except self.connection_errors as err:
self.logger.error(
"Connection error ! ({})".format(err)
)
break
except KombuSerializationError as exc:
self.logger.error(
"Kombu serialization error: invalid message "
"received: {}".format(exc))
except Exception as err:
self.logger.error("Error: {} ({})".format(
err,
type(err)
))
print_exc(file=stdout)
break
self.disconnect()
if self.RUN:
self.logger.error(
'Connection lost, try to reconnect in few seconds ...'
)
reconnect = True
self.wait_connection(timeout=5)
self.logger.debug("End of thread ...")
def stop(self):
self.logger.debug("Stop thread ...")
self.RUN = False
def connect(self):
"""
Create the connection
Inits the producers
Init the only channel
Declares exchanges
"""
if not self.connected:
self.logger.info(
"Connect to AMQP Broker (%s:%s)" % (self.host, self.port))
self.conn = Connection(self.amqp_uri)
try:
self.logger.debug(" + Connect")
self.conn.connect()
self.logger.info("Connected to AMQP Broker.")
self.producers = pools.Producers(limit=10)
self.connected = True
except Exception as err:
self.conn.release()
self.logger.error("Impossible to connect ({})".format(err))
if self.connected:
self.logger.debug(" + Open channel")
try:
self.chan = self.conn.channel()
self.logger.info("Channel openned. Ready to send messages")
try:
# Declare exchange
self.logger.debug("Declare exchanges")
for exchange_name in self.exchanges:
self.logger.debug(" + {}".format(exchange_name))
self.exchanges[exchange_name](self.chan).declare()
except Exception as err:
self.logger.error(
"Impossible to declare exchange ({})".format(err))
except Exception as err:
self.logger.error(err)
else:
self.logger.debug("Already connected")
def get_exchange(self, name):
"""
Returns an exchange if stored in self.exchanges.
Otherwise, creates it and returns it
:param string name: name of the exchange to get/create
:rtype: Exchange|None
"""
if name:
try:
return self.exchanges[name]
except Exception:
if name == "amq.direct":
self.exchanges[name] = Exchange(
name, "direct", durable=True)
else:
self.exchanges[name] = Exchange(
name, "topic", durable=True, auto_delete=False)
return self.exchanges[name]
else:
return None
def init_queue(self, reconnect=False):
"""
Loads queue settings
Creates queues and attaches the same channel to each of them
Binds queues to exchange
Revives or creates consumers
calls consume
"""
if self.queues:
self.logger.debug("Init queues")
for queue_name in self.queues.keys():
self.logger.debug(" + {}".format(queue_name))
qsettings = self.queues[queue_name]
if not qsettings['queue']:
self.logger.debug(" + Create queue")
# copy list
routing_keys = list(qsettings['routing_keys'])
routing_key = None
if len(routing_keys):
routing_key = routing_keys[0]
routing_keys = routing_keys[1:]
exchange = self.get_exchange(qsettings['exchange_name'])
if qsettings['exchange_name'] == "amq.direct" \
and not routing_key:
routing_key = queue_name
self.logger.debug(
"exchange: '{}', exclusive: {},"
" auto_delete: {},no_ack: {}"
.format(
qsettings['exchange_name'],
qsettings['exclusive'],
qsettings['auto_delete'],
qsettings['no_ack']
)
)
qsettings['queue'] = Queue(
queue_name,
exchange=exchange,
routing_key=routing_key,
exclusive=qsettings['exclusive'],
auto_delete=qsettings['auto_delete'],
no_ack=qsettings['no_ack'],
channel=self.conn.channel())
qsettings['queue'].declare()
if len(routing_keys):
self.logger.debug(" + Bind on all routing keys")
for routing_key in routing_keys:
self.logger.debug(
" + routing_key: '{}'".format(routing_key)
)
try:
qsettings['queue'].bind_to(
exchange=exchange,
routing_key=routing_key
)
except Exception:
self.logger.error(
"You need upgrade your Kombu version ({})"
.format(__version__)
)
if qsettings['consumer'] and not reconnect:
qsettings['consumer'].revive(self.chan)
elif not qsettings['consumer'] or reconnect:
self.logger.debug(" + Create Consumer")
qsettings['consumer'] = self.conn.Consumer(
qsettings['queue'], callbacks=[qsettings['callback']])
self.logger.debug(" + Consume queue")
qsettings['consumer'].consume()
if self.on_ready:
self.on_ready()
else:
self.logger.info('Queue already inited')
def add_queue(
self,
queue_name,
routing_keys,
callback,
exchange_name=None,
no_ack=True,
exclusive=False,
auto_delete=True
):
"""
Initializes the queue configuration
maps the callback on the queue
"""
c_routing_keys = []
if not isinstance(routing_keys, list):
if isinstance(routing_keys, basestring):
c_routing_keys = [routing_keys]
else:
c_routing_keys = routing_keys
# aka if rk is nor a list nor a basetring, leave it empty
if not exchange_name:
exchange_name = self.exchange_name
self.queues[queue_name] = {
'queue': False,
'consumer': False,
'queue_name': queue_name,
'routing_keys': c_routing_keys,
'callback': callback,
'exchange_name': exchange_name,
'no_ack': no_ack,
'exclusive': exclusive,
'auto_delete': auto_delete
}
def publish(
self,
msg,
routing_key,
exchange_name=None,
serializer="json",
compression=None,
content_type=None,
content_encoding=None
):
"""
Tries to publish an event
In case of failure, tries to reconnect and retry until (self.max_retries)
:returns: operation success status
:rtype: bool
"""
self.logger.warning("Publishing from old.rabbitmq.Amqp is deprecated")
operation_success = False
retries = 0
while not operation_success and retries < self.max_retries:
retries += 1
if self.connected:
if not exchange_name:
exchange_name = self.exchange_name
with self.producers[self.conn].acquire(block=True) as producer:
try:
_msg = msg.copy()
Amqp._clean_msg_for_serialization(_msg)
exchange = self.get_exchange(
exchange_name.encode('utf-8'))
producer.publish(
_msg,
serializer=serializer,
compression=compression,
routing_key=routing_key,
exchange=exchange
)
self.logger.debug('publish {} in exchange {}'.format(
routing_key,
exchange_name
))
operation_success = True
except AmqpStructError:
self.logger.warning(
'Malformed message: routing key is too long. '
'Cancelling message')
return False
except Exception:
self.logger.error(
' + Impossible to send {}'
.format(traceback.format_exc())
)
self.disconnect()
self.connect()
self.init_queue(reconnect=False)
else:
self.logger.error('Not connected ... try reconnecting')
self.connect()
if not operation_success:
# Event and it's information are buffered until next send retry
self.logger.info('Retry count {}'.format(
retries
))
if not operation_success:
# Event and it's information are buffered until next send retry
self.logger.error('Too much retries for event {}, give up'.format(
routing_key
))
return operation_success
@staticmethod
def _clean_msg_for_serialization(msg):
from bson import objectid
for key in msg:
if isinstance(msg[key], objectid.ObjectId):
msg[key] = str(msg[key])
def cancel_queues(self):
if self.connected:
for queue_name in self.queues.keys():
if self.queues[queue_name]['consumer']:
self.logger.debug(
" + Cancel consumer on {}".format(queue_name)
)
try:
self.queues[queue_name]['consumer'].cancel()
except Exception:
pass
del(self.queues[queue_name]['consumer'])
self.queues[queue_name]['consumer'] = False
del(self.queues[queue_name]['queue'])
self.queues[queue_name]['queue'] = False
def disconnect(self):
if self.connected:
self.logger.info("Disconnect from AMQP Broker")
self.cancel_queues()
# Force producers closing to permit a clean reconnect after
# ... especially on timeout errors
self.producers[self.conn].force_close_all()
for exchange in self.exchanges:
del exchange
self.exchanges = {}
try:
pools.reset()
except Exception as err:
self.logger.error(
"Impossible to reset kombu pools: {} ({})".format(
err, type(err)))
try:
self.conn.release()
del self.conn
except Exception as err:
self.logger.error(
"Impossible to release connection: {} ({})".format(
err, type(err)))
self.connected = False
def wait_connection(self, timeout=5):
i = 0
while self.RUN and not self.connected and i < (timeout * 2):
sleep(0.5)
i += 1
def read_config(self, name):
filename = join(root_path, 'etc', '{0}.conf'.format(name))
import ConfigParser
self.config = ConfigParser.RawConfigParser()
try:
self.config.read(filename)
section = 'master'
self.host = self.config.get(section, "host")
self.port = self.config.getint(section, "port")
self.userid = self.config.get(section, "userid")
self.password = self.config.get(section, "password")
self.virtual_host = self.config.get(section, "virtual_host")
self.exchange_name = self.config.get(section, "exchange_name")
except Exception as err:
self.logger.error(
"Can't to load configurations ({}), use default ...".format(
err
))
def __del__(self):
self.stop()
def do_produce(use_predef_msgs=False):
conn = Connection(amqp_hosts, failover_strategy='round-robin' )
conn.ensure_connection(errback=on_ens_conn_err_cb)
conn.connect()
# bind xchg and Qs to the rmq connection, declare primary exchange
bound_priTopicXchg = priTopicXchg(conn)
bound_priTopicXchg.declare()
"""
# and all explicit Qs in rmq
for i in priTopicExplicitQs:
_bound_q = i(conn)
try:
_bound_q.declare()
except Exception as e:
print("unable to declare, exception type [%s], [%s]" %
(type(e), repr(e)))
_bound_q.delete()
_bound_q.declare()
"""
producer = conn.Producer(serializer='json')
if use_predef_msgs:
for msg in msgs:
if msg['topic'] == PRI_TOPIC_NAME:
xchg = priTopicXchg
qs = priTopicExplicitQs
else:
print("unknown topic [%s]" % msg['topic'])
print("sending messages[%s], xchg[%s], topic[%s], routing[%s]"
% (msg['msg'], xchg, msg['topic'], msg['routing']))
producer.publish(
msg['msg']
, exchange=xchg
, declare=[xchg] + qs # let declaration of the exchange and the explicit Qs
#,compression='zlib'
, compression='bzip2'
, routing_key=msg['routing']
# apparently expiration per message dont really work in kombu 3.0.32
#exipration=10 # 60*15 # 15 minutes
#,properties=properties
#,x-message-ttl=1000
)
print("all predefined messages sent")
try:
while True:
var = input('')
print(var.split('route'))
tmp = [i.strip(' ') for i in var.split('route')]
if len(tmp) != 2:
print("invalid msg [%s], need to be of form: [aaa bbb route x.y.z]" %
var)
continue
try:
bound_priTopicXchg.publish(
bound_priTopicXchg.Message(tmp[0]),
routing_key=tmp[1]
)
except conn.connection_errors + conn.channel_errors:
print("connection [%s] went down , reconnect to the next one" %
conn.info())
conn.close()
conn.ensure_connection(errback=on_ens_conn_err_cb)
bound_priTopicXchg = priTopicXchg(conn)
bound_priTopicXchg.publish(
bound_priTopicXchg.Message(tmp[0]),
routing_key=tmp[1]
)
except (EOFError, KeyboardInterrupt):
print("done")
from kombu import Connection from tapiriik.settings import RABBITMQ_BROKER_URL mq = Connection(RABBITMQ_BROKER_URL) mq.connect()
def kombu_connection_test():
connection = Connection('amqp://*****:*****@121.41.4.137:5672//')
connection.connect()
print connection.connected
connection.release()
class _AMQPServerWrapper(object):
def __init__(self, amqp_url):
self.__connection = Connection(amqp_url)
self.__connection.connect()
self.__monitors = {}
self.__running = True
self.__consumer = Consumer(self.__connection, on_message=self.__on_message)
self.__consumer_gl = gevent.spawn(self.__consumer_greenlet_main)
def __consumer_greenlet_main(self):
gevent.sleep(0)
while self.__running:
self.__consumer.consume()
try:
self.__connection.drain_events(timeout=0.1)
except Exception as ex: # NOQA: assigned but not used (left in for super-duper-low-level-debug)
# print("was woken because {}".format(ex))
pass
gevent.sleep(0)
# print("---loop")
def __on_message(self, msg):
ct = msg.delivery_info['consumer_tag']
assert ct in self.__monitors, \
"Message from consumer '{}', but we are not monitoring that (list={})".format(
msg.delivery_info['consumer_tag'], self.__monitors.keys())
mon = self.__monitors[ct]
mon['event_cb'](msg, msg.body)
@property
def connected(self):
return self.__connection.connected
def create_add_monitor(self, exchange, routing_key, event_cb, queue_name=None):
mname = "ex={} rk={} qn={}".format(exchange, routing_key, queue_name)
if mname in self.__monitors:
mon = self.__monitors[mname]
mon["event_cb"] = event_cb
else:
if queue_name is None:
queue_name = ''
exclusive = True
else:
exclusive = False
ex = Exchange(exchange, 'topic')
queue = Queue(exchange=ex, routing_key=routing_key, exclusive=exclusive)
bound_queue = queue.bind(self.__connection)
self.__consumer.add_queue(bound_queue)
bound_queue.consume(mname, self.__on_message)
mon = {
"event_cb": event_cb,
"exchange": ex
}
self.__monitors[mname] = mon
return mon['exchange']
def inject(self, exchange, routing_key, payload):
prod = Producer(self.__connection, exchange=exchange, routing_key=routing_key)
prod.publish(payload)
def test_helper_sync_send_msg(self, exchange, ex_rk, send_rk, payload):
ex = Exchange(exchange, 'topic')
queue = Queue(exchange=ex, routing_key=ex_rk + '.*', exclusive=True, channel=self.__connection)
queue.declare()
prod = Producer(self.__connection, exchange=ex, routing_key=send_rk)
prod.publish(payload)
return queue
def test_helper_sync_recv_msg(self, queue):
for tick in range(10):
msg = queue.get()
if msg is not None:
break
time.sleep(1)
return msg
def do_consume(user_qs):
print("about to listen no queues [%s]" %
", ".join(list(map(lambda x: x, user_qs))))
conn = Connection(amqp_hosts, failover_strategy='round-robin')
# try to get a connection no matter what
while True:
try:
conn.ensure_connection(errback=on_ens_conn_err_cb)
conn.connect()
except Exception as e:
print("connection error failed on exception [%s]" % repr(e))
conn.release()
continue
if conn.connected:
break
else:
print("connection failed in some way, retry")
chan = conn.channel()
global bound_cons_Q
cons_Q = Queue(common.uuid(), queue_arguments=q_expires)
bound_cons_Q = cons_Q(chan)
bound_cons_Q.declare()
# first bind to some control route
bound_cons_Q.bind_to(priTopicXchg, routing_key='manage.#')
for i in user_qs:
if '*' in i or '#' in i:
# create the wildcard route_key bind
bound_cons_Q.bind_to(priTopicXchg, routing_key=i)
else:
for j in allQs:
if i == j.as_dict()['name']:
bound_cons_Q.bind_to(priTopicXchg, routing_key=j.as_dict()['routing_key'])
cons = Consumer(
chan,
accept=['json'],
queues=bound_cons_Q,
callbacks=[on_msg_cb_1, on_msg_cb_2]
)
print("queue set to [%s]" % bound_cons_Q.as_dict(recurse=True))
cons.consume()
while True:
try:
conn.drain_events()
except conn.connection_errors + conn.channel_errors as e:
print("connection [%s] went down (error[%s]), trying to "
"connect to the next one" % (conn.info(), repr(e)))
conn.close()
conn.release()
conn.ensure_connection(errback=on_ens_conn_err_cb)
conn.connect()
chan = conn.channel()
cons_Q.bind(chan)
cons = Consumer(
chan,
accept=['json'],
queues=bound_cons_Q,
callbacks=[on_msg_cb_1, on_msg_cb_2]
)
cons.consume()
class test_Consumer:
def setup(self):
self.connection = Connection(transport=Transport)
self.connection.connect()
assert self.connection.connection.connected
self.exchange = Exchange('foo', 'direct')
def test_accept(self):
a = Consumer(self.connection)
assert a.accept is None
b = Consumer(self.connection, accept=['json', 'pickle'])
assert b.accept == {
'application/json', 'application/x-python-serialize',
}
c = Consumer(self.connection, accept=b.accept)
assert b.accept == c.accept
def test_enter_exit_cancel_raises(self):
c = Consumer(self.connection)
c.cancel = Mock(name='Consumer.cancel')
c.cancel.side_effect = KeyError('foo')
with c:
pass
c.cancel.assert_called_with()
def test_receive_callback_accept(self):
message = Mock(name='Message')
message.errors = []
callback = Mock(name='on_message')
c = Consumer(self.connection, accept=['json'], on_message=callback)
c.on_decode_error = None
c.channel = Mock(name='channel')
c.channel.message_to_python = None
c._receive_callback(message)
callback.assert_called_with(message)
assert message.accept == c.accept
def test_accept__content_disallowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q], exchange=self.exchange, serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], callbacks=[callback]) as consumer:
with pytest.raises(consumer.ContentDisallowed):
conn.drain_events(timeout=1)
callback.assert_not_called()
def test_accept__content_allowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q], exchange=self.exchange, serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], accept=['pickle'],
callbacks=[callback]):
conn.drain_events(timeout=1)
callback.assert_called()
body, message = callback.call_args[0]
assert body['complex']
def test_set_no_channel(self):
c = Consumer(None)
assert c.channel is None
c.revive(Mock())
assert c.channel
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
assert consumer.no_ack
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
assert q in consumer.queues
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
assert q in consumer.queues
assert not q.declare.call_count
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
assert consumer.consume() is None
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')]
consumer._active_tags = {'a': 1, 'b': 2}
assert not consumer.consuming_from(Queue('c'))
assert not consumer.consuming_from('c')
assert not consumer.consuming_from(Queue('d'))
assert not consumer.consuming_from('d')
assert consumer.consuming_from(Queue('a'))
assert consumer.consuming_from(Queue('b'))
assert consumer.consuming_from('b')
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, 'message_to_python')
channel.message_to_python = None
try:
message = Mock()
message.errors = []
message.decode.return_value = 'Hello'
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with('Hello', message)
finally:
channel.message_to_python = m2p
def test_receive_callback__message_errors(self):
channel = self.connection.channel()
channel.message_to_python = None
c = channel.Consumer()
message = Mock()
try:
raise KeyError('foo')
except KeyError:
message.errors = [sys.exc_info()]
message._reraise_error.side_effect = KeyError()
with pytest.raises(KeyError):
c._receive_callback(message)
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
callbacks = [lambda x, y: x,
lambda x, y: x]
consumer = Consumer(channel, queue, auto_declare=True,
callbacks=callbacks)
assert consumer.callbacks == callbacks
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
assert consumer.queues[0] is not queue
assert consumer.queues[0].is_bound
assert consumer.queues[0].exchange.is_bound
assert consumer.queues[0].exchange is not self.exchange
for meth in ('exchange_declare',
'queue_declare',
'queue_bind',
'basic_consume'):
assert meth in channel
assert channel.called.count('basic_consume') == 1
assert consumer._active_tags
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
assert not consumer._active_tags
def test_consumer_tag_prefix(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, tag_prefix='consumer_')
consumer.consume()
assert consumer._active_tags[queue.name].startswith('consumer_')
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=False)
assert consumer.queues[0] is not queue
assert consumer.queues[0].is_bound
assert consumer.queues[0].exchange.is_bound
assert consumer.queues[0].exchange is not self.exchange
for meth in ('exchange_declare',
'queue_declare',
'basic_consume'):
assert meth not in channel
consumer.declare()
for meth in ('exchange_declare',
'queue_declare',
'queue_bind'):
assert meth in channel
assert 'basic_consume' not in channel
consumer.consume()
assert 'basic_consume' in channel
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
assert 'basic_cancel' in channel
assert not consumer._active_tags
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
assert context is consumer
assert consumer._active_tags
res = consumer.__exit__(None, None, None)
assert not res
assert 'basic_cancel' in channel
assert not consumer._active_tags
def test_flow(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
assert 'flow' in channel
def test_qos(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
assert 'basic_qos' in channel
def test_purge(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
assert channel.called.count('queue_purge') == 4
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
assert channel.called.count('exchange_declare') == 4
assert channel.called.count('queue_declare') == 4
assert channel.called.count('queue_bind') == 4
assert channel.called.count('basic_consume') == 4
assert len(consumer._active_tags) == 4
consumer.cancel()
assert channel.called.count('basic_cancel') == 4
assert not len(consumer._active_tags)
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
assert 'basic_ack' in channel
assert 'message_to_python' in channel
assert received[0] == {'foo': 'bar'}
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with pytest.raises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject' in channel
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with pytest.raises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject' in channel
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject:requeue' in channel
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with pytest.raises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject:requeue' in channel
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
with pytest.raises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with pytest.raises(ValueError):
consumer._receive_callback({'foo': 'bar'})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({'foo': 'bar'})
assert thrown
m, exc = thrown[0]
assert json.loads(m) == {'foo': 'bar'}
assert isinstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.recover()
assert 'basic_recover' in channel
def test_revive(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
assert consumer.channel is channel2
assert consumer.queues[0].channel is channel2
assert consumer.queues[0].exchange.channel is channel2
def test_revive__with_prefetch_count(self):
channel = Mock(name='channel')
b1 = Queue('qname1', self.exchange, 'rkey')
Consumer(channel, [b1], prefetch_count=14)
channel.basic_qos.assert_called_with(0, 14, False)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
assert repr(Consumer(channel, [b1]))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
assert p.connection is None
class test_Producer:
def setup(self):
self.exchange = Exchange('foo', 'direct')
self.connection = Connection(transport=Transport)
self.connection.connect()
assert self.connection.connection.connected
assert not self.exchange.is_bound
def test_repr(self):
p = Producer(self.connection)
assert repr(p)
def test_pickle(self):
chan = Mock()
producer = Producer(chan, serializer='pickle')
p2 = pickle.loads(pickle.dumps(producer))
assert p2.serializer == producer.serializer
def test_no_channel(self):
p = Producer(None)
assert not p._channel
@patch('kombu.messaging.maybe_declare')
def test_maybe_declare(self, maybe_declare):
p = self.connection.Producer()
q = Queue('foo')
p.maybe_declare(q)
maybe_declare.assert_called_with(q, p.channel, False)
@patch('kombu.common.maybe_declare')
def test_maybe_declare_when_entity_false(self, maybe_declare):
p = self.connection.Producer()
p.maybe_declare(None)
maybe_declare.assert_not_called()
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
# creates Exchange clone at bind
assert p.exchange is not self.exchange
assert p.exchange.is_bound
# auto_declare declares exchange'
assert 'exchange_declare' in channel
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
assert p.exchange.is_bound
# auto_declare=False does not declare exchange
assert 'exchange_declare' not in channel
# p.declare() declares exchange')
p.declare()
assert 'exchange_declare' in channel
def test_prepare(self):
message = {'the quick brown fox': 'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, headers={})
assert json.loads(m) == message
assert ctype == 'application/json'
assert cencoding == 'utf-8'
def test_prepare_compression(self):
message = {'the quick brown fox': 'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
headers = {}
m, ctype, cencoding = p._prepare(message, compression='zlib',
headers=headers)
assert ctype == 'application/json'
assert cencoding == 'utf-8'
assert headers['compression'] == 'application/x-gzip'
import zlib
assert json.loads(zlib.decompress(m).decode('utf-8')) == message
def test_prepare_custom_content_type(self):
message = 'the quick brown fox'.encode('utf-8')
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='custom')
assert m == message
assert ctype == 'custom'
assert cencoding == 'binary'
m, ctype, cencoding = p._prepare(message, content_type='custom',
content_encoding='alien')
assert m == message
assert ctype == 'custom'
assert cencoding == 'alien'
def test_prepare_is_already_unicode(self):
message = 'the quick brown fox'
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='text/plain')
assert m == message.encode('utf-8')
assert ctype == 'text/plain'
assert cencoding == 'utf-8'
m, ctype, cencoding = p._prepare(message, content_type='text/plain',
content_encoding='utf-8')
assert m == message.encode('utf-8')
assert ctype == 'text/plain'
assert cencoding == 'utf-8'
def test_publish_with_Exchange_instance(self):
p = self.connection.Producer()
p.channel = Mock()
p.publish('hello', exchange=Exchange('foo'), delivery_mode='transient')
assert p._channel.basic_publish.call_args[1]['exchange'] == 'foo'
def test_publish_with_expiration(self):
p = self.connection.Producer()
p.channel = Mock()
p.publish('hello', exchange=Exchange('foo'), expiration=10)
properties = p._channel.prepare_message.call_args[0][5]
assert properties['expiration'] == '10000'
def test_publish_with_reply_to(self):
p = self.connection.Producer()
p.channel = Mock()
p.publish('hello', exchange=Exchange('foo'), reply_to=Queue('foo'))
properties = p._channel.prepare_message.call_args[0][5]
assert properties['reply_to'] == 'foo'
def test_set_on_return(self):
chan = Mock()
chan.events = defaultdict(Mock)
p = Producer(ChannelPromise(lambda: chan), on_return='on_return')
p.channel
chan.events['basic_return'].add.assert_called_with('on_return')
def test_publish_retry_calls_ensure(self):
p = Producer(Mock())
p._connection = Mock()
ensure = p.connection.ensure = Mock()
p.publish('foo', exchange='foo', retry=True)
ensure.assert_called()
def test_publish_retry_with_declare(self):
p = self.connection.Producer()
p.maybe_declare = Mock()
p.connection.ensure = Mock()
ex = Exchange('foo')
p._publish('hello', 0, '', '', {}, {}, 'rk', 0, 0, ex, declare=[ex])
p.maybe_declare.assert_called_with(ex)
def test_revive_when_channel_is_connection(self):
p = self.connection.Producer()
p.exchange = Mock()
new_conn = Connection('memory://')
defchan = new_conn.default_channel
p.revive(new_conn)
assert p.channel is defchan
p.exchange.revive.assert_called_with(defchan)
def test_enter_exit(self):
p = self.connection.Producer()
p.release = Mock()
assert p.__enter__() is p
p.__exit__()
p.release.assert_called_with()
def test_connection_property_handles_AttributeError(self):
p = self.connection.Producer()
p.channel = object()
p.__connection__ = None
assert p.connection is None
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
message = {'the quick brown fox': 'jumps over the lazy dog'}
ret = p.publish(message, routing_key='process')
assert 'prepare_message' in channel
assert 'basic_publish' in channel
m, exc, rkey = ret
assert json.loads(m['body']) == message
assert m['content_type'] == 'application/json'
assert m['content_encoding'] == 'utf-8'
assert m['priority'] == 0
assert m['properties']['delivery_mode'] == 2
assert exc == p.exchange.name
assert rkey == 'process'
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
assert not p.exchange.name
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
assert p.channel is chan2
assert p.exchange.channel is chan2
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
assert on_return in chan.events['basic_return']
assert p.on_return
class test_Producer(TestCase):
def setUp(self):
self.exchange = Exchange("foo", "direct")
self.connection = Connection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.assertFalse(self.exchange.is_bound)
def test_pickle(self):
chan = Mock()
producer = Producer(chan, serializer="pickle")
p2 = pickle.loads(pickle.dumps(producer))
self.assertEqual(p2.serializer, producer.serializer)
def test_no_channel(self):
p = Producer(None)
self.assertFalse(p._channel)
@patch("kombu.common.maybe_declare")
def test_maybe_declare(self, maybe_declare):
p = self.connection.Producer()
q = Queue("foo")
p.maybe_declare(q)
maybe_declare.assert_called_with(q, p.channel, False)
@patch("kombu.common.maybe_declare")
def test_maybe_declare_when_entity_false(self, maybe_declare):
p = self.connection.Producer()
p.maybe_declare(None)
self.assertFalse(maybe_declare.called)
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
self.assertIsNot(p.exchange, self.exchange, "creates Exchange clone at bind")
self.assertTrue(p.exchange.is_bound)
self.assertIn("exchange_declare", channel, "auto_declare declares exchange")
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn("exchange_declare", channel, "auto_declare=False does not declare exchange")
p.declare()
self.assertIn("exchange_declare", channel, "p.declare() declares exchange")
def test_prepare(self):
message = {"the quick brown fox": "jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, headers={})
self.assertDictEqual(message, anyjson.loads(m))
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
def test_prepare_compression(self):
message = {"the quick brown fox": "jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
headers = {}
m, ctype, cencoding = p._prepare(message, compression="zlib", headers=headers)
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
self.assertEqual(headers["compression"], "application/x-gzip")
import zlib
self.assertEqual(anyjson.loads(zlib.decompress(m).decode("utf-8")), message)
def test_prepare_custom_content_type(self):
message = "the quick brown fox".encode("utf-8")
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="custom")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "binary")
m, ctype, cencoding = p._prepare(message, content_type="custom", content_encoding="alien")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "alien")
def test_prepare_is_already_unicode(self):
message = "the quick brown fox"
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="text/plain")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
m, ctype, cencoding = p._prepare(message, content_type="text/plain", content_encoding="utf-8")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
def test_publish_with_Exchange_instance(self):
p = self.connection.Producer()
p.channel = Mock()
p.publish("hello", exchange=Exchange("foo"), delivery_mode="transient")
self.assertEqual(p._channel.basic_publish.call_args[1]["exchange"], "foo")
def test_set_on_return(self):
chan = Mock()
chan.events = defaultdict(Mock)
p = Producer(ChannelPromise(lambda: chan), on_return="on_return")
p.channel
chan.events["basic_return"].add.assert_called_with("on_return")
def test_publish_retry_calls_ensure(self):
p = Producer(Mock())
p._connection = Mock()
ensure = p.connection.ensure = Mock()
p.publish("foo", exchange="foo", retry=True)
self.assertTrue(ensure.called)
def test_publish_retry_with_declare(self):
p = self.connection.Producer()
p.maybe_declare = Mock()
p.connection.ensure = Mock()
ex = Exchange("foo")
p._publish("hello", 0, "", "", {}, {}, "rk", 0, 0, ex, declare=[ex])
p.maybe_declare.assert_called_with(ex)
def test_revive_when_channel_is_connection(self):
p = self.connection.Producer()
p.exchange = Mock()
new_conn = Connection("memory://")
defchan = new_conn.default_channel
p.revive(new_conn)
self.assertIs(p.channel, defchan)
p.exchange.revive.assert_called_with(defchan)
def test_enter_exit(self):
p = self.connection.Producer()
p.release = Mock()
self.assertIs(p.__enter__(), p)
p.__exit__()
p.release.assert_called_with()
def test_connection_property_handles_AttributeError(self):
p = self.connection.Producer()
p.channel = object()
p.__connection__ = None
self.assertIsNone(p.connection)
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
message = {"the quick brown fox": "jumps over the lazy dog"}
ret = p.publish(message, routing_key="process")
self.assertIn("prepare_message", channel)
self.assertIn("basic_publish", channel)
m, exc, rkey = ret
self.assertDictEqual(message, anyjson.loads(m["body"]))
self.assertDictContainsSubset(
{"content_type": "application/json", "content_encoding": "utf-8", "priority": 0}, m
)
self.assertDictContainsSubset({"delivery_mode": 2}, m["properties"])
self.assertEqual(exc, p.exchange.name)
self.assertEqual(rkey, "process")
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
self.assertFalse(p.exchange.name)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
self.assertTrue(on_return in chan.events["basic_return"])
self.assertTrue(p.on_return)
class test_Consumer(Case):
def setUp(self):
self.connection = Connection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange('foo', 'direct')
def test_accept(self):
a = Consumer(self.connection)
self.assertIsNone(a.accept)
b = Consumer(self.connection, accept=['json', 'pickle'])
self.assertSetEqual(
b.accept,
set(['application/json', 'application/x-python-serialize']),
)
c = Consumer(self.connection, accept=b.accept)
self.assertSetEqual(b.accept, c.accept)
def test_enter_exit_cancel_raises(self):
c = Consumer(self.connection)
c.cancel = Mock(name='Consumer.cancel')
c.cancel.side_effect = KeyError('foo')
with c:
pass
c.cancel.assert_called_with()
def test_receive_callback_accept(self):
message = Mock(name='Message')
message.errors = []
callback = Mock(name='on_message')
c = Consumer(self.connection, accept=['json'], on_message=callback)
c.on_decode_error = None
c.channel = Mock(name='channel')
c.channel.message_to_python = None
c._receive_callback(message)
callback.assert_called_with(message)
self.assertSetEqual(message.accept, c.accept)
def test_accept__content_disallowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q],
exchange=self.exchange,
serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], callbacks=[callback]) as consumer:
with self.assertRaises(consumer.ContentDisallowed):
conn.drain_events(timeout=1)
self.assertFalse(callback.called)
def test_accept__content_allowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q],
exchange=self.exchange,
serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], accept=['pickle'],
callbacks=[callback]):
conn.drain_events(timeout=1)
self.assertTrue(callback.called)
body, message = callback.call_args[0]
self.assertTrue(body['complex'])
def test_set_no_channel(self):
c = Consumer(None)
self.assertIsNone(c.channel)
c.revive(Mock())
self.assertTrue(c.channel)
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
self.assertFalse(q.declare.call_count)
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
self.assertIsNone(consumer.consume())
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')]
consumer._active_tags = {'a': 1, 'b': 2}
self.assertFalse(consumer.consuming_from(Queue('c')))
self.assertFalse(consumer.consuming_from('c'))
self.assertFalse(consumer.consuming_from(Queue('d')))
self.assertFalse(consumer.consuming_from('d'))
self.assertTrue(consumer.consuming_from(Queue('a')))
self.assertTrue(consumer.consuming_from(Queue('b')))
self.assertTrue(consumer.consuming_from('b'))
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, 'message_to_python')
channel.message_to_python = None
try:
message = Mock()
message.errors = []
message.decode.return_value = 'Hello'
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with('Hello', message)
finally:
channel.message_to_python = m2p
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
callbacks = [lambda x, y: x, lambda x, y: x]
consumer = Consumer(channel,
queue,
auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare', 'queue_declare', 'queue_bind',
'basic_consume'):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count('basic_consume'), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_consumer_tag_prefix(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, tag_prefix='consumer_')
consumer.consume()
self.assertTrue(
consumer._active_tags[queue.name].startswith('consumer_'), )
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare', 'queue_declare', 'basic_consume'):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ('exchange_declare', 'queue_declare', 'queue_bind'):
self.assertIn(meth, channel)
self.assertNotIn('basic_consume', channel)
consumer.consume()
self.assertIn('basic_consume', channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn('flow', channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn('basic_qos', channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count('queue_purge'), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count('exchange_declare'), 4)
self.assertEqual(channel.called.count('queue_declare'), 4)
self.assertEqual(channel.called.count('queue_bind'), 4)
self.assertEqual(channel.called.count('basic_consume'), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count('basic_cancel'), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_ack', channel)
self.assertIn('message_to_python', channel)
self.assertEqual(received[0], {'foo': 'bar'})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
with self.assertRaises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({'foo': 'bar'})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({'foo': 'bar'})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.loads(m), {'foo': 'bar'})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn('basic_recover', channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
self.assertTrue(repr(Consumer(channel, [b1])))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
self.assertIsNone(p.connection)
class test_Consumer(Case):
def setUp(self):
self.connection = Connection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange('foo', 'direct')
def test_accept(self):
a = Consumer(self.connection)
self.assertIsNone(a.accept)
b = Consumer(self.connection, accept=['json', 'pickle'])
self.assertSetEqual(
b.accept,
set(['application/json', 'application/x-python-serialize']),
)
c = Consumer(self.connection, accept=b.accept)
self.assertSetEqual(b.accept, c.accept)
def test_enter_exit_cancel_raises(self):
c = Consumer(self.connection)
c.cancel = Mock(name='Consumer.cancel')
c.cancel.side_effect = KeyError('foo')
with c:
pass
c.cancel.assert_called_with()
def test_receive_callback_accept(self):
message = Mock(name='Message')
message.errors = []
callback = Mock(name='on_message')
c = Consumer(self.connection, accept=['json'], on_message=callback)
c.on_decode_error = None
c.channel = Mock(name='channel')
c.channel.message_to_python = None
c._receive_callback(message)
callback.assert_called_with(message)
self.assertSetEqual(message.accept, c.accept)
def test_accept__content_disallowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q], exchange=self.exchange, serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], callbacks=[callback]) as consumer:
with self.assertRaises(consumer.ContentDisallowed):
conn.drain_events(timeout=1)
self.assertFalse(callback.called)
def test_accept__content_allowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q], exchange=self.exchange, serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], accept=['pickle'],
callbacks=[callback]):
conn.drain_events(timeout=1)
self.assertTrue(callback.called)
body, message = callback.call_args[0]
self.assertTrue(body['complex'])
def test_set_no_channel(self):
c = Consumer(None)
self.assertIsNone(c.channel)
c.revive(Mock())
self.assertTrue(c.channel)
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
self.assertFalse(q.declare.call_count)
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
self.assertIsNone(consumer.consume())
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')]
consumer._active_tags = {'a': 1, 'b': 2}
self.assertFalse(consumer.consuming_from(Queue('c')))
self.assertFalse(consumer.consuming_from('c'))
self.assertFalse(consumer.consuming_from(Queue('d')))
self.assertFalse(consumer.consuming_from('d'))
self.assertTrue(consumer.consuming_from(Queue('a')))
self.assertTrue(consumer.consuming_from(Queue('b')))
self.assertTrue(consumer.consuming_from('b'))
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, 'message_to_python')
channel.message_to_python = None
try:
message = Mock()
message.errors = []
message.decode.return_value = 'Hello'
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with('Hello', message)
finally:
channel.message_to_python = m2p
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
callbacks = [lambda x, y: x,
lambda x, y: x]
consumer = Consumer(channel, queue, auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare',
'queue_declare',
'queue_bind',
'basic_consume'):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count('basic_consume'), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare',
'queue_declare',
'basic_consume'):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ('exchange_declare',
'queue_declare',
'queue_bind'):
self.assertIn(meth, channel)
self.assertNotIn('basic_consume', channel)
consumer.consume()
self.assertIn('basic_consume', channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn('flow', channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn('basic_qos', channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count('queue_purge'), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count('exchange_declare'), 4)
self.assertEqual(channel.called.count('queue_declare'), 4)
self.assertEqual(channel.called.count('queue_bind'), 4)
self.assertEqual(channel.called.count('basic_consume'), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count('basic_cancel'), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_ack', channel)
self.assertIn('message_to_python', channel)
self.assertEqual(received[0], {'foo': 'bar'})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
with self.assertRaises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({'foo': 'bar'})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({'foo': 'bar'})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.loads(m), {'foo': 'bar'})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn('basic_recover', channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
self.assertTrue(repr(Consumer(channel, [b1])))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
self.assertIsNone(p.connection)
class Amqp(Thread):
def __init__(
self,
host="localhost",
port=5672,
userid="guest",
password="******",
virtual_host="canopsis",
exchange_name="canopsis",
logging_name="Amqp",
logging_level=INFO,
read_config_file=True,
auto_connect=True,
on_ready=None,
max_retries=5,
):
super(Amqp, self).__init__()
self.logger = getLogger(logging_name)
self.host = host
self.port = port
self.userid = userid
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.logging_level = logging_level
if read_config_file:
self.read_config("amqp")
self.amqp_uri = "amqp://{0}:{1}@{2}:{3}/{4}".format(
self.userid, self.password, self.host, self.port, self.virtual_host
)
self.logger.setLevel(logging_level)
# Event sent try count before event drop in case of connection problem
if max_retries != 5:
self.logger.info(u"Custom retries value : {} {}".format(max_retries, type(max_retries)))
self.max_retries = max_retries
self.exchange_name_events = exchange_name + ".events"
self.exchange_name_alerts = exchange_name + ".alerts"
self.exchange_name_incidents = exchange_name + ".incidents"
self.chan = None
self.conn = None
self.connected = False
self.on_ready = on_ready
self.RUN = True
self.exchanges = {}
self.queues = {}
self.paused = False
self.connection_errors = (ConnectionError, error, IOError, OSError)
# Create exchange
self.logger.debug("Create exchanges object")
for exchange_name in [
self.exchange_name,
self.exchange_name_events,
self.exchange_name_alerts,
self.exchange_name_incidents,
]:
self.logger.debug(u" + {0}".format(exchange_name))
self.get_exchange(exchange_name)
if auto_connect:
self.connect()
self.logger.debug("Object canamqp initialized")
def run(self):
self.logger.debug("Start thread ...")
reconnect = False
while self.RUN:
self.connect()
if self.connected:
self.init_queue(reconnect=reconnect)
self.logger.debug("Drain events ...")
while self.RUN:
try:
if not self.paused:
self.conn.drain_events(timeout=0.5)
else:
sleep(0.5)
except timeout:
pass
except self.connection_errors as err:
self.logger.error(u"Connection error ! ({})".format(err))
break
except Exception as err:
self.logger.error(u"Error: {} ({})".format(err, type(err)))
print_exc(file=stdout)
break
self.disconnect()
if self.RUN:
self.logger.error("Connection lost, try to reconnect in few seconds ...")
reconnect = True
self.wait_connection(timeout=5)
self.logger.debug("End of thread ...")
def stop(self):
self.logger.debug("Stop thread ...")
self.RUN = False
def connect(self):
if not self.connected:
self.logger.info("Connect to AMQP Broker (%s:%s)" % (self.host, self.port))
self.conn = Connection(self.amqp_uri)
try:
self.logger.debug(" + Connect")
self.conn.connect()
self.logger.info("Connected to AMQP Broker.")
self.producers = pools.Producers(limit=10)
self.connected = True
except Exception as err:
self.conn.release()
self.logger.error(u"Impossible to connect ({})".format(err))
if self.connected:
self.logger.debug(" + Open channel")
try:
self.chan = self.conn.channel()
self.logger.debug("Channel openned. Ready to send messages")
try:
# Declare exchange
self.logger.debug("Declare exchanges")
for exchange_name in self.exchanges:
self.logger.debug(u" + {}".format(exchange_name))
self.exchanges[exchange_name](self.chan).declare()
except Exception as err:
self.logger.error(u"Impossible to declare exchange ({})".format(err))
except Exception as err:
self.logger.error(err)
else:
self.logger.debug("Already connected")
def get_exchange(self, name):
if name:
try:
return self.exchanges[name]
except:
if name == "amq.direct":
self.exchanges[name] = Exchange(name, "direct", durable=True)
else:
self.exchanges[name] = Exchange(name, "topic", durable=True, auto_delete=False)
return self.exchanges[name]
else:
return None
def init_queue(self, reconnect=False):
if self.queues:
self.logger.debug("Init queues")
for queue_name in self.queues.keys():
self.logger.debug(u" + {}".format(queue_name))
qsettings = self.queues[queue_name]
if not qsettings["queue"]:
self.logger.debug(" + Create queue")
# copy list
routing_keys = list(qsettings["routing_keys"])
routing_key = None
if len(routing_keys):
routing_key = routing_keys[0]
routing_keys = routing_keys[1:]
exchange = self.get_exchange(qsettings["exchange_name"])
if qsettings["exchange_name"] == "amq.direct" and not routing_key:
routing_key = queue_name
self.logger.debug(
(u"exchange: '{}', exclusive: {}," + " auto_delete: {},no_ack: {}").format(
qsettings["exchange_name"],
qsettings["exclusive"],
qsettings["auto_delete"],
qsettings["no_ack"],
)
)
qsettings["queue"] = Queue(
queue_name,
exchange=exchange,
routing_key=routing_key,
exclusive=qsettings["exclusive"],
auto_delete=qsettings["auto_delete"],
no_ack=qsettings["no_ack"],
channel=self.conn.channel(),
)
qsettings["queue"].declare()
if len(routing_keys):
self.logger.debug(" + Bind on all routing keys")
for routing_key in routing_keys:
self.logger.debug(" + routing_key: '{}'".format(routing_key))
try:
qsettings["queue"].bind_to(exchange=exchange, routing_key=routing_key)
except:
self.logger.error(u"You need upgrade your Kombu version ({})".format(__version__))
if not qsettings["consumer"] or reconnect:
self.logger.debug(" + Create Consumer")
qsettings["consumer"] = self.conn.Consumer(qsettings["queue"], callbacks=[qsettings["callback"]])
self.logger.debug(" + Consume queue")
qsettings["consumer"].consume()
if self.on_ready:
self.on_ready()
def add_queue(
self, queue_name, routing_keys, callback, exchange_name=None, no_ack=True, exclusive=False, auto_delete=True
):
c_routing_keys = []
if not isinstance(routing_keys, list):
if isinstance(routing_keys, basestring):
c_routing_keys = [routing_keys]
else:
c_routing_keys = routing_keys
if not exchange_name:
exchange_name = self.exchange_name
self.queues[queue_name] = {
"queue": False,
"consumer": False,
"queue_name": queue_name,
"routing_keys": c_routing_keys,
"callback": callback,
"exchange_name": exchange_name,
"no_ack": no_ack,
"exclusive": exclusive,
"auto_delete": auto_delete,
}
def publish(
self,
msg,
routing_key,
exchange_name=None,
serializer="json",
compression=None,
content_type=None,
content_encoding=None,
):
operation_success = False
retries = 0
while not operation_success and retries < self.max_retries:
retries += 1
if self.connected:
if not exchange_name:
exchange_name = self.exchange_name
with self.producers[self.conn].acquire(block=True) as producer:
try:
_msg = msg.copy()
Amqp._clean_msg_for_serialization(_msg)
producer.publish(
_msg,
serializer=serializer,
compression=compression,
routing_key=routing_key,
exchange=self.get_exchange(exchange_name),
)
self.logger.debug("publish {} in exchange {}".format(routing_key, exchange_name))
operation_success = True
except Exception as e:
self.logger.error(u" + Impossible to send {}".format(e))
self.disconnect()
self.connect()
else:
self.logger.error("Not connected ... try reconnecting")
self.connect()
if not operation_success:
# Event and it's information are buffered until next send retry
self.logger.info(u"Retry count {}".format(retries))
if not operation_success:
# Event and it's information are buffered until next send retry
self.logger.error(u"Too much retries for event {}, give up".format(routing_key))
@staticmethod
def _clean_msg_for_serialization(msg):
from bson import objectid
for key in msg:
if isinstance(msg[key], objectid.ObjectId):
msg[key] = str(msg[key])
def cancel_queues(self):
if self.connected:
for queue_name in self.queues.keys():
if self.queues[queue_name]["consumer"]:
self.logger.debug(u" + Cancel consumer on {}".format(queue_name))
try:
self.queues[queue_name]["consumer"].cancel()
except:
pass
del (self.queues[queue_name]["consumer"])
self.queues[queue_name]["consumer"] = False
del (self.queues[queue_name]["queue"])
self.queues[queue_name]["queue"] = False
def disconnect(self):
if self.connected:
self.logger.info("Disconnect from AMQP Broker")
self.cancel_queues()
for exchange in self.exchanges:
del exchange
self.exchanges = {}
try:
pools.reset()
except Exception as err:
self.logger.error(u"Impossible to reset kombu pools: {} ({})".format(err, type(err)))
try:
self.conn.release()
del self.conn
except Exception as err:
self.logger.error(u"Impossible to release connection: {} ({})".format(err, type(err)))
self.connected = False
def wait_connection(self, timeout=5):
i = 0
while self.RUN and not self.connected and i < (timeout * 2):
try:
sleep(0.5)
except:
pass
i += 1
def read_config(self, name):
filename = join(sys_prefix, "etc", u"{0}.conf".format(name))
import ConfigParser
self.config = ConfigParser.RawConfigParser()
try:
self.config.read(filename)
section = "master"
self.host = self.config.get(section, "host")
self.port = self.config.getint(section, "port")
self.userid = self.config.get(section, "userid")
self.password = self.config.get(section, "password")
self.virtual_host = self.config.get(section, "virtual_host")
self.exchange_name = self.config.get(section, "exchange_name")
except Exception as err:
self.logger.error(u"Can't to load configurations ({}), use default ...".format(err))
def __del__(self):
self.stop()
from kombu import Connection
connection = Connection('amqp://*****:*****@localhost:5672//')
connection.connect()
#check whether connected boolean
connection.connected
connection.close()
#better practice, releases resource if in a connection pool else close
connection.release()
