def __init__(self, hostname, port, vhost, userid, password):
"""connects to broker and provides convenience methods"""
self.broker = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host=vhost)
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
def send_messages(ui, repo, node, **kwargs):
# Read the config
get_configuration(ui)
# Let the user know what is going on
print "Sending messages to %s" % CONF["BROKER_HOST"]
print "Please do not interrupt..."
try:
# Get the push data
# TODO: Support pulling from the pushlog db
pushdata = get_push_data(ui, repo, node)
# If they don't want AMQP, switch the backend
if CONF["BROKER_PROTOCOL"] == "STOMP":
backend = "carrot.backends.pystomp.Backend"
else:
backend = None
# Connect once for all the messages we will send
connection = BrokerConnection(
hostname=CONF["BROKER_HOST"],
port=CONF["BROKER_PORT"],
userid=CONF["BROKER_USER"],
password=CONF["BROKER_PASS"],
backend_cls=backend,
)
# Send the overall push message. Most consumers likely
# only care about this message
print " Sending the %s message..." % CONF["LABEL_PUSH"]
send_push_message(connection, pushdata)
# Also send messages for each changeset as consumers may be
# interested in those as well
print " Sending individual %s messages..." % CONF["LABEL_CHANGESET"]
for changedata in pushdata[CONF["LABEL_CHANGESETS"]]:
changedata["repository"] = pushdata["repository"]
send_changeset_message(connection, changedata)
try:
connection.close()
except:
# We don't care about connection close failures
pass
except Exception, e:
# Something went wrong...
print "ERROR: Hook returned an exception: %s" % e
if CONF["HG_FAIL_ON_MSG_FAIL"]:
# Admin wants the hook to fail on the message send failure
print "Please try pushing again."
return 1
else:
# Admin wants the hook to succeed on the message send failure
print "Ignoring and continuing the push..."
return 0
def send_msg(routing_key, data):
conn = BrokerConnection(
hostname=settings.BROKER_HOST,
port=settings.BROKER_PORT,
userid=settings.BROKER_USER,
password=settings.BROKER_PASSWORD,
virtual_host=settings.BROKER_VHOST)
publisher = Publisher(connection=conn,
exchange="django_send",
routing_key=routing_key,
exchange_type="topic",
)
publisher.send(data)
publisher.close()
conn.close()
def main():
logging.basicConfig(level=logging.DEBUG)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
publisher = Publisher(auto_declare=False,
connection=conn,
exchange=args.exchange,
routing_key=args.key)
logging.info("Declaring exchange: %s" % args.exchange)
publisher.backend.exchange_declare(exchange=args.exchange,
type="topic",
durable=False,
auto_delete=False)
while True:
line = sys.stdin.readline()
if not line:
break
logging.debug("Sending message '%s'" % line.strip())
publisher.send(line.strip())
publisher.close()
def establish_connection(hostname=None,
userid=None,
password=None,
virtual_host=None,
port=None,
ssl=None,
insist=None,
connect_timeout=None,
backend_cls=None,
defaults=conf):
"""Establish a connection to the message broker."""
if insist is None:
insist = defaults.BROKER_INSIST
if ssl is None:
ssl = defaults.BROKER_USE_SSL
if connect_timeout is None:
connect_timeout = defaults.BROKER_CONNECTION_TIMEOUT
return BrokerConnection(hostname or defaults.BROKER_HOST,
userid or defaults.BROKER_USER,
password or defaults.BROKER_PASSWORD,
virtual_host or defaults.BROKER_VHOST,
port or defaults.BROKER_PORT,
backend_cls=backend_cls or defaults.BROKER_BACKEND,
insist=insist,
ssl=ssl,
connect_timeout=connect_timeout)
def setUp(self):
# single global RPC server thread, ok in test framework
global test_rpc_service
if not test_rpc_service:
test_rpc_service = MockRPCServiceProvider()
if not self.test_connection:
self.test_connection = BrokerConnection(**settings.AMQP_CONNECTION)
def __init__(self,
hostname='localhost',
port=5672,
userid="collectoruser",
password="******",
**options):
conn = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host='collectorvhost')
try:
self.publisher = Publisher(connection=conn,
exchange="collector.response",
exchange_type='direct',
routing_key="response",
serializer="json")
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="collector",
routing_key="collector.*.*")
def cast(msg, event_type, topic, priority):
yagi.config.setup(config_path='/etc/yagi.conf')
conf = yagi.config.config_with('rabbit_broker')
host = conf('host')
port = conf('port')
user = conf('user')
exchange = conf('exchange')
password = conf('password')
vhost = conf('vhost')
message_dict = {
'message_id': str(uuid.uuid4()),
'event_type': event_type,
'publisher_id': 'some_publisher',
'priority': priority,
'timestamp': str(datetime.datetime.utcnow()),
'payload': msg
}
conn = BrokerConnection(hostname=host, port=port, userid=user,
password=password, virtual_host=vhost)
publisher = Publisher(connection=conn, exchange=exchange,
routing_key="%s.%s" % (topic, priority), durable=False,
exchange_type='topic')
publisher.send(message_dict)
publisher.close()
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
def __init__(self, host_name, port, userid, password, virtual_host,
encoder_class):
self.q_connection = BrokerConnection(hostname=host_name,
port=port,
userid=userid,
password=password,
virtual_host=virtual_host)
self.encoder = encoder_class()
def add(self):
broker = BrokerConnection(hostname=self.hostname,
port=self.port,
userid=self.username,
password=self.password,
virtual_host=self.vhost)
with self.lock:
self._brokers.append(broker)
def add_many(self, limit):
with self.lock:
for _ in range(0, limit):
broker = BrokerConnection(hostname=self.hostname,
port=self.port,
userid=self.username,
password=self.password,
virtual_host=self.vhost)
self._brokers.append(broker)
def __init__(self, host_name, port, userid, password, virtual_host,
encoder_class):
self.q_connection = BrokerConnection(hostname=host_name,
port=port,
userid=userid,
password=password,
virtual_host=virtual_host)
self.encoder = encoder_class()
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
def test_with_statement(self):
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Publisher(connection=conn, exchange="F", routing_key="G") \
as publisher:
self.assertFalse(publisher._closed)
self.assertTrue(conn._closed)
self.assertTrue(publisher._closed)
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Consumer(connection=conn,
queue="E",
exchange="F",
routing_key="G") as consumer:
self.assertFalse(consumer._closed)
self.assertTrue(conn._closed)
self.assertTrue(consumer._closed)
def test_messaging(self):
m = TMessaging(connection=BrokerConnection(backend_cls=PyQueueBackend))
self.assertTrue(m)
self.assertEquals(m.fetch(), None)
mdata = {"name": "Cosmo Kramer"}
m.send(mdata)
next_msg = m.fetch()
next_msg_data = next_msg.decode()
self.assertEquals(next_msg_data, mdata)
self.assertEquals(m.fetch(), None)
def __init__(self, hostname, port, user_id, password, virtual_host, encoder_class):
self.queue_connection = BrokerConnection(
hostname=hostname,
port=port,
userid=user_id,
password=password,
virtual_host=virtual_host
)
self.encoder = encoder_class()
# Setup / Teardown Rabbit plumbing when spider opens / closes
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
def establish_consumer_connection(self, consumer):
# for now all consumers use the same queue connection
# That may not be the case forever
config = conf.config_with("rabbit_broker")
reconnect_delay = int(config("reconnect_delay"))
max_wait = int(config("max_wait"))
# This just sets the connection string. It doesn't actually
# connect to the AMQP server yet.
connection = BrokerConnection(hostname=config("host"),
port=config("port"),
userid=config("user"),
password=config("password"),
virtual_host=config("vhost"),
ssl=confbool("ssl"))
auto_delete = consumer.config("auto_delete") == "True" or False
durable = consumer.config("durable") == "True" or False
exdurable = confbool(consumer.config("exchange_durable"))
exauto_delete = confbool(consumer.config("exchange_auto_delete"))
# try a few times to connect, we might have lost the connection
retries = 0
while True:
try:
carrot_consumer = NotQuiteSoStupidConsumer(
connection=connection,
warn_if_exists=True,
exchange=consumer.config("exchange"),
exchange_type=consumer.config("exchange_type"),
routing_key=consumer.config("routing_key"),
queue=consumer.queue_name,
auto_delete=auto_delete,
durable=durable,
exchange_durable=exdurable,
exchange_auto_delete=exauto_delete,
)
consumer.connect(connection, carrot_consumer)
LOG.info("Connection established for %s" % consumer.queue_name)
break
except amqplib.client_0_8.exceptions.AMQPConnectionException, e:
LOG.error("AMQP protocol error connecting to for queue %s" %
consumer.queue_name)
LOG.exception(e)
except socket.error, e:
# lost connection?
pass
def _wait_connection(self,time_to_wait=60):
"""Waits for RabbitMQ connection. """
_connected = False
while(not _connected):
try:
self.amqp_connection = BrokerConnection(**self.amqp_connection_settings)
if self.amqp_connection.connection:
_connected = True
logging.debug('%s connected to AMQP @ %s:%i' % (
self.service_name,
self.amqp_connection_settings['hostname'],
self.amqp_connection_settings['port']))
except socket_error:
logging.warn("%s waiting %i sec for RabbitMQ connection..." % (
self.service_name, time_to_wait))
sleep(time_to_wait)
def get_carrot_connection(config):
backend = config.get('queue.library', 'pyamqplib')
log.info("Carrot connnection using %s backend" % backend)
try:
port = int(config.get('queue.port', PORT))
except ValueError:
port = PORT
userid = config.get('queue.user_id', USERID)
password = config.get('queue.password', PASSWORD)
hostname = config.get('queue.host', HOSTNAME)
virtual_host = config.get('queue.virtual_host', VIRTUAL_HOST)
backend_cls = 'carrot.backends.%s.Backend' % backend
return BrokerConnection(hostname=hostname, port=port,
userid=userid, password=password,
virtual_host=virtual_host,
backend_cls=backend_cls)
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn,
queue="test",
exchange="test",
routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn,
exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
def main():
logging.basicConfig(level=logging.INFO)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
consumer = Consumer(
auto_declare=False,
connection=conn,
exclusive=True,
auto_delete=True,
)
try:
logging.info("Creating exchange: %s" % args.exchange)
consumer.backend.exchange_declare(exchange=args.exchange,
type="topic",
durable=False,
auto_delete=True)
except Exception, e:
logging.warning("Failed to create exchange: %s" % e)
def create_connection():
return BrokerConnection(backend_cls=StompBackend,
**test_stomp_connection_args())
from carrot.connection import BrokerConnection
from config import BROKER_CONFIG, PRINTER_CONFIG
from pointofsale import printer
if __name__ == '__main__':
conn = BrokerConnection(**BROKER_CONFIG)
registry = printer.PrinterRegistry(PRINTER_CONFIG['registry'],
PRINTER_CONFIG['auth_token'])
p = printer.TestPrinter(PRINTER_CONFIG['name'], registry, conn)
pm_handler = printer.PrinterMessageThread(p)
pm_handler.start()
while True:
pm_handler.join(1)
if not pm_handler.is_alive():
break
class IMessageBroker(Consumer, Thread):
"""Interface for communicating JSON over AMQP.
Consumer thread interface that can also publish a response for each request with unique id.
Consumer and publisher use the same AMQP connection.
Subclass this to implement a worker thread. See the example in the tests.
"""
# implemented in subclass
service_name = None
amqp_connection = None
amqp_connection_settings = None
exchange_name = None
topic = None
# dynamic instance variables
binding_key = None
response_routing_key = None
req_queue_name = None
def __init__(self, *args, **kwargs):
logging.debug("%s starting up" % self.__unicode__())
# set instance variables
self.binding_key = '.'.join([self.topic, 'request', '*'])
self.response_routing_key = '.'.join([self.topic, 'response', '%s'])
self.req_queue_name = '%s_req' % self.service_name
# initialize the thread
Thread.__init__(self, *args, **kwargs)
# wait for AMQP connection, declare exchange and request queue
# and bind the consumer
self._wait_connection()
self._declare_channel()
self._init_consumer()
def __exit__(self, e_type, e_value, e_trace):
logging.debug('%s exiting' % self.__unicode__())
if e_type:
logging.error(e_type(e_value))
def __unicode__(self):
return self.service_name
def _declare_channel(self):
"""Declares exchange and request queue."""
backend = self.amqp_connection.create_backend()
backend.exchange_declare(
exchange=self.exchange_name,
type="topic",
durable=True,
auto_delete=False,)
backend.queue_declare(
queue=self.req_queue_name,
durable=True,
exclusive=False,
auto_delete=False,)
backend.queue_bind(self.req_queue_name,self.exchange_name,self.binding_key)
logging.debug("%s queue %s bound to %s" % (
self.exchange_name,
self.req_queue_name,
self.binding_key))
def _wait_connection(self,time_to_wait=60):
"""Waits for RabbitMQ connection. """
_connected = False
while(not _connected):
try:
self.amqp_connection = BrokerConnection(**self.amqp_connection_settings)
if self.amqp_connection.connection:
_connected = True
logging.debug('%s connected to AMQP @ %s:%i' % (
self.service_name,
self.amqp_connection_settings['hostname'],
self.amqp_connection_settings['port']))
except socket_error:
logging.warn("%s waiting %i sec for RabbitMQ connection..." % (
self.service_name, time_to_wait))
sleep(time_to_wait)
def _init_consumer(self):
Consumer.__init__(self,
connection=self.amqp_connection,
exchange=self.exchange_name,
exchange_type="topic",
queue=self.req_queue_name,
)
# register the callback method
self.register_callback(self._request_filter) # calls "dispatch"
self._stop = Event()
self._stop.set() # set to stopped state
def _request_filter(self, message_data, message):
"""Filter for incoming AMQP requests.
Expects JSON input as message payload. It has a metadata wrapper with keys
"q" and "qid", for actual message data and an identifier, respectively.
Example:
{"qid": 293742, "q": {"hello my friend . we speak over the RPC now . "}}
"""
logging.debug("consumer received message: \n%s" % message.delivery_info)
#routing_key = message.delivery_info['routing_key']
qid = None
try:
data = json.loads(message_data)
qid = data.get('qid')
request = data.get('q')
self.dispatch(message, request, qid)
except:
logging.error(sys.exc_info()[1])
return
def dispatch(self, message, request, qid):
"""AMQP "request" callback handler.
- message: carrot Message
- request: JSON data
- qid: unique request identifier
Do whatever your worker is supposed to do here.
You can return a response by calling self.return_response(json_response, qid).
"""
raise NotImplementedError, "Write your own subclass"
def return_response(self, response, qid):
"""AMQP "response" handler.
Publishes a response message to a temporary queue.
- response is json, qid is string
"""
message = json.dumps(response)
routing_key = self.response_routing_key % qid
logging.debug("response to %s with routing_key: %s, message: \n%s" % (self.exchange_name, routing_key, message))
try:
publisher = Publisher(
connection=self.amqp_connection,
exchange=self.exchange_name,
exchange_type="topic",
routing_key=routing_key,
)
publisher.send(message)
publisher.close()
except:
"""Trying to send with broken connection.
Handle gracefully by waiting for connection and publish again."""
logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1]))
self._wait_connection(5)
self.return_response(response, qid)
@property
def stopped(self):
return self._stop.isSet()
def run(self):
self._stop.clear() # started
logging.info('%s entering AMQP consumer loop' % self.__unicode__())
it = self.iterconsume()
while not self.stopped:
try:
it.next()
except KeyboardInterrupt:
logging.warn("KB interrupt")
except (AMQPConnectionException, IOError, AttributeError):
"""AMQPConnectionException or IOError if the AMQ socket closes.
AttributeError if channel has been closed.
Handle gracefully by waiting for connection and re-enter consumer loop,
unless the service stop flag is set.
"""
if self.stopped:
break
# else reconnect
logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1]))
self._wait_connection()
self._init_consumer()
self._stop.clear() # continue in started mode
it = self.iterconsume()
continue
except StopIteration:
"""Happens at cancel() and on lost connection,
if IOError exception handler could not restore consumption loop."""
logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1]))
logging.info('%s break out of consumer loop' % self.__unicode__())
self._stop.set()
break
except:
logging.error("BUG")
logging.error('%s %s' % (self.__unicode__(), sys.exc_info()[1]))
traceback.print_tb(sys.exc_info()[2],10)
try:
self.amqp_connection.close()
except:
logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1]))
logging.info('%s thread exiting' % self.__unicode__())
exit()
def stop(self,*args):
"""Stop consumer"""
logging.debug('%s stop' % self.__unicode__())
try:
try:
# set the Event
self._stop.set()
# release the consumer from iterconsume
self.cancel()
except:
pass
#logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1]))
finally:
try:
self.amqp_connection.close()
except IOError:
logging.debug(sys.exc_info()[1])
if self.amqp_connection.pool:
self.amqp_connection.release()
class GenericPublisher(object):
def __init__(self, config, exchange=None, connect=True):
self.config = config
self.exchange = exchange
self.connection = None
if connect:
self.connect()
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
self.connection = None
# Used to publish a pulse message to the proper exchange
def publish(self, message):
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a message given
if not message:
raise MalformedMessage(message)
# Have the message prepare and validate itself
message._prepare()
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker publisher
self.publisher = Publisher(connection=self.connection,
exchange=self.exchange,
exchange_type="topic",
routing_key=message.routing_key)
# The message is actually a simple envelope format with a payload and
# some metadata
final_data = {}
final_data['payload'] = message.data
final_data['_meta'] = message.metadata.copy()
final_data['_meta'].update({
'exchange':
self.exchange,
'routing_key':
message.routing_key,
'serializer':
self.config.serializer,
'sent':
time_to_string(datetime.now(timezone(self.config.broker_timezone)))
})
# Send the message
self.publisher.send(final_data, serializer=self.config.serializer)
# Close the publishing connection
self.publisher.close()
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
if __name__ == '__main__':
connection = BrokerConnection(hostname='localhost',
port=5672,
userid='guest',
password='******',
virtual_host=None)
consumer = Consumer(connection=connection,
queue="transactions",
exchange="transactions")
def print_message(message_data, message):
print(message_data)
message.ack()
consumer.register_callback(print_message)
consumer.wait()
class GenericConsumer(object):
def __init__(self, config, exchange=None, connect=True, heartbeat=False,
**kwargs):
self.config = config
self.exchange = exchange
self.connection = None
self.durable = False
self.applabel = ''
self.heartbeat = heartbeat
for x in ['applabel','topic','callback','durable']:
if x in kwargs:
setattr(self, x, kwargs[x])
del kwargs[x]
if connect:
self.connect()
# Sets vairables
def configure(self, **kwargs):
for x in kwargs:
setattr(self, x, kwargs[x])
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
# Support purging messages that are already in the queue on the broker
# TODO: I think this is only supported by the amqp backend
def purge_existing_messages(self):
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Purge the queue of existing messages
self.connection.create_backend().queue_purge(self.applabel)
# Blocks and calls the callback when a message comes into the queue
# For info on one script listening to multiple channels, see
# http://ask.github.com/carrot/changelog.html#id1
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
def create_backend():
return PyQueueBackend(connection=BrokerConnection())
def __init__(self, hostname, port, vhost, userid, password):
"""connects to broker and provides convenience methods"""
self.broker = BrokerConnection(hostname=hostname, port=port,
userid=userid, password=password,
virtual_host=vhost)
class Connection:
def __init__(self, hostname, port, vhost, userid, password):
"""connects to broker and provides convenience methods"""
self.broker = BrokerConnection(hostname=hostname, port=port,
userid=userid, password=password,
virtual_host=vhost)
def __del__(self):
self.broker.close()
def declare(self, exchange, exchange_type, binding="", queue=""):
"""declares the exchange, the queue and binds the queue to the exchange
exchange - exchange name
exchange_type - direct, topic, fanout
binding - binding to queue (optional)
queue - queue to bind to exchange using binding (optional)
"""
if (binding and not queue) or (queue and not binding):
if queue and not exchange_type == "fanout":
raise Error("binding and queue are not mutually exclusive")
consumer = Consumer(connection=self.broker,
exchange=exchange, exchange_type=exchange_type,
routing_key=binding, queue=queue)
consumer.declare()
consumer.close()
def consume(self, queue, limit=None, callback=None, auto_declare=False):
"""consume messages in queue
queue - name of queue
limit - amount of messages to iterate through (default: no limit)
callback - method to call when a new message is received
must take two arguments: message_data, message
must send the acknowledgement: message.ack()
default: print message to stdout and send ack
auto_declare - automatically declare the queue (default: false)
"""
if not callback:
callback = _consume_callback
consumer = Consumer(connection=self.broker, queue=queue,
auto_declare=auto_declare)
consumer.register_callback(callback)
for message in consumer.iterqueue(limit=limit, infinite=False):
consumer.receive(message.payload, message)
consumer.close()
def publish(self, exchange, routing_key, message,
auto_declare=False, persistent=True):
"""publish a message to exchange using routing_key
exchange - name of exchange
routing_key - interpretation of routing key depends on exchange type
message - message content to send
auto_declare - automatically declare the exchange (default: false)
persistent - store message on disk as well as memory (default: True)
"""
delivery_mode = 2
if not persistent:
delivery_mode = 1
publisher = Publisher(connection=self.broker,
exchange=exchange, routing_key=routing_key,
auto_declare=auto_declare)
publisher.send(message, delivery_mode=delivery_mode)
publisher.close()
class Connection:
def __init__(self, hostname, port, vhost, userid, password):
"""connects to broker and provides convenience methods"""
self.broker = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host=vhost)
def __del__(self):
self.broker.close()
def declare(self, exchange, exchange_type, binding="", queue=""):
"""declares the exchange, the queue and binds the queue to the exchange
exchange - exchange name
exchange_type - direct, topic, fanout
binding - binding to queue (optional)
queue - queue to bind to exchange using binding (optional)
"""
if (binding and not queue) or (queue and not binding):
if queue and not exchange_type == "fanout":
raise Error("binding and queue are not mutually exclusive")
consumer = Consumer(connection=self.broker,
exchange=exchange,
exchange_type=exchange_type,
routing_key=binding,
queue=queue)
consumer.declare()
consumer.close()
def consume(self, queue, limit=None, callback=None, auto_declare=False):
"""consume messages in queue
queue - name of queue
limit - amount of messages to iterate through (default: no limit)
callback - method to call when a new message is received
must take two arguments: message_data, message
must send the acknowledgement: message.ack()
default: print message to stdout and send ack
auto_declare - automatically declare the queue (default: false)
"""
if not callback:
callback = _consume_callback
consumer = Consumer(connection=self.broker,
queue=queue,
auto_declare=auto_declare)
consumer.register_callback(callback)
for message in consumer.iterqueue(limit=limit, infinite=False):
consumer.receive(message.payload, message)
consumer.close()
def publish(self,
exchange,
routing_key,
message,
auto_declare=False,
persistent=True):
"""publish a message to exchange using routing_key
exchange - name of exchange
routing_key - interpretation of routing key depends on exchange type
message - message content to send
auto_declare - automatically declare the exchange (default: false)
persistent - store message on disk as well as memory (default: True)
"""
delivery_mode = 2
if not persistent:
delivery_mode = 1
publisher = Publisher(connection=self.broker,
exchange=exchange,
routing_key=routing_key,
auto_declare=auto_declare)
publisher.send(message, delivery_mode=delivery_mode)
publisher.close()
def establish_test_connection():
return BrokerConnection(**test_connection_args())
"""Message-classes."""
import os, subprocess
from os.path import join, split, splitext, exists
from carrot.connection import BrokerConnection
conn = BrokerConnection(hostname="localhost", port=5672,
userid="test", password="******",
virtual_host="test")
arrot.messaging import Consumer
consumer = Consumer(connection=conn, queue="feed",
exchange="feed", routing_key="importer")
def import_feed_callback(message_data, message):
feed_url = message_data["import_feed"]
print("Got feed import message for: %s" % feed_url)
# something importing this feed url
# import_feed(feed_url)
message.ack()
consumer.register_callback(import_feed_callback)
consumer.wait() # Go into the consumer loop.
class GenericConsumer(object):
def __init__(self,
config,
exchange=None,
connect=True,
heartbeat=False,
**kwargs):
self.config = config
self.exchange = exchange
self.connection = None
self.durable = False
self.applabel = ''
self.heartbeat = heartbeat
for x in ['applabel', 'topic', 'callback', 'durable']:
if x in kwargs:
setattr(self, x, kwargs[x])
del kwargs[x]
if connect:
self.connect()
# Sets vairables
def configure(self, **kwargs):
for x in kwargs:
setattr(self, x, kwargs[x])
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
# Support purging messages that are already in the queue on the broker
# TODO: I think this is only supported by the amqp backend
def purge_existing_messages(self):
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Purge the queue of existing messages
self.connection.create_backend().queue_purge(self.applabel)
# Blocks and calls the callback when a message comes into the queue
# For info on one script listening to multiple channels, see
# http://ask.github.com/carrot/changelog.html#id1
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(
queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(
queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
import jsonrpclib
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/")
consumer = Consumer(connection=conn, queue="po_box", exchange="sorting_room", routing_key="jason")
def amqp_callback(message_data, message):
server = jsonrpclib.Server("http://localhost:8080")
server.ping(message_data)
print jsonrpclib.history.response
message.ack()
consumer.register_callback(amqp_callback)
consumer.wait() # Go into the consumer loop.
conn.close()
class GenericPublisher(object):
def __init__(self, config, exchange=None, connect=True):
self.config = config
self.exchange = exchange
self.connection = None
if connect:
self.connect()
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
self.connection = None
# Used to publish a pulse message to the proper exchange
def publish(self, message):
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a message given
if not message:
raise MalformedMessage(message)
# Have the message prepare and validate itself
message._prepare()
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker publisher
self.publisher = Publisher(connection=self.connection,
exchange=self.exchange,
exchange_type="topic",
routing_key=message.routing_key)
# The message is actually a simple envelope format with a payload and
# some metadata
final_data = {}
final_data['payload'] = message.data
final_data['_meta'] = message.metadata.copy()
final_data['_meta'].update({
'exchange': self.exchange,
'routing_key': message.routing_key,
'serializer': self.config.serializer,
'sent': time_to_string(datetime.now(timezone(self.config.broker_timezone)))
})
# Send the message
self.publisher.send(final_data, serializer=self.config.serializer)
# Close the publishing connection
self.publisher.close()
class AmqpbusTests(unittest.TestCase):
"""Tests the AMQP communication.
"""
test_connection = None
def setUp(self):
# single global RPC server thread, ok in test framework
global test_rpc_service
if not test_rpc_service:
test_rpc_service = MockRPCServiceProvider()
if not self.test_connection:
self.test_connection = BrokerConnection(**settings.AMQP_CONNECTION)
def tearDown(self):
test_rpc_service.stop()
self.test_connection.close()
def test_consumer(self):
"""AMQP->RPC->AMQP
Send a AMQP message, and test RPCConsumer response.
Uses a mock RPC server that the consumer will call.
Check the consumer AMQP response with test consumer.
"""
class TestConsumer(IMessageBroker):
service_name = 'TestConsumer'
exchange_name = 'Test'
topic = 'test'
amqp_connection_settings = settings.AMQP_CONNECTION
def dispatch(self, message, request, qid):
""" AMQP -> RPC dispatcher.
"""
logging.info('dispatching AMQP -> RPC')
response = {}
if qid:
# call remote RPC
# this message has an qid so it expects a response
response['msg'] = test_rpc_service.push(request)
message.ack()
# return the response to AMQP, the caller should be listening ..
self.return_response(response,qid)
else:
# no qid, so do something stateless ..
print request
message.ack()
try:
consumer = TestConsumer()
self.assertEquals('test.request.*',consumer.binding_key)
self.assertEquals('test.response.%s',consumer.response_routing_key)
consumer.start()
test_rpc_service.start()
test_rpc_service.messages = []
# allow consumer to start
sleep(0.2)
self.assert_(not consumer.stopped)
self.assert_(consumer.isAlive())
# test variables
qid = str(randrange(0,999999))
jsondata = {'msg':'hello rpc'}
_publisher = Publisher(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
routing_key='test.request.'+qid,
)
# test channel
backend = self.test_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,)
backend.queue_bind("test",'Test','test.response.'+qid)
_consumer = Consumer(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
queue="test",
)
_consumer.discard_all()
logging.debug('publishing JSON message to RPC')
data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid})
_publisher.send(data_on_the_wire)
# allow data to pass the wire
sleep(0.2)
# retrieve dispatcher response
response = _consumer.fetch()
self.assert_(response, 'No response')
data = json.loads(response.payload)
self.assert_(len(data['msg']) > 0) # assert non-empty response
self.assertEquals('ok',data['msg'])
# check dispatcher RPC function
self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive')
self.assertEquals(test_rpc_service.messages[0], jsondata)
finally:
try:
consumer.stop()
except:
pass
try:
_consumer.close()
except:
pass
test_rpc_service.stop()
self.assert_(consumer.stopped)
