class Worker(object):
def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options):
conn = BrokerConnection(
hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost"
)
try:
self.publisher = Publisher(
connection=conn,
exchange="collector.response",
exchange_type="direct",
routing_key="response",
serializer="json",
)
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(
connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*"
)
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
# print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj["tags"]:
if driver:
value = driver.read_value(tag["device"], tag["address"])
else:
value = "BAD_DRIVER"
values.append({"name": tag["name"], "current_value": value, "read_at": "2009-01-01"})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
def create_README_consumer(amqpconn):
from carrot.messaging import Consumer
consumer = Consumer(connection=amqpconn,
queue=README_QUEUE, exchange=README_EXCHANGE,
routing_key=README_ROUTING_KEY)
tcallbacks = CallbacksTestable()
consumer.register_callback(tcallbacks.import_feed)
consumer.register_callback(tcallbacks.dump_message)
return consumer, tcallbacks
def consumer(self):
conf = self.app.conf
consumer = Consumer(self.connection,
queue=conf.CELERY_EVENT_QUEUE,
exchange=conf.CELERY_EVENT_EXCHANGE,
exchange_type=conf.CELERY_EVENT_EXCHANGE_TYPE,
routing_key=conf.CELERY_EVENT_ROUTING_KEY,
no_ack=True)
consumer.register_callback(self._receive)
return consumer
def consume(self, queue, limit=None, callback=None, auto_declare=False):
"""consume messages in queue
queue - name of queue
limit - amount of messages to iterate through (default: no limit)
callback - method to call when a new message is received
must take two arguments: message_data, message
must send the acknowledgement: message.ack()
default: print message to stdout and send ack
auto_declare - automatically declare the queue (default: false)
"""
if not callback:
callback = _consume_callback
consumer = Consumer(connection=self.broker,
queue=queue,
auto_declare=auto_declare)
consumer.register_callback(callback)
for message in consumer.iterqueue(limit=limit, infinite=False):
consumer.receive(message.payload, message)
consumer.close()
def _init_consumer(self):
Consumer.__init__(self,
connection=self.amqp_connection,
exchange=self.exchange_name,
exchange_type="topic",
queue=self.req_queue_name,
)
# register the callback method
self.register_callback(self._request_filter) # calls "dispatch"
self._stop = Event()
self._stop.set() # set to stopped state
def test_consumer_options(self):
opposite_defaults = {
"queue": "xyxyxyxy",
"exchange": "xyxyxyxy",
"routing_key": "xyxyxyxy",
"durable": False,
"exclusive": True,
"auto_delete": True,
"exchange_type": "topic",
}
consumer = Consumer(connection=self.conn, **opposite_defaults)
for opt_name, opt_value in opposite_defaults.items():
self.assertEquals(getattr(consumer, opt_name), opt_value)
consumer.close()
def test_consumer_options(self):
opposite_defaults = {
"queue": "xyxyxyxy",
"exchange": "xyxyxyxy",
"routing_key": "xyxyxyxy",
"durable": False,
"exclusive": True,
"auto_delete": True,
"exchange_type": "topic",
}
consumer = Consumer(connection=self.conn, **opposite_defaults)
for opt_name, opt_value in opposite_defaults.items():
self.assertEquals(getattr(consumer, opt_name), opt_value)
consumer.close()
def create_consumer(self, **options):
queue = "%s%s" % (self.queue, self.nextq())
return Consumer(connection=self.conn,
queue=queue,
exchange=self.exchange,
routing_key=self.routing_key,
**options)
def get_consumer(config, queue_name, routing_key):
return Consumer(connection=get_carrot_connection(config),
queue=queue_name,
routing_key=routing_key,
exchange=config.get('ckan.site_id'),
exchange_type=EXCHANGE_TYPE,
durable=True, auto_delete=False)
def declare(self, exchange, exchange_type, binding="", queue=""):
"""declares the exchange, the queue and binds the queue to the exchange
exchange - exchange name
exchange_type - direct, topic, fanout
binding - binding to queue (optional)
queue - queue to bind to exchange using binding (optional)
"""
if (binding and not queue) or (queue and not binding):
if queue and not exchange_type == "fanout":
raise Error("binding and queue are not mutually exclusive")
consumer = Consumer(connection=self.broker,
exchange=exchange, exchange_type=exchange_type,
routing_key=binding, queue=queue)
consumer.declare()
consumer.close()
def get_consumer(queue_name, routing_key):
return Consumer(connection=get_carrot_connection(),
queue=queue_name,
routing_key=routing_key,
exchange=EXCHANGE_NAME,
exchange_type=EXCHANGE_TYPE,
durable=True,
auto_delete=False)
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn, queue="test",
exchange="test", routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn, exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
def request_with_response():
"""Example of requesting over AMQP from another process, not necessarily Python."""
qid = str(randrange(0,999))
data = {'qid': qid, 'q': 'what time is it?'}
broker = ExampleBroker()
publisher = Publisher(
connection = broker.amqp_connection,
exchange = broker.exchange_name,
exchange_type = "topic",
routing_key = broker.binding_key,
)
# declare test queue for receiving response message
backend = broker.amqp_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,)
backend.queue_bind(
"test",
broker.exchange_name,
broker.response_routing_key % qid)
consumer = Consumer(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
queue="test",
)
consumer.discard_all()
# send message
publisher.send(json.dumps(data))
# allow data to pass the wire
sleep(0.2)
# retrieve response
response = consumer.fetch()
payload = json.loads(response.payload)
print "Response from AMQP: %s" % payload
def request_with_response():
"""Example of requesting over AMQP from another process, not necessarily Python."""
qid = str(randrange(0, 999))
data = {'qid': qid, 'q': 'what time is it?'}
broker = ExampleBroker()
publisher = Publisher(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
routing_key=broker.binding_key,
)
# declare test queue for receiving response message
backend = broker.amqp_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,
)
backend.queue_bind("test", broker.exchange_name,
broker.response_routing_key % qid)
consumer = Consumer(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
queue="test",
)
consumer.discard_all()
# send message
publisher.send(json.dumps(data))
# allow data to pass the wire
sleep(0.2)
# retrieve response
response = consumer.fetch()
payload = json.loads(response.payload)
print "Response from AMQP: %s" % payload
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
class AmqpMessageProcessor(object):
purge_every = datetime.timedelta(minutes=10)
conn_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2))
def __init__(self, connection):
self.pygo_mp = PyGoWaveClientMessageProcessor()
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json"
)
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def wait(self, limit=None):
self.consumer.wait(limit)
def send(self, routing_key, message_data):
self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1)
def receive(self, message_data, message):
routing_key = message.amqp_message.routing_key
msg_dict = self.pygo_mp.process(routing_key, message_data)
for out_rkey, messages in msg_dict.iteritems():
self.send(out_rkey, messages)
# Cleanup time?
if datetime.datetime.now() > self.next_purge:
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def get_responses(logger=logging):
connection = DjangoBrokerConnection()
consumer = Consumer(connection=connection,
exchange="collector.response",
queue="responses",
routing_key="response")
for message in consumer.iterqueue():
responses = message.payload
for resp in responses:
logger.debug("resp=%s" % resp)
try:
tag = Tag.objects.get(name=resp['name'])
tag.save_with_history(resp['current_value'])
except Exception as ex:
logger.error(ex)
#print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
message.ack()
consumer.close()
connection.close()
def get_responses(logger=logging):
connection = DjangoBrokerConnection()
consumer = Consumer(connection=connection,
exchange="collector.response",
queue="responses",
routing_key="response")
for message in consumer.iterqueue():
responses = message.payload
for resp in responses:
logger.debug("resp=%s" % resp )
try:
tag=Tag.objects.get(name=resp['name'])
tag.save_with_history(resp['current_value'])
except Exception as ex:
logger.error(ex)
#print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
message.ack()
consumer.close()
connection.close()
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn,
queue="test",
exchange="test",
routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn,
exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
def do_consume():
consumer = Consumer(ctx.broker, exchange='testx', queue='testq',
routing_key='testq', exclusive=True, durable=False)
consumer.register_callback(got_message_cb)
try:
consumer.wait(limit=1)
except StopIteration:
pass
except GreenletExit:
pass
finally:
consumer.close()
def get_consumer(queue_name, routing_key):
connection = get_carrot_connection()
try:
return Consumer(connection=connection,
queue=queue_name,
routing_key=routing_key,
exchange=EXCHANGE_NAME,
exchange_type=EXCHANGE_TYPE,
durable=True,
auto_delete=False)
except socket.error, e:
log.error('Error connecting to RabbitMQ with settings: %r',
connection.__dict__)
raise
def save_hits():
conn = DjangoAMQPConnection()
consumer = Consumer(connection=conn, queue='requestrequesthitqueue',
exchange='request',
routing_key='request.*', exchange_type='topic')
count = 0
messages = []
for message in consumer.iterqueue():
messages.append(message)
request = message.decode()
request.path = convert_unicode_to_string(request.path[:255])
request.data = convert_unicode_to_string(request.data[:255])
request.referer = convert_unicode_to_string(request.referer[:255])
request.user_agent =\
convert_unicode_to_string(request.user_agent[:255])
request.language = convert_unicode_to_string(request.language[:255])
request.save()
count += 1
logger.info("Saved {0} requests".format(count))
[m.ack() for m in messages]
logger.debug("Acknowledged all messages")
consumer.close()
conn.close()
def create_README_consumer(amqpconn):
from carrot.messaging import Consumer
consumer = Consumer(connection=amqpconn,
queue=README_QUEUE, exchange=README_EXCHANGE,
routing_key=README_ROUTING_KEY)
tcallbacks = CallbacksTestable()
consumer.register_callback(tcallbacks.import_feed)
consumer.register_callback(tcallbacks.dump_message)
return consumer, tcallbacks
def get(self, exchange_name, queue_name, routing_key):
def msg_callback(message_data, message):
message.ack()
self.write(simplejson.dumps(message_data))
self.finish()
consumer = Consumer(connection=conn, queue=queue_name,
exchange=exchange_name, routing_key=routing_key)
consumer.register_callback(msg_callback)
try:
consumer.wait(1)
except StopIteration:
consumer.close()
def get_consumer_set(connection, queues=None, **options):
"""Get the :class:`carrot.messaging.ConsumerSet`` for a queue
configuration.
Defaults to the queues in :const:`CELERY_QUEUES`.
"""
queues = queues or conf.get_queues()
cset = ConsumerSet(connection)
for queue_name, queue_options in queues.items():
queue_options = dict(queue_options)
queue_options["routing_key"] = queue_options.pop("binding_key", None)
consumer = Consumer(connection,
queue=queue_name,
backend=cset.backend,
**queue_options)
cset.consumers.append(consumer)
return cset
def __init__(self, connection):
self.pygo_mp = PyGoWaveClientMessageProcessor()
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json"
)
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def test_with_statement(self):
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Publisher(connection=conn, exchange="F", routing_key="G") \
as publisher:
self.assertFalse(publisher._closed)
self.assertTrue(conn._closed)
self.assertTrue(publisher._closed)
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Consumer(connection=conn,
queue="E",
exchange="F",
routing_key="G") as consumer:
self.assertFalse(consumer._closed)
self.assertTrue(conn._closed)
self.assertTrue(consumer._closed)
def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options):
conn = BrokerConnection(
hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost"
)
try:
self.publisher = Publisher(
connection=conn,
exchange="collector.response",
exchange_type="direct",
routing_key="response",
serializer="json",
)
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(
connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*"
)
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def __init__(self, connection):
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection,
exchange="wavelet.direct",
exchange_type="direct",
delivery_mode=1,
serializer="json",
)
self.out_queue = {}
self.purge_connections()
def disabled_publisher_mandatory_flag_regr16(self):
"""
Test that the publisher "mandatory" flag
raises exceptions at appropriate times.
"""
routing_key = 'black_hole'
assert self.conn.connection is not None
message = {'foo': 'mandatory'}
# sanity check cleanup from last test
assert not self.create_consumer().backend.queue_exists(routing_key)
publisher = self.create_publisher()
# this should just get discarded silently, it's not mandatory
publisher.send(message, routing_key=routing_key, mandatory=False)
# This raises an unspecified exception because there is no queue to
# deliver to
self.assertRaises(Exception,
publisher.send,
message,
routing_key=routing_key,
mandatory=True)
# now bind a queue to it
consumer = Consumer(connection=self.conn,
queue=routing_key,
exchange=self.exchange,
routing_key=routing_key,
durable=False,
exclusive=True)
# check that it exists
assert self.create_consumer().backend.queue_exists(routing_key)
# this should now get routed to our consumer with no exception
publisher.send(message, routing_key=routing_key, mandatory=True)
def consume(self, queue, limit=None, callback=None, auto_declare=False):
"""consume messages in queue
queue - name of queue
limit - amount of messages to iterate through (default: no limit)
callback - method to call when a new message is received
must take two arguments: message_data, message
must send the acknowledgement: message.ack()
default: print message to stdout and send ack
auto_declare - automatically declare the queue (default: false)
"""
if not callback:
callback = _consume_callback
consumer = Consumer(connection=self.broker, queue=queue,
auto_declare=auto_declare)
consumer.register_callback(callback)
for message in consumer.iterqueue(limit=limit, infinite=False):
consumer.receive(message.payload, message)
consumer.close()
def main():
connection = BrokerConnection(
hostname = "localhost",
port = 5672,
userid = "test",
password = "******",
virtual_host = "test.com",
)
consumer = Consumer(
connection = connection,
queue = "messages",
exchange = "messages",
routing_key = "awesome",
)
consumer.register_callback(worker_callback)
try:
consumer.wait()
except KeyboardInterrupt:
consumer.close()
def main():
logging.basicConfig(level=logging.INFO)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
consumer = Consumer(
auto_declare=False,
connection=conn,
exclusive=True,
auto_delete=True,
)
try:
logging.info("Creating exchange: %s" % args.exchange)
consumer.backend.exchange_declare(exchange=args.exchange,
type="topic",
durable=False,
auto_delete=True)
except Exception, e:
logging.warning("Failed to create exchange: %s" % e)
def declare(self, exchange, exchange_type, binding="", queue=""):
"""declares the exchange, the queue and binds the queue to the exchange
exchange - exchange name
exchange_type - direct, topic, fanout
binding - binding to queue (optional)
queue - queue to bind to exchange using binding (optional)
"""
if (binding and not queue) or (queue and not binding):
if queue and not exchange_type == "fanout":
raise Error("binding and queue are not mutually exclusive")
consumer = Consumer(connection=self.broker,
exchange=exchange,
exchange_type=exchange_type,
routing_key=binding,
queue=queue)
consumer.declare()
consumer.close()
#!/usr/bin/env python
from carrot.messaging import Consumer
from carrot.connection import BrokerConnection
conn = BrokerConnection(hostname='localhost',
port=5672,
userid='test',
password='******',
virtual_host='test')
consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="fst",
routing_key="easyip.*")
#publisher = Publisher(connection=conn, exchange="fst", exchange_type='topic',
# routing_key="easyip.incomming", serializer="json")
def get_using_callback(message_data, message):
print "%s" % message_data
message.ack()
consumer.register_callback(get_using_callback)
consumer.wait()
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
conn = BrokerConnection(hostname="localhost", port=61617,
userid="admin", password="******",
virtual_host="/")
consumer = Consumer(connection=conn, queue="feed",
exchange="feed", routing_key="importer")
def import_feed_callback(message_data, message):
feed_url = message_data["import_feed"]
#print("Got feed import message for: %s" % feed_url)
# something importing this feed url
# import_feed(feed_url)
message.ack()
consumer.register_callback(import_feed_callback)
consumer.wait() # Go into the consumer loop.
class SignalAMQP(Signal):
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def send(self, message, **kw):
""" Transfer message to bus. Message can be any simple python type. """
self.publisher.send(message, **kw)
def callback(self, message_data, message):
""" Consumer callback function. Send Django singnal."""
try:
sender = message_data['sender']
except:
sender = None
super(SignalAMQP, self).send(sender=sender,
message_data=message_data,
message=message)
message.ack()
if self.debug:
print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message
return True
def get_backend(self):
return self.conn.get_backend_cls()(DjangoBrokerConnection())
def queue_bind(self, queue_list):
""" Bind another queues to current exchange """
routing_keys = []
backend = self.get_backend()
for x in queue_list:
backend.queue_bind(queue=x, exchange=self.exchange, \
routing_key='%s_%s' % (self.exchange, x))
routing_keys.append('%s_%s' % (self.exchange, x))
return routing_keys
def listen(self):
""" Run consumer loop thread """
cl = ConsumerLoop()
cl.consumer = self.consumer
cl.start()
return cl
def stop(self):
""" Unactivate this signal """
self.conn.close()
def get_consumer(self, broker, **kwargs):
return Consumer(broker, **kwargs)
class GenericConsumer(object):
def __init__(self,
config,
exchange=None,
connect=True,
heartbeat=False,
**kwargs):
self.config = config
self.exchange = exchange
self.connection = None
self.durable = False
self.applabel = ''
self.heartbeat = heartbeat
for x in ['applabel', 'topic', 'callback', 'durable']:
if x in kwargs:
setattr(self, x, kwargs[x])
del kwargs[x]
if connect:
self.connect()
# Sets vairables
def configure(self, **kwargs):
for x in kwargs:
setattr(self, x, kwargs[x])
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
# Support purging messages that are already in the queue on the broker
# TODO: I think this is only supported by the amqp backend
def purge_existing_messages(self):
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Purge the queue of existing messages
self.connection.create_backend().queue_purge(self.applabel)
# Blocks and calls the callback when a message comes into the queue
# For info on one script listening to multiple channels, see
# http://ask.github.com/carrot/changelog.html#id1
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(
queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(
queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
if __name__ == '__main__':
connection = BrokerConnection(hostname='localhost',
port=5672,
userid='guest',
password='******',
virtual_host=None)
consumer = Consumer(connection=connection,
queue="transactions",
exchange="transactions")
def print_message(message_data, message):
print(message_data)
message.ack()
consumer.register_callback(print_message)
consumer.wait()
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
if __name__ == '__main__':
connection = BrokerConnection(hostname='localhost', port=5672,
userid='guest', password='******',
virtual_host=None)
consumer = Consumer(connection=connection, queue="transactions",
exchange="transactions")
def print_message(message_data, message):
print(message_data)
message.ack()
consumer.register_callback(print_message)
consumer.wait()
def create_consumer(self, **options):
return Consumer(connection=self.conn,
queue=self.queue,
exchange=self.exchange,
routing_key=self.routing_key,
**options)
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(
queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(
queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
import jsonrpclib
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/")
consumer = Consumer(connection=conn, queue="po_box", exchange="sorting_room", routing_key="jason")
def amqp_callback(message_data, message):
server = jsonrpclib.Server("http://localhost:8080")
server.ping(message_data)
print jsonrpclib.history.response
message.ack()
consumer.register_callback(amqp_callback)
consumer.wait() # Go into the consumer loop.
conn.close()
#!/usr/bin/env python
from carrot.messaging import Consumer
from carrot.connection import BrokerConnection
conn=BrokerConnection(hostname='localhost', port=5672,
userid='test', password='******',
virtual_host='test')
consumer = Consumer(connection=conn, queue="feed",
exchange_type='topic',
exchange="fst", routing_key="easyip.*")
#publisher = Publisher(connection=conn, exchange="fst", exchange_type='topic',
# routing_key="easyip.incomming", serializer="json")
def get_using_callback(message_data, message):
print "%s" % message_data
message.ack()
consumer.register_callback(get_using_callback)
consumer.wait()
def test_consumer(self):
"""AMQP->RPC->AMQP
Send a AMQP message, and test RPCConsumer response.
Uses a mock RPC server that the consumer will call.
Check the consumer AMQP response with test consumer.
"""
class TestConsumer(IMessageBroker):
service_name = 'TestConsumer'
exchange_name = 'Test'
topic = 'test'
amqp_connection_settings = settings.AMQP_CONNECTION
def dispatch(self, message, request, qid):
""" AMQP -> RPC dispatcher.
"""
logging.info('dispatching AMQP -> RPC')
response = {}
if qid:
# call remote RPC
# this message has an qid so it expects a response
response['msg'] = test_rpc_service.push(request)
message.ack()
# return the response to AMQP, the caller should be listening ..
self.return_response(response,qid)
else:
# no qid, so do something stateless ..
print request
message.ack()
try:
consumer = TestConsumer()
self.assertEquals('test.request.*',consumer.binding_key)
self.assertEquals('test.response.%s',consumer.response_routing_key)
consumer.start()
test_rpc_service.start()
test_rpc_service.messages = []
# allow consumer to start
sleep(0.2)
self.assert_(not consumer.stopped)
self.assert_(consumer.isAlive())
# test variables
qid = str(randrange(0,999999))
jsondata = {'msg':'hello rpc'}
_publisher = Publisher(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
routing_key='test.request.'+qid,
)
# test channel
backend = self.test_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,)
backend.queue_bind("test",'Test','test.response.'+qid)
_consumer = Consumer(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
queue="test",
)
_consumer.discard_all()
logging.debug('publishing JSON message to RPC')
data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid})
_publisher.send(data_on_the_wire)
# allow data to pass the wire
sleep(0.2)
# retrieve dispatcher response
response = _consumer.fetch()
self.assert_(response, 'No response')
data = json.loads(response.payload)
self.assert_(len(data['msg']) > 0) # assert non-empty response
self.assertEquals('ok',data['msg'])
# check dispatcher RPC function
self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive')
self.assertEquals(test_rpc_service.messages[0], jsondata)
finally:
try:
consumer.stop()
except:
pass
try:
_consumer.close()
except:
pass
test_rpc_service.stop()
self.assert_(consumer.stopped)
class GenericConsumer(object):
def __init__(self, config, exchange=None, connect=True, heartbeat=False,
**kwargs):
self.config = config
self.exchange = exchange
self.connection = None
self.durable = False
self.applabel = ''
self.heartbeat = heartbeat
for x in ['applabel','topic','callback','durable']:
if x in kwargs:
setattr(self, x, kwargs[x])
del kwargs[x]
if connect:
self.connect()
# Sets vairables
def configure(self, **kwargs):
for x in kwargs:
setattr(self, x, kwargs[x])
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
# Support purging messages that are already in the queue on the broker
# TODO: I think this is only supported by the amqp backend
def purge_existing_messages(self):
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Purge the queue of existing messages
self.connection.create_backend().queue_purge(self.applabel)
# Blocks and calls the callback when a message comes into the queue
# For info on one script listening to multiple channels, see
# http://ask.github.com/carrot/changelog.html#id1
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
class PyGoWaveMessageProcessor(object):
"""
Handle all incoming messages.
Routing key structure:
<participant_conn_guid>.<wavelet_id>.[waveop|clientop]
participant_conn_guid is created every time the user enters the wave viewer.
Multiple instances of the wave viewer can be opened per user at the same time.
A participant can "listen" to messages from multiple wavlets, by subscribing
to multiple routing keys.
In this scenario, a message may be sent to a queue, that the participant has
not subscribed to. This is intended and it's RabbitMQ's job to drop the
message. This keeps things simple for now and may be changed in the future.
Message comsumers can be optimized in a multi-threading environment. If
one consumer starts handling messages of a particular wavelet, it should
block others from handling them.
Some messages are handled synchronously (i.e. the client does not perform
any actions and waits for the server's response). Those are in particular:
WAVELET_ADD_PARTICIPANT
"""
purge_every = datetime.timedelta(minutes=10)
conn_min_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2))
def __init__(self, connection):
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection,
exchange="wavelet.direct",
exchange_type="direct",
delivery_mode=1,
serializer="json",
)
self.out_queue = {}
self.purge_connections()
def broadcast(self, wavelet, type, property, except_connections=[]):
"""
Send messages to all participants.
`wavelet` must be a Wavelet object.
`except_connections` is a list of ParticipantConn objects to be
excluded from the broadcast.
"""
msg_dict = {
"type": type,
"property": property
}
logger.debug("Broadcasting Message:\n" + repr(msg_dict))
for p in wavelet.participants.all():
for conn in p.connections.all():
if not conn in except_connections:
if self.out_queue.has_key(conn.rx_key):
self.out_queue[conn.rx_key].append(msg_dict)
else:
self.out_queue[conn.rx_key] = [msg_dict]
def emit(self, to, type, property, except_connections=[]):
"""
Collect messages to be sent.
`to` must be a ParticipantConn object.
"""
msg_dict = {
"type": type,
"property": property
}
logger.debug("Emiting Message to %s/%d:\n%s" % (to.participant.name, to.id, repr(msg_dict)))
if self.out_queue.has_key(to.rx_key):
self.out_queue[to.rx_key].append(msg_dict)
else:
self.out_queue[to.rx_key] = [msg_dict]
def wait(self, limit=None):
self.consumer.wait(limit)
def send(self, message_data, routing_key):
self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1)
def receive(self, message_data, message):
rkey = message.amqp_message.routing_key
participant_conn_key, wavelet_id, message_category = rkey.split(".")
if message_category != "clientop":
return
logger.debug("Received Message from %s.%s.%s:\n%s" % (participant_conn_key, wavelet_id, message_category, repr(message_data)))
# Get participant connection
try:
pconn = ParticipantConn.objects.get(tx_key=participant_conn_key)
except ObjectDoesNotExist:
logger.error("{%s} ParticipantConn not found" % (rkey))
return # Fail silently
# Get wavelet
try:
wavelet = pconn.participant.wavelets.get(id=wavelet_id)
except ObjectDoesNotExist:
logger.error("{%s} Wavelet not found (or not participating)" % (rkey))
return # Fail silently
# Handle message and reply to sender and/or broadcast an event
self.out_queue = {}
if isinstance(message_data, list): # multi-message?
for sub_message in message_data:
if not self.handle_participant_message(wavelet, pconn, sub_message): break
else:
self.handle_participant_message(wavelet, pconn, message_data)
for receiver, messages in self.out_queue.iteritems():
self.send(messages, "%s.%s.waveop" % (receiver, wavelet_id))
self.out_queue = {}
# Cleanup time?
if datetime.datetime.now() > self.next_purge:
self.purge_connections()
def handle_participant_message(self, wavelet, pconn, message):
"""
Handle a participant's operation.
If True is returned, go on with processing the next message.
If False is returned, discard any following messages.
"""
participant = pconn.participant
if message.has_key(u"type"):
if message["type"] == "WAVELET_OPEN":
logger.info("[%s/%d@%s] Opening wavelet" % (participant.name, pconn.id, wavelet.wave.id))
pconn.wavelets.add(wavelet)
# I know this is neat :)
self.emit(pconn, "WAVELET_OPEN", {
"wavelet": wavelet.serialize(),
"blips": wavelet.serialize_blips(),
})
elif message["type"] == "PARTICIPANT_INFO":
logger.info("[%s/%d@%s] Sending participant information" % (participant.name, pconn.id, wavelet.wave.id))
p_info = {}
for p_id in message["property"]:
try:
p_info[p_id] = Participant.objects.get(id=p_id).serialize()
except ObjectDoesNotExist:
p_info[p_id] = None
self.emit(pconn, "PARTICIPANT_INFO", p_info)
elif message["type"] == "PARTICIPANT_SEARCH":
if len(message["property"]) < getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0):
self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "TOO_SHORT", "data": getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0)})
logger.debug("[%s/%d@%s] Participant search query too short" % (participant.name, pconn.id, wavelet.wave.id))
else:
logger.info("[%s/%d@%s] Performing participant search" % (participant.name, pconn.id, wavelet.wave.id))
lst = []
for p in Participant.objects.filter(name__icontains=message["property"]).exclude(id=participant.id):
lst.append(p.id)
self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "OK", "data": lst})
elif message["type"] == "GADGET_LIST":
all_gadgets = map(lambda g: {"id": g.id, "uploaded_by": g.by_user.participants.all()[0].name, "name": g.title, "descr": g.description, "url": g.url}, Gadget.objects.all())
logger.info("[%s/%d@%s] Sending Gadget list" % (participant.name, pconn.id, wavelet.wave.id))
self.emit(pconn, "GADGET_LIST", all_gadgets)
elif message["type"] == "WAVELET_ADD_PARTICIPANT":
# Find participant
try:
p = Participant.objects.get(id=message["property"])
except ObjectDoesNotExist:
logger.error("[%s/%d@%s] Target participant '%s' not found" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
return # Fail silently (TODO: report error to user)
# Check if already participating
if wavelet.participants.filter(id=message["property"]).count() > 0:
logger.error("[%s/%d@%s] Target participant '%s' already there" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
return # Fail silently (TODO: report error to user)
wavelet.participants.add(p)
logger.info("[%s/%d@%s] Added new participant '%s'" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
self.broadcast(wavelet, "WAVELET_ADD_PARTICIPANT", message["property"])
elif message["type"] == "WAVELET_REMOVE_SELF":
self.broadcast(wavelet, "WAVELET_REMOVE_PARTICIPANT", participant.id)
wavelet.participants.remove(participant) # Bye bye
pconn.wavelets.remove(wavelet) # Also for your connection
logger.info("[%s/%d@%s] Participant removed himself" % (participant.name, pconn.id, wavelet.wave.id))
if wavelet.participants.count() == 0: # Oh my god, you killed the Wave! You bastard!
logger.info("[%s/%d@%s] Wave got killed!" % (participant.name, pconn.id, wavelet.wave.id))
wavelet.wave.delete()
return False
elif message["type"] == "OPERATION_MESSAGE_BUNDLE":
# Build OpManager
newdelta = OpManager(wavelet.wave.id, wavelet.id)
newdelta.unserialize(message["property"]["operations"])
version = message["property"]["version"]
# Transform
for delta in wavelet.deltas.filter(version__gt=version):
for op in delta.getOpManager().operations:
newdelta.transform(op) # Trash results (an existing delta cannot be changed)
# Apply
wavelet.applyOperations(newdelta.operations)
# Raise version and store
wavelet.version += 1
wavelet.save()
Delta.createByOpManager(newdelta, wavelet.version).save()
# Create tentative checksums
blipsums = wavelet.blipsums()
# Respond
self.emit(pconn, "OPERATION_MESSAGE_BUNDLE_ACK", {"version": wavelet.version, "blipsums": blipsums})
self.broadcast(wavelet, "OPERATION_MESSAGE_BUNDLE", {"version": wavelet.version, "operations": newdelta.serialize(), "blipsums": blipsums}, [pconn])
logger.debug("[%s/%d@%s] Processed delta #%d -> v%d" % (participant.name, pconn.id, wavelet.wave.id, version, wavelet.version))
else:
logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message))
else:
logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message))
return True
def purge_connections(self):
"""
This method will check all connections to the server and throw out all
closed ones.
This can be run asynchronously every 10 minutes or so.
"""
for conn in ParticipantConn.objects.all():
for wavelet in conn.wavelets.all():
if not self.queue_exists("%s.%s.waveop" % (conn.rx_key, wavelet.id)):
wavelet.participant_conns.remove(conn)
logger.info("[%s/%d@%s] Connection to wavelet closed" % (conn.participant.name, conn.id, wavelet.wave.id))
if conn.wavelets.count() == 0 and datetime.datetime.now() > conn.created + self.conn_min_lifetime:
conn_id, conn_participant_name = conn.id, conn.participant.name
conn.delete()
logger.info("[%s/%d] Connection to server closed" % (conn_participant_name, conn_id))
self.next_purge = datetime.datetime.now() + self.purge_every
def queue_exists(self, queue):
"""
Check if a queue exists, i.e. a user is connected to it (because
auto-delete is always turned on).
"""
logger.debug("Checking queue %s" % (queue))
qex = self.publisher.backend.queue_exists(queue)
# Re-open the channel if it was closed (this is a pyamqlib issue)
if self.publisher.backend.channel.connection == None:
self.publisher.backend.channel = self.publisher.backend.connection.connection.channel()
return qex
def __init__(self, message_type, queue, context):
Consumer.__init__(self, context.broker, routing_key=message_type, queue=queue)
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
import redis
conn = BrokerConnection(hostname="localhost", port=5672,
userid="guest", password="******",
virtual_host="/")
consumer = Consumer(connection=conn, queue="feed",
exchange="feed", routing_key="importer")
db = redis.Redis()
def message_callback(message_data, message):
db.incr(message_data["test_name"])
message.ack()
consumer.register_callback(message_callback)
consumer.wait()
def __init__(self, channel, context):
queue = 'eb_%s' % uuid1().hex
Consumer.__init__(self, context.broker,
exchange=_exchange_for_channel(channel),
queue=queue)
class Worker(object):
def __init__(self,
hostname='localhost',
port=5672,
userid="collectoruser",
password="******",
**options):
conn = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host='collectorvhost')
try:
self.publisher = Publisher(connection=conn,
exchange="collector.response",
exchange_type='direct',
routing_key="response",
serializer="json")
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="collector",
routing_key="collector.*.*")
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
#print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj['tags']:
if driver:
value = driver.read_value(tag['device'],
tag['address'])
else:
value = "BAD_DRIVER"
values.append({
'name': tag['name'],
'current_value': value,
'read_at': '2009-01-01'
})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
