class Worker(object):
def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options):
conn = BrokerConnection(
hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost"
)
try:
self.publisher = Publisher(
connection=conn,
exchange="collector.response",
exchange_type="direct",
routing_key="response",
serializer="json",
)
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(
connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*"
)
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
# print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj["tags"]:
if driver:
value = driver.read_value(tag["device"], tag["address"])
else:
value = "BAD_DRIVER"
values.append({"name": tag["name"], "current_value": value, "read_at": "2009-01-01"})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
def do_consume():
consumer = Consumer(ctx.broker, exchange='testx', queue='testq',
routing_key='testq', exclusive=True, durable=False)
consumer.register_callback(got_message_cb)
try:
consumer.wait(limit=1)
except StopIteration:
pass
except GreenletExit:
pass
finally:
consumer.close()
def get(self, exchange_name, queue_name, routing_key):
def msg_callback(message_data, message):
message.ack()
self.write(simplejson.dumps(message_data))
self.finish()
consumer = Consumer(connection=conn, queue=queue_name,
exchange=exchange_name, routing_key=routing_key)
consumer.register_callback(msg_callback)
try:
consumer.wait(1)
except StopIteration:
consumer.close()
class AmqpMessageProcessor(object):
purge_every = datetime.timedelta(minutes=10)
conn_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2))
def __init__(self, connection):
self.pygo_mp = PyGoWaveClientMessageProcessor()
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json"
)
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def wait(self, limit=None):
self.consumer.wait(limit)
def send(self, routing_key, message_data):
self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1)
def receive(self, message_data, message):
routing_key = message.amqp_message.routing_key
msg_dict = self.pygo_mp.process(routing_key, message_data)
for out_rkey, messages in msg_dict.iteritems():
self.send(out_rkey, messages)
# Cleanup time?
if datetime.datetime.now() > self.next_purge:
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def main():
connection = BrokerConnection(
hostname = "localhost",
port = 5672,
userid = "test",
password = "******",
virtual_host = "test.com",
)
consumer = Consumer(
connection = connection,
queue = "messages",
exchange = "messages",
routing_key = "awesome",
)
consumer.register_callback(worker_callback)
try:
consumer.wait()
except KeyboardInterrupt:
consumer.close()
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
if __name__ == '__main__':
connection = BrokerConnection(hostname='localhost',
port=5672,
userid='guest',
password='******',
virtual_host=None)
consumer = Consumer(connection=connection,
queue="transactions",
exchange="transactions")
def print_message(message_data, message):
print(message_data)
message.ack()
consumer.register_callback(print_message)
consumer.wait()
class GenericConsumer(object):
def __init__(self, config, exchange=None, connect=True, heartbeat=False,
**kwargs):
self.config = config
self.exchange = exchange
self.connection = None
self.durable = False
self.applabel = ''
self.heartbeat = heartbeat
for x in ['applabel','topic','callback','durable']:
if x in kwargs:
setattr(self, x, kwargs[x])
del kwargs[x]
if connect:
self.connect()
# Sets vairables
def configure(self, **kwargs):
for x in kwargs:
setattr(self, x, kwargs[x])
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
# Support purging messages that are already in the queue on the broker
# TODO: I think this is only supported by the amqp backend
def purge_existing_messages(self):
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Purge the queue of existing messages
self.connection.create_backend().queue_purge(self.applabel)
# Blocks and calls the callback when a message comes into the queue
# For info on one script listening to multiple channels, see
# http://ask.github.com/carrot/changelog.html#id1
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
import jsonrpclib
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/")
consumer = Consumer(connection=conn, queue="po_box", exchange="sorting_room", routing_key="jason")
def amqp_callback(message_data, message):
server = jsonrpclib.Server("http://localhost:8080")
server.ping(message_data)
print jsonrpclib.history.response
message.ack()
consumer.register_callback(amqp_callback)
consumer.wait() # Go into the consumer loop.
conn.close()
class Worker(object):
def __init__(self,
hostname='localhost',
port=5672,
userid="collectoruser",
password="******",
**options):
conn = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host='collectorvhost')
try:
self.publisher = Publisher(connection=conn,
exchange="collector.response",
exchange_type='direct',
routing_key="response",
serializer="json")
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="collector",
routing_key="collector.*.*")
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
#print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj['tags']:
if driver:
value = driver.read_value(tag['device'],
tag['address'])
else:
value = "BAD_DRIVER"
values.append({
'name': tag['name'],
'current_value': value,
'read_at': '2009-01-01'
})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
class PyGoWaveMessageProcessor(object):
"""
Handle all incoming messages.
Routing key structure:
<participant_conn_guid>.<wavelet_id>.[waveop|clientop]
participant_conn_guid is created every time the user enters the wave viewer.
Multiple instances of the wave viewer can be opened per user at the same time.
A participant can "listen" to messages from multiple wavlets, by subscribing
to multiple routing keys.
In this scenario, a message may be sent to a queue, that the participant has
not subscribed to. This is intended and it's RabbitMQ's job to drop the
message. This keeps things simple for now and may be changed in the future.
Message comsumers can be optimized in a multi-threading environment. If
one consumer starts handling messages of a particular wavelet, it should
block others from handling them.
Some messages are handled synchronously (i.e. the client does not perform
any actions and waits for the server's response). Those are in particular:
WAVELET_ADD_PARTICIPANT
"""
purge_every = datetime.timedelta(minutes=10)
conn_min_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2))
def __init__(self, connection):
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection,
exchange="wavelet.direct",
exchange_type="direct",
delivery_mode=1,
serializer="json",
)
self.out_queue = {}
self.purge_connections()
def broadcast(self, wavelet, type, property, except_connections=[]):
"""
Send messages to all participants.
`wavelet` must be a Wavelet object.
`except_connections` is a list of ParticipantConn objects to be
excluded from the broadcast.
"""
msg_dict = {
"type": type,
"property": property
}
logger.debug("Broadcasting Message:\n" + repr(msg_dict))
for p in wavelet.participants.all():
for conn in p.connections.all():
if not conn in except_connections:
if self.out_queue.has_key(conn.rx_key):
self.out_queue[conn.rx_key].append(msg_dict)
else:
self.out_queue[conn.rx_key] = [msg_dict]
def emit(self, to, type, property, except_connections=[]):
"""
Collect messages to be sent.
`to` must be a ParticipantConn object.
"""
msg_dict = {
"type": type,
"property": property
}
logger.debug("Emiting Message to %s/%d:\n%s" % (to.participant.name, to.id, repr(msg_dict)))
if self.out_queue.has_key(to.rx_key):
self.out_queue[to.rx_key].append(msg_dict)
else:
self.out_queue[to.rx_key] = [msg_dict]
def wait(self, limit=None):
self.consumer.wait(limit)
def send(self, message_data, routing_key):
self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1)
def receive(self, message_data, message):
rkey = message.amqp_message.routing_key
participant_conn_key, wavelet_id, message_category = rkey.split(".")
if message_category != "clientop":
return
logger.debug("Received Message from %s.%s.%s:\n%s" % (participant_conn_key, wavelet_id, message_category, repr(message_data)))
# Get participant connection
try:
pconn = ParticipantConn.objects.get(tx_key=participant_conn_key)
except ObjectDoesNotExist:
logger.error("{%s} ParticipantConn not found" % (rkey))
return # Fail silently
# Get wavelet
try:
wavelet = pconn.participant.wavelets.get(id=wavelet_id)
except ObjectDoesNotExist:
logger.error("{%s} Wavelet not found (or not participating)" % (rkey))
return # Fail silently
# Handle message and reply to sender and/or broadcast an event
self.out_queue = {}
if isinstance(message_data, list): # multi-message?
for sub_message in message_data:
if not self.handle_participant_message(wavelet, pconn, sub_message): break
else:
self.handle_participant_message(wavelet, pconn, message_data)
for receiver, messages in self.out_queue.iteritems():
self.send(messages, "%s.%s.waveop" % (receiver, wavelet_id))
self.out_queue = {}
# Cleanup time?
if datetime.datetime.now() > self.next_purge:
self.purge_connections()
def handle_participant_message(self, wavelet, pconn, message):
"""
Handle a participant's operation.
If True is returned, go on with processing the next message.
If False is returned, discard any following messages.
"""
participant = pconn.participant
if message.has_key(u"type"):
if message["type"] == "WAVELET_OPEN":
logger.info("[%s/%d@%s] Opening wavelet" % (participant.name, pconn.id, wavelet.wave.id))
pconn.wavelets.add(wavelet)
# I know this is neat :)
self.emit(pconn, "WAVELET_OPEN", {
"wavelet": wavelet.serialize(),
"blips": wavelet.serialize_blips(),
})
elif message["type"] == "PARTICIPANT_INFO":
logger.info("[%s/%d@%s] Sending participant information" % (participant.name, pconn.id, wavelet.wave.id))
p_info = {}
for p_id in message["property"]:
try:
p_info[p_id] = Participant.objects.get(id=p_id).serialize()
except ObjectDoesNotExist:
p_info[p_id] = None
self.emit(pconn, "PARTICIPANT_INFO", p_info)
elif message["type"] == "PARTICIPANT_SEARCH":
if len(message["property"]) < getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0):
self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "TOO_SHORT", "data": getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0)})
logger.debug("[%s/%d@%s] Participant search query too short" % (participant.name, pconn.id, wavelet.wave.id))
else:
logger.info("[%s/%d@%s] Performing participant search" % (participant.name, pconn.id, wavelet.wave.id))
lst = []
for p in Participant.objects.filter(name__icontains=message["property"]).exclude(id=participant.id):
lst.append(p.id)
self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "OK", "data": lst})
elif message["type"] == "GADGET_LIST":
all_gadgets = map(lambda g: {"id": g.id, "uploaded_by": g.by_user.participants.all()[0].name, "name": g.title, "descr": g.description, "url": g.url}, Gadget.objects.all())
logger.info("[%s/%d@%s] Sending Gadget list" % (participant.name, pconn.id, wavelet.wave.id))
self.emit(pconn, "GADGET_LIST", all_gadgets)
elif message["type"] == "WAVELET_ADD_PARTICIPANT":
# Find participant
try:
p = Participant.objects.get(id=message["property"])
except ObjectDoesNotExist:
logger.error("[%s/%d@%s] Target participant '%s' not found" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
return # Fail silently (TODO: report error to user)
# Check if already participating
if wavelet.participants.filter(id=message["property"]).count() > 0:
logger.error("[%s/%d@%s] Target participant '%s' already there" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
return # Fail silently (TODO: report error to user)
wavelet.participants.add(p)
logger.info("[%s/%d@%s] Added new participant '%s'" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
self.broadcast(wavelet, "WAVELET_ADD_PARTICIPANT", message["property"])
elif message["type"] == "WAVELET_REMOVE_SELF":
self.broadcast(wavelet, "WAVELET_REMOVE_PARTICIPANT", participant.id)
wavelet.participants.remove(participant) # Bye bye
pconn.wavelets.remove(wavelet) # Also for your connection
logger.info("[%s/%d@%s] Participant removed himself" % (participant.name, pconn.id, wavelet.wave.id))
if wavelet.participants.count() == 0: # Oh my god, you killed the Wave! You bastard!
logger.info("[%s/%d@%s] Wave got killed!" % (participant.name, pconn.id, wavelet.wave.id))
wavelet.wave.delete()
return False
elif message["type"] == "OPERATION_MESSAGE_BUNDLE":
# Build OpManager
newdelta = OpManager(wavelet.wave.id, wavelet.id)
newdelta.unserialize(message["property"]["operations"])
version = message["property"]["version"]
# Transform
for delta in wavelet.deltas.filter(version__gt=version):
for op in delta.getOpManager().operations:
newdelta.transform(op) # Trash results (an existing delta cannot be changed)
# Apply
wavelet.applyOperations(newdelta.operations)
# Raise version and store
wavelet.version += 1
wavelet.save()
Delta.createByOpManager(newdelta, wavelet.version).save()
# Create tentative checksums
blipsums = wavelet.blipsums()
# Respond
self.emit(pconn, "OPERATION_MESSAGE_BUNDLE_ACK", {"version": wavelet.version, "blipsums": blipsums})
self.broadcast(wavelet, "OPERATION_MESSAGE_BUNDLE", {"version": wavelet.version, "operations": newdelta.serialize(), "blipsums": blipsums}, [pconn])
logger.debug("[%s/%d@%s] Processed delta #%d -> v%d" % (participant.name, pconn.id, wavelet.wave.id, version, wavelet.version))
else:
logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message))
else:
logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message))
return True
def purge_connections(self):
"""
This method will check all connections to the server and throw out all
closed ones.
This can be run asynchronously every 10 minutes or so.
"""
for conn in ParticipantConn.objects.all():
for wavelet in conn.wavelets.all():
if not self.queue_exists("%s.%s.waveop" % (conn.rx_key, wavelet.id)):
wavelet.participant_conns.remove(conn)
logger.info("[%s/%d@%s] Connection to wavelet closed" % (conn.participant.name, conn.id, wavelet.wave.id))
if conn.wavelets.count() == 0 and datetime.datetime.now() > conn.created + self.conn_min_lifetime:
conn_id, conn_participant_name = conn.id, conn.participant.name
conn.delete()
logger.info("[%s/%d] Connection to server closed" % (conn_participant_name, conn_id))
self.next_purge = datetime.datetime.now() + self.purge_every
def queue_exists(self, queue):
"""
Check if a queue exists, i.e. a user is connected to it (because
auto-delete is always turned on).
"""
logger.debug("Checking queue %s" % (queue))
qex = self.publisher.backend.queue_exists(queue)
# Re-open the channel if it was closed (this is a pyamqlib issue)
if self.publisher.backend.channel.connection == None:
self.publisher.backend.channel = self.publisher.backend.connection.connection.channel()
return qex
class GenericConsumer(object):
def __init__(self,
config,
exchange=None,
connect=True,
heartbeat=False,
**kwargs):
self.config = config
self.exchange = exchange
self.connection = None
self.durable = False
self.applabel = ''
self.heartbeat = heartbeat
for x in ['applabel', 'topic', 'callback', 'durable']:
if x in kwargs:
setattr(self, x, kwargs[x])
del kwargs[x]
if connect:
self.connect()
# Sets vairables
def configure(self, **kwargs):
for x in kwargs:
setattr(self, x, kwargs[x])
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
# Support purging messages that are already in the queue on the broker
# TODO: I think this is only supported by the amqp backend
def purge_existing_messages(self):
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Purge the queue of existing messages
self.connection.create_backend().queue_purge(self.applabel)
# Blocks and calls the callback when a message comes into the queue
# For info on one script listening to multiple channels, see
# http://ask.github.com/carrot/changelog.html#id1
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(
queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(
queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
#!/usr/bin/env python
from carrot.messaging import Consumer
from carrot.connection import BrokerConnection
conn=BrokerConnection(hostname='localhost', port=5672,
userid='test', password='******',
virtual_host='test')
consumer = Consumer(connection=conn, queue="feed",
exchange_type='topic',
exchange="fst", routing_key="easyip.*")
#publisher = Publisher(connection=conn, exchange="fst", exchange_type='topic',
# routing_key="easyip.incomming", serializer="json")
def get_using_callback(message_data, message):
print "%s" % message_data
message.ack()
consumer.register_callback(get_using_callback)
consumer.wait()
