class Worker(object):
def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options):
conn = BrokerConnection(
hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost"
)
try:
self.publisher = Publisher(
connection=conn,
exchange="collector.response",
exchange_type="direct",
routing_key="response",
serializer="json",
)
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(
connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*"
)
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
# print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj["tags"]:
if driver:
value = driver.read_value(tag["device"], tag["address"])
else:
value = "BAD_DRIVER"
values.append({"name": tag["name"], "current_value": value, "read_at": "2009-01-01"})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
def return_response(self, response, qid):
"""AMQP "response" handler.
Publishes a response message to a temporary queue.
- response is json, qid is string
"""
message = json.dumps(response)
routing_key = self.response_routing_key % qid
logging.debug("response to %s with routing_key: %s, message: \n%s" % (self.exchange_name, routing_key, message))
try:
publisher = Publisher(
connection=self.amqp_connection,
exchange=self.exchange_name,
exchange_type="topic",
routing_key=routing_key,
)
publisher.send(message)
publisher.close()
except:
"""Trying to send with broken connection.
Handle gracefully by waiting for connection and publish again."""
logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1]))
self._wait_connection(5)
self.return_response(response, qid)
def cast(msg, event_type, topic, priority):
yagi.config.setup(config_path='/etc/yagi.conf')
conf = yagi.config.config_with('rabbit_broker')
host = conf('host')
port = conf('port')
user = conf('user')
exchange = conf('exchange')
password = conf('password')
vhost = conf('vhost')
message_dict = {
'message_id': str(uuid.uuid4()),
'event_type': event_type,
'publisher_id': 'some_publisher',
'priority': priority,
'timestamp': str(datetime.datetime.utcnow()),
'payload': msg
}
conn = BrokerConnection(hostname=host, port=port, userid=user,
password=password, virtual_host=vhost)
publisher = Publisher(connection=conn, exchange=exchange,
routing_key="%s.%s" % (topic, priority), durable=False,
exchange_type='topic')
publisher.send(message_dict)
publisher.close()
def _send_message(connection, routing_key, data, envelope):
# Make a message envelope if we are told to have one
if envelope:
message = {}
message["payload"] = data
message["_meta"] = {
"exchange": CONF["MSG_EXCHANGE"],
"routing_key": routing_key,
"sent": mercurial.util.datestr(None, CONF["DATE_FORMAT"]),
# TODO: Support more than just JSON
"serializer": "json",
}
else:
message = data
# Set up our broker publisher
publisher = Publisher(
connection=connection, exchange=CONF["MSG_EXCHANGE"], exchange_type="topic", routing_key=routing_key
)
# Send the message
# TODO: Support more than just JSON
publisher.send(message)
# Close the publishing connection
publisher.close()
def main():
logging.basicConfig(level=logging.DEBUG)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
publisher = Publisher(auto_declare=False,
connection=conn,
exchange=args.exchange,
routing_key=args.key)
logging.info("Declaring exchange: %s" % args.exchange)
publisher.backend.exchange_declare(exchange=args.exchange,
type="topic",
durable=False,
auto_delete=False)
while True:
line = sys.stdin.readline()
if not line:
break
logging.debug("Sending message '%s'" % line.strip())
publisher.send(line.strip())
publisher.close()
def publish(self,
exchange,
routing_key,
message,
auto_declare=False,
persistent=True):
"""publish a message to exchange using routing_key
exchange - name of exchange
routing_key - interpretation of routing key depends on exchange type
message - message content to send
auto_declare - automatically declare the exchange (default: false)
persistent - store message on disk as well as memory (default: True)
"""
delivery_mode = 2
if not persistent:
delivery_mode = 1
publisher = Publisher(connection=self.broker,
exchange=exchange,
routing_key=routing_key,
auto_declare=auto_declare)
publisher.send(message, delivery_mode=delivery_mode)
publisher.close()
def cast(msg, event_type, topic, priority):
yagi.config.setup(config_path="/etc/yagi.conf")
conf = yagi.config.config_with("rabbit_broker")
host = conf("host")
port = conf("port")
user = conf("user")
exchange = conf("exchange")
password = conf("password")
vhost = conf("vhost")
message_dict = {
"message_id": str(uuid.uuid4()),
"event_type": event_type,
"publisher_id": "some_publisher",
"priority": priority,
"timestamp": str(datetime.datetime.utcnow()),
"payload": msg,
}
conn = BrokerConnection(hostname=host, port=port, userid=user, password=password, virtual_host=vhost)
publisher = Publisher(
connection=conn,
exchange=exchange,
routing_key="%s.%s" % (topic, priority),
durable=False,
exchange_type="topic",
)
publisher.send(message_dict)
publisher.close()
def _send_message(routing_key, request):
conn = DjangoAMQPConnection()
publisher = Publisher(connection=conn, exchange='request',
exchange_type='topic', routing_key=routing_key,
serializer='pickle')
publisher.send(request)
publisher.close()
conn.close()
def reply(routing_key, data):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn, exchange="django", routing_key=routing_key, exchange_type="topic")
publisher.send(data)
publisher.close()
conn.close()
def sender_callback(self, routing_key, data):
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn,
exchange="django_send",
routing_key=routing_key,
exchange_type="topic",
)
publisher.send(data)
publisher.close()
conn.close()
print "Sent object change/delete message for %s" % routing_key
def request_no_response():
"""No response"""
data = {'q': 'hello AMQP backend, I am Python'}
broker = ExampleBroker()
publisher = Publisher(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
routing_key=broker.binding_key,
)
publisher.send(json.dumps(data))
def request_no_response():
"""No response"""
data = {'q': 'hello AMQP backend, I am Python'}
broker = ExampleBroker()
publisher = Publisher(
connection = broker.amqp_connection,
exchange = broker.exchange_name,
exchange_type = "topic",
routing_key = broker.binding_key,
)
publisher.send(json.dumps(data))
def test_amqp(self):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher, Consumer
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
publisher.send("test")
publisher.close()
connection.close()
def test_amqp(self):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher, Consumer
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
publisher.send("test")
publisher.close()
connection.close()
def send_msg(routing_key, data):
conn = BrokerConnection(
hostname=settings.BROKER_HOST,
port=settings.BROKER_PORT,
userid=settings.BROKER_USER,
password=settings.BROKER_PASSWORD,
virtual_host=settings.BROKER_VHOST)
publisher = Publisher(connection=conn,
exchange="django_send",
routing_key=routing_key,
exchange_type="topic",
)
publisher.send(data)
publisher.close()
conn.close()
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn, queue="test",
exchange="test", routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn, exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
def send_requests(requests, **options):
logger = logging
if 'logger' in options.keys():
logger = options['logger']
"""Send a import request message to be picked up by workers."""
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
for req in requests:
routing_key=req['driver_routing_key']
publisher.send(req, routing_key=routing_key)
logger.debug("Sent request with routing_key %s:%s" %( routing_key,req, ))
publisher.close()
connection.close()
class AMQPTransmitter(Transmitter):
def __init__(self, hostname='localhost', port=5672,
userid=None, password=None, virtual_host=None,
exchange=None):
self.hostname = hostname
self.port = port
self.userid = userid
self.password = password
self.virtual_host = None
self.exchange = exchange
self.connection = BrokerConnection(hostname=self.hostname, port=self.port,
userid=self.userid, password=self.password,
virtual_host=self.virtual_host)
self.publisher = Publisher(connection=self.connection,
exchange=exchange)
def transmit(self, message):
self.publisher.send(message)
def main():
connection = BrokerConnection(
hostname = "localhost",
port = 5672,
userid = "test",
password = "******",
virtual_host = "test.com",
)
publisher = Publisher(
connection = connection,
exchange = "messages",
routing_key = "awesome",
)
for i in xrange(100):
publisher.send({"a": i})
publisher.close()
class RabbitMQPipeline(object):
def __init__(self, hostname, port, user_id, password, virtual_host, encoder_class):
self.queue_connection = BrokerConnection(
hostname=hostname,
port=port,
userid=user_id,
password=password,
virtual_host=virtual_host
)
self.encoder = encoder_class()
# Setup / Teardown Rabbit plumbing when spider opens / closes
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
@classmethod
def from_settings(cls, settings):
hostname = settings.get('BROKER_HOST')
port = settings.get('BROKER_PORT')
user_id = settings.get('BROKER_USERID')
password = settings.get('BROKER_PASSWORD')
virtual_host = settings.get('BROKER_VIRTUAL_HOST')
encoder_class = settings.get('QUEUE_SERIALIZER', ScrapyJSONEncoder)
return cls(hostname, port, user_id, password, virtual_host, encoder_class)
def spider_opened(self, spider):
self.publisher = Publisher(
connection=self.queue_connection,
exchange='',
routing_key=spider.name
)
def spider_closed(self, spider):
self.publisher.close()
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
def _process_item(self, item, spider):
self.publisher.send(self.encoder.encode(dict(item)))
return item
def request_with_response():
"""Example of requesting over AMQP from another process, not necessarily Python."""
qid = str(randrange(0,999))
data = {'qid': qid, 'q': 'what time is it?'}
broker = ExampleBroker()
publisher = Publisher(
connection = broker.amqp_connection,
exchange = broker.exchange_name,
exchange_type = "topic",
routing_key = broker.binding_key,
)
# declare test queue for receiving response message
backend = broker.amqp_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,)
backend.queue_bind(
"test",
broker.exchange_name,
broker.response_routing_key % qid)
consumer = Consumer(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
queue="test",
)
consumer.discard_all()
# send message
publisher.send(json.dumps(data))
# allow data to pass the wire
sleep(0.2)
# retrieve response
response = consumer.fetch()
payload = json.loads(response.payload)
print "Response from AMQP: %s" % payload
class MessageQueuePipeline(object):
def __init__(self, host_name, port, userid, password, virtual_host,
encoder_class):
self.q_connection = BrokerConnection(hostname=host_name,
port=port,
userid=userid,
password=password,
virtual_host=virtual_host)
self.encoder = encoder_class()
@classmethod
def from_settings(cls, settings):
host_name = settings.get('BROKER_HOST', 'localhost')
port = settings.get('BROKER_PORT', 5672)
userid = settings.get('BROKER_USERID', "guest")
password = settings.get('BROKER_PASSWORD', "guest")
virtual_host = settings.get('BROKER_VIRTUAL_HOST', "/")
encoder_class = settings.get('MESSAGE_Q_SERIALIZER', ScrapyJSONEncoder)
return cls(host_name, port, userid, password, virtual_host,
encoder_class)
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.publisher = Publisher(connection=self.q_connection,
exchange="",
routing_key=spider.name)
def spider_closed(self, spider):
self.publisher.close()
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
def _process_item(self, item, spider):
self.publisher.send({"scraped_data": self.encoder.encode(dict(item))})
return item
def publish(self, exchange, routing_key, message,
auto_declare=False, persistent=True):
"""publish a message to exchange using routing_key
exchange - name of exchange
routing_key - interpretation of routing key depends on exchange type
message - message content to send
auto_declare - automatically declare the exchange (default: false)
persistent - store message on disk as well as memory (default: True)
"""
delivery_mode = 2
if not persistent:
delivery_mode = 1
publisher = Publisher(connection=self.broker,
exchange=exchange, routing_key=routing_key,
auto_declare=auto_declare)
publisher.send(message, delivery_mode=delivery_mode)
publisher.close()
def send_requests(requests, **options):
logger = logging
if 'logger' in options.keys():
logger = options['logger']
"""Send a import request message to be picked up by workers."""
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
for req in requests:
routing_key = req['driver_routing_key']
publisher.send(req, routing_key=routing_key)
logger.debug("Sent request with routing_key %s:%s" % (
routing_key,
req,
))
publisher.close()
connection.close()
class AmqpMessageProcessor(object):
purge_every = datetime.timedelta(minutes=10)
conn_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2))
def __init__(self, connection):
self.pygo_mp = PyGoWaveClientMessageProcessor()
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json"
)
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def wait(self, limit=None):
self.consumer.wait(limit)
def send(self, routing_key, message_data):
self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1)
def receive(self, message_data, message):
routing_key = message.amqp_message.routing_key
msg_dict = self.pygo_mp.process(routing_key, message_data)
for out_rkey, messages in msg_dict.iteritems():
self.send(out_rkey, messages)
# Cleanup time?
if datetime.datetime.now() > self.next_purge:
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def request_with_response():
"""Example of requesting over AMQP from another process, not necessarily Python."""
qid = str(randrange(0, 999))
data = {'qid': qid, 'q': 'what time is it?'}
broker = ExampleBroker()
publisher = Publisher(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
routing_key=broker.binding_key,
)
# declare test queue for receiving response message
backend = broker.amqp_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,
)
backend.queue_bind("test", broker.exchange_name,
broker.response_routing_key % qid)
consumer = Consumer(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
queue="test",
)
consumer.discard_all()
# send message
publisher.send(json.dumps(data))
# allow data to pass the wire
sleep(0.2)
# retrieve response
response = consumer.fetch()
payload = json.loads(response.payload)
print "Response from AMQP: %s" % payload
def _dispatch_message(self, message):
if not message.claim(self.context.db):
return
try:
publisher = Publisher(self.context.broker, exchange=message.options.exchange,
exchange_type=message.options.exchange_type)
publisher.send(message.message,
routing_key = message.options.routing_key,
delivery_mode = message.options.delivery_mode,
mandatory = message.options.mandatory,
priority = message.options.priority)
publisher.close()
except:
log.error("Error dispatching deferred message %s: %s" % (message, traceback.format_exc()))
self.error_reschedule(message)
return False
else:
log.debug("Dispatched message %s" % message)
# sent with no problems, done with it.
self.context.db.delete(message)
return True
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn,
queue="test",
exchange="test",
routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn,
exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
class MessageQueuePipeline(object):
def __init__(self, host_name, port, userid, password, virtual_host, encoder_class):
self.q_connection = BrokerConnection(hostname=host_name, port=port,
userid=userid, password=password,
virtual_host=virtual_host)
self.encoder = encoder_class()
@classmethod
def from_settings(cls, settings):
host_name = settings.get('BROKER_HOST', 'localhost')
port = settings.get('BROKER_PORT', 5672)
userid = settings.get('BROKER_USERID', "guest")
password = settings.get('BROKER_PASSWORD', "guest")
virtual_host = settings.get('BROKER_VIRTUAL_HOST', "/")
encoder_class = settings.get('MESSAGE_Q_SERIALIZER', ScrapyJSONEncoder)
return cls(host_name, port, userid, password, virtual_host, encoder_class)
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.publisher = Publisher(connection=self.q_connection,
exchange="", routing_key=spider.name)
def spider_closed(self, spider):
self.publisher.close()
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
def _process_item(self, item, spider):
self.publisher.send({"scraped_data": self.encoder.encode(dict(item))})
return item
def main():
logging.basicConfig(level=logging.DEBUG)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
publisher = Publisher(
auto_declare = False,
connection = conn,
exchange = args.exchange,
routing_key = args.key
)
logging.info("Declaring exchange: %s" % args.exchange)
publisher.backend.exchange_declare(exchange=args.exchange, type="topic", durable=False, auto_delete=False)
while True:
line = sys.stdin.readline()
if not line:
break
logging.debug("Sending message '%s'" % line.strip())
publisher.send(line.strip())
publisher.close()
class GenericPublisher(object):
def __init__(self, config, exchange=None, connect=True):
self.config = config
self.exchange = exchange
self.connection = None
if connect:
self.connect()
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
self.connection = None
# Used to publish a pulse message to the proper exchange
def publish(self, message):
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a message given
if not message:
raise MalformedMessage(message)
# Have the message prepare and validate itself
message._prepare()
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker publisher
self.publisher = Publisher(connection=self.connection,
exchange=self.exchange,
exchange_type="topic",
routing_key=message.routing_key)
# The message is actually a simple envelope format with a payload and
# some metadata
final_data = {}
final_data['payload'] = message.data
final_data['_meta'] = message.metadata.copy()
final_data['_meta'].update({
'exchange':
self.exchange,
'routing_key':
message.routing_key,
'serializer':
self.config.serializer,
'sent':
time_to_string(datetime.now(timezone(self.config.broker_timezone)))
})
# Send the message
self.publisher.send(final_data, serializer=self.config.serializer)
# Close the publishing connection
self.publisher.close()
class GenericPublisher(object):
def __init__(self, config, exchange=None, connect=True):
self.config = config
self.exchange = exchange
self.connection = None
if connect:
self.connect()
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
self.connection = None
# Used to publish a pulse message to the proper exchange
def publish(self, message):
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a message given
if not message:
raise MalformedMessage(message)
# Have the message prepare and validate itself
message._prepare()
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker publisher
self.publisher = Publisher(connection=self.connection,
exchange=self.exchange,
exchange_type="topic",
routing_key=message.routing_key)
# The message is actually a simple envelope format with a payload and
# some metadata
final_data = {}
final_data['payload'] = message.data
final_data['_meta'] = message.metadata.copy()
final_data['_meta'].update({
'exchange': self.exchange,
'routing_key': message.routing_key,
'serializer': self.config.serializer,
'sent': time_to_string(datetime.now(timezone(self.config.broker_timezone)))
})
# Send the message
self.publisher.send(final_data, serializer=self.config.serializer)
# Close the publishing connection
self.publisher.close()
class PyGoWaveMessageProcessor(object):
"""
Handle all incoming messages.
Routing key structure:
<participant_conn_guid>.<wavelet_id>.[waveop|clientop]
participant_conn_guid is created every time the user enters the wave viewer.
Multiple instances of the wave viewer can be opened per user at the same time.
A participant can "listen" to messages from multiple wavlets, by subscribing
to multiple routing keys.
In this scenario, a message may be sent to a queue, that the participant has
not subscribed to. This is intended and it's RabbitMQ's job to drop the
message. This keeps things simple for now and may be changed in the future.
Message comsumers can be optimized in a multi-threading environment. If
one consumer starts handling messages of a particular wavelet, it should
block others from handling them.
Some messages are handled synchronously (i.e. the client does not perform
any actions and waits for the server's response). Those are in particular:
WAVELET_ADD_PARTICIPANT
"""
purge_every = datetime.timedelta(minutes=10)
conn_min_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2))
def __init__(self, connection):
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection,
exchange="wavelet.direct",
exchange_type="direct",
delivery_mode=1,
serializer="json",
)
self.out_queue = {}
self.purge_connections()
def broadcast(self, wavelet, type, property, except_connections=[]):
"""
Send messages to all participants.
`wavelet` must be a Wavelet object.
`except_connections` is a list of ParticipantConn objects to be
excluded from the broadcast.
"""
msg_dict = {
"type": type,
"property": property
}
logger.debug("Broadcasting Message:\n" + repr(msg_dict))
for p in wavelet.participants.all():
for conn in p.connections.all():
if not conn in except_connections:
if self.out_queue.has_key(conn.rx_key):
self.out_queue[conn.rx_key].append(msg_dict)
else:
self.out_queue[conn.rx_key] = [msg_dict]
def emit(self, to, type, property, except_connections=[]):
"""
Collect messages to be sent.
`to` must be a ParticipantConn object.
"""
msg_dict = {
"type": type,
"property": property
}
logger.debug("Emiting Message to %s/%d:\n%s" % (to.participant.name, to.id, repr(msg_dict)))
if self.out_queue.has_key(to.rx_key):
self.out_queue[to.rx_key].append(msg_dict)
else:
self.out_queue[to.rx_key] = [msg_dict]
def wait(self, limit=None):
self.consumer.wait(limit)
def send(self, message_data, routing_key):
self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1)
def receive(self, message_data, message):
rkey = message.amqp_message.routing_key
participant_conn_key, wavelet_id, message_category = rkey.split(".")
if message_category != "clientop":
return
logger.debug("Received Message from %s.%s.%s:\n%s" % (participant_conn_key, wavelet_id, message_category, repr(message_data)))
# Get participant connection
try:
pconn = ParticipantConn.objects.get(tx_key=participant_conn_key)
except ObjectDoesNotExist:
logger.error("{%s} ParticipantConn not found" % (rkey))
return # Fail silently
# Get wavelet
try:
wavelet = pconn.participant.wavelets.get(id=wavelet_id)
except ObjectDoesNotExist:
logger.error("{%s} Wavelet not found (or not participating)" % (rkey))
return # Fail silently
# Handle message and reply to sender and/or broadcast an event
self.out_queue = {}
if isinstance(message_data, list): # multi-message?
for sub_message in message_data:
if not self.handle_participant_message(wavelet, pconn, sub_message): break
else:
self.handle_participant_message(wavelet, pconn, message_data)
for receiver, messages in self.out_queue.iteritems():
self.send(messages, "%s.%s.waveop" % (receiver, wavelet_id))
self.out_queue = {}
# Cleanup time?
if datetime.datetime.now() > self.next_purge:
self.purge_connections()
def handle_participant_message(self, wavelet, pconn, message):
"""
Handle a participant's operation.
If True is returned, go on with processing the next message.
If False is returned, discard any following messages.
"""
participant = pconn.participant
if message.has_key(u"type"):
if message["type"] == "WAVELET_OPEN":
logger.info("[%s/%d@%s] Opening wavelet" % (participant.name, pconn.id, wavelet.wave.id))
pconn.wavelets.add(wavelet)
# I know this is neat :)
self.emit(pconn, "WAVELET_OPEN", {
"wavelet": wavelet.serialize(),
"blips": wavelet.serialize_blips(),
})
elif message["type"] == "PARTICIPANT_INFO":
logger.info("[%s/%d@%s] Sending participant information" % (participant.name, pconn.id, wavelet.wave.id))
p_info = {}
for p_id in message["property"]:
try:
p_info[p_id] = Participant.objects.get(id=p_id).serialize()
except ObjectDoesNotExist:
p_info[p_id] = None
self.emit(pconn, "PARTICIPANT_INFO", p_info)
elif message["type"] == "PARTICIPANT_SEARCH":
if len(message["property"]) < getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0):
self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "TOO_SHORT", "data": getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0)})
logger.debug("[%s/%d@%s] Participant search query too short" % (participant.name, pconn.id, wavelet.wave.id))
else:
logger.info("[%s/%d@%s] Performing participant search" % (participant.name, pconn.id, wavelet.wave.id))
lst = []
for p in Participant.objects.filter(name__icontains=message["property"]).exclude(id=participant.id):
lst.append(p.id)
self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "OK", "data": lst})
elif message["type"] == "GADGET_LIST":
all_gadgets = map(lambda g: {"id": g.id, "uploaded_by": g.by_user.participants.all()[0].name, "name": g.title, "descr": g.description, "url": g.url}, Gadget.objects.all())
logger.info("[%s/%d@%s] Sending Gadget list" % (participant.name, pconn.id, wavelet.wave.id))
self.emit(pconn, "GADGET_LIST", all_gadgets)
elif message["type"] == "WAVELET_ADD_PARTICIPANT":
# Find participant
try:
p = Participant.objects.get(id=message["property"])
except ObjectDoesNotExist:
logger.error("[%s/%d@%s] Target participant '%s' not found" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
return # Fail silently (TODO: report error to user)
# Check if already participating
if wavelet.participants.filter(id=message["property"]).count() > 0:
logger.error("[%s/%d@%s] Target participant '%s' already there" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
return # Fail silently (TODO: report error to user)
wavelet.participants.add(p)
logger.info("[%s/%d@%s] Added new participant '%s'" % (participant.name, pconn.id, wavelet.wave.id, message["property"]))
self.broadcast(wavelet, "WAVELET_ADD_PARTICIPANT", message["property"])
elif message["type"] == "WAVELET_REMOVE_SELF":
self.broadcast(wavelet, "WAVELET_REMOVE_PARTICIPANT", participant.id)
wavelet.participants.remove(participant) # Bye bye
pconn.wavelets.remove(wavelet) # Also for your connection
logger.info("[%s/%d@%s] Participant removed himself" % (participant.name, pconn.id, wavelet.wave.id))
if wavelet.participants.count() == 0: # Oh my god, you killed the Wave! You bastard!
logger.info("[%s/%d@%s] Wave got killed!" % (participant.name, pconn.id, wavelet.wave.id))
wavelet.wave.delete()
return False
elif message["type"] == "OPERATION_MESSAGE_BUNDLE":
# Build OpManager
newdelta = OpManager(wavelet.wave.id, wavelet.id)
newdelta.unserialize(message["property"]["operations"])
version = message["property"]["version"]
# Transform
for delta in wavelet.deltas.filter(version__gt=version):
for op in delta.getOpManager().operations:
newdelta.transform(op) # Trash results (an existing delta cannot be changed)
# Apply
wavelet.applyOperations(newdelta.operations)
# Raise version and store
wavelet.version += 1
wavelet.save()
Delta.createByOpManager(newdelta, wavelet.version).save()
# Create tentative checksums
blipsums = wavelet.blipsums()
# Respond
self.emit(pconn, "OPERATION_MESSAGE_BUNDLE_ACK", {"version": wavelet.version, "blipsums": blipsums})
self.broadcast(wavelet, "OPERATION_MESSAGE_BUNDLE", {"version": wavelet.version, "operations": newdelta.serialize(), "blipsums": blipsums}, [pconn])
logger.debug("[%s/%d@%s] Processed delta #%d -> v%d" % (participant.name, pconn.id, wavelet.wave.id, version, wavelet.version))
else:
logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message))
else:
logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message))
return True
def purge_connections(self):
"""
This method will check all connections to the server and throw out all
closed ones.
This can be run asynchronously every 10 minutes or so.
"""
for conn in ParticipantConn.objects.all():
for wavelet in conn.wavelets.all():
if not self.queue_exists("%s.%s.waveop" % (conn.rx_key, wavelet.id)):
wavelet.participant_conns.remove(conn)
logger.info("[%s/%d@%s] Connection to wavelet closed" % (conn.participant.name, conn.id, wavelet.wave.id))
if conn.wavelets.count() == 0 and datetime.datetime.now() > conn.created + self.conn_min_lifetime:
conn_id, conn_participant_name = conn.id, conn.participant.name
conn.delete()
logger.info("[%s/%d] Connection to server closed" % (conn_participant_name, conn_id))
self.next_purge = datetime.datetime.now() + self.purge_every
def queue_exists(self, queue):
"""
Check if a queue exists, i.e. a user is connected to it (because
auto-delete is always turned on).
"""
logger.debug("Checking queue %s" % (queue))
qex = self.publisher.backend.queue_exists(queue)
# Re-open the channel if it was closed (this is a pyamqlib issue)
if self.publisher.backend.channel.connection == None:
self.publisher.backend.channel = self.publisher.backend.connection.connection.channel()
return qex
from carrot.connection import BrokerConnection
from carrot.messaging import Publisher
import sys
conn = BrokerConnection(hostname="localhost", port=5672,
userid="guest", password="******",
virtual_host="/")
publisher = Publisher(connection=conn,
exchange="sorting_room",
routing_key="jason")
publisher.send({"po_box": sys.argv[1]})
publisher.close()
def send(exchange, routing_key, msg):
publisher = Publisher(connection=conn,
exchange=exchange,
routing_key=routing_key)
publisher.send(msg)
publisher.close()
def test_consumer(self):
"""AMQP->RPC->AMQP
Send a AMQP message, and test RPCConsumer response.
Uses a mock RPC server that the consumer will call.
Check the consumer AMQP response with test consumer.
"""
class TestConsumer(IMessageBroker):
service_name = 'TestConsumer'
exchange_name = 'Test'
topic = 'test'
amqp_connection_settings = settings.AMQP_CONNECTION
def dispatch(self, message, request, qid):
""" AMQP -> RPC dispatcher.
"""
logging.info('dispatching AMQP -> RPC')
response = {}
if qid:
# call remote RPC
# this message has an qid so it expects a response
response['msg'] = test_rpc_service.push(request)
message.ack()
# return the response to AMQP, the caller should be listening ..
self.return_response(response,qid)
else:
# no qid, so do something stateless ..
print request
message.ack()
try:
consumer = TestConsumer()
self.assertEquals('test.request.*',consumer.binding_key)
self.assertEquals('test.response.%s',consumer.response_routing_key)
consumer.start()
test_rpc_service.start()
test_rpc_service.messages = []
# allow consumer to start
sleep(0.2)
self.assert_(not consumer.stopped)
self.assert_(consumer.isAlive())
# test variables
qid = str(randrange(0,999999))
jsondata = {'msg':'hello rpc'}
_publisher = Publisher(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
routing_key='test.request.'+qid,
)
# test channel
backend = self.test_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,)
backend.queue_bind("test",'Test','test.response.'+qid)
_consumer = Consumer(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
queue="test",
)
_consumer.discard_all()
logging.debug('publishing JSON message to RPC')
data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid})
_publisher.send(data_on_the_wire)
# allow data to pass the wire
sleep(0.2)
# retrieve dispatcher response
response = _consumer.fetch()
self.assert_(response, 'No response')
data = json.loads(response.payload)
self.assert_(len(data['msg']) > 0) # assert non-empty response
self.assertEquals('ok',data['msg'])
# check dispatcher RPC function
self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive')
self.assertEquals(test_rpc_service.messages[0], jsondata)
finally:
try:
consumer.stop()
except:
pass
try:
_consumer.close()
except:
pass
test_rpc_service.stop()
self.assert_(consumer.stopped)
from carrot.connection import BrokerConnection
from carrot.messaging import Publisher
conn = BrokerConnection(hostname="localhost", port=61617,
userid="admin", password="******",
virtual_host="/")
publisher = Publisher(connection=conn,
exchange="feed", routing_key="importer")
for i in range(30):
publisher.send({"name":"foo", "i":i})
publisher.close()
userid="guest", password="******",
virtual_host="/")
publisher = Publisher(connection=conn,
exchange="feed", routing_key="importer")
num = 10000
test_name = "concurrency_test_%d" % db.incr("concurrency_num_tests")
print "Running test %s at %d messages" % (test_name, num)
start_time = time.time()
for i in xrange(num):
publisher.send({
"download_url": "http://cnn.com/rss/edition.rss",
"rate_id": "243234-234234234-234234234234-234234234234",
"search_id": "234234234-234234234234-234234234-234234234",
"test_name": test_name,
"vendor": "hertz"})
publisher.close()
print "Done publishing in %f seconds" % (time.time() - start_time)
# Wait until all the results have been published
amount = 0
while amount != num:
amount = int(db.get(test_name) or 0)
print "Done in %f seconds" % (time.time() - start_time)
class Worker(object):
def __init__(self,
hostname='localhost',
port=5672,
userid="collectoruser",
password="******",
**options):
conn = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host='collectorvhost')
try:
self.publisher = Publisher(connection=conn,
exchange="collector.response",
exchange_type='direct',
routing_key="response",
serializer="json")
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="collector",
routing_key="collector.*.*")
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
#print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj['tags']:
if driver:
value = driver.read_value(tag['device'],
tag['address'])
else:
value = "BAD_DRIVER"
values.append({
'name': tag['name'],
'current_value': value,
'read_at': '2009-01-01'
})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
port = 1000 + easyip.EASYIP_PORT
buf = 1024
addr = (host, port)
# Create socket and bind to address
UDPSock = socket(AF_INET, SOCK_DGRAM)
UDPSock.bind(addr)
print "waiting"
# Receive messages
while 1:
data, addr = UDPSock.recvfrom(buf)
if not data:
print "Client has exited!"
break
else:
packet = easyip.Packet(data)
print "\nReceived message '", packet, "' with %s" % (packet.payload, )
response = easyip.Factory.response(packet)
UDPSock.sendto(response.pack(), addr)
print "Responded to %s, %s" % addr
print "Putting incomming on queue"
publisher.send({'address': addr, 'payload': packet.payload})
print "Closing"
# Close socket
UDPSock.close()
publisher.close()
def reply(data, exchange, routing_key, connection=None, connect_timeout=None,
**kwargs):
pub = Publisher(connection, exchange=exchange,
routing_key=routing_key, **kwargs)
pub.send(data)
class SignalAMQP(Signal):
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def send(self, message, **kw):
""" Transfer message to bus. Message can be any simple python type. """
self.publisher.send(message, **kw)
def callback(self, message_data, message):
""" Consumer callback function. Send Django singnal."""
try:
sender = message_data['sender']
except:
sender = None
super(SignalAMQP, self).send(sender=sender,
message_data=message_data,
message=message)
message.ack()
if self.debug:
print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message
return True
def get_backend(self):
return self.conn.get_backend_cls()(DjangoBrokerConnection())
def queue_bind(self, queue_list):
""" Bind another queues to current exchange """
routing_keys = []
backend = self.get_backend()
for x in queue_list:
backend.queue_bind(queue=x, exchange=self.exchange, \
routing_key='%s_%s' % (self.exchange, x))
routing_keys.append('%s_%s' % (self.exchange, x))
return routing_keys
def listen(self):
""" Run consumer loop thread """
cl = ConsumerLoop()
cl.consumer = self.consumer
cl.start()
return cl
def stop(self):
""" Unactivate this signal """
self.conn.close()
