def cast(msg, event_type, topic, priority):
yagi.config.setup(config_path="/etc/yagi.conf")
conf = yagi.config.config_with("rabbit_broker")
host = conf("host")
port = conf("port")
user = conf("user")
exchange = conf("exchange")
password = conf("password")
vhost = conf("vhost")
message_dict = {
"message_id": str(uuid.uuid4()),
"event_type": event_type,
"publisher_id": "some_publisher",
"priority": priority,
"timestamp": str(datetime.datetime.utcnow()),
"payload": msg,
}
conn = BrokerConnection(hostname=host, port=port, userid=user, password=password, virtual_host=vhost)
publisher = Publisher(
connection=conn,
exchange=exchange,
routing_key="%s.%s" % (topic, priority),
durable=False,
exchange_type="topic",
)
publisher.send(message_dict)
publisher.close()
def return_response(self, response, qid):
"""AMQP "response" handler.
Publishes a response message to a temporary queue.
- response is json, qid is string
"""
message = json.dumps(response)
routing_key = self.response_routing_key % qid
logging.debug("response to %s with routing_key: %s, message: \n%s" % (self.exchange_name, routing_key, message))
try:
publisher = Publisher(
connection=self.amqp_connection,
exchange=self.exchange_name,
exchange_type="topic",
routing_key=routing_key,
)
publisher.send(message)
publisher.close()
except:
"""Trying to send with broken connection.
Handle gracefully by waiting for connection and publish again."""
logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1]))
self._wait_connection(5)
self.return_response(response, qid)
class Worker(object):
def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options):
conn = BrokerConnection(
hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost"
)
try:
self.publisher = Publisher(
connection=conn,
exchange="collector.response",
exchange_type="direct",
routing_key="response",
serializer="json",
)
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(
connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*"
)
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
# print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj["tags"]:
if driver:
value = driver.read_value(tag["device"], tag["address"])
else:
value = "BAD_DRIVER"
values.append({"name": tag["name"], "current_value": value, "read_at": "2009-01-01"})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
def _send_message(connection, routing_key, data, envelope):
# Make a message envelope if we are told to have one
if envelope:
message = {}
message["payload"] = data
message["_meta"] = {
"exchange": CONF["MSG_EXCHANGE"],
"routing_key": routing_key,
"sent": mercurial.util.datestr(None, CONF["DATE_FORMAT"]),
# TODO: Support more than just JSON
"serializer": "json",
}
else:
message = data
# Set up our broker publisher
publisher = Publisher(
connection=connection, exchange=CONF["MSG_EXCHANGE"], exchange_type="topic", routing_key=routing_key
)
# Send the message
# TODO: Support more than just JSON
publisher.send(message)
# Close the publishing connection
publisher.close()
def __init__(self,
hostname='localhost',
port=5672,
userid="collectoruser",
password="******",
**options):
conn = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host='collectorvhost')
try:
self.publisher = Publisher(connection=conn,
exchange="collector.response",
exchange_type='direct',
routing_key="response",
serializer="json")
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="collector",
routing_key="collector.*.*")
def _send_message(routing_key, request):
conn = DjangoAMQPConnection()
publisher = Publisher(connection=conn, exchange='request',
exchange_type='topic', routing_key=routing_key,
serializer='pickle')
publisher.send(request)
publisher.close()
conn.close()
def reply(routing_key, data):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn, exchange="django", routing_key=routing_key, exchange_type="topic")
publisher.send(data)
publisher.close()
conn.close()
def sender_callback(self, routing_key, data):
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn,
exchange="django_send",
routing_key=routing_key,
exchange_type="topic",
)
publisher.send(data)
publisher.close()
conn.close()
print "Sent object change/delete message for %s" % routing_key
def request_no_response():
"""No response"""
data = {'q': 'hello AMQP backend, I am Python'}
broker = ExampleBroker()
publisher = Publisher(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
routing_key=broker.binding_key,
)
publisher.send(json.dumps(data))
def request_no_response():
"""No response"""
data = {'q': 'hello AMQP backend, I am Python'}
broker = ExampleBroker()
publisher = Publisher(
connection = broker.amqp_connection,
exchange = broker.exchange_name,
exchange_type = "topic",
routing_key = broker.binding_key,
)
publisher.send(json.dumps(data))
def test_amqp(self):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher, Consumer
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
publisher.send("test")
publisher.close()
connection.close()
def send_msg(routing_key, data):
conn = BrokerConnection(
hostname=settings.BROKER_HOST,
port=settings.BROKER_PORT,
userid=settings.BROKER_USER,
password=settings.BROKER_PASSWORD,
virtual_host=settings.BROKER_VHOST)
publisher = Publisher(connection=conn,
exchange="django_send",
routing_key=routing_key,
exchange_type="topic",
)
publisher.send(data)
publisher.close()
conn.close()
def create_publisher(self, exchange=None, routing_key=None, **options):
exchange = exchange or self.exchange
routing_key = routing_key or self.routing_key
return Publisher(connection=self.conn,
exchange=exchange,
routing_key=routing_key,
**options)
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn, queue="test",
exchange="test", routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn, exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
def enable(self):
conf = self.app.conf
self.enabled = True
self.publisher = Publisher(self.connection,
exchange=conf.CELERY_EVENT_EXCHANGE,
exchange_type=conf.CELERY_EVENT_EXCHANGE_TYPE,
routing_key=conf.CELERY_EVENT_ROUTING_KEY,
serializer=conf.CELERY_EVENT_SERIALIZER)
def send_requests(requests, **options):
logger = logging
if 'logger' in options.keys():
logger = options['logger']
"""Send a import request message to be picked up by workers."""
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
for req in requests:
routing_key=req['driver_routing_key']
publisher.send(req, routing_key=routing_key)
logger.debug("Sent request with routing_key %s:%s" %( routing_key,req, ))
publisher.close()
connection.close()
def publish(self, message):
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a message given
if not message:
raise MalformedMessage(message)
# Have the message prepare and validate itself
message._prepare()
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker publisher
self.publisher = Publisher(connection=self.connection,
exchange=self.exchange,
exchange_type="topic",
routing_key=message.routing_key)
# The message is actually a simple envelope format with a payload and
# some metadata
final_data = {}
final_data['payload'] = message.data
final_data['_meta'] = message.metadata.copy()
final_data['_meta'].update({
'exchange':
self.exchange,
'routing_key':
message.routing_key,
'serializer':
self.config.serializer,
'sent':
time_to_string(datetime.now(timezone(self.config.broker_timezone)))
})
# Send the message
self.publisher.send(final_data, serializer=self.config.serializer)
# Close the publishing connection
self.publisher.close()
def main():
connection = BrokerConnection(
hostname = "localhost",
port = 5672,
userid = "test",
password = "******",
virtual_host = "test.com",
)
publisher = Publisher(
connection = connection,
exchange = "messages",
routing_key = "awesome",
)
for i in xrange(100):
publisher.send({"a": i})
publisher.close()
class AMQPTransmitter(Transmitter):
def __init__(self, hostname='localhost', port=5672,
userid=None, password=None, virtual_host=None,
exchange=None):
self.hostname = hostname
self.port = port
self.userid = userid
self.password = password
self.virtual_host = None
self.exchange = exchange
self.connection = BrokerConnection(hostname=self.hostname, port=self.port,
userid=self.userid, password=self.password,
virtual_host=self.virtual_host)
self.publisher = Publisher(connection=self.connection,
exchange=exchange)
def transmit(self, message):
self.publisher.send(message)
class RabbitMQPipeline(object):
def __init__(self, hostname, port, user_id, password, virtual_host, encoder_class):
self.queue_connection = BrokerConnection(
hostname=hostname,
port=port,
userid=user_id,
password=password,
virtual_host=virtual_host
)
self.encoder = encoder_class()
# Setup / Teardown Rabbit plumbing when spider opens / closes
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
@classmethod
def from_settings(cls, settings):
hostname = settings.get('BROKER_HOST')
port = settings.get('BROKER_PORT')
user_id = settings.get('BROKER_USERID')
password = settings.get('BROKER_PASSWORD')
virtual_host = settings.get('BROKER_VIRTUAL_HOST')
encoder_class = settings.get('QUEUE_SERIALIZER', ScrapyJSONEncoder)
return cls(hostname, port, user_id, password, virtual_host, encoder_class)
def spider_opened(self, spider):
self.publisher = Publisher(
connection=self.queue_connection,
exchange='',
routing_key=spider.name
)
def spider_closed(self, spider):
self.publisher.close()
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
def _process_item(self, item, spider):
self.publisher.send(self.encoder.encode(dict(item)))
return item
def request_with_response():
"""Example of requesting over AMQP from another process, not necessarily Python."""
qid = str(randrange(0,999))
data = {'qid': qid, 'q': 'what time is it?'}
broker = ExampleBroker()
publisher = Publisher(
connection = broker.amqp_connection,
exchange = broker.exchange_name,
exchange_type = "topic",
routing_key = broker.binding_key,
)
# declare test queue for receiving response message
backend = broker.amqp_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,)
backend.queue_bind(
"test",
broker.exchange_name,
broker.response_routing_key % qid)
consumer = Consumer(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
queue="test",
)
consumer.discard_all()
# send message
publisher.send(json.dumps(data))
# allow data to pass the wire
sleep(0.2)
# retrieve response
response = consumer.fetch()
payload = json.loads(response.payload)
print "Response from AMQP: %s" % payload
class MessageQueuePipeline(object):
def __init__(self, host_name, port, userid, password, virtual_host,
encoder_class):
self.q_connection = BrokerConnection(hostname=host_name,
port=port,
userid=userid,
password=password,
virtual_host=virtual_host)
self.encoder = encoder_class()
@classmethod
def from_settings(cls, settings):
host_name = settings.get('BROKER_HOST', 'localhost')
port = settings.get('BROKER_PORT', 5672)
userid = settings.get('BROKER_USERID', "guest")
password = settings.get('BROKER_PASSWORD', "guest")
virtual_host = settings.get('BROKER_VIRTUAL_HOST', "/")
encoder_class = settings.get('MESSAGE_Q_SERIALIZER', ScrapyJSONEncoder)
return cls(host_name, port, userid, password, virtual_host,
encoder_class)
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.publisher = Publisher(connection=self.q_connection,
exchange="",
routing_key=spider.name)
def spider_closed(self, spider):
self.publisher.close()
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
def _process_item(self, item, spider):
self.publisher.send({"scraped_data": self.encoder.encode(dict(item))})
return item
def publish(self, exchange, routing_key, message,
auto_declare=False, persistent=True):
"""publish a message to exchange using routing_key
exchange - name of exchange
routing_key - interpretation of routing key depends on exchange type
message - message content to send
auto_declare - automatically declare the exchange (default: false)
persistent - store message on disk as well as memory (default: True)
"""
delivery_mode = 2
if not persistent:
delivery_mode = 1
publisher = Publisher(connection=self.broker,
exchange=exchange, routing_key=routing_key,
auto_declare=auto_declare)
publisher.send(message, delivery_mode=delivery_mode)
publisher.close()
class AmqpMessageProcessor(object):
purge_every = datetime.timedelta(minutes=10)
conn_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2))
def __init__(self, connection):
self.pygo_mp = PyGoWaveClientMessageProcessor()
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json"
)
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def wait(self, limit=None):
self.consumer.wait(limit)
def send(self, routing_key, message_data):
self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1)
def receive(self, message_data, message):
routing_key = message.amqp_message.routing_key
msg_dict = self.pygo_mp.process(routing_key, message_data)
for out_rkey, messages in msg_dict.iteritems():
self.send(out_rkey, messages)
# Cleanup time?
if datetime.datetime.now() > self.next_purge:
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def request_with_response():
"""Example of requesting over AMQP from another process, not necessarily Python."""
qid = str(randrange(0, 999))
data = {'qid': qid, 'q': 'what time is it?'}
broker = ExampleBroker()
publisher = Publisher(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
routing_key=broker.binding_key,
)
# declare test queue for receiving response message
backend = broker.amqp_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,
)
backend.queue_bind("test", broker.exchange_name,
broker.response_routing_key % qid)
consumer = Consumer(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
queue="test",
)
consumer.discard_all()
# send message
publisher.send(json.dumps(data))
# allow data to pass the wire
sleep(0.2)
# retrieve response
response = consumer.fetch()
payload = json.loads(response.payload)
print "Response from AMQP: %s" % payload
def main():
logging.basicConfig(level=logging.DEBUG)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
publisher = Publisher(auto_declare=False,
connection=conn,
exchange=args.exchange,
routing_key=args.key)
logging.info("Declaring exchange: %s" % args.exchange)
publisher.backend.exchange_declare(exchange=args.exchange,
type="topic",
durable=False,
auto_delete=False)
while True:
line = sys.stdin.readline()
if not line:
break
logging.debug("Sending message '%s'" % line.strip())
publisher.send(line.strip())
publisher.close()
def cast(msg, event_type, topic, priority):
yagi.config.setup(config_path='/etc/yagi.conf')
conf = yagi.config.config_with('rabbit_broker')
host = conf('host')
port = conf('port')
user = conf('user')
exchange = conf('exchange')
password = conf('password')
vhost = conf('vhost')
message_dict = {
'message_id': str(uuid.uuid4()),
'event_type': event_type,
'publisher_id': 'some_publisher',
'priority': priority,
'timestamp': str(datetime.datetime.utcnow()),
'payload': msg
}
conn = BrokerConnection(hostname=host, port=port, userid=user,
password=password, virtual_host=vhost)
publisher = Publisher(connection=conn, exchange=exchange,
routing_key="%s.%s" % (topic, priority), durable=False,
exchange_type='topic')
publisher.send(message_dict)
publisher.close()
def publish(self,
exchange,
routing_key,
message,
auto_declare=False,
persistent=True):
"""publish a message to exchange using routing_key
exchange - name of exchange
routing_key - interpretation of routing key depends on exchange type
message - message content to send
auto_declare - automatically declare the exchange (default: false)
persistent - store message on disk as well as memory (default: True)
"""
delivery_mode = 2
if not persistent:
delivery_mode = 1
publisher = Publisher(connection=self.broker,
exchange=exchange,
routing_key=routing_key,
auto_declare=auto_declare)
publisher.send(message, delivery_mode=delivery_mode)
publisher.close()
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn,
queue="test",
exchange="test",
routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn,
exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
def _dispatch_message(self, message):
if not message.claim(self.context.db):
return
try:
publisher = Publisher(self.context.broker, exchange=message.options.exchange,
exchange_type=message.options.exchange_type)
publisher.send(message.message,
routing_key = message.options.routing_key,
delivery_mode = message.options.delivery_mode,
mandatory = message.options.mandatory,
priority = message.options.priority)
publisher.close()
except:
log.error("Error dispatching deferred message %s: %s" % (message, traceback.format_exc()))
self.error_reschedule(message)
return False
else:
log.debug("Dispatched message %s" % message)
# sent with no problems, done with it.
self.context.db.delete(message)
return True
class MessageQueuePipeline(object):
def __init__(self, host_name, port, userid, password, virtual_host, encoder_class):
self.q_connection = BrokerConnection(hostname=host_name, port=port,
userid=userid, password=password,
virtual_host=virtual_host)
self.encoder = encoder_class()
@classmethod
def from_settings(cls, settings):
host_name = settings.get('BROKER_HOST', 'localhost')
port = settings.get('BROKER_PORT', 5672)
userid = settings.get('BROKER_USERID', "guest")
password = settings.get('BROKER_PASSWORD', "guest")
virtual_host = settings.get('BROKER_VIRTUAL_HOST', "/")
encoder_class = settings.get('MESSAGE_Q_SERIALIZER', ScrapyJSONEncoder)
return cls(host_name, port, userid, password, virtual_host, encoder_class)
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.publisher = Publisher(connection=self.q_connection,
exchange="", routing_key=spider.name)
def spider_closed(self, spider):
self.publisher.close()
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
def _process_item(self, item, spider):
self.publisher.send({"scraped_data": self.encoder.encode(dict(item))})
return item
def main():
logging.basicConfig(level=logging.DEBUG)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
publisher = Publisher(
auto_declare = False,
connection = conn,
exchange = args.exchange,
routing_key = args.key
)
logging.info("Declaring exchange: %s" % args.exchange)
publisher.backend.exchange_declare(exchange=args.exchange, type="topic", durable=False, auto_delete=False)
while True:
line = sys.stdin.readline()
if not line:
break
logging.debug("Sending message '%s'" % line.strip())
publisher.send(line.strip())
publisher.close()
def __init__(self, connection):
self.pygo_mp = PyGoWaveClientMessageProcessor()
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json"
)
self.pygo_mp.purge_connections()
self.next_purge = datetime.datetime.now() + self.purge_every
def test_amqp(self):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher, Consumer
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
publisher.send("test")
publisher.close()
connection.close()
def test_with_statement(self):
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Publisher(connection=conn, exchange="F", routing_key="G") \
as publisher:
self.assertFalse(publisher._closed)
self.assertTrue(conn._closed)
self.assertTrue(publisher._closed)
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Consumer(connection=conn,
queue="E",
exchange="F",
routing_key="G") as consumer:
self.assertFalse(consumer._closed)
self.assertTrue(conn._closed)
self.assertTrue(consumer._closed)
def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options):
conn = BrokerConnection(
hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost"
)
try:
self.publisher = Publisher(
connection=conn,
exchange="collector.response",
exchange_type="direct",
routing_key="response",
serializer="json",
)
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(
connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*"
)
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def __init__(self, connection):
self.consumer = Consumer(
connection,
queue="wavelet_rpc_singlethread",
exchange="wavelet.topic",
routing_key="#.#.clientop",
exchange_type="topic",
serializer="json",
auto_ack=True,
)
self.consumer.register_callback(self.receive)
self.publisher = Publisher(
connection,
exchange="wavelet.direct",
exchange_type="direct",
delivery_mode=1,
serializer="json",
)
self.out_queue = {}
self.purge_connections()
def publish(self, message):
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a message given
if not message:
raise MalformedMessage(message)
# Have the message prepare and validate itself
message._prepare()
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker publisher
self.publisher = Publisher(connection=self.connection,
exchange=self.exchange,
exchange_type="topic",
routing_key=message.routing_key)
# The message is actually a simple envelope format with a payload and
# some metadata
final_data = {}
final_data['payload'] = message.data
final_data['_meta'] = message.metadata.copy()
final_data['_meta'].update({
'exchange': self.exchange,
'routing_key': message.routing_key,
'serializer': self.config.serializer,
'sent': time_to_string(datetime.now(timezone(self.config.broker_timezone)))
})
# Send the message
self.publisher.send(final_data, serializer=self.config.serializer)
# Close the publishing connection
self.publisher.close()
def send_requests(requests, **options):
logger = logging
if 'logger' in options.keys():
logger = options['logger']
"""Send a import request message to be picked up by workers."""
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
for req in requests:
routing_key = req['driver_routing_key']
publisher.send(req, routing_key=routing_key)
logger.debug("Sent request with routing_key %s:%s" % (
routing_key,
req,
))
publisher.close()
connection.close()
def create_README_publisher(amqpconn):
from carrot.messaging import Publisher
publisher = Publisher(connection=amqpconn, exchange=README_EXCHANGE,
routing_key=README_ROUTING_KEY)
return publisher
class GenericPublisher(object):
def __init__(self, config, exchange=None, connect=True):
self.config = config
self.exchange = exchange
self.connection = None
if connect:
self.connect()
# Connect to the message broker
def connect(self):
if not self.connection:
self.connection = BrokerConnection(hostname=self.config.host,
port=self.config.port,
userid=self.config.user,
password=self.config.password,
virtual_host=self.config.vhost)
# Disconnect from the message broker
def disconnect(self):
if self.connection:
self.connection.close()
self.connection = None
# Used to publish a pulse message to the proper exchange
def publish(self, message):
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a message given
if not message:
raise MalformedMessage(message)
# Have the message prepare and validate itself
message._prepare()
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker publisher
self.publisher = Publisher(connection=self.connection,
exchange=self.exchange,
exchange_type="topic",
routing_key=message.routing_key)
# The message is actually a simple envelope format with a payload and
# some metadata
final_data = {}
final_data['payload'] = message.data
final_data['_meta'] = message.metadata.copy()
final_data['_meta'].update({
'exchange':
self.exchange,
'routing_key':
message.routing_key,
'serializer':
self.config.serializer,
'sent':
time_to_string(datetime.now(timezone(self.config.broker_timezone)))
})
# Send the message
self.publisher.send(final_data, serializer=self.config.serializer)
# Close the publishing connection
self.publisher.close()
def get_publisher(self, broker, **kwargs):
return Publisher(broker, **kwargs)
from carrot.connection import BrokerConnection
from carrot.messaging import Publisher
import sys
conn = BrokerConnection(hostname="localhost", port=5672,
userid="guest", password="******",
virtual_host="/")
publisher = Publisher(connection=conn,
exchange="sorting_room",
routing_key="jason")
publisher.send({"po_box": sys.argv[1]})
publisher.close()
def send(exchange, routing_key, msg):
publisher = Publisher(connection=conn,
exchange=exchange,
routing_key=routing_key)
publisher.send(msg)
publisher.close()
def create_publisher(self, **options):
return Publisher(connection=self.conn,
exchange=self.exchange,
routing_key=self.routing_key,
**options)
def get_publisher(routing_key):
return Publisher(connection=get_carrot_connection(),
exchange=EXCHANGE_NAME,
exchange_type=EXCHANGE_TYPE,
routing_key=routing_key)
def spider_opened(self, spider):
self.publisher = Publisher(connection=self.q_connection,
exchange="",
routing_key=spider.name)
def get_publisher(config):
return Publisher(connection=get_carrot_connection(config),
exchange=config.get('ckan.site_id'),
exchange_type=EXCHANGE_TYPE)
class Worker(object):
def __init__(self,
hostname='localhost',
port=5672,
userid="collectoruser",
password="******",
**options):
conn = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host='collectorvhost')
try:
self.publisher = Publisher(connection=conn,
exchange="collector.response",
exchange_type='direct',
routing_key="response",
serializer="json")
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="collector",
routing_key="collector.*.*")
def run(self):
self.consumer.register_callback(self._get_request_callback)
print "Waiting..."
self.consumer.wait()
def _get_request_callback(self, message_data, message):
#print "message_data=%s" % message_data
drivers = []
drivers.append(message_data)
DRIVER_KEY = "driver_routing_key"
TAGS_KEY = "tags"
for obj in drivers:
driver = None
if DRIVER_KEY in obj and TAGS_KEY in obj:
module_name = obj["driver_routing_key"]
try:
__import__(module_name)
module = sys.modules[module_name]
driver = module.Driver()
except:
print "could not import module %s" % module_name
values = []
for tag in obj['tags']:
if driver:
value = driver.read_value(tag['device'],
tag['address'])
else:
value = "BAD_DRIVER"
values.append({
'name': tag['name'],
'current_value': value,
'read_at': '2009-01-01'
})
self.publisher.send(values)
else:
print "Badly formated request %s" % obj
message.ack()
class WorkerException(Exception):
pass
class ConnectionException(WorkerException):
pass
class SignalAMQP(Signal):
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def send(self, message, **kw):
""" Transfer message to bus. Message can be any simple python type. """
self.publisher.send(message, **kw)
def callback(self, message_data, message):
""" Consumer callback function. Send Django singnal."""
try:
sender = message_data['sender']
except:
sender = None
super(SignalAMQP, self).send(sender=sender,
message_data=message_data,
message=message)
message.ack()
if self.debug:
print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message
return True
def get_backend(self):
return self.conn.get_backend_cls()(DjangoBrokerConnection())
def queue_bind(self, queue_list):
""" Bind another queues to current exchange """
routing_keys = []
backend = self.get_backend()
for x in queue_list:
backend.queue_bind(queue=x, exchange=self.exchange, \
routing_key='%s_%s' % (self.exchange, x))
routing_keys.append('%s_%s' % (self.exchange, x))
return routing_keys
def listen(self):
""" Run consumer loop thread """
cl = ConsumerLoop()
cl.consumer = self.consumer
cl.start()
return cl
def stop(self):
""" Unactivate this signal """
self.conn.close()
#!/usr/bin/env python
# coding=utf-8
from socket import socket, AF_INET, SOCK_DGRAM
from fstlib import easyip
from carrot.messaging import Publisher
from carrot.connection import BrokerConnection
conn = BrokerConnection(hostname='localhost',
port=5672,
userid='test',
password='******',
virtual_host='test')
publisher = Publisher(connection=conn,
exchange="fst",
exchange_type='topic',
routing_key="easyip.incomming",
serializer="json")
# Set the socket parameters
host = "localhost"
port = 1000 + easyip.EASYIP_PORT
buf = 1024
addr = (host, port)
# Create socket and bind to address
UDPSock = socket(AF_INET, SOCK_DGRAM)
UDPSock.bind(addr)
print "waiting"
