class Worker(object): def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options): conn = BrokerConnection( hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost" ) try: self.publisher = Publisher( connection=conn, exchange="collector.response", exchange_type="direct", routing_key="response", serializer="json", ) except Exception as ex: raise Worker.ConnectionException(ex) self.consumer = Consumer( connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*" ) def run(self): self.consumer.register_callback(self._get_request_callback) print "Waiting..." self.consumer.wait() def _get_request_callback(self, message_data, message): # print "message_data=%s" % message_data drivers = [] drivers.append(message_data) DRIVER_KEY = "driver_routing_key" TAGS_KEY = "tags" for obj in drivers: driver = None if DRIVER_KEY in obj and TAGS_KEY in obj: module_name = obj["driver_routing_key"] try: __import__(module_name) module = sys.modules[module_name] driver = module.Driver() except: print "could not import module %s" % module_name values = [] for tag in obj["tags"]: if driver: value = driver.read_value(tag["device"], tag["address"]) else: value = "BAD_DRIVER" values.append({"name": tag["name"], "current_value": value, "read_at": "2009-01-01"}) self.publisher.send(values) else: print "Badly formated request %s" % obj message.ack() class WorkerException(Exception): pass class ConnectionException(WorkerException): pass
def return_response(self, response, qid): """AMQP "response" handler. Publishes a response message to a temporary queue. - response is json, qid is string """ message = json.dumps(response) routing_key = self.response_routing_key % qid logging.debug("response to %s with routing_key: %s, message: \n%s" % (self.exchange_name, routing_key, message)) try: publisher = Publisher( connection=self.amqp_connection, exchange=self.exchange_name, exchange_type="topic", routing_key=routing_key, ) publisher.send(message) publisher.close() except: """Trying to send with broken connection. Handle gracefully by waiting for connection and publish again.""" logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1])) self._wait_connection(5) self.return_response(response, qid)
def cast(msg, event_type, topic, priority): yagi.config.setup(config_path='/etc/yagi.conf') conf = yagi.config.config_with('rabbit_broker') host = conf('host') port = conf('port') user = conf('user') exchange = conf('exchange') password = conf('password') vhost = conf('vhost') message_dict = { 'message_id': str(uuid.uuid4()), 'event_type': event_type, 'publisher_id': 'some_publisher', 'priority': priority, 'timestamp': str(datetime.datetime.utcnow()), 'payload': msg } conn = BrokerConnection(hostname=host, port=port, userid=user, password=password, virtual_host=vhost) publisher = Publisher(connection=conn, exchange=exchange, routing_key="%s.%s" % (topic, priority), durable=False, exchange_type='topic') publisher.send(message_dict) publisher.close()
def _send_message(connection, routing_key, data, envelope): # Make a message envelope if we are told to have one if envelope: message = {} message["payload"] = data message["_meta"] = { "exchange": CONF["MSG_EXCHANGE"], "routing_key": routing_key, "sent": mercurial.util.datestr(None, CONF["DATE_FORMAT"]), # TODO: Support more than just JSON "serializer": "json", } else: message = data # Set up our broker publisher publisher = Publisher( connection=connection, exchange=CONF["MSG_EXCHANGE"], exchange_type="topic", routing_key=routing_key ) # Send the message # TODO: Support more than just JSON publisher.send(message) # Close the publishing connection publisher.close()
def main(): logging.basicConfig(level=logging.DEBUG) args = parser.parse_args() conn = BrokerConnection( hostname=args.hostname, virtual_host=args.vhost, userid=args.user, password=args.password, ) publisher = Publisher(auto_declare=False, connection=conn, exchange=args.exchange, routing_key=args.key) logging.info("Declaring exchange: %s" % args.exchange) publisher.backend.exchange_declare(exchange=args.exchange, type="topic", durable=False, auto_delete=False) while True: line = sys.stdin.readline() if not line: break logging.debug("Sending message '%s'" % line.strip()) publisher.send(line.strip()) publisher.close()
def publish(self, exchange, routing_key, message, auto_declare=False, persistent=True): """publish a message to exchange using routing_key exchange - name of exchange routing_key - interpretation of routing key depends on exchange type message - message content to send auto_declare - automatically declare the exchange (default: false) persistent - store message on disk as well as memory (default: True) """ delivery_mode = 2 if not persistent: delivery_mode = 1 publisher = Publisher(connection=self.broker, exchange=exchange, routing_key=routing_key, auto_declare=auto_declare) publisher.send(message, delivery_mode=delivery_mode) publisher.close()
def cast(msg, event_type, topic, priority): yagi.config.setup(config_path="/etc/yagi.conf") conf = yagi.config.config_with("rabbit_broker") host = conf("host") port = conf("port") user = conf("user") exchange = conf("exchange") password = conf("password") vhost = conf("vhost") message_dict = { "message_id": str(uuid.uuid4()), "event_type": event_type, "publisher_id": "some_publisher", "priority": priority, "timestamp": str(datetime.datetime.utcnow()), "payload": msg, } conn = BrokerConnection(hostname=host, port=port, userid=user, password=password, virtual_host=vhost) publisher = Publisher( connection=conn, exchange=exchange, routing_key="%s.%s" % (topic, priority), durable=False, exchange_type="topic", ) publisher.send(message_dict) publisher.close()
def _send_message(routing_key, request): conn = DjangoAMQPConnection() publisher = Publisher(connection=conn, exchange='request', exchange_type='topic', routing_key=routing_key, serializer='pickle') publisher.send(request) publisher.close() conn.close()
def reply(routing_key, data): from carrot.connection import DjangoBrokerConnection from carrot.messaging import Publisher conn = DjangoBrokerConnection() publisher = Publisher(connection=conn, exchange="django", routing_key=routing_key, exchange_type="topic") publisher.send(data) publisher.close() conn.close()
def sender_callback(self, routing_key, data): conn = DjangoBrokerConnection() publisher = Publisher(connection=conn, exchange="django_send", routing_key=routing_key, exchange_type="topic", ) publisher.send(data) publisher.close() conn.close() print "Sent object change/delete message for %s" % routing_key
def request_no_response(): """No response""" data = {'q': 'hello AMQP backend, I am Python'} broker = ExampleBroker() publisher = Publisher( connection=broker.amqp_connection, exchange=broker.exchange_name, exchange_type="topic", routing_key=broker.binding_key, ) publisher.send(json.dumps(data))
def request_no_response(): """No response""" data = {'q': 'hello AMQP backend, I am Python'} broker = ExampleBroker() publisher = Publisher( connection = broker.amqp_connection, exchange = broker.exchange_name, exchange_type = "topic", routing_key = broker.binding_key, ) publisher.send(json.dumps(data))
def test_amqp(self): from carrot.connection import DjangoBrokerConnection from carrot.messaging import Publisher, Consumer connection = DjangoBrokerConnection() publisher = Publisher(connection=connection, exchange="collector", exchange_type='topic', routing_key="collector.driver", serializer='json') publisher.send("test") publisher.close() connection.close()
def send_msg(routing_key, data): conn = BrokerConnection( hostname=settings.BROKER_HOST, port=settings.BROKER_PORT, userid=settings.BROKER_USER, password=settings.BROKER_PASSWORD, virtual_host=settings.BROKER_VHOST) publisher = Publisher(connection=conn, exchange="django_send", routing_key=routing_key, exchange_type="topic", ) publisher.send(data) publisher.close() conn.close()
def test_consumer_interface(self): to_send = ['No', 'soup', 'for', 'you!'] messages = [] def cb(message_data, message): messages.append(message_data) conn = BrokerConnection(backend_cls='memory') consumer = Consumer(connection=conn, queue="test", exchange="test", routing_key="test") consumer.register_callback(cb) publisher = Publisher(connection=conn, exchange="test", routing_key="test") for i in to_send: publisher.send(i) it = consumer.iterconsume() for i in range(len(to_send)): it.next() self.assertEqual(messages, to_send)
def send_requests(requests, **options): logger = logging if 'logger' in options.keys(): logger = options['logger'] """Send a import request message to be picked up by workers.""" connection = DjangoBrokerConnection() publisher = Publisher(connection=connection, exchange="collector", exchange_type='topic', routing_key="collector.driver", serializer='json') for req in requests: routing_key=req['driver_routing_key'] publisher.send(req, routing_key=routing_key) logger.debug("Sent request with routing_key %s:%s" %( routing_key,req, )) publisher.close() connection.close()
class AMQPTransmitter(Transmitter): def __init__(self, hostname='localhost', port=5672, userid=None, password=None, virtual_host=None, exchange=None): self.hostname = hostname self.port = port self.userid = userid self.password = password self.virtual_host = None self.exchange = exchange self.connection = BrokerConnection(hostname=self.hostname, port=self.port, userid=self.userid, password=self.password, virtual_host=self.virtual_host) self.publisher = Publisher(connection=self.connection, exchange=exchange) def transmit(self, message): self.publisher.send(message)
def main(): connection = BrokerConnection( hostname = "localhost", port = 5672, userid = "test", password = "******", virtual_host = "test.com", ) publisher = Publisher( connection = connection, exchange = "messages", routing_key = "awesome", ) for i in xrange(100): publisher.send({"a": i}) publisher.close()
class RabbitMQPipeline(object): def __init__(self, hostname, port, user_id, password, virtual_host, encoder_class): self.queue_connection = BrokerConnection( hostname=hostname, port=port, userid=user_id, password=password, virtual_host=virtual_host ) self.encoder = encoder_class() # Setup / Teardown Rabbit plumbing when spider opens / closes dispatcher.connect(self.spider_opened, signals.spider_opened) dispatcher.connect(self.spider_closed, signals.spider_closed) @classmethod def from_settings(cls, settings): hostname = settings.get('BROKER_HOST') port = settings.get('BROKER_PORT') user_id = settings.get('BROKER_USERID') password = settings.get('BROKER_PASSWORD') virtual_host = settings.get('BROKER_VIRTUAL_HOST') encoder_class = settings.get('QUEUE_SERIALIZER', ScrapyJSONEncoder) return cls(hostname, port, user_id, password, virtual_host, encoder_class) def spider_opened(self, spider): self.publisher = Publisher( connection=self.queue_connection, exchange='', routing_key=spider.name ) def spider_closed(self, spider): self.publisher.close() def process_item(self, item, spider): return deferToThread(self._process_item, item, spider) def _process_item(self, item, spider): self.publisher.send(self.encoder.encode(dict(item))) return item
def request_with_response(): """Example of requesting over AMQP from another process, not necessarily Python.""" qid = str(randrange(0,999)) data = {'qid': qid, 'q': 'what time is it?'} broker = ExampleBroker() publisher = Publisher( connection = broker.amqp_connection, exchange = broker.exchange_name, exchange_type = "topic", routing_key = broker.binding_key, ) # declare test queue for receiving response message backend = broker.amqp_connection.create_backend() backend.queue_declare( queue="test", durable=False, exclusive=False, auto_delete=True,) backend.queue_bind( "test", broker.exchange_name, broker.response_routing_key % qid) consumer = Consumer( connection=broker.amqp_connection, exchange=broker.exchange_name, exchange_type="topic", queue="test", ) consumer.discard_all() # send message publisher.send(json.dumps(data)) # allow data to pass the wire sleep(0.2) # retrieve response response = consumer.fetch() payload = json.loads(response.payload) print "Response from AMQP: %s" % payload
class MessageQueuePipeline(object): def __init__(self, host_name, port, userid, password, virtual_host, encoder_class): self.q_connection = BrokerConnection(hostname=host_name, port=port, userid=userid, password=password, virtual_host=virtual_host) self.encoder = encoder_class() @classmethod def from_settings(cls, settings): host_name = settings.get('BROKER_HOST', 'localhost') port = settings.get('BROKER_PORT', 5672) userid = settings.get('BROKER_USERID', "guest") password = settings.get('BROKER_PASSWORD', "guest") virtual_host = settings.get('BROKER_VIRTUAL_HOST', "/") encoder_class = settings.get('MESSAGE_Q_SERIALIZER', ScrapyJSONEncoder) return cls(host_name, port, userid, password, virtual_host, encoder_class) @classmethod def from_crawler(cls, crawler): o = cls(crawler) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o def spider_opened(self, spider): self.publisher = Publisher(connection=self.q_connection, exchange="", routing_key=spider.name) def spider_closed(self, spider): self.publisher.close() def process_item(self, item, spider): return deferToThread(self._process_item, item, spider) def _process_item(self, item, spider): self.publisher.send({"scraped_data": self.encoder.encode(dict(item))}) return item
def send_requests(requests, **options): logger = logging if 'logger' in options.keys(): logger = options['logger'] """Send a import request message to be picked up by workers.""" connection = DjangoBrokerConnection() publisher = Publisher(connection=connection, exchange="collector", exchange_type='topic', routing_key="collector.driver", serializer='json') for req in requests: routing_key = req['driver_routing_key'] publisher.send(req, routing_key=routing_key) logger.debug("Sent request with routing_key %s:%s" % ( routing_key, req, )) publisher.close() connection.close()
class AmqpMessageProcessor(object): purge_every = datetime.timedelta(minutes=10) conn_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2)) def __init__(self, connection): self.pygo_mp = PyGoWaveClientMessageProcessor() self.consumer = Consumer( connection, queue="wavelet_rpc_singlethread", exchange="wavelet.topic", routing_key="#.#.clientop", exchange_type="topic", serializer="json", auto_ack=True, ) self.consumer.register_callback(self.receive) self.publisher = Publisher( connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json" ) self.pygo_mp.purge_connections() self.next_purge = datetime.datetime.now() + self.purge_every def wait(self, limit=None): self.consumer.wait(limit) def send(self, routing_key, message_data): self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1) def receive(self, message_data, message): routing_key = message.amqp_message.routing_key msg_dict = self.pygo_mp.process(routing_key, message_data) for out_rkey, messages in msg_dict.iteritems(): self.send(out_rkey, messages) # Cleanup time? if datetime.datetime.now() > self.next_purge: self.pygo_mp.purge_connections() self.next_purge = datetime.datetime.now() + self.purge_every
def request_with_response(): """Example of requesting over AMQP from another process, not necessarily Python.""" qid = str(randrange(0, 999)) data = {'qid': qid, 'q': 'what time is it?'} broker = ExampleBroker() publisher = Publisher( connection=broker.amqp_connection, exchange=broker.exchange_name, exchange_type="topic", routing_key=broker.binding_key, ) # declare test queue for receiving response message backend = broker.amqp_connection.create_backend() backend.queue_declare( queue="test", durable=False, exclusive=False, auto_delete=True, ) backend.queue_bind("test", broker.exchange_name, broker.response_routing_key % qid) consumer = Consumer( connection=broker.amqp_connection, exchange=broker.exchange_name, exchange_type="topic", queue="test", ) consumer.discard_all() # send message publisher.send(json.dumps(data)) # allow data to pass the wire sleep(0.2) # retrieve response response = consumer.fetch() payload = json.loads(response.payload) print "Response from AMQP: %s" % payload
def _dispatch_message(self, message): if not message.claim(self.context.db): return try: publisher = Publisher(self.context.broker, exchange=message.options.exchange, exchange_type=message.options.exchange_type) publisher.send(message.message, routing_key = message.options.routing_key, delivery_mode = message.options.delivery_mode, mandatory = message.options.mandatory, priority = message.options.priority) publisher.close() except: log.error("Error dispatching deferred message %s: %s" % (message, traceback.format_exc())) self.error_reschedule(message) return False else: log.debug("Dispatched message %s" % message) # sent with no problems, done with it. self.context.db.delete(message) return True
def main(): logging.basicConfig(level=logging.DEBUG) args = parser.parse_args() conn = BrokerConnection( hostname=args.hostname, virtual_host=args.vhost, userid=args.user, password=args.password, ) publisher = Publisher( auto_declare = False, connection = conn, exchange = args.exchange, routing_key = args.key ) logging.info("Declaring exchange: %s" % args.exchange) publisher.backend.exchange_declare(exchange=args.exchange, type="topic", durable=False, auto_delete=False) while True: line = sys.stdin.readline() if not line: break logging.debug("Sending message '%s'" % line.strip()) publisher.send(line.strip()) publisher.close()
class GenericPublisher(object): def __init__(self, config, exchange=None, connect=True): self.config = config self.exchange = exchange self.connection = None if connect: self.connect() # Connect to the message broker def connect(self): if not self.connection: self.connection = BrokerConnection(hostname=self.config.host, port=self.config.port, userid=self.config.user, password=self.config.password, virtual_host=self.config.vhost) # Disconnect from the message broker def disconnect(self): if self.connection: self.connection.close() self.connection = None # Used to publish a pulse message to the proper exchange def publish(self, message): # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a message given if not message: raise MalformedMessage(message) # Have the message prepare and validate itself message._prepare() # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker publisher self.publisher = Publisher(connection=self.connection, exchange=self.exchange, exchange_type="topic", routing_key=message.routing_key) # The message is actually a simple envelope format with a payload and # some metadata final_data = {} final_data['payload'] = message.data final_data['_meta'] = message.metadata.copy() final_data['_meta'].update({ 'exchange': self.exchange, 'routing_key': message.routing_key, 'serializer': self.config.serializer, 'sent': time_to_string(datetime.now(timezone(self.config.broker_timezone))) }) # Send the message self.publisher.send(final_data, serializer=self.config.serializer) # Close the publishing connection self.publisher.close()
class PyGoWaveMessageProcessor(object): """ Handle all incoming messages. Routing key structure: <participant_conn_guid>.<wavelet_id>.[waveop|clientop] participant_conn_guid is created every time the user enters the wave viewer. Multiple instances of the wave viewer can be opened per user at the same time. A participant can "listen" to messages from multiple wavlets, by subscribing to multiple routing keys. In this scenario, a message may be sent to a queue, that the participant has not subscribed to. This is intended and it's RabbitMQ's job to drop the message. This keeps things simple for now and may be changed in the future. Message comsumers can be optimized in a multi-threading environment. If one consumer starts handling messages of a particular wavelet, it should block others from handling them. Some messages are handled synchronously (i.e. the client does not perform any actions and waits for the server's response). Those are in particular: WAVELET_ADD_PARTICIPANT """ purge_every = datetime.timedelta(minutes=10) conn_min_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2)) def __init__(self, connection): self.consumer = Consumer( connection, queue="wavelet_rpc_singlethread", exchange="wavelet.topic", routing_key="#.#.clientop", exchange_type="topic", serializer="json", auto_ack=True, ) self.consumer.register_callback(self.receive) self.publisher = Publisher( connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json", ) self.out_queue = {} self.purge_connections() def broadcast(self, wavelet, type, property, except_connections=[]): """ Send messages to all participants. `wavelet` must be a Wavelet object. `except_connections` is a list of ParticipantConn objects to be excluded from the broadcast. """ msg_dict = { "type": type, "property": property } logger.debug("Broadcasting Message:\n" + repr(msg_dict)) for p in wavelet.participants.all(): for conn in p.connections.all(): if not conn in except_connections: if self.out_queue.has_key(conn.rx_key): self.out_queue[conn.rx_key].append(msg_dict) else: self.out_queue[conn.rx_key] = [msg_dict] def emit(self, to, type, property, except_connections=[]): """ Collect messages to be sent. `to` must be a ParticipantConn object. """ msg_dict = { "type": type, "property": property } logger.debug("Emiting Message to %s/%d:\n%s" % (to.participant.name, to.id, repr(msg_dict))) if self.out_queue.has_key(to.rx_key): self.out_queue[to.rx_key].append(msg_dict) else: self.out_queue[to.rx_key] = [msg_dict] def wait(self, limit=None): self.consumer.wait(limit) def send(self, message_data, routing_key): self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1) def receive(self, message_data, message): rkey = message.amqp_message.routing_key participant_conn_key, wavelet_id, message_category = rkey.split(".") if message_category != "clientop": return logger.debug("Received Message from %s.%s.%s:\n%s" % (participant_conn_key, wavelet_id, message_category, repr(message_data))) # Get participant connection try: pconn = ParticipantConn.objects.get(tx_key=participant_conn_key) except ObjectDoesNotExist: logger.error("{%s} ParticipantConn not found" % (rkey)) return # Fail silently # Get wavelet try: wavelet = pconn.participant.wavelets.get(id=wavelet_id) except ObjectDoesNotExist: logger.error("{%s} Wavelet not found (or not participating)" % (rkey)) return # Fail silently # Handle message and reply to sender and/or broadcast an event self.out_queue = {} if isinstance(message_data, list): # multi-message? for sub_message in message_data: if not self.handle_participant_message(wavelet, pconn, sub_message): break else: self.handle_participant_message(wavelet, pconn, message_data) for receiver, messages in self.out_queue.iteritems(): self.send(messages, "%s.%s.waveop" % (receiver, wavelet_id)) self.out_queue = {} # Cleanup time? if datetime.datetime.now() > self.next_purge: self.purge_connections() def handle_participant_message(self, wavelet, pconn, message): """ Handle a participant's operation. If True is returned, go on with processing the next message. If False is returned, discard any following messages. """ participant = pconn.participant if message.has_key(u"type"): if message["type"] == "WAVELET_OPEN": logger.info("[%s/%d@%s] Opening wavelet" % (participant.name, pconn.id, wavelet.wave.id)) pconn.wavelets.add(wavelet) # I know this is neat :) self.emit(pconn, "WAVELET_OPEN", { "wavelet": wavelet.serialize(), "blips": wavelet.serialize_blips(), }) elif message["type"] == "PARTICIPANT_INFO": logger.info("[%s/%d@%s] Sending participant information" % (participant.name, pconn.id, wavelet.wave.id)) p_info = {} for p_id in message["property"]: try: p_info[p_id] = Participant.objects.get(id=p_id).serialize() except ObjectDoesNotExist: p_info[p_id] = None self.emit(pconn, "PARTICIPANT_INFO", p_info) elif message["type"] == "PARTICIPANT_SEARCH": if len(message["property"]) < getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0): self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "TOO_SHORT", "data": getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0)}) logger.debug("[%s/%d@%s] Participant search query too short" % (participant.name, pconn.id, wavelet.wave.id)) else: logger.info("[%s/%d@%s] Performing participant search" % (participant.name, pconn.id, wavelet.wave.id)) lst = [] for p in Participant.objects.filter(name__icontains=message["property"]).exclude(id=participant.id): lst.append(p.id) self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "OK", "data": lst}) elif message["type"] == "GADGET_LIST": all_gadgets = map(lambda g: {"id": g.id, "uploaded_by": g.by_user.participants.all()[0].name, "name": g.title, "descr": g.description, "url": g.url}, Gadget.objects.all()) logger.info("[%s/%d@%s] Sending Gadget list" % (participant.name, pconn.id, wavelet.wave.id)) self.emit(pconn, "GADGET_LIST", all_gadgets) elif message["type"] == "WAVELET_ADD_PARTICIPANT": # Find participant try: p = Participant.objects.get(id=message["property"]) except ObjectDoesNotExist: logger.error("[%s/%d@%s] Target participant '%s' not found" % (participant.name, pconn.id, wavelet.wave.id, message["property"])) return # Fail silently (TODO: report error to user) # Check if already participating if wavelet.participants.filter(id=message["property"]).count() > 0: logger.error("[%s/%d@%s] Target participant '%s' already there" % (participant.name, pconn.id, wavelet.wave.id, message["property"])) return # Fail silently (TODO: report error to user) wavelet.participants.add(p) logger.info("[%s/%d@%s] Added new participant '%s'" % (participant.name, pconn.id, wavelet.wave.id, message["property"])) self.broadcast(wavelet, "WAVELET_ADD_PARTICIPANT", message["property"]) elif message["type"] == "WAVELET_REMOVE_SELF": self.broadcast(wavelet, "WAVELET_REMOVE_PARTICIPANT", participant.id) wavelet.participants.remove(participant) # Bye bye pconn.wavelets.remove(wavelet) # Also for your connection logger.info("[%s/%d@%s] Participant removed himself" % (participant.name, pconn.id, wavelet.wave.id)) if wavelet.participants.count() == 0: # Oh my god, you killed the Wave! You bastard! logger.info("[%s/%d@%s] Wave got killed!" % (participant.name, pconn.id, wavelet.wave.id)) wavelet.wave.delete() return False elif message["type"] == "OPERATION_MESSAGE_BUNDLE": # Build OpManager newdelta = OpManager(wavelet.wave.id, wavelet.id) newdelta.unserialize(message["property"]["operations"]) version = message["property"]["version"] # Transform for delta in wavelet.deltas.filter(version__gt=version): for op in delta.getOpManager().operations: newdelta.transform(op) # Trash results (an existing delta cannot be changed) # Apply wavelet.applyOperations(newdelta.operations) # Raise version and store wavelet.version += 1 wavelet.save() Delta.createByOpManager(newdelta, wavelet.version).save() # Create tentative checksums blipsums = wavelet.blipsums() # Respond self.emit(pconn, "OPERATION_MESSAGE_BUNDLE_ACK", {"version": wavelet.version, "blipsums": blipsums}) self.broadcast(wavelet, "OPERATION_MESSAGE_BUNDLE", {"version": wavelet.version, "operations": newdelta.serialize(), "blipsums": blipsums}, [pconn]) logger.debug("[%s/%d@%s] Processed delta #%d -> v%d" % (participant.name, pconn.id, wavelet.wave.id, version, wavelet.version)) else: logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message)) else: logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message)) return True def purge_connections(self): """ This method will check all connections to the server and throw out all closed ones. This can be run asynchronously every 10 minutes or so. """ for conn in ParticipantConn.objects.all(): for wavelet in conn.wavelets.all(): if not self.queue_exists("%s.%s.waveop" % (conn.rx_key, wavelet.id)): wavelet.participant_conns.remove(conn) logger.info("[%s/%d@%s] Connection to wavelet closed" % (conn.participant.name, conn.id, wavelet.wave.id)) if conn.wavelets.count() == 0 and datetime.datetime.now() > conn.created + self.conn_min_lifetime: conn_id, conn_participant_name = conn.id, conn.participant.name conn.delete() logger.info("[%s/%d] Connection to server closed" % (conn_participant_name, conn_id)) self.next_purge = datetime.datetime.now() + self.purge_every def queue_exists(self, queue): """ Check if a queue exists, i.e. a user is connected to it (because auto-delete is always turned on). """ logger.debug("Checking queue %s" % (queue)) qex = self.publisher.backend.queue_exists(queue) # Re-open the channel if it was closed (this is a pyamqlib issue) if self.publisher.backend.channel.connection == None: self.publisher.backend.channel = self.publisher.backend.connection.connection.channel() return qex
from carrot.connection import BrokerConnection from carrot.messaging import Publisher import sys conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/") publisher = Publisher(connection=conn, exchange="sorting_room", routing_key="jason") publisher.send({"po_box": sys.argv[1]}) publisher.close()
def send(exchange, routing_key, msg): publisher = Publisher(connection=conn, exchange=exchange, routing_key=routing_key) publisher.send(msg) publisher.close()
def test_consumer(self): """AMQP->RPC->AMQP Send a AMQP message, and test RPCConsumer response. Uses a mock RPC server that the consumer will call. Check the consumer AMQP response with test consumer. """ class TestConsumer(IMessageBroker): service_name = 'TestConsumer' exchange_name = 'Test' topic = 'test' amqp_connection_settings = settings.AMQP_CONNECTION def dispatch(self, message, request, qid): """ AMQP -> RPC dispatcher. """ logging.info('dispatching AMQP -> RPC') response = {} if qid: # call remote RPC # this message has an qid so it expects a response response['msg'] = test_rpc_service.push(request) message.ack() # return the response to AMQP, the caller should be listening .. self.return_response(response,qid) else: # no qid, so do something stateless .. print request message.ack() try: consumer = TestConsumer() self.assertEquals('test.request.*',consumer.binding_key) self.assertEquals('test.response.%s',consumer.response_routing_key) consumer.start() test_rpc_service.start() test_rpc_service.messages = [] # allow consumer to start sleep(0.2) self.assert_(not consumer.stopped) self.assert_(consumer.isAlive()) # test variables qid = str(randrange(0,999999)) jsondata = {'msg':'hello rpc'} _publisher = Publisher( connection=self.test_connection, exchange='Test', exchange_type="topic", routing_key='test.request.'+qid, ) # test channel backend = self.test_connection.create_backend() backend.queue_declare( queue="test", durable=False, exclusive=False, auto_delete=True,) backend.queue_bind("test",'Test','test.response.'+qid) _consumer = Consumer( connection=self.test_connection, exchange='Test', exchange_type="topic", queue="test", ) _consumer.discard_all() logging.debug('publishing JSON message to RPC') data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid}) _publisher.send(data_on_the_wire) # allow data to pass the wire sleep(0.2) # retrieve dispatcher response response = _consumer.fetch() self.assert_(response, 'No response') data = json.loads(response.payload) self.assert_(len(data['msg']) > 0) # assert non-empty response self.assertEquals('ok',data['msg']) # check dispatcher RPC function self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive') self.assertEquals(test_rpc_service.messages[0], jsondata) finally: try: consumer.stop() except: pass try: _consumer.close() except: pass test_rpc_service.stop() self.assert_(consumer.stopped)
from carrot.connection import BrokerConnection from carrot.messaging import Publisher conn = BrokerConnection(hostname="localhost", port=61617, userid="admin", password="******", virtual_host="/") publisher = Publisher(connection=conn, exchange="feed", routing_key="importer") for i in range(30): publisher.send({"name":"foo", "i":i}) publisher.close()
userid="guest", password="******", virtual_host="/") publisher = Publisher(connection=conn, exchange="feed", routing_key="importer") num = 10000 test_name = "concurrency_test_%d" % db.incr("concurrency_num_tests") print "Running test %s at %d messages" % (test_name, num) start_time = time.time() for i in xrange(num): publisher.send({ "download_url": "http://cnn.com/rss/edition.rss", "rate_id": "243234-234234234-234234234234-234234234234", "search_id": "234234234-234234234234-234234234-234234234", "test_name": test_name, "vendor": "hertz"}) publisher.close() print "Done publishing in %f seconds" % (time.time() - start_time) # Wait until all the results have been published amount = 0 while amount != num: amount = int(db.get(test_name) or 0) print "Done in %f seconds" % (time.time() - start_time)
class Worker(object): def __init__(self, hostname='localhost', port=5672, userid="collectoruser", password="******", **options): conn = BrokerConnection(hostname=hostname, port=port, userid=userid, password=password, virtual_host='collectorvhost') try: self.publisher = Publisher(connection=conn, exchange="collector.response", exchange_type='direct', routing_key="response", serializer="json") except Exception as ex: raise Worker.ConnectionException(ex) self.consumer = Consumer(connection=conn, queue="feed", exchange_type='topic', exchange="collector", routing_key="collector.*.*") def run(self): self.consumer.register_callback(self._get_request_callback) print "Waiting..." self.consumer.wait() def _get_request_callback(self, message_data, message): #print "message_data=%s" % message_data drivers = [] drivers.append(message_data) DRIVER_KEY = "driver_routing_key" TAGS_KEY = "tags" for obj in drivers: driver = None if DRIVER_KEY in obj and TAGS_KEY in obj: module_name = obj["driver_routing_key"] try: __import__(module_name) module = sys.modules[module_name] driver = module.Driver() except: print "could not import module %s" % module_name values = [] for tag in obj['tags']: if driver: value = driver.read_value(tag['device'], tag['address']) else: value = "BAD_DRIVER" values.append({ 'name': tag['name'], 'current_value': value, 'read_at': '2009-01-01' }) self.publisher.send(values) else: print "Badly formated request %s" % obj message.ack() class WorkerException(Exception): pass class ConnectionException(WorkerException): pass
port = 1000 + easyip.EASYIP_PORT buf = 1024 addr = (host, port) # Create socket and bind to address UDPSock = socket(AF_INET, SOCK_DGRAM) UDPSock.bind(addr) print "waiting" # Receive messages while 1: data, addr = UDPSock.recvfrom(buf) if not data: print "Client has exited!" break else: packet = easyip.Packet(data) print "\nReceived message '", packet, "' with %s" % (packet.payload, ) response = easyip.Factory.response(packet) UDPSock.sendto(response.pack(), addr) print "Responded to %s, %s" % addr print "Putting incomming on queue" publisher.send({'address': addr, 'payload': packet.payload}) print "Closing" # Close socket UDPSock.close() publisher.close()
def reply(data, exchange, routing_key, connection=None, connect_timeout=None, **kwargs): pub = Publisher(connection, exchange=exchange, routing_key=routing_key, **kwargs) pub.send(data)
class SignalAMQP(Signal): def __init__(self, exchange, queue, routing_key=None, debug=None): """ Initialise signal, conection, exchange, queue and run consumer @return: Django Signal like object. @param exchange: name of exchange for this signal @type exchange: string @param queue: name queue of this signal @type queue: string @param routing_key: name of routing_key betwen exchange and queue of this signal @type routing_key: string @param debug: debug flag @type debug: bool Example: >>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1") >>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2") >>> amqp_signal_1.queue_bind(['q2']) >>> def amqp_handler(sender, **kwargs): print "AMPQ handler:", sender, kwargs >>> amqp_signal_2.connect(amqp_handler, sender=None) >>> amqp_signal_1.send("Hello world!") """ super(SignalAMQP, self).__init__(providing_args=["message"]) self.exchange = exchange self.queue = queue self.routing_key = routing_key self.debug = debug is None and settings.DEBUG or debug self.conn = DjangoBrokerConnection() self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\ routing_key=self.routing_key) self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\ exchange=self.exchange, routing_key=self.routing_key) self.consumer.register_callback(self.callback) self.cl = self.listen() def send(self, message, **kw): """ Transfer message to bus. Message can be any simple python type. """ self.publisher.send(message, **kw) def callback(self, message_data, message): """ Consumer callback function. Send Django singnal.""" try: sender = message_data['sender'] except: sender = None super(SignalAMQP, self).send(sender=sender, message_data=message_data, message=message) message.ack() if self.debug: print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message return True def get_backend(self): return self.conn.get_backend_cls()(DjangoBrokerConnection()) def queue_bind(self, queue_list): """ Bind another queues to current exchange """ routing_keys = [] backend = self.get_backend() for x in queue_list: backend.queue_bind(queue=x, exchange=self.exchange, \ routing_key='%s_%s' % (self.exchange, x)) routing_keys.append('%s_%s' % (self.exchange, x)) return routing_keys def listen(self): """ Run consumer loop thread """ cl = ConsumerLoop() cl.consumer = self.consumer cl.start() return cl def stop(self): """ Unactivate this signal """ self.conn.close()