def reply(routing_key, data): from carrot.connection import DjangoBrokerConnection from carrot.messaging import Publisher conn = DjangoBrokerConnection() publisher = Publisher(connection=conn, exchange="django", routing_key=routing_key, exchange_type="topic") publisher.send(data) publisher.close() conn.close()
def receive_a_message(): logger = get_logger() conn = DjangoBrokerConnection() m = MyMessager(connection=conn).fetch() if m: msg = simplejson.loads(m.body) logger.info("Message receieved: %s" % msg.get("message")) m.ack() conn.close()
def setUp(self): conn = DjangoBrokerConnection() consumer = StatsConsumer(connection=conn) consumer.discard_all() conn.close() consumer.close() self.s = StatsCollector() self.assertEquals(self.s.total_tasks_processed, 0) self.assertEquals(self.s.total_tasks_processed_by_type, {}) self.assertEquals(self.s.total_task_time_running, 0.0) self.assertEquals(self.s.total_task_time_running_by_type, {})
def sender_callback(self, routing_key, data): conn = DjangoBrokerConnection() publisher = Publisher(connection=conn, exchange="django_send", routing_key=routing_key, exchange_type="topic", ) publisher.send(data) publisher.close() conn.close() print "Sent object change/delete message for %s" % routing_key
def test_amqp(self): from carrot.connection import DjangoBrokerConnection from carrot.messaging import Publisher, Consumer connection = DjangoBrokerConnection() publisher = Publisher(connection=connection, exchange="collector", exchange_type='topic', routing_key="collector.driver", serializer='json') publisher.send("test") publisher.close() connection.close()
def publish(self, **data): """Publish statistics to be collected later by :class:`StatsCollector`. :param data: An arbitrary Python object containing the statistics to be published. """ if not self.enabled: return connection = DjangoBrokerConnection() publisher = StatsPublisher(connection=connection) publisher.send({"type": self.type, "data": data}) publisher.close() connection.close()
def even_time_distribution(task, size, time_window, iterable, **apply_kwargs): """With an iterator yielding task args, kwargs tuples, evenly distribute the processing of its tasks throughout the time window available. :param task: The kind of task (a :class:`celery.task.base.Task`.) :param size: Total number of elements the iterator gives. :param time_window: Total time available, in minutes. :param iterable: Iterable yielding task args, kwargs tuples. :param \*\*apply_kwargs: Additional keyword arguments to be passed on to :func:`celery.execute.apply_async`. Example >>> class RefreshAllFeeds(Task): ... ... def run(self, **kwargs): ... feeds = Feed.objects.all() ... total = feeds.count() ... ... time_window = REFRESH_FEEDS_EVERY_INTERVAL_MINUTES ... ... def iter_feed_task_args(iterable): ... for feed in iterable: ... yield ([feed.feed_url], {}) # args, kwargs tuple ... ... it = iter_feed_task_args(feeds.iterator()) ... ... even_time_distribution(RefreshFeedTask, total, ... time_window, it) """ bucketsize = size / time_window buckets = chunks(iterable, int(bucketsize)) connection = DjangoBrokerConnection() try: for bucket_count, bucket in enumerate(buckets): # Skew the countdown for items in this bucket by one. seconds_eta = (60 * bucket_count if bucket_count else None) for args, kwargs in bucket: task.apply_async(args=args, kwargs=kwargs, connection=connection, countdown=seconds_eta, **apply_kwargs) finally: connection.close()
def discard_all(connect_timeout=AMQP_CONNECTION_TIMEOUT): """Discard all waiting tasks. This will ignore all tasks waiting for execution, and they will be deleted from the messaging server. :returns: the number of tasks discarded. :rtype: int """ amqp_connection = DjangoBrokerConnection(connect_timeout=connect_timeout) consumer = TaskConsumer(connection=amqp_connection) discarded_count = consumer.discard_all() amqp_connection.close() return discarded_count
def run(self, connect_timeout=conf.AMQP_CONNECTION_TIMEOUT): """Run all tasks in the taskset. :returns: A :class:`celery.result.TaskSetResult` instance. Example >>> ts = TaskSet(RefreshFeedTask, args=[ ... (["http://foo.com/rss"], {}), ... (["http://bar.com/rss"], {}), ... ]) >>> result = ts.run() >>> result.taskset_id "d2c9b261-8eff-4bfb-8459-1e1b72063514" >>> result.subtask_ids ["b4996460-d959-49c8-aeb9-39c530dcde25", "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"] >>> result.waiting() True >>> time.sleep(10) >>> result.ready() True >>> result.successful() True >>> result.failed() False >>> result.join() [True, True] """ taskset_id = gen_unique_id() from celery.conf import ALWAYS_EAGER if ALWAYS_EAGER: subtasks = [apply(self.task, args, kwargs) for args, kwargs in self.arguments] return TaskSetResult(taskset_id, subtasks) conn = DjangoBrokerConnection(connect_timeout=connect_timeout) publisher = TaskPublisher(connection=conn, exchange=self.task.exchange) subtasks = [apply_async(self.task, args, kwargs, taskset_id=taskset_id, publisher=publisher) for args, kwargs in self.arguments] publisher.close() conn.close() return TaskSetResult(taskset_id, subtasks)
def send_requests(requests, **options): logger = logging if 'logger' in options.keys(): logger = options['logger'] """Send a import request message to be picked up by workers.""" connection = DjangoBrokerConnection() publisher = Publisher(connection=connection, exchange="collector", exchange_type='topic', routing_key="collector.driver", serializer='json') for req in requests: routing_key=req['driver_routing_key'] publisher.send(req, routing_key=routing_key) logger.debug("Sent request with routing_key %s:%s" %( routing_key,req, )) publisher.close() connection.close()
def send_requests(requests, **options): logger = logging if 'logger' in options.keys(): logger = options['logger'] """Send a import request message to be picked up by workers.""" connection = DjangoBrokerConnection() publisher = Publisher(connection=connection, exchange="collector", exchange_type='topic', routing_key="collector.driver", serializer='json') for req in requests: routing_key = req['driver_routing_key'] publisher.send(req, routing_key=routing_key) logger.debug("Sent request with routing_key %s:%s" % ( routing_key, req, )) publisher.close() connection.close()
def get_responses(logger=logging): connection = DjangoBrokerConnection() consumer = Consumer(connection=connection, exchange="collector.response", queue="responses", routing_key="response") for message in consumer.iterqueue(): responses = message.payload for resp in responses: logger.debug("resp=%s" % resp) try: tag = Tag.objects.get(name=resp['name']) tag.save_with_history(resp['current_value']) except Exception as ex: logger.error(ex) #print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,) message.ack() consumer.close() connection.close()
def get_responses(logger=logging): connection = DjangoBrokerConnection() consumer = Consumer(connection=connection, exchange="collector.response", queue="responses", routing_key="response") for message in consumer.iterqueue(): responses = message.payload for resp in responses: logger.debug("resp=%s" % resp ) try: tag=Tag.objects.get(name=resp['name']) tag.save_with_history(resp['current_value']) except Exception as ex: logger.error(ex) #print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,) message.ack() consumer.close() connection.close()
class SignalAMQP(Signal): def __init__(self, exchange, queue, routing_key=None, debug=None): """ Initialise signal, conection, exchange, queue and run consumer @return: Django Signal like object. @param exchange: name of exchange for this signal @type exchange: string @param queue: name queue of this signal @type queue: string @param routing_key: name of routing_key betwen exchange and queue of this signal @type routing_key: string @param debug: debug flag @type debug: bool Example: >>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1") >>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2") >>> amqp_signal_1.queue_bind(['q2']) >>> def amqp_handler(sender, **kwargs): print "AMPQ handler:", sender, kwargs >>> amqp_signal_2.connect(amqp_handler, sender=None) >>> amqp_signal_1.send("Hello world!") """ super(SignalAMQP, self).__init__(providing_args=["message"]) self.exchange = exchange self.queue = queue self.routing_key = routing_key self.debug = debug is None and settings.DEBUG or debug self.conn = DjangoBrokerConnection() self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\ routing_key=self.routing_key) self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\ exchange=self.exchange, routing_key=self.routing_key) self.consumer.register_callback(self.callback) self.cl = self.listen() def send(self, message, **kw): """ Transfer message to bus. Message can be any simple python type. """ self.publisher.send(message, **kw) def callback(self, message_data, message): """ Consumer callback function. Send Django singnal.""" try: sender = message_data['sender'] except: sender = None super(SignalAMQP, self).send(sender=sender, message_data=message_data, message=message) message.ack() if self.debug: print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message return True def get_backend(self): return self.conn.get_backend_cls()(DjangoBrokerConnection()) def queue_bind(self, queue_list): """ Bind another queues to current exchange """ routing_keys = [] backend = self.get_backend() for x in queue_list: backend.queue_bind(queue=x, exchange=self.exchange, \ routing_key='%s_%s' % (self.exchange, x)) routing_keys.append('%s_%s' % (self.exchange, x)) return routing_keys def listen(self): """ Run consumer loop thread """ cl = ConsumerLoop() cl.consumer = self.consumer cl.start() return cl def stop(self): """ Unactivate this signal """ self.conn.close()
def apply_async( task, args=None, kwargs=None, countdown=None, eta=None, routing_key=None, exchange=None, task_id=None, immediate=None, mandatory=None, priority=None, connection=None, connect_timeout=AMQP_CONNECTION_TIMEOUT, serializer=None, **opts ): """Run a task asynchronously by the celery daemon(s). :param task: The task to run (a callable object, or a :class:`Task` instance :param args: The positional arguments to pass on to the task (a ``list``). :param kwargs: The keyword arguments to pass on to the task (a ``dict``) :param countdown: Number of seconds into the future that the task should execute. Defaults to immediate delivery (Do not confuse that with the ``immediate`` setting, they are unrelated). :param eta: A :class:`datetime.datetime` object that describes the absolute time when the task should execute. May not be specified if ``countdown`` is also supplied. (Do not confuse this with the ``immediate`` setting, they are unrelated). :keyword routing_key: The routing key used to route the task to a worker server. :keyword exchange: The named exchange to send the task to. Defaults to :attr:`celery.task.base.Task.exchange`. :keyword immediate: Request immediate delivery. Will raise an exception if the task cannot be routed to a worker immediately. (Do not confuse this parameter with the ``countdown`` and ``eta`` settings, as they are unrelated). :keyword mandatory: Mandatory routing. Raises an exception if there's no running workers able to take on this task. :keyword connection: Re-use existing AMQP connection. The ``connect_timeout`` argument is not respected if this is set. :keyword connect_timeout: The timeout in seconds, before we give up on establishing a connection to the AMQP server. :keyword priority: The task priority, a number between ``0`` and ``9``. :keyword serializer: A string identifying the default serialization method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting. Can be ``pickle`` ``json``, ``yaml``, or any custom serialization methods that have been registered with :mod:`carrot.serialization.registry`. """ args = args or [] kwargs = kwargs or {} routing_key = routing_key or getattr(task, "routing_key", None) exchange = exchange or getattr(task, "exchange", None) immediate = immediate or getattr(task, "immediate", None) mandatory = mandatory or getattr(task, "mandatory", None) priority = priority or getattr(task, "priority", None) serializer = serializer or getattr(task, "serializer", None) taskset_id = opts.get("taskset_id") publisher = opts.get("publisher") retries = opts.get("retries") if countdown: eta = datetime.now() + timedelta(seconds=countdown) from celery.conf import ALWAYS_EAGER if ALWAYS_EAGER: return apply(task, args, kwargs) need_to_close_connection = False if not publisher: if not connection: connection = DjangoBrokerConnection(connect_timeout=connect_timeout) need_to_close_connection = True publisher = TaskPublisher(connection=connection) delay_task = publisher.delay_task if taskset_id: delay_task = curry(publisher.delay_task_in_set, taskset_id) task_id = delay_task( task.name, args, kwargs, task_id=task_id, retries=retries, routing_key=routing_key, exchange=exchange, mandatory=mandatory, immediate=immediate, serializer=serializer, priority=priority, eta=eta, ) if need_to_close_connection: publisher.close() connection.close() return AsyncResult(task_id)
def send_a_message(msg): conn = DjangoBrokerConnection() MyMessager(connection=conn).send({"message": msg}) conn.close()
def discard_all(): conn = DjangoBrokerConnection() MyMessager(connection=conn).consumer.discard_all() conn.close()