Beispiel #1
0
def reply(routing_key, data):
    from carrot.connection import DjangoBrokerConnection
    from carrot.messaging import Publisher

    conn = DjangoBrokerConnection()
    publisher = Publisher(connection=conn, exchange="django", routing_key=routing_key, exchange_type="topic")
    publisher.send(data)
    publisher.close()
    conn.close()
Beispiel #2
0
def receive_a_message():
    logger = get_logger()
    conn = DjangoBrokerConnection()
    m = MyMessager(connection=conn).fetch()
    if m:
        msg = simplejson.loads(m.body)
        logger.info("Message receieved: %s" % msg.get("message"))
        m.ack()
    conn.close()
Beispiel #3
0
 def setUp(self):
     conn = DjangoBrokerConnection()
     consumer = StatsConsumer(connection=conn)
     consumer.discard_all()
     conn.close()
     consumer.close()
     self.s = StatsCollector()
     self.assertEquals(self.s.total_tasks_processed, 0)
     self.assertEquals(self.s.total_tasks_processed_by_type, {})
     self.assertEquals(self.s.total_task_time_running, 0.0)
     self.assertEquals(self.s.total_task_time_running_by_type, {})
Beispiel #4
0
 def sender_callback(self, routing_key, data):
     conn = DjangoBrokerConnection()
     publisher = Publisher(connection=conn,
                           exchange="django_send",
                           routing_key=routing_key,
                           exchange_type="topic",
                           )
     publisher.send(data)
     publisher.close()
     conn.close()
     print "Sent object change/delete message for %s" % routing_key
Beispiel #5
0
 def test_amqp(self):
     from carrot.connection import DjangoBrokerConnection
     from carrot.messaging import Publisher, Consumer
     
     connection = DjangoBrokerConnection()
     publisher = Publisher(connection=connection,
                           exchange="collector",
                           exchange_type='topic',
                           routing_key="collector.driver",
                           serializer='json')
     publisher.send("test")
     publisher.close()
     connection.close()
Beispiel #6
0
    def test_amqp(self):
        from carrot.connection import DjangoBrokerConnection
        from carrot.messaging import Publisher, Consumer

        connection = DjangoBrokerConnection()
        publisher = Publisher(connection=connection,
                              exchange="collector",
                              exchange_type='topic',
                              routing_key="collector.driver",
                              serializer='json')
        publisher.send("test")
        publisher.close()
        connection.close()
Beispiel #7
0
    def publish(self, **data):
        """Publish statistics to be collected later by
        :class:`StatsCollector`.

        :param data: An arbitrary Python object containing the statistics
            to be published.

        """
        if not self.enabled:
            return
        connection = DjangoBrokerConnection()
        publisher = StatsPublisher(connection=connection)
        publisher.send({"type": self.type, "data": data})
        publisher.close()
        connection.close()
Beispiel #8
0
def even_time_distribution(task, size, time_window, iterable, **apply_kwargs):
    """With an iterator yielding task args, kwargs tuples, evenly distribute
    the processing of its tasks throughout the time window available.

    :param task: The kind of task (a :class:`celery.task.base.Task`.)
    :param size: Total number of elements the iterator gives.
    :param time_window: Total time available, in minutes.
    :param iterable: Iterable yielding task args, kwargs tuples.
    :param \*\*apply_kwargs: Additional keyword arguments to be passed on to
        :func:`celery.execute.apply_async`.

    Example

        >>> class RefreshAllFeeds(Task):
        ...
        ...     def run(self, **kwargs):
        ...         feeds = Feed.objects.all()
        ...         total = feeds.count()
        ...
        ...         time_window = REFRESH_FEEDS_EVERY_INTERVAL_MINUTES
        ...
        ...         def iter_feed_task_args(iterable):
        ...             for feed in iterable:
        ...                 yield ([feed.feed_url], {}) # args, kwargs tuple
        ...
        ...         it = iter_feed_task_args(feeds.iterator())
        ...
        ...         even_time_distribution(RefreshFeedTask, total,
        ...                                time_window, it)

    """

    bucketsize = size / time_window
    buckets = chunks(iterable, int(bucketsize))

    connection = DjangoBrokerConnection()
    try:
        for bucket_count, bucket in enumerate(buckets):
            # Skew the countdown for items in this bucket by one.
            seconds_eta = (60 * bucket_count if bucket_count else None)

            for args, kwargs in bucket:
                task.apply_async(args=args, kwargs=kwargs,
                                 connection=connection,
                                 countdown=seconds_eta,
                                 **apply_kwargs)
    finally:
        connection.close()
Beispiel #9
0
def discard_all(connect_timeout=AMQP_CONNECTION_TIMEOUT):
    """Discard all waiting tasks.

    This will ignore all tasks waiting for execution, and they will
    be deleted from the messaging server.

    :returns: the number of tasks discarded.

    :rtype: int

    """
    amqp_connection = DjangoBrokerConnection(connect_timeout=connect_timeout)
    consumer = TaskConsumer(connection=amqp_connection)
    discarded_count = consumer.discard_all()
    amqp_connection.close()
    return discarded_count
    def test_DjangoBrokerConnection(self):
        try:
            from django.conf import settings
        except ImportError:
            sys.stderr.write("Django is not installed. \
                Not testing django specific features.\n")
            return
        configured_or_configure(settings,
                                CARROT_BACKEND=CARROT_BACKEND,
                                BROKER_HOST=BROKER_HOST,
                                BROKER_PORT=BROKER_PORT,
                                BROKER_VHOST=BROKER_VHOST,
                                BROKER_USER=BROKER_USER,
                                BROKER_PASSWORD=BROKER_PASSWORD)

        expected_values = {
            "backend_cls": CARROT_BACKEND,
            "hostname": BROKER_HOST,
            "port": BROKER_PORT,
            "virtual_host": BROKER_VHOST,
            "userid": BROKER_USER,
            "password": BROKER_PASSWORD
        }

        conn = DjangoBrokerConnection()
        self.assertTrue(isinstance(conn, BrokerConnection))

        for val_name, val_value in expected_values.items():
            self.assertEquals(getattr(conn, val_name, None), val_value)
Beispiel #11
0
    def run(self, connect_timeout=conf.AMQP_CONNECTION_TIMEOUT):
        """Run all tasks in the taskset.

        :returns: A :class:`celery.result.TaskSetResult` instance.

        Example

            >>> ts = TaskSet(RefreshFeedTask, args=[
            ...         (["http://foo.com/rss"], {}),
            ...         (["http://bar.com/rss"], {}),
            ... ])
            >>> result = ts.run()
            >>> result.taskset_id
            "d2c9b261-8eff-4bfb-8459-1e1b72063514"
            >>> result.subtask_ids
            ["b4996460-d959-49c8-aeb9-39c530dcde25",
            "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
            >>> result.waiting()
            True
            >>> time.sleep(10)
            >>> result.ready()
            True
            >>> result.successful()
            True
            >>> result.failed()
            False
            >>> result.join()
            [True, True]

        """
        taskset_id = gen_unique_id()

        from celery.conf import ALWAYS_EAGER
        if ALWAYS_EAGER:
            subtasks = [apply(self.task, args, kwargs)
                            for args, kwargs in self.arguments]
            return TaskSetResult(taskset_id, subtasks)

        conn = DjangoBrokerConnection(connect_timeout=connect_timeout)
        publisher = TaskPublisher(connection=conn,
                                  exchange=self.task.exchange)
        subtasks = [apply_async(self.task, args, kwargs,
                                taskset_id=taskset_id, publisher=publisher)
                        for args, kwargs in self.arguments]
        publisher.close()
        conn.close()
        return TaskSetResult(taskset_id, subtasks)
Beispiel #12
0
def get_connection_info():
    broker_connection = DjangoBrokerConnection()
    carrot_backend = broker_connection.backend_cls
    if carrot_backend and not isinstance(carrot_backend, str):
        carrot_backend = carrot_backend.__name__
    port = broker_connection.port or \
                broker_connection.get_backend_cls().default_port
    port = port and ":%s" % port or ""
    vhost = broker_connection.virtual_host
    if not vhost.startswith("/"):
        vhost = "/" + vhost
    return "%(carrot_backend)s://%(userid)s@%(host)s%(port)s%(vhost)s" % {
                "carrot_backend": carrot_backend,
                "userid": broker_connection.userid,
                "host": broker_connection.hostname,
                "port": port,
                "vhost": vhost}
Beispiel #13
0
def send_requests(requests, **options):
    logger = logging
    if 'logger' in options.keys():
        logger = options['logger']
        
    """Send a import request message to be picked up by workers."""
    connection = DjangoBrokerConnection()
    publisher = Publisher(connection=connection,
                              exchange="collector",
                              exchange_type='topic',
                              routing_key="collector.driver",
                              serializer='json')
    for req in requests:
        routing_key=req['driver_routing_key']        
        publisher.send(req, routing_key=routing_key)
        logger.debug("Sent request with routing_key %s:%s" %( routing_key,req,  ))
    publisher.close()
    connection.close()
Beispiel #14
0
def send_requests(requests, **options):
    logger = logging
    if 'logger' in options.keys():
        logger = options['logger']
    """Send a import request message to be picked up by workers."""
    connection = DjangoBrokerConnection()
    publisher = Publisher(connection=connection,
                          exchange="collector",
                          exchange_type='topic',
                          routing_key="collector.driver",
                          serializer='json')
    for req in requests:
        routing_key = req['driver_routing_key']
        publisher.send(req, routing_key=routing_key)
        logger.debug("Sent request with routing_key %s:%s" % (
            routing_key,
            req,
        ))
    publisher.close()
    connection.close()
Beispiel #15
0
    def __init__(self, exchange, queue, routing_key=None, debug=None):
        """
			Initialise signal, conection, exchange, queue and run consumer 
			@return: Django Signal like object.

			@param exchange: name of exchange for this signal
			@type exchange: string

			@param queue: name queue of this signal
			@type queue: string

			@param routing_key: name of routing_key betwen exchange and queue of this signal
			@type routing_key: string

			@param debug: debug flag
			@type debug: bool

			Example:
			>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
			>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
			>>> amqp_signal_1.queue_bind(['q2'])
			>>> def amqp_handler(sender, **kwargs):
				print "AMPQ handler:", sender, kwargs
			>>> amqp_signal_2.connect(amqp_handler, sender=None)
			>>> amqp_signal_1.send("Hello world!")
		"""
        super(SignalAMQP, self).__init__(providing_args=["message"])

        self.exchange = exchange
        self.queue = queue
        self.routing_key = routing_key
        self.debug = debug is None and settings.DEBUG or debug

        self.conn = DjangoBrokerConnection()
        self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
             routing_key=self.routing_key)
        self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
           exchange=self.exchange, routing_key=self.routing_key)
        self.consumer.register_callback(self.callback)

        self.cl = self.listen()
Beispiel #16
0
def get_responses(logger=logging):
    connection = DjangoBrokerConnection()
    consumer = Consumer(connection=connection,
                        exchange="collector.response",
                        queue="responses",
                        routing_key="response")

    for message in consumer.iterqueue():
        responses = message.payload
        for resp in responses:
            logger.debug("resp=%s" % resp)
            try:
                tag = Tag.objects.get(name=resp['name'])
                tag.save_with_history(resp['current_value'])
            except Exception as ex:
                logger.error(ex)
            #print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
        message.ack()

    consumer.close()
    connection.close()
Beispiel #17
0
def get_responses(logger=logging):
    connection = DjangoBrokerConnection()
    consumer = Consumer(connection=connection,
                              exchange="collector.response",
                              queue="responses",
                              routing_key="response")
    
    for message in consumer.iterqueue():
        responses = message.payload
        for resp in responses:
            logger.debug("resp=%s" % resp )
            try:
                tag=Tag.objects.get(name=resp['name'])
                tag.save_with_history(resp['current_value'])
            except Exception as ex:
                logger.error(ex)
            #print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
        message.ack()
        
    consumer.close()
    connection.close()
Beispiel #18
0
	def __init__(self, exchange, queue, routing_key=None, debug=None):
		"""
			Initialise signal, conection, exchange, queue and run consumer 
			@return: Django Signal like object.

			@param exchange: name of exchange for this signal
			@type exchange: string

			@param queue: name queue of this signal
			@type queue: string

			@param routing_key: name of routing_key betwen exchange and queue of this signal
			@type routing_key: string

			@param debug: debug flag
			@type debug: bool

			Example:
			>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
			>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
			>>> amqp_signal_1.queue_bind(['q2'])
			>>> def amqp_handler(sender, **kwargs):
				print "AMPQ handler:", sender, kwargs
			>>> amqp_signal_2.connect(amqp_handler, sender=None)
			>>> amqp_signal_1.send("Hello world!")
		"""
		super(SignalAMQP, self).__init__(providing_args=["message"])

		self.exchange = exchange
		self.queue = queue
		self.routing_key = routing_key
		self.debug = debug is None and settings.DEBUG or debug

		self.conn = DjangoBrokerConnection()
		self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
							routing_key=self.routing_key)
		self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
					exchange=self.exchange, routing_key=self.routing_key)
		self.consumer.register_callback(self.callback)

		self.cl = self.listen()
Beispiel #19
0
class SignalAMQP(Signal):
	def __init__(self, exchange, queue, routing_key=None, debug=None):
		"""
			Initialise signal, conection, exchange, queue and run consumer 
			@return: Django Signal like object.

			@param exchange: name of exchange for this signal
			@type exchange: string

			@param queue: name queue of this signal
			@type queue: string

			@param routing_key: name of routing_key betwen exchange and queue of this signal
			@type routing_key: string

			@param debug: debug flag
			@type debug: bool

			Example:
			>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
			>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
			>>> amqp_signal_1.queue_bind(['q2'])
			>>> def amqp_handler(sender, **kwargs):
				print "AMPQ handler:", sender, kwargs
			>>> amqp_signal_2.connect(amqp_handler, sender=None)
			>>> amqp_signal_1.send("Hello world!")
		"""
		super(SignalAMQP, self).__init__(providing_args=["message"])

		self.exchange = exchange
		self.queue = queue
		self.routing_key = routing_key
		self.debug = debug is None and settings.DEBUG or debug

		self.conn = DjangoBrokerConnection()
		self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
							routing_key=self.routing_key)
		self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
					exchange=self.exchange, routing_key=self.routing_key)
		self.consumer.register_callback(self.callback)

		self.cl = self.listen()

	def send(self, message, **kw):
		""" Transfer message to bus. Message can be any simple python type. """
		self.publisher.send(message, **kw)

	def callback(self, message_data, message):
		""" Consumer callback function. Send Django singnal."""
		try:
			sender = message_data['sender'] 
		except:
			sender = None
		super(SignalAMQP, self).send(sender=sender, message_data=message_data, message=message)
		message.ack()
		if self.debug:
			print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message
		return True

	def get_backend(self):
		return self.conn.get_backend_cls()(DjangoBrokerConnection())

	def queue_bind(self, queue_list):
		""" Bind another queues to current exchange """
		routing_keys = []
		backend = self.get_backend()
		for x in queue_list:
			backend.queue_bind(queue=x, exchange=self.exchange, \
						routing_key='%s_%s' % (self.exchange, x))
			routing_keys.append('%s_%s' % (self.exchange, x))
		return routing_keys

	def listen(self):
		""" Run consumer loop thread """
		cl = ConsumerLoop()
		cl.consumer = self.consumer
		cl.start()
		return cl

	def stop(self):
		""" Unactivate this signal """
		self.conn.close()
Beispiel #20
0
def apply_async(
    task,
    args=None,
    kwargs=None,
    countdown=None,
    eta=None,
    routing_key=None,
    exchange=None,
    task_id=None,
    immediate=None,
    mandatory=None,
    priority=None,
    connection=None,
    connect_timeout=AMQP_CONNECTION_TIMEOUT,
    serializer=None,
    **opts
):
    """Run a task asynchronously by the celery daemon(s).

    :param task: The task to run (a callable object, or a :class:`Task`
        instance

    :param args: The positional arguments to pass on to the task (a ``list``).

    :param kwargs: The keyword arguments to pass on to the task (a ``dict``)

    :param countdown: Number of seconds into the future that the task should
        execute. Defaults to immediate delivery (Do not confuse that with
        the ``immediate`` setting, they are unrelated).

    :param eta: A :class:`datetime.datetime` object that describes the
        absolute time when the task should execute. May not be specified
        if ``countdown`` is also supplied. (Do not confuse this with the
        ``immediate`` setting, they are unrelated).

    :keyword routing_key: The routing key used to route the task to a worker
        server.

    :keyword exchange: The named exchange to send the task to. Defaults to
        :attr:`celery.task.base.Task.exchange`.

    :keyword immediate: Request immediate delivery. Will raise an exception
        if the task cannot be routed to a worker immediately.
        (Do not confuse this parameter with the ``countdown`` and ``eta``
        settings, as they are unrelated).

    :keyword mandatory: Mandatory routing. Raises an exception if there's
        no running workers able to take on this task.

    :keyword connection: Re-use existing AMQP connection.
        The ``connect_timeout`` argument is not respected if this is set.

    :keyword connect_timeout: The timeout in seconds, before we give up
        on establishing a connection to the AMQP server.

    :keyword priority: The task priority, a number between ``0`` and ``9``.

    :keyword serializer: A string identifying the default serialization
        method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting.
        Can be ``pickle`` ``json``, ``yaml``, or any custom serialization
        methods that have been registered with
        :mod:`carrot.serialization.registry`.

    """
    args = args or []
    kwargs = kwargs or {}
    routing_key = routing_key or getattr(task, "routing_key", None)
    exchange = exchange or getattr(task, "exchange", None)
    immediate = immediate or getattr(task, "immediate", None)
    mandatory = mandatory or getattr(task, "mandatory", None)
    priority = priority or getattr(task, "priority", None)
    serializer = serializer or getattr(task, "serializer", None)
    taskset_id = opts.get("taskset_id")
    publisher = opts.get("publisher")
    retries = opts.get("retries")
    if countdown:
        eta = datetime.now() + timedelta(seconds=countdown)

    from celery.conf import ALWAYS_EAGER

    if ALWAYS_EAGER:
        return apply(task, args, kwargs)

    need_to_close_connection = False
    if not publisher:
        if not connection:
            connection = DjangoBrokerConnection(connect_timeout=connect_timeout)
            need_to_close_connection = True
        publisher = TaskPublisher(connection=connection)

    delay_task = publisher.delay_task
    if taskset_id:
        delay_task = curry(publisher.delay_task_in_set, taskset_id)

    task_id = delay_task(
        task.name,
        args,
        kwargs,
        task_id=task_id,
        retries=retries,
        routing_key=routing_key,
        exchange=exchange,
        mandatory=mandatory,
        immediate=immediate,
        serializer=serializer,
        priority=priority,
        eta=eta,
    )

    if need_to_close_connection:
        publisher.close()
        connection.close()

    return AsyncResult(task_id)
Beispiel #21
0
 def get_backend(self):
     return self.conn.get_backend_cls()(DjangoBrokerConnection())
Beispiel #22
0
class SignalAMQP(Signal):
    def __init__(self, exchange, queue, routing_key=None, debug=None):
        """
			Initialise signal, conection, exchange, queue and run consumer 
			@return: Django Signal like object.

			@param exchange: name of exchange for this signal
			@type exchange: string

			@param queue: name queue of this signal
			@type queue: string

			@param routing_key: name of routing_key betwen exchange and queue of this signal
			@type routing_key: string

			@param debug: debug flag
			@type debug: bool

			Example:
			>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
			>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
			>>> amqp_signal_1.queue_bind(['q2'])
			>>> def amqp_handler(sender, **kwargs):
				print "AMPQ handler:", sender, kwargs
			>>> amqp_signal_2.connect(amqp_handler, sender=None)
			>>> amqp_signal_1.send("Hello world!")
		"""
        super(SignalAMQP, self).__init__(providing_args=["message"])

        self.exchange = exchange
        self.queue = queue
        self.routing_key = routing_key
        self.debug = debug is None and settings.DEBUG or debug

        self.conn = DjangoBrokerConnection()
        self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
             routing_key=self.routing_key)
        self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
           exchange=self.exchange, routing_key=self.routing_key)
        self.consumer.register_callback(self.callback)

        self.cl = self.listen()

    def send(self, message, **kw):
        """ Transfer message to bus. Message can be any simple python type. """
        self.publisher.send(message, **kw)

    def callback(self, message_data, message):
        """ Consumer callback function. Send Django singnal."""
        try:
            sender = message_data['sender']
        except:
            sender = None
        super(SignalAMQP, self).send(sender=sender,
                                     message_data=message_data,
                                     message=message)
        message.ack()
        if self.debug:
            print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message
        return True

    def get_backend(self):
        return self.conn.get_backend_cls()(DjangoBrokerConnection())

    def queue_bind(self, queue_list):
        """ Bind another queues to current exchange """
        routing_keys = []
        backend = self.get_backend()
        for x in queue_list:
            backend.queue_bind(queue=x, exchange=self.exchange, \
               routing_key='%s_%s' % (self.exchange, x))
            routing_keys.append('%s_%s' % (self.exchange, x))
        return routing_keys

    def listen(self):
        """ Run consumer loop thread """
        cl = ConsumerLoop()
        cl.consumer = self.consumer
        cl.start()
        return cl

    def stop(self):
        """ Unactivate this signal """
        self.conn.close()
def _get_carrot_object(klass, **kwargs):
    "Helper function to create Publisher and Consumer objects."
    return klass(connection=DjangoBrokerConnection(),
                 exchange=settings.EXCHANGE,
                 routing_key=settings.ROUTING_KEY,
                 **kwargs)
Beispiel #24
0
def send_a_message(msg):
    conn = DjangoBrokerConnection()
    MyMessager(connection=conn).send({"message": msg})
    conn.close()
Beispiel #25
0
def discard_all():
    conn = DjangoBrokerConnection()
    MyMessager(connection=conn).consumer.discard_all()
    conn.close()