コード例 #1
0
ファイル: log.py プロジェクト: soad241/django-request-lite
def _send_message(routing_key, request):
    conn = DjangoAMQPConnection()
    publisher = Publisher(connection=conn, exchange='request', 
                          exchange_type='topic', routing_key=routing_key, 
                          serializer='pickle')
    publisher.send(request)
    publisher.close()
    conn.close()
コード例 #2
0
ファイル: testconn.py プロジェクト: idan/celery
def receive_a_message():
    logger = get_logger()
    conn = DjangoAMQPConnection()
    m = MyMessager(connection=conn).fetch()
    if m:
        msg = simplejson.loads(m.body)
        logger.info("Message receieved: %s" % msg.get("message"))
        m.ack()
    conn.close()
コード例 #3
0
ファイル: test_monitoring.py プロジェクト: idan/celery
 def setUp(self):
     conn = DjangoAMQPConnection()
     consumer = StatsConsumer(connection=conn)
     consumer.discard_all()
     conn.close()
     consumer.close()
     self.s = StatsCollector()
     self.assertEquals(self.s.total_tasks_processed, 0)
     self.assertEquals(self.s.total_tasks_processed_by_type, {})
     self.assertEquals(self.s.total_task_time_running, 0.0)
     self.assertEquals(self.s.total_task_time_running_by_type, {})
コード例 #4
0
ファイル: monitoring.py プロジェクト: idan/celery
    def publish(self, **data):
        """Publish statistics to be collected later by
        :class:`StatsCollector`.

        :param data: An arbitrary Python object containing the statistics
            to be published.

        """
        if not self.enabled:
            return
        connection = DjangoAMQPConnection()
        publisher = StatsPublisher(connection=connection)
        publisher.send({"type": self.type, "data": data})
        publisher.close()
        connection.close()
コード例 #5
0
ファイル: strategy.py プロジェクト: idan/celery
def even_time_distribution(task, size, time_window, iterable, **apply_kwargs):
    """With an iterator yielding task args, kwargs tuples, evenly distribute
    the processing of its tasks throughout the time window available.

    :param task: The kind of task (a :class:`celery.task.base.Task`.)
    :param size: Total number of elements the iterator gives.
    :param time_window: Total time available, in minutes.
    :param iterable: Iterable yielding task args, kwargs tuples.
    :param \*\*apply_kwargs: Additional keyword arguments to be passed on to
        :func:`celery.execute.apply_async`.

    Example

        >>> class RefreshAllFeeds(Task):
        ...
        ...     def run(self, **kwargs):
        ...         feeds = Feed.objects.all()
        ...         total = feeds.count()
        ...
        ...         time_window = REFRESH_FEEDS_EVERY_INTERVAL_MINUTES
        ...
        ...         def iter_feed_task_args(iterable):
        ...             for feed in iterable:
        ...                 yield ([feed.feed_url], {}) # args, kwargs tuple
        ...
        ...         it = iter_feed_task_args(feeds.iterator())
        ...
        ...         even_time_distribution(RefreshFeedTask, total,
        ...                                time_window, it)

    """

    bucketsize = size / time_window
    buckets = chunks(iterable, int(bucketsize))

    connection = DjangoAMQPConnection()
    try:
        for bucket_count, bucket in enumerate(buckets):
            # Skew the countdown for items in this bucket by one.
            seconds_eta = (60 * bucket_count if bucket_count else None)

            for args, kwargs in bucket:
                task.apply_async(args=args, kwargs=kwargs,
                                 connection=connection,
                                 countdown=seconds_eta,
                                 **apply_kwargs)
    finally:
        connection.close()
コード例 #6
0
ファイル: task.py プロジェクト: simonw/celery
def discard_all():
    """Discard all waiting tasks.

    This will ignore all tasks waiting for execution, and they will
    be deleted from the messaging server.

    :returns: the number of tasks discarded.

    :rtype: int

    """
    amqp_connection = DjangoAMQPConnection()
    consumer = TaskConsumer(connection=amqp_connection)
    discarded_count = consumer.discard_all()
    amqp_connection.close()
    return discarded_count
コード例 #7
0
ファイル: base.py プロジェクト: idan/celery
    def run(self, connect_timeout=AMQP_CONNECTION_TIMEOUT):
        """Run all tasks in the taskset.

        :returns: A :class:`celery.result.TaskSetResult` instance.

        Example

            >>> ts = TaskSet(RefreshFeedTask, [
            ...         ["http://foo.com/rss", {}],
            ...         ["http://bar.com/rss", {}],
            ... )
            >>> result = ts.run()
            >>> result.taskset_id
            "d2c9b261-8eff-4bfb-8459-1e1b72063514"
            >>> result.subtask_ids
            ["b4996460-d959-49c8-aeb9-39c530dcde25",
            "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
            >>> result.waiting()
            True
            >>> time.sleep(10)
            >>> result.ready()
            True
            >>> result.successful()
            True
            >>> result.failed()
            False
            >>> result.join()
            [True, True]

        """
        taskset_id = gen_unique_id()

        from celery.conf import ALWAYS_EAGER
        if ALWAYS_EAGER:
            subtasks = [apply(self.task, args, kwargs)
                            for args, kwargs in self.arguments]
            return TaskSetResult(taskset_id, subtasks)

        conn = DjangoAMQPConnection(connect_timeout=connect_timeout)
        publisher = TaskPublisher(connection=conn,
                                  exchange=self.task.exchange)
        subtasks = [apply_async(self.task, args, kwargs,
                                taskset_id=taskset_id, publisher=publisher)
                        for args, kwargs in self.arguments]
        publisher.close()
        conn.close()
        return TaskSetResult(taskset_id, subtasks)
コード例 #8
0
ファイル: task.py プロジェクト: simonw/celery
    def run(self):
        """Run all tasks in the taskset.

        :returns: A :class:`celery.result.TaskSetResult` instance.

        Example

            >>> ts = TaskSet(RefreshFeedTask, [
            ...         ["http://foo.com/rss", {}],
            ...         ["http://bar.com/rss", {}],
            ... )
            >>> result = ts.run()
            >>> result.taskset_id
            "d2c9b261-8eff-4bfb-8459-1e1b72063514"
            >>> result.subtask_ids
            ["b4996460-d959-49c8-aeb9-39c530dcde25",
            "598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
            >>> result.waiting()
            True
            >>> time.sleep(10)
            >>> result.ready()
            True
            >>> result.successful()
            True
            >>> result.failed()
            False
            >>> result.join()
            [True, True]

        """
        taskset_id = str(uuid.uuid4())
        conn = DjangoAMQPConnection()
        publisher = TaskPublisher(connection=conn)
        subtask_ids = [
            publisher.delay_task_in_set(
                task_name=self.task_name, taskset_id=taskset_id, task_args=arg, task_kwargs=kwarg
            )
            for arg, kwarg in self.arguments
        ]
        publisher.close()
        conn.close()
        return TaskSetResult(taskset_id, subtask_ids)
コード例 #9
0
ファイル: task.py プロジェクト: simonw/celery
def apply_async(
    task, args=None, kwargs=None, routing_key=None, immediate=None, mandatory=None, connect_timeout=None, priority=None
):
    """Run a task asynchronously by the celery daemon(s).

    :param task: The task to run (a callable object, or a :class:`Task`
        instance

    :param args: The positional arguments to pass on to the task (a ``list``).

    :param kwargs: The keyword arguments to pass on to the task (a ``dict``)


    :keyword routing_key: The routing key used to route the task to a worker
        server.

    :keyword immediate: Request immediate delivery. Will raise an exception
        if the task cannot be routed to a worker immediately.

    :keyword mandatory: Mandatory routing. Raises an exception if there's
        no running workers able to take on this task.

    :keyword connect_timeout: The timeout in seconds, before we give up
        on establishing a connection to the AMQP server.

    :keyword priority: The task priority, a number between ``0`` and ``9``.

    """
    if not args:
        args = []
    if not kwargs:
        kwargs = []
    message_opts = {"routing_key": routing_key, "immediate": immediate, "mandatory": mandatory, "priority": priority}
    for option_name, option_value in message_opts.items():
        message_opts[option_name] = getattr(task, option_name, option_value)

    conn = DjangoAMQPConnection(connect_timeout=connect_timeout)
    publisher = TaskPublisher(connection=conn)
    task_id = publisher.delay_task(task.name, args, kwargs, **message_opts)
    publisher.close()
    conn.close()
    return AsyncResult(task_id)
コード例 #10
0
ファイル: log.py プロジェクト: soad241/django-request-lite
def save_hits():
    conn = DjangoAMQPConnection()
    consumer = Consumer(connection=conn, queue='requestrequesthitqueue', 
                        exchange='request', 
                        routing_key='request.*', exchange_type='topic')
    count = 0
    messages = []
    for message in consumer.iterqueue():
        messages.append(message)
        request = message.decode()
        request.path = convert_unicode_to_string(request.path[:255])
        request.data = convert_unicode_to_string(request.data[:255])
        request.referer = convert_unicode_to_string(request.referer[:255])
        request.user_agent =\
               convert_unicode_to_string(request.user_agent[:255])
        request.language = convert_unicode_to_string(request.language[:255])
        request.save()
        count += 1
    logger.info("Saved {0} requests".format(count))
    [m.ack() for m in messages]
    logger.debug("Acknowledged all messages")
    consumer.close()
    conn.close()
コード例 #11
0
ファイル: execute.py プロジェクト: idan/celery
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None,
        routing_key=None, exchange=None,
        immediate=None, mandatory=None, priority=None, connection=None,
        connect_timeout=AMQP_CONNECTION_TIMEOUT, **opts):
    """Run a task asynchronously by the celery daemon(s).

    :param task: The task to run (a callable object, or a :class:`Task`
        instance

    :param args: The positional arguments to pass on to the task (a ``list``).

    :param kwargs: The keyword arguments to pass on to the task (a ``dict``)

    :param countdown: Number of seconds into the future that the task should
        execute. Defaults to immediate delivery (Do not confuse that with
        the ``immediate`` setting, they are unrelated).

    :param eta: A :class:`datetime.datetime` object that describes the
        absolute time when the task should execute. May not be specified
        if ``countdown`` is also supplied. (Do not confuse this with the
        ``immediate`` setting, they are unrelated).

    :keyword routing_key: The routing key used to route the task to a worker
        server.

    :keyword exchange: The named exchange to send the task to. Defaults to
        :attr:`celery.task.base.Task.exchange`.

    :keyword immediate: Request immediate delivery. Will raise an exception
        if the task cannot be routed to a worker immediately.
        (Do not confuse this parameter with the ``countdown`` and ``eta``
        settings, as they are unrelated).

    :keyword mandatory: Mandatory routing. Raises an exception if there's
        no running workers able to take on this task.

    :keyword connection: Re-use existing AMQP connection.
        The ``connect_timeout`` argument is not respected if this is set.

    :keyword connect_timeout: The timeout in seconds, before we give up
        on establishing a connection to the AMQP server.

    :keyword priority: The task priority, a number between ``0`` and ``9``.

    """
    args = args or []
    kwargs = kwargs or {}
    routing_key = routing_key or getattr(task, "routing_key", None)
    exchange = exchange or getattr(task, "exchange", None)
    immediate = immediate or getattr(task, "immediate", None)
    mandatory = mandatory or getattr(task, "mandatory", None)
    priority = priority or getattr(task, "priority", None)
    taskset_id = opts.get("taskset_id")
    publisher = opts.get("publisher")
    if countdown:
        eta = datetime.now() + timedelta(seconds=countdown)

    from celery.conf import ALWAYS_EAGER
    if ALWAYS_EAGER:
        return apply(task, args, kwargs)

    need_to_close_connection = False
    if not publisher:
        if not connection:
            connection = DjangoAMQPConnection(connect_timeout=connect_timeout)
            need_to_close_connection = True
        publisher = TaskPublisher(connection=connection)

    delay_task = publisher.delay_task
    if taskset_id:
        delay_task = curry(publisher.delay_task_in_set, taskset_id)

    task_id = delay_task(task.name, args, kwargs,
                         routing_key=routing_key, exchange=exchange,
                         mandatory=mandatory, immediate=immediate,
                         priority=priority, eta=eta)

    if need_to_close_connection:
        publisher.close()
        connection.close()

    return AsyncResult(task_id)
コード例 #12
0
ファイル: testconn.py プロジェクト: idan/celery
def discard_all():
    conn = DjangoAMQPConnection()
    MyMessager(connection=conn).consumer.discard_all()
    conn.close()
コード例 #13
0
ファイル: testconn.py プロジェクト: idan/celery
def send_a_message(msg):
    conn = DjangoAMQPConnection()
    MyMessager(connection=conn).send({"message": msg})
    conn.close()