Beispiel #1
0
    def flush(self):
        if self._state == TERMINATE:
            return
        # cancel all tasks that haven't been accepted so that NACK is sent.
        for job in values(self._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()

        self.maintain_pool()

        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                owned_by = {}
                for job in values(self._cache):
                    writer = _get_job_writer(job)
                    if writer is not None:
                        owned_by[writer] = job

                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if gen.__name__ == "_write_job" and gen_not_started(gen):
                            # hasn't started writing the job so can
                            # discard the task, but we must also remove
                            # it from the Pool._cache.
                            try:
                                job = owned_by[gen]
                            except KeyError:
                                pass
                            else:
                                # removes from Pool._cache
                                job.discard()
                            self._active_writers.discard(gen)
                        else:
                            try:
                                job = owned_by[gen]
                            except KeyError:
                                pass
                            else:
                                job_proc = job._write_to
                                if job_proc._is_alive():
                                    self._flush_writer(job_proc, gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
            self._active_writes.clear()
            self._busy_workers.clear()
Beispiel #2
0
    def test_compat_indices(self):
        assert not any(key.isupper() for key in DEFAULTS)
        assert not any(key.islower() for key in _OLD_DEFAULTS)
        assert not any(key.isupper() for key in _TO_OLD_KEY)
        assert not any(key.islower() for key in _TO_NEW_KEY)
        assert not any(key.isupper() for key in SETTING_KEYS)
        assert not any(key.islower() for key in _OLD_SETTING_KEYS)
        assert not any(value.isupper() for value in values(_TO_NEW_KEY))
        assert not any(value.islower() for value in values(_TO_OLD_KEY))

        for key in _TO_NEW_KEY:
            assert key in _OLD_SETTING_KEYS
        for key in _TO_OLD_KEY:
            assert key in SETTING_KEYS
Beispiel #3
0
    def test_compat_indices(self):
        self.assertFalse(any(key.isupper() for key in DEFAULTS))
        self.assertFalse(any(key.islower() for key in _OLD_DEFAULTS))
        self.assertFalse(any(key.isupper() for key in _TO_OLD_KEY))
        self.assertFalse(any(key.islower() for key in _TO_NEW_KEY))
        self.assertFalse(any(key.isupper() for key in SETTING_KEYS))
        self.assertFalse(any(key.islower() for key in _OLD_SETTING_KEYS))
        self.assertFalse(any(value.isupper() for value in values(_TO_NEW_KEY)))
        self.assertFalse(any(value.islower() for value in values(_TO_OLD_KEY)))

        for key in _TO_NEW_KEY:
            self.assertIn(key, _OLD_SETTING_KEYS)
        for key in _TO_OLD_KEY:
            self.assertIn(key, SETTING_KEYS)
Beispiel #4
0
 def filtertests(self, group, names):
     tests = self.groups[group]
     try:
         return ([tests[n] for n in names] if names
                 else list(values(tests)))
     except KeyError as exc:
         raise KeyError('Unknown test name: {0}'.format(exc))
Beispiel #5
0
 def periodic(self):
     for worker in values(self.state.workers):
         if not worker.alive:
             try:
                 self.on_node_lost(worker)
             finally:
                 self.state.workers.pop(worker.hostname, None)
Beispiel #6
0
    def run(self, force_ipython=False, force_bpython=False,
            force_python=False, without_tasks=False, eventlet=False,
            gevent=False, **kwargs):
        sys.path.insert(0, os.getcwd())
        if eventlet:
            import_module('celery.concurrency.eventlet')
        if gevent:
            import_module('celery.concurrency.gevent')
        import celery
        import celery.task.base
        self.app.loader.import_default_modules()
        self.locals = {'celery': self.app,
                       'Task': celery.Task,
                       'chord': celery.chord,
                       'group': celery.group,
                       'chain': celery.chain,
                       'chunks': celery.chunks,
                       'xmap': celery.xmap,
                       'xstarmap': celery.xstarmap,
                       'subtask': celery.subtask}

        if not without_tasks:
            self.locals.update(dict(
                (task.__name__, task) for task in values(self.app.tasks)
                if not task.name.startswith('celery.')),
            )

        if force_python:
            return self.invoke_fallback_shell()
        elif force_bpython:
            return self.invoke_bpython_shell()
        elif force_ipython:
            return self.invoke_ipython_shell()
        return self.invoke_default_shell()
Beispiel #7
0
 def find_app(self, app):
     try:
         sym = self.symbol_by_name(app)
     except AttributeError:
         # last part was not an attribute, but a module
         sym = import_from_cwd(app)
     if isinstance(sym, ModuleType):
         try:
             return sym.app
         except AttributeError:
             try:
                 return sym.celery
             except AttributeError:
                 if getattr(sym, '__path__', None):
                     try:
                         return self.find_app(
                             '{0}.celery:'.format(app.replace(':', '')),
                         )
                     except ImportError:
                         pass
                 for suspect in values(vars(sym)):
                     if isinstance(suspect, Celery):
                         return suspect
                 raise
     return sym
Beispiel #8
0
    def _get(self):
        # If the first bucket is always returning items, we would never
        # get to fetch items from the other buckets. So we always iterate over
        # all the buckets and put any ready items into a queue called
        # "immediate". This queue is always checked for cached items first.
        try:
            return 0, self._get_immediate()
        except Empty:
            pass

        remaining_times = []
        for bucket in values(self.buckets):
            remaining = bucket.expected_time()
            if not remaining:
                try:
                    # Just put any ready items into the immediate queue.
                    self.immediate.append(bucket.get_nowait())
                except Empty:
                    pass
                except RateLimitExceeded:
                    remaining_times.append(bucket.expected_time())
            else:
                remaining_times.append(remaining)

        # Try the immediate queue again.
        try:
            return 0, self._get_immediate()
        except Empty:
            if not remaining_times:
                # No items in any of the buckets.
                raise

            # There's items, but have to wait before we can retrieve them,
            # return the shortest remaining time.
            return min(remaining_times), None
Beispiel #9
0
    def flush(self):
        # cancel all tasks that have not been accepted so that NACK is sent.
        for job in values(self._pool._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()
        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._pool._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if gen.__name__ == "_write_job" and gen.gi_frame and gen.gi_frame.f_lasti != -1:
                            # has not started writing the job so can
                            # safely discard
                            self._active_writers.discard(gen)
                        else:
                            try:
                                next(gen)
                            except StopIteration:
                                self._active_writers.discard(gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
Beispiel #10
0
        def on_process_up(proc):
            """Called when a process has started."""
            # If we got the same fd as a previous process then we will also
            # receive jobs in the old buffer, so we need to reset the
            # job._write_to and job._scheduled_for attributes used to recover
            # message boundaries when processes exit.
            infd = proc.inqW_fd
            for job in values(cache):
                if job._write_to and job._write_to.inqW_fd == infd:
                    job._write_to = proc
                if job._scheduled_for and job._scheduled_for.inqW_fd == infd:
                    job._scheduled_for = proc
            fileno_to_outq[proc.outqR_fd] = proc
            # maintain_pool is called whenever a process exits.
            add_reader(
                proc.sentinel, event_process_exit, hub, proc.sentinel,
            )

            assert not isblocking(proc.outq._reader)

            # handle_result_event is called when the processes outqueue is
            # readable.
            add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd)

            waiting_to_start.add(proc)
            hub.call_later(
                self._proc_alive_timeout, verify_process_alive, proc,
            )
Beispiel #11
0
def find_app(app, symbol_by_name=symbol_by_name, imp=import_from_cwd):
    """Find app by name."""
    from .base import Celery

    try:
        sym = symbol_by_name(app, imp=imp)
    except AttributeError:
        # last part was not an attribute, but a module
        sym = imp(app)
    if isinstance(sym, ModuleType) and ":" not in app:
        try:
            found = sym.app
            if isinstance(found, ModuleType):
                raise AttributeError()
        except AttributeError:
            try:
                found = sym.celery
                if isinstance(found, ModuleType):
                    raise AttributeError()
            except AttributeError:
                if getattr(sym, "__path__", None):
                    try:
                        return find_app("{0}.celery".format(app), symbol_by_name=symbol_by_name, imp=imp)
                    except ImportError:
                        pass
                for suspect in values(vars(sym)):
                    if isinstance(suspect, Celery):
                        return suspect
                raise
            else:
                return found
        else:
            return found
    return sym
Beispiel #12
0
    def flush(self):
        if self._state == TERMINATE:
            return
        # cancel all tasks that have not been accepted so that NACK is sent.
        for job in values(self._cache):
            if not job._accepted:
                job._cancel()

        # clear the outgoing buffer as the tasks will be redelivered by
        # the broker anyway.
        if self.outbound_buffer:
            self.outbound_buffer.clear()
        try:
            # ...but we must continue writing the payloads we already started
            # to keep message boundaries.
            # The messages may be NACK'ed later if synack is enabled.
            if self._state == RUN:
                # flush outgoing buffers
                intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
                while self._active_writers:
                    writers = list(self._active_writers)
                    for gen in writers:
                        if (gen.__name__ == '_write_job' and
                                gen_not_started(gen)):
                            # has not started writing the job so can
                            # discard the task, but we must also remove
                            # it from the Pool._cache.
                            job_to_discard = None
                            for job in values(self._cache):
                                if job._writer() is gen:  # _writer is saferef
                                    # removes from Pool._cache
                                    job_to_discard = job
                                    break
                            if job_to_discard:
                                job_to_discard.discard()
                            self._active_writers.discard(gen)
                        else:
                            try:
                                next(gen)
                            except StopIteration:
                                self._active_writers.discard(gen)
                    # workers may have exited in the meantime.
                    self.maintain_pool()
                    sleep(next(intervals))  # don't busyloop
        finally:
            self.outbound_buffer.clear()
            self._active_writers.clear()
Beispiel #13
0
 def periodic(self):
     dirty = set()
     for worker in values(self.state.workers):
         if not worker.alive:
             dirty.add(worker)
             self.on_node_lost(worker)
     for worker in dirty:
         self.state.workers.pop(worker.hostname, None)
Beispiel #14
0
 def periodic(self):
     workers = self.state.workers
     dirty = set()
     for worker in values(workers):
         if not worker.alive:
             dirty.add(worker)
             self.on_node_lost(worker)
     for worker in dirty:
         workers.pop(worker.hostname, None)
Beispiel #15
0
    def tick(self):
        if not self.rdb.exists(self.key):
            logger.warn("key: {} not in rdb".format(self.key))
            for e in values(self.schedule):
                self.rdb.zadd(
                    self.key,
                    {jsonpickle.encode(e): self._when(e,
                                                      e.is_due()[1]) or 0})

        tasks = self.rdb.zrangebyscore(
            self.key,
            0,
            self.adjust(mktime(self.app.now().timetuple()), drift=0.010),
            withscores=True) or []

        next_times = [
            self.max_interval,
        ]

        for task, score in tasks:
            entry = jsonpickle.decode(task)
            is_due, next_time_to_run = self.is_due(entry)

            next_times.append(next_time_to_run)
            if is_due:
                next_entry = self.reserve(entry)
                try:
                    linfo("add task entry: {} to publisher".format(entry.name))
                    result = self.apply_async(entry)
                except Exception as exc:
                    error('Message Error: %s\n%s',
                          exc,
                          traceback.format_stack(),
                          exc_info=True)
                else:
                    debug('%s sent. id->%s', entry.task, result.id)
                self.rdb.zrem(self.key, task)
                self.rdb.zadd(
                    self.key, {
                        jsonpickle.encode(next_entry):
                        self._when(next_entry, next_time_to_run) or 0
                    })

        next_task = self.rdb.zrangebyscore(self.key,
                                           0,
                                           MAXINT,
                                           withscores=True,
                                           num=1,
                                           start=0)
        if not next_task:
            linfo("no next task found")
            return min(next_times)
        entry = jsonpickle.decode(next_task[0][0])
        next_times.append(self.is_due(entry)[1])

        return min(next_times)
Beispiel #16
0
 def setup_includes(self, includes):
     # Update celery_include to have all known task modules, so that we
     # ensure all task modules are imported in case an execv happens.
     prev = tuple(self.app.conf.CELERY_INCLUDE)
     if includes:
         prev += tuple(includes)
         [self.app.loader.import_task_module(m) for m in includes]
     self.include = includes
     task_modules = {task.__class__.__module__ for task in values(self.app.tasks)}
     self.app.conf.CELERY_INCLUDE = tuple(set(prev) | task_modules)
Beispiel #17
0
 def start(self, c):
     info('mingle: searching for neighbors')
     I = c.app.control.inspect(timeout=1.0, connection=c.connection)
     replies = I.hello()
     if replies:
         for reply in values(replies):
             c.app.clock.adjust(reply['clock'])
             revoked.update(reply['revoked'])
         info('mingle: synced with %s', ', '.join(replies))
     else:
         info('mingle: no one here')
Beispiel #18
0
 def __enter__(self):
     # The __warningregistry__'s need to be in a pristine state for tests
     # to work properly.
     warnings.resetwarnings()
     for v in list(values(sys.modules)):
         if getattr(v, '__warningregistry__', None):
             v.__warningregistry__ = {}
     self.warnings_manager = warnings.catch_warnings(record=True)
     self.warnings = self.warnings_manager.__enter__()
     warnings.simplefilter('always', self.expected)
     return self
Beispiel #19
0
 def setup_includes(self, includes):
     # Update celery_include to have all known task modules, so that we
     # ensure all task modules are imported in case an execv happens.
     prev = tuple(self.app.conf.CELERY_INCLUDE)
     if includes:
         prev += tuple(includes)
         [self.app.loader.import_task_module(m) for m in includes]
     self.include = includes
     task_modules = set(task.__class__.__module__
                        for task in values(self.app.tasks))
     self.app.conf.CELERY_INCLUDE = tuple(set(prev) | task_modules)
Beispiel #20
0
 def setup_includes(self, includes):
     # Update celery_include to have all known task modules, so that we
     # ensure all task modules are imported in case an execv happens.
     inc = self.app.conf.CELERY_INCLUDE
     if includes:
         includes = str_to_list(includes)
         inc = self.app.conf.CELERY_INCLUDE = tuple(inc) + tuple(includes)
     self.include = includes
     task_modules = set(task.__class__.__module__
                        for task in values(self.app.tasks))
     self.app.conf.CELERY_INCLUDE = tuple(set(inc) | task_modules)
Beispiel #21
0
 def populate_heap(self, event_t=event_t, heapify=heapq.heapify):
     """Populate the heap with the data contained in the schedule."""
     priority = 5
     self._heap = []
     for entry in values(self.schedule):
         is_due, next_call_delay = entry.is_due()
         self._heap.append(
             event_t(
                 self._when(entry, 0 if is_due else next_call_delay) or 0,
                 priority, entry))
     heapify(self._heap)
Beispiel #22
0
 def __enter__(self):
     # The __warningregistry__'s need to be in a pristine state for tests
     # to work properly.
     warnings.resetwarnings()
     for v in list(values(sys.modules)):
         if getattr(v, '__warningregistry__', None):
             v.__warningregistry__ = {}
     self.warnings_manager = warnings.catch_warnings(record=True)
     self.warnings = self.warnings_manager.__enter__()
     warnings.simplefilter('always', self.expected)
     return self
Beispiel #23
0
 def start(self, c):
     info('mingle: searching for neighbors')
     I = c.app.control.inspect(timeout=1.0, connection=c.connection)
     replies = I.hello()
     if replies:
         for reply in values(replies):
             c.app.clock.adjust(reply['clock'])
             revoked.update(reply['revoked'])
         info('mingle: synced with %s', ', '.join(replies))
     else:
         info('mingle: no one here')
Beispiel #24
0
    def finalize(self):
        with self._finalize_mutex:
            if not self.finalized:
                self.finalized = True
                load_shared_tasks(self)

                pending = self._pending
                while pending:
                    maybe_evaluate(pending.popleft())

                for task in values(self._tasks):
                    task.bind(self)
Beispiel #25
0
    def finalize(self):
        with self._finalize_mutex:
            if not self.finalized:
                self.finalized = True
                load_shared_tasks(self)

                pending = self._pending
                while pending:
                    maybe_evaluate(pending.popleft())

                for task in values(self._tasks):
                    task.bind(self)
Beispiel #26
0
 def setup_includes(self, includes):
     # Update celery_include to have all known task modules, so that we
     # ensure all task modules are imported in case an execv happens.
     inc = self.app.conf.CELERY_INCLUDE
     if includes:
         if isinstance(includes, string_t):
             includes = includes.split(',')
         inc = self.app.conf.CELERY_INCLUDE = tuple(inc) + tuple(includes)
     self.include = includes
     task_modules = set(task.__class__.__module__
                        for task in values(self.app.tasks))
     self.app.conf.CELERY_INCLUDE = tuple(set(inc) | task_modules)
Beispiel #27
0
 def on_close(self):
     # Clear internal queues to get rid of old messages.
     # They can't be acked anyway, as a delivery tag is specific
     # to the current channel.
     if self.controller and self.controller.semaphore:
         self.controller.semaphore.clear()
     if self.timer:
         self.timer.clear()
     for bucket in values(self.task_buckets):
         if bucket:
             bucket.clear_pending()
     reserved_requests.clear()
     if self.pool and self.pool.flush:
         self.pool.flush()
Beispiel #28
0
    def finalize(self, auto=False):
        with self._finalize_mutex:
            if not self.finalized:
                if auto and not self.autofinalize:
                    raise RuntimeError("Contract breach: app not finalized")
                self.finalized = True
                _announce_app_finalized(self)

                pending = self._pending
                while pending:
                    maybe_evaluate(pending.popleft())

                for task in values(self._tasks):
                    task.bind(self)
Beispiel #29
0
 def on_close(self):
     # Clear internal queues to get rid of old messages.
     # They can't be acked anyway, as a delivery tag is specific
     # to the current channel.
     if self.controller and self.controller.semaphore:
         self.controller.semaphore.clear()
     if self.timer:
         self.timer.clear()
     for bucket in values(self.task_buckets):
         if bucket:
             bucket.clear_pending()
     reserved_requests.clear()
     if self.pool and self.pool.flush:
         self.pool.flush()
Beispiel #30
0
    def finalize(self, auto=False):
        with self._finalize_mutex:
            if not self.finalized:
                if auto and not self.autofinalize:
                    raise RuntimeError('Contract breach: app not finalized')
                self.finalized = True
                load_shared_tasks(self)

                pending = self._pending
                while pending:
                    maybe_evaluate(pending.popleft())

                for task in values(self._tasks):
                    task.bind(self)
Beispiel #31
0
 def on_process_up(proc):
     # If we got the same fd as a previous process then we will also
     # receive jobs in the old buffer, so we need to reset the
     # _write_to and _scheduled_for tracking values used to recover
     # message boundaries when processes exit.
     infd = proc.inqW_fd
     for job in values(pool._cache):
         if job._write_to and job._write_to.inqW_fd == infd:
             job._write_to = proc
         if job._scheduled_for and job._scheduled_for.inqW_fd == infd:
             job._scheduled_for = proc
     fileno_to_outq[proc.outqR_fd] = proc
     hub_add(proc.sentinel, maintain_pool, READ | ERR)
     hub_add(proc.outqR_fd, handle_result_event, READ | ERR)
Beispiel #32
0
    def human_write_stats(self):
        if self.write_stats is None:
            return 'N/A'
        vals = list(values(self.write_stats))
        total = sum(vals)

        def per(v, total):
            return '{0:.2f}%'.format((float(v) / total) * 100.0 if v else 0)

        return {
            'total': total,
            'avg': per(total / len(self.write_stats) if total else 0, total),
            'all': ', '.join(per(v, total) for v in vals)
        }
Beispiel #33
0
    def as_dict(self):
        """Whole set as serializable dictionary.

        Example:
            >>> s = LimitedSet(maxlen=200)
            >>> r = LimitedSet(maxlen=200)
            >>> for i in range(500):
            ...     s.add(i)
            ...
            >>> r.update(s.as_dict())
            >>> r == s
            True
        """
        return {key: inserted for inserted, key in values(self._data)}
Beispiel #34
0
 def on_process_up(proc):
     # If we got the same fd as a previous process then we will also
     # receive jobs in the old buffer, so we need to reset the
     # _write_to and _scheduled_for tracking values used to recover
     # message boundaries when processes exit.
     infd = proc.inqW_fd
     for job in values(pool._cache):
         if job._write_to and job._write_to.inqW_fd == infd:
             job._write_to = proc
         if job._scheduled_for and job._scheduled_for.inqW_fd == infd:
             job._scheduled_for = proc
     fileno_to_outq[proc.outqR_fd] = proc
     hub_add(proc.sentinel, maintain_pool, READ | ERR)
     hub_add(proc.outqR_fd, handle_result_event, READ | ERR)
Beispiel #35
0
    def human_write_stats(self):
        if self.write_stats is None:
            return "N/A"
        vals = list(values(self.write_stats))
        total = sum(vals)

        def per(v, total):
            return "{0:.2f}%".format((float(v) / total) * 100.0 if v else 0)

        return {
            "total": total,
            "avg": per(total / len(self.write_stats) if total else 0, total),
            "all": ", ".join(per(v, total) for v in vals),
        }
Beispiel #36
0
    def setup_schedule(self):
        """ PersistentScheduler + s/self.scheduler/self._store['entries'] """
        try:
            self._store = self.persistence.open(self.schedule_filename,
                                                writeback=True)
        except Exception as exc:
            error('Removing corrupted schedule file %r: %r',
                  self.schedule_filename,
                  exc,
                  exc_info=True)
            self._remove_db()
            self._store = self.persistence.open(self.schedule_filename,
                                                writeback=True)
        else:
            try:
                self._store['entries']
            except KeyError:
                # new schedule db
                self._store['entries'] = {}
            else:
                if '__version__' not in self._store:
                    warning('DB Reset: Account for new __version__ field')
                    self._store.clear()  # remove schedule at 2.2.2 upgrade.
                elif 'tz' not in self._store:
                    warning('DB Reset: Account for new tz field')
                    self._store.clear()  # remove schedule at 3.0.8 upgrade
                elif 'utc_enabled' not in self._store:
                    warning('DB Reset: Account for new utc_enabled field')
                    self._store.clear()  # remove schedule at 3.0.9 upgrade

        tz = self.app.conf.CELERY_TIMEZONE
        stored_tz = self._store.get('tz')
        if stored_tz is not None and stored_tz != tz:
            warning('Reset: Timezone changed from %r to %r', stored_tz, tz)
            self._store.clear()  # Timezone changed, reset db!
        utc = self.app.conf.CELERY_ENABLE_UTC
        stored_utc = self._store.get('utc_enabled')
        if stored_utc is not None and stored_utc != utc:
            choices = {True: 'enabled', False: 'disabled'}
            warning('Reset: UTC changed from %s to %s', choices[stored_utc],
                    choices[utc])
            self._store.clear()  # UTC setting changed, reset db!
        entries = self._store.setdefault('entries', {})
        self.merge_inplace(self.app.conf.CELERYBEAT_SCHEDULE)
        self.install_default_entries(self._store['entries'])
        self._store.update(__version__=__version__, tz=tz, utc_enabled=utc)
        self.sync()
        debug('Current schedule:\n' +
              '\n'.join(repr(entry) for entry in values(entries)))
Beispiel #37
0
def iter_open_logger_fds():
    seen = set()
    loggers = (list(values(logging.Logger.manager.loggerDict)) +
               [logging.getLogger(None)])
    for l in loggers:
        try:
            for handler in l.handlers:
                try:
                    if handler not in seen:  # pragma: no cover
                        yield handler.stream
                        seen.add(handler)
                except AttributeError:
                    pass
        except AttributeError:  # PlaceHolder does not have handlers
            pass
Beispiel #38
0
    def tick(self, min=min, **kwargs):
        if self.lock:
            logger.debug('beat: Extending lock...')
            redis(self.app).pexpire(self.lock_key, int(self.lock_timeout * 1000))

        remaining_times = []
        try:
            for entry in values(self.schedule):
                next_time_to_run = self.maybe_due(entry, **self._maybe_due_kwargs)
                if next_time_to_run:
                    remaining_times.append(next_time_to_run)
        except RuntimeError:
            logger.debug('beat: RuntimeError', exc_info=True)

        return min(remaining_times + [self.max_interval])
Beispiel #39
0
 def start(self, c):
     info('mingle: searching for neighbors')
     I = c.app.control.inspect(timeout=1.0, connection=c.connection)
     replies = I.hello()
     if replies:
         for reply in values(replies):
             try:
                 other_clock, other_revoked = MINGLE_GET_FIELDS(reply)
             except KeyError:  # reply from pre-3.1 worker
                 pass
             c.app.clock.adjust(other_clock)
             revoked.update(other_revoked)
         info('mingle: synced with %s', ', '.join(replies))
     else:
         info('mingle: no one here')
Beispiel #40
0
 def start(self, c):
     info('mingle: searching for neighbors')
     I = c.app.control.inspect(timeout=1.0, connection=c.connection)
     replies = I.hello()
     if replies:
         for reply in values(replies):
             try:
                 other_clock, other_revoked = MINGLE_GET_FIELDS(reply)
             except KeyError:  # reply from pre-3.1 worker
                 pass
             c.app.clock.adjust(other_clock)
             revoked.update(other_revoked)
         info('mingle: synced with %s', ', '.join(replies))
     else:
         info('mingle: no one here')
Beispiel #41
0
 def _finalize_steps(self, steps):
     last = self._find_last()
     self._firstpass(steps)
     it = ((C, C.requires) for C in values(steps))
     G = self.graph = DependencyGraph(
         it,
         formatter=self.GraphFormatter(root=last),
     )
     if last:
         for obj in G:
             if obj != last:
                 G.add_edge(last, obj)
     try:
         return G.topsort()
     except KeyError as exc:
         raise KeyError('unknown bootstep: %s' % exc)
Beispiel #42
0
    def human_write_stats(self):
        if self.write_stats is None:
            return "N/A"
        vals = list(values(self.write_stats))
        total = sum(vals)

        def per(v, total):
            return "{0:.2f}%".format((float(v) / total) * 100.0 if v else 0)

        return {
            "total": total,
            "avg": per(total / len(self.write_stats) if total else 0, total),
            "all": ", ".join(per(v, total) for v in vals),
            "raw": ", ".join(map(str, vals)),
            "inqueues": {"total": len(self._all_inqueues), "active": len(self._active_writes)},
        }
Beispiel #43
0
    def setup_schedule(self):
        """ PersistentScheduler + s/self.scheduler/self._store['entries'] """
        try:
            self._store = self.persistence.open(self.schedule_filename,
                                                writeback=True)
        except Exception as exc:
            error('Removing corrupted schedule file %r: %r',
                  self.schedule_filename, exc, exc_info=True)
            self._remove_db()
            self._store = self.persistence.open(self.schedule_filename,
                                                writeback=True)
        else:
            try:
                self._store['entries']
            except KeyError:
                # new schedule db
                self._store['entries'] = {}
            else:
                if '__version__' not in self._store:
                    warning('DB Reset: Account for new __version__ field')
                    self._store.clear()   # remove schedule at 2.2.2 upgrade.
                elif 'tz' not in self._store:
                    warning('DB Reset: Account for new tz field')
                    self._store.clear()   # remove schedule at 3.0.8 upgrade
                elif 'utc_enabled' not in self._store:
                    warning('DB Reset: Account for new utc_enabled field')
                    self._store.clear()   # remove schedule at 3.0.9 upgrade

        tz = self.app.conf.CELERY_TIMEZONE
        stored_tz = self._store.get('tz')
        if stored_tz is not None and stored_tz != tz:
            warning('Reset: Timezone changed from %r to %r', stored_tz, tz)
            self._store.clear()   # Timezone changed, reset db!
        utc = self.app.conf.CELERY_ENABLE_UTC
        stored_utc = self._store.get('utc_enabled')
        if stored_utc is not None and stored_utc != utc:
            choices = {True: 'enabled', False: 'disabled'}
            warning('Reset: UTC changed from %s to %s',
                    choices[stored_utc], choices[utc])
            self._store.clear()   # UTC setting changed, reset db!
        entries = self._store.setdefault('entries', {})
        self.merge_inplace(self.app.conf.CELERYBEAT_SCHEDULE)
        self.install_default_entries(self._store['entries'])
        self._store.update(__version__=__version__, tz=tz, utc_enabled=utc)
        self.sync()
        debug('Current schedule:\n' + '\n'.join(
            repr(entry) for entry in values(entries)))
Beispiel #44
0
    def _run(self,
             ipython=False,
             bpython=False,
             python=False,
             without_tasks=False,
             eventlet=False,
             gevent=False,
             **kwargs):
        sys.path.insert(0, os.getcwd())
        if eventlet:
            import_module("celery.concurrency.eventlet")
        if gevent:
            import_module("celery.concurrency.gevent")
        import celery
        import celery.task.base

        self.app.loader.import_default_modules()

        # pylint: disable=attribute-defined-outside-init
        self.locals = {
            "app": self.app,
            "celery": self.app,
            "Task": celery.Task,
            "chord": celery.chord,
            "group": celery.group,
            "chain": celery.chain,
            "chunks": celery.chunks,
            "xmap": celery.xmap,
            "xstarmap": celery.xstarmap,
            "subtask": celery.subtask,
            "signature": celery.signature,
        }

        if not without_tasks:
            self.locals.update({
                task.__name__: task
                for task in values(self.app.tasks)
                if not task.name.startswith("celery.")
            })

        if python:
            return self.invoke_fallback_shell()
        elif bpython:
            return self.invoke_bpython_shell()
        elif ipython:
            return self.invoke_ipython_shell()
        return self.invoke_default_shell()
Beispiel #45
0
 def init_groups(self):
     acc = defaultdict(list)
     for attr in dir(self):
         if not _is_descriptor(self, attr):
             meth = getattr(self, attr)
             try:
                 groups = meth.__func__.__testgroup__
             except AttributeError:
                 pass
             else:
                 for g in groups:
                     acc[g].append(meth)
     # sort the tests by the order in which they are defined in the class
     for g in values(acc):
         g[:] = sorted(g, key=lambda m: m.__func__.__testsort__)
     self.groups = dict(
         (name, testgroup(*tests)) for name, tests in items(acc))
Beispiel #46
0
    def _run(self,
             ipython=False,
             bpython=False,
             python=False,
             without_tasks=False,
             eventlet=False,
             gevent=False,
             **kwargs):
        sys.path.insert(0, os.getcwd())
        if eventlet:
            import_module('celery.concurrency.eventlet')
        if gevent:
            import_module('celery.concurrency.gevent')
        import celery
        import celery.task.base
        self.app.loader.import_default_modules()

        # pylint: disable=attribute-defined-outside-init
        self.locals = {
            'app': self.app,
            'celery': self.app,
            'Task': celery.Task,
            'chord': celery.chord,
            'group': celery.group,
            'chain': celery.chain,
            'chunks': celery.chunks,
            'xmap': celery.xmap,
            'xstarmap': celery.xstarmap,
            'subtask': celery.subtask,
            'signature': celery.signature,
        }

        if not without_tasks:
            self.locals.update({
                task.__name__: task
                for task in values(self.app.tasks)
                if not task.name.startswith('celery.')
            })

        if python:
            return self.invoke_fallback_shell()
        elif bpython:
            return self.invoke_bpython_shell()
        elif ipython:
            return self.invoke_ipython_shell()
        return self.invoke_default_shell()
Beispiel #47
0
    def human_write_stats(self):
        if self.write_stats is None:
            return "N/A"
        vals = list(values(self.write_stats))
        total = sum(vals)

        def per(v, total):
            return "{0:.2%}".format((float(v) / total) if v else 0)

        return {
            "total": total,
            "avg": per(total / len(self.write_stats) if total else 0, total),
            "all": ", ".join(per(v, total) for v in vals),
            "raw": ", ".join(map(str, vals)),
            "strategy": SCHED_STRATEGY_TO_NAME.get(self.sched_strategy, self.sched_strategy),
            "inqueues": {"total": len(self._all_inqueues), "active": len(self._active_writes)},
        }
Beispiel #48
0
    def run(
        self,
        force_ipython=False,
        force_bpython=False,
        force_python=False,
        without_tasks=False,
        eventlet=False,
        gevent=False,
        **kwargs
    ):
        sys.path.insert(0, os.getcwd())
        if eventlet:
            import_module("celery.concurrency.eventlet")
        if gevent:
            import_module("celery.concurrency.gevent")
        import celery
        import celery.task.base

        self.app.loader.import_default_modules()
        self.locals = {
            "app": self.app,
            "celery": self.app,
            "Task": celery.Task,
            "chord": celery.chord,
            "group": celery.group,
            "chain": celery.chain,
            "chunks": celery.chunks,
            "xmap": celery.xmap,
            "xstarmap": celery.xstarmap,
            "subtask": celery.subtask,
            "signature": celery.signature,
        }

        if not without_tasks:
            self.locals.update(
                {task.__name__: task for task in values(self.app.tasks) if not task.name.startswith("celery.")}
            )

        if force_python:
            return self.invoke_fallback_shell()
        elif force_bpython:
            return self.invoke_bpython_shell()
        elif force_ipython:
            return self.invoke_ipython_shell()
        return self.invoke_default_shell()
Beispiel #49
0
    def finalize(self, auto=False):
        """Finalizes the app by loading built-in tasks,
        and evaluating pending task decorators."""
        with self._finalize_mutex:
            if not self.finalized:
                if auto and not self.autofinalize:
                    raise RuntimeError('Contract breach: app not finalized')
                self.finalized = True
                _announce_app_finalized(self)

                pending = self._pending
                while pending:
                    maybe_evaluate(pending.popleft())

                for task in values(self._tasks):
                    task.bind(self)

                self.on_after_finalize.send(sender=self)
Beispiel #50
0
    def finalize(self, auto=False):
        """Finalizes the app by loading built-in tasks,
        and evaluating pending task decorators."""
        with self._finalize_mutex:
            if not self.finalized:
                if auto and not self.autofinalize:
                    raise RuntimeError('Contract breach: app not finalized')
                self.finalized = True
                _announce_app_finalized(self)

                pending = self._pending
                while pending:
                    maybe_evaluate(pending.popleft())

                for task in values(self._tasks):
                    task.bind(self)

                self.on_after_finalize.send(sender=self)
Beispiel #51
0
 def on_process_up(proc):
     """Called when a WORKER_UP message is received from process."""
     # If we got the same fd as a previous process then we will also
     # receive jobs in the old buffer, so we need to reset the
     # job._write_to and job._scheduled_for attributes used to recover
     # message boundaries when processes exit.
     infd = proc.inqW_fd
     for job in values(cache):
         if job._write_to and job._write_to.inqW_fd == infd:
             job._write_to = proc
         if job._scheduled_for and job._scheduled_for.inqW_fd == infd:
             job._scheduled_for = proc
     fileno_to_outq[proc.outqR_fd] = proc
     # maintain_pool is called whenever a process exits.
     add_reader(proc.sentinel, maintain_pool)
     # handle_result_event is called when the processes outqueue is
     # readable.
     add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd)
Beispiel #52
0
    def human_write_stats(self):
        if self.write_stats is None:
            return 'N/A'
        vals = list(values(self.write_stats))
        total = sum(vals)

        def per(v, total):
            return '{0:.2f}%'.format((float(v) / total) * 100.0 if v else 0)

        return {
            'total': total,
            'avg': per(total / len(self.write_stats) if total else 0, total),
            'all': ', '.join(per(v, total) for v in vals),
            'raw': ', '.join(map(str, vals)),
            'inqueues': {
                'total': len(self._all_inqueues),
                'active': len(self._active_writes),
            }
        }
Beispiel #53
0
 def start(self, c):
     info('mingle: searching for neighbors')
     I = c.app.control.inspect(timeout=1.0, connection=c.connection)
     replies = I.hello(c.hostname, revoked._data) or {}
     replies.pop(c.hostname, None)
     if replies:
         info('mingle: hello %s! sync with me',
              ', '.join(reply for reply, value in items(replies) if value))
         for reply in values(replies):
             if reply:
                 try:
                     other_clock, other_revoked = MINGLE_GET_FIELDS(reply)
                 except KeyError:  # reply from pre-3.1 worker
                     pass
                 else:
                     c.app.clock.adjust(other_clock)
                     revoked.update(other_revoked)
     else:
         info('mingle: all alone')
Beispiel #54
0
 def find_app(self, app):
     try:
         sym = self.symbol_by_name(app)
     except AttributeError:
         # last part was not an attribute, but a module
         sym = import_from_cwd(app)
     if isinstance(sym, ModuleType):
         try:
             return sym.celery
         except AttributeError:
             if getattr(sym, '__path__', None):
                 return self.find_app('{0}.celery:'.format(
                                      app.replace(':', '')))
             from celery.app.base import Celery
             for suspect in values(vars(sym)):
                 if isinstance(suspect, Celery):
                     return suspect
             raise
     return sym
Beispiel #55
0
    def schedule(self):
        update = False
        if not self._initial_read:
            debug('DatabaseScheduler: initial read')
            update = True
            self._initial_read = True
        elif self.schedule_changed():
            info('DatabaseScheduler: Schedule changed.')
            update = True

        if update:
            self.sync()
            self._schedule = self.all_as_schedule()
            if logger.isEnabledFor(logging.DEBUG):
                debug(
                    'Current schedule:\n%s',
                    '\n'.join(repr(entry) for entry in values(self._schedule)),
                )
        return self._schedule
Beispiel #56
0
    def run(self,
            force_ipython=False,
            force_bpython=False,
            force_python=False,
            without_tasks=False,
            eventlet=False,
            gevent=False,
            **kwargs):
        sys.path.insert(0, os.getcwd())
        if eventlet:
            import_module('celery.concurrency.eventlet')
        if gevent:
            import_module('celery.concurrency.gevent')
        import celery
        import celery.task.base
        self.app.loader.import_default_modules()
        self.locals = {
            'app': self.app,
            'celery': self.app,
            'Task': celery.Task,
            'chord': celery.chord,
            'group': celery.group,
            'chain': celery.chain,
            'chunks': celery.chunks,
            'xmap': celery.xmap,
            'xstarmap': celery.xstarmap,
            'subtask': celery.subtask,
            'signature': celery.signature
        }

        if not without_tasks:
            self.locals.update(
                dict((task.__name__, task) for task in values(self.app.tasks)
                     if not task.name.startswith('celery.')), )

        if force_python:
            return self.invoke_fallback_shell()
        elif force_bpython:
            return self.invoke_bpython_shell()
        elif force_ipython:
            return self.invoke_ipython_shell()
        return self.invoke_default_shell()
Beispiel #57
0
    def human_write_stats(self):
        if self.write_stats is None:
            return 'N/A'
        vals = list(values(self.write_stats))
        total = sum(vals)

        def per(v, total):
            return '{0:.2%}'.format((float(v) / total) if v else 0)

        return {
            'total': total,
            'avg': per(total / len(self.write_stats) if total else 0, total),
            'all': ', '.join(per(v, total) for v in vals),
            'raw': ', '.join(map(str, vals)),
            'strategy': SCHED_STRATEGY_TO_NAME[self.sched_strategy],
            'inqueues': {
                'total': len(self._all_inqueues),
                'active': len(self._active_writes),
            }
        }
Beispiel #58
0
 def __enter__(self):
     # The __warningregistry__'s need to be in a pristine state for tests
     # to work properly.
     warnings.resetwarnings()
     for v in list(values(sys.modules)):
         # do not evaluate Django moved modules and other lazily
         # initialized modules.
         if v and not _is_magic_module(v):
             # use raw __getattribute__ to protect even better from
             # lazily loaded modules
             try:
                 object.__getattribute__(v, '__warningregistry__')
             except AttributeError:
                 pass
             else:
                 object.__setattr__(v, '__warningregistry__', {})
     self.warnings_manager = warnings.catch_warnings(record=True)
     self.warnings = self.warnings_manager.__enter__()
     warnings.simplefilter('always', self.expected)
     return self