def __init__(self, task): self._task = task self.app = current_app._get_current_object() self.name = self._task.name self.task = self._task.task self.schedule = self._task.schedule self.args = self._task.args self.kwargs = self._task.kwargs self.options = { "queue": self._task.queue, "exchange": self._task.exchange, "routing_key": self._task.routing_key, "expires": self._task.expires, "soft_time_limit": self._task.soft_time_limit, "enabled": self._task.enabled, } if self._task.total_run_count is None: self._task.total_run_count = 0 self.total_run_count = self._task.total_run_count if not self._task.last_run_at: self._task.last_run_at = self._default_now() self.last_run_at = self._task.last_run_at
def __init__(self, model, app=None): """Initialize the model entry.""" self.app = app or current_app._get_current_object() self.name = "{}_{}".format(model.name, model.pk) self.task = model.task self.args = model.args self.kwargs = model.kwargs try: self.schedule = model.schedule except model.DoesNotExist: logger.error( "Disabling schedule %s that was removed from database", self.name, ) self._disable(model) self.options = {} if model.queue: self.options["queue"] = model.queue self.options["headers"] = {} self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() self.last_run_at = model.last_run_at
def __init__(self, model): self.app = current_app._get_current_object() self.name = model.name self.task = model.task self.schedule = model.schedule try: self.args = loads(model.args or "[]") self.kwargs = loads(model.kwargs or "{}") except ValueError: logging.error("Failed to serialize arguments for %s.", self.name, exc_info=1) logging.warning("Disabling %s", self.name) model.no_changes = True model.enabled = False model.save() self.options = { "queue": model.queue, "exchange": model.exchange, "routing_key": model.routing_key, "expires": model.expires, } self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() orig = self.last_run_at = model.last_run_at if not is_naive(self.last_run_at): self.last_run_at = self.last_run_at.replace(tzinfo=None) assert orig.hour == self.last_run_at.hour # timezone sanity
def __init__(self, model): self.app = current_app._get_current_object() self.name = model.name self.task = model.task try: self.schedule = model.schedule except model.DoesNotExist: logger.error('Schedule was removed from database') logger.warning('Disabling %s', self.name) self._disable(model) try: self.args = loads(model.args or '[]') self.kwargs = loads(model.kwargs or '{}') except ValueError: logging.error('Failed to serialize arguments for %s.', self.name, exc_info=1) logging.warning('Disabling %s', self.name) self._disable(model) self.options = { 'queue': model.queue, 'exchange': model.exchange, 'routing_key': model.routing_key, 'expires': model.expires } self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() orig = self.last_run_at = model.last_run_at if not is_naive(self.last_run_at): self.last_run_at = self.last_run_at.replace(tzinfo=None) assert orig.hour == self.last_run_at.hour # timezone sanity
def __init__(self, model): self.app = current_app._get_current_object() self.name = model.name self.task = model.task self.schedule = model.schedule try: self.args = loads(model.args or '[]') self.kwargs = loads(model.kwargs or '{}') except ValueError: logging.error('Failed to serialize arguments for %s.', self.name, exc_info=1) logging.warning('Disabling %s', self.name) model.no_changes = True model.enabled = False model.save() expires = model.expires or model.expires_at self.options = { 'queue': model.queue, 'exchange': model.exchange, 'routing_key': model.routing_key, 'expires': expires } self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() orig = self.last_run_at = model.last_run_at if not is_naive(self.last_run_at): self.last_run_at = self.last_run_at.replace(tzinfo=None) assert orig.hour == self.last_run_at.hour # timezone sanity
def rabbitmq_queue_size(): """Returns the rabbitmq queue size. Two things to know about the queue size: 1. It's not 100% accurate, but the size is generally near that number 2. I can't think of a second thing, but that first thing is pretty important. """ # FIXME: 2015-04-23: This is busted. from celery import current_app # FIXME: This uses a private method, but I'm not sure how else to # figure this out, either. app = current_app._get_current_object() conn = app.connection() chan = conn.default_channel # FIXME: This hard-codes the exchange, but I'm not sure how else # to figure it out. queue = chan.queue_declare('celery', passive=True) return queue.message_count
def setup_security(allowed_serializers=None, key=None, cert=None, store=None, digest='sha1', serializer='json', app=None): """See :meth:`@Celery.setup_security`.""" if app is None: from celery import current_app app = current_app._get_current_object() disable_untrusted_serializers(allowed_serializers) conf = app.conf if conf.CELERY_TASK_SERIALIZER != 'auth': return try: from OpenSSL import crypto # noqa except ImportError: raise ImproperlyConfigured(SSL_NOT_INSTALLED) key = key or conf.CELERY_SECURITY_KEY cert = cert or conf.CELERY_SECURITY_CERTIFICATE store = store or conf.CELERY_SECURITY_CERT_STORE if not (key and cert and store): raise ImproperlyConfigured(SETTING_MISSING) with open(key) as kf: with open(cert) as cf: register_auth(kf.read(), cf.read(), store, digest, serializer) registry._set_default_serializer('auth')
def __new__(cls, name, bases, attrs): new = super(TaskType, cls).__new__ task_module = attrs.get('__module__') or '__main__' # - Abstract class: abstract attribute should not be inherited. abstract = attrs.pop('abstract', None) if abstract or not attrs.get('autoregister', True): return new(cls, name, bases, attrs) # The 'app' attribute is now a property, with the real app located # in the '_app' attribute. Previously this was a regular attribute, # so we should support classes defining it. app = attrs.pop('_app', None) or attrs.pop('app', None) # Attempt to inherit app from one the bases if not isinstance(app, Proxy) and app is None: for base in bases: if getattr(base, '_app', None): app = base._app break else: app = current_app._get_current_object() attrs['_app'] = app # - Automatically generate missing/empty name. task_name = attrs.get('name') if not task_name: attrs['name'] = task_name = gen_task_name(app, name, task_module) if not attrs.get('_decorated'): # non decorated tasks must also be shared in case # an app is created multiple times due to modules # imported under multiple names. # Hairy stuff, here to be compatible with 2.x. # People should not use non-abstract task classes anymore, # use the task decorator. from celery._state import connect_on_app_finalize unique_name = '.'.join([task_module, name]) if unique_name not in cls._creation_count: # the creation count is used as a safety # so that the same task is not added recursively # to the set of constructors. cls._creation_count[unique_name] = 1 connect_on_app_finalize(_CompatShared( unique_name, lambda app: TaskType.__new__(cls, name, bases, dict(attrs, _app=app)), )) # - Create and register class. # Because of the way import happens (recursively) # we may or may not be the first time the task tries to register # with the framework. There should only be one class for each task # name, so we always return the registered version. tasks = app._tasks if task_name not in tasks: tasks.register(new(cls, name, bases, attrs)) instance = tasks[task_name] instance.bind(app) return instance.__class__
def __init__(self, model): self.app = current_app._get_current_object() self.name = model.name self.task = model.task try: self.schedule = model.schedule except model.DoesNotExist: logger.error('Schedule was removed from database') logger.warning('Disabling %s', self.name) self._disable(model) try: self.args = loads(model.args or '[]') self.kwargs = loads(model.kwargs or '{}') except ValueError: logging.error('Failed to serialize arguments for %s.', self.name, exc_info=1) logging.warning('Disabling %s', self.name) self._disable(model) self.options = {'queue': model.queue, 'exchange': model.exchange, 'routing_key': model.routing_key, 'expires': model.expires} self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() orig = self.last_run_at = model.last_run_at if not is_naive(self.last_run_at): self.last_run_at = self.last_run_at.replace(tzinfo=None) assert orig.hour == self.last_run_at.hour # timezone sanity
def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._logger.debug('DS executing tick') try: self._has_lock = False with self._mutex: # noinspection PyProtectedMember node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = System.get_my_machine_id() try: lock = self._persistent.get(self._lock_name) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True self._logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock self._logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock[ 'timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen self._logger.debug( 'DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) self._logger.debug('DS stealing lock from {0}'.format( lock['name'])) self._load_schedule() self._has_lock = True else: self._logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} self._logger.debug('DS refreshing lock') self._persistent.set(self._lock_name, lock) if self._has_lock is True: self._logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due( entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass self._logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval except Exception as ex: self._logger.debug('DS got error during tick: {0}'.format(ex)) return self.max_interval
def app(env_config, default_config): """Flask application fixture.""" app = create_app(**default_config) # FIXME: Needs fixing flask_celeryext, # which once creates the first celery app, the flask_app that is set # is never released from the global state, even if you create a new # celery application. We need to unset the "flask_app" manually. from celery import current_app as cca cca = cca._get_current_object() delattr(cca, "flask_app") celery_app = create_celery_app(app) # FIXME: When https://github.com/inveniosoftware/flask-celeryext/issues/35 # is closed and Flask-CeleryExt is released, this can be removed. class _TestAppContextTask(Task): abstract = True def __call__(self, *args, **kwargs): if flask_current_app: return Task.__call__(self, *args, **kwargs) with self.app.flask_app.app_context(): return Task.__call__(self, *args, **kwargs) celery_app.Task = _TestAppContextTask celery_app.set_current() with app.app_context(): yield app
def __init__(self, task): self._task = task self.app = current_app._get_current_object() self.name = self._task.name self.task = self._task.task self.schedule = self._task.schedule self.args = self._task.args self.kwargs = self._task.kwargs self.options = { 'queue': self._task.queue, 'exchange': self._task.exchange, 'routing_key': self._task.routing_key, 'expires': self._task.expires, 'soft_time_limit': self._task.soft_time_limit, 'enabled': self._task.enabled } if self._task.total_run_count is None: self._task.total_run_count = 0 self.total_run_count = self._task.total_run_count if not self._task.last_run_at: self._task.last_run_at = self._default_now() self.last_run_at = self._task.last_run_at
def __new__(cls, name, bases, attrs): new = super(TaskType, cls).__new__ task_module = attrs.get('__module__') or '__main__' # The 'app' attribute is now a property, with the real app located # in the '_app' attribute. Previously this was a regular attribute, # so we should support classes defining it. app = attrs.pop('_app', None) or attrs.pop('app', None) # Attempt to inherit app from one the bases if not isinstance(app, Proxy) and app is None: for base in bases: if getattr(base, '_app', None): app = base._app break else: app = current_app._get_current_object() attrs['_app'] = app # - Automatically generate missing/empty name. task_name = attrs.get('name') if not task_name: attrs['name'] = task_name = gen_task_name(app, name, task_module) # - Create and register class. # Because of the way import happens (recursively) # we may or may not be the first time the task tries to register # with the framework. There should only be one class for each task # name, so we always return the registered version. tasks = app._tasks if task_name not in tasks: tasks.register(new(cls, name, bases, attrs)) instance = tasks[task_name] instance.bind(app) return instance.__class__
def __init__(self, task): self._task = task self.app = current_app._get_current_object() self.name = self._task.key # passing key here as the task name is a human use only field. self.task = self._task.task self.schedule = self._task.schedule self.args = self._task.args self.kwargs = self._task.kwargs self.options = { 'queue': self._task.queue, 'exchange': self._task.exchange, 'routing_key': self._task.routing_key, 'expires': self._task.expires } if not self._task.total_run_count: self._task.total_run_count = 0 self.total_run_count = self._task.total_run_count if not self._task.last_run_at: # subtract some time from the current time to populate the last time # that the task was run so that a newly scheduled task does not get missed time_subtract = (self.app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or 30) self._task.last_run_at = self._default_now() - datetime.timedelta(seconds=time_subtract) self.save() self.last_run_at = self._task.last_run_at
def __init__(self, model, app=None): self.app = app or current_app._get_current_object() self.name = model.name self.task = model.task try: self.schedule = model.schedule except model.DoesNotExist: logger.error( 'Disabling schedule %s that was removed from database', self.name, ) self._disable(model) try: self.args = loads(model.args or '[]') self.kwargs = loads(model.kwargs or '{}') except ValueError as exc: logger.exception( 'Removing schedule %s for argument deseralization error: %r', self.name, exc, ) self._disable(model) self.options = { 'queue': model.queue, 'exchange': model.exchange, 'routing_key': model.routing_key, 'expires': model.expires, } self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() self.last_run_at = make_aware(model.last_run_at)
def __init__(self, model): self.app = current_app._get_current_object() self.name = model.name self.task = model.task self.schedule = model.schedule try: self.args = deserialize(model.args or u'[]') self.kwargs = deserialize(model.kwargs or u'{}') except ValueError: logging.error('Failed to serialize arguments for %s.', self.name, exc_info=1) logging.warning('Disabling %s', self.name) model.no_changes = True model.enabled = False model.save() self.options = {'queue': model.queue, 'exchange': model.exchange, 'routing_key': model.routing_key, 'expires': model.expires} self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() orig = self.last_run_at = model.last_run_at if not is_naive(self.last_run_at): self.last_run_at = self.last_run_at.replace(tzinfo=None) assert orig.hour == self.last_run_at.hour # timezone sanity
def setup_security(allowed_serializers=None, key=None, cert=None, store=None, digest=None, serializer='json', app=None): """See :meth:`@Celery.setup_security`.""" if app is None: from celery import current_app app = current_app._get_current_object() _disable_insecure_serializers(allowed_serializers) # check conf for sane security settings conf = app.conf if conf.task_serializer != 'auth' or conf.accept_content != ['auth']: raise ImproperlyConfigured(SETTING_MISSING) key = key or conf.security_key cert = cert or conf.security_certificate store = store or conf.security_cert_store digest = digest or conf.security_digest if not (key and cert and store): raise ImproperlyConfigured(SECURITY_SETTING_MISSING) with open(key, 'r') as kf: with open(cert, 'r') as cf: register_auth(kf.read(), cf.read(), store, digest, serializer) registry._set_default_serializer('auth')
def __init__(self, model): self.app = current_app._get_current_object() self.name = model.name self.task = model.task self.schedule = model.schedule try: self.args = loads(model.args or '[]') self.kwargs = loads(model.kwargs or '{}') except ValueError: # disable because of error deserializing args/kwargs model.no_changes = True model.enabled = False model.save() raise self.options = {'queue': model.queue, 'exchange': model.exchange, 'routing_key': model.routing_key, 'expires': model.expires} self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() orig = self.last_run_at = model.last_run_at if not is_naive(self.last_run_at): self.last_run_at = self.last_run_at.replace(tzinfo=None) assert orig.hour == self.last_run_at.hour # timezone sanity
def __init__(self, task): self._task = task self.app = current_app._get_current_object() self.name = self._task['name'] self.task = self._task['name'] # Fill out schedule if self._task['schedule_type'] == 'crontab': self.schedule = celery.schedules.crontab( minute=self._task['crontab']['minute'], hour=self._task['crontab']['hour'], day_of_week=self._task['crontab']['day_of_week'], day_of_month=self._task['crontab']['day_of_month'], month_of_year=self._task['crontab']['month_of_year']) elif self._task['schedule_type'] == 'interval': self.schedule = celery.schedules.schedule( datetime.timedelta( **{ self._task['interval']['period']: self._task['interval']['every'] })) self.args = self._task['args'] self.kwargs = self._task['kwargs'] self.options = {'enabled': self._task['enabled']} if 'last_run_at' not in self._task: self._task['last_run_at'] = self._default_now() self.last_run_at = toUTC(self._task['last_run_at']) if 'run_immediately' not in self._task: self._task['run_immediately'] = False
def setup_security(allowed_serializers=None, key=None, cert=None, store=None, digest='sha1', serializer='json', app=None): """See :meth:`@Celery.setup_security`.""" if app is None: from celery import current_app app = current_app._get_current_object() _disable_insecure_serializers(allowed_serializers) conf = app.conf if conf.task_serializer != 'auth': return try: from OpenSSL import crypto # noqa except ImportError: raise ImproperlyConfigured(SSL_NOT_INSTALLED) key = key or conf.security_key cert = cert or conf.security_certificate store = store or conf.security_cert_store if not (key and cert and store): raise ImproperlyConfigured(SETTING_MISSING) with open(key) as kf: with open(cert) as cf: register_auth(kf.read(), cf.read(), store, digest, serializer) registry._set_default_serializer('auth')
def __init__(self, task): """Initialize the task.""" self._task = task self.app = current_app._get_current_object() self.name = self._task.name self.task = self._task.task self.schedule = self._task.schedule self.args = self._task.args self.kwargs = self._task.kwargs # self.options = { # 'queue': self._task.queue, # 'exchange': self._task.exchange, # 'routing_key': self._task.routing_key, # 'expires': self._task.expires, # 'soft_time_limit': self._task.soft_time_limit # } if self._task.total_run_count is None: self._task.total_run_count = 0 self.total_run_count = self._task.total_run_count if not self._task.last_run_at: self._task.last_run_at = self._default_now() self.last_run_at = self._task.last_run_at
def setup_security( allowed_serializers=None, key=None, cert=None, store=None, digest=None, serializer="json", app=None, ): """See :meth:`@Celery.setup_security`.""" if app is None: from celery import current_app app = current_app._get_current_object() _disable_insecure_serializers(allowed_serializers) # check conf for sane security settings conf = app.conf if conf.task_serializer != "auth" or conf.accept_content != ["auth"]: raise ImproperlyConfigured(SETTING_MISSING) key = key or conf.security_key cert = cert or conf.security_certificate store = store or conf.security_cert_store digest = digest or conf.security_digest if not (key and cert and store): raise ImproperlyConfigured(SECURITY_SETTING_MISSING) with open(key, "r") as kf: with open(cert, "r") as cf: register_auth(kf.read(), cf.read(), store, digest, serializer) registry._set_default_serializer("auth")
def _reset_celery_thread_pool(): # Send signal to restart thread pool app = current_app._get_current_object() app.control.broadcast( 'pool_restart', arguments={'reload': True}, destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False)
def __init__(self, model, Session, app=None, **kw): """Initialize the model entry.""" self.app = app or current_app._get_current_object() self.session = kw.get('session') self.Session = Session self.model = model self.name = model.name self.task = model.task try: self.schedule = model.schedule logger.debug('schedule: {}'.format(self.schedule)) except Exception as e: logger.error(e) logger.error( 'Disabling schedule %s that was removed from database', self.name, ) self._disable(model) try: self.args = loads(model.args or '[]') self.kwargs = loads(model.kwargs or '{}') except ValueError as exc: logger.exception( 'Removing schedule %s for argument deseralization error: %r', self.name, exc, ) self._disable(model) self.options = {} for option in [ 'queue', 'exchange', 'routing_key', 'expires', 'priority' ]: value = getattr(model, option) if value is None: continue self.options[option] = value self.total_run_count = model.total_run_count self.enabled = model.enabled if not model.last_run_at: model.last_run_at = self._default_now() self.last_run_at = model.last_run_at timezone = self.app.timezone # 因为从数据库读取的 last_run_at 可能没有时区信息,所以这里必须加上时区信息 self.last_run_at = maybe_make_aware(self.last_run_at, timezone) # self.options['expires'] 同理 if 'expires' in self.options: expires = self.options['expires'] self.options['expires'] = maybe_make_aware(expires, timezone)
def main( ctx, test, list_config, disable_ticker_messaging, log_ticker_minimal, task, queue, version, ): """ Use this tool to start all or part of the tasks. """ if list_config: if list_config == "currencies": import pprint click.echo(pprint.pprint(s.SYMBOLS_PER_EXCHANGE)) elif list_config == "exchanges": click.echo("\n".join(s.EXCHANGES)) ctx.exit() beat_kwargs = dict( disable_ticker_messaging=disable_ticker_messaging, log_ticker_minimal=log_ticker_minimal, ) if test: ticker.test(**beat_kwargs) ctx.exit() if version: from xtcryptosignals import __title__, __version__ click.echo("{} {}".format(__title__, __version__)) ctx.exit() from celery import current_app from celery.bin import worker app = current_app._get_current_object() app.config_from_object("xtcryptosignals.tasks.celeryconfig") # tasks passed by argument or default tasks = list(task) _prepare_celery_beat(app, tasks=tasks, beat_kwargs=beat_kwargs) _prepare_queue(app, tasks=tasks, queue=queue) worker = worker.worker(app=app) worker.run( beat=True, queues=[queue], schedule_filename=f"celerybeat-schedule-{queue}.db", loglevel=ticker.logging.INFO, )
def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._has_lock = False try: logger.debug('DS executing tick') self._mutex.acquire(wait=10) node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = Configuration.get('ovs.core.uniqueid') try: lock = self._persistent.get('{0}_lock'.format(self._namespace)) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen logger.debug('DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) logger.debug( 'DS stealing lock from {0}'.format(lock['name'])) self._load_schedule() self._has_lock = True else: logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} logger.debug('DS refreshing lock') self._persistent.set('{0}_lock'.format(self._namespace), lock) finally: self._mutex.release() if self._has_lock is True: logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due(entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval
def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._has_lock = False try: logger.debug('DS executing tick') self._mutex.acquire(wait=10) node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = System.get_my_machine_id() try: lock = self._persistent.get('{0}_lock'.format(self._namespace)) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen logger.debug('DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) logger.debug( 'DS stealing lock from {0}'.format(lock['name'])) self._load_schedule() self._has_lock = True else: logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} logger.debug('DS refreshing lock') self._persistent.set('{0}_lock'.format(self._namespace), lock) finally: self._mutex.release() if self._has_lock is True: logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due(entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval
def __init__(self, task, db): self._task = task self.db = db self.app = current_app._get_current_object() self.name = self._task["name"] self.task = self._task["task"] if "interval" in self._task: interval = self._task["interval"] if "period" in interval and "every" in interval: period = interval["period"] every = interval["every"] else: self._task["enabled"] = False return self.schedule = celery.schedules.schedule( datetime.timedelta(**{period: every})) if "crontab" in self._task: tab = self._task["crontab"] if "minute" in tab and "hour" in tab and "day_of_week" in tab and "day_of_month" in tab and "month_of_year" in tab: minute = tab["minute"] hour = tab["hour"] day_of_week = tab["day_of_week"] day_of_month = tab["day_of_month"] month_of_year = tab["month_of_year"] else: self._task["enabled"] = False return self.schedule = celery.schedules.crontab( minute=minute, hour=hour, day_of_week=day_of_week, day_of_month=day_of_month, month_of_year=month_of_year) self.args = self._task["args"] if "args" in self._task else "" self.kwargs = self._task["kwargs"] if "args" in self._task else "" self.options = { 'queue': self._task["queue"] if "queue" in self._task else "", 'exchange': self._task["exchange"] if "exchange" in self._task else "", 'routing_key': self._task["routing_key"] if "routing_key" in self._task else "", 'expires': self._task["expires"] if "expires" in self._task else None } if "total_run_count" not in self._task: self._task["total_run_count"] = 0 self.total_run_count = self._task["total_run_count"] if "last_run_at" not in self._task: self._task["last_run_at"] = self._default_now() self.last_run_at = self._task["last_run_at"]
def handle(self, *args, **options): periodic_task_id = options['periodic-task-id'] ignore_result = options['ignore_result'] if options['timeout']: timeout = options['timeout'] else: timeout = None def confirm(prompt): if options['confirm']: return True try: return input("%s [y/n] > " % (prompt, )).lower() == "y" except KeyboardInterrupt: raise CommandError("Please confirm the question.") try: periodic_task = PeriodicTask.objects.get(pk=periodic_task_id) except PeriodicTask.DoesNotExist: raise CommandError("PeriodicTask with id %s does not exist." % (periodic_task_id)) if periodic_task.last_run_at: msg = ("The task %s was last run on %s\n" "Are you sure you want to resubmit this periodic task?" % (self.style.NOTICE(periodic_task), self.style.NOTICE(periodic_task.last_run_at))) else: msg = ("The task %s has never been run before.\n" "Are you sure you want to submit this periodic task?" % (self.style.NOTICE(periodic_task), )) if not confirm(msg): raise CommandError( 'Please confirm as you need to know what you are doing.') from celery import current_app model_entry = ModelEntry(periodic_task) app = current_app._get_current_object() task = app.tasks.get(model_entry.task) if task: async_result = task.apply_async(model_entry.args, model_entry.kwargs, **model_entry.options) else: async_result = app.send_task(model_entry.task, model_entry.args, model_entry.kwargs, **model_entry.options) periodic_task.last_run_at = app.now() periodic_task.save() if not ignore_result: result = async_result.get(timeout=timeout) self.stdout.write(result)
def __init__(self, *args, **kwargs): self._schedule = None self._has_made_initial_read = False self.app = kwargs.get('app', current_app._get_current_object()) self.PeriodicTask = import_from_string( self.app.conf['CELERYSTORE_PERIODIC_TASK']) self.TaskSchedule = import_from_string( self.app.conf['CELERYSTORE_TASK_SCHEDULE']) self._latest_change = self.PeriodicTask.get_latest_change_to_schedule() super(StoreScheduler, self).__init__(*args, **kwargs)
def __init__(self, **kwargs): super(CaerusEntry, self).__init__() attrs = self.__dict__.keys() self.app = kwargs.pop('app', current_app._get_current_object()) for akey, aval in kwargs.items(): setattr(self, akey, aval) if self.last_run_at is None: self.last_run_at = self._default_now()
def _trace_task_ret(name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request): app = app or current_app._get_current_object() accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) args, kwargs = loads(body, content_type, content_encoding, accept=accept) request.update(args=args, kwargs=kwargs, **extra_request) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T)
def run_celery_server(): # celery worker -l debug -A tasks --beat application = current_app._get_current_object() worker_process = worker.worker(app=application) options = { 'broker': 'amqp://super@localhost//', 'loglevel': 'INFO', 'traceback': False, } worker_process.run(**options)
def main(options={}): """ Be a worker, with the given options as if present on the command line. """ app = current_app._get_current_object() wrkr = worker.worker(app=app) opts = copy.copy(default_options) opts.update(options) wrkr.run(**opts)
def startWorker(self): from celery import current_app from celery.bin import worker application = current_app._get_current_object() worker = worker.worker(app=application) options = { 'broker': KrakenConfiguration.restApi.config['CELERY_BROKER_URL'], 'loglevel': 'INFO', 'traceback': True, } worker.run(**options)
def get_event_store(): """Get or set the current event store.""" app = current_app._get_current_object() if not hasattr(Config, 'DEFAULT_BACKEND'): return None if hasattr(app, 'event_store'): return app.event_store else: app.event_store = EventStore(Config.DEFAULT_BACKEND) return app.event_store
def __call__(self, app=None, *args, **kwargs): a = current_app._get_current_object() w = worker.worker(app=a) options = { 'loglevel': 'INFO', 'concurrency': 2, 'without-gossip': True } options.update(kwargs) w.run(*args, **options)
def submit_urls(urls, contributor): celery_app = current_app._get_current_object() assert celery_app.conf.GITHUB_USERNAME is not None assert celery_app.conf.GITHUB_PASSWORD is not None github = Github(celery_app.conf.GITHUB_USERNAME, celery_app.conf.GITHUB_PASSWORD) git = Git(TEST_LISTS_REPO) git.pull(["-u", celery_app.conf.TARGET_REPO.split("/")[0], "master"]) branch_name = ("user-contribution/{0}".format( datetime.now().strftime("%Y%m%dT%H%M%S") )) git.checkout(["-b", branch_name]) try: country_names = set() for url in urls: country_code = url['country'].upper() if (country_code not in country_codes.keys() and country_code != "GLOBAL"): raise Exception("Invalid country code '{0}'".format(country_code)) if country_code == "GLOBAL": country_names.add("Global") else: country_names.add(country_codes[country_code]) append_url_to_csv( country_code=url['country'].lower(), url=url['url'], category_code=url['category'], source=contributor, notes=url['notes'] ) for untracked_file in git.list_untracked(): git.add([untracked_file]) commit_title = "Add user contributed URLs for countries {0}".format( ' '.join(list(country_names)) ) commit_msg = "This Pull Request was automatically generated from the web" git.add(["."]) git.commit([ "-m", commit_title ]) git.push(["-u", "origin", branch_name]) repo = github.get_repo(celery_app.conf.TARGET_REPO) head = "{0}:{1}".format(celery_app.conf.GITHUB_USERNAME, branch_name) print "Repo %s" % celery_app.conf.TARGET_REPO print "Head %s" % head repo.create_pull(commit_title, commit_msg, head=head, base="master") finally: git.checkout(["master"])
def __init__(self, task): self._task = task self.app = current_app._get_current_object() self.name = self._task['name'] self.task = TASK self.is_active = self._task['is_active'] self.schedule = celery.schedules.schedule(datetime.timedelta(seconds=self._task['params']['interval'])) self.options = {} self.args = (self._task['params']['url'],) self.total_run_count = self._task['total_run_count'] self.last_run_at = self._task['last_run_at']
def start_celery_worker(): from celery import current_app from celery.bin import worker celery_app = current_app._get_current_object() worker = worker.worker(app=celery_app) options = { 'broker': app.config['CELERY_BROKER_URL'], 'loglevel': 'INFO', 'traceback': True } worker.run(**options)
def __init__(self, task,db): self._task = task self.db=db self.app = current_app._get_current_object() self.name = self._task["name"] self.task = self._task["task"] if "interval" in self._task: interval= self._task["interval"] if "period" in interval and "every" in interval: period=interval["period"] every= interval["every"] else: self._task["enabled"]=False return self.schedule = celery.schedules.schedule(datetime.timedelta(**{period: every})) if "crontab" in self._task: tab= self._task["crontab"] if "minute" in tab and "hour" in tab and "day_of_week" in tab and "day_of_month" in tab and "month_of_year" in tab: minute=tab["minute"] hour= tab["hour"] day_of_week= tab["day_of_week"] day_of_month= tab["day_of_month"] month_of_year= tab["month_of_year"] else: self._task["enabled"]=False return self.schedule = celery.schedules.crontab(minute=minute, hour=hour, day_of_week=day_of_week, day_of_month=day_of_month, month_of_year=month_of_year) self.args = self._task["args"] if "args" in self._task else "" self.kwargs = self._task["kwargs"] if "args" in self._task else "" self.options = { 'queue': self._task["queue"] if "queue" in self._task else "" , 'exchange': self._task["exchange"] if "exchange" in self._task else "", 'routing_key': self._task["routing_key"] if "routing_key" in self._task else "", 'expires': self._task["expires"] if "expires" in self._task else None } if "total_run_count" not in self._task: self._task["total_run_count"] = 0 self.total_run_count = self._task["total_run_count"] if "last_run_at" not in self._task: self._task["last_run_at"] = self._default_now() self.last_run_at = self._task["last_run_at"]
def _trace_task_ret( name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request ): app = app or current_app._get_current_object() embed = None if content_type: accept = prepare_accept_content(app.conf.accept_content) args, kwargs, embed = loads(body, content_type, content_encoding, accept=accept) else: args, kwargs, embed = body hostname = gethostname() request.update({"args": args, "kwargs": kwargs, "hostname": hostname, "is_eager": False}, **embed or {}) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T)
def app(env_config, default_config): """Flask application fixture.""" app = create_app(**default_config) # FIXME: Needs fixing flask_celeryext, # which once creates the first celery app, the flask_app that is set # is never released from the global state, even if you create a new # celery application. We need to unset the "flask_app" manually. from celery import current_app as cca cca = cca._get_current_object() delattr(cca, "flask_app") celery_app = create_celery_app(app) celery_app.set_current() with app.app_context(): yield app
def __init__(self, svc_model=None, task_model=None): self.app = current_app._get_current_object() self.svc_model = svc_model self.task_model = task_model self.options = {} self.name = task_model.Name self.args = [svc_model.Id, task_model.Command] self.schedule = crontab(*task_model.Schedule.split()) self.total_run_count = task_model.TotalRunCount or 0 task_model.LastRunAt = task_model.LastRunAt or "0001-01-01T00:00:00Z" if isinstance(task_model.LastRunAt, basestring): task_model.LastRunAt = parser.parse(task_model.LastRunAt) self.last_run_at = task_model.LastRunAt
def __init__(self, model, app=None): """Initialize the model entry.""" self.app = app or current_app._get_current_object() self.name = model.name self.task = model.task try: self.schedule = model.schedule except model.DoesNotExist: logger.error( 'Disabling schedule %s that was removed from database', self.name, ) self._disable(model) try: self.args = loads(model.args or '[]') self.kwargs = loads(model.kwargs or '{}') except ValueError as exc: logger.exception( 'Removing schedule %s for argument deseralization error: %r', self.name, exc, ) self._disable(model) self.options = {} for option in ['queue', 'exchange', 'routing_key', 'expires', 'priority']: value = getattr(model, option) if value is None: continue self.options[option] = value self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() last_run_at = model.last_run_at if getattr(settings, 'DJANGO_CELERY_BEAT_TZ_AWARE', True): last_run_at = make_aware(last_run_at) self.last_run_at = last_run_at
def _trace_task_ret(name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request): app = app or current_app._get_current_object() embed = None if content_type: accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) args, kwargs, embed = loads( body, content_type, content_encoding, accept=accept, ) else: args, kwargs = body hostname = socket.gethostname() request.update({ 'args': args, 'kwargs': kwargs, 'hostname': hostname, 'is_eager': False, }, **embed or {}) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T)
def __init__(self, db_task): self.db_task = db_task self.app = current_app._get_current_object() self.name = db_task.name self.task = db_task.task self.enabled = db_task.enabled self.schedule = celery.schedules.crontab(minute=db_task.crontab_minute, hour=db_task.crontab_hour, day_of_week=db_task.crontab_day_of_week, day_of_month=db_task.crontab_day_of_month, month_of_year=db_task.crontab_month_of_year) self.args = db_task.args self.kwargs = db_task.kwargs self.options = {} if not self.db_task.last_run_at: self.db_task.last_run_at = self._default_now() self.last_run_at = self.db_task.last_run_at
def __init__(self, model): self.app = current_app._get_current_object() self.name = model.name self.task = model.task self.schedule = model.schedule self.args = model.args self.kwargs = model.kwargs self.options = dict( queue=model.queue, exchange=model.exchange, routing_key=model.routing_key, expires=model.expires, ) self.total_run_count = model.total_run_count self.model = model if not model.last_run_at: model.last_run_at = self._default_now() orig = self.last_run_at = model.last_run_at if not is_naive(self.last_run_at): self.last_run_at = self.last_run_at.replace(tzinfo=None) assert orig.hour == self.last_run_at.hour # timezone sanity
def execute_and_trace(name, uuid, args, kwargs, request=None, **opts): """This is a pickleable method used as a target when applying to pools. It's the same as:: >>> trace_task(name, *args, **kwargs)[0] """ global _current_app_for_proc if _current_app_for_proc is None: _current_app_for_proc = current_app._get_current_object() task = _current_app_for_proc.tasks[name] try: hostname = opts.get("hostname") setps("celeryd", name, hostname, rate_limit=True) try: if task.__tracer__ is None: task.__tracer__ = build_tracer(name, task, **opts) return task.__tracer__(uuid, args, kwargs, request)[0] finally: setps("celeryd", "-idle-", hostname, rate_limit=True) except Exception, exc: return report_internal_error(task, exc)
def register_task(name, task, schedule, args=[], kwargs={}, relative=False, options={}, celery_app=None): if not isinstance(schedule, (schedules.crontab, schedules.schedule)): raise TypeError if celery_app is None: celery_app = current_app._get_current_object() if task not in celery_app.tasks: raise ValueError('task \'%s\' is not in %s' % (task, celery_app)) models.session.begin() try: entry = CaerusEntry( name=name, task=task, args=args, kwargs=kwargs, relative=relative, options=options, schedule=schedule) models.session.add(entry) CaerusEntryMeta.set_latest_modified(celery_app.now()) models.session.commit() except Exception, e: models.session.rollback() raise
def unpickle_backend(cls, args, kwargs): """Return an unpickled backend.""" return cls(*args, app=current_app._get_current_object(), **kwargs)
def test_AsyncResult(self): x = self.app.AsyncResult('1') self.assertIs(x.app, self.app) r = loads(dumps(x)) # not set as current, so ends up as default app after reduce self.assertIs(r.app, current_app._get_current_object())