def __init__(self, task):
self._task = task
self.app = current_app._get_current_object()
self.name = self._task.name
self.task = self._task.task
self.schedule = self._task.schedule
self.args = self._task.args
self.kwargs = self._task.kwargs
self.options = {
"queue": self._task.queue,
"exchange": self._task.exchange,
"routing_key": self._task.routing_key,
"expires": self._task.expires,
"soft_time_limit": self._task.soft_time_limit,
"enabled": self._task.enabled,
}
if self._task.total_run_count is None:
self._task.total_run_count = 0
self.total_run_count = self._task.total_run_count
if not self._task.last_run_at:
self._task.last_run_at = self._default_now()
self.last_run_at = self._task.last_run_at
def __init__(self, model, app=None):
"""Initialize the model entry."""
self.app = app or current_app._get_current_object()
self.name = "{}_{}".format(model.name, model.pk)
self.task = model.task
self.args = model.args
self.kwargs = model.kwargs
try:
self.schedule = model.schedule
except model.DoesNotExist:
logger.error(
"Disabling schedule %s that was removed from database",
self.name,
)
self._disable(model)
self.options = {}
if model.queue:
self.options["queue"] = model.queue
self.options["headers"] = {}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
self.last_run_at = model.last_run_at
def __init__(self, model):
self.app = current_app._get_current_object()
self.name = model.name
self.task = model.task
self.schedule = model.schedule
try:
self.args = loads(model.args or "[]")
self.kwargs = loads(model.kwargs or "{}")
except ValueError:
logging.error("Failed to serialize arguments for %s.", self.name, exc_info=1)
logging.warning("Disabling %s", self.name)
model.no_changes = True
model.enabled = False
model.save()
self.options = {
"queue": model.queue,
"exchange": model.exchange,
"routing_key": model.routing_key,
"expires": model.expires,
}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
orig = self.last_run_at = model.last_run_at
if not is_naive(self.last_run_at):
self.last_run_at = self.last_run_at.replace(tzinfo=None)
assert orig.hour == self.last_run_at.hour # timezone sanity
def __init__(self, model):
self.app = current_app._get_current_object()
self.name = model.name
self.task = model.task
try:
self.schedule = model.schedule
except model.DoesNotExist:
logger.error('Schedule was removed from database')
logger.warning('Disabling %s', self.name)
self._disable(model)
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError:
logging.error('Failed to serialize arguments for %s.',
self.name,
exc_info=1)
logging.warning('Disabling %s', self.name)
self._disable(model)
self.options = {
'queue': model.queue,
'exchange': model.exchange,
'routing_key': model.routing_key,
'expires': model.expires
}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
orig = self.last_run_at = model.last_run_at
if not is_naive(self.last_run_at):
self.last_run_at = self.last_run_at.replace(tzinfo=None)
assert orig.hour == self.last_run_at.hour # timezone sanity
def __init__(self, model):
self.app = current_app._get_current_object()
self.name = model.name
self.task = model.task
self.schedule = model.schedule
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError:
logging.error('Failed to serialize arguments for %s.',
self.name,
exc_info=1)
logging.warning('Disabling %s', self.name)
model.no_changes = True
model.enabled = False
model.save()
expires = model.expires or model.expires_at
self.options = {
'queue': model.queue,
'exchange': model.exchange,
'routing_key': model.routing_key,
'expires': expires
}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
orig = self.last_run_at = model.last_run_at
if not is_naive(self.last_run_at):
self.last_run_at = self.last_run_at.replace(tzinfo=None)
assert orig.hour == self.last_run_at.hour # timezone sanity
def rabbitmq_queue_size():
"""Returns the rabbitmq queue size.
Two things to know about the queue size:
1. It's not 100% accurate, but the size is generally near that
number
2. I can't think of a second thing, but that first thing is
pretty important.
"""
# FIXME: 2015-04-23: This is busted.
from celery import current_app
# FIXME: This uses a private method, but I'm not sure how else to
# figure this out, either.
app = current_app._get_current_object()
conn = app.connection()
chan = conn.default_channel
# FIXME: This hard-codes the exchange, but I'm not sure how else
# to figure it out.
queue = chan.queue_declare('celery', passive=True)
return queue.message_count
def setup_security(allowed_serializers=None, key=None, cert=None, store=None,
digest='sha1', serializer='json', app=None):
"""See :meth:`@Celery.setup_security`."""
if app is None:
from celery import current_app
app = current_app._get_current_object()
disable_untrusted_serializers(allowed_serializers)
conf = app.conf
if conf.CELERY_TASK_SERIALIZER != 'auth':
return
try:
from OpenSSL import crypto # noqa
except ImportError:
raise ImproperlyConfigured(SSL_NOT_INSTALLED)
key = key or conf.CELERY_SECURITY_KEY
cert = cert or conf.CELERY_SECURITY_CERTIFICATE
store = store or conf.CELERY_SECURITY_CERT_STORE
if not (key and cert and store):
raise ImproperlyConfigured(SETTING_MISSING)
with open(key) as kf:
with open(cert) as cf:
register_auth(kf.read(), cf.read(), store, digest, serializer)
registry._set_default_serializer('auth')
def __new__(cls, name, bases, attrs):
new = super(TaskType, cls).__new__
task_module = attrs.get('__module__') or '__main__'
# - Abstract class: abstract attribute should not be inherited.
abstract = attrs.pop('abstract', None)
if abstract or not attrs.get('autoregister', True):
return new(cls, name, bases, attrs)
# The 'app' attribute is now a property, with the real app located
# in the '_app' attribute. Previously this was a regular attribute,
# so we should support classes defining it.
app = attrs.pop('_app', None) or attrs.pop('app', None)
# Attempt to inherit app from one the bases
if not isinstance(app, Proxy) and app is None:
for base in bases:
if getattr(base, '_app', None):
app = base._app
break
else:
app = current_app._get_current_object()
attrs['_app'] = app
# - Automatically generate missing/empty name.
task_name = attrs.get('name')
if not task_name:
attrs['name'] = task_name = gen_task_name(app, name, task_module)
if not attrs.get('_decorated'):
# non decorated tasks must also be shared in case
# an app is created multiple times due to modules
# imported under multiple names.
# Hairy stuff, here to be compatible with 2.x.
# People should not use non-abstract task classes anymore,
# use the task decorator.
from celery._state import connect_on_app_finalize
unique_name = '.'.join([task_module, name])
if unique_name not in cls._creation_count:
# the creation count is used as a safety
# so that the same task is not added recursively
# to the set of constructors.
cls._creation_count[unique_name] = 1
connect_on_app_finalize(_CompatShared(
unique_name,
lambda app: TaskType.__new__(cls, name, bases,
dict(attrs, _app=app)),
))
# - Create and register class.
# Because of the way import happens (recursively)
# we may or may not be the first time the task tries to register
# with the framework. There should only be one class for each task
# name, so we always return the registered version.
tasks = app._tasks
if task_name not in tasks:
tasks.register(new(cls, name, bases, attrs))
instance = tasks[task_name]
instance.bind(app)
return instance.__class__
def __init__(self, model):
self.app = current_app._get_current_object()
self.name = model.name
self.task = model.task
try:
self.schedule = model.schedule
except model.DoesNotExist:
logger.error('Schedule was removed from database')
logger.warning('Disabling %s', self.name)
self._disable(model)
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError:
logging.error('Failed to serialize arguments for %s.', self.name,
exc_info=1)
logging.warning('Disabling %s', self.name)
self._disable(model)
self.options = {'queue': model.queue,
'exchange': model.exchange,
'routing_key': model.routing_key,
'expires': model.expires}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
orig = self.last_run_at = model.last_run_at
if not is_naive(self.last_run_at):
self.last_run_at = self.last_run_at.replace(tzinfo=None)
assert orig.hour == self.last_run_at.hour # timezone sanity
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._logger.debug('DS executing tick')
try:
self._has_lock = False
with self._mutex:
# noinspection PyProtectedMember
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = System.get_my_machine_id()
try:
lock = self._persistent.get(self._lock_name)
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
self._logger.debug('DS there was no lock in tick')
else:
if lock['name'] == node_name:
# The current node holds the lock
self._logger.debug('DS keeps own lock')
self._has_lock = True
elif node_timestamp - lock[
'timestamp'] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
self._logger.debug(
'DS last lock refresh is {0}s old'.format(
node_timestamp - lock['timestamp']))
self._logger.debug('DS stealing lock from {0}'.format(
lock['name']))
self._load_schedule()
self._has_lock = True
else:
self._logger.debug('DS lock is not ours')
if self._has_lock is True:
lock = {'name': node_name, 'timestamp': node_timestamp}
self._logger.debug('DS refreshing lock')
self._persistent.set(self._lock_name, lock)
if self._has_lock is True:
self._logger.debug('DS executing tick workload')
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(
entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
self._logger.debug('DS executing tick workload - done')
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
except Exception as ex:
self._logger.debug('DS got error during tick: {0}'.format(ex))
return self.max_interval
def app(env_config, default_config):
"""Flask application fixture."""
app = create_app(**default_config)
# FIXME: Needs fixing flask_celeryext,
# which once creates the first celery app, the flask_app that is set
# is never released from the global state, even if you create a new
# celery application. We need to unset the "flask_app" manually.
from celery import current_app as cca
cca = cca._get_current_object()
delattr(cca, "flask_app")
celery_app = create_celery_app(app)
# FIXME: When https://github.com/inveniosoftware/flask-celeryext/issues/35
# is closed and Flask-CeleryExt is released, this can be removed.
class _TestAppContextTask(Task):
abstract = True
def __call__(self, *args, **kwargs):
if flask_current_app:
return Task.__call__(self, *args, **kwargs)
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
celery_app.Task = _TestAppContextTask
celery_app.set_current()
with app.app_context():
yield app
def __init__(self, task):
self._task = task
self.app = current_app._get_current_object()
self.name = self._task.name
self.task = self._task.task
self.schedule = self._task.schedule
self.args = self._task.args
self.kwargs = self._task.kwargs
self.options = {
'queue': self._task.queue,
'exchange': self._task.exchange,
'routing_key': self._task.routing_key,
'expires': self._task.expires,
'soft_time_limit': self._task.soft_time_limit,
'enabled': self._task.enabled
}
if self._task.total_run_count is None:
self._task.total_run_count = 0
self.total_run_count = self._task.total_run_count
if not self._task.last_run_at:
self._task.last_run_at = self._default_now()
self.last_run_at = self._task.last_run_at
def __new__(cls, name, bases, attrs):
new = super(TaskType, cls).__new__
task_module = attrs.get('__module__') or '__main__'
# The 'app' attribute is now a property, with the real app located
# in the '_app' attribute. Previously this was a regular attribute,
# so we should support classes defining it.
app = attrs.pop('_app', None) or attrs.pop('app', None)
# Attempt to inherit app from one the bases
if not isinstance(app, Proxy) and app is None:
for base in bases:
if getattr(base, '_app', None):
app = base._app
break
else:
app = current_app._get_current_object()
attrs['_app'] = app
# - Automatically generate missing/empty name.
task_name = attrs.get('name')
if not task_name:
attrs['name'] = task_name = gen_task_name(app, name, task_module)
# - Create and register class.
# Because of the way import happens (recursively)
# we may or may not be the first time the task tries to register
# with the framework. There should only be one class for each task
# name, so we always return the registered version.
tasks = app._tasks
if task_name not in tasks:
tasks.register(new(cls, name, bases, attrs))
instance = tasks[task_name]
instance.bind(app)
return instance.__class__
def __init__(self, task):
self._task = task
self.app = current_app._get_current_object()
self.name = self._task.key # passing key here as the task name is a human use only field.
self.task = self._task.task
self.schedule = self._task.schedule
self.args = self._task.args
self.kwargs = self._task.kwargs
self.options = {
'queue': self._task.queue,
'exchange': self._task.exchange,
'routing_key': self._task.routing_key,
'expires': self._task.expires
}
if not self._task.total_run_count:
self._task.total_run_count = 0
self.total_run_count = self._task.total_run_count
if not self._task.last_run_at:
# subtract some time from the current time to populate the last time
# that the task was run so that a newly scheduled task does not get missed
time_subtract = (self.app.conf.CELERYBEAT_MAX_LOOP_INTERVAL or 30)
self._task.last_run_at = self._default_now() - datetime.timedelta(seconds=time_subtract)
self.save()
self.last_run_at = self._task.last_run_at
def __init__(self, model, app=None):
self.app = app or current_app._get_current_object()
self.name = model.name
self.task = model.task
try:
self.schedule = model.schedule
except model.DoesNotExist:
logger.error(
'Disabling schedule %s that was removed from database',
self.name,
)
self._disable(model)
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError as exc:
logger.exception(
'Removing schedule %s for argument deseralization error: %r',
self.name,
exc,
)
self._disable(model)
self.options = {
'queue': model.queue,
'exchange': model.exchange,
'routing_key': model.routing_key,
'expires': model.expires,
}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
self.last_run_at = make_aware(model.last_run_at)
def setup_security(allowed_serializers=None,
key=None,
cert=None,
store=None,
digest='sha1',
serializer='json',
app=None):
"""See :meth:`@Celery.setup_security`."""
if app is None:
from celery import current_app
app = current_app._get_current_object()
disable_untrusted_serializers(allowed_serializers)
conf = app.conf
if conf.CELERY_TASK_SERIALIZER != 'auth':
return
try:
from OpenSSL import crypto # noqa
except ImportError:
raise ImproperlyConfigured(SSL_NOT_INSTALLED)
key = key or conf.CELERY_SECURITY_KEY
cert = cert or conf.CELERY_SECURITY_CERTIFICATE
store = store or conf.CELERY_SECURITY_CERT_STORE
if not (key and cert and store):
raise ImproperlyConfigured(SETTING_MISSING)
with open(key) as kf:
with open(cert) as cf:
register_auth(kf.read(), cf.read(), store, digest, serializer)
registry._set_default_serializer('auth')
def app(env_config, default_config):
"""Flask application fixture."""
app = create_app(**default_config)
# FIXME: Needs fixing flask_celeryext,
# which once creates the first celery app, the flask_app that is set
# is never released from the global state, even if you create a new
# celery application. We need to unset the "flask_app" manually.
from celery import current_app as cca
cca = cca._get_current_object()
delattr(cca, "flask_app")
celery_app = create_celery_app(app)
# FIXME: When https://github.com/inveniosoftware/flask-celeryext/issues/35
# is closed and Flask-CeleryExt is released, this can be removed.
class _TestAppContextTask(Task):
abstract = True
def __call__(self, *args, **kwargs):
if flask_current_app:
return Task.__call__(self, *args, **kwargs)
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
celery_app.Task = _TestAppContextTask
celery_app.set_current()
with app.app_context():
yield app
def __init__(self, model):
self.app = current_app._get_current_object()
self.name = model.name
self.task = model.task
self.schedule = model.schedule
try:
self.args = deserialize(model.args or u'[]')
self.kwargs = deserialize(model.kwargs or u'{}')
except ValueError:
logging.error('Failed to serialize arguments for %s.', self.name,
exc_info=1)
logging.warning('Disabling %s', self.name)
model.no_changes = True
model.enabled = False
model.save()
self.options = {'queue': model.queue,
'exchange': model.exchange,
'routing_key': model.routing_key,
'expires': model.expires}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
orig = self.last_run_at = model.last_run_at
if not is_naive(self.last_run_at):
self.last_run_at = self.last_run_at.replace(tzinfo=None)
assert orig.hour == self.last_run_at.hour # timezone sanity
def setup_security(allowed_serializers=None, key=None, cert=None, store=None,
digest=None, serializer='json', app=None):
"""See :meth:`@Celery.setup_security`."""
if app is None:
from celery import current_app
app = current_app._get_current_object()
_disable_insecure_serializers(allowed_serializers)
# check conf for sane security settings
conf = app.conf
if conf.task_serializer != 'auth' or conf.accept_content != ['auth']:
raise ImproperlyConfigured(SETTING_MISSING)
key = key or conf.security_key
cert = cert or conf.security_certificate
store = store or conf.security_cert_store
digest = digest or conf.security_digest
if not (key and cert and store):
raise ImproperlyConfigured(SECURITY_SETTING_MISSING)
with open(key, 'r') as kf:
with open(cert, 'r') as cf:
register_auth(kf.read(), cf.read(), store, digest, serializer)
registry._set_default_serializer('auth')
def __init__(self, model):
self.app = current_app._get_current_object()
self.name = model.name
self.task = model.task
self.schedule = model.schedule
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError:
# disable because of error deserializing args/kwargs
model.no_changes = True
model.enabled = False
model.save()
raise
self.options = {'queue': model.queue,
'exchange': model.exchange,
'routing_key': model.routing_key,
'expires': model.expires}
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
orig = self.last_run_at = model.last_run_at
if not is_naive(self.last_run_at):
self.last_run_at = self.last_run_at.replace(tzinfo=None)
assert orig.hour == self.last_run_at.hour # timezone sanity
def __init__(self, task):
self._task = task
self.app = current_app._get_current_object()
self.name = self._task['name']
self.task = self._task['name']
# Fill out schedule
if self._task['schedule_type'] == 'crontab':
self.schedule = celery.schedules.crontab(
minute=self._task['crontab']['minute'],
hour=self._task['crontab']['hour'],
day_of_week=self._task['crontab']['day_of_week'],
day_of_month=self._task['crontab']['day_of_month'],
month_of_year=self._task['crontab']['month_of_year'])
elif self._task['schedule_type'] == 'interval':
self.schedule = celery.schedules.schedule(
datetime.timedelta(
**{
self._task['interval']['period']:
self._task['interval']['every']
}))
self.args = self._task['args']
self.kwargs = self._task['kwargs']
self.options = {'enabled': self._task['enabled']}
if 'last_run_at' not in self._task:
self._task['last_run_at'] = self._default_now()
self.last_run_at = toUTC(self._task['last_run_at'])
if 'run_immediately' not in self._task:
self._task['run_immediately'] = False
def setup_security(allowed_serializers=None, key=None, cert=None, store=None,
digest='sha1', serializer='json', app=None):
"""See :meth:`@Celery.setup_security`."""
if app is None:
from celery import current_app
app = current_app._get_current_object()
_disable_insecure_serializers(allowed_serializers)
conf = app.conf
if conf.task_serializer != 'auth':
return
try:
from OpenSSL import crypto # noqa
except ImportError:
raise ImproperlyConfigured(SSL_NOT_INSTALLED)
key = key or conf.security_key
cert = cert or conf.security_certificate
store = store or conf.security_cert_store
if not (key and cert and store):
raise ImproperlyConfigured(SETTING_MISSING)
with open(key) as kf:
with open(cert) as cf:
register_auth(kf.read(), cf.read(), store, digest, serializer)
registry._set_default_serializer('auth')
def __init__(self, task):
"""Initialize the task."""
self._task = task
self.app = current_app._get_current_object()
self.name = self._task.name
self.task = self._task.task
self.schedule = self._task.schedule
self.args = self._task.args
self.kwargs = self._task.kwargs
# self.options = {
# 'queue': self._task.queue,
# 'exchange': self._task.exchange,
# 'routing_key': self._task.routing_key,
# 'expires': self._task.expires,
# 'soft_time_limit': self._task.soft_time_limit
# }
if self._task.total_run_count is None:
self._task.total_run_count = 0
self.total_run_count = self._task.total_run_count
if not self._task.last_run_at:
self._task.last_run_at = self._default_now()
self.last_run_at = self._task.last_run_at
def setup_security(
allowed_serializers=None,
key=None,
cert=None,
store=None,
digest=None,
serializer="json",
app=None,
):
"""See :meth:`@Celery.setup_security`."""
if app is None:
from celery import current_app
app = current_app._get_current_object()
_disable_insecure_serializers(allowed_serializers)
# check conf for sane security settings
conf = app.conf
if conf.task_serializer != "auth" or conf.accept_content != ["auth"]:
raise ImproperlyConfigured(SETTING_MISSING)
key = key or conf.security_key
cert = cert or conf.security_certificate
store = store or conf.security_cert_store
digest = digest or conf.security_digest
if not (key and cert and store):
raise ImproperlyConfigured(SECURITY_SETTING_MISSING)
with open(key, "r") as kf:
with open(cert, "r") as cf:
register_auth(kf.read(), cf.read(), store, digest, serializer)
registry._set_default_serializer("auth")
def _reset_celery_thread_pool():
# Send signal to restart thread pool
app = current_app._get_current_object()
app.control.broadcast(
'pool_restart',
arguments={'reload': True},
destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)],
reply=False)
def __init__(self, model, Session, app=None, **kw):
"""Initialize the model entry."""
self.app = app or current_app._get_current_object()
self.session = kw.get('session')
self.Session = Session
self.model = model
self.name = model.name
self.task = model.task
try:
self.schedule = model.schedule
logger.debug('schedule: {}'.format(self.schedule))
except Exception as e:
logger.error(e)
logger.error(
'Disabling schedule %s that was removed from database',
self.name,
)
self._disable(model)
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError as exc:
logger.exception(
'Removing schedule %s for argument deseralization error: %r',
self.name,
exc,
)
self._disable(model)
self.options = {}
for option in [
'queue', 'exchange', 'routing_key', 'expires', 'priority'
]:
value = getattr(model, option)
if value is None:
continue
self.options[option] = value
self.total_run_count = model.total_run_count
self.enabled = model.enabled
if not model.last_run_at:
model.last_run_at = self._default_now()
self.last_run_at = model.last_run_at
timezone = self.app.timezone
# 因为从数据库读取的 last_run_at 可能没有时区信息，所以这里必须加上时区信息
self.last_run_at = maybe_make_aware(self.last_run_at, timezone)
# self.options['expires'] 同理
if 'expires' in self.options:
expires = self.options['expires']
self.options['expires'] = maybe_make_aware(expires, timezone)
def main(
ctx,
test,
list_config,
disable_ticker_messaging,
log_ticker_minimal,
task,
queue,
version,
):
"""
Use this tool to start all or part of the tasks.
"""
if list_config:
if list_config == "currencies":
import pprint
click.echo(pprint.pprint(s.SYMBOLS_PER_EXCHANGE))
elif list_config == "exchanges":
click.echo("\n".join(s.EXCHANGES))
ctx.exit()
beat_kwargs = dict(
disable_ticker_messaging=disable_ticker_messaging,
log_ticker_minimal=log_ticker_minimal,
)
if test:
ticker.test(**beat_kwargs)
ctx.exit()
if version:
from xtcryptosignals import __title__, __version__
click.echo("{} {}".format(__title__, __version__))
ctx.exit()
from celery import current_app
from celery.bin import worker
app = current_app._get_current_object()
app.config_from_object("xtcryptosignals.tasks.celeryconfig")
# tasks passed by argument or default
tasks = list(task)
_prepare_celery_beat(app, tasks=tasks, beat_kwargs=beat_kwargs)
_prepare_queue(app, tasks=tasks, queue=queue)
worker = worker.worker(app=app)
worker.run(
beat=True,
queues=[queue],
schedule_filename=f"celerybeat-schedule-{queue}.db",
loglevel=ticker.logging.INFO,
)
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._has_lock = False
try:
logger.debug('DS executing tick')
self._mutex.acquire(wait=10)
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = Configuration.get('ovs.core.uniqueid')
try:
lock = self._persistent.get('{0}_lock'.format(self._namespace))
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
logger.debug('DS there was no lock in tick')
else:
if lock['name'] == node_name:
# The current node holds the lock
logger.debug('DS keeps own lock')
self._has_lock = True
elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
logger.debug('DS last lock refresh is {0}s old'.format(
node_timestamp - lock['timestamp']))
logger.debug(
'DS stealing lock from {0}'.format(lock['name']))
self._load_schedule()
self._has_lock = True
else:
logger.debug('DS lock is not ours')
if self._has_lock is True:
lock = {'name': node_name,
'timestamp': node_timestamp}
logger.debug('DS refreshing lock')
self._persistent.set('{0}_lock'.format(self._namespace), lock)
finally:
self._mutex.release()
if self._has_lock is True:
logger.debug('DS executing tick workload')
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
logger.debug('DS executing tick workload - done')
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._has_lock = False
try:
logger.debug('DS executing tick')
self._mutex.acquire(wait=10)
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = System.get_my_machine_id()
try:
lock = self._persistent.get('{0}_lock'.format(self._namespace))
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
logger.debug('DS there was no lock in tick')
else:
if lock['name'] == node_name:
# The current node holds the lock
logger.debug('DS keeps own lock')
self._has_lock = True
elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
logger.debug('DS last lock refresh is {0}s old'.format(
node_timestamp - lock['timestamp']))
logger.debug(
'DS stealing lock from {0}'.format(lock['name']))
self._load_schedule()
self._has_lock = True
else:
logger.debug('DS lock is not ours')
if self._has_lock is True:
lock = {'name': node_name,
'timestamp': node_timestamp}
logger.debug('DS refreshing lock')
self._persistent.set('{0}_lock'.format(self._namespace), lock)
finally:
self._mutex.release()
if self._has_lock is True:
logger.debug('DS executing tick workload')
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
logger.debug('DS executing tick workload - done')
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
def __init__(self, task, db):
self._task = task
self.db = db
self.app = current_app._get_current_object()
self.name = self._task["name"]
self.task = self._task["task"]
if "interval" in self._task:
interval = self._task["interval"]
if "period" in interval and "every" in interval:
period = interval["period"]
every = interval["every"]
else:
self._task["enabled"] = False
return
self.schedule = celery.schedules.schedule(
datetime.timedelta(**{period: every}))
if "crontab" in self._task:
tab = self._task["crontab"]
if "minute" in tab and "hour" in tab and "day_of_week" in tab and "day_of_month" in tab and "month_of_year" in tab:
minute = tab["minute"]
hour = tab["hour"]
day_of_week = tab["day_of_week"]
day_of_month = tab["day_of_month"]
month_of_year = tab["month_of_year"]
else:
self._task["enabled"] = False
return
self.schedule = celery.schedules.crontab(
minute=minute,
hour=hour,
day_of_week=day_of_week,
day_of_month=day_of_month,
month_of_year=month_of_year)
self.args = self._task["args"] if "args" in self._task else ""
self.kwargs = self._task["kwargs"] if "args" in self._task else ""
self.options = {
'queue':
self._task["queue"] if "queue" in self._task else "",
'exchange':
self._task["exchange"] if "exchange" in self._task else "",
'routing_key':
self._task["routing_key"] if "routing_key" in self._task else "",
'expires':
self._task["expires"] if "expires" in self._task else None
}
if "total_run_count" not in self._task:
self._task["total_run_count"] = 0
self.total_run_count = self._task["total_run_count"]
if "last_run_at" not in self._task:
self._task["last_run_at"] = self._default_now()
self.last_run_at = self._task["last_run_at"]
def handle(self, *args, **options):
periodic_task_id = options['periodic-task-id']
ignore_result = options['ignore_result']
if options['timeout']:
timeout = options['timeout']
else:
timeout = None
def confirm(prompt):
if options['confirm']:
return True
try:
return input("%s [y/n] > " % (prompt, )).lower() == "y"
except KeyboardInterrupt:
raise CommandError("Please confirm the question.")
try:
periodic_task = PeriodicTask.objects.get(pk=periodic_task_id)
except PeriodicTask.DoesNotExist:
raise CommandError("PeriodicTask with id %s does not exist." %
(periodic_task_id))
if periodic_task.last_run_at:
msg = ("The task %s was last run on %s\n"
"Are you sure you want to resubmit this periodic task?" %
(self.style.NOTICE(periodic_task),
self.style.NOTICE(periodic_task.last_run_at)))
else:
msg = ("The task %s has never been run before.\n"
"Are you sure you want to submit this periodic task?" %
(self.style.NOTICE(periodic_task), ))
if not confirm(msg):
raise CommandError(
'Please confirm as you need to know what you are doing.')
from celery import current_app
model_entry = ModelEntry(periodic_task)
app = current_app._get_current_object()
task = app.tasks.get(model_entry.task)
if task:
async_result = task.apply_async(model_entry.args,
model_entry.kwargs,
**model_entry.options)
else:
async_result = app.send_task(model_entry.task, model_entry.args,
model_entry.kwargs,
**model_entry.options)
periodic_task.last_run_at = app.now()
periodic_task.save()
if not ignore_result:
result = async_result.get(timeout=timeout)
self.stdout.write(result)
def __init__(self, *args, **kwargs):
self._schedule = None
self._has_made_initial_read = False
self.app = kwargs.get('app', current_app._get_current_object())
self.PeriodicTask = import_from_string(
self.app.conf['CELERYSTORE_PERIODIC_TASK'])
self.TaskSchedule = import_from_string(
self.app.conf['CELERYSTORE_TASK_SCHEDULE'])
self._latest_change = self.PeriodicTask.get_latest_change_to_schedule()
super(StoreScheduler, self).__init__(*args, **kwargs)
def __init__(self, **kwargs):
super(CaerusEntry, self).__init__()
attrs = self.__dict__.keys()
self.app = kwargs.pop('app', current_app._get_current_object())
for akey, aval in kwargs.items():
setattr(self, akey, aval)
if self.last_run_at is None:
self.last_run_at = self._default_now()
def _trace_task_ret(name, uuid, request, body, content_type,
content_encoding, loads=loads_message, app=None,
**extra_request):
app = app or current_app._get_current_object()
accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT)
args, kwargs = loads(body, content_type, content_encoding, accept=accept)
request.update(args=args, kwargs=kwargs, **extra_request)
R, I, T, Rstr = trace_task(app.tasks[name],
uuid, args, kwargs, request, app=app)
return (1, R, T) if I else (0, Rstr, T)
def run_celery_server():
# celery worker -l debug -A tasks --beat
application = current_app._get_current_object()
worker_process = worker.worker(app=application)
options = {
'broker': 'amqp://super@localhost//',
'loglevel': 'INFO',
'traceback': False,
}
worker_process.run(**options)
def main(options={}):
"""
Be a worker, with the given options as if present on the command line.
"""
app = current_app._get_current_object()
wrkr = worker.worker(app=app)
opts = copy.copy(default_options)
opts.update(options)
wrkr.run(**opts)
def startWorker(self):
from celery import current_app
from celery.bin import worker
application = current_app._get_current_object()
worker = worker.worker(app=application)
options = {
'broker': KrakenConfiguration.restApi.config['CELERY_BROKER_URL'],
'loglevel': 'INFO',
'traceback': True,
}
worker.run(**options)
def get_event_store():
"""Get or set the current event store."""
app = current_app._get_current_object()
if not hasattr(Config, 'DEFAULT_BACKEND'):
return None
if hasattr(app, 'event_store'):
return app.event_store
else:
app.event_store = EventStore(Config.DEFAULT_BACKEND)
return app.event_store
def __call__(self, app=None, *args, **kwargs):
a = current_app._get_current_object()
w = worker.worker(app=a)
options = {
'loglevel': 'INFO',
'concurrency': 2,
'without-gossip': True
}
options.update(kwargs)
w.run(*args, **options)
def submit_urls(urls, contributor):
celery_app = current_app._get_current_object()
assert celery_app.conf.GITHUB_USERNAME is not None
assert celery_app.conf.GITHUB_PASSWORD is not None
github = Github(celery_app.conf.GITHUB_USERNAME,
celery_app.conf.GITHUB_PASSWORD)
git = Git(TEST_LISTS_REPO)
git.pull(["-u", celery_app.conf.TARGET_REPO.split("/")[0], "master"])
branch_name = ("user-contribution/{0}".format(
datetime.now().strftime("%Y%m%dT%H%M%S")
))
git.checkout(["-b", branch_name])
try:
country_names = set()
for url in urls:
country_code = url['country'].upper()
if (country_code not in country_codes.keys() and
country_code != "GLOBAL"):
raise Exception("Invalid country code '{0}'".format(country_code))
if country_code == "GLOBAL":
country_names.add("Global")
else:
country_names.add(country_codes[country_code])
append_url_to_csv(
country_code=url['country'].lower(),
url=url['url'],
category_code=url['category'],
source=contributor,
notes=url['notes']
)
for untracked_file in git.list_untracked():
git.add([untracked_file])
commit_title = "Add user contributed URLs for countries {0}".format(
' '.join(list(country_names))
)
commit_msg = "This Pull Request was automatically generated from the web"
git.add(["."])
git.commit([
"-m",
commit_title
])
git.push(["-u", "origin", branch_name])
repo = github.get_repo(celery_app.conf.TARGET_REPO)
head = "{0}:{1}".format(celery_app.conf.GITHUB_USERNAME, branch_name)
print "Repo %s" % celery_app.conf.TARGET_REPO
print "Head %s" % head
repo.create_pull(commit_title, commit_msg,
head=head,
base="master")
finally:
git.checkout(["master"])
def __init__(self, task):
self._task = task
self.app = current_app._get_current_object()
self.name = self._task['name']
self.task = TASK
self.is_active = self._task['is_active']
self.schedule = celery.schedules.schedule(datetime.timedelta(seconds=self._task['params']['interval']))
self.options = {}
self.args = (self._task['params']['url'],)
self.total_run_count = self._task['total_run_count']
self.last_run_at = self._task['last_run_at']
def start_celery_worker():
from celery import current_app
from celery.bin import worker
celery_app = current_app._get_current_object()
worker = worker.worker(app=celery_app)
options = {
'broker': app.config['CELERY_BROKER_URL'],
'loglevel': 'INFO',
'traceback': True
}
worker.run(**options)
def __init__(self, task,db):
self._task = task
self.db=db
self.app = current_app._get_current_object()
self.name = self._task["name"]
self.task = self._task["task"]
if "interval" in self._task:
interval= self._task["interval"]
if "period" in interval and "every" in interval:
period=interval["period"]
every= interval["every"]
else:
self._task["enabled"]=False
return
self.schedule = celery.schedules.schedule(datetime.timedelta(**{period: every}))
if "crontab" in self._task:
tab= self._task["crontab"]
if "minute" in tab and "hour" in tab and "day_of_week" in tab and "day_of_month" in tab and "month_of_year" in tab:
minute=tab["minute"]
hour= tab["hour"]
day_of_week= tab["day_of_week"]
day_of_month= tab["day_of_month"]
month_of_year= tab["month_of_year"]
else:
self._task["enabled"]=False
return
self.schedule = celery.schedules.crontab(minute=minute,
hour=hour,
day_of_week=day_of_week,
day_of_month=day_of_month,
month_of_year=month_of_year)
self.args = self._task["args"] if "args" in self._task else ""
self.kwargs = self._task["kwargs"] if "args" in self._task else ""
self.options = {
'queue': self._task["queue"] if "queue" in self._task else "" ,
'exchange': self._task["exchange"] if "exchange" in self._task else "",
'routing_key': self._task["routing_key"] if "routing_key" in self._task else "",
'expires': self._task["expires"] if "expires" in self._task else None
}
if "total_run_count" not in self._task:
self._task["total_run_count"] = 0
self.total_run_count = self._task["total_run_count"]
if "last_run_at" not in self._task:
self._task["last_run_at"] = self._default_now()
self.last_run_at = self._task["last_run_at"]
def _trace_task_ret(
name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request
):
app = app or current_app._get_current_object()
embed = None
if content_type:
accept = prepare_accept_content(app.conf.accept_content)
args, kwargs, embed = loads(body, content_type, content_encoding, accept=accept)
else:
args, kwargs, embed = body
hostname = gethostname()
request.update({"args": args, "kwargs": kwargs, "hostname": hostname, "is_eager": False}, **embed or {})
R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app)
return (1, R, T) if I else (0, Rstr, T)
def app(env_config, default_config):
"""Flask application fixture."""
app = create_app(**default_config)
# FIXME: Needs fixing flask_celeryext,
# which once creates the first celery app, the flask_app that is set
# is never released from the global state, even if you create a new
# celery application. We need to unset the "flask_app" manually.
from celery import current_app as cca
cca = cca._get_current_object()
delattr(cca, "flask_app")
celery_app = create_celery_app(app)
celery_app.set_current()
with app.app_context():
yield app
def __init__(self, svc_model=None, task_model=None):
self.app = current_app._get_current_object()
self.svc_model = svc_model
self.task_model = task_model
self.options = {}
self.name = task_model.Name
self.args = [svc_model.Id, task_model.Command]
self.schedule = crontab(*task_model.Schedule.split())
self.total_run_count = task_model.TotalRunCount or 0
task_model.LastRunAt = task_model.LastRunAt or "0001-01-01T00:00:00Z"
if isinstance(task_model.LastRunAt, basestring):
task_model.LastRunAt = parser.parse(task_model.LastRunAt)
self.last_run_at = task_model.LastRunAt
def __init__(self, model, app=None):
"""Initialize the model entry."""
self.app = app or current_app._get_current_object()
self.name = model.name
self.task = model.task
try:
self.schedule = model.schedule
except model.DoesNotExist:
logger.error(
'Disabling schedule %s that was removed from database',
self.name,
)
self._disable(model)
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError as exc:
logger.exception(
'Removing schedule %s for argument deseralization error: %r',
self.name, exc,
)
self._disable(model)
self.options = {}
for option in ['queue', 'exchange', 'routing_key', 'expires',
'priority']:
value = getattr(model, option)
if value is None:
continue
self.options[option] = value
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
last_run_at = model.last_run_at
if getattr(settings, 'DJANGO_CELERY_BEAT_TZ_AWARE', True):
last_run_at = make_aware(last_run_at)
self.last_run_at = last_run_at
def _trace_task_ret(name, uuid, request, body, content_type,
content_encoding, loads=loads_message, app=None,
**extra_request):
app = app or current_app._get_current_object()
embed = None
if content_type:
accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT)
args, kwargs, embed = loads(
body, content_type, content_encoding, accept=accept,
)
else:
args, kwargs = body
hostname = socket.gethostname()
request.update({
'args': args, 'kwargs': kwargs,
'hostname': hostname, 'is_eager': False,
}, **embed or {})
R, I, T, Rstr = trace_task(app.tasks[name],
uuid, args, kwargs, request, app=app)
return (1, R, T) if I else (0, Rstr, T)
def __init__(self, db_task):
self.db_task = db_task
self.app = current_app._get_current_object()
self.name = db_task.name
self.task = db_task.task
self.enabled = db_task.enabled
self.schedule = celery.schedules.crontab(minute=db_task.crontab_minute,
hour=db_task.crontab_hour,
day_of_week=db_task.crontab_day_of_week,
day_of_month=db_task.crontab_day_of_month,
month_of_year=db_task.crontab_month_of_year)
self.args = db_task.args
self.kwargs = db_task.kwargs
self.options = {}
if not self.db_task.last_run_at:
self.db_task.last_run_at = self._default_now()
self.last_run_at = self.db_task.last_run_at
def __init__(self, model):
self.app = current_app._get_current_object()
self.name = model.name
self.task = model.task
self.schedule = model.schedule
self.args = model.args
self.kwargs = model.kwargs
self.options = dict(
queue=model.queue,
exchange=model.exchange,
routing_key=model.routing_key,
expires=model.expires,
)
self.total_run_count = model.total_run_count
self.model = model
if not model.last_run_at:
model.last_run_at = self._default_now()
orig = self.last_run_at = model.last_run_at
if not is_naive(self.last_run_at):
self.last_run_at = self.last_run_at.replace(tzinfo=None)
assert orig.hour == self.last_run_at.hour # timezone sanity
def execute_and_trace(name, uuid, args, kwargs, request=None, **opts):
"""This is a pickleable method used as a target when applying to pools.
It's the same as::
>>> trace_task(name, *args, **kwargs)[0]
"""
global _current_app_for_proc
if _current_app_for_proc is None:
_current_app_for_proc = current_app._get_current_object()
task = _current_app_for_proc.tasks[name]
try:
hostname = opts.get("hostname")
setps("celeryd", name, hostname, rate_limit=True)
try:
if task.__tracer__ is None:
task.__tracer__ = build_tracer(name, task, **opts)
return task.__tracer__(uuid, args, kwargs, request)[0]
finally:
setps("celeryd", "-idle-", hostname, rate_limit=True)
except Exception, exc:
return report_internal_error(task, exc)
def register_task(name, task, schedule, args=[], kwargs={}, relative=False, options={}, celery_app=None):
if not isinstance(schedule, (schedules.crontab, schedules.schedule)):
raise TypeError
if celery_app is None:
celery_app = current_app._get_current_object()
if task not in celery_app.tasks:
raise ValueError('task \'%s\' is not in %s' % (task, celery_app))
models.session.begin()
try:
entry = CaerusEntry(
name=name,
task=task,
args=args,
kwargs=kwargs,
relative=relative,
options=options,
schedule=schedule)
models.session.add(entry)
CaerusEntryMeta.set_latest_modified(celery_app.now())
models.session.commit()
except Exception, e:
models.session.rollback()
raise
def unpickle_backend(cls, args, kwargs):
"""Return an unpickled backend."""
return cls(*args, app=current_app._get_current_object(), **kwargs)
def test_AsyncResult(self):
x = self.app.AsyncResult('1')
self.assertIs(x.app, self.app)
r = loads(dumps(x))
# not set as current, so ends up as default app after reduce
self.assertIs(r.app, current_app._get_current_object())
