def configure_app(app):
app.user_options['worker'].add(
Option('--with-gate-keeper', action='store_true',
default=False, help='Enable gate keeper'))
app.user_options['worker'].add(
Option('--gate-keeper-bucket-size', action='store',
type='int', default=5,
help='The gate keeper bucket size'))
app.steps['worker'].add(GateKeeper)
def __init__(self, *args, **kwargs):
super(YowCelery, self).__init__(*args, **kwargs)
self.steps['worker'].add(YowsupStep)
self.user_options['worker'].update(
[Option('--yowlogin', dest='login', metavar="phone:b64password", default=None,
help='''WhatsApp login credentials, in the format phonenumber:password,
where password is base64 encoded.'''),
Option('--yowconfig', dest='config', default=None,
help='Path to config file containing authentication info.'),
Option('--yowunmoxie', dest='unmoxie', action="store_true", default=False,
help="Disable E2E Encryption")])
def make_app():
app = Celery()
if celery_version.major > 3:
app.user_options['preload'].add(add_preload_arguments)
else:
app.user_options['preload'].add(
Option('-i',
'--ini',
default=None,
help='Paste ini configuration file.'))
app.user_options['preload'].add(
Option('--ini-var',
default=None,
action='append',
help='Comma separated list of key=value to pass to ini.'))
return app
def create_celery_app(app):
"""
This function integrates celery into Flask, see ref here:
http://flask.pocoo.org/docs/0.10/patterns/celery/
"""
# Create a new celery object and configure it using the apps config
celery = Celery(app.import_name, broker=app.config['CELERY_BROKER_URL'])
app.config['CELERY_RESULT_BACKEND'] = app.config['CELERY_BROKER_URL']
celery.conf.update(app.config)
# Add the env option
option = Option('-e',
'--env',
choices=['dev', 'local', 'prod'],
default='local',
dest='env')
celery.user_options['beat'].add(option)
celery.user_options['worker'].add(option)
# Create a sub class of the celery Task class that exectures within the
# application's context.
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
def configure_app(app):
app.user_options['worker'].add(
Option('--with-logging-server',
action='store_true',
default=False,
help='Enable logging server'))
app.user_options['worker'].add(
Option('--logging-server-logdir',
action='store',
help='logdir location'))
app.user_options['worker'].add(
Option('--logging-server-handler-cache-size',
action='store',
type='int',
default=100,
help='Maximum number of file handlers that can be open at any '
'given time'))
app.steps['worker'].add(ZMQLoggingServerBootstep)
parser.add_argument(
'-i', '--ini', default=None,
help='Paste ini configuration file.'
)
parser.add_argument(
'--ini-var', default=None,
help='Comma separated list of key=value to pass to ini.'
)
celery_app = Celery()
if celery_version.major > 3:
celery_app.user_options['preload'].add(add_preload_arguments)
else:
celery_app.user_options['preload'].add(Option(
'-i', '--ini', default=None,
help='Paste ini configuration file.'
))
celery_app.user_options['preload'].add(Option(
'--ini-var', default=None,
help='Comma separated list of key=value to pass to ini.'
))
ini_file = None
def boolify(config, *names):
"""Make config variables boolean.
Celery wants ``False`` instead of ``"false"`` for CELERY_ALWAYS_EAGER.
"""
for n in names:
from sentry_sdk.integrations.celery import CeleryIntegration
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')
app = Celery('project')
#def add_worker_arguments(parser):
# parser.add_argument(
# '--enable-my-option', action='store_true', default=False,
# help='Enable custom option.',
# ),
#app.user_options['worker'].add(add_worker_arguments)
#app.user_options['beat'].add(add_worker_arguments)
app.user_options['worker'].add(
Option('--sentry', dest='dsn', default=None, help='Sentry DSN.'))
app.user_options['beat'].add(
Option('--sentry', dest='dsn', default=None, help='Sentry DSN.'))
class CustomArgs(bootsteps.Step):
def __init__(self, worker_beat, dsn, **options):
# initialize sentry for celery
if dsn:
sentry_sdk.init(dsn[0], integrations=[CeleryIntegration()])
app.steps['worker'].add(CustomArgs)
app.steps['beat'].add(CustomArgs)
# Using a string here means the worker doesn't have to serialize
log = logging.getLogger(__name__)
register(
"date_json",
json_dumps,
json_loads,
content_type="application/x-date_json",
content_encoding="utf-8",
)
celery = Celery()
celery.user_options["preload"].add(
Option(
"--ini",
dest="ini",
default=None,
help="Specifies pyramid configuration file location.",
)
)
@user_preload_options.connect
def on_preload_parsed(options, **kwargs):
"""
This actually configures celery from pyramid config file
"""
celery.conf["INI_PYRAMID"] = options["ini"]
import appenlight_client.client as e_client
ini_location = options["ini"]
if not ini_location:
import tarfile
from subprocess import Popen
import redis
from celery import Celery, bootsteps
from celery.bin import Option
from celery.utils.log import get_task_logger
from dominion import routines
from firmwares.models import Firmware
APP = Celery('tasks', backend='rpc://', broker='amqp://guest@localhost//')
APP.user_options['worker'].add(
Option('--base-systems',
dest='base_systems',
default='/var/dominion',
help='The path to the directory which contains chroot environments '
'which, in turn, contain the Debian base system'))
APP.user_options['worker'].add(
Option('--builder-location',
dest='builder_location',
default='/var/dominion/pieman',
help=''))
APP.user_options['worker'].add(
Option('--redis-host', dest='redis_host', default='localhost', help=''))
APP.user_options['worker'].add(
Option('--redis-port', dest='redis_port', default=6379, help=''))
APP.user_options['worker'].add(Option('--workspace', dest='workspace',
help=''))
BUILD_FAILED = 1
BUILDS_NUMBER_KEY = 'builds_number'
'openspending.tasks.dataset.load_source': {
'queue': 'loading'
},
'openspending.tasks.dataset.load_budgetdatapackage': {
'queue': 'loading'
},
'openspending.tasks.dataset.index_dataset': {
'queue': 'loading'
},
}
# Add a user option to celery where the configuration file for pylons
# can be provided.
celery.user_options['preload'].add(
Option('-p',
'--pylons-ini-file',
help='Pylons .ini file to use for environment configuration'), )
@signals.user_preload_options.connect
def on_preload_parsed(options, **kwargs):
"""
Parse user options for celery. It only configures the environment
if a pylons ini file is provided, else it does nothing.
"""
if 'pylons_ini_file' in options:
# Get the filename of the pylons ini file provided
pylons_config = os.path.realpath(options['pylons_ini_file'])
# Configure the pylons environment with the pylons config
_configure_pylons(pylons_config)
class MailgunArgs(bootsteps.Step):
def __init__(
self, worker, mailgun_domain, mailgun_email, mailgun_api_key, **options
):
MailgunAPITask.mailgun_domain = mailgun_domain[0]
MailgunAPITask.mailgun_email = mailgun_email[0]
MailgunAPITask.mailgun_api_key = mailgun_api_key[0]
logger = get_task_logger(__name__)
app = Celery("tasks")
app.conf.broker_url = "redis://redis:6379/0"
app.user_options["worker"].add(
Option("--domain", dest="mailgun_domain", default=None, help="Mailgun domain")
)
app.user_options["worker"].add(
Option(
"--email",
dest="mailgun_email",
default=None,
help='Mailgun "from" email address.',
)
)
app.user_options["worker"].add(
Option("--key", dest="mailgun_api_key", default=None, help="Mailgun API key")
)
from core.config import cfg
from PIL import Image
# Tools
import json
import time
import requests
import datetime
# Celery configuration
app = Celery('frame_works')
app.config_from_object(celeryconfig) # Configs file
# Celery arguments -> server and port
app.user_options['worker'].add(
Option('--server', default='127.0.0.1')
)
app.user_options['worker'].add(
Option('--port', default='5000')
)
server_addr=""
class CustomArgs(bootsteps.Step):
def __init__(self, worker, server, port, **options):
global server_addr
print("\nSERVER -> " + str(server))
print("PORT -> " + str(port))
server_str=""
from celery.utils.log import get_task_logger
from .context import Context
'''
Celery worker setup
'''
ARRAY_CONFIGS = "array_configs"
# logger for the worker tasks.
logger = get_task_logger(__name__)
# global execution context
context = None
app = Celery('pureelk.worker', include=['pureelk.tasks'])
app.user_options['preload'].add(
Option(
"-a",
'--array-configs',
default='',
help='Path to array configs',
))
@signals.user_preload_options.connect
def handle_preload_options(options, **kwargs):
global context
context = Context(options[ARRAY_CONFIGS])
'task': 'occam-history-worker',
'schedule': timedelta(minutes=5),
}
}
celery_args = {}
redis_config = web_app.config.get('redis_config')
if redis_config:
redis_url = make_redis_url(redis_config)
celery_args['backend'] = redis_url
celery_args['broker'] = redis_url
background_app = Celery('occam', **celery_args)
background_app.conf.update(CELERYBEAT_SCHEDULE=COLLECTOR_SCHEDULE)
background_app.user_options['preload'].add(
Option('--occam-config', help='Path to an Occam configuration file.'))
@signals.user_preload_options.connect
def on_preload_parsed(options, **kwargs):
occam_config_path = options['occam_config']
occam_config = attach_occam_config_to_app(occam_config_path)
redis_conf = occam_config.get("redis_config", {})
redis_url = make_redis_url(redis_conf)
update_celery_backend_broker(redis_url)
def update_celery_backend_broker(redis_url):
global background_app
background_app.conf.update(BROKER_URL=redis_url,
from celery import bootsteps
from celery.bin import Option
from snake.config import config_parser
from snake.config import snake_config
from snake.core.celery import celery
class CustomArgs(bootsteps.Step):
"""Custom arguments for celery.
This allows for a custom configuration file to be passed throught the
command line. Mainly used for testing.
"""
def __init__(self, worker, worker_config, **options): # pylint: disable=super-init-not-called, unused-argument
if worker_config:
# NOTE: While the core will have the original settings, as the worker
# is in effect standalone this should not result in any configuration
# clashing!
config_parser.load_config(worker_config[0])
worker.app.conf.update(**snake_config)
app = celery # pylint: disable=invalid-name
app.user_options['worker'].add(
Option('--worker_config',
dest='worker_config',
default=None,
help='Custom worker configuration'))
app.steps['worker'].add(CustomArgs)
from __future__ import absolute_import
from celery import Celery
from celery.bin import Option
from trader import celeryconfig
app = Celery('trader')
app.config_from_object(celeryconfig)
app.user_options['trader'].add(
Option('--enable_replay_mode', action='store_true', default=False,
help='To enable mode for replaying of historic trade data processing'), # noqa
)
# from celery import bootsteps
"""
class MyBootstep(bootsteps.Step):
def __init__(self, worker, enable_replay_mode=False, **options):
print "AAAAAAA ", enable_replay_mode
with open("/tmp/qwe", "w") as f:
f.write(str("AAAAAAA " + str( enable_replay_mode) + "\n"))
def start(self, parent):
# our step is started together with all other Worker/Consumer
# bootsteps.
print('{0!r} is starting'.format(parent))
"""
if __name__ == '__main__':
# app.steps['trader'].add(MyBootstep)
# move that into __init__. It seems celery apps usually follow same package pattern as flask.
celeros_app = Celery()
# BEWARE : https://github.com/celery/celery/issues/3050
# doing this prevent setting config from command line
# from . import config
# celeros_app.config_from_object(config)
# setting up custom bootstep to start ROS node and pass ROS arguments to it
# for worker ( running on robot )
celeros_app.steps['worker'].add(PyrosBoot)
celeros_app.steps['consumer'].add(BatteryWatcher)
celeros_app.user_options['worker'].add(
Option('-R',
'--ros-arg',
action="append",
help='Arguments for ros initialisation'))
# and for beat ( running on concert )
celeros_app.user_options['beat'].add(
Option('-R',
'--ros-arg',
action="append",
help='Arguments for ros initialisation'))
#############################
# Basic task for simple tests
#############################
from celery.utils.log import get_task_logger
_logger = get_task_logger(__name__)
from celery import Celery
from celery import signals
import grsnp.hypergeom4 as hpgm
from celery.bin import Option
from celery.exceptions import Reject, MaxRetriesExceededError
import os
# celery
app = Celery('grsnp')
app.config_from_object('grsnp.celeryconfiguration_optimizer')
app.user_options['preload'].add(
Option('-d', '--data_dir', default='',
help='Set the directory containing the database. Required. Use absolute path. Example: /home/username/db_#.##_#.##.####/.'),
)
sett = {}
# acks_late allows us to remove jobs for which we do not have the corresponding data
@app.task(acks_late=True,ignore_result = False,max_retries =5)
def calculate_bkg_gf_overlap(gf_path=None,list_bkg_paths=None,**kwargs):
"""Calculates the overlaps stats between the genomic feature and the backgrounds provided.
gf_path: The relative path to the genomic feature bed file. EX: 'grsnp_db_75_plus/hg19/genes/evofold.bed.gz'
list_bkg_paths: A list containing relative paths to the backgrounds. EX: ['custom_data/backgrounds/hg19/bkg1.gz','custom_data/backgrounds/hg19/bkg2.gz']
"""
global sett
data_dir = sett["data_dir"]
full_gf_path = os.path.join(data_dir,gf_path)
full_bkg_paths = [os.path.join(data_dir,x) for x in list_bkg_paths]
try:
if self._db_session is None:
config = ConfigParser.RawConfigParser()
config.read(SqlAlchemySessionTask.ini_config_path)
connection_string = config.get('app:main', 'sqlalchemy.url')
engine = create_engine(connection_string)
DBSession.configure(bind=engine)
self._db_session = DBSession
return self._db_session
email_celery_app = Celery(CELERY_APP_NAME,
broker='amqp://guest@localhost//',
backend='rpc',
include=['trailcam_email_service.celery.tasks'])
email_celery_app.user_options['worker'].add(
Option('--ini', dest='ini_config', default=None, help='Ini config file'))
class CustomArgs(bootsteps.Step):
def __init__(self, worker, ini_config, **options):
SqlAlchemySessionTask.ini_config_path = ini_config
email_celery_app.steps['worker'].add(CustomArgs)
email_celery_app.conf.update(result_expires=3600, )
if __name__ == '__main__':
email_celery_app.start()
'CELERYD_CONCURRENCY': None,
'CELERY_TIMEZONE': None,
'CELERYBEAT_SCHEDULE': {
'name': {
'task': 'task',
'schedule': timedelta(seconds=5)
}
}
}
celery = Celery()
celery.config_from_object(CELERY_CONFIG)
celery.user_options['preload'].add(
Option('--ini',
dest='ini',
default=None,
help='Specifies pyramid configuration file location.'))
@user_preload_options.connect
def on_preload_parsed(options, **kwargs):
"""
This actually configures celery from pyramid config file
"""
celery.conf['INI_PYRAMID'] = options['ini']
ini_location = options['ini']
if not ini_location:
raise Exception('You need to pass pyramid ini location using '
'--ini=filename.ini argument to the worker')
env = bootstrap(ini_location)
try:
from celery.bin import Option
import celery.signals
from celery.utils.log import get_task_logger
import six
from pyramid.settings import aslist
from pyramid.paster import bootstrap
import redis
import sqlalchemy as sa
from sqlalchemy import orm
from occams_datastore import models
from occams_datastore.models.events import register
app = Celery(__name__)
app.user_options['preload'].add(Option('--ini', help='Pyramid config file'))
log = get_task_logger(__name__)
#
# Dedicated Celery application database session.
# DO NOT USE THIS SESSION IN THE WSGI APP
#
# TODO: Haven't figured out how to make this a non-global variable for
# celery tasks...
#
Session = orm.scoped_session(orm.sessionmaker())
register(Session)
def includeme(config):
'--ini',
default=None,
help='Paste ini configuration file.')
parser.add_argument(
'--ini-var',
default=None,
help='Comma separated list of key=value to pass to ini.')
celery_app = get_current_app()
if celery_version.major > 3:
celery_app.user_options['preload'].add(add_preload_arguments)
else:
celery_app.user_options['preload'].add(
Option('-i',
'--ini',
default=None,
help='Paste ini configuration file.'))
celery_app.user_options['preload'].add(
Option('--ini-var',
default=None,
help='Comma separated list of key=value to pass to ini.'))
ini_file = None
def boolify(config, *names):
"""Make config variables boolean.
Celery wants ``False`` instead of ``"false"`` for CELERY_ALWAYS_EAGER.
"""
for n in names:
# -*- coding: utf-8 -*- # author liutaihua([email protected]) ''' celery的初始化app入口 ''' import os import sys import requests import json import celery from celery import Celery from celery.datastructures import LimitedSet from celery.worker import state from celery.bin import Option from config.config import REVOKES_MAX, REVOKE_EXPIRES state.revoked = LimitedSet(maxlen=REVOKES_MAX, expires=REVOKE_EXPIRES) os.environ['CELERY_CONFIG_MODULE'] = 'config.celeryconfig' hera_app = Celery() hera_app.user_options['worker'].add( Option('--result-backend', dest='result_backend', default='amqp', help='amqp or redis use for store thre celery result') )