def UnitApp(name=None, set_as_current=False, log=UnitLogging, **kwargs): app = Celery(name or 'celery.tests', set_as_current=set_as_current, log=log, **kwargs) app.add_defaults(deepcopy(CELERY_TEST_CONFIG)) return app
def test_add_defaults(self): app = Celery(set_as_current=False) self.assertFalse(app.configured) _conf = {'FOO': 300} conf = lambda: _conf app.add_defaults(conf) self.assertIn(conf, app._pending_defaults) self.assertFalse(app.configured) self.assertEqual(app.conf.FOO, 300) self.assertTrue(app.configured) self.assertFalse(app._pending_defaults) # defaults not pickled appr = loads(dumps(app)) with self.assertRaises(AttributeError): appr.conf.FOO # add more defaults after configured conf2 = {'FOO': 'BAR'} app.add_defaults(conf2) self.assertEqual(app.conf.FOO, 'BAR') self.assertIn(_conf, app.conf.defaults) self.assertIn(conf2, app.conf.defaults)
def TestApp(name=None, config=None, enable_logging=False, set_as_current=False, log=UnitLogging, backend=None, broker=None, **kwargs): """App used for testing.""" from . import tasks # noqa config = dict(deepcopy(DEFAULT_TEST_CONFIG), **config or {}) if broker is not None: config.pop("broker_url", None) if backend is not None: config.pop("result_backend", None) log = None if enable_logging else log test_app = Celery(name or "celery.tests", set_as_current=set_as_current, log=log, broker=broker, backend=backend, **kwargs) test_app.add_defaults(config) return test_app
def UnitApp(name=None, set_as_current=False, log=UnitLogging, broker='memory://', backend='cache+memory://', **kwargs): app = Celery(name or 'celery.tests', set_as_current=set_as_current, log=log, broker=broker, backend=backend, **kwargs) app.add_defaults(deepcopy(CELERY_TEST_CONFIG)) return app
def TestApp(name=None, set_as_current=False, log=UnitLogging, broker='memory://', backend=None, **kwargs): """App used for testing.""" test_app = Celery(name or 'celery.tests', set_as_current=set_as_current, log=log, broker=broker, backend=backend or 'cache+memory://', **kwargs) test_app.add_defaults(deepcopy(CELERY_TEST_CONFIG)) return test_app
def build_app(config=None): config = merge_configs( {k: v for (k, (_, v)) in DEFAULT_CONFIG.items()}, config or {} ) config["task_queues"] = CELERY_QUEUES + [ Queue(queue, Exchange(queue), routing_key=queue) for queue in config.get("task_queues", ()) ] logger.debug("Creating a Celery app with %s", config) # Instantiate the Celery app app = Celery(broker=config["task_broker"], task_cls="swh.scheduler.task:SWHTask") app.add_defaults(CELERY_DEFAULT_CONFIG) app.add_defaults(config) return app
def TestApp(name=None, config=None, enable_logging=False, set_as_current=False, log=UnitLogging, backend=None, broker=None, **kwargs): """App used for testing.""" from . import tasks # noqa config = dict(deepcopy(DEFAULT_TEST_CONFIG), **config or {}) if broker is not None: config.pop('broker_url', None) if backend is not None: config.pop('result_backend', None) log = None if enable_logging else log test_app = Celery( name or 'celery.tests', set_as_current=set_as_current, log=log, broker=broker, backend=backend, **kwargs) test_app.add_defaults(config) return test_app
Load settings into celery as late as possible. ''' config_updates = { 'BROKER_URL': settings.broker_url, 'CELERY_ROUTES': { 'yalp.pipeline.tasks.process_message': { 'queue': settings.parser_queue, }, }, } config_updates.update(settings.celery_advanced) return config_updates app = Celery() app.add_defaults(lazy_update_app_config) class PipelineTask(Task): ''' Pipeline celery task. ''' abstract = True _config = None _parsers = None _outputers = None _output_queue = None _output_exchange = None @property def config(self):
# Authors: Nikolay Melnik <*****@*****.**> import logging from logging import config as logging_config from flask.ext.admin import Admin from celery import Celery from app import create_app from rest_api import Api __version__ = '2016.08.08' app = create_app() celery = Celery('cloudml') celery.add_defaults(lambda: app.config) api = Api(app) admin = Admin(app, 'CloudML Admin Interface') logging_config.dictConfig(app.config['LOGGING']) # Note: to see sqlalchemy SQL statements, uncomment the following, # and watch out for the logs will be huge # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) if app.config.get('SEND_ERROR_EMAILS'): from logging.handlers import SMTPHandler mail_handler = SMTPHandler( mailhost=(app.config['EMAIL_HOST'], app.config['EMAIL_PORT']), fromaddr=app.config['SERVER_EMAIL'],
} # Final Celery settings return { "beat_schedule": schedule, "broker_url": config.celery_broker_url, "timezone": config.schedule_timezone, } # ============================================================================= # The Celery app # ============================================================================= celery_app = Celery() celery_app.add_defaults(get_celery_settings_dict) # celery_app.autodiscover_tasks([CELERY_APP_NAME], # related_name=CELERY_TASKS_MODULE) _ = ''' @celery_app.on_configure.connect def _app_on_configure(**kwargs) -> None: log.critical("@celery_app.on_configure: {!r}", kwargs) @celery_app.on_after_configure.connect def _app_on_after_configure(**kwargs) -> None: log.critical("@celery_app.on_after_configure: {!r}", kwargs) '''
import os from celery import Celery import django # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'powersong.settings') django.setup() app = Celery('powersong') app.config_from_object('django.conf:settings', namespace='CELERY') app.add_defaults({ 'CELERYD_HIJACK_ROOT_LOGGER': False, }) app.autodiscover_tasks() app.control.rate_limit('powersong.tasks.strava_task', '40/m') app.control.rate_limit('powersong.tasks.lastfm_task', '250/m') app.control.rate_limit('powersong.tasks.spotify_task', '250/m') @app.task(bind=True) def debug_task(self): print('Request: {0!r}'.format(self.request))
from operator import StackOperatorRedis # allow the usage of ssh config file by fabric env.use_ssh_config = True env.forward_agent = True # load config from file via environ variable config = os.environ.get('GACHETTE_SETTINGS', './config.rc') dd = imp.new_module('config') with open(config) as config_file: dd.__file__ = config_file.name exec(compile(config_file.read(), config_file.name, 'exec'), dd.__dict__) celery = Celery() celery.add_defaults(dd) # get settings key_filename = None if not hasattr(dd, "BUILD_KEY_FILENAME") else dd.BUILD_KEY_FILENAME host = dd.BUILD_HOST MAGIC_DOMAIN = "main" def send_notification(data): """ Send notification using Pubsub Redis """ red = Redis(dd.REDIS_HOST, int(dd.REDIS_PORT)) red.publish("all", ['publish', data])
from . import sentry from . import voevent __all__ = ('app', ) __version__ = get_versions()['version'] del get_versions # Use redis broker, because it supports locks (and thus singleton tasks). app = Celery(__name__, broker='redis://', autofinalize=False) """Celery application object.""" # Register LVAlert and VOEvent subsystems. lvalert.install(app) voevent.install(app) # Register all tasks. app.autodiscover_tasks([__name__]) # Add default configuration. app.add_defaults(playground) app.finalize() # Customize configuration from environment variable. app.config_from_envvar('CELERY_CONFIG_MODULE', silent=True) # Use the same URL for both the result backend and the broker. app.conf['result_backend'] = app.conf.broker_url sentry.configure()
from __future__ import absolute_import import os from celery import Celery from django.conf import settings # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') app = Celery('cyano') app.add_defaults({ 'CELERYD_HIJACK_ROOT_LOGGER': False, }) # Using a string here means the worker will not have to # pickle the object when using Windows. app.config_from_object('django.conf:settings') app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) @app.task(bind=True) def debug_task(self): print('Request: {0!r}'.format(self.request))
import os import random import logging logger = logging.getLogger(__name__) from celery import Celery redis_hostname = os.getenv('REDIS_HOSTNAME') app = Celery('hello', broker='redis://{}/0'.format(redis_hostname)) app.add_defaults( dict(CELERY_RESULT_BACKEND='redis://{}/1'.format(redis_hostname), CELERY_ACCEPT_CONTENT=['json'], CELERY_TASK_SERIALIZER='json', CELERY_RESULT_SERIALIZER='json')) @app.task(bind=True) def simple(self, something=None): logger.warning('SIMPLE simple {}'.format(something)) return 'simple return'
from flask import Flask from flask.ext.pymongo import PyMongo from flask_debugtoolbar import DebugToolbarExtension from raven.contrib.flask import Sentry from celery import Celery from utils import (configure_error_handlers, configure_app, configure_template_filters) app = Flask(__name__) configure_app(app, 'config.DevelopmentConfig') configure_error_handlers(app) # configure_middleware_handlers(app) configure_template_filters(app) toolbar = DebugToolbarExtension(app) mongo = PyMongo(app) if not app.config.get('DEBUG'): sentry = Sentry(app) celery = Celery() celery.add_defaults(app.config) from core.views import * if __name__ == '__main__': app.run()