def init_app(celery_app: celery.Celery, init_ts_logger: bool = False, add_json_handler: bool = False): ts_service = celery_app.conf['TS_SERVICE_NAME'] ts_service = ts_service.replace('-', '_') log_level = get_log_level(celery_app.conf['TS_LOG_LEVEL']) log_filter = CeleryTaskFilter() json_handler = ts_json_handler('celery', ts_service, log_filter) if add_json_handler else None if init_ts_logger: logger = setup_ts_logger(ts_service, log_level, log_filter, json_handler) if add_json_handler: logger.addHandler(json_handler) logger.info('setting up ts_logger') def _setup_logger(): def do_setup_logging(**kwargs): stream_handler = ts_stream_handler(log_filter) del celery.utils.log.task_logger.handlers[:] del celery.utils.log.worker_logger.handlers[:] celery.utils.log.task_logger.propagate = True celery.utils.log.worker_logger.propagate = True celery.utils.log.task_logger.name = ts_service celery.utils.log.worker_logger.name = ts_service celery.utils.log.task_logger.setLevel(log_level) celery.utils.log.worker_logger.setLevel(log_level) if add_json_handler: celery.utils.log.task_logger.addHandler(json_handler) celery.utils.log.worker_logger.addHandler(json_handler) else: celery.utils.log.task_logger.addHandler(stream_handler) celery.utils.log.worker_logger.addHandler(stream_handler) return do_setup_logging setup_logging.connect(_setup_logger(), weak=False)
def create_app(config): """ Create a fully configured Celery application object. Args: config (Config): A reference to a lightflow configuration object. Returns: Celery: A fully configured Celery application object. """ # configure the celery logging system with the lightflow settings setup_logging.connect(partial(_initialize_logging, config), weak=False) # patch Celery to use cloudpickle instead of pickle for serialisation patch_celery() # create the main celery app and load the configuration app = Celery('lightflow') app.conf.update(**config.celery) # overwrite user supplied settings to make sure celery works with lightflow app.conf.update(task_serializer='pickle', accept_content=['pickle'], result_serializer='pickle', task_default_queue=JobType.Task, task_queues=(Queue(JobType.Task, routing_key=JobType.Task), Queue(JobType.Workflow, routing_key=JobType.Workflow), Queue(JobType.Dag, routing_key=JobType.Dag))) if isinstance(app.conf.include, list): app.conf.include.extend(LIGHTFLOW_INCLUDE) else: if len(app.conf.include) > 0: raise ConfigOverwriteError( 'The content in the include config will be overwritten') app.conf.include = LIGHTFLOW_INCLUDE return app
celery.conf.accept_content = {'pickle'} creds = AccountCreationCredentials( **{ field: conf.get(*field.split('_')) for field in AccountCreationCredentials._fields }, ) # if in debug mode, disable celery logging so that stdin / stdout / stderr # don't get tampered with (otherwise, interactive debuggers won't work) if os.environ.get('CREATE_DEBUG', ''): # pylint: disable=unused-argument def no_logging(*args, **kwargs): pass setup_logging.connect(no_logging) def failure_handler(exc, task_id, args, kwargs, einfo): """Handle errors in Celery tasks by reporting via ocflib. We want to report actual errors, not just validation errors. Unfortunately it's hard to pick them out. For now, we just ignore ValueErrors and report everything else. It's likely that we'll need to revisit that some time in the future. """ if isinstance(exc, ValueError): return try:
from compass.actions import progress_update from compass.db import database from compass.tasks.client import celery from compass.utils import flags from compass.utils import logsetting from compass.utils import setting_wrapper as setting def tasks_setup_logging(**_): """Setup logging options from compass setting.""" flags.init() flags.OPTIONS.logfile = setting.CELERY_LOGFILE logsetting.init() setup_logging.connect(tasks_setup_logging) @celery.task(name="compass.tasks.pollswitch") def pollswitch(ip_addr, req_obj='mac', oper="SCAN"): """Query switch and return expected result. :param ip_addr: switch ip address. :type ip_addr: str :param reqObj: the object requested to query from switch. :type reqObj: str :param oper: the operation to query the switch (SCAN, GET, SET). :type oper: str """ with database.session(): poll_switch.poll_switch(ip_addr, req_obj='mac', oper="SCAN")
broker=conf.get('celery', 'broker'), backend=conf.get('celery', 'backend'), ) creds = AccountCreationCredentials(**{ field: conf.get(*field.split('_')) for field in AccountCreationCredentials._fields }) # if in debug mode, disable celery logging so that stdin / stdout / stderr # don't get tampered with (otherwise, interactive debuggers won't work) if os.environ.get('CREATE_DEBUG', ''): def no_logging(*args, **kwargs): pass setup_logging.connect(no_logging) def failure_handler(exc, task_id, args, kwargs, einfo): """Handle errors in Celery tasks by reporting via ocflib. We want to report actual errors, not just validation errors. Unfortunately it's hard to pick them out. For now, we just ignore ValueErrors and report everything else. It's likely that we'll need to revisit that some time in the future. """ if isinstance(exc, ValueError): return try:
LOG.info("Tasks.py: VENV=" + VENV) TESTS_DIR = os.path.join(VENV, 'tests') # if VENV is None: # raise RuntimeError("You must first activate the virtualenv (e.g. workon my_environment).") LOG_CONFIG = 'logging.conf' MEMCACHED_META_PREFIX = 'f5test-task-' # URL_REGEX = r'(\(?\bhttp://[-A-Za-z0-9+&@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%=~_()|])' MAX_LOG_LINE = 1024 # Setup logging only once when celery is initialized. # Never use --log-config with nosetests when running under celery! def _setup_logging(**kw): logging.config.fileConfig(os.path.join(VENV, LOG_CONFIG)) # after_setup_task_logger.connect(_setup_logging) setup_logging.connect(_setup_logging) # This unloads all our modules, thus forcing nose to reload all tests. def _clean_sys_modules(tests_path=TESTS_DIR): for name, module in sys.modules.items(): if (module and inspect.ismodule(module) and hasattr(module, '__file__')) \ and module.__file__.startswith(tests_path): del sys.modules[name] class MyMemoryHandler(BufferingHandler): def __init__(self, task, level, *args, **kwargs): super(MyMemoryHandler, self).__init__(*args, **kwargs) self.task = task
qualname = log_params['qualname'] if 'qualname' in log_params else name if qualname == 'root': qualname = None logger = logging.getLogger(qualname) level = getattr(logging, log_params['level']) logger.setLevel(level) for handler in logger.handlers: handler.setFormatter(formatter) setup_logging.connect(setup_logging_from_paste_ini) def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME, default_conf_file=None): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is. By default it defaults to 'mediagoblin.ini'. Note that if celery_setup_elsewhere is set in your config file, this simply won't work. """ if not default_conf_file:
from celery import Celery from celery.signals import setup_logging from log_util import LogUtil app = Celery('task_demo1', broker='redis://127.0.0.1:6379/0') # add logging config conf = { 'filename': 'log/demo1.log', 'level': 'DEBUG', } LogUtil(**conf) fn = lambda **kwargs: logging.getLogger() setup_logging.connect(fn) CELERYBEAT_SCHEDULE = { 'add-every-10-second': { 'task': 'demo1.add', 'schedule': timedelta(seconds=10), 'args': (3, 2) }, } CELERY_TIMEZONE = 'Asia/Shanghai' @app.task def add(x, y): logging.debug('calculate {0} plus {1} equals {2}'.format(x, y, x + y)) return x + y
import logging import os from tlog.config import Config Config.load() from celery.signals import setup_logging from tlog.logger import logger from celery import Celery from celery.loaders.base import BaseLoader def init_logging(*arg, **kwargs): logger.set_logger('worker.log') return logging.getLogger() setup_logging.connect(init_logging) celery = Celery( 'tlog.worker.app', broker=Config.data['celery']['broker'], backend=Config.data['celery']['backend'], include=[ 'tlog.worker.receiver', ], ) def main(): celery.start() if __name__ == '__main__': main()
logging_conf_file = 'paste.ini' # allow users to set up explicitly which paste file to check via the # PASTE_CONFIG environment variable logging_conf_file = os.environ.get('PASTE_CONFIG', logging_conf_file) if not os.path.exists(logging_conf_file): raise OSError('{} does not exist. Logging can not be set up.'.format( logging_conf_file)) logging.config.fileConfig(logging_conf_file) hook_runall('celery_logging_setup') setup_logging.connect(setup_logging_from_paste_ini) def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME, default_conf_file=None): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable MEDIAGOBLIN_CONFIG to specify where this config file is. By default it defaults to 'mediagoblin.ini'. Note that if celery_setup_elsewhere is set in your config file, this simply won't work. """
from __future__ import absolute_import import deployer.logger from celery.signals import setup_logging __version__ = '0.5.1' __author__ = 'sukrit' deployer.logger.init_logging() setup_logging.connect(deployer.logger.init_celery_logging)
from __future__ import absolute_import from celery.signals import setup_logging import orchestrator.logger __version__ = '0.5.3' __author__ = 'sukrit' orchestrator.logger.init_logging() setup_logging.connect(orchestrator.logger.init_celery_logging)
# stdout_logger = logging.getLogger('STDOUT') # sl = StreamToLogger(stdout_logger, logging.INFO) # sys.stdout = sl # # stderr_logger = logging.getLogger('STDERR') # sl = StreamToLogger(stderr_logger, logging.ERROR) # sys.stderr = sl # stdout_logger.addHandler(handler) # stderr_logger.addHandler(handler) log = logging.getLogger('celery') log.addHandler(handler) log.setLevel(loglevel) return log NewFrameHandler = RotatingFileHandler( '/data/log/service_platform/newframe.log', maxBytes=100 * 1024 * 1024, backupCount=2) new_frame_log = logging.getLogger('newframe') new_frame_log.addHandler(NewFrameHandler) new_frame_log.setLevel(logging.DEBUG) setup_logging.connect(initialize_logger) if __name__ == '__main__': app.start()
VENV = os.environ.get('VIRTUAL_ENV', '../../../') TESTS_DIR = os.path.join(VENV, 'tests') # if VENV is None: # raise RuntimeError("You must first activate the virtualenv (e.g. workon my_environment).") LOG_CONFIG = 'logging.conf' MEMCACHED_META_PREFIX = 'f5test-task-' # URL_REGEX = r'(\(?\bhttp://[-A-Za-z0-9+&@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%=~_()|])' MAX_LOG_LINE = 1024 # Setup logging only once when celery is initialized. # Never use --log-config with nosetests when running under celery! def _setup_logging(**kw): logging.config.fileConfig(os.path.join(VENV, LOG_CONFIG)) # after_setup_task_logger.connect(_setup_logging) setup_logging.connect(_setup_logging) # This unloads all our modules, thus forcing nose to reload all tests. def _clean_sys_modules(tests_path=TESTS_DIR): for name, module in sys.modules.items(): if (module and inspect.ismodule(module) and hasattr(module, '__file__')) \ and module.__file__.startswith(tests_path): del sys.modules[name] class MyMemoryHandler(BufferingHandler): def __init__(self, task, level, *args, **kwargs): super(MyMemoryHandler, self).__init__(*args, **kwargs) self.task = task
formatter = CustomJsonFormatter('(timestamp) (level) (name) (message)') # override celery logger with json-logger def setup_json_logging(loglevel=logging.INFO, **kwargs): logHandler = logging.StreamHandler() logHandler.setFormatter(formatter) logger.addHandler(logHandler) logger.setLevel(loglevel) return logger # signal celery to override logger setup setup_logging.connect(setup_json_logging) @app.on_after_configure.connect def setup_periodic_tasks(sender, **kwargs): # Calls test('hello') every 10 seconds. sender.add_periodic_task(10.0, test.s('hello'), name='add every 10') # Calls test('world') every 30 seconds sender.add_periodic_task(30.0, test.s('world'), expires=10) # Executes every Monday morning at 7:30 a.m. sender.add_periodic_task( crontab(minute="*"), test.s('Happy Mondays!'),