def init_app(celery_app: celery.Celery,
init_ts_logger: bool = False,
add_json_handler: bool = False):
ts_service = celery_app.conf['TS_SERVICE_NAME']
ts_service = ts_service.replace('-', '_')
log_level = get_log_level(celery_app.conf['TS_LOG_LEVEL'])
log_filter = CeleryTaskFilter()
json_handler = ts_json_handler('celery', ts_service,
log_filter) if add_json_handler else None
if init_ts_logger:
logger = setup_ts_logger(ts_service, log_level, log_filter,
json_handler)
if add_json_handler:
logger.addHandler(json_handler)
logger.info('setting up ts_logger')
def _setup_logger():
def do_setup_logging(**kwargs):
stream_handler = ts_stream_handler(log_filter)
del celery.utils.log.task_logger.handlers[:]
del celery.utils.log.worker_logger.handlers[:]
celery.utils.log.task_logger.propagate = True
celery.utils.log.worker_logger.propagate = True
celery.utils.log.task_logger.name = ts_service
celery.utils.log.worker_logger.name = ts_service
celery.utils.log.task_logger.setLevel(log_level)
celery.utils.log.worker_logger.setLevel(log_level)
if add_json_handler:
celery.utils.log.task_logger.addHandler(json_handler)
celery.utils.log.worker_logger.addHandler(json_handler)
else:
celery.utils.log.task_logger.addHandler(stream_handler)
celery.utils.log.worker_logger.addHandler(stream_handler)
return do_setup_logging
setup_logging.connect(_setup_logger(), weak=False)
def create_app(config):
""" Create a fully configured Celery application object.
Args:
config (Config): A reference to a lightflow configuration object.
Returns:
Celery: A fully configured Celery application object.
"""
# configure the celery logging system with the lightflow settings
setup_logging.connect(partial(_initialize_logging, config), weak=False)
# patch Celery to use cloudpickle instead of pickle for serialisation
patch_celery()
# create the main celery app and load the configuration
app = Celery('lightflow')
app.conf.update(**config.celery)
# overwrite user supplied settings to make sure celery works with lightflow
app.conf.update(task_serializer='pickle',
accept_content=['pickle'],
result_serializer='pickle',
task_default_queue=JobType.Task,
task_queues=(Queue(JobType.Task, routing_key=JobType.Task),
Queue(JobType.Workflow,
routing_key=JobType.Workflow),
Queue(JobType.Dag, routing_key=JobType.Dag)))
if isinstance(app.conf.include, list):
app.conf.include.extend(LIGHTFLOW_INCLUDE)
else:
if len(app.conf.include) > 0:
raise ConfigOverwriteError(
'The content in the include config will be overwritten')
app.conf.include = LIGHTFLOW_INCLUDE
return app
celery.conf.accept_content = {'pickle'}
creds = AccountCreationCredentials(
**{
field: conf.get(*field.split('_'))
for field in AccountCreationCredentials._fields
}, )
# if in debug mode, disable celery logging so that stdin / stdout / stderr
# don't get tampered with (otherwise, interactive debuggers won't work)
if os.environ.get('CREATE_DEBUG', ''):
# pylint: disable=unused-argument
def no_logging(*args, **kwargs):
pass
setup_logging.connect(no_logging)
def failure_handler(exc, task_id, args, kwargs, einfo):
"""Handle errors in Celery tasks by reporting via ocflib.
We want to report actual errors, not just validation errors. Unfortunately
it's hard to pick them out. For now, we just ignore ValueErrors and report
everything else.
It's likely that we'll need to revisit that some time in the future.
"""
if isinstance(exc, ValueError):
return
try:
from compass.actions import progress_update
from compass.db import database
from compass.tasks.client import celery
from compass.utils import flags
from compass.utils import logsetting
from compass.utils import setting_wrapper as setting
def tasks_setup_logging(**_):
"""Setup logging options from compass setting."""
flags.init()
flags.OPTIONS.logfile = setting.CELERY_LOGFILE
logsetting.init()
setup_logging.connect(tasks_setup_logging)
@celery.task(name="compass.tasks.pollswitch")
def pollswitch(ip_addr, req_obj='mac', oper="SCAN"):
"""Query switch and return expected result.
:param ip_addr: switch ip address.
:type ip_addr: str
:param reqObj: the object requested to query from switch.
:type reqObj: str
:param oper: the operation to query the switch (SCAN, GET, SET).
:type oper: str
"""
with database.session():
poll_switch.poll_switch(ip_addr, req_obj='mac', oper="SCAN")
broker=conf.get('celery', 'broker'),
backend=conf.get('celery', 'backend'),
)
creds = AccountCreationCredentials(**{
field:
conf.get(*field.split('_'))
for field in AccountCreationCredentials._fields
})
# if in debug mode, disable celery logging so that stdin / stdout / stderr
# don't get tampered with (otherwise, interactive debuggers won't work)
if os.environ.get('CREATE_DEBUG', ''):
def no_logging(*args, **kwargs):
pass
setup_logging.connect(no_logging)
def failure_handler(exc, task_id, args, kwargs, einfo):
"""Handle errors in Celery tasks by reporting via ocflib.
We want to report actual errors, not just validation errors. Unfortunately
it's hard to pick them out. For now, we just ignore ValueErrors and report
everything else.
It's likely that we'll need to revisit that some time in the future.
"""
if isinstance(exc, ValueError):
return
try:
LOG.info("Tasks.py: VENV=" + VENV)
TESTS_DIR = os.path.join(VENV, 'tests')
# if VENV is None:
# raise RuntimeError("You must first activate the virtualenv (e.g. workon my_environment).")
LOG_CONFIG = 'logging.conf'
MEMCACHED_META_PREFIX = 'f5test-task-'
# URL_REGEX = r'(\(?\bhttp://[-A-Za-z0-9+&@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%=~_()|])'
MAX_LOG_LINE = 1024
# Setup logging only once when celery is initialized.
# Never use --log-config with nosetests when running under celery!
def _setup_logging(**kw):
logging.config.fileConfig(os.path.join(VENV, LOG_CONFIG))
# after_setup_task_logger.connect(_setup_logging)
setup_logging.connect(_setup_logging)
# This unloads all our modules, thus forcing nose to reload all tests.
def _clean_sys_modules(tests_path=TESTS_DIR):
for name, module in sys.modules.items():
if (module and inspect.ismodule(module) and hasattr(module, '__file__')) \
and module.__file__.startswith(tests_path):
del sys.modules[name]
class MyMemoryHandler(BufferingHandler):
def __init__(self, task, level, *args, **kwargs):
super(MyMemoryHandler, self).__init__(*args, **kwargs)
self.task = task
qualname = log_params['qualname'] if 'qualname' in log_params else name
if qualname == 'root':
qualname = None
logger = logging.getLogger(qualname)
level = getattr(logging, log_params['level'])
logger.setLevel(level)
for handler in logger.handlers:
handler.setFormatter(formatter)
setup_logging.connect(setup_logging_from_paste_ini)
def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME,
default_conf_file=None):
"""
Transform this module into a celery config module by reading the
mediagoblin config file. Set the environment variable
MEDIAGOBLIN_CONFIG to specify where this config file is.
By default it defaults to 'mediagoblin.ini'.
Note that if celery_setup_elsewhere is set in your config file,
this simply won't work.
"""
if not default_conf_file:
from celery import Celery
from celery.signals import setup_logging
from log_util import LogUtil
app = Celery('task_demo1', broker='redis://127.0.0.1:6379/0')
# add logging config
conf = {
'filename': 'log/demo1.log',
'level': 'DEBUG',
}
LogUtil(**conf)
fn = lambda **kwargs: logging.getLogger()
setup_logging.connect(fn)
CELERYBEAT_SCHEDULE = {
'add-every-10-second': {
'task': 'demo1.add',
'schedule': timedelta(seconds=10),
'args': (3, 2)
},
}
CELERY_TIMEZONE = 'Asia/Shanghai'
@app.task
def add(x, y):
logging.debug('calculate {0} plus {1} equals {2}'.format(x, y, x + y))
return x + y
import logging
import os
from tlog.config import Config
Config.load()
from celery.signals import setup_logging
from tlog.logger import logger
from celery import Celery
from celery.loaders.base import BaseLoader
def init_logging(*arg, **kwargs):
logger.set_logger('worker.log')
return logging.getLogger()
setup_logging.connect(init_logging)
celery = Celery(
'tlog.worker.app',
broker=Config.data['celery']['broker'],
backend=Config.data['celery']['backend'],
include=[
'tlog.worker.receiver',
],
)
def main():
celery.start()
if __name__ == '__main__':
main()
logging_conf_file = 'paste.ini'
# allow users to set up explicitly which paste file to check via the
# PASTE_CONFIG environment variable
logging_conf_file = os.environ.get('PASTE_CONFIG', logging_conf_file)
if not os.path.exists(logging_conf_file):
raise OSError('{} does not exist. Logging can not be set up.'.format(
logging_conf_file))
logging.config.fileConfig(logging_conf_file)
hook_runall('celery_logging_setup')
setup_logging.connect(setup_logging_from_paste_ini)
def setup_self(check_environ_for_conf=True,
module_name=OUR_MODULENAME,
default_conf_file=None):
"""
Transform this module into a celery config module by reading the
mediagoblin config file. Set the environment variable
MEDIAGOBLIN_CONFIG to specify where this config file is.
By default it defaults to 'mediagoblin.ini'.
Note that if celery_setup_elsewhere is set in your config file,
this simply won't work.
"""
from __future__ import absolute_import import deployer.logger from celery.signals import setup_logging __version__ = '0.5.1' __author__ = 'sukrit' deployer.logger.init_logging() setup_logging.connect(deployer.logger.init_celery_logging)
from __future__ import absolute_import from celery.signals import setup_logging import orchestrator.logger __version__ = '0.5.3' __author__ = 'sukrit' orchestrator.logger.init_logging() setup_logging.connect(orchestrator.logger.init_celery_logging)
# stdout_logger = logging.getLogger('STDOUT')
# sl = StreamToLogger(stdout_logger, logging.INFO)
# sys.stdout = sl
#
# stderr_logger = logging.getLogger('STDERR')
# sl = StreamToLogger(stderr_logger, logging.ERROR)
# sys.stderr = sl
# stdout_logger.addHandler(handler)
# stderr_logger.addHandler(handler)
log = logging.getLogger('celery')
log.addHandler(handler)
log.setLevel(loglevel)
return log
NewFrameHandler = RotatingFileHandler(
'/data/log/service_platform/newframe.log',
maxBytes=100 * 1024 * 1024,
backupCount=2)
new_frame_log = logging.getLogger('newframe')
new_frame_log.addHandler(NewFrameHandler)
new_frame_log.setLevel(logging.DEBUG)
setup_logging.connect(initialize_logger)
if __name__ == '__main__':
app.start()
VENV = os.environ.get('VIRTUAL_ENV', '../../../')
TESTS_DIR = os.path.join(VENV, 'tests')
# if VENV is None:
# raise RuntimeError("You must first activate the virtualenv (e.g. workon my_environment).")
LOG_CONFIG = 'logging.conf'
MEMCACHED_META_PREFIX = 'f5test-task-'
# URL_REGEX = r'(\(?\bhttp://[-A-Za-z0-9+&@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%=~_()|])'
MAX_LOG_LINE = 1024
# Setup logging only once when celery is initialized.
# Never use --log-config with nosetests when running under celery!
def _setup_logging(**kw):
logging.config.fileConfig(os.path.join(VENV, LOG_CONFIG))
# after_setup_task_logger.connect(_setup_logging)
setup_logging.connect(_setup_logging)
# This unloads all our modules, thus forcing nose to reload all tests.
def _clean_sys_modules(tests_path=TESTS_DIR):
for name, module in sys.modules.items():
if (module and inspect.ismodule(module) and hasattr(module, '__file__')) \
and module.__file__.startswith(tests_path):
del sys.modules[name]
class MyMemoryHandler(BufferingHandler):
def __init__(self, task, level, *args, **kwargs):
super(MyMemoryHandler, self).__init__(*args, **kwargs)
self.task = task
formatter = CustomJsonFormatter('(timestamp) (level) (name) (message)')
# override celery logger with json-logger
def setup_json_logging(loglevel=logging.INFO, **kwargs):
logHandler = logging.StreamHandler()
logHandler.setFormatter(formatter)
logger.addHandler(logHandler)
logger.setLevel(loglevel)
return logger
# signal celery to override logger setup
setup_logging.connect(setup_json_logging)
@app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
# Calls test('hello') every 10 seconds.
sender.add_periodic_task(10.0, test.s('hello'), name='add every 10')
# Calls test('world') every 30 seconds
sender.add_periodic_task(30.0, test.s('world'), expires=10)
# Executes every Monday morning at 7:30 a.m.
sender.add_periodic_task(
crontab(minute="*"),
test.s('Happy Mondays!'),