def test_configure_syslog(self, m_config, m_after_setup_logger, m_after_setup_task_logger): app = MagicMock() m_config.log.syslog = True module.configure_syslog(app) app.conf.update.assert_called_with(CELERYD_LOG_COLOR=False) m_after_setup_logger.connect.assert_called_once_with(module.setup_log) m_after_setup_task_logger.connect.\ assert_called_once_with(module.setup_log)
import multiprocessing import config.parser as config from celery import Celery from fasteners import interprocess_locked from brain.models.sqlobjects import Probe from irma.common.base.exceptions import IrmaDatabaseResultNotFound, \ IrmaDatabaseError from irma.common.plugin_result import PluginResult log = logging.getLogger(__name__) probe_app = Celery('probetasks') config.conf_probe_celery(probe_app) config.configure_syslog(probe_app) # Time to cache the probe list # to avoid asking to rabbitmq PROBELIST_CACHE_TIME = 30 manager = multiprocessing.Manager() available_probes = manager.list() interprocess_lock_path = config.get_lock_path() def register(name, display_name, category, mimetype_regexp, session): try: probe = Probe.get_by_name(name, session) log.info("probe %s already registred " "updating parameters: "
# You may obtain a copy of the License in the top-level directory # of this distribution and at: # # http://www.apache.org/licenses/LICENSE-2.0 # # No part of the project, including this file, may be copied, # modified, propagated, or distributed except according to the # terms contained in the LICENSE file. import celery import config.parser as config from brain.helpers.celerytasks import async_call frontend_app = celery.Celery('frontendtasks') config.conf_frontend_celery(frontend_app) config.configure_syslog(frontend_app) def scan_launched(frontend_scanid, scan_request): async_call(frontend_app, "frontend.tasks", "scan_launched", args=[frontend_scanid, scan_request]) def scan_result(frontend_scanid, filename, probe, result): async_call(frontend_app, "frontend.tasks", "scan_result", args=[frontend_scanid, filename, probe, result])
from brain.models.sqlobjects import User, Job, Scan import brain.controllers.ftpctrl as ftp_ctrl import brain.controllers.scanctrl as scan_ctrl import brain.controllers.probectrl as probe_ctrl import brain.controllers.frontendtasks as celery_frontend from brain.helpers.sql import session_transaction from irma.common.base.utils import IrmaTaskReturn, IrmaScanRequest RETRY_MAX_DELAY = 30 # Get celery's logger log = get_task_logger(__name__) scan_app = celery.Celery('scan_tasks') config.conf_brain_celery(scan_app) config.configure_syslog(scan_app) interprocess_lock_path = config.get_lock_path() # IRMA specific debug messages are enables through # config file Section: log / Key: debug if config.debug_enabled(): def after_setup_logger_handler(sender=None, logger=None, loglevel=None, logfile=None, format=None, colorize=None, **kwds): config.setup_debug_logger(logging.getLogger(__name__)) log.debug("debug is enabled") celery.signals.after_setup_logger.connect(after_setup_logger_handler) celery.signals.after_setup_task_logger.connect(after_setup_logger_handler) delay = 1
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License in the top-level directory # of this distribution and at: # # http://www.apache.org/licenses/LICENSE-2.0 # # No part of the project, including this file, may be copied, # modified, propagated, or distributed except according to the # terms contained in the LICENSE file. import celery import config.parser as config from probe.helpers.celerytasks import async_call # declare a new Remote Brain application brain_app = celery.Celery('braintasks') config.conf_brain_celery(brain_app) config.configure_syslog(brain_app) # ============ # Task calls # ============ def register_probe(name, category, mimetype_regexp): """ send a task to the brain to register local probes""" task = async_call(brain_app, "brain.tasks", "register_probe", args=[name, category, mimetype_regexp]) return task
logfile=None, format=None, colorize=None, **kwds): config.setup_debug_logger(logging.getLogger(__name__)) log.debug("debug is enabled") celery.signals.after_setup_logger.connect(after_setup_logger_handler) celery.signals.after_setup_task_logger.connect(after_setup_logger_handler) # disable insecure serializer (disabled by default from 3.x.x) if (kombu.VERSION.major) < 3: kombu.disable_insecure_serializers() # declare a new Local Probe application probe_app = Celery("probe.tasks") config.conf_probe_celery(probe_app) config.configure_syslog(probe_app) # discover plugins located at specified path plugin_path = os.path.abspath("modules") if not os.path.exists(plugin_path): log.error("path {0} is invalid, cannot load probes".format(plugin_path)) sys.exit(1) manager = PluginManager() manager.discover(plugin_path) # determine dynamically queues to connect to using plugin names probes = manager.get_all_plugins() if not probes: log.error("No probe found, exiting application") sys.exit(1)
from lib.common.utils import to_unicode ############################################################################## # celery application configuration ############################################################################## log = get_task_logger(__name__) # disable insecure serializer (disabled by default from 3.x.x) if (kombu.VERSION.major) < 3: kombu.disable_insecure_serializers() # declare a new application app = Celery("probe.tasks") config.conf_probe_celery(app) config.configure_syslog(app) # discover plugins located at specified path plugin_path = os.path.abspath("modules") if not os.path.exists(plugin_path): log.error("path {0} is invalid, cannot load probes".format(plugin_path)) sys.exit(1) manager = PluginManager() manager.discover(plugin_path) # determine dynamically queues to connect to using plugin names probes = PluginManager().get_all_plugins() if not probes: log.error("No probe found, exiting application") sys.exit(1)
import config.parser as config import celery import logging from celery.utils.log import get_task_logger import brain.controllers.probectrl as probe_ctrl import brain.controllers.frontendtasks as celery_frontend from brain.helpers.sql import session_query # Get celery's logger log = get_task_logger(__name__) results_app = celery.Celery('resultstasks') config.conf_results_celery(results_app) config.configure_syslog(results_app) # IRMA specific debug messages are enables through # config file Section: log / Key: debug if config.debug_enabled(): def after_setup_logger_handler(sender=None, logger=None, loglevel=None, logfile=None, format=None, colorize=None, **kwds): config.setup_debug_logger(logging.getLogger(__name__)) log.debug("debug is enabled")
from brain.models.sqlobjects import User, Job, Scan import brain.controllers.scanctrl as scan_ctrl import brain.controllers.probectrl as probe_ctrl import brain.controllers.probetasks as celery_probe import brain.controllers.frontendtasks as celery_frontend from brain.helpers.sql import session_transaction from lib.irma.common.utils import IrmaTaskReturn, IrmaScanStatus, \ IrmaScanRequest # Get celery's logger log = get_task_logger(__name__) scan_app = celery.Celery('scantasks') config.conf_brain_celery(scan_app) config.configure_syslog(scan_app) interprocess_lock_path = config.get_lock_path() # IRMA specific debug messages are enables through # config file Section: log / Key: debug if config.debug_enabled(): def after_setup_logger_handler(sender=None, logger=None, loglevel=None, logfile=None, format=None, colorize=None, **kwds): config.setup_debug_logger(logging.getLogger(__name__)) log.debug("debug is enabled") celery.signals.after_setup_logger.connect(after_setup_logger_handler) celery.signals.after_setup_task_logger.connect(after_setup_logger_handler) # Refresh all probes before starting
# You may obtain a copy of the License in the top-level directory # of this distribution and at: # # http://www.apache.org/licenses/LICENSE-2.0 # # No part of the project, including this file, may be copied, # modified, propagated, or distributed except according to the # terms contained in the LICENSE file. import celery import config.parser as config from probe.helpers.celerytasks import async_call # declare a new Remote Brain application brain_app = celery.Celery('braintasks') config.conf_brain_celery(brain_app) config.configure_syslog(brain_app) # ============ # Task calls # ============ def register_probe(name, display_name, category, mimetype_regexp): """ send a task to the brain to register local probes""" task = async_call(brain_app, "brain.scan_tasks", "register_probe", args=[name, display_name, category, mimetype_regexp]) return task
from celery.utils.log import get_task_logger import brain.controllers.userctrl as user_ctrl import brain.controllers.scanctrl as scan_ctrl import brain.controllers.jobctrl as job_ctrl import brain.controllers.probetasks as celery_probe import brain.controllers.frontendtasks as celery_frontend import brain.controllers.ftpctrl as ftp_ctrl from lib.irma.common.utils import IrmaTaskReturn, IrmaScanStatus from lib.common.utils import UUID # Get celery's logger log = get_task_logger(__name__) scan_app = Celery('scantasks') config.conf_brain_celery(scan_app) config.configure_syslog(scan_app) results_app = Celery('resultstasks') config.conf_results_celery(results_app) config.configure_syslog(results_app) # =================== # Tasks declaration # =================== @scan_app.task(acks_late=True) def probe_list(): try: probe_list = celery_probe.get_probelist() if len(probe_list) > 0: