def test_configure_syslog(self, m_config, m_after_setup_logger,
m_after_setup_task_logger):
app = MagicMock()
m_config.log.syslog = True
module.configure_syslog(app)
app.conf.update.assert_called_with(CELERYD_LOG_COLOR=False)
m_after_setup_logger.connect.assert_called_once_with(module.setup_log)
m_after_setup_task_logger.connect.\
assert_called_once_with(module.setup_log)
def test_configure_syslog(self, m_config, m_after_setup_logger,
m_after_setup_task_logger):
app = MagicMock()
m_config.log.syslog = True
module.configure_syslog(app)
app.conf.update.assert_called_with(CELERYD_LOG_COLOR=False)
m_after_setup_logger.connect.assert_called_once_with(module.setup_log)
m_after_setup_task_logger.connect.\
assert_called_once_with(module.setup_log)
import multiprocessing
import config.parser as config
from celery import Celery
from fasteners import interprocess_locked
from brain.models.sqlobjects import Probe
from irma.common.base.exceptions import IrmaDatabaseResultNotFound, \
IrmaDatabaseError
from irma.common.plugin_result import PluginResult
log = logging.getLogger(__name__)
probe_app = Celery('probetasks')
config.conf_probe_celery(probe_app)
config.configure_syslog(probe_app)
# Time to cache the probe list
# to avoid asking to rabbitmq
PROBELIST_CACHE_TIME = 30
manager = multiprocessing.Manager()
available_probes = manager.list()
interprocess_lock_path = config.get_lock_path()
def register(name, display_name, category, mimetype_regexp, session):
try:
probe = Probe.get_by_name(name, session)
log.info("probe %s already registred "
"updating parameters: "
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.
import celery
import config.parser as config
from brain.helpers.celerytasks import async_call
frontend_app = celery.Celery('frontendtasks')
config.conf_frontend_celery(frontend_app)
config.configure_syslog(frontend_app)
def scan_launched(frontend_scanid, scan_request):
async_call(frontend_app,
"frontend.tasks",
"scan_launched",
args=[frontend_scanid, scan_request])
def scan_result(frontend_scanid, filename, probe, result):
async_call(frontend_app,
"frontend.tasks",
"scan_result",
args=[frontend_scanid, filename, probe, result])
from brain.models.sqlobjects import User, Job, Scan
import brain.controllers.ftpctrl as ftp_ctrl
import brain.controllers.scanctrl as scan_ctrl
import brain.controllers.probectrl as probe_ctrl
import brain.controllers.frontendtasks as celery_frontend
from brain.helpers.sql import session_transaction
from irma.common.base.utils import IrmaTaskReturn, IrmaScanRequest
RETRY_MAX_DELAY = 30
# Get celery's logger
log = get_task_logger(__name__)
scan_app = celery.Celery('scan_tasks')
config.conf_brain_celery(scan_app)
config.configure_syslog(scan_app)
interprocess_lock_path = config.get_lock_path()
# IRMA specific debug messages are enables through
# config file Section: log / Key: debug
if config.debug_enabled():
def after_setup_logger_handler(sender=None, logger=None, loglevel=None,
logfile=None, format=None,
colorize=None, **kwds):
config.setup_debug_logger(logging.getLogger(__name__))
log.debug("debug is enabled")
celery.signals.after_setup_logger.connect(after_setup_logger_handler)
celery.signals.after_setup_task_logger.connect(after_setup_logger_handler)
delay = 1
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.
import celery
import config.parser as config
from probe.helpers.celerytasks import async_call
# declare a new Remote Brain application
brain_app = celery.Celery('braintasks')
config.conf_brain_celery(brain_app)
config.configure_syslog(brain_app)
# ============
# Task calls
# ============
def register_probe(name, category, mimetype_regexp):
""" send a task to the brain to register local probes"""
task = async_call(brain_app, "brain.tasks", "register_probe",
args=[name, category, mimetype_regexp])
return task
logfile=None, format=None,
colorize=None, **kwds):
config.setup_debug_logger(logging.getLogger(__name__))
log.debug("debug is enabled")
celery.signals.after_setup_logger.connect(after_setup_logger_handler)
celery.signals.after_setup_task_logger.connect(after_setup_logger_handler)
# disable insecure serializer (disabled by default from 3.x.x)
if (kombu.VERSION.major) < 3:
kombu.disable_insecure_serializers()
# declare a new Local Probe application
probe_app = Celery("probe.tasks")
config.conf_probe_celery(probe_app)
config.configure_syslog(probe_app)
# discover plugins located at specified path
plugin_path = os.path.abspath("modules")
if not os.path.exists(plugin_path):
log.error("path {0} is invalid, cannot load probes".format(plugin_path))
sys.exit(1)
manager = PluginManager()
manager.discover(plugin_path)
# determine dynamically queues to connect to using plugin names
probes = manager.get_all_plugins()
if not probes:
log.error("No probe found, exiting application")
sys.exit(1)
from lib.common.utils import to_unicode
##############################################################################
# celery application configuration
##############################################################################
log = get_task_logger(__name__)
# disable insecure serializer (disabled by default from 3.x.x)
if (kombu.VERSION.major) < 3:
kombu.disable_insecure_serializers()
# declare a new application
app = Celery("probe.tasks")
config.conf_probe_celery(app)
config.configure_syslog(app)
# discover plugins located at specified path
plugin_path = os.path.abspath("modules")
if not os.path.exists(plugin_path):
log.error("path {0} is invalid, cannot load probes".format(plugin_path))
sys.exit(1)
manager = PluginManager()
manager.discover(plugin_path)
# determine dynamically queues to connect to using plugin names
probes = PluginManager().get_all_plugins()
if not probes:
log.error("No probe found, exiting application")
sys.exit(1)
import config.parser as config
import celery
import logging
from celery.utils.log import get_task_logger
import brain.controllers.probectrl as probe_ctrl
import brain.controllers.frontendtasks as celery_frontend
from brain.helpers.sql import session_query
# Get celery's logger
log = get_task_logger(__name__)
results_app = celery.Celery('resultstasks')
config.conf_results_celery(results_app)
config.configure_syslog(results_app)
# IRMA specific debug messages are enables through
# config file Section: log / Key: debug
if config.debug_enabled():
def after_setup_logger_handler(sender=None,
logger=None,
loglevel=None,
logfile=None,
format=None,
colorize=None,
**kwds):
config.setup_debug_logger(logging.getLogger(__name__))
log.debug("debug is enabled")
from brain.models.sqlobjects import User, Job, Scan
import brain.controllers.scanctrl as scan_ctrl
import brain.controllers.probectrl as probe_ctrl
import brain.controllers.probetasks as celery_probe
import brain.controllers.frontendtasks as celery_frontend
from brain.helpers.sql import session_transaction
from lib.irma.common.utils import IrmaTaskReturn, IrmaScanStatus, \
IrmaScanRequest
# Get celery's logger
log = get_task_logger(__name__)
scan_app = celery.Celery('scantasks')
config.conf_brain_celery(scan_app)
config.configure_syslog(scan_app)
interprocess_lock_path = config.get_lock_path()
# IRMA specific debug messages are enables through
# config file Section: log / Key: debug
if config.debug_enabled():
def after_setup_logger_handler(sender=None, logger=None, loglevel=None,
logfile=None, format=None,
colorize=None, **kwds):
config.setup_debug_logger(logging.getLogger(__name__))
log.debug("debug is enabled")
celery.signals.after_setup_logger.connect(after_setup_logger_handler)
celery.signals.after_setup_task_logger.connect(after_setup_logger_handler)
# Refresh all probes before starting
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.
import celery
import config.parser as config
from probe.helpers.celerytasks import async_call
# declare a new Remote Brain application
brain_app = celery.Celery('braintasks')
config.conf_brain_celery(brain_app)
config.configure_syslog(brain_app)
# ============
# Task calls
# ============
def register_probe(name, display_name, category, mimetype_regexp):
""" send a task to the brain to register local probes"""
task = async_call(brain_app,
"brain.scan_tasks",
"register_probe",
args=[name, display_name, category, mimetype_regexp])
return task
from celery.utils.log import get_task_logger
import brain.controllers.userctrl as user_ctrl
import brain.controllers.scanctrl as scan_ctrl
import brain.controllers.jobctrl as job_ctrl
import brain.controllers.probetasks as celery_probe
import brain.controllers.frontendtasks as celery_frontend
import brain.controllers.ftpctrl as ftp_ctrl
from lib.irma.common.utils import IrmaTaskReturn, IrmaScanStatus
from lib.common.utils import UUID
# Get celery's logger
log = get_task_logger(__name__)
scan_app = Celery('scantasks')
config.conf_brain_celery(scan_app)
config.configure_syslog(scan_app)
results_app = Celery('resultstasks')
config.conf_results_celery(results_app)
config.configure_syslog(results_app)
# ===================
# Tasks declaration
# ===================
@scan_app.task(acks_late=True)
def probe_list():
try:
probe_list = celery_probe.get_probelist()
if len(probe_list) > 0:
from lib.common.utils import to_unicode
##############################################################################
# celery application configuration
##############################################################################
log = get_task_logger(__name__)
# disable insecure serializer (disabled by default from 3.x.x)
if (kombu.VERSION.major) < 3:
kombu.disable_insecure_serializers()
# declare a new application
app = Celery("probe.tasks")
config.conf_probe_celery(app)
config.configure_syslog(app)
# discover plugins located at specified path
plugin_path = os.path.abspath("modules")
if not os.path.exists(plugin_path):
log.error("path {0} is invalid, cannot load probes".format(plugin_path))
sys.exit(1)
manager = PluginManager()
manager.discover(plugin_path)
# determine dynamically queues to connect to using plugin names
probes = PluginManager().get_all_plugins()
if not probes:
log.error("No probe found, exiting application")
sys.exit(1)
