def run(self): """ """ algo_config = config_loader.load('/opt/anna-molly/config/services.json') algo_config = algo_config.get(self.plugin, {None: None}) for service, options in algo_config.iteritems(): if service and options: option = {'service': service, 'params': options, 'plugin': self.plugin} app.task_runner.delay(SeasonalDecomposition, option) return True
def run(self): try: algo_config = config_loader.load(os.path.join(os.path.dirname(__file__), "../../config/services.json")) algo_config = algo_config.get(self.plugin_name)["worker_options"] except AttributeError: return None for service, options in algo_config.iteritems(): if service and options: params = {"params": options, "service": service} app.task_runner.delay(self.plugin, params) return True
def setup(options): CONFIG = config.load(options.config) WHITELIST = CONFIG['router']['whitelist'] log.debug("Whitelist: %s" % (WHITELIST)) BLACKLIST = [re.compile(x) for x in CONFIG['router']['blacklist']] log.debug("Blacklist: %s" % (BLACKLIST)) for pattern, mappings in WHITELIST.iteritems(): for _models in mappings: for model, default in _models.iteritems(): handler = partial(getattr(models, model), defaults=default) EE.add_listener(pattern, handler, count=-1) return CONFIG
def run(self): """ """ try: algo_config = config_loader.load('/opt/anna-molly/config/services.json') algo_config = algo_config.get(self.plugin) except AttributeError: return None for service, options in algo_config.iteritems(): if service and options: option = {'params': options, 'service': service} app.task_runner.delay(SeasonalDecomposition, option) return True
def run(self): try: algo_config = config_loader.load( os.path.join(os.path.dirname(__file__), '../../config/services.json')) algo_config = algo_config.get(self.plugin_name)['worker_options'] except AttributeError: return None for service, options in algo_config.iteritems(): if service and options: params = {'params': options, 'service': service} app.task_runner.delay(self.plugin, params) return True
def setup(options): """Load the config file and add listeners for whitelisted metrics. :param options: app options """ global WHITELIST, BLACKLIST CONFIG = config.load(options.config) WHITELIST = CONFIG['router']['whitelist'] log.debug("Whitelist: %s" % (WHITELIST)) BLACKLIST = [re.compile(x) for x in CONFIG['router']['blacklist']] log.debug("Blacklist: %s" % (BLACKLIST)) for pattern, mappings in WHITELIST.iteritems(): for _models in mappings: for model, default in _models.iteritems(): handler = partial(getattr(models, model), defaults=default) EE.add_listener(pattern, handler, count=-1) return CONFIG
""" The Celery Scheduler """ from datetime import timedelta import os.path import sys ROOT = os.path.join(os.path.dirname(__file__), '..') sys.path.append(ROOT) from lib.modules import config from lib.plugins.poll_task import PollTask analyzer_file = os.path.join(os.path.dirname(__file__), '../config/analyzer.json') analyzer_config = config.load(analyzer_file) BROKER_URL = analyzer_config['celery']['broker']['host'] # services config services_file = os.path.join(os.path.dirname(__file__), '../config/services.json') services_config = config.load(services_file) CELERYBEAT_SCHEDULE = {} for algorithm, algo_config in services_config.iteritems(): scheduler_options = algo_config['scheduler_options'] CELERYBEAT_SCHEDULE[algorithm] = { 'task': 'lib.app.task_runner', 'schedule': timedelta(seconds=scheduler_options['interval_secs']), 'args': (PollTask, scheduler_options['plugin_args']) }
import sys import os.path import traceback from celery import Celery from twitter.common import log from lib.modules import config CONFIG = config.load( os.path.join(os.path.dirname(__file__), '../config/analyzer.json')) app = Celery('anna-molly', broker=CONFIG['celery']['broker']['host']) app.conf.update(CELERY_IMPORTS=('celery.task.http'), CELERY_RESULT_BACKEND=CONFIG['celery']['backend']['host'], CELERY_RESULT_PERSISTENT=True, CELERY_TASK_RESULT_EXPIRES=None) @app.task() def task_runner(plugin, params): """ Run Forrest Run! :param plugin: plugin name :param params: plugin parameters :param dry_run: Default=False :return: """ try: plugin = plugin(config=CONFIG, logger=log, options=params) return plugin.run() except:
import sys from datetime import timedelta sys.path.append('../') from lib.modules import config from lib.plugins.poll_tukeys_filter import PollTukeysFilter from lib.plugins.poll_seasonal_decomposition import PollSeasonalDecomposition configfile = '/opt/anna-molly/config/analyzer.json' CONFIG = config.load(configfile) BROKER_URL = "redis://127.0.0.1:6379/0" CELERYBEAT_SCHEDULE = { '1': { 'task': 'app.task_runner', 'schedule': timedelta(seconds=30), 'args': (PollTukeysFilter, {}) } }
import sys import os.path import traceback from celery import Celery from twitter.common import log from lib.modules import config CONFIG = config.load(os.path.join(os.path.dirname(__file__), "../config/analyzer.json")) app = Celery("anna-molly", broker=CONFIG["celery"]["broker"]["host"]) app.conf.update( CELERY_IMPORTS=("celery.task.http"), CELERY_RESULT_BACKEND=CONFIG["celery"]["backend"]["host"], CELERY_RESULT_PERSISTENT=True, CELERY_TASK_RESULT_EXPIRES=None, ) @app.task() def task_runner(plugin, params): """ Run Forrest Run! :param plugin: plugin name :param params: plugin parameters :param dry_run: Default=False :return: """ try: plugin = plugin(config=CONFIG, logger=log, options=params) return plugin.run()
The Celery Scheduler """ from datetime import timedelta import os.path import sys ROOT = os.path.join(os.path.dirname(__file__), '..') sys.path.append(ROOT) from lib.modules import config from lib.plugins.poll_task import PollTask analyzer_file = os.path.join(os.path.dirname(__file__), '../config/analyzer.json') analyzer_config = config.load(analyzer_file) BROKER_URL = analyzer_config['celery']['broker']['host'] # services config services_file = os.path.join(os.path.dirname(__file__), '../config/services.json') services_config = config.load(services_file) CELERYBEAT_SCHEDULE = {} for algorithm, algo_config in services_config.iteritems(): scheduler_options = algo_config['scheduler_options'] CELERYBEAT_SCHEDULE[algorithm] = { 'task': 'lib.app.task_runner', 'schedule': timedelta(seconds=scheduler_options['interval_secs']), 'args': (PollTask, scheduler_options['plugin_args'])