from celery import Celery
from lib.config import ALERTS, LOGGING
from logging.config import dictConfig
print ALERTS
# Alert files to include
alerts_include = []
for alert in ALERTS.keys():
alerts_include.append('.'.join((alert).split('.')[:-1]))
alerts_include = list(set(alerts_include))
print alerts_include
app = Celery('alerts',
broker='amqp://',
backend='amqp://',
include=alerts_include)
CELERY_DISABLE_RATE_LIMITS = True
CELERYD_CONCURRENCY = 1
CELERY_IGNORE_RESULT = True
CELERY_DEFAULT_QUEUE = 'celery-default'
CELERY_QUEUES = {
'celery-default': {
"exchange": "celery-default",
"binding_key": "celery-default",
},
}
def load_and_register_alerts(self):
existing_alert_schedules = self.fetch_schedule_dict()
alert_schedules = {}
for alert_name, params in ALERTS.items():
# Register alerts in celery
try:
alert_tokens = alert_name.split(".")
alert_module_name = alert_tokens[0]
alert_classname = alert_tokens[-1]
alert_module = import_module(alert_module_name)
alert_class = getattr(alert_module, alert_classname)
current_app.register_task(alert_class())
except ImportError as e:
logger.exception("Error importing {0}: {1}".format(
alert_name, e))
pass
except Exception as e:
logger.exception("Generic error registering {0}: {1}".format(
alert_name, e))
pass
alert_schedule = {
"name": alert_name,
"task": alert_name,
"enabled": True,
}
if 'args' in params:
alert_schedule['args'] = params['args']
if 'kwargs' in params:
alert_schedule['kwargs'] = params['kwargs']
if isinstance(params['schedule'], timedelta):
alert_schedule['schedule_type'] = 'interval'
alert_schedule['celery_schedule'] = {
"every": params['schedule'].total_seconds(),
"period": "seconds"
}
elif isinstance(params['schedule'], crontab):
alert_schedule['schedule_type'] = 'crontab'
alert_schedule['celery_schedule'] = {
"minute": params['schedule']._orig_minute,
"hour": params['schedule']._orig_hour,
"day_of_week": params['schedule']._orig_day_of_week,
"day_of_month": params['schedule']._orig_day_of_month,
"month_of_year": params['schedule']._orig_month_of_year,
}
if alert_name not in existing_alert_schedules:
logger.debug("Inserting schedule for {0} into mongodb".format(
alert_name))
updated_alert_schedule = alert_schedule
else:
existing_schedule = existing_alert_schedules[alert_name]
logger.debug(
"Updating existing schedule ({0}) with new information into mongodb"
.format(alert_name))
existing_schedule['schedule_type'] = alert_schedule[
'schedule_type']
existing_schedule['celery_schedule'] = alert_schedule[
'celery_schedule']
updated_alert_schedule = existing_schedule
alert_schedules[alert_name] = PeriodicTask(
**updated_alert_schedule).to_dict()
self.update_schedules(alert_schedules)
import os
from celery import Celery
from importlib import import_module
from lib.config import ALERTS, LOGGING, RABBITMQ
from logging.config import dictConfig
# Alert files to include
alerts_include = []
for alert in ALERTS.keys():
alerts_include.append(".".join((alert).split(".")[:-1]))
alerts_include = list(set(alerts_include))
# XXX TBD this should get wrapped into an object that provides pyconfig
if os.getenv("OPTIONS_MQPROTOCOL", "amqp") == "sqs":
BROKER_URL = "sqs://@"
BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1]}
CELERY_RESULT_BACKEND = None
alert_queue_name = os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4]
CELERY_DEFAULT_QUEUE = alert_queue_name
CELERY_QUEUES = {
alert_queue_name: {"exchange": alert_queue_name, "binding_key": alert_queue_name}
}
else:
BROKER_URL = "amqp://{0}:{1}@{2}:{3}//".format(
RABBITMQ["mquser"], RABBITMQ["mqpassword"], RABBITMQ["mqserver"], RABBITMQ["mqport"]
)
CELERY_QUEUES = {
"celery-default": {"exchange": "celery-default", "binding_key": "celery-default"}
}
CELERY_DEFAULT_QUEUE = 'celery-default'
def load_and_register_alerts(self):
existing_alert_schedules = self.fetch_schedule_dict()
alert_schedules = {}
for alert_name, params in ALERTS.items():
# Register alerts in celery
try:
alert_tokens = alert_name.split(".")
alert_module_name = alert_tokens[0]
alert_classname = alert_tokens[-1]
alert_module = import_module(alert_module_name)
alert_class = getattr(alert_module, alert_classname)
current_app.register_task(alert_class())
except ImportError as e:
logger.exception("Error importing {0}: {1}".format(
alert_name, e))
pass
except Exception as e:
logger.exception("Generic error registering {0}: {1}".format(
alert_name, e))
pass
full_path_name = "{0}.{1}".format(alert_module_name,
alert_classname)
alert_schedule = {
"_id": str(ObjectId()),
"_cls": "PeriodicTask",
"name": full_path_name,
"task": full_path_name,
"enabled": True,
"args": [],
"kwargs": {},
}
if 'args' in params:
alert_schedule['args'] = params['args']
if 'kwargs' in params:
alert_schedule['kwargs'] = params['kwargs']
if isinstance(params['schedule'], timedelta):
alert_schedule['schedule_type'] = 'interval'
alert_schedule['interval'] = {
"every": params['schedule'].total_seconds(),
"period": "seconds"
}
alert_schedule['schedule_string'] = "{0} {1}".format(
params['schedule'].total_seconds(), "seconds")
elif isinstance(params['schedule'], crontab):
alert_schedule['schedule_type'] = 'crontab'
alert_schedule['crontab'] = {
"minute": params['schedule']._orig_minute,
"hour": params['schedule']._orig_hour,
"day_of_week": params['schedule']._orig_day_of_week,
"day_of_month": params['schedule']._orig_day_of_month,
"month_of_year": params['schedule']._orig_month_of_year,
}
alert_schedule[
'schedule_string'] = "{0} {1} {2} {3} {4}".format(
params['schedule']._orig_minute,
params['schedule']._orig_hour,
params['schedule']._orig_day_of_week,
params['schedule']._orig_day_of_month,
params['schedule']._orig_month_of_year,
)
if alert_name not in existing_alert_schedules:
logger.debug("Inserting schedule for {0} into mongodb".format(
full_path_name))
updated_alert_schedule = alert_schedule
else:
# Update schedule if it differs from file to api
del existing_alert_schedules[alert_name][
existing_alert_schedules[alert_name]['schedule_type']]
existing_alert_schedules[alert_name][
'schedule_type'] = alert_schedule['schedule_type']
existing_alert_schedules[alert_name][
alert_schedule['schedule_type']] = alert_schedule[
alert_schedule['schedule_type']]
existing_alert_schedules[alert_name][
'schedule_string'] = alert_schedule['schedule_string']
updated_alert_schedule = existing_alert_schedules[alert_name]
alert_schedules[alert_name] = updated_alert_schedule
resp = requests.post(url=RESTAPI_URL + "/updatealertschedules",
data=json.dumps(alert_schedules),
auth=self._restapi_jwt)
if not resp.ok:
raise Exception(
"Received error {0} from rest api when updating alerts schedules"
.format(resp.status_code))