def test_bad_canopsis_event_raises(self):
event = {}
with AmqpConnection(self.amqp_url) as ac:
amqp_pub = AmqpPublisher(ac, Mock())
with self.assertRaises(KeyError):
amqp_pub.canopsis_event(event, self.amqp_exname)
def test_bad_canopsis_event_raises(self):
event = {}
with AmqpConnection(self.amqp_url) as ac:
amqp_pub = AmqpPublisher(ac, Mock())
with self.assertRaises(KeyError):
amqp_pub.canopsis_event(event, self.amqp_exname)
def __init__(
self, namespace, confnamespace='object', storage=None,
autolog=False, amqp_pub=None, *args, **kwargs
):
super(Archiver, self).__init__()
self.namespace = namespace
self.namespace_log = namespace + '_log'
# Bulk operation configuration
self.last_bulk_insert_date = time()
self.bulk_ids = []
# How many events can be buffered
self.bulk_amount = 500
# What is the maximum duration until bulk insert
self.bulk_delay = 3
self.incoming_events = {}
self.autolog = autolog
self.logger.debug("Init Archiver on %s" % namespace)
self.account = Account(user="******", group="root")
if not storage:
self.logger.debug(" + Get storage")
self.storage = get_storage(
namespace=namespace,
logging_level=self.log_lvl
)
else:
self.storage = storage
self.conf_storage = get_storage(
namespace=confnamespace,
logging_level=self.log_lvl
)
self.conf_collection = self.conf_storage.get_backend(confnamespace)
self.collection = self.storage.get_backend(namespace)
if amqp_pub is None:
self.amqp_pub = AmqpPublisher(
get_default_amqp_connection(), self.logger)
self.reset_stealthy_event_duration = time()
self.reset_stats()
def __init__(self, logger=None, *args, **kwargs):
"""__init__
:param *args:
:param **kwargs:
"""
super(ContextGraphImport, self).__init__(logger, *args, **kwargs)
if logger is not None:
self.logger = logger
# Entities asked for update
self.entities_to_update = {}
# Entities to update
self.update = {}
self.delete = []
self.amqp_pub = AmqpPublisher(get_default_amqp_conn(), self.logger)
def __init__(self, amqp_pub=None):
"""
:param amqp_pub canopsis.common.amqp.AmqpPublisher:
"""
self.logger = Logger.get('watcher', LOG_PATH)
self.watcher_storage = Middleware.get_middleware_by_uri(
'mongodb-default-watcher://')
self.alert_storage = Middleware.get_middleware_by_uri(
'mongodb-periodical-alarm://')
self.sla_storage = Middleware.get_middleware_by_uri(
'storage-default-sla://')
self.context_graph = ContextGraph(self.logger)
self.pbehavior_manager = PBehaviorManager(
*PBehaviorManager.provide_default_basics())
self.amqp_pub = amqp_pub
if amqp_pub is None:
self.amqp_pub = AmqpPublisher(get_default_amqp_conn(), self.logger)
def provide_default_basics(cls, logger):
"""
Returns the default collection for the manager.
! Do not use in tests !
:rtype: (canopsis.common.collection.MongoCollection,
canopsis.common.amqp.AmqpPublisher)
"""
store = MongoStore.get_default()
collection = store.get_collection(name=cls.COLLECTION)
amqp_pub = AmqpPublisher(get_default_amqp_conn(), logger)
return (MongoCollection(collection), amqp_pub)
def get_default_app(logger=None, webconf=None, amqp_conn=None, amqp_pub=None):
if webconf is None:
webconf = Configuration.load(WebServer.CONF_PATH, Ini)
if logger is None:
logger = Logger.get('webserver', WebServer.LOG_FILE)
if amqp_conn is None:
amqp_conn = get_default_amqp_connection()
if amqp_pub is None:
amqp_pub = AmqpPublisher(amqp_conn, logger)
# Declare WSGI application
ws = WebServer(config=webconf, logger=logger, amqp_pub=amqp_pub).init_app()
app = ws.application
return app
def __init__(self, amqp_pub=None):
"""
:param amqp_pub canopsis.common.amqp.AmqpPublisher:
"""
self.logger = Logger.get('watcher', LOG_PATH)
self.watcher_storage = Middleware.get_middleware_by_uri(
'mongodb-default-watcher://')
self.alert_storage = Middleware.get_middleware_by_uri(
'mongodb-periodical-alarm://')
self.sla_storage = Middleware.get_middleware_by_uri(
'storage-default-sla://')
self.context_graph = ContextGraph(self.logger)
self.pbehavior_manager = PBehaviorManager(
*PBehaviorManager.provide_default_basics()
)
self.amqp_pub = amqp_pub
if amqp_pub is None:
self.amqp_pub = AmqpPublisher(get_default_amqp_conn(), self.logger)
def provide_default_basics(cls):
"""
Provide logger, config, storages...
! Do not use in tests !
:rtype: Union[canopsis.confng.simpleconf.Configuration
logging.Logger,
canopsis.storage.core.Storage,
canopsis.common.ethereal_data.EtherealData,
canopsis.storage.core.Storage,
canopsis.context_graph.manager.ContextGraph,
canopsis.watcher.manager.Watcher]
"""
config = Configuration.load(Alerts.CONF_PATH, Ini)
conf_store = Configuration.load(MongoStore.CONF_PATH, Ini)
mongo = MongoStore(config=conf_store)
config_collection = MongoCollection(
mongo.get_collection(name=cls.CONFIG_COLLECTION))
filter_ = {'crecord_type': 'statusmanagement'}
config_data = EtherealData(collection=config_collection,
filter_=filter_)
logger = Logger.get('alerts', cls.LOG_PATH)
alerts_storage = Middleware.get_middleware_by_uri(
cls.ALERTS_STORAGE_URI
)
filter_storage = Middleware.get_middleware_by_uri(
cls.FILTER_STORAGE_URI
)
context_manager = ContextGraph(logger)
watcher_manager = Watcher()
pbehavior_manager = PBehaviorManager(*PBehaviorManager.provide_default_basics())
amqp_pub = AmqpPublisher(get_default_amqp_conn(), logger)
event_publisher = StatEventPublisher(logger, amqp_pub)
return (config, logger, alerts_storage, config_data,
filter_storage, context_manager, watcher_manager,
event_publisher, pbehavior_manager)
def __init__(self,
next_amqp_queues=[],
next_balanced=False,
name="worker1",
beat_interval=60,
logging_level=INFO,
exchange_name=DIRECT_EXCHANGE_NAME,
routing_keys=[],
camqp_custom=None,
max_retries=5,
*args,
**kwargs):
super(Engine, self).__init__()
self.logging_level = logging_level
self.debug = logging_level == DEBUG
init = Init()
self.logger = init.getLogger(name, logging_level=self.logging_level)
log_handler = FileHandler(filename=join(
root_path, 'var', 'log', 'engines', '{0}.log'.format(name)))
log_handler.setFormatter(
Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
# Log in file
self.logger.addHandler(log_handler)
self.RUN = True
self.name = name
# Set parametrized Amqp for testing purposes
if camqp_custom is None:
self.Amqp = Amqp
else:
self.Amqp = camqp_custom
# self.amqp handles the consumption of events from rabbitmq. The
# publication of events from self.amqp is deprecated.
self.amqp = None
# self.beat_amqp_publisher and self.work_amqp_publisher handle the
# publication of events (they are separated to prevent sharing a
# channel between two threads).
self.beat_amqp_publisher = AmqpPublisher(get_default_amqp_connection(),
self.logger)
self.work_amqp_publisher = AmqpPublisher(get_default_amqp_connection(),
self.logger)
self.amqp_queue = "Engine_{0}".format(self.name)
self.routing_keys = routing_keys
self.exchange_name = exchange_name
self.perfdata_retention = 3600
self.next_amqp_queues = next_amqp_queues
self.get_amqp_queue = cycle(self.next_amqp_queues)
# Get from internal or external queue
self.next_balanced = next_balanced
self.max_retries = max_retries
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
self.thd_warn_sec_per_evt = 0.6
self.thd_crit_sec_per_evt = 0.9
self.beat_interval = beat_interval
self.beat_last = time()
self.create_queue = True
self.send_stats_event = True
self.rk_on_error = []
self.last_stat = int(time())
self.logger.info("Engine initialized")
class Engine(object):
etype = 'Engine'
def __init__(self,
next_amqp_queues=[],
next_balanced=False,
name="worker1",
beat_interval=60,
logging_level=INFO,
exchange_name=DIRECT_EXCHANGE_NAME,
routing_keys=[],
camqp_custom=None,
max_retries=5,
*args,
**kwargs):
super(Engine, self).__init__()
self.logging_level = logging_level
self.debug = logging_level == DEBUG
init = Init()
self.logger = init.getLogger(name, logging_level=self.logging_level)
log_handler = FileHandler(filename=join(
root_path, 'var', 'log', 'engines', '{0}.log'.format(name)))
log_handler.setFormatter(
Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
# Log in file
self.logger.addHandler(log_handler)
self.RUN = True
self.name = name
# Set parametrized Amqp for testing purposes
if camqp_custom is None:
self.Amqp = Amqp
else:
self.Amqp = camqp_custom
# self.amqp handles the consumption of events from rabbitmq. The
# publication of events from self.amqp is deprecated.
self.amqp = None
# self.beat_amqp_publisher and self.work_amqp_publisher handle the
# publication of events (they are separated to prevent sharing a
# channel between two threads).
self.beat_amqp_publisher = AmqpPublisher(get_default_amqp_connection(),
self.logger)
self.work_amqp_publisher = AmqpPublisher(get_default_amqp_connection(),
self.logger)
self.amqp_queue = "Engine_{0}".format(self.name)
self.routing_keys = routing_keys
self.exchange_name = exchange_name
self.perfdata_retention = 3600
self.next_amqp_queues = next_amqp_queues
self.get_amqp_queue = cycle(self.next_amqp_queues)
# Get from internal or external queue
self.next_balanced = next_balanced
self.max_retries = max_retries
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
self.thd_warn_sec_per_evt = 0.6
self.thd_crit_sec_per_evt = 0.9
self.beat_interval = beat_interval
self.beat_last = time()
self.create_queue = True
self.send_stats_event = True
self.rk_on_error = []
self.last_stat = int(time())
self.logger.info("Engine initialized")
def new_amqp_queue(self, amqp_queue, routing_keys, on_amqp_event,
exchange_name):
self.amqp.add_queue(queue_name=amqp_queue,
routing_keys=routing_keys,
callback=on_amqp_event,
exchange_name=exchange_name,
no_ack=True,
exclusive=False,
auto_delete=False)
def pre_run(self):
pass
def post_run(self):
pass
def run(self):
def ready():
self.logger.info(" + Ready!")
self.logger.info("Start Engine with pid {0}".format(getpid()))
self.amqp = self.Amqp(logging_level=self.logging_level,
logging_name="{0}-amqp".format(self.name),
on_ready=ready,
max_retries=self.max_retries)
if self.create_queue:
self.new_amqp_queue(self.amqp_queue, self.routing_keys,
self.on_amqp_event, self.exchange_name)
self.amqp.start()
self.pre_run()
while self.RUN:
# Beat
if self.beat_interval:
now = time()
if now > (self.beat_last + self.beat_interval):
self._beat()
self.beat_last = now
try:
sleep(1)
except Exception as err:
self.logger.error("Error in break time: {0}".format(err))
self.RUN = False
except KeyboardInterrupt:
self.logger.info(u'Stop request')
self.RUN = False
self.post_run()
self.logger.info("Stop Engine")
self.stop()
self.logger.info("End of Engine")
def on_amqp_event(self, event, msg):
try:
self._work(event, msg)
except Exception as err:
if event['rk'] not in self.rk_on_error:
self.logger.error(err)
self.logger.error("Impossible to deal with: {0}".format(event))
self.rk_on_error.append(event['rk'])
self.next_queue(event)
def _work(self, event, msg=None, *args, **kargs):
start = time()
error = False
try:
if self.debug:
if 'processing' not in event:
event['processing'] = {}
event['processing'][self.etype] = start
wevent = self.work(event, msg, *args, **kargs)
if wevent != DROP:
if isinstance(wevent, dict):
event = wevent
self.next_queue(event)
except Exception as err:
error = True
self.logger.error("Worker raise exception: {0}".format(err))
self.logger.error(format_exc())
if error:
self.counter_error += 1
elapsed = time() - start
if elapsed > 3:
self.logger.warning("Elapsed time %.2f seconds" % elapsed)
self.counter_event += 1
self.counter_worktime += elapsed
def work(self, event, amqp_msg):
return event
def next_queue(self, event):
if self.next_balanced:
queue_name = self.get_amqp_queue.next()
if queue_name:
try:
self.work_amqp_publisher.direct_event(event, queue_name)
except Exception as e:
self.logger.exception("Unable to send event to next queue")
else:
for queue_name in self.next_amqp_queues:
try:
self.work_amqp_publisher.direct_event(event, queue_name)
except Exception as e:
self.logger.exception("Unable to send event to next queue")
def _beat(self):
now = int(time())
if self.last_stat + 60 <= now:
self.logger.debug(" + Send stats")
self.last_stat = now
evt_per_sec = 0
sec_per_evt = 0
if self.counter_event:
evt_per_sec = float(self.counter_event) / self.beat_interval
self.logger.debug(" + %0.2f event(s)/seconds", evt_per_sec)
if self.counter_worktime and self.counter_event:
sec_per_evt = self.counter_worktime / self.counter_event
self.logger.debug(" + %0.5f seconds/event", sec_per_evt)
# Submit event
if self.send_stats_event and self.counter_event != 0:
state = 0
if sec_per_evt > self.thd_warn_sec_per_evt:
state = 1
if sec_per_evt > self.thd_crit_sec_per_evt:
state = 2
perf_data_array = [{
'retention': self.perfdata_retention,
'metric': 'cps_evt_per_sec',
'value': round(evt_per_sec, 2),
'unit': 'evt'
}, {
'retention': self.perfdata_retention,
'metric': 'cps_sec_per_evt',
'value': round(sec_per_evt, 5),
'unit': 's',
'warn': self.thd_warn_sec_per_evt,
'crit': self.thd_crit_sec_per_evt
}]
self.logger.debug(" + State: {0}".format(state))
event = forger(connector="Engine",
connector_name="engine",
event_type="check",
source_type="resource",
resource=self.amqp_queue,
state=state,
state_type=1,
output="%0.2f evt/sec, %0.5f sec/evt" %
(evt_per_sec, sec_per_evt),
perf_data_array=perf_data_array)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send perfdata")
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
try:
self.beat()
except Exception as err:
self.logger.error("Beat raise exception: {0}".format(err))
self.logger.error(print_exc())
def beat(self):
pass
def stop(self):
self.RUN = False
# cancel self consumer
self.amqp.cancel_queues()
self.amqp.stop()
self.amqp.join()
self.logger.debug(" + Stopped")
class Lock(object):
def __init__(self, engine, name, *args, **kwargs):
super(Engine.Lock, self).__init__()
self.name = name
self.lock_id = '{0}.{1}'.format(engine.etype, name)
self.storage = get_storage(
namespace='lock',
logging_level=engine.logging_level,
account=Account(user='******', group='root')).get_backend()
self.engine = engine
self.lock = {}
def own(self):
now = time()
last = self.lock.get('t', now)
if self.lock.get('l', False) \
and (now - last) < self.engine.beat_interval:
self.engine.logger.debug(
'Another engine {0} is already holding the lock {1}'.
format(self.engine.etype, self.name))
return False
else:
self.engine.logger.debug(
'Lock {1} on engine {0}, processing...'.format(
self.engine.etype, self.name))
return True
return False
def __enter__(self):
lock = self.storage.find_and_modify(query={'_id': self.lock_id},
update={'$set': {
'l': True
}},
upsert=True)
if lock is not None:
self.lock = lock
if 't' not in self.lock:
self.lock['t'] = 0
return self
def __exit__(self, type, value, tb):
if self.own():
self.engine.logger.debug(
'Release lock {1} on engine {0}'.format(
self.engine.etype, self.name))
self.storage.save({
'_id': self.lock_id,
'l': False,
't': time()
})
@classmethod
def release(cls, lock_id, object_storage):
object_storage.update({'_id': lock_id},
{'$set': {
'l': False,
't': time()
}},
upsert=True)
class Watcher:
"""Watcher class"""
def __init__(self, amqp_pub=None):
"""
:param amqp_pub canopsis.common.amqp.AmqpPublisher:
"""
self.logger = Logger.get('watcher', LOG_PATH)
self.watcher_storage = Middleware.get_middleware_by_uri(
'mongodb-default-watcher://')
self.alert_storage = Middleware.get_middleware_by_uri(
'mongodb-periodical-alarm://')
self.sla_storage = Middleware.get_middleware_by_uri(
'storage-default-sla://')
self.context_graph = ContextGraph(self.logger)
self.pbehavior_manager = PBehaviorManager(
*PBehaviorManager.provide_default_basics()
)
self.amqp_pub = amqp_pub
if amqp_pub is None:
self.amqp_pub = AmqpPublisher(get_default_amqp_conn())
def get_watcher(self, watcher_id):
"""Retreive from database the watcher specified by is watcher id.
:param str watcher_id: the watcher id
:return dict: the wanted watcher. None, if no watcher match the
watcher_id
"""
watcher = self.context_graph.get_entities_by_id(watcher_id)
try:
return watcher[0]
except IndexError:
return None
def create_watcher(self, body):
"""
Create watcher entity in context and link to entities.
:param dict body: watcher conf
"""
watcher_id = body['_id']
try:
watcher_finder = json.loads(body['mfilter'])
except ValueError:
self.logger.error('can t decode mfilter')
return None
except KeyError:
self.logger.error('no filter')
return None
depends_list = self.context_graph.get_entities(
query=watcher_finder,
projection={'_id': 1}
)
self.watcher_storage.put_element(body)
depend_list = []
for entity_id in depends_list:
depend_list.append(entity_id['_id'])
entity = ContextGraph.create_entity_dict(
id=watcher_id,
name=body['display_name'],
etype='watcher',
impact=[],
depends=depend_list
)
# adding the fields specific to the Watcher entities
entity['mfilter'] = body['mfilter']
entity['state'] = 0
try:
self.context_graph.create_entity(entity)
except ValueError:
self.context_graph.update_entity(entity)
self.compute_state(watcher_id)
return True # TODO: return really something
def update_watcher(self, watcher_id, updated_field):
"""Update the watcher specified by is watcher id with updated_field.
Raise a ValueError, if the watcher_id do not match any entity.
:param str watcher_id: the watcher_id of the watcher to update
:param dict updated_field: the fields to update
:returns: the updated Watcher
:rtype: <Watcher>
"""
watcher = self.get_watcher(watcher_id)
if watcher is None:
raise ValueError("No watcher found for the following"
" id: {}".format(watcher_id))
if "mfilter" in watcher.keys() and "mfilter" in updated_field.keys():
if updated_field['mfilter'] != watcher['mfilter']:
watcher['mfilter'] = updated_field['mfilter']
query = json.loads(updated_field['mfilter'])
entities = self.context_graph.get_entities(
query=query, projection={'_id': 1})
watcher["depends"] = [entity["_id"] for entity in entities]
for key in updated_field:
if key == "infos": # update fields inside infos
for info_key in updated_field["infos"]:
watcher["infos"][info_key] = updated_field["infos"][
info_key]
watcher[key] = updated_field[key]
self.context_graph.update_entity(watcher)
def delete_watcher(self, watcher_id):
"""
Delete watcher & disable watcher entity in context.
:param string watcher_id: watcher_id
:returns: the mongodb dict response
"""
self.context_graph.delete_entity(watcher_id)
self.sla_storage.remove_elements(ids=[watcher_id])
return self.watcher_storage.remove_elements(ids=[watcher_id])
def alarm_changed(self, alarm_id):
"""
Launch a computation of a watcher state.
:param alarm_id: alarm id
"""
watchers = self.context_graph.get_entities(query={'type': 'watcher'})
for i in watchers:
if alarm_id in i['depends']:
self.compute_state(i['_id'])
def compute_watchers(self):
"""
Compute all watchers states.
"""
watchers = list(self.watcher_storage.get_elements(query={}))
for watcher in watchers:
self.compute_state(watcher['_id'])
def compute_state(self, watcher_id):
"""
Send an event watcher with the new state of the watcher.
:param watcher_id: watcher id
"""
try:
watcher_entity = self.context_graph.get_entities(
query={'_id': watcher_id})[0]
except IndexError:
return None
entities = watcher_entity['depends']
query = {"_id": {"$in": entities},
"enabled": True}
cursor = self.context_graph.get_entities(query=query,
projection={"_id": 1})
entities = []
for ent in cursor:
entities.append(ent["_id"])
display_name = watcher_entity['name']
alarm_list = list(self.alert_storage._backend.find({
'$and': [
{'d': {'$in': entities}},
{
'$or': [
{'v.resolved': None},
{'v.resolved': {'$exists': False}}
]
}
]
}))
states = []
for alarm in alarm_list:
active_pb = self.pbehavior_manager.get_active_pbehaviors(
[alarm['d']]
)
if len(active_pb) == 0:
states.append(alarm['v']['state']['val'])
nb_entities = len(entities)
nb_crit = states.count(Check.CRITICAL)
nb_major = states.count(Check.MAJOR)
nb_minor = states.count(Check.MINOR)
nb_ok = nb_entities - (nb_crit + nb_major + nb_minor)
# here add selection for calculation method actually it's worst state
# by default and think to add pbehavior in tab
computed_state = self.worst_state(nb_crit, nb_major, nb_minor)
output = '{0} ok, {1} minor, {2} major, {3} critical'.format(
nb_ok, nb_minor, nb_major, nb_crit)
if computed_state != watcher_entity.get('state', None):
watcher_entity['state'] = computed_state
self.context_graph.update_entity(watcher_entity)
self.publish_event(
display_name,
computed_state,
output,
watcher_entity['_id']
)
def compute_slas(self):
"""
Launch the sla calcul for each watchers.
"""
watcher_list = self.context_graph.get_entities(
query={'type': 'watcher',
'infos.enabled': True})
for watcher in watcher_list:
self.sla_compute(watcher['_id'], watcher['infos']['state'])
def publish_event(self, display_name, computed_state, output, _id):
"""
Publish an event watcher on amqp.
TODO: move that elsewhere (not specific to watchers)
:param display_name: watcher display_name
:param computed_state: watcher state
:param output: watcher output
"""
event = forger(
connector="canopsis",
connector_name="engine",
event_type="watcher",
source_type="component",
component=_id,
state=computed_state,
output=output,
perf_data_array=[],
display_name=display_name)
self.amqp_pub.canopsis_event(event)
def sla_compute(self, watcher_id, state):
"""
Launch the sla calcul.
:param watcher_id: watcher id
:param state: watcher state
"""
# sla_tab = list(
# self.sla_storage.get_elements(query={'_id': watcher_id}))[0]
# sla_tab['states'][state] = sla_tab['states'][state] + 1
# self.sla_storage.put_element(sla_tab)
# watcher_conf = list(
# self[self.WATCHER_STORAGE].get_elements(
# query={'_id': watcher_id})
# )[0]
# sla = Sla(self[self.WATCHER_STORAGE],
# 'test/de/rk/on/verra/plus/tard',
# watcher_conf['sla_output_tpl'],
# watcher_conf['sla_timewindow'],
# watcher_conf['sla_warning'],
# watcher_conf['alert_level'],
# watcher_conf['display_name'])
# self.logger.critical('{0}'.format((
# sla_tab['states']/
# (sla_tab['states'][1] +
# sla_tab['states'][2] +
# sla_tab['states'][3]))))
pass
@staticmethod
def worst_state(nb_crit, nb_major, nb_minor):
"""Calculate the worst state.
:param int nb_crit: critical number
:param int nb_major: major number
:param int nb_minor: minor number
:return int state: return the worst state
"""
if nb_crit > 0:
return 3
elif nb_major > 0:
return 2
elif nb_minor > 0:
return 1
return 0
class ContextGraphImport(ContextGraph):
"""The manager in charge of an import of a context.
"""
# TODO add a feature to restore the context if an error occured during
# while is pushed into the database
K_LINKS = "links"
K_FROM = "from"
K_TO = "to"
K_CIS = "cis"
K_ID = "_id"
K_NAME = "name"
K_TYPE = "type"
K_DEPENDS = "depends"
K_IMPACT = "impact"
K_MEASUREMENTS = "measurements"
K_INFOS = "infos"
K_ACTION = "action"
K_ENABLE = "enable"
K_DISABLE = "disable"
K_PROPERTIES = "action_properties"
K_ENABLED = "enabled"
# If you add an action, remember to add in the a_pattern string in method
# import_checker
A_DELETE = "delete"
A_CREATE = "create"
A_SET = "set"
A_UPDATE = "update"
A_DISABLE = "disable"
A_ENABLE = "enable"
__A_PATTERN = "^delete$|^create$|^update$|^set$|^disable$|^enable$"
__T_PATTERN = "^resource$|^component$|^connector$|^watcher$"
__CI_REQUIRED = [K_ID, K_ACTION, K_TYPE]
__LINK_REQUIRED = [K_FROM, K_TO, K_ACTION]
CIS_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"required": __CI_REQUIRED,
"uniqueItems": True,
"properties": {
K_ID: {
"type": "string"
},
K_NAME: {
"type": "string"
},
K_MEASUREMENTS: {
"type": "array",
"items": {
"type": "string"
}
},
K_INFOS: {
"type": "object"
},
K_ACTION: {
"type": "string",
"pattern": __A_PATTERN
},
K_TYPE: {
"type": "string",
"pattern": __T_PATTERN
},
K_PROPERTIES: {
"type": "object"
}
}
}
LINKS_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"uniqueItems": True,
"type": "object",
"required": __LINK_REQUIRED,
"properties": {
K_ID: {
"type": "string"
},
K_FROM: {
"type": "array",
"items": {
"type": "string"
}
},
K_TO: {
"type": "string"
},
K_INFOS: {
"type": "object"
},
K_ACTION: {
"type": "string",
"pattern": __A_PATTERN
},
K_PROPERTIES: {
"type": "object"
}
}
}
def __init__(self, logger=None, *args, **kwargs):
"""__init__
:param *args:
:param **kwargs:
"""
super(ContextGraphImport, self).__init__(logger, *args, **kwargs)
if logger is not None:
self.logger = logger
# Entities asked for update
self.entities_to_update = {}
# Entities to update
self.update = {}
self.delete = []
self.amqp_pub = AmqpPublisher(get_default_amqp_conn(), self.logger)
@classmethod
def check_element(cls, element, type_):
"""Check an element with a schema schema specified by his type.
:param element: the element to check
:param type_: the expected type of the element
:raise: ValueError if the type_ is not correct
:raise: ValidationError if the element does not match the schema
"""
if type_ == cls.K_LINKS:
schema = cls.LINKS_SCHEMA
elif type_ == cls.K_CIS:
schema = cls.CIS_SCHEMA
else:
raise ValueError("Unknowed type {0}\n".format(type_))
jsonschema.validate(element, schema)
state = element[ContextGraphImport.K_ACTION]
if state == ContextGraphImport.A_DISABLE\
or state == ContextGraphImport.A_ENABLE:
element[ContextGraphImport.K_PROPERTIES][state]
def clean_attributes(self):
del self.entities_to_update
del self.update
del self.delete
self.entities_to_update = dict()
self.update = dict()
self.delete = list()
def __get_entities_to_update(self, file_):
"""Return every entities id required for the update
If a ci or link does not match the schema a ValidationError is raised.
:param json: the json with every actions required for the update
:param rtype: a dict with the entity id as a key and the entity as
a value
"""
# a set so no duplicate ids without effort and lower time complexity
ids_cis = set()
ids_links = set()
def __get_entities_to_update_links(file_):
"""Parse the file_ to extract every link"""
fd = open(file_, 'r')
for link in ijson.items(fd, "{0}.item".format(self.K_LINKS)):
self.check_element(link, self.K_LINKS)
for id_ in link[self.K_FROM]:
ids_links.add(id_)
# ids_cis.add(ci[self.K_ID])
ids_links.add(link[self.K_TO])
fd.close()
def __get_entities_to_update_cis(file_):
"""Parse the file_ to extract every CI"""
fd = open(file_, 'r')
for ci in ijson.items(fd, "{0}.item".format(self.K_CIS)):
self.check_element(ci, self.K_CIS)
ids_cis.add(ci[self.K_ID])
# we need to retreive every related entity to update the links
if ci[self.K_ACTION] == self.A_DELETE:
# FIXME do the get_entities_by_id in one call Then add all
# impacts depends
entity = self.get_entities_by_id(ci[self.K_ID])[0]
for id_ in entity["depends"] + entity["impact"]:
ids_cis.add(id_)
fd.close()
cis_thd = ExceptionThread(group=None,
target=__get_entities_to_update_cis,
name="cis_thread",
args=(file_, ))
links_thd = ExceptionThread(group=None,
target=__get_entities_to_update_links,
name="links_thread",
args=(file_, ))
threads = [cis_thd, links_thd]
cis_thd.start()
links_thd.start()
cis_thd.join()
links_thd.join()
# Unqueue an raise existing exception
for thread in threads:
try:
excep = thread.except_queue.get_nowait()
except Queue.Empty:
pass
else:
self.logger.error("Exception in {0}".format(thread.getName()))
self.logger.exception(excep)
raise excep
ids = ids_links.union(ids_cis)
result = self.get_entities_by_id(list(ids))
ctx = {}
# transform "depends" and "impact" list in set for improved performance
for doc in result:
doc[self.K_DEPENDS] = set(doc[self.K_DEPENDS])
doc[self.K_IMPACT] = set(doc[self.K_IMPACT])
ctx[doc[self.K_ID]] = doc
return ctx
def __a_delete_entity(self, ci):
"""Update the entities related with the entity to be deleted disigned
by ci and store them into self.update. Add the id of entity to be
deleted into self.delete.
If the entity to be deleted is not initially store in the context,
a ValueError will be raised.
:param ci: the ci (see the JSON specification).
"""
id_ = ci[self.K_ID]
try:
entity = self.entities_to_update[id_]
except KeyError:
desc = "No entity found for the following id : {}".format(id_)
raise ValueError(desc)
# Update the depends/impact link
for ent_id in entity[self.K_DEPENDS]:
if ent_id in self.delete:
# the entity of id ent_id is already deleted, skipping
continue
if ent_id not in self.update:
self.update[ent_id] = self.entities_to_update[ent_id].copy()
try:
if id_ in self.update[ent_id][self.K_IMPACT]:
self.update[ent_id][self.K_IMPACT].remove(id_)
except ValueError:
raise ValueError("Try to remove {0} from impacts field of "
"entity {1}.".format(id_, ent_id))
# Update the impact/depends link
for ent_id in entity[self.K_IMPACT]:
if ent_id in self.delete:
# the entity of id ent_id is already deleted, skipping
continue
if ent_id not in self.update:
self.update[ent_id] = self.entities_to_update[ent_id].copy()
try:
if id_ in self.update[ent_id][self.K_DEPENDS]:
self.update[ent_id][self.K_DEPENDS].remove(id_)
except ValueError:
raise ValueError("Try to remove {0} from impacts field of "
"entity {1}.".format(id_, ent_id))
if id_ in self.update:
self.update.pop(id_)
self.delete.append(id_)
def __a_update_entity(self, ci):
"""Update the entity with the information stored into the ci and store
the result into self.update.
If the entity to be updated is not initially store in the context,
a ValueError will be raised.
:param ci: the ci (see the JSON specification).
"""
if ci[self.K_ID] not in self.entities_to_update:
desc = "The ci of id {0} does not match any existing"\
" entity.".format(ci[self.K_ID])
raise ValueError(desc)
entity = self.entities_to_update[ci[self.K_ID]]
for key in [self.K_ACTION, self.K_PROPERTIES]:
try:
del ci[key]
except KeyError:
msg = "No key {0} in ci of id {1}."
self.logger.debug(msg.format(key, ci[self.K_ID]))
for key in ci:
entity[key] = ci[key]
self.update[ci[self.K_ID]] = entity
def __a_set_entity(self, ci):
"""Create an entity with a ci and store it into self.update
If the new entity is initially store in the context, a ValueError will
be raised.
:param ci: the ci (see the JSON specification).
"""
# TODO handle the creation of the name if needed and if the id
# match the id scheme used in canopsis
if ci[self.K_ID] in self.entities_to_update:
desc = ("The ci of id {} match an existing entity. Updating it.".
format(ci["_id"]))
self.logger.info(desc)
entity_to_update = self.entities_to_update[ci[self.K_ID]].copy()
dict_merge(entity_to_update, ci)
ci = entity_to_update
# set default value for required fields
if self.K_NAME not in ci:
ci[self.K_NAME] = ci[self.K_ID]
if self.K_DEPENDS not in ci:
ci[self.K_DEPENDS] = set()
else:
ci[self.K_DEPENDS] = set(ci[self.K_DEPENDS])
if self.K_IMPACT not in ci:
ci[self.K_IMPACT] = set()
else:
ci[self.K_IMPACT] = set(ci[self.K_IMPACT])
if self.K_MEASUREMENTS not in ci:
ci[self.K_MEASUREMENTS] = []
if self.K_INFOS not in ci:
ci[self.K_INFOS] = {}
for key in [self.K_ACTION, self.K_PROPERTIES]:
try:
del ci[key]
except KeyError:
self.logger.debug("No key {0} in ci of id {1}.".format(
key, ci[self.K_ID]))
entity = {}
for key in ci:
entity[key] = ci[key]
self.update[ci[self.K_ID]] = entity
def __a_create_entity(self, ci):
"""Create an entity with a ci and store it into self.update
If the new entity is initially store in the context, a ValueError will
be raised.
:param ci: the ci (see the JSON specification).
"""
# TODO handle the creation of the name if needed and if the id
# match the id scheme used in canopsis
if ci[self.K_ID] in self.entities_to_update:
desc = ("The ci of id {} match an existing entity. Overriding it.".
format(ci["_id"]))
self.logger.info(desc)
# set default value for required fields
if self.K_NAME not in ci:
ci[self.K_NAME] = ci[self.K_ID]
if self.K_DEPENDS not in ci:
ci[self.K_DEPENDS] = set()
else:
ci[self.K_DEPENDS] = set(ci[self.K_DEPENDS])
if self.K_IMPACT not in ci:
ci[self.K_IMPACT] = set()
else:
ci[self.K_IMPACT] = set(ci[self.K_IMPACT])
if self.K_MEASUREMENTS not in ci:
ci[self.K_MEASUREMENTS] = []
if self.K_INFOS not in ci:
ci[self.K_INFOS] = {}
for key in [self.K_ACTION, self.K_PROPERTIES]:
try:
del ci[key]
except KeyError:
self.logger.debug("No key {0} in ci of id {1}.".format(
key, ci[self.K_ID]))
entity = {}
for key in ci:
entity[key] = ci[key]
self.update[ci[self.K_ID]] = entity
def __change_state_entity(self, ci, state):
"""Change the state (enable/disable) of an entity and store the result
into self.update.
If state does not match enable or disable, a ValueError will be raised.
:param ci: the ci (see the JSON specification).
:param state: if the state is "disable", the timestamp of the
deactivation of the entity will be store into the fields infos.disable.
Same behaviour with "enable" but the timestamp will be store into
infos.enable.
"""
if state != self.A_DISABLE and state != self.A_ENABLE:
raise ValueError("{0} is not a valid state.".format(state))
id_ = ci[self.K_ID]
if id_ not in self.entities_to_update:
desc = "The ci of id {0} does not match any existing"\
" entity.".format(id_)
raise ValueError(desc)
# If the entity is not in the update dict, add it
if id_ not in self.update:
self.update[id_] = self.entities_to_update[id_].copy()
if state == self.A_DISABLE:
key_history = "disable_history"
key = self.K_DISABLE
self.update[id_][self.K_ENABLED] = False
else:
key_history = "enable_history"
key = self.K_ENABLE
self.update[id_][self.K_ENABLED] = True
# Update entity the fields enable/disable of infos
timestamp = ci[self.K_PROPERTIES][key]
if not isinstance(timestamp, list):
if timestamp is None:
timestamp = []
else:
timestamp = [timestamp]
if self.update[id_].has_key(key_history):
if self.update[id_][key_history] is None:
self.update[id_][key_history] = timestamp
else:
self.update[id_][key_history] += timestamp
else:
self.update[id_][key_history] = timestamp
def __a_disable_entity(self, ci):
"""Disable an entity defined by ci. For more information, see
__change_state.
:param ci: the ci (see the JSON specification).
"""
self.__change_state_entity(ci, self.K_DISABLE)
def __a_enable_entity(self, ci):
"""Enable an entity defined by ci. For more information, see
__change_state.
:param ci: the ci (see the JSON specification).
"""
self.__change_state_entity(ci, self.K_ENABLE)
def __a_delete_link(self, link):
"""Delete a link between two entity and store the modify entities
into self.udpate.
:param link: the link that identify a link (see the JSON specification)
"""
for id_ in link[self.K_FROM]:
if id_ not in self.update:
self.update[id_] = self.entities_to_update[id_]
if link[self.K_TO] not in self.update:
to_ = link[self.K_TO]
self.update[to_] = self.entities_to_update[to_]
for id_ in link[self.K_FROM]:
self.update[id_][self.K_IMPACT].remove(link[self.K_TO])
self.update[link[self.K_TO]][self.K_DEPENDS].remove(id_)
def __a_update_link(self, link):
raise NotImplementedError()
def __a_create_link(self, link):
"""Create a link between two entity and store the modify entities
into self.udpate.
:param link: the link that identify a link (see the JSON specification)
"""
if link[self.K_TO] not in self.update:
to_ = link[self.K_TO]
self.update[to_] = self.entities_to_update[to_]
for ci_id in link[self.K_FROM]:
if ci_id not in self.update:
self.update[ci_id] = self.entities_to_update[ci_id]
self.update[ci_id][self.K_IMPACT].add(link[self.K_TO])
self.update[link[self.K_TO]][self.K_DEPENDS].add(ci_id)
def __a_disable_link(self, link):
raise NotImplementedError()
def __a_enable_link(self, link):
raise NotImplementedError()
@classmethod
def __superficial_check(cls, fd):
"""Check if the cis and links field are a list. If not, raise a
jsonschema.ValidationError. It move the cursor of the fd to back 0."""
# cis and links store if a field cis and links are found in the import
# *_start store if a the beginning of a json array are found in the cis
# and links fields of an import
# *_end store if a the end of a json array are found in the cis and
# links fields of an import
cis = False
cis_start = False
cis_end = False
links = False
links_start = False
links_end = False
parser = ijson.parse(fd)
for prefix, event, _ in parser:
if prefix == "cis":
cis = True
if event == "end_array":
cis_end = True
if event == "start_array":
cis_start = True
if prefix == "links":
links = True
if event == "end_array":
links_end = True
if event == "start_array":
links_start = True
fd.seek(0)
cis_status = (cis, cis_start, cis_end)
links_status = (links, links_start, links_end)
# ok is a filter to ascertain if a cis/link field of an import is
# correct.
ok = [(True, True, True), (False, False, False)]
if cis_status in ok and links_status in ok:
return True
elif cis_status not in ok and links_status not in ok:
raise jsonschema.ValidationError(
"CIS and LINKS should be an array.")
elif cis_status not in ok:
raise jsonschema.ValidationError("CIS should be an array.")
elif links_status not in ok:
raise jsonschema.ValidationError("LINKS should be an array.")
def import_context(self, uuid):
"""Import a new context.
:param uuid: the uuid of the import to process
:return type: a tuple (updated entities, deleted entities)
"""
file_ = ImportKey.IMPORT_FILE.format(uuid)
fd = open(file_, 'r')
# In case the previous import failed and/or raise an exception, we\
# clean now
self.clean_attributes()
start = time.time()
self.__superficial_check(fd)
end = time.time()
self.logger.debug("Import {0} : superficial"
" check {1}.".format(uuid,
execution_time(end - start)))
start = time.time()
self.entities_to_update = self.__get_entities_to_update(file_)
end = time.time()
self.logger.debug("Import {0} : get_entities_to"
"_update {1}.".format(uuid,
execution_time(end - start)))
# create/update watcher tracker
watcher_tracker = set()
# Process cis list
start = time.time()
for ci in ijson.items(fd, "{0}.item".format(self.K_CIS)):
self.logger.debug("Current ci : {0}".format(ci))
if ci[self.K_ACTION] == self.A_DELETE:
self.__a_delete_entity(ci)
elif ci[self.K_ACTION] == self.A_CREATE:
if ci.get(self.K_TYPE, '') == 'watcher':
watcher_tracker.add(ci.get(self.K_ID))
self.__a_create_entity(ci)
elif ci[self.K_ACTION] == self.A_UPDATE:
if ci.get(self.K_TYPE, '') == 'watcher':
watcher_tracker.add(ci.get(self.K_ID))
self.__a_update_entity(ci)
elif ci[self.K_ACTION] == self.A_SET:
self.__a_set_entity(ci)
elif ci[self.K_ACTION] == self.A_DISABLE:
self.__a_disable_entity(ci)
elif ci[self.K_ACTION] == self.A_ENABLE:
self.__a_enable_entity(ci)
else:
raise ValueError("The action {0} is not recognized\n".format(
ci[self.K_ACTION]))
end = time.time()
self.logger.debug("Import {0} : update cis {1}.".format(
uuid, execution_time(end - start)))
fd.seek(0)
# Process link list
start = time.time()
for link in ijson.items(fd, "{0}.item".format(self.K_LINKS)):
self.logger.debug("Current link : {0}".format(link))
if link[self.K_ACTION] == self.A_DELETE:
self.__a_delete_link(link)
elif link[self.K_ACTION] == self.A_CREATE:
self.__a_create_link(link)
elif link[self.K_ACTION] == self.A_UPDATE:
self.__a_update_link(link)
elif link[self.K_ACTION] == self.A_DISABLE:
self.__a_disable_link(link)
elif link[self.K_ACTION] == self.A_ENABLE:
self.__a_enable_link(link)
else:
raise ValueError("The action {0} is not recognized".format(
link[self.K_ACTION]))
end = time.time()
self.logger.debug("Import {0} : update links"
" {1}.".format(uuid, execution_time(end - start)))
for id_ in self.update:
if id_ in self.delete:
desc = "The entity {0} to be deleted is updated in "\
"the same import. Update aborted.".format(id_)
raise ValueError(desc)
updated_entities = len(self.update)
deleted_entities = len(self.delete)
for entity in self.update.values():
entity[self.K_IMPACT] = list(entity[self.K_IMPACT])
entity[self.K_DEPENDS] = list(entity[self.K_DEPENDS])
start = time.time()
self._put_entities(self.update.values())
# send updatewatcher event
for _id in watcher_tracker:
event = forger(connector="watcher",
connector_name="watcher",
event_type="updatewatcher",
source_type="component",
component=_id)
self.amqp_pub.canopsis_event(event)
end = time.time()
self.logger.debug("Import {0} : push updated"
" entities {1}.".format(uuid,
execution_time(end - start)))
start = time.time()
self._delete_entities(self.delete)
end = time.time()
self.logger.debug("Import {0} : delete entities"
" {1}.".format(uuid, execution_time(end - start)))
self.clean_attributes()
return updated_entities, deleted_entities
def test_json_document(self):
jdoc = {'bla': 'bla'}
with AmqpConnection(self.amqp_url) as ac:
amqp_pub = AmqpPublisher(ac, Mock())
amqp_pub.json_document(jdoc, self.amqp_exname, '#')
def test_json_document(self):
jdoc = {'bla': 'bla'}
with AmqpConnection(self.amqp_url) as ac:
amqp_pub = AmqpPublisher(ac, Mock())
amqp_pub.json_document(jdoc, self.amqp_exname, '#')
class Engine(object):
etype = 'Engine'
def __init__(self,
next_amqp_queues=[],
next_balanced=False,
name="worker1",
beat_interval=60,
logging_level=INFO,
exchange_name=DIRECT_EXCHANGE_NAME,
routing_keys=[],
camqp_custom=None,
max_retries=5,
*args, **kwargs):
super(Engine, self).__init__()
self.logging_level = logging_level
self.debug = logging_level == DEBUG
init = Init()
self.logger = init.getLogger(name, logging_level=self.logging_level)
log_handler = FileHandler(
filename=join(
root_path, 'var', 'log', 'engines', '{0}.log'.format(name)
)
)
log_handler.setFormatter(
Formatter(
"%(asctime)s %(levelname)s %(name)s %(message)s"
)
)
# Log in file
self.logger.addHandler(log_handler)
self.RUN = True
self.name = name
# Set parametrized Amqp for testing purposes
if camqp_custom is None:
self.Amqp = Amqp
else:
self.Amqp = camqp_custom
# self.amqp handles the consumption of events from rabbitmq. The
# publication of events from self.amqp is deprecated.
self.amqp = None
# self.beat_amqp_publisher and self.work_amqp_publisher handle the
# publication of events (they are separated to prevent sharing a
# channel between two threads).
self.beat_amqp_publisher = AmqpPublisher(
get_default_amqp_connection(), self.logger)
self.work_amqp_publisher = AmqpPublisher(
get_default_amqp_connection(), self.logger)
self.amqp_queue = "Engine_{0}".format(self.name)
self.routing_keys = routing_keys
self.exchange_name = exchange_name
self.perfdata_retention = 3600
self.next_amqp_queues = next_amqp_queues
self.get_amqp_queue = cycle(self.next_amqp_queues)
# Get from internal or external queue
self.next_balanced = next_balanced
self.max_retries = max_retries
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
self.thd_warn_sec_per_evt = 0.6
self.thd_crit_sec_per_evt = 0.9
self.beat_interval = beat_interval
self.beat_last = time()
self.create_queue = True
self.send_stats_event = True
self.rk_on_error = []
self.last_stat = int(time())
self.logger.info("Engine initialized")
def new_amqp_queue(self,
amqp_queue,
routing_keys,
on_amqp_event,
exchange_name):
self.amqp.add_queue(
queue_name=amqp_queue,
routing_keys=routing_keys,
callback=on_amqp_event,
exchange_name=exchange_name,
no_ack=True,
exclusive=False,
auto_delete=False
)
def pre_run(self):
pass
def post_run(self):
pass
def run(self):
def ready():
self.logger.info(" + Ready!")
self.logger.info("Start Engine with pid {0}".format(getpid()))
self.amqp = self.Amqp(
logging_level=self.logging_level,
logging_name="{0}-amqp".format(self.name),
on_ready=ready,
max_retries=self.max_retries
)
if self.create_queue:
self.new_amqp_queue(
self.amqp_queue,
self.routing_keys,
self.on_amqp_event,
self.exchange_name
)
self.amqp.start()
self.pre_run()
while self.RUN:
# Beat
if self.beat_interval:
now = time()
if now > (self.beat_last + self.beat_interval):
self._beat()
self.beat_last = now
try:
sleep(1)
except Exception as err:
self.logger.error("Error in break time: {0}".format(err))
self.RUN = False
except KeyboardInterrupt:
self.logger.info(u'Stop request')
self.RUN = False
self.post_run()
self.logger.info("Stop Engine")
self.stop()
self.logger.info("End of Engine")
def on_amqp_event(self, event, msg):
try:
self._work(event, msg)
except Exception as err:
if event['rk'] not in self.rk_on_error:
self.logger.error(err)
self.logger.error("Impossible to deal with: {0}".format(event))
self.rk_on_error.append(event['rk'])
self.next_queue(event)
def _work(self, event, msg=None, *args, **kargs):
start = time()
error = False
try:
if self.debug:
if 'processing' not in event:
event['processing'] = {}
event['processing'][self.etype] = start
wevent = self.work(event, msg, *args, **kargs)
if wevent != DROP:
if isinstance(wevent, dict):
event = wevent
self.next_queue(event)
except Exception as err:
error = True
self.logger.error("Worker raise exception: {0}".format(err))
self.logger.error(format_exc())
if error:
self.counter_error += 1
elapsed = time() - start
if elapsed > 3:
self.logger.warning("Elapsed time %.2f seconds" % elapsed)
self.counter_event += 1
self.counter_worktime += elapsed
def work(self, event, amqp_msg):
return event
def next_queue(self, event):
if self.next_balanced:
queue_name = self.get_amqp_queue.next()
if queue_name:
try:
self.work_amqp_publisher.direct_event(event, queue_name)
except Exception as e:
self.logger.exception("Unable to send event to next queue")
else:
for queue_name in self.next_amqp_queues:
try:
self.work_amqp_publisher.direct_event(event, queue_name)
except Exception as e:
self.logger.exception("Unable to send event to next queue")
def _beat(self):
now = int(time())
if self.last_stat + 60 <= now:
self.logger.debug(" + Send stats")
self.last_stat = now
evt_per_sec = 0
sec_per_evt = 0
if self.counter_event:
evt_per_sec = float(self.counter_event) / self.beat_interval
self.logger.debug(" + %0.2f event(s)/seconds", evt_per_sec)
if self.counter_worktime and self.counter_event:
sec_per_evt = self.counter_worktime / self.counter_event
self.logger.debug(" + %0.5f seconds/event", sec_per_evt)
# Submit event
if self.send_stats_event and self.counter_event != 0:
state = 0
if sec_per_evt > self.thd_warn_sec_per_evt:
state = 1
if sec_per_evt > self.thd_crit_sec_per_evt:
state = 2
perf_data_array = [
{
'retention': self.perfdata_retention,
'metric': 'cps_evt_per_sec',
'value': round(evt_per_sec, 2),
'unit': 'evt'
}, {
'retention': self.perfdata_retention,
'metric': 'cps_sec_per_evt',
'value': round(sec_per_evt, 5),
'unit': 's',
'warn': self.thd_warn_sec_per_evt,
'crit': self.thd_crit_sec_per_evt
}
]
self.logger.debug(" + State: {0}".format(state))
event = forger(
connector="Engine",
connector_name="engine",
event_type="check",
source_type="resource",
resource=self.amqp_queue,
state=state,
state_type=1,
output="%0.2f evt/sec, %0.5f sec/evt" % (
evt_per_sec, sec_per_evt),
perf_data_array=perf_data_array
)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send perfdata")
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
try:
self.beat()
except Exception as err:
self.logger.error("Beat raise exception: {0}".format(err))
self.logger.error(print_exc())
def beat(self):
pass
def stop(self):
self.RUN = False
# cancel self consumer
self.amqp.cancel_queues()
self.amqp.stop()
self.amqp.join()
self.logger.debug(" + Stopped")
class Lock(object):
def __init__(self, engine, name, *args, **kwargs):
super(Engine.Lock, self).__init__()
self.name = name
self.lock_id = '{0}.{1}'.format(engine.etype, name)
self.storage = get_storage(
namespace='lock',
logging_level=engine.logging_level,
account=Account(user='******', group='root')
).get_backend()
self.engine = engine
self.lock = {}
def own(self):
now = time()
last = self.lock.get('t', now)
if self.lock.get('l', False) \
and (now - last) < self.engine.beat_interval:
self.engine.logger.debug(
'Another engine {0} is already holding the lock {1}'.
format(self.engine.etype, self.name))
return False
else:
self.engine.logger.debug(
'Lock {1} on engine {0}, processing...'.format(
self.engine.etype, self.name))
return True
return False
def __enter__(self):
lock = self.storage.find_and_modify(
query={'_id': self.lock_id},
update={'$set': {'l': True}},
upsert=True
)
if lock is not None:
self.lock = lock
if 't' not in self.lock:
self.lock['t'] = 0
return self
def __exit__(self, type, value, tb):
if self.own():
self.engine.logger.debug(
'Release lock {1} on engine {0}'.format(
self.engine.etype, self.name))
self.storage.save({
'_id': self.lock_id,
'l': False,
't': time()
})
@classmethod
def release(cls, lock_id, object_storage):
object_storage.update(
{'_id': lock_id},
{'$set': {
'l': False,
't': time()
}},
upsert=True
)
def test_canopsis_event(self):
with AmqpConnection(self.amqp_url) as ac:
amqp_pub = AmqpPublisher(ac, Mock())
amqp_pub.canopsis_event(self.event, self.amqp_exname)
def test_canopsis_event(self):
with AmqpConnection(self.amqp_url) as ac:
amqp_pub = AmqpPublisher(ac, Mock())
amqp_pub.canopsis_event(self.event, self.amqp_exname)
class Archiver(Middleware):
def __init__(
self, namespace, confnamespace='object', storage=None,
autolog=False, amqp_pub=None, *args, **kwargs
):
super(Archiver, self).__init__()
self.namespace = namespace
self.namespace_log = namespace + '_log'
# Bulk operation configuration
self.last_bulk_insert_date = time()
self.bulk_ids = []
# How many events can be buffered
self.bulk_amount = 500
# What is the maximum duration until bulk insert
self.bulk_delay = 3
self.incoming_events = {}
self.autolog = autolog
self.logger.debug("Init Archiver on %s" % namespace)
self.account = Account(user="******", group="root")
if not storage:
self.logger.debug(" + Get storage")
self.storage = get_storage(
namespace=namespace,
logging_level=self.log_lvl
)
else:
self.storage = storage
self.conf_storage = get_storage(
namespace=confnamespace,
logging_level=self.log_lvl
)
self.conf_collection = self.conf_storage.get_backend(confnamespace)
self.collection = self.storage.get_backend(namespace)
if amqp_pub is None:
self.amqp_pub = AmqpPublisher(get_default_amqp_connection())
self.reset_stealthy_event_duration = time()
self.reset_stats()
def reset_stats(self):
self.stats = {
'update': 0,
'insert ' + self.namespace: 0,
'insert ' + self.namespace_log: 0
}
def beat(self):
self.logger.info(
(
'DB documents stats : ' +
'update: {} in events, ' +
'insert: {} in events, ' +
'insert: {} in events_log').format(
self.stats['update'],
self.stats['insert ' + self.namespace],
self.stats['insert ' + self.namespace_log]
)
)
self.reset_stats()
def process_insert_operations_collection(self, operations, collection):
self.stats['insert ' + collection] += len(operations)
if operations:
# is there any event to process ?
backend = self.storage.get_backend(collection)
bulk = backend.initialize_unordered_bulk_op()
for operation in operations:
record = Record(operation['event'])
record.type = "event"
event = record.dump()
bulk.insert(event)
try:
bulk.execute({'w': 0})
except BulkWriteError as bwe:
self.logger.warning(bwe.details)
self.logger.info('inserted log events {}'.format(len(operations)))
def process_update_operations(self, operations):
self.stats['update'] += len(operations)
if operations:
# is there any event to process ?
backend = self.storage.get_backend('events')
bulk = backend.initialize_unordered_bulk_op()
for operation in operations:
bulk.find(operation['query']).update(operation['update'])
bulk.execute({'w': 0})
def process_insert_operations(self, insert_operations):
events = {}
events_log = {}
# Avoid same RK insert
for operation in insert_operations:
if '_id' not in operation['event']:
self.logger.error(
'Unable to find _id value in event {}'.format(
operation['event']
)
)
else:
_id = operation['event']['_id']
if operation['collection'] == self.namespace:
events[_id] = operation
elif operation['collection'] == self.namespace_log:
_id = '{}.{}'.format(_id, time())
operation['event']['_id'] = _id
events_log[_id] = operation
else:
self.logger.critical(
'Wrong operation type {}'.format(
operation['collection']
)
)
self.process_insert_operations_collection(
events.values(),
'events'
)
self.process_insert_operations_collection(
events_log.values(),
'events_log'
)
def reload_configuration(self):
# Default values
self.restore_event = True
self.bagot_freq = 10
self.bagot_time = 3600
self.stealthy_time = 360
self.stealthy_show = 360
state_config = self.conf_collection.find_one(
{'crecord_type': 'statusmanagement'}
)
if state_config is not None:
self.bagot_freq = state_config.get('bagot_freq', self.bagot_freq)
self.bagot_time = state_config.get('bagot_time', self.bagot_time)
self.stealthy_time = state_config.get(
'stealthy_time',
self.stealthy_time
)
self.stealthy_show = state_config.get(
'stealthy_show',
self.stealthy_show
)
self.restore_event = state_config.get(
'restore_event',
self.restore_event
)
self.logger.debug(
(
'Archiver configuration ->\n' +
' bagot_freq {}\n' +
' bagot_time {}\n' +
' stealthy_show {}\n' +
' stealthy_time {}'
).format(
self.bagot_freq,
self.bagot_time,
self.stealthy_show,
self.stealthy_time
)
)
def reset_status_event(self, reset_type):
"""Trigger event status reset to off/on going status if event are in
BAGOT or STEALTHY status.
:param reset_type: event status to consider and change.
:type int: This is en enum, can be either BAGOT or STEALTHY
"""
if reset_type not in [BAGOT, STEALTHY]:
self.logger.info('wrong reset type given, will not process')
return
# Dynamic method parameter depends on reset type input
compare_property = {
BAGOT: 'last_state_change',
STEALTHY: 'ts_first_stealthy'
}[reset_type]
configuration_delay = {
BAGOT: self.bagot_time,
STEALTHY: self.stealthy_show
}[reset_type]
event_cursor = self.collection.find(
{
'crecord_type': 'event',
'status': reset_type
}
)
# Change all potention reset type events
for event in event_cursor:
# This is a bagot event.
is_show_delay_passed = \
time() - event[compare_property] >= configuration_delay
# Check the stealthy intervals
if is_show_delay_passed:
self.logger.info(
'Event {} no longer in status {}'.format(
event['rk'],
reset_type
)
)
new_status = ONGOING if event['state'] else OFF
self.set_status(event, new_status)
event['pass_status'] = 1
self.amqp_pub.canopsis_event(event)
def is_bagot(self, event):
"""
Args:
event map of the current evet
Returns:
``True`` if the event is bagot
``False`` otherwise
"""
ts_curr = event['timestamp']
ts_first_bagot = event.get('ts_first_bagot', 0)
ts_diff_bagot = (ts_curr - ts_first_bagot)
freq = event.get('bagot_freq', -1)
result = ts_diff_bagot <= self.bagot_time and freq >= self.bagot_freq
return result
def is_stealthy(self, event, d_status):
"""
Args:
event map of the current evet
d_status status of the previous event
Returns:
``True`` if the event is stealthy
``False`` otherwise
"""
ts_diff = event['timestamp'] - event['ts_first_stealthy']
result = ts_diff <= self.stealthy_time and d_status != STEALTHY
return result
def set_status(self, event, status, devent=None):
"""
Args:
event map of the current event
status status of the current event
"""
log = 'Status is set to {} for event {}'.format(status, event['rk'])
bagot_freq = event.get('bagot_freq', 0)
values = {
OFF: {
'freq': bagot_freq,
'name': 'Off'
},
ONGOING: {
'freq': bagot_freq,
'name': 'On going'
},
STEALTHY: {
'freq': bagot_freq,
'name': 'Stealthy'
},
BAGOT: {
'freq': bagot_freq + 1,
'name': 'Bagot'
},
CANCELED: {
'freq': bagot_freq,
'name': 'Cancelled'
}
}
self.logger.debug(log.format(values[status]['name']))
# This is an additional check as stealthy
# status is not properly managed until now
if status != STEALTHY:
event['status'] = status
elif devent['state'] != 0 and event['state'] == 0:
delta = time() - event['last_state_change']
if delta < self.stealthy_time:
event['status'] = status
event['bagot_freq'] = values[status]['freq']
if status not in [STEALTHY, BAGOT]:
event['ts_first_stealthy'] = 0
def check_stealthy(self, devent, ts):
"""
Args:
devent map of the previous event
ts timestamp of the current event
Returns:
``True`` if the event should stay stealthy
``False`` otherwise
"""
result = False
if devent['status'] == STEALTHY:
result = (ts - devent['ts_first_stealthy']) <= self.stealthy_show
return result
def check_statuses(self, event, devent):
"""
Args:
event map of the current event
devent map of the previous evet
"""
if event.get('pass_status', 0):
event['pass_status'] = 0
return
event_ts = event['timestamp']
event['bagot_freq'] = devent.get('bagot_freq', 0)
event['ts_first_stealthy'] = devent.get('ts_first_stealthy', 0)
event['ts_first_bagot'] = devent.get('ts_first_bagot', 0)
dstate = devent['state']
# Increment frequency if state changed and set first occurences
if ((not dstate and event['state']) or
dstate and not event['state']):
if event['state']:
event['ts_first_stealthy'] = event_ts
else:
event['ts_first_stealthy'] = event_ts
event['bagot_freq'] += 1
if not event['ts_first_bagot']:
event['ts_first_bagot'] = event_ts
# Out of bagot interval, reset variables
if event['ts_first_bagot'] - event_ts > self.bagot_time:
event['ts_first_bagot'] = 0
event['bagot_freq'] = 0
# If not canceled, proceed to check the status
if (devent.get('status', ONGOING) != CANCELED
or (dstate != event['state']
and (event['state'] == OFF
or dstate == OFF))):
# Check the stealthy intervals
if self.check_stealthy(devent, event_ts):
if self.is_bagot(event):
self.set_status(event, BAGOT)
else:
self.set_status(event, STEALTHY, devent=devent)
# Else proceed normally
else:
if (event['state'] == OFF):
# If still non-alert, can only be OFF
if (not self.is_bagot(event)
and not self.is_stealthy(event, devent['status'])):
self.set_status(event, OFF)
elif self.is_bagot(event):
self.set_status(event, BAGOT)
elif self.is_stealthy(event, devent['status']):
self.set_status(event, STEALTHY, devent=devent)
else:
# If not bagot/stealthy, can only be ONGOING
if (not self.is_bagot(event)
and not self.is_stealthy(event, devent['status'])):
self.set_status(event, ONGOING)
elif self.is_bagot(event):
self.set_status(event, BAGOT)
elif self.is_stealthy(event, devent['status']):
if devent['status'] == OFF:
self.set_status(event, ONGOING)
else:
self.set_status(event, STEALTHY, devent=devent)
else:
self.set_status(event, CANCELED)
def check_event(self, _id, event):
"""
This method aims to buffer and process incoming events.
Processing is done on buffer to reduce database operations.
"""
# As this was not done until now... setting event primary key
event['_id'] = _id
# Buffering event informations
self.bulk_ids.append(_id)
self.incoming_events[_id] = event
# Processing many events condition computation
bulk_modulo = len(self.bulk_ids) % self.bulk_amount
elapsed_time = time() - self.last_bulk_insert_date
# When enough event buffered/time elapsed
# processing events buffers
if bulk_modulo == 0 or elapsed_time > self.bulk_delay:
insert_operations = []
update_operations = []
query = {'_id': {'$in': self.bulk_ids}}
devents = {}
# Put previous events in pretty data structure
backend = self.storage.get_backend(self.namespace)
for devent in backend.find(query):
devents[devent['_id']] = devent
# Try to match previous and new incoming event
for _id in self.incoming_events:
event = self.incoming_events[_id]
devent = None
if _id in devents:
devent = devents[_id]
# Effective archiver processing call
operations = self.process_an_event(_id, event, devent)
for operation in operations:
if operation['type'] == 'insert':
insert_operations.append(operation)
else:
update_operations.append(operation)
self.process_insert_operations(insert_operations)
self.process_update_operations(update_operations)
# Buld processing done, reseting informations
self.bulk_ids = []
self.incoming_events = {}
self.last_bulk_insert_date = time()
# Half useless retro compatibility
if 'state' in event and event['state']:
return _id
def process_an_event(self, _id, event, devent):
operations = []
changed = False
new_event = False
state = event['state']
state_type = event['state_type']
now = int(time())
event['timestamp'] = event.get('timestamp', now)
try:
# Get old record
exclusion_fields = {
'perf_data_array',
'processing'
}
if not devent:
new_event = True
# may have side effects on acks/cancels
devent = {}
old_state = devent['state']
old_state_type = devent['state_type']
event['last_state_change'] = devent.get(
'last_state_change',
event['timestamp']
)
if state != old_state:
event['previous_state'] = old_state
if state != old_state or state_type != old_state_type:
changed = True
self.check_statuses(event, devent)
except Exception:
# No old record
event['ts_first_stealthy'] = 0
changed = True
old_state = state
if changed:
# Tests if change is from alert to non alert
if ('last_state_change' in event
and (state == 0 or (state > 0 and old_state == 0))):
event['previous_state_change_ts'] = event['last_state_change']
event['last_state_change'] = event.get('timestamp', now)
if new_event:
# copy avoid side effects
operations.append({
'type': 'insert',
'event': event.copy(),
'collection': 'events'
})
else:
change = {}
# Clear ack related information when event goes ok state
# Ack is kept in the current event below
if event['state'] == 0 and devent.get('state') != 0:
for key in (
'ticket_declared_author',
'ticket_declared_date',
'ticket_date',
'ticket'
):
change[key] = None
# keep ack information if status does not reset event
if 'ack' in devent:
if event['status'] == 0:
was_ack = devent.get('ack', {}).get('isAck', False)
# save was ack for stats purposes
change['ack'] = {
'wasAck': was_ack
}
else:
change['ack'] = devent['ack']
# remove was ack for accurate stats
# when event change to any alert state
if 'wasAck' in change['ack']:
del change['ack']['wasAck']
# keep cancel information if status does not reset event
if 'cancel' in devent:
if event['status'] not in [0, 1]:
change['cancel'] = devent['cancel']
else:
change['cancel'] = {}
# Remove ticket information in case state is back to normal
# (both ack and ticket declaration case)
if 'ticket_declared_author' in devent and event['status'] == 0:
change['ticket_declared_author'] = None
change['ticket_declared_date'] = None
# Remove ticket information in case state is back to normal
# (ticket number declaration only case)
if 'ticket' in devent and event['status'] == 0:
del devent['ticket']
if 'ticket_date' in devent:
del devent['ticket_date']
# Generate diff change from old event to new event
for key in event:
if key not in exclusion_fields:
if (key in event and
key in devent and
devent[key] != event[key]):
change[key] = event[key]
elif key in event and key not in devent:
change[key] = event[key]
# Manage keep state key that allow
# from UI to keep the choosen state
# into until next ok state
event_reset = False
# When a event is ok again, dismiss keep_state statement
if devent.get('keep_state') and event['state'] == 0:
change['keep_state'] = False
event_reset = True
# assume we do not just received a keep state and
# if keep state was sent previously
# then override state of new event
if 'keep_state' not in event:
if not event_reset and devent.get('keep_state'):
change['state'] = devent['state']
# Keep previous output
if 'keep_state' in event:
change['change_state_output'] = event['output']
change['output'] = devent.get('output', '')
if change:
operations.append(
{
'type': 'update',
'update': {'$set': change},
'query': {'_id': _id},
'collection': 'events'
}
)
# I think that is the right condition to log
have_to_log = event.get('previous_state', state) != state
if have_to_log:
# store ack information to log collection
if 'ack' in devent:
event['ack'] = devent['ack']
self.logger.info(' + State changed, have to log {}'.format(_id))
# copy avoid side effects
operations.append(
{
'type': 'insert',
'event': event.copy(),
'collection': 'events_log'
}
)
return operations
def __init__(self,
next_amqp_queues=[],
next_balanced=False,
name="worker1",
beat_interval=60,
logging_level=INFO,
exchange_name=DIRECT_EXCHANGE_NAME,
routing_keys=[],
camqp_custom=None,
max_retries=5,
*args, **kwargs):
super(Engine, self).__init__()
self.logging_level = logging_level
self.debug = logging_level == DEBUG
init = Init()
self.logger = init.getLogger(name, logging_level=self.logging_level)
log_handler = FileHandler(
filename=join(
root_path, 'var', 'log', 'engines', '{0}.log'.format(name)
)
)
log_handler.setFormatter(
Formatter(
"%(asctime)s %(levelname)s %(name)s %(message)s"
)
)
# Log in file
self.logger.addHandler(log_handler)
self.RUN = True
self.name = name
# Set parametrized Amqp for testing purposes
if camqp_custom is None:
self.Amqp = Amqp
else:
self.Amqp = camqp_custom
# self.amqp handles the consumption of events from rabbitmq. The
# publication of events from self.amqp is deprecated.
self.amqp = None
# self.beat_amqp_publisher and self.work_amqp_publisher handle the
# publication of events (they are separated to prevent sharing a
# channel between two threads).
self.beat_amqp_publisher = AmqpPublisher(
get_default_amqp_connection(), self.logger)
self.work_amqp_publisher = AmqpPublisher(
get_default_amqp_connection(), self.logger)
self.amqp_queue = "Engine_{0}".format(self.name)
self.routing_keys = routing_keys
self.exchange_name = exchange_name
self.perfdata_retention = 3600
self.next_amqp_queues = next_amqp_queues
self.get_amqp_queue = cycle(self.next_amqp_queues)
# Get from internal or external queue
self.next_balanced = next_balanced
self.max_retries = max_retries
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
self.thd_warn_sec_per_evt = 0.6
self.thd_crit_sec_per_evt = 0.9
self.beat_interval = beat_interval
self.beat_last = time()
self.create_queue = True
self.send_stats_event = True
self.rk_on_error = []
self.last_stat = int(time())
self.logger.info("Engine initialized")
def _amqp_setup(self):
self.amqp_conn = AmqpConnection(self.AMQP_URL)
self.amqp_pub = AmqpPublisher(self.amqp_conn)
class Archiver(Middleware):
def __init__(
self, namespace, confnamespace='object', storage=None,
autolog=False, amqp_pub=None, *args, **kwargs
):
super(Archiver, self).__init__()
self.namespace = namespace
self.namespace_log = namespace + '_log'
# Bulk operation configuration
self.last_bulk_insert_date = time()
self.bulk_ids = []
# How many events can be buffered
self.bulk_amount = 500
# What is the maximum duration until bulk insert
self.bulk_delay = 3
self.incoming_events = {}
self.autolog = autolog
self.logger.debug("Init Archiver on %s" % namespace)
self.account = Account(user="******", group="root")
if not storage:
self.logger.debug(" + Get storage")
self.storage = get_storage(
namespace=namespace,
logging_level=self.log_lvl
)
else:
self.storage = storage
self.conf_storage = get_storage(
namespace=confnamespace,
logging_level=self.log_lvl
)
self.conf_collection = self.conf_storage.get_backend(confnamespace)
self.collection = self.storage.get_backend(namespace)
if amqp_pub is None:
self.amqp_pub = AmqpPublisher(
get_default_amqp_connection(), self.logger)
self.reset_stealthy_event_duration = time()
self.reset_stats()
def reset_stats(self):
self.stats = {
'update': 0,
'insert ' + self.namespace: 0,
'insert ' + self.namespace_log: 0
}
def beat(self):
self.logger.info(
(
'DB documents stats : ' +
'update: {} in events, ' +
'insert: {} in events, ' +
'insert: {} in events_log').format(
self.stats['update'],
self.stats['insert ' + self.namespace],
self.stats['insert ' + self.namespace_log]
)
)
self.reset_stats()
def process_insert_operations_collection(self, operations, collection):
self.stats['insert ' + collection] += len(operations)
if operations:
# is there any event to process ?
backend = self.storage.get_backend(collection)
bulk = backend.initialize_unordered_bulk_op()
for operation in operations:
record = Record(operation['event'])
record.type = "event"
event = record.dump()
bulk.insert(event)
try:
bulk.execute({'w': 0})
except BulkWriteError as bwe:
self.logger.warning(bwe.details)
self.logger.info('inserted log events {}'.format(len(operations)))
def process_update_operations(self, operations):
self.stats['update'] += len(operations)
if operations:
# is there any event to process ?
backend = self.storage.get_backend('events')
bulk = backend.initialize_unordered_bulk_op()
for operation in operations:
bulk.find(operation['query']).update(operation['update'])
bulk.execute({'w': 0})
def process_insert_operations(self, insert_operations):
events = {}
events_log = {}
# Avoid same RK insert
for operation in insert_operations:
if '_id' not in operation['event']:
self.logger.error(
'Unable to find _id value in event {}'.format(
operation['event']
)
)
else:
_id = operation['event']['_id']
if operation['collection'] == self.namespace:
events[_id] = operation
elif operation['collection'] == self.namespace_log:
_id = '{}.{}'.format(_id, time())
operation['event']['_id'] = _id
events_log[_id] = operation
else:
self.logger.critical(
'Wrong operation type {}'.format(
operation['collection']
)
)
self.process_insert_operations_collection(
events.values(),
'events'
)
self.process_insert_operations_collection(
events_log.values(),
'events_log'
)
def reload_configuration(self):
# Default values
self.restore_event = True
self.bagot_freq = 10
self.bagot_time = 3600
self.stealthy_time = 360
self.stealthy_show = 360
state_config = self.conf_collection.find_one(
{'crecord_type': 'statusmanagement'}
)
if state_config is not None:
self.bagot_freq = state_config.get('bagot_freq', self.bagot_freq)
self.bagot_time = state_config.get('bagot_time', self.bagot_time)
self.stealthy_time = state_config.get(
'stealthy_time',
self.stealthy_time
)
self.stealthy_show = state_config.get(
'stealthy_show',
self.stealthy_show
)
self.restore_event = state_config.get(
'restore_event',
self.restore_event
)
self.logger.debug(
(
'Archiver configuration ->\n' +
' bagot_freq {}\n' +
' bagot_time {}\n' +
' stealthy_show {}\n' +
' stealthy_time {}'
).format(
self.bagot_freq,
self.bagot_time,
self.stealthy_show,
self.stealthy_time
)
)
def reset_status_event(self, reset_type):
"""Trigger event status reset to off/on going status if event are in
BAGOT or STEALTHY status.
:param reset_type: event status to consider and change.
:type int: This is en enum, can be either BAGOT or STEALTHY
"""
if reset_type not in [BAGOT, STEALTHY]:
self.logger.info('wrong reset type given, will not process')
return
# Dynamic method parameter depends on reset type input
compare_property = {
BAGOT: 'last_state_change',
STEALTHY: 'ts_first_stealthy'
}[reset_type]
configuration_delay = {
BAGOT: self.bagot_time,
STEALTHY: self.stealthy_show
}[reset_type]
event_cursor = self.collection.find(
{
'crecord_type': 'event',
'status': reset_type
}
)
# Change all potention reset type events
for event in event_cursor:
# This is a bagot event.
is_show_delay_passed = \
time() - event[compare_property] >= configuration_delay
# Check the stealthy intervals
if is_show_delay_passed:
self.logger.info(
'Event {} no longer in status {}'.format(
event['rk'],
reset_type
)
)
new_status = ONGOING if event['state'] else OFF
self.set_status(event, new_status)
event['pass_status'] = 1
self.amqp_pub.canopsis_event(event)
def is_bagot(self, event):
"""
Args:
event map of the current evet
Returns:
``True`` if the event is bagot
``False`` otherwise
"""
ts_curr = event['timestamp']
ts_first_bagot = event.get('ts_first_bagot', 0)
ts_diff_bagot = (ts_curr - ts_first_bagot)
freq = event.get('bagot_freq', -1)
result = ts_diff_bagot <= self.bagot_time and freq >= self.bagot_freq
return result
def is_stealthy(self, event, d_status):
"""
Args:
event map of the current evet
d_status status of the previous event
Returns:
``True`` if the event is stealthy
``False`` otherwise
"""
ts_diff = event['timestamp'] - event['ts_first_stealthy']
result = ts_diff <= self.stealthy_time and d_status != STEALTHY
return result
def set_status(self, event, status, devent=None):
"""
Args:
event map of the current event
status status of the current event
"""
log = 'Status is set to {} for event {}'.format(status, event['rk'])
bagot_freq = event.get('bagot_freq', 0)
values = {
OFF: {
'freq': bagot_freq,
'name': 'Off'
},
ONGOING: {
'freq': bagot_freq,
'name': 'On going'
},
STEALTHY: {
'freq': bagot_freq,
'name': 'Stealthy'
},
BAGOT: {
'freq': bagot_freq + 1,
'name': 'Bagot'
},
CANCELED: {
'freq': bagot_freq,
'name': 'Cancelled'
}
}
self.logger.debug(log.format(values[status]['name']))
# This is an additional check as stealthy
# status is not properly managed until now
if status != STEALTHY:
event['status'] = status
elif devent['state'] != 0 and event['state'] == 0:
delta = time() - event['last_state_change']
if delta < self.stealthy_time:
event['status'] = status
event['bagot_freq'] = values[status]['freq']
if status not in [STEALTHY, BAGOT]:
event['ts_first_stealthy'] = 0
def check_stealthy(self, devent, ts):
"""
Args:
devent map of the previous event
ts timestamp of the current event
Returns:
``True`` if the event should stay stealthy
``False`` otherwise
"""
result = False
if devent['status'] == STEALTHY:
result = (ts - devent['ts_first_stealthy']) <= self.stealthy_show
return result
def check_statuses(self, event, devent):
"""
Args:
event map of the current event
devent map of the previous evet
"""
if event.get('pass_status', 0):
event['pass_status'] = 0
return
event_ts = event['timestamp']
event['bagot_freq'] = devent.get('bagot_freq', 0)
event['ts_first_stealthy'] = devent.get('ts_first_stealthy', 0)
event['ts_first_bagot'] = devent.get('ts_first_bagot', 0)
dstate = devent['state']
# Increment frequency if state changed and set first occurences
if ((not dstate and event['state']) or
dstate and not event['state']):
if event['state']:
event['ts_first_stealthy'] = event_ts
else:
event['ts_first_stealthy'] = event_ts
event['bagot_freq'] += 1
if not event['ts_first_bagot']:
event['ts_first_bagot'] = event_ts
# Out of bagot interval, reset variables
if event['ts_first_bagot'] - event_ts > self.bagot_time:
event['ts_first_bagot'] = 0
event['bagot_freq'] = 0
# If not canceled, proceed to check the status
if (devent.get('status', ONGOING) != CANCELED
or (dstate != event['state']
and (event['state'] == OFF
or dstate == OFF))):
# Check the stealthy intervals
if self.check_stealthy(devent, event_ts):
if self.is_bagot(event):
self.set_status(event, BAGOT)
else:
self.set_status(event, STEALTHY, devent=devent)
# Else proceed normally
else:
if (event['state'] == OFF):
# If still non-alert, can only be OFF
if (not self.is_bagot(event)
and not self.is_stealthy(event, devent['status'])):
self.set_status(event, OFF)
elif self.is_bagot(event):
self.set_status(event, BAGOT)
elif self.is_stealthy(event, devent['status']):
self.set_status(event, STEALTHY, devent=devent)
else:
# If not bagot/stealthy, can only be ONGOING
if (not self.is_bagot(event)
and not self.is_stealthy(event, devent['status'])):
self.set_status(event, ONGOING)
elif self.is_bagot(event):
self.set_status(event, BAGOT)
elif self.is_stealthy(event, devent['status']):
if devent['status'] == OFF:
self.set_status(event, ONGOING)
else:
self.set_status(event, STEALTHY, devent=devent)
else:
self.set_status(event, CANCELED)
def check_event(self, _id, event):
"""
This method aims to buffer and process incoming events.
Processing is done on buffer to reduce database operations.
"""
# As this was not done until now... setting event primary key
event['_id'] = _id
# Buffering event informations
self.bulk_ids.append(_id)
self.incoming_events[_id] = event
# Processing many events condition computation
bulk_modulo = len(self.bulk_ids) % self.bulk_amount
elapsed_time = time() - self.last_bulk_insert_date
# When enough event buffered/time elapsed
# processing events buffers
if bulk_modulo == 0 or elapsed_time > self.bulk_delay:
insert_operations = []
update_operations = []
query = {'_id': {'$in': self.bulk_ids}}
devents = {}
# Put previous events in pretty data structure
backend = self.storage.get_backend(self.namespace)
for devent in backend.find(query):
devents[devent['_id']] = devent
# Try to match previous and new incoming event
for _id in self.incoming_events:
event = self.incoming_events[_id]
devent = None
if _id in devents:
devent = devents[_id]
# Effective archiver processing call
operations = self.process_an_event(_id, event, devent)
for operation in operations:
if operation['type'] == 'insert':
insert_operations.append(operation)
else:
update_operations.append(operation)
self.process_insert_operations(insert_operations)
self.process_update_operations(update_operations)
# Buld processing done, reseting informations
self.bulk_ids = []
self.incoming_events = {}
self.last_bulk_insert_date = time()
# Half useless retro compatibility
if 'state' in event and event['state']:
return _id
def process_an_event(self, _id, event, devent):
operations = []
changed = False
new_event = False
state = event['state']
state_type = event['state_type']
now = int(time())
event['timestamp'] = event.get('timestamp', now)
try:
# Get old record
exclusion_fields = {
'perf_data_array',
'processing'
}
if not devent:
new_event = True
# may have side effects on acks/cancels
devent = {}
old_state = devent['state']
old_state_type = devent['state_type']
event['last_state_change'] = devent.get(
'last_state_change',
event['timestamp']
)
if state != old_state:
event['previous_state'] = old_state
if state != old_state or state_type != old_state_type:
changed = True
self.check_statuses(event, devent)
except Exception:
# No old record
event['ts_first_stealthy'] = 0
changed = True
old_state = state
if changed:
# Tests if change is from alert to non alert
if ('last_state_change' in event
and (state == 0 or (state > 0 and old_state == 0))):
event['previous_state_change_ts'] = event['last_state_change']
event['last_state_change'] = event.get('timestamp', now)
if new_event:
# copy avoid side effects
operations.append({
'type': 'insert',
'event': event.copy(),
'collection': 'events'
})
else:
change = {}
# Clear ack related information when event goes ok state
# Ack is kept in the current event below
if event['state'] == 0 and devent.get('state') != 0:
for key in (
'ticket_declared_author',
'ticket_declared_date',
'ticket_date',
'ticket'
):
change[key] = None
# keep ack information if status does not reset event
if 'ack' in devent:
if event['status'] == 0:
was_ack = devent.get('ack', {}).get('isAck', False)
# save was ack for stats purposes
change['ack'] = {
'wasAck': was_ack
}
else:
change['ack'] = devent['ack']
# remove was ack for accurate stats
# when event change to any alert state
if 'wasAck' in change['ack']:
del change['ack']['wasAck']
# keep cancel information if status does not reset event
if 'cancel' in devent:
if event['status'] not in [0, 1]:
change['cancel'] = devent['cancel']
else:
change['cancel'] = {}
# Remove ticket information in case state is back to normal
# (both ack and ticket declaration case)
if 'ticket_declared_author' in devent and event['status'] == 0:
change['ticket_declared_author'] = None
change['ticket_declared_date'] = None
# Remove ticket information in case state is back to normal
# (ticket number declaration only case)
if 'ticket' in devent and event['status'] == 0:
del devent['ticket']
if 'ticket_date' in devent:
del devent['ticket_date']
# Generate diff change from old event to new event
for key in event:
if key not in exclusion_fields:
if (key in event and
key in devent and
devent[key] != event[key]):
change[key] = event[key]
elif key in event and key not in devent:
change[key] = event[key]
# Manage keep state key that allow
# from UI to keep the choosen state
# into until next ok state
event_reset = False
# When a event is ok again, dismiss keep_state statement
if devent.get('keep_state') and event['state'] == 0:
change['keep_state'] = False
event_reset = True
# assume we do not just received a keep state and
# if keep state was sent previously
# then override state of new event
if 'keep_state' not in event:
if not event_reset and devent.get('keep_state'):
change['state'] = devent['state']
# Keep previous output
if 'keep_state' in event:
change['change_state_output'] = event['output']
change['output'] = devent.get('output', '')
if change:
operations.append(
{
'type': 'update',
'update': {'$set': change},
'query': {'_id': _id},
'collection': 'events'
}
)
# I think that is the right condition to log
have_to_log = event.get('previous_state', state) != state
if have_to_log:
# store ack information to log collection
if 'ack' in devent:
event['ack'] = devent['ack']
self.logger.info(' + State changed, have to log {}'.format(_id))
# copy avoid side effects
operations.append(
{
'type': 'insert',
'event': event.copy(),
'collection': 'events_log'
}
)
return operations
def _amqp_setup(self):
logger = logging.getLogger("test_base")
self.amqp_conn = AmqpConnection(self.AMQP_URL)
self.amqp_pub = AmqpPublisher(self.amqp_conn, logger)
class Watcher:
"""Watcher class"""
def __init__(self, amqp_pub=None):
"""
:param amqp_pub canopsis.common.amqp.AmqpPublisher:
"""
self.logger = Logger.get('watcher', LOG_PATH)
self.watcher_storage = Middleware.get_middleware_by_uri(
'mongodb-default-watcher://')
self.alert_storage = Middleware.get_middleware_by_uri(
'mongodb-periodical-alarm://')
self.sla_storage = Middleware.get_middleware_by_uri(
'storage-default-sla://')
self.context_graph = ContextGraph(self.logger)
self.pbehavior_manager = PBehaviorManager(
*PBehaviorManager.provide_default_basics()
)
self.amqp_pub = amqp_pub
if amqp_pub is None:
self.amqp_pub = AmqpPublisher(get_default_amqp_conn(), self.logger)
def get_watcher(self, watcher_id):
"""Retreive from database the watcher specified by is watcher id.
:param str watcher_id: the watcher id
:return dict: the wanted watcher. None, if no watcher match the
watcher_id
"""
watcher = self.context_graph.get_entities_by_id(watcher_id)
try:
return watcher[0]
except IndexError:
return None
def create_watcher(self, body):
"""
Create watcher entity in context and link to entities.
:param dict body: watcher conf
"""
watcher_id = body['_id']
try:
watcher_finder = json.loads(body['mfilter'])
except ValueError:
self.logger.error('can t decode mfilter')
return None
except KeyError:
self.logger.error('no filter')
return None
depends_list = self.context_graph.get_entities(
query=watcher_finder,
projection={'_id': 1}
)
self.watcher_storage.put_element(body)
depend_list = []
for entity_id in depends_list:
depend_list.append(entity_id['_id'])
entity = ContextGraph.create_entity_dict(
id=watcher_id,
name=body['display_name'],
etype='watcher',
impact=[],
depends=depend_list
)
# adding the fields specific to the Watcher entities
entity['mfilter'] = body['mfilter']
entity['state'] = 0
try:
self.context_graph.create_entity(entity)
except ValueError:
self.context_graph.update_entity(entity)
self.compute_state(watcher_id)
return True # TODO: return really something
def update_watcher(self, watcher_id, updated_field):
"""Update the watcher specified by is watcher id with updated_field.
Raise a ValueError, if the watcher_id do not match any entity.
:param str watcher_id: the watcher_id of the watcher to update
:param dict updated_field: the fields to update
:returns: the updated Watcher
:rtype: <Watcher>
"""
watcher = self.get_watcher(watcher_id)
if watcher is None:
raise ValueError("No watcher found for the following"
" id: {}".format(watcher_id))
if "mfilter" in watcher.keys() and "mfilter" in updated_field.keys():
if updated_field['mfilter'] != watcher['mfilter']:
watcher['mfilter'] = updated_field['mfilter']
query = json.loads(updated_field['mfilter'])
entities = self.context_graph.get_entities(
query=query, projection={'_id': 1})
watcher["depends"] = [entity["_id"] for entity in entities]
for key in updated_field:
if key == "infos": # update fields inside infos
for info_key in updated_field["infos"]:
watcher["infos"][info_key] = updated_field["infos"][
info_key]
watcher[key] = updated_field[key]
self.context_graph.update_entity(watcher)
def delete_watcher(self, watcher_id):
"""
Delete watcher & disable watcher entity in context.
:param string watcher_id: watcher_id
:returns: the mongodb dict response
"""
self.context_graph.delete_entity(watcher_id)
self.sla_storage.remove_elements(ids=[watcher_id])
return self.watcher_storage.remove_elements(ids=[watcher_id])
def alarm_changed(self, alarm_id):
"""
Launch a computation of a watcher state.
:param alarm_id: alarm id
"""
watchers = self.context_graph.get_entities(query={'type': 'watcher'})
for i in watchers:
if alarm_id in i['depends']:
self.compute_state(i['_id'])
def compute_watchers(self):
"""
Compute all watchers states.
"""
watchers = list(self.watcher_storage.get_elements(query={}))
for watcher in watchers:
self.compute_state(watcher['_id'])
def compute_state(self, watcher_id):
"""
Send an event watcher with the new state of the watcher.
:param watcher_id: watcher id
"""
try:
watcher_entity = self.context_graph.get_entities(
query={'_id': watcher_id})[0]
except IndexError:
return None
entities = watcher_entity['depends']
query = {"_id": {"$in": entities},
"enabled": True}
cursor = self.context_graph.get_entities(query=query,
projection={"_id": 1})
entities = []
for ent in cursor:
entities.append(ent["_id"])
display_name = watcher_entity['name']
alarm_list = list(self.alert_storage._backend.find({
'$and': [
{'d': {'$in': entities}},
{
'$or': [
{'v.resolved': None},
{'v.resolved': {'$exists': False}}
]
}
]
}))
states = []
for alarm in alarm_list:
pbh_alarm = self.pbehavior_manager.get_pbehaviors_by_eid(alarm['d'])
active_pbh = []
now = int(time.time())
for pbh in pbh_alarm:
if self.pbehavior_manager.check_active_pbehavior(now, pbh):
active_pbh.append(pbh)
if len(active_pbh) == 0:
states.append(alarm['v']['state']['val'])
nb_entities = len(entities)
nb_crit = states.count(Check.CRITICAL)
nb_major = states.count(Check.MAJOR)
nb_minor = states.count(Check.MINOR)
nb_ok = nb_entities - (nb_crit + nb_major + nb_minor)
# here add selection for calculation method actually it's worst state
# by default and think to add pbehavior in tab
computed_state = self.worst_state(nb_crit, nb_major, nb_minor)
output = '{0} ok, {1} minor, {2} major, {3} critical'.format(
nb_ok, nb_minor, nb_major, nb_crit)
if computed_state != watcher_entity.get('state', None):
watcher_entity['state'] = computed_state
self.context_graph.update_entity(watcher_entity)
self.publish_event(
display_name,
computed_state,
output,
watcher_entity['_id']
)
def compute_slas(self):
"""
Launch the sla calcul for each watchers.
"""
watcher_list = self.context_graph.get_entities(
query={'type': 'watcher',
'infos.enabled': True})
for watcher in watcher_list:
self.sla_compute(watcher['_id'], watcher['infos']['state'])
def publish_event(self, display_name, computed_state, output, _id):
"""
Publish an event watcher on amqp.
TODO: move that elsewhere (not specific to watchers)
:param display_name: watcher display_name
:param computed_state: watcher state
:param output: watcher output
"""
event = forger(
connector="canopsis",
connector_name="engine",
event_type="watcher",
source_type="component",
component=_id,
state=computed_state,
output=output,
perf_data_array=[],
display_name=display_name)
self.amqp_pub.canopsis_event(event)
def sla_compute(self, watcher_id, state):
"""
Launch the sla calcul.
:param watcher_id: watcher id
:param state: watcher state
"""
# sla_tab = list(
# self.sla_storage.get_elements(query={'_id': watcher_id}))[0]
# sla_tab['states'][state] = sla_tab['states'][state] + 1
# self.sla_storage.put_element(sla_tab)
# watcher_conf = list(
# self[self.WATCHER_STORAGE].get_elements(
# query={'_id': watcher_id})
# )[0]
# sla = Sla(self[self.WATCHER_STORAGE],
# 'test/de/rk/on/verra/plus/tard',
# watcher_conf['sla_output_tpl'],
# watcher_conf['sla_timewindow'],
# watcher_conf['sla_warning'],
# watcher_conf['alert_level'],
# watcher_conf['display_name'])
# self.logger.critical('{0}'.format((
# sla_tab['states']/
# (sla_tab['states'][1] +
# sla_tab['states'][2] +
# sla_tab['states'][3]))))
pass
@staticmethod
def worst_state(nb_crit, nb_major, nb_minor):
"""Calculate the worst state.
:param int nb_crit: critical number
:param int nb_major: major number
:param int nb_minor: minor number
:return int state: return the worst state
"""
if nb_crit > 0:
return 3
elif nb_major > 0:
return 2
elif nb_minor > 0:
return 1
return 0