def check_db(self):
"""
Check if database service is available.
:rtype: ServiceState
"""
existing_cols = self.db_store.client.collection_names()
for collection_name in self.CHECK_COLLECTIONS:
# Existence test
if collection_name not in existing_cols:
msg = 'Missing collection {}'.format(collection_name)
return ServiceState(message=msg)
# Read test
collection = self.db_store.get_collection(name=collection_name)
mongo_collection = MongoCollection(collection)
try:
mongo_collection.find({}, limit=1)
except Exception as exc:
return ServiceState(message='Find error: {}'.format(exc))
return ServiceState()
def check_db(self):
"""
Check if database service is available.
:rtype: ServiceState
"""
existing_cols = self.db_store.client.collection_names()
for collection_name in self.CHECK_COLLECTIONS:
# Existence test
if collection_name not in existing_cols:
msg = 'Missing collection {}'.format(collection_name)
return ServiceState(message=msg)
# Read test
collection = self.db_store.get_collection(name=collection_name)
mongo_collection = MongoCollection(collection)
try:
mongo_collection.find({}, limit=1)
except Exception as exc:
return ServiceState(message='Find error: {}'.format(exc))
return ServiceState()
class TestMongoCollection(unittest.TestCase):
def setUp(self):
output = StringIO()
self.logger = Logger.get('test', output, OutputStream)
self.storage = Middleware.get_middleware_by_uri(
'storage-default-testmongocollection://')
self.collection = MongoCollection(collection=self.storage._backend,
logger=self.logger)
self.id_ = 'testid'
def tearDown(self):
"""Teardown"""
self.collection.remove()
def test_insert(self):
res = self.collection.insert(document={'_id': self.id_})
self.assertEqual(res, self.id_)
res2 = self.collection.insert(document={'up': 'down'})
self.assertTrue(isinstance(res, string_types))
self.assertNotEqual(res, res2)
def test_update(self):
res = self.collection.update(query={'_id': self.id_},
document={'strange': 'charm'})
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 0)
res = self.collection.update(query={'_id': self.id_},
document={'yin': 'yang'},
upsert=True)
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 1)
res = self.collection.find_one(self.id_)
self.assertEqual(res['yin'], 'yang')
self.assertTrue('strange' not in res)
def test_remove(self):
res = self.collection.insert(document={
'_id': self.id_,
'top': 'bottom'
})
self.assertIsNotNone(res)
res = self.collection.remove(query={'_id': self.id_})
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 1)
# Deleting non-existing object doesn't throw error
res = self.collection.remove(query={})
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 0)
def test_find(self):
res = self.collection.insert(document={'_id': self.id_, 'yin': 'yang'})
self.assertIsNotNone(res)
res = self.collection.find(query={'_id': self.id_})
self.assertTrue(isinstance(res, Cursor))
res = list(res)
self.assertTrue(isinstance(res, list))
self.assertEqual(res[0]['yin'], 'yang')
def test_find_one(self):
res = self.collection.insert(document={'_id': self.id_, 'up': 'down'})
self.assertIsNotNone(res)
res = self.collection.insert(document={'strange': 'charm'})
self.assertIsNotNone(res)
res = self.collection.find_one(query={'_id': self.id_})
self.assertTrue(isinstance(res, dict))
self.assertEqual(res['up'], 'down')
def test_is_successfull(self):
dico = {'ok': 1.0, 'n': 2}
self.assertTrue(MongoCollection.is_successfull(dico))
dico = {'ok': 666.667, 'n': 1}
self.assertFalse(MongoCollection.is_successfull(dico))
dico = {'n': 2}
self.assertFalse(MongoCollection.is_successfull(dico))
class RuleManager(object):
"""
Manager for event filter rules.
"""
def __init__(self, logger):
self.logger = logger
self.rule_collection = MongoCollection(
MongoStore.get_default().get_collection(RULE_COLLECTION))
def get_by_id(self, rule_id):
"""
Get an event filter rule given its id.
:param str rule_id: the id of the rule.
:rtype: Dict[str, Any]
"""
return self.rule_collection.find_one({
RuleField.id: rule_id
})
def create(self, rule):
"""
Create a new rule and return its id.
:param Dict[str, Any] rule:
:rtype: str
:raises: InvalidRuleError if the rule is invalid. CollectionError if
the creation fails.
"""
rule_id = str(uuid4())
rule[RuleField.id] = rule_id
self.validate(rule_id, rule)
self.rule_collection.insert(rule)
return rule_id
def remove_with_id(self, rule_id):
"""
Remove a rule given its id.
:param str rule_id: the id of the rule. CollectionError if the
creation fails.
"""
self.rule_collection.remove({
RuleField.id: rule_id
})
def list(self):
"""
Return a list of all the rules.
:rtype: List[Dict[str, Any]]
"""
return list(self.rule_collection.find({}))
def update(self, rule_id, rule):
"""
Update a rule given its id.
:param str rule_id: the id of the rule.
:param Dict[str, Any] rule:
:raises: InvalidRuleError if the rule is invalid. CollectionError if
the creation fails.
"""
self.validate(rule_id, rule)
self.rule_collection.update({
RuleField.id: rule_id
}, rule, upsert=False)
def validate(self, rule_id, rule):
"""
Check that the rule is valid.
The pattern and external_data fields are not validated by this method.
:param Dict[str, Any] view:
:raises: InvalidRuleError if it is invalid.
"""
# Validate id field
if rule.get(RuleField.id, rule_id) != rule_id:
raise InvalidRuleError(
'The {0} field should not be modified.'.format(RuleField.id))
# Check that there are no unexpected fields in the rule
unexpected_fields = set(rule.keys()).difference(RuleField.values)
if unexpected_fields:
raise InvalidRuleError(
'Unexpected fields: {0}.'.format(', '.join(unexpected_fields)))
# Validate the type field
if RuleField.type not in rule:
raise InvalidRuleError(
'The {0} field is required.'.format(RuleField.type))
if rule.get(RuleField.type) not in RuleType.values:
raise InvalidRuleError(
'The {0} field should be one of: {1}.'.format(
RuleField.type,
', '.join(RuleType.values)))
# Validate the priority field
if not isinstance(rule.get(RuleField.priority, 0), int):
raise InvalidRuleError(
'The {0} field should be an integer.'.format(
RuleField.priority))
# Validate the enabled field
if not isinstance(rule.get(RuleField.enabled, True), bool):
raise InvalidRuleError(
'The {0} field should be a boolean.'.format(
RuleField.enabled))
if rule.get(RuleField.type) != RuleType.enrichment:
# Check that the enrichment fields are not defined for
# non-enrichment rules.
unexpected_fields = set(rule.keys()).intersection(
ENRICHMENT_FIELDS)
if unexpected_fields:
raise InvalidRuleError(
'The following fields should only be defined for '
'enrichment rules: {0}.'.format(
', '.join(unexpected_fields)))
else:
# Validate the actions field of the enrichment rules.
if RuleField.actions not in rule:
raise InvalidRuleError(
'The {0} field is required for enrichment rules.'.format(
RuleField.actions))
if not isinstance(rule.get(RuleField.actions), list):
raise InvalidRuleError(
'The {0} field should be a list.'.format(
RuleField.actions))
if not rule.get(RuleField.actions):
raise InvalidRuleError(
'The {0} field should contain at least one action.'.format(
RuleField.actions))
# Validate the on_success field of the enrichment rules.
outcome = rule.get(RuleField.on_success)
if outcome and outcome not in RuleOutcome.values:
raise InvalidRuleError(
'The {0} field should be one of: {1}.'.format(
RuleField.on_success,
', '.join(RuleOutcome.values)))
# Validate the on_failure field of the enrichment rules.
outcome = rule.get(RuleField.on_failure)
if outcome and outcome not in RuleOutcome.values:
raise InvalidRuleError(
'The {0} field should be one of: {1}.'.format(
RuleField.on_failure,
', '.join(RuleOutcome.values)))
class HeartbeatManager(object):
"""
Heartbeat service manager abstraction.
"""
COLLECTION = 'heartbeat'
LOG_PATH = 'var/log/heartbeat.log'
@classmethod
def provide_default_basics(cls):
"""
Provide logger, config, storages...
! Do not use in tests !
:rtype: Union[logging.Logger,
canopsis.common.collection.MongoCollection]
"""
store = MongoStore.get_default()
collection = store.get_collection(name=cls.COLLECTION)
return (Logger.get('action',
cls.LOG_PATH), MongoCollection(collection))
def __init__(self, logger, collection):
"""
:param `~.logger.Logger` logger: object.
:param `~.common.collection.MongoCollection` collection: object.
"""
self.__logger = logger
self.__collection = MongoCollection(collection)
def create(self, heartbeat):
"""
Create a new Heartbeat.
:param `HeartBeat` heartbeat: a Heartbeat model.
:returns: a created Heartbeat ID.
:rtype: `str`.
:raises: (`.HeartbeatPatternExistsError`,
`pymongo.errors.PyMongoError`,
`~.common.collection.CollectionError`, ).
"""
if self.get(heartbeat.id):
raise HeartbeatPatternExistsError()
return self.__collection.insert(heartbeat.to_dict())
def get(self, heartbeat_id=None):
"""
Get Heartbeat by ID or a list of Heartbeats
when calling with default arguments.
:param `Optional[str]` heartbeat_id:
:returns: list of Heartbeat documents if **heartbeat_id** is None
else single Heartbeat document or None if not found.
:raises: (`pymongo.errors.PyMongoError`, ).
"""
if heartbeat_id:
return self.__collection.find_one({"_id": heartbeat_id})
return [x for x in self.__collection.find({})]
def delete(self, heartbeat_id):
"""
Delete Heartbeat by ID.
:param `str` heartbeat_id: Heartbeat ID.
:return:
:raises: (`~.common.collection.CollectionError`, ).
"""
return self.__collection.remove({"_id": heartbeat_id})
class engine(Engine):
etype = 'event_filter'
def __init__(self, *args, **kargs):
super(engine, self).__init__(*args, **kargs)
self.mg_store = MongoStore.get_default()
self.collection = MongoCollection(
self.mg_store.get_collection("object"))
self.name = kargs['name']
self.drop_event_count = 0
self.pass_event_count = 0
self.__load_rules()
def pre_run(self):
self.beat()
def a_override(self, event, action):
"""Override a field from event or add a new one if it does not have
one.
"""
afield = action.get('field', None)
avalue = action.get('value', None)
# This must be a hard check because value can be a boolean or a null
# integer
if afield is None or avalue is None:
self.logger.error(
"Malformed action ('field' and 'value' required): {}".format(
action))
return False
if afield not in event:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
# afield is in event
if not isinstance(avalue, list):
if isinstance(event[afield], list):
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
event[afield].append(avalue)
else:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
else:
# operation field is supported only for list values
op = action.get('operation', 'append')
if op == 'override':
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
elif op == 'append':
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
if isinstance(event[afield], list):
event[afield] += avalue
else:
event[afield] = [event[afield]] + avalue
return True
else:
self.logger.error(
"Operation '{}' unsupported (action '{}')".format(
op, action))
return False
def a_remove(self, event, action):
"""Remove an event from a field in event or the whole field if no
element is specified.
"""
akey = action.get('key', None)
aelement = action.get('element', None)
del_met = action.get('met', 0)
if akey:
if aelement:
if del_met:
for i, met in enumerate(event[akey]):
if met['name'] == aelement:
del event[akey][i]
break
elif isinstance(event[akey], dict):
del event[akey][aelement]
elif isinstance(event[akey], list):
del event[akey][event[akey].index(aelement)]
self.logger.debug(u" + {}: Removed: '{}' from '{}'".format(
event['rk'], aelement, akey))
else:
del event[akey]
self.logger.debug(u" + {}: Removed: '{}'".format(
event['rk'], akey))
return True
else:
self.logger.error(
u"Action malformed (needs 'key' and/or 'element'): {}".format(
action))
return False
def a_modify(self, event, action, name):
"""
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
derogated = False
atype = action.get('type')
actionMap = {'override': self.a_override, 'remove': self.a_remove}
if atype in actionMap:
derogated = actionMap[atype](event, action)
else:
self.logger.warning(u"Unknown action '{}'".format(atype))
# If the event was derogated, fill some informations
if derogated:
self.logger.debug(u"Event changed by rule '{}'".format(name))
return None
def a_drop(self, event, action, name):
""" Drop the event.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event dropped by rule '{}'".format(name))
self.drop_event_count += 1
return DROP
def a_pass(self, event, action, name):
"""Pass the event to the next queue.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event passed by rule '{}'".format(name))
self.pass_event_count += 1
return event
def a_route(self, event, action, name):
"""
Change the route to which an event will be sent
Args:
event: map of the event to be modified
action: map of type action
name: of the rule
Returns:
``None``
"""
if "route" in action:
self.next_amqp_queues = [action["route"]]
self.logger.debug(u"Event re-routed by rule '{}'".format(name))
else:
self.logger.error(
u"Action malformed (needs 'route'): {}".format(action))
return None
def a_exec_job(self, event, action, name):
records = self.collection.find({
'crecord_type': 'job',
'_id': action['job']
})
for record in records:
job = copy.deepcopy(record)
job['context'] = event
try:
self.work_amqp_publisher.direct_event(job, 'Engine_scheduler')
except Exception as e:
self.logger.exception("Unable to send job")
time.sleep(1)
return True
def a_snooze(self, event, action, name):
"""
Snooze event checks
:param dict event: event to be snoozed
:param dict action: action
:param str name: name of the rule
:returns: True if a snooze has been sent, False otherwise
:rtype: boolean
"""
if event.get('event_type') == 'snooze':
return False
# Only check events can trigger an auto-snooze
if event.get('event_type') != 'check':
return False
# A check OK cannot trigger an auto-snooze
if event.get('state') == 0:
return False
# Alerts manager caching
if not hasattr(self, 'am'):
self.am = Alerts(*Alerts.provide_default_basics())
# Context manager caching
if not hasattr(self, 'cm'):
self.cm = ContextGraph(self.logger)
entity_id = self.cm.get_id(event)
current_alarm = self.am.get_current_alarm(entity_id)
if current_alarm is None:
snooze = {
'connector': event.get('connector', ''),
'connector_name': event.get('connector_name', ''),
'source_type': event.get('source_type', ''),
'component': event.get('component', ''),
'event_type': 'snooze',
'duration': action['duration'],
'author': 'event_filter',
'output': 'Auto snooze generated by rule "{}"'.format(name),
'timestamp': int(time.time())
}
if event.get('resource', ''):
snooze['resource'] = event['resource']
try:
self.work_amqp_publisher.direct_event(snooze,
'Engine_event_filter')
except Exception as e:
self.logger.exception("Unable to send snooze event")
return True
return False
def a_baseline(self, event, actions, name):
"""a_baseline
:param event:
:param action: baseline conf in event filter
:param name:
"""
event['baseline_name'] = actions['baseline_name']
event['check_frequency'] = actions['check_frequency']
try:
self.work_amqp_publisher.direct_event(event, 'Engine_baseline')
except Exception as e:
self.logger.exception("Unable to send baseline event")
def apply_actions(self, event, actions):
pass_event = False
actionMap = {
'drop': self.a_drop,
'pass': self.a_pass,
'override': self.a_modify,
'remove': self.a_modify,
'execjob': self.a_exec_job,
'route': self.a_route,
'snooze': self.a_snooze,
'baseline': self.a_baseline
}
for name, action in actions:
if action['type'] in actionMap:
ret = actionMap[action['type'].lower()](event, action, name)
if ret:
pass_event = True
else:
self.logger.warning(u"Unknown action '{}'".format(action))
return pass_event
def work(self, event, *xargs, **kwargs):
rk = get_routingkey(event)
default_action = self.configuration.get('default_action', 'pass')
# list of supported actions
rules = self.configuration.get('rules', [])
to_apply = []
self.logger.debug(u'event {}'.format(event))
# When list configuration then check black and
# white lists depending on json configuration
for filterItem in rules:
actions = filterItem.get('actions')
name = filterItem.get('name', 'no_name')
self.logger.debug(u'rule {}'.format(filterItem))
self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
# Try filter rules on current event
if filterItem['mfilter'] and check(filterItem['mfilter'], event):
self.logger.debug(u'Event: {}, filter matches'.format(
event.get('rk', event)))
if 'pbehaviors' in filterItem:
pbehaviors = filterItem.get('pbehaviors', {})
list_in = pbehaviors.get('in', [])
list_out = pbehaviors.get('out', [])
if list_in or list_out:
pbm = singleton_per_scope(PBehaviorManager)
cm = singleton_per_scope(ContextGraph)
entity = cm.get_entity(event)
entity_id = cm.get_entity_id(entity)
result = pbm.check_pbehaviors(entity_id, list_in,
list_out)
if not result:
break
for action in actions:
if action['type'].lower() == 'drop':
self.apply_actions(event, to_apply)
return self.a_drop(event, None, name)
to_apply.append((name, action))
if filterItem.get('break', 0):
self.logger.debug(
u' + Filter {} broke the next filters processing'.
format(filterItem.get('name', 'filter')))
break
if len(to_apply):
if self.apply_actions(event, to_apply):
self.logger.debug(u'Event before sent to next engine: %s' %
event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
# No rules matched
if default_action == 'drop':
self.logger.debug("Event '%s' dropped by default action" % (rk))
self.drop_event_count += 1
return DROP
self.logger.debug("Event '%s' passed by default action" % (rk))
self.pass_event_count += 1
self.logger.debug(u'Event before sent to next engine: %s' % event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
def __load_rules(self):
tmp_rules = []
records = self.collection.find({
'crecord_type': 'filter',
'enable': True
})
records.sort('priority', 1)
for record in records:
record_dump = copy.deepcopy(record)
self.set_loaded(record_dump)
try:
record_dump["mfilter"] = loads(record_dump["mfilter"])
except Exception:
self.logger.info(u'Invalid mfilter {}, filter {}'.format(
record_dump['mfilter'],
record_dump['name'],
))
self.logger.debug(u'Loading record_dump:')
self.logger.debug(record_dump)
tmp_rules.append(record_dump)
self.configuration = {
'rules': tmp_rules,
'default_action': self.find_default_action()
}
def beat(self, *args, **kargs):
""" Configuration reload for realtime ui changes handling """
self.logger.debug(u'Reload configuration rules')
self.__load_rules()
self.logger.debug('Loaded {} rules'.format(
len(self.configuration['rules'])))
self.send_stat_event()
def set_loaded(self, record):
if 'run_once' in record and not record['run_once']:
self.collection.update({"_id": record['_id']},
{"$set": {
'run_once': True
}})
self.logger.info('record {} has been run once'.format(
record['_id']))
def send_stat_event(self):
""" Send AMQP Event for drop and pass metrics """
message_dropped = '{} event dropped since {}'.format(
self.drop_event_count, self.beat_interval)
message_passed = '{} event passed since {}'.format(
self.pass_event_count, self.beat_interval)
event = forger(connector='Engine',
connector_name='engine',
event_type='check',
source_type='resource',
resource=self.amqp_queue + '_data',
state=0,
state_type=1,
output=message_dropped,
perf_data_array=[{
'metric': 'pass_event',
'value': self.pass_event_count,
'type': 'GAUGE'
}, {
'metric': 'drop_event',
'value': self.drop_event_count,
'type': 'GAUGE'
}])
self.logger.debug(message_dropped)
self.logger.debug(message_passed)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send stat event")
self.drop_event_count = 0
self.pass_event_count = 0
def find_default_action(self):
"""Find the default action stored and returns it, else assume it
default action is pass.
"""
records = self.collection.find_one({'crecord_type': 'defaultrule'})
if records:
return records[0]["action"]
self.logger.debug(
"No default action found. Assuming default action is pass")
return 'pass'
class ViewAdapter(object):
"""
Adapter for the view collection.
"""
def __init__(self, logger):
self.logger = logger
self.view_collection = MongoCollection(
MongoStore.get_default().get_collection(VIEWS_COLLECTION))
self.group_collection = MongoCollection(
MongoStore.get_default().get_collection(GROUPS_COLLECTION))
def get_by_id(self, view_id):
"""
Get a view given its id.
:param str view_id: the id of the view.
"""
return self.view_collection.find_one({ViewField.id: view_id})
def create(self, view):
"""
Create a new view and return its id.
:param Dict[str, Any] view:
:rtype: str
"""
view_id = str(uuid4())
view[ViewField.id] = view_id
self.validate(view_id, view)
self.view_collection.insert(view)
return view_id
def update(self, view_id, view):
"""
Update a view given its id.
:param str view_id: the id of the view.
:param Dict[str, Any] view:
"""
self.validate(view_id, view)
self.view_collection.update({ViewField.id: view_id},
view,
upsert=False)
def remove_with_id(self, view_id):
"""
Remove a view given its id.
:param str view_id: the id of the view.
"""
self.view_collection.remove({ViewField.id: view_id})
def list(self, name=None, title=None):
"""
Return a list of views, optionally filtered by name or title.
:param str name:
:param str title:
:rtype: List[Dict[str, Any]]
:raises: InvalidFilterError
"""
view_filter = {}
if name is not None and title is not None:
raise InvalidFilterError(
'Cannot filter both by name and by title.')
if name is not None:
view_filter[ViewField.name] = name
if title is not None:
view_filter[ViewField.title] = title
views = self.view_collection.find(view_filter)
groups = self.group_collection.find({})
response_groups = {}
for group in groups:
group_id = group[GroupField.id]
del group[GroupField.id]
group[GroupField.views] = []
response_groups[group_id] = group
for view in views:
try:
group_id = view[ViewField.group_id]
except KeyError:
# This should never happen as long as the collections are only
# modified using the API
self.logger.exception(
'The view {0} is missing the group_id field.'.format(
view[ViewField.id]))
continue
try:
response_groups[group_id][GroupField.views].append(view)
except KeyError:
# This should never happen as long as the collections are only
# modified using the API
self.logger.exception('No group with id: {0}'.format(group_id))
return {ViewResponseField.groups: response_groups}
def validate(self, view_id, view):
"""
Check that the view is valid, return InvalidViewError if it is not.
:param Dict[str, Any] view:
"""
# Validate id field
if view.get(ViewField.id, view_id) != view_id:
raise InvalidViewError(
'The {0} field should not be modified'.format(ViewField.id))
# Validate group_id field
if ViewField.group_id not in view:
raise InvalidViewError('The view should have a group_id field.')
group_id = view[ViewField.group_id]
group = self.group_collection.find_one({GroupField.id: group_id})
if not group:
raise InvalidViewError('No group with id: {0}'.format(group_id))
# Validate name field
if ViewField.name not in view:
raise InvalidViewError('The view should have a name field.')
view_name = view[ViewField.name]
same_name_view = self.view_collection.find_one({
ViewField.id: {
'$ne': view_id
},
ViewField.name:
view_name
})
if same_name_view:
raise InvalidViewError(
'There is already a view with the name: {0}'.format(view_name))
# Validate title field
if ViewField.title not in view:
raise InvalidViewError('The view should have a title field.')
class GroupAdapter(object):
"""
Adapter for the group collection.
"""
def __init__(self, logger):
self.logger = logger
self.group_collection = MongoCollection(
MongoStore.get_default().get_collection(GROUPS_COLLECTION))
self.view_collection = MongoCollection(
MongoStore.get_default().get_collection(VIEWS_COLLECTION))
def get_by_id(self, group_id, name=None, title=None):
"""
Get a group given its id.
:param str group_id: the id of the group.
:param str name:
:param str title:
:rtype: Dict[str, Any]
"""
group = self.group_collection.find_one({GroupField.id: group_id})
if group:
group[GroupField.views] = self.get_views(group_id, name, title)
return group
def get_views(self, group_id, name=None, title=None):
"""
Returns the list of views of a group, optionally filtered by name or
title.
:param str group_id:
:param str name:
:param str title:
:rtype: List[Dict[str, Any]]
:raises: InvalidFilterError
"""
view_filter = {ViewField.group_id: group_id}
if name is not None and title is not None:
raise InvalidFilterError(
'Cannot filter both by name and by title.')
if name is not None:
view_filter[ViewField.name] = name
if title is not None:
view_filter[ViewField.title] = title
return list(self.view_collection.find(view_filter))
def is_empty(self, group_id):
"""
Return True if a group is empty.
:param str group_id:
:rtype: bool
"""
return self.view_collection.find({
ViewField.group_id: group_id
}).limit(1).count() == 0
def create(self, group):
"""
Create a new group.
:param Dict group:
:rtype: str
:raises: InvalidGroupError
"""
group_id = str(uuid4())
group[GroupField.id] = group_id
self.validate(group_id, group)
self.group_collection.insert(group)
return group_id
def update(self, group_id, group):
"""
Update a group given its id.
:param str group_id:
:param Dict group:
:raises: InvalidGroupError
"""
self.validate(group_id, group)
self.group_collection.update({GroupField.id: group_id},
group,
upsert=False)
def remove_with_id(self, group_id):
"""
Remove a group given its id.
:param str group_id:
:raises: NonEmptyGroupError
"""
if not self.is_empty(group_id):
raise NonEmptyGroupError()
self.group_collection.remove({GroupField.id: group_id})
def list(self, name=None):
"""
Return a list of groups, optionally filtered by name.
:param str name:
:rtype: List[Dict[str, Any]]
"""
group_filter = {}
if name is not None:
group_filter[GroupField.name] = name
return list(self.group_collection.find(group_filter))
def validate(self, group_id, group):
"""
Check that the gorup is valid, return InvalidGroupError if it is not.
:param Dict[str, Any] view:
:raises: InvalidGroupError
"""
if group.get(GroupField.id, group_id) != group_id:
raise InvalidGroupError(
'The {0} field should not be modified'.format(GroupField.id))
if GroupField.name not in group:
raise InvalidGroupError('The group should have a name field.')
group_name = group[GroupField.name]
same_name_group = self.group_collection.find_one({
GroupField.id: {
'$ne': group_id
},
GroupField.name:
group_name
})
if same_name_group:
raise InvalidGroupError(
'There is already a group with the name: {0}'.format(
group_name))
def exists(self, group_id):
"""
Return True if a group exists.
:param str group_id:
:rtype: bool
"""
group = self.group_collection.find_one({GroupField.id: group_id})
return group is not None
class engine(Engine):
etype = 'event_filter'
def __init__(self, *args, **kargs):
super(engine, self).__init__(*args, **kargs)
self.mg_store = MongoStore.get_default()
self.collection = MongoCollection(self.mg_store.get_collection("object"))
self.name = kargs['name']
self.drop_event_count = 0
self.pass_event_count = 0
self.__load_rules()
def pre_run(self):
self.beat()
def a_override(self, event, action):
"""Override a field from event or add a new one if it does not have
one.
"""
afield = action.get('field', None)
avalue = action.get('value', None)
# This must be a hard check because value can be a boolean or a null
# integer
if afield is None or avalue is None:
self.logger.error(
"Malformed action ('field' and 'value' required): {}".format(
action
)
)
return False
if afield not in event:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
# afield is in event
if not isinstance(avalue, list):
if isinstance(event[afield], list):
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
event[afield].append(avalue)
else:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
else:
# operation field is supported only for list values
op = action.get('operation', 'append')
if op == 'override':
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
elif op == 'append':
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
if isinstance(event[afield], list):
event[afield] += avalue
else:
event[afield] = [event[afield]] + avalue
return True
else:
self.logger.error(
"Operation '{}' unsupported (action '{}')".format(
op, action
)
)
return False
def a_remove(self, event, action):
"""Remove an event from a field in event or the whole field if no
element is specified.
"""
akey = action.get('key', None)
aelement = action.get('element', None)
del_met = action.get('met', 0)
if akey:
if aelement:
if del_met:
for i, met in enumerate(event[akey]):
if met['name'] == aelement:
del event[akey][i]
break
elif isinstance(event[akey], dict):
del event[akey][aelement]
elif isinstance(event[akey], list):
del event[akey][event[akey].index(aelement)]
self.logger.debug(u" + {}: Removed: '{}' from '{}'".format(
event['rk'],
aelement,
akey))
else:
del event[akey]
self.logger.debug(u" + {}: Removed: '{}'".format(
event['rk'],
akey))
return True
else:
self.logger.error(
u"Action malformed (needs 'key' and/or 'element'): {}".format(
action))
return False
def a_modify(self, event, action, name):
"""
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
derogated = False
atype = action.get('type')
actionMap = {
'override': self.a_override,
'remove': self.a_remove
}
if atype in actionMap:
derogated = actionMap[atype](event, action)
else:
self.logger.warning(u"Unknown action '{}'".format(atype))
# If the event was derogated, fill some informations
if derogated:
self.logger.debug(u"Event changed by rule '{}'".format(name))
return None
def a_drop(self, event, action, name):
""" Drop the event.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event dropped by rule '{}'".format(name))
self.drop_event_count += 1
return DROP
def a_pass(self, event, action, name):
"""Pass the event to the next queue.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event passed by rule '{}'".format(name))
self.pass_event_count += 1
return event
def a_route(self, event, action, name):
"""
Change the route to which an event will be sent
Args:
event: map of the event to be modified
action: map of type action
name: of the rule
Returns:
``None``
"""
if "route" in action:
self.next_amqp_queues = [action["route"]]
self.logger.debug(u"Event re-routed by rule '{}'".format(name))
else:
self.logger.error(
u"Action malformed (needs 'route'): {}".format(action))
return None
def a_exec_job(self, event, action, name):
records = self.collection.find(
{'crecord_type': 'job', '_id': action['job']}
)
for record in records:
job = copy.deepcopy(record)
job['context'] = event
try:
self.work_amqp_publisher.direct_event(job, 'Engine_scheduler')
except Exception as e:
self.logger.exception("Unable to send job")
time.sleep(1)
return True
def a_snooze(self, event, action, name):
"""
Snooze event checks
:param dict event: event to be snoozed
:param dict action: action
:param str name: name of the rule
:returns: True if a snooze has been sent, False otherwise
:rtype: boolean
"""
if event.get('event_type') == 'snooze':
return False
# Only check events can trigger an auto-snooze
if event.get('event_type') != 'check':
return False
# A check OK cannot trigger an auto-snooze
if event.get('state') == 0:
return False
# Alerts manager caching
if not hasattr(self, 'am'):
self.am = Alerts(*Alerts.provide_default_basics())
# Context manager caching
if not hasattr(self, 'cm'):
self.cm = ContextGraph(self.logger)
entity_id = self.cm.get_id(event)
current_alarm = self.am.get_current_alarm(entity_id)
if current_alarm is None:
snooze = {
'connector': event.get('connector', ''),
'connector_name': event.get('connector_name', ''),
'source_type': event.get('source_type', ''),
'component': event.get('component', ''),
'event_type': 'snooze',
'duration': action['duration'],
'author': 'event_filter',
'output': 'Auto snooze generated by rule "{}"'.format(name),
'timestamp': int(time.time())
}
if event.get('resource', ''):
snooze['resource'] = event['resource']
try:
self.work_amqp_publisher.direct_event(
snooze, 'Engine_event_filter')
except Exception as e:
self.logger.exception("Unable to send snooze event")
return True
return False
def a_baseline(self, event, actions, name):
"""a_baseline
:param event:
:param action: baseline conf in event filter
:param name:
"""
event['baseline_name'] = actions['baseline_name']
event['check_frequency'] = actions['check_frequency']
try:
self.work_amqp_publisher.direct_event(
event, 'Engine_baseline')
except Exception as e:
self.logger.exception("Unable to send baseline event")
def apply_actions(self, event, actions):
pass_event = False
actionMap = {
'drop': self.a_drop,
'pass': self.a_pass,
'override': self.a_modify,
'remove': self.a_modify,
'execjob': self.a_exec_job,
'route': self.a_route,
'snooze': self.a_snooze,
'baseline': self.a_baseline
}
for name, action in actions:
if action['type'] in actionMap:
ret = actionMap[action['type'].lower()](event, action, name)
if ret:
pass_event = True
else:
self.logger.warning(u"Unknown action '{}'".format(action))
return pass_event
def work(self, event, *xargs, **kwargs):
rk = get_routingkey(event)
default_action = self.configuration.get('default_action', 'pass')
# list of supported actions
rules = self.configuration.get('rules', [])
to_apply = []
self.logger.debug(u'event {}'.format(event))
# When list configuration then check black and
# white lists depending on json configuration
for filterItem in rules:
actions = filterItem.get('actions')
name = filterItem.get('name', 'no_name')
self.logger.debug(u'rule {}'.format(filterItem))
self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
# Try filter rules on current event
if filterItem['mfilter'] and check(filterItem['mfilter'], event):
self.logger.debug(
u'Event: {}, filter matches'.format(event.get('rk', event))
)
if 'pbehaviors' in filterItem:
pbehaviors = filterItem.get('pbehaviors', {})
list_in = pbehaviors.get('in', [])
list_out = pbehaviors.get('out', [])
if list_in or list_out:
pbm = singleton_per_scope(PBehaviorManager)
cm = singleton_per_scope(ContextGraph)
entity = cm.get_entity(event)
entity_id = cm.get_entity_id(entity)
result = pbm.check_pbehaviors(
entity_id, list_in, list_out
)
if not result:
break
for action in actions:
if action['type'].lower() == 'drop':
self.apply_actions(event, to_apply)
return self.a_drop(event, None, name)
to_apply.append((name, action))
if filterItem.get('break', 0):
self.logger.debug(
u' + Filter {} broke the next filters processing'
.format(
filterItem.get('name', 'filter')
)
)
break
if len(to_apply):
if self.apply_actions(event, to_apply):
self.logger.debug(
u'Event before sent to next engine: %s' % event
)
event['rk'] = event['_id'] = get_routingkey(event)
return event
# No rules matched
if default_action == 'drop':
self.logger.debug("Event '%s' dropped by default action" % (rk))
self.drop_event_count += 1
return DROP
self.logger.debug("Event '%s' passed by default action" % (rk))
self.pass_event_count += 1
self.logger.debug(u'Event before sent to next engine: %s' % event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
def __load_rules(self):
tmp_rules = []
records = self.collection.find(
{'crecord_type': 'filter', 'enable': True})
records.sort('priority', 1)
for record in records:
record_dump = copy.deepcopy(record)
self.set_loaded(record_dump)
try:
record_dump["mfilter"] = loads(record_dump["mfilter"])
except Exception:
self.logger.info(u'Invalid mfilter {}, filter {}'.format(
record_dump['mfilter'],
record_dump['name'],
))
self.logger.debug(u'Loading record_dump:')
self.logger.debug(record_dump)
tmp_rules.append(record_dump)
self.configuration = {
'rules': tmp_rules,
'default_action': self.find_default_action()
}
def beat(self, *args, **kargs):
""" Configuration reload for realtime ui changes handling """
self.logger.debug(u'Reload configuration rules')
self.__load_rules()
self.logger.debug(
'Loaded {} rules'.format(len(self.configuration['rules']))
)
self.send_stat_event()
def set_loaded(self, record):
if 'run_once' in record and not record['run_once']:
self.collection.update({"_id": record['_id']}, {"$set": {'run_once': True}})
self.logger.info(
'record {} has been run once'.format(record['_id'])
)
def send_stat_event(self):
""" Send AMQP Event for drop and pass metrics """
message_dropped = '{} event dropped since {}'.format(
self.drop_event_count,
self.beat_interval
)
message_passed = '{} event passed since {}'.format(
self.pass_event_count,
self.beat_interval
)
event = forger(
connector='Engine',
connector_name='engine',
event_type='check',
source_type='resource',
resource=self.amqp_queue + '_data',
state=0,
state_type=1,
output=message_dropped,
perf_data_array=[
{'metric': 'pass_event',
'value': self.pass_event_count,
'type': 'GAUGE'},
{'metric': 'drop_event',
'value': self.drop_event_count,
'type': 'GAUGE'}
]
)
self.logger.debug(message_dropped)
self.logger.debug(message_passed)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send stat event")
self.drop_event_count = 0
self.pass_event_count = 0
def find_default_action(self):
"""Find the default action stored and returns it, else assume it
default action is pass.
"""
records = self.collection.find_one({'crecord_type': 'defaultrule'})
if records:
return records[0]["action"]
self.logger.debug(
"No default action found. Assuming default action is pass"
)
return 'pass'