def check_db(self): """ Check if database service is available. :rtype: ServiceState """ existing_cols = self.db_store.client.collection_names() for collection_name in self.CHECK_COLLECTIONS: # Existence test if collection_name not in existing_cols: msg = 'Missing collection {}'.format(collection_name) return ServiceState(message=msg) # Read test collection = self.db_store.get_collection(name=collection_name) mongo_collection = MongoCollection(collection) try: mongo_collection.find({}, limit=1) except Exception as exc: return ServiceState(message='Find error: {}'.format(exc)) return ServiceState()
def check_db(self): """ Check if database service is available. :rtype: ServiceState """ existing_cols = self.db_store.client.collection_names() for collection_name in self.CHECK_COLLECTIONS: # Existence test if collection_name not in existing_cols: msg = 'Missing collection {}'.format(collection_name) return ServiceState(message=msg) # Read test collection = self.db_store.get_collection(name=collection_name) mongo_collection = MongoCollection(collection) try: mongo_collection.find({}, limit=1) except Exception as exc: return ServiceState(message='Find error: {}'.format(exc)) return ServiceState()
class TestMongoCollection(unittest.TestCase): def setUp(self): output = StringIO() self.logger = Logger.get('test', output, OutputStream) self.storage = Middleware.get_middleware_by_uri( 'storage-default-testmongocollection://') self.collection = MongoCollection(collection=self.storage._backend, logger=self.logger) self.id_ = 'testid' def tearDown(self): """Teardown""" self.collection.remove() def test_insert(self): res = self.collection.insert(document={'_id': self.id_}) self.assertEqual(res, self.id_) res2 = self.collection.insert(document={'up': 'down'}) self.assertTrue(isinstance(res, string_types)) self.assertNotEqual(res, res2) def test_update(self): res = self.collection.update(query={'_id': self.id_}, document={'strange': 'charm'}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 0) res = self.collection.update(query={'_id': self.id_}, document={'yin': 'yang'}, upsert=True) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 1) res = self.collection.find_one(self.id_) self.assertEqual(res['yin'], 'yang') self.assertTrue('strange' not in res) def test_remove(self): res = self.collection.insert(document={ '_id': self.id_, 'top': 'bottom' }) self.assertIsNotNone(res) res = self.collection.remove(query={'_id': self.id_}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 1) # Deleting non-existing object doesn't throw error res = self.collection.remove(query={}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 0) def test_find(self): res = self.collection.insert(document={'_id': self.id_, 'yin': 'yang'}) self.assertIsNotNone(res) res = self.collection.find(query={'_id': self.id_}) self.assertTrue(isinstance(res, Cursor)) res = list(res) self.assertTrue(isinstance(res, list)) self.assertEqual(res[0]['yin'], 'yang') def test_find_one(self): res = self.collection.insert(document={'_id': self.id_, 'up': 'down'}) self.assertIsNotNone(res) res = self.collection.insert(document={'strange': 'charm'}) self.assertIsNotNone(res) res = self.collection.find_one(query={'_id': self.id_}) self.assertTrue(isinstance(res, dict)) self.assertEqual(res['up'], 'down') def test_is_successfull(self): dico = {'ok': 1.0, 'n': 2} self.assertTrue(MongoCollection.is_successfull(dico)) dico = {'ok': 666.667, 'n': 1} self.assertFalse(MongoCollection.is_successfull(dico)) dico = {'n': 2} self.assertFalse(MongoCollection.is_successfull(dico))
class RuleManager(object): """ Manager for event filter rules. """ def __init__(self, logger): self.logger = logger self.rule_collection = MongoCollection( MongoStore.get_default().get_collection(RULE_COLLECTION)) def get_by_id(self, rule_id): """ Get an event filter rule given its id. :param str rule_id: the id of the rule. :rtype: Dict[str, Any] """ return self.rule_collection.find_one({ RuleField.id: rule_id }) def create(self, rule): """ Create a new rule and return its id. :param Dict[str, Any] rule: :rtype: str :raises: InvalidRuleError if the rule is invalid. CollectionError if the creation fails. """ rule_id = str(uuid4()) rule[RuleField.id] = rule_id self.validate(rule_id, rule) self.rule_collection.insert(rule) return rule_id def remove_with_id(self, rule_id): """ Remove a rule given its id. :param str rule_id: the id of the rule. CollectionError if the creation fails. """ self.rule_collection.remove({ RuleField.id: rule_id }) def list(self): """ Return a list of all the rules. :rtype: List[Dict[str, Any]] """ return list(self.rule_collection.find({})) def update(self, rule_id, rule): """ Update a rule given its id. :param str rule_id: the id of the rule. :param Dict[str, Any] rule: :raises: InvalidRuleError if the rule is invalid. CollectionError if the creation fails. """ self.validate(rule_id, rule) self.rule_collection.update({ RuleField.id: rule_id }, rule, upsert=False) def validate(self, rule_id, rule): """ Check that the rule is valid. The pattern and external_data fields are not validated by this method. :param Dict[str, Any] view: :raises: InvalidRuleError if it is invalid. """ # Validate id field if rule.get(RuleField.id, rule_id) != rule_id: raise InvalidRuleError( 'The {0} field should not be modified.'.format(RuleField.id)) # Check that there are no unexpected fields in the rule unexpected_fields = set(rule.keys()).difference(RuleField.values) if unexpected_fields: raise InvalidRuleError( 'Unexpected fields: {0}.'.format(', '.join(unexpected_fields))) # Validate the type field if RuleField.type not in rule: raise InvalidRuleError( 'The {0} field is required.'.format(RuleField.type)) if rule.get(RuleField.type) not in RuleType.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.type, ', '.join(RuleType.values))) # Validate the priority field if not isinstance(rule.get(RuleField.priority, 0), int): raise InvalidRuleError( 'The {0} field should be an integer.'.format( RuleField.priority)) # Validate the enabled field if not isinstance(rule.get(RuleField.enabled, True), bool): raise InvalidRuleError( 'The {0} field should be a boolean.'.format( RuleField.enabled)) if rule.get(RuleField.type) != RuleType.enrichment: # Check that the enrichment fields are not defined for # non-enrichment rules. unexpected_fields = set(rule.keys()).intersection( ENRICHMENT_FIELDS) if unexpected_fields: raise InvalidRuleError( 'The following fields should only be defined for ' 'enrichment rules: {0}.'.format( ', '.join(unexpected_fields))) else: # Validate the actions field of the enrichment rules. if RuleField.actions not in rule: raise InvalidRuleError( 'The {0} field is required for enrichment rules.'.format( RuleField.actions)) if not isinstance(rule.get(RuleField.actions), list): raise InvalidRuleError( 'The {0} field should be a list.'.format( RuleField.actions)) if not rule.get(RuleField.actions): raise InvalidRuleError( 'The {0} field should contain at least one action.'.format( RuleField.actions)) # Validate the on_success field of the enrichment rules. outcome = rule.get(RuleField.on_success) if outcome and outcome not in RuleOutcome.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.on_success, ', '.join(RuleOutcome.values))) # Validate the on_failure field of the enrichment rules. outcome = rule.get(RuleField.on_failure) if outcome and outcome not in RuleOutcome.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.on_failure, ', '.join(RuleOutcome.values)))
class HeartbeatManager(object): """ Heartbeat service manager abstraction. """ COLLECTION = 'heartbeat' LOG_PATH = 'var/log/heartbeat.log' @classmethod def provide_default_basics(cls): """ Provide logger, config, storages... ! Do not use in tests ! :rtype: Union[logging.Logger, canopsis.common.collection.MongoCollection] """ store = MongoStore.get_default() collection = store.get_collection(name=cls.COLLECTION) return (Logger.get('action', cls.LOG_PATH), MongoCollection(collection)) def __init__(self, logger, collection): """ :param `~.logger.Logger` logger: object. :param `~.common.collection.MongoCollection` collection: object. """ self.__logger = logger self.__collection = MongoCollection(collection) def create(self, heartbeat): """ Create a new Heartbeat. :param `HeartBeat` heartbeat: a Heartbeat model. :returns: a created Heartbeat ID. :rtype: `str`. :raises: (`.HeartbeatPatternExistsError`, `pymongo.errors.PyMongoError`, `~.common.collection.CollectionError`, ). """ if self.get(heartbeat.id): raise HeartbeatPatternExistsError() return self.__collection.insert(heartbeat.to_dict()) def get(self, heartbeat_id=None): """ Get Heartbeat by ID or a list of Heartbeats when calling with default arguments. :param `Optional[str]` heartbeat_id: :returns: list of Heartbeat documents if **heartbeat_id** is None else single Heartbeat document or None if not found. :raises: (`pymongo.errors.PyMongoError`, ). """ if heartbeat_id: return self.__collection.find_one({"_id": heartbeat_id}) return [x for x in self.__collection.find({})] def delete(self, heartbeat_id): """ Delete Heartbeat by ID. :param `str` heartbeat_id: Heartbeat ID. :return: :raises: (`~.common.collection.CollectionError`, ). """ return self.__collection.remove({"_id": heartbeat_id})
class engine(Engine): etype = 'event_filter' def __init__(self, *args, **kargs): super(engine, self).__init__(*args, **kargs) self.mg_store = MongoStore.get_default() self.collection = MongoCollection( self.mg_store.get_collection("object")) self.name = kargs['name'] self.drop_event_count = 0 self.pass_event_count = 0 self.__load_rules() def pre_run(self): self.beat() def a_override(self, event, action): """Override a field from event or add a new one if it does not have one. """ afield = action.get('field', None) avalue = action.get('value', None) # This must be a hard check because value can be a boolean or a null # integer if afield is None or avalue is None: self.logger.error( "Malformed action ('field' and 'value' required): {}".format( action)) return False if afield not in event: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True # afield is in event if not isinstance(avalue, list): if isinstance(event[afield], list): self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) event[afield].append(avalue) else: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True else: # operation field is supported only for list values op = action.get('operation', 'append') if op == 'override': self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True elif op == 'append': self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) if isinstance(event[afield], list): event[afield] += avalue else: event[afield] = [event[afield]] + avalue return True else: self.logger.error( "Operation '{}' unsupported (action '{}')".format( op, action)) return False def a_remove(self, event, action): """Remove an event from a field in event or the whole field if no element is specified. """ akey = action.get('key', None) aelement = action.get('element', None) del_met = action.get('met', 0) if akey: if aelement: if del_met: for i, met in enumerate(event[akey]): if met['name'] == aelement: del event[akey][i] break elif isinstance(event[akey], dict): del event[akey][aelement] elif isinstance(event[akey], list): del event[akey][event[akey].index(aelement)] self.logger.debug(u" + {}: Removed: '{}' from '{}'".format( event['rk'], aelement, akey)) else: del event[akey] self.logger.debug(u" + {}: Removed: '{}'".format( event['rk'], akey)) return True else: self.logger.error( u"Action malformed (needs 'key' and/or 'element'): {}".format( action)) return False def a_modify(self, event, action, name): """ Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ derogated = False atype = action.get('type') actionMap = {'override': self.a_override, 'remove': self.a_remove} if atype in actionMap: derogated = actionMap[atype](event, action) else: self.logger.warning(u"Unknown action '{}'".format(atype)) # If the event was derogated, fill some informations if derogated: self.logger.debug(u"Event changed by rule '{}'".format(name)) return None def a_drop(self, event, action, name): """ Drop the event. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event dropped by rule '{}'".format(name)) self.drop_event_count += 1 return DROP def a_pass(self, event, action, name): """Pass the event to the next queue. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event passed by rule '{}'".format(name)) self.pass_event_count += 1 return event def a_route(self, event, action, name): """ Change the route to which an event will be sent Args: event: map of the event to be modified action: map of type action name: of the rule Returns: ``None`` """ if "route" in action: self.next_amqp_queues = [action["route"]] self.logger.debug(u"Event re-routed by rule '{}'".format(name)) else: self.logger.error( u"Action malformed (needs 'route'): {}".format(action)) return None def a_exec_job(self, event, action, name): records = self.collection.find({ 'crecord_type': 'job', '_id': action['job'] }) for record in records: job = copy.deepcopy(record) job['context'] = event try: self.work_amqp_publisher.direct_event(job, 'Engine_scheduler') except Exception as e: self.logger.exception("Unable to send job") time.sleep(1) return True def a_snooze(self, event, action, name): """ Snooze event checks :param dict event: event to be snoozed :param dict action: action :param str name: name of the rule :returns: True if a snooze has been sent, False otherwise :rtype: boolean """ if event.get('event_type') == 'snooze': return False # Only check events can trigger an auto-snooze if event.get('event_type') != 'check': return False # A check OK cannot trigger an auto-snooze if event.get('state') == 0: return False # Alerts manager caching if not hasattr(self, 'am'): self.am = Alerts(*Alerts.provide_default_basics()) # Context manager caching if not hasattr(self, 'cm'): self.cm = ContextGraph(self.logger) entity_id = self.cm.get_id(event) current_alarm = self.am.get_current_alarm(entity_id) if current_alarm is None: snooze = { 'connector': event.get('connector', ''), 'connector_name': event.get('connector_name', ''), 'source_type': event.get('source_type', ''), 'component': event.get('component', ''), 'event_type': 'snooze', 'duration': action['duration'], 'author': 'event_filter', 'output': 'Auto snooze generated by rule "{}"'.format(name), 'timestamp': int(time.time()) } if event.get('resource', ''): snooze['resource'] = event['resource'] try: self.work_amqp_publisher.direct_event(snooze, 'Engine_event_filter') except Exception as e: self.logger.exception("Unable to send snooze event") return True return False def a_baseline(self, event, actions, name): """a_baseline :param event: :param action: baseline conf in event filter :param name: """ event['baseline_name'] = actions['baseline_name'] event['check_frequency'] = actions['check_frequency'] try: self.work_amqp_publisher.direct_event(event, 'Engine_baseline') except Exception as e: self.logger.exception("Unable to send baseline event") def apply_actions(self, event, actions): pass_event = False actionMap = { 'drop': self.a_drop, 'pass': self.a_pass, 'override': self.a_modify, 'remove': self.a_modify, 'execjob': self.a_exec_job, 'route': self.a_route, 'snooze': self.a_snooze, 'baseline': self.a_baseline } for name, action in actions: if action['type'] in actionMap: ret = actionMap[action['type'].lower()](event, action, name) if ret: pass_event = True else: self.logger.warning(u"Unknown action '{}'".format(action)) return pass_event def work(self, event, *xargs, **kwargs): rk = get_routingkey(event) default_action = self.configuration.get('default_action', 'pass') # list of supported actions rules = self.configuration.get('rules', []) to_apply = [] self.logger.debug(u'event {}'.format(event)) # When list configuration then check black and # white lists depending on json configuration for filterItem in rules: actions = filterItem.get('actions') name = filterItem.get('name', 'no_name') self.logger.debug(u'rule {}'.format(filterItem)) self.logger.debug(u'filter is {}'.format(filterItem['mfilter'])) # Try filter rules on current event if filterItem['mfilter'] and check(filterItem['mfilter'], event): self.logger.debug(u'Event: {}, filter matches'.format( event.get('rk', event))) if 'pbehaviors' in filterItem: pbehaviors = filterItem.get('pbehaviors', {}) list_in = pbehaviors.get('in', []) list_out = pbehaviors.get('out', []) if list_in or list_out: pbm = singleton_per_scope(PBehaviorManager) cm = singleton_per_scope(ContextGraph) entity = cm.get_entity(event) entity_id = cm.get_entity_id(entity) result = pbm.check_pbehaviors(entity_id, list_in, list_out) if not result: break for action in actions: if action['type'].lower() == 'drop': self.apply_actions(event, to_apply) return self.a_drop(event, None, name) to_apply.append((name, action)) if filterItem.get('break', 0): self.logger.debug( u' + Filter {} broke the next filters processing'. format(filterItem.get('name', 'filter'))) break if len(to_apply): if self.apply_actions(event, to_apply): self.logger.debug(u'Event before sent to next engine: %s' % event) event['rk'] = event['_id'] = get_routingkey(event) return event # No rules matched if default_action == 'drop': self.logger.debug("Event '%s' dropped by default action" % (rk)) self.drop_event_count += 1 return DROP self.logger.debug("Event '%s' passed by default action" % (rk)) self.pass_event_count += 1 self.logger.debug(u'Event before sent to next engine: %s' % event) event['rk'] = event['_id'] = get_routingkey(event) return event def __load_rules(self): tmp_rules = [] records = self.collection.find({ 'crecord_type': 'filter', 'enable': True }) records.sort('priority', 1) for record in records: record_dump = copy.deepcopy(record) self.set_loaded(record_dump) try: record_dump["mfilter"] = loads(record_dump["mfilter"]) except Exception: self.logger.info(u'Invalid mfilter {}, filter {}'.format( record_dump['mfilter'], record_dump['name'], )) self.logger.debug(u'Loading record_dump:') self.logger.debug(record_dump) tmp_rules.append(record_dump) self.configuration = { 'rules': tmp_rules, 'default_action': self.find_default_action() } def beat(self, *args, **kargs): """ Configuration reload for realtime ui changes handling """ self.logger.debug(u'Reload configuration rules') self.__load_rules() self.logger.debug('Loaded {} rules'.format( len(self.configuration['rules']))) self.send_stat_event() def set_loaded(self, record): if 'run_once' in record and not record['run_once']: self.collection.update({"_id": record['_id']}, {"$set": { 'run_once': True }}) self.logger.info('record {} has been run once'.format( record['_id'])) def send_stat_event(self): """ Send AMQP Event for drop and pass metrics """ message_dropped = '{} event dropped since {}'.format( self.drop_event_count, self.beat_interval) message_passed = '{} event passed since {}'.format( self.pass_event_count, self.beat_interval) event = forger(connector='Engine', connector_name='engine', event_type='check', source_type='resource', resource=self.amqp_queue + '_data', state=0, state_type=1, output=message_dropped, perf_data_array=[{ 'metric': 'pass_event', 'value': self.pass_event_count, 'type': 'GAUGE' }, { 'metric': 'drop_event', 'value': self.drop_event_count, 'type': 'GAUGE' }]) self.logger.debug(message_dropped) self.logger.debug(message_passed) try: self.beat_amqp_publisher.canopsis_event(event) except Exception as e: self.logger.exception("Unable to send stat event") self.drop_event_count = 0 self.pass_event_count = 0 def find_default_action(self): """Find the default action stored and returns it, else assume it default action is pass. """ records = self.collection.find_one({'crecord_type': 'defaultrule'}) if records: return records[0]["action"] self.logger.debug( "No default action found. Assuming default action is pass") return 'pass'
class ViewAdapter(object): """ Adapter for the view collection. """ def __init__(self, logger): self.logger = logger self.view_collection = MongoCollection( MongoStore.get_default().get_collection(VIEWS_COLLECTION)) self.group_collection = MongoCollection( MongoStore.get_default().get_collection(GROUPS_COLLECTION)) def get_by_id(self, view_id): """ Get a view given its id. :param str view_id: the id of the view. """ return self.view_collection.find_one({ViewField.id: view_id}) def create(self, view): """ Create a new view and return its id. :param Dict[str, Any] view: :rtype: str """ view_id = str(uuid4()) view[ViewField.id] = view_id self.validate(view_id, view) self.view_collection.insert(view) return view_id def update(self, view_id, view): """ Update a view given its id. :param str view_id: the id of the view. :param Dict[str, Any] view: """ self.validate(view_id, view) self.view_collection.update({ViewField.id: view_id}, view, upsert=False) def remove_with_id(self, view_id): """ Remove a view given its id. :param str view_id: the id of the view. """ self.view_collection.remove({ViewField.id: view_id}) def list(self, name=None, title=None): """ Return a list of views, optionally filtered by name or title. :param str name: :param str title: :rtype: List[Dict[str, Any]] :raises: InvalidFilterError """ view_filter = {} if name is not None and title is not None: raise InvalidFilterError( 'Cannot filter both by name and by title.') if name is not None: view_filter[ViewField.name] = name if title is not None: view_filter[ViewField.title] = title views = self.view_collection.find(view_filter) groups = self.group_collection.find({}) response_groups = {} for group in groups: group_id = group[GroupField.id] del group[GroupField.id] group[GroupField.views] = [] response_groups[group_id] = group for view in views: try: group_id = view[ViewField.group_id] except KeyError: # This should never happen as long as the collections are only # modified using the API self.logger.exception( 'The view {0} is missing the group_id field.'.format( view[ViewField.id])) continue try: response_groups[group_id][GroupField.views].append(view) except KeyError: # This should never happen as long as the collections are only # modified using the API self.logger.exception('No group with id: {0}'.format(group_id)) return {ViewResponseField.groups: response_groups} def validate(self, view_id, view): """ Check that the view is valid, return InvalidViewError if it is not. :param Dict[str, Any] view: """ # Validate id field if view.get(ViewField.id, view_id) != view_id: raise InvalidViewError( 'The {0} field should not be modified'.format(ViewField.id)) # Validate group_id field if ViewField.group_id not in view: raise InvalidViewError('The view should have a group_id field.') group_id = view[ViewField.group_id] group = self.group_collection.find_one({GroupField.id: group_id}) if not group: raise InvalidViewError('No group with id: {0}'.format(group_id)) # Validate name field if ViewField.name not in view: raise InvalidViewError('The view should have a name field.') view_name = view[ViewField.name] same_name_view = self.view_collection.find_one({ ViewField.id: { '$ne': view_id }, ViewField.name: view_name }) if same_name_view: raise InvalidViewError( 'There is already a view with the name: {0}'.format(view_name)) # Validate title field if ViewField.title not in view: raise InvalidViewError('The view should have a title field.')
class GroupAdapter(object): """ Adapter for the group collection. """ def __init__(self, logger): self.logger = logger self.group_collection = MongoCollection( MongoStore.get_default().get_collection(GROUPS_COLLECTION)) self.view_collection = MongoCollection( MongoStore.get_default().get_collection(VIEWS_COLLECTION)) def get_by_id(self, group_id, name=None, title=None): """ Get a group given its id. :param str group_id: the id of the group. :param str name: :param str title: :rtype: Dict[str, Any] """ group = self.group_collection.find_one({GroupField.id: group_id}) if group: group[GroupField.views] = self.get_views(group_id, name, title) return group def get_views(self, group_id, name=None, title=None): """ Returns the list of views of a group, optionally filtered by name or title. :param str group_id: :param str name: :param str title: :rtype: List[Dict[str, Any]] :raises: InvalidFilterError """ view_filter = {ViewField.group_id: group_id} if name is not None and title is not None: raise InvalidFilterError( 'Cannot filter both by name and by title.') if name is not None: view_filter[ViewField.name] = name if title is not None: view_filter[ViewField.title] = title return list(self.view_collection.find(view_filter)) def is_empty(self, group_id): """ Return True if a group is empty. :param str group_id: :rtype: bool """ return self.view_collection.find({ ViewField.group_id: group_id }).limit(1).count() == 0 def create(self, group): """ Create a new group. :param Dict group: :rtype: str :raises: InvalidGroupError """ group_id = str(uuid4()) group[GroupField.id] = group_id self.validate(group_id, group) self.group_collection.insert(group) return group_id def update(self, group_id, group): """ Update a group given its id. :param str group_id: :param Dict group: :raises: InvalidGroupError """ self.validate(group_id, group) self.group_collection.update({GroupField.id: group_id}, group, upsert=False) def remove_with_id(self, group_id): """ Remove a group given its id. :param str group_id: :raises: NonEmptyGroupError """ if not self.is_empty(group_id): raise NonEmptyGroupError() self.group_collection.remove({GroupField.id: group_id}) def list(self, name=None): """ Return a list of groups, optionally filtered by name. :param str name: :rtype: List[Dict[str, Any]] """ group_filter = {} if name is not None: group_filter[GroupField.name] = name return list(self.group_collection.find(group_filter)) def validate(self, group_id, group): """ Check that the gorup is valid, return InvalidGroupError if it is not. :param Dict[str, Any] view: :raises: InvalidGroupError """ if group.get(GroupField.id, group_id) != group_id: raise InvalidGroupError( 'The {0} field should not be modified'.format(GroupField.id)) if GroupField.name not in group: raise InvalidGroupError('The group should have a name field.') group_name = group[GroupField.name] same_name_group = self.group_collection.find_one({ GroupField.id: { '$ne': group_id }, GroupField.name: group_name }) if same_name_group: raise InvalidGroupError( 'There is already a group with the name: {0}'.format( group_name)) def exists(self, group_id): """ Return True if a group exists. :param str group_id: :rtype: bool """ group = self.group_collection.find_one({GroupField.id: group_id}) return group is not None
class engine(Engine): etype = 'event_filter' def __init__(self, *args, **kargs): super(engine, self).__init__(*args, **kargs) self.mg_store = MongoStore.get_default() self.collection = MongoCollection(self.mg_store.get_collection("object")) self.name = kargs['name'] self.drop_event_count = 0 self.pass_event_count = 0 self.__load_rules() def pre_run(self): self.beat() def a_override(self, event, action): """Override a field from event or add a new one if it does not have one. """ afield = action.get('field', None) avalue = action.get('value', None) # This must be a hard check because value can be a boolean or a null # integer if afield is None or avalue is None: self.logger.error( "Malformed action ('field' and 'value' required): {}".format( action ) ) return False if afield not in event: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True # afield is in event if not isinstance(avalue, list): if isinstance(event[afield], list): self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) event[afield].append(avalue) else: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True else: # operation field is supported only for list values op = action.get('operation', 'append') if op == 'override': self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True elif op == 'append': self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) if isinstance(event[afield], list): event[afield] += avalue else: event[afield] = [event[afield]] + avalue return True else: self.logger.error( "Operation '{}' unsupported (action '{}')".format( op, action ) ) return False def a_remove(self, event, action): """Remove an event from a field in event or the whole field if no element is specified. """ akey = action.get('key', None) aelement = action.get('element', None) del_met = action.get('met', 0) if akey: if aelement: if del_met: for i, met in enumerate(event[akey]): if met['name'] == aelement: del event[akey][i] break elif isinstance(event[akey], dict): del event[akey][aelement] elif isinstance(event[akey], list): del event[akey][event[akey].index(aelement)] self.logger.debug(u" + {}: Removed: '{}' from '{}'".format( event['rk'], aelement, akey)) else: del event[akey] self.logger.debug(u" + {}: Removed: '{}'".format( event['rk'], akey)) return True else: self.logger.error( u"Action malformed (needs 'key' and/or 'element'): {}".format( action)) return False def a_modify(self, event, action, name): """ Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ derogated = False atype = action.get('type') actionMap = { 'override': self.a_override, 'remove': self.a_remove } if atype in actionMap: derogated = actionMap[atype](event, action) else: self.logger.warning(u"Unknown action '{}'".format(atype)) # If the event was derogated, fill some informations if derogated: self.logger.debug(u"Event changed by rule '{}'".format(name)) return None def a_drop(self, event, action, name): """ Drop the event. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event dropped by rule '{}'".format(name)) self.drop_event_count += 1 return DROP def a_pass(self, event, action, name): """Pass the event to the next queue. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event passed by rule '{}'".format(name)) self.pass_event_count += 1 return event def a_route(self, event, action, name): """ Change the route to which an event will be sent Args: event: map of the event to be modified action: map of type action name: of the rule Returns: ``None`` """ if "route" in action: self.next_amqp_queues = [action["route"]] self.logger.debug(u"Event re-routed by rule '{}'".format(name)) else: self.logger.error( u"Action malformed (needs 'route'): {}".format(action)) return None def a_exec_job(self, event, action, name): records = self.collection.find( {'crecord_type': 'job', '_id': action['job']} ) for record in records: job = copy.deepcopy(record) job['context'] = event try: self.work_amqp_publisher.direct_event(job, 'Engine_scheduler') except Exception as e: self.logger.exception("Unable to send job") time.sleep(1) return True def a_snooze(self, event, action, name): """ Snooze event checks :param dict event: event to be snoozed :param dict action: action :param str name: name of the rule :returns: True if a snooze has been sent, False otherwise :rtype: boolean """ if event.get('event_type') == 'snooze': return False # Only check events can trigger an auto-snooze if event.get('event_type') != 'check': return False # A check OK cannot trigger an auto-snooze if event.get('state') == 0: return False # Alerts manager caching if not hasattr(self, 'am'): self.am = Alerts(*Alerts.provide_default_basics()) # Context manager caching if not hasattr(self, 'cm'): self.cm = ContextGraph(self.logger) entity_id = self.cm.get_id(event) current_alarm = self.am.get_current_alarm(entity_id) if current_alarm is None: snooze = { 'connector': event.get('connector', ''), 'connector_name': event.get('connector_name', ''), 'source_type': event.get('source_type', ''), 'component': event.get('component', ''), 'event_type': 'snooze', 'duration': action['duration'], 'author': 'event_filter', 'output': 'Auto snooze generated by rule "{}"'.format(name), 'timestamp': int(time.time()) } if event.get('resource', ''): snooze['resource'] = event['resource'] try: self.work_amqp_publisher.direct_event( snooze, 'Engine_event_filter') except Exception as e: self.logger.exception("Unable to send snooze event") return True return False def a_baseline(self, event, actions, name): """a_baseline :param event: :param action: baseline conf in event filter :param name: """ event['baseline_name'] = actions['baseline_name'] event['check_frequency'] = actions['check_frequency'] try: self.work_amqp_publisher.direct_event( event, 'Engine_baseline') except Exception as e: self.logger.exception("Unable to send baseline event") def apply_actions(self, event, actions): pass_event = False actionMap = { 'drop': self.a_drop, 'pass': self.a_pass, 'override': self.a_modify, 'remove': self.a_modify, 'execjob': self.a_exec_job, 'route': self.a_route, 'snooze': self.a_snooze, 'baseline': self.a_baseline } for name, action in actions: if action['type'] in actionMap: ret = actionMap[action['type'].lower()](event, action, name) if ret: pass_event = True else: self.logger.warning(u"Unknown action '{}'".format(action)) return pass_event def work(self, event, *xargs, **kwargs): rk = get_routingkey(event) default_action = self.configuration.get('default_action', 'pass') # list of supported actions rules = self.configuration.get('rules', []) to_apply = [] self.logger.debug(u'event {}'.format(event)) # When list configuration then check black and # white lists depending on json configuration for filterItem in rules: actions = filterItem.get('actions') name = filterItem.get('name', 'no_name') self.logger.debug(u'rule {}'.format(filterItem)) self.logger.debug(u'filter is {}'.format(filterItem['mfilter'])) # Try filter rules on current event if filterItem['mfilter'] and check(filterItem['mfilter'], event): self.logger.debug( u'Event: {}, filter matches'.format(event.get('rk', event)) ) if 'pbehaviors' in filterItem: pbehaviors = filterItem.get('pbehaviors', {}) list_in = pbehaviors.get('in', []) list_out = pbehaviors.get('out', []) if list_in or list_out: pbm = singleton_per_scope(PBehaviorManager) cm = singleton_per_scope(ContextGraph) entity = cm.get_entity(event) entity_id = cm.get_entity_id(entity) result = pbm.check_pbehaviors( entity_id, list_in, list_out ) if not result: break for action in actions: if action['type'].lower() == 'drop': self.apply_actions(event, to_apply) return self.a_drop(event, None, name) to_apply.append((name, action)) if filterItem.get('break', 0): self.logger.debug( u' + Filter {} broke the next filters processing' .format( filterItem.get('name', 'filter') ) ) break if len(to_apply): if self.apply_actions(event, to_apply): self.logger.debug( u'Event before sent to next engine: %s' % event ) event['rk'] = event['_id'] = get_routingkey(event) return event # No rules matched if default_action == 'drop': self.logger.debug("Event '%s' dropped by default action" % (rk)) self.drop_event_count += 1 return DROP self.logger.debug("Event '%s' passed by default action" % (rk)) self.pass_event_count += 1 self.logger.debug(u'Event before sent to next engine: %s' % event) event['rk'] = event['_id'] = get_routingkey(event) return event def __load_rules(self): tmp_rules = [] records = self.collection.find( {'crecord_type': 'filter', 'enable': True}) records.sort('priority', 1) for record in records: record_dump = copy.deepcopy(record) self.set_loaded(record_dump) try: record_dump["mfilter"] = loads(record_dump["mfilter"]) except Exception: self.logger.info(u'Invalid mfilter {}, filter {}'.format( record_dump['mfilter'], record_dump['name'], )) self.logger.debug(u'Loading record_dump:') self.logger.debug(record_dump) tmp_rules.append(record_dump) self.configuration = { 'rules': tmp_rules, 'default_action': self.find_default_action() } def beat(self, *args, **kargs): """ Configuration reload for realtime ui changes handling """ self.logger.debug(u'Reload configuration rules') self.__load_rules() self.logger.debug( 'Loaded {} rules'.format(len(self.configuration['rules'])) ) self.send_stat_event() def set_loaded(self, record): if 'run_once' in record and not record['run_once']: self.collection.update({"_id": record['_id']}, {"$set": {'run_once': True}}) self.logger.info( 'record {} has been run once'.format(record['_id']) ) def send_stat_event(self): """ Send AMQP Event for drop and pass metrics """ message_dropped = '{} event dropped since {}'.format( self.drop_event_count, self.beat_interval ) message_passed = '{} event passed since {}'.format( self.pass_event_count, self.beat_interval ) event = forger( connector='Engine', connector_name='engine', event_type='check', source_type='resource', resource=self.amqp_queue + '_data', state=0, state_type=1, output=message_dropped, perf_data_array=[ {'metric': 'pass_event', 'value': self.pass_event_count, 'type': 'GAUGE'}, {'metric': 'drop_event', 'value': self.drop_event_count, 'type': 'GAUGE'} ] ) self.logger.debug(message_dropped) self.logger.debug(message_passed) try: self.beat_amqp_publisher.canopsis_event(event) except Exception as e: self.logger.exception("Unable to send stat event") self.drop_event_count = 0 self.pass_event_count = 0 def find_default_action(self): """Find the default action stored and returns it, else assume it default action is pass. """ records = self.collection.find_one({'crecord_type': 'defaultrule'}) if records: return records[0]["action"] self.logger.debug( "No default action found. Assuming default action is pass" ) return 'pass'