class CanopsisVersionManager(object): """ Canopsis version manager abstraction. """ COLLECTION = "configuration" VERSION_FIELD = "version" __DOCUMENT_ID = "canopsis_version" def __init__(self, collection): """ :param collection: `pymongo.collection.Collection` object. """ self.__collection = MongoCollection(collection) def find_canopsis_version_document(self): """ Find Canopsis version document. :returns: Canopsis version document or None if not found. :raises: (`pymongo.errors.PyMongoError`, ). """ return self.__collection.find_one({ '_id': self.__DOCUMENT_ID }) def put_canopsis_version_document(self, version): """ Put Canopsis version document (upsert). :param version: `str` Canopsis version. :raises: (`canopsis.common.collection.CollectionError`, ). """ self.__collection.update( { '_id': self.__DOCUMENT_ID }, { '_id': self.__DOCUMENT_ID, self.VERSION_FIELD: version }, upsert=True )
class CanopsisVersionManager(object): """ Canopsis version manager abstraction. """ COLLECTION = "configuration" EDITION_FIELD = "edition" STACK_FIELD = "stack" VERSION_FIELD = "version" __DOCUMENT_ID = "canopsis_version" def __init__(self, collection): """ :param collection: `pymongo.collection.Collection` object. """ self.__collection = MongoCollection(collection) def find_canopsis_document(self): """ Find Canopsis version document. :returns: Canopsis version document or None if not found. :raises: (`pymongo.errors.PyMongoError`, ). """ return self.__collection.find_one({'_id': self.__DOCUMENT_ID}) def put_canopsis_document(self, edition, stack, version): """ Put Canopsis version document (upsert). :param version: `str` Canopsis version. :raises: (`canopsis.common.collection.CollectionError`, ). """ document = {} if edition is not None: document[self.EDITION_FIELD] = edition if stack is not None: document[self.STACK_FIELD] = stack if version is not None: document[self.VERSION_FIELD] = version if len(document) > 0: resp = self.__collection.update({'_id': self.__DOCUMENT_ID}, {'$set': document}, upsert=True) return self.__collection.is_successfull(resp) return True
class PBehaviorManager(object): """ PBehavior manager class. """ PB_STORAGE_URI = 'mongodb-default-pbehavior://' LOG_PATH = 'var/log/pbehaviormanager.log' LOG_NAME = 'pbehaviormanager' CONF_PATH = 'etc/pbehavior/manager.conf' PBH_CAT = "PBEHAVIOR" _UPDATE_FLAG = 'updatedExisting' __TYPE_ERR = "id_ must be a list of string or a string" @classmethod def provide_default_basics(cls): """ Provide the default configuration and logger objects for PBehaviorManager. Do not use those defaults for tests. :return: config, logger, storage :rtype: Union[dict, logging.Logger, canopsis.storage.core.Storage] """ logger = Logger.get(cls.LOG_NAME, cls.LOG_PATH) pb_storage = Middleware.get_middleware_by_uri(cls.PB_STORAGE_URI) config = Configuration.load(PBehaviorManager.CONF_PATH, Ini) return config, logger, pb_storage def __init__(self, config, logger, pb_storage): """ :param dict config: configuration :param pb_storage: PBehavior Storage object """ super(PBehaviorManager, self).__init__() kwargs = {"logger": logger} self.context = singleton_per_scope(ContextGraph, kwargs=kwargs) self.logger = logger self.pb_storage = pb_storage self.config = config self.config_data = self.config.get(self.PBH_CAT, {}) self.default_tz = self.config_data.get("default_timezone", "Europe/Paris") # this line allow us to raise an exception pytz.UnknownTimeZoneError, # if the timezone defined in the pbehabior configuration file is wrong pytz.timezone(self.default_tz) self.pb_store = MongoCollection(MongoStore.get_default().get_collection('default_pbehavior')) self.currently_active_pb = set() def get(self, _id, query=None): """Get pbehavior by id. :param str id: pbehavior id :param dict query: filtering options """ return self.pb_storage.get_elements(ids=_id, query=query) def create( self, name, filter, author, tstart, tstop, rrule='', enabled=True, comments=None, connector='canopsis', connector_name='canopsis', type_=PBehavior.DEFAULT_TYPE, reason='', timezone=None, exdate=None): """ Method creates pbehavior record :param str name: filtering options :param dict filter: a mongo filter that match entities from canopsis context :param str author: the name of the user/app that has generated the pbehavior :param timestamp tstart: timestamp that correspond to the start of the pbehavior :param timestamp tstop: timestamp that correspond to the end of the pbehavior :param str rrule: reccurent rule that is compliant with rrule spec :param bool enabled: boolean to know if pbhevior is enabled or disabled :param list of dict comments: a list of comments made by users :param str connector: a string representing the type of connector that has generated the pbehavior :param str connector_name: a string representing the name of connector that has generated the pbehavior :param str type_: associated type_ for this pbh :param str reason: associated reason for this pbh :param str timezone: the timezone of the new pbehabior. If no timezone are given, use the default one. See the pbehavior documentation for more information. :param list of str| str exdate: a list of string representation of a date following this pattern "YYYY/MM/DD HH:MM:00 TIMEZONE". The hour use the 24 hours clock system and the timezone is the name of the timezone. The month, the day of the month, the hour, the minute and second are zero-padded. :raises ValueError: invalid RRULE :raises pytz.UnknownTimeZoneError: invalid timezone :return: created element eid :rtype: str """ if timezone is None: timezone = self.default_tz if exdate is None: exdate = [] # this line allow us to raise an exception pytz.UnknownTimeZoneError, # if the timezone defined in the pbehabior configuration file is wrong pytz.timezone(timezone) if enabled in [True, "True", "true"]: enabled = True elif enabled in [False, "False", "false"]: enabled = False else: raise ValueError("The enabled value does not match a boolean") if not isinstance(exdate, list): exdate = [exdate] check_valid_rrule(rrule) if comments is not None: for comment in comments: if "author" in comment: if not isinstance(comment["author"], string_types): raise ValueError("The author field must be an string") else: raise ValueError("The author field is missing") if "message" in comment: if not isinstance(comment["message"], string_types): raise ValueError("The message field must be an string") else: raise ValueError("The message field is missing") pb_kwargs = { PBehavior.NAME: name, PBehavior.FILTER: filter, PBehavior.AUTHOR: author, PBehavior.TSTART: tstart, PBehavior.TSTOP: tstop, PBehavior.RRULE: rrule, PBehavior.ENABLED: enabled, PBehavior.COMMENTS: comments, PBehavior.CONNECTOR: connector, PBehavior.CONNECTOR_NAME: connector_name, PBehavior.TYPE: type_, PBehavior.REASON: reason, PBehavior.TIMEZONE: timezone, PBehavior.EXDATE: exdate, PBehavior.EIDS: [] } data = PBehavior(**pb_kwargs) if not data.comments or not isinstance(data.comments, list): data.update(comments=[]) else: for comment in data.comments: comment.update({'_id': str(uuid4())}) result = self.pb_storage.put_element(element=data.to_dict()) return result def get_pbehaviors_by_eid(self, id_): """Retreive from database every pbehavior that contains the given id_ in the PBehavior.EIDS field. :param list,str: the id(s) as a str or a list of string :returns: a list of pbehavior, with the isActive key in pbehavior is active when queried. :rtype: list """ if not isinstance(id_, (list, string_types)): raise TypeError(self.__TYPE_ERR) if isinstance(id_, list): for element in id_: if not isinstance(element, string_types): raise TypeError(self.__TYPE_ERR) else: id_ = [id_] cursor = self.pb_storage.get_elements( query={PBehavior.EIDS: {"$in": id_}} ) pbehaviors = [] now = int(time()) for pb in cursor: if pb['tstart'] <= now and (pb['tstop'] is None or pb['tstop'] >= now): pb['isActive'] = True else: pb['isActive'] = False pbehaviors.append(pb) return pbehaviors def read(self, _id=None): """Get pbehavior or list pbehaviors. :param str _id: pbehavior id, _id may be equal to None """ result = self.get(_id) return result if _id else list(result) def update(self, _id, **kwargs): """ Update pbehavior record :param str _id: pbehavior id :param dict kwargs: values pbehavior fields. If a field is None, it will **not** be updated. :raises ValueError: invalid RRULE or no pbehavior with given _id """ pb_value = self.get(_id) if pb_value is None: raise ValueError("The id does not match any pebahvior") check_valid_rrule(kwargs.get('rrule', '')) pbehavior = PBehavior(**self.get(_id)) new_data = {k: v for k, v in kwargs.items() if v is not None} pbehavior.update(**new_data) result = self.pb_storage.put_element( element=new_data, _id=_id ) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return pbehavior.to_dict() return None def upsert(self, pbehavior): """ Creates or update the given pbehavior. This function uses MongoStore/MongoCollection instead of Storage. :param canopsis.models.pbehavior.PBehavior pbehavior: :rtype: bool, dict :returns: success, update result """ r = self.pb_store.update({'_id': pbehavior._id}, pbehavior.to_dict(), upsert=True) if r.get('updatedExisting', False) and r.get('nModified') == 1: return True, r elif r.get('updatedExisting', None) is False and r.get('nModified') == 0 and r.get('ok') == 1.0: return True, r else: return False, r def delete(self, _id=None, _filter=None): """ Delete pbehavior record :param str _id: pbehavior id """ result = self.pb_storage.remove_elements( ids=_id, _filter=_filter ) return self._check_response(result) def _update_pbehavior(self, pbehavior_id, query): result = self.pb_storage._update( spec={'_id': pbehavior_id}, document=query, multi=False, cache=False ) return result def create_pbehavior_comment(self, pbehavior_id, author, message): """ Сreate comment for pbehavior. :param str pbehavior_id: pbehavior id :param str author: author of the comment :param str message: text of the comment """ comment_id = str(uuid4()) comment = { Comment.ID: comment_id, Comment.AUTHOR: author, Comment.TS: timegm(datetime.utcnow().timetuple()), Comment.MESSAGE: message } query = {'$addToSet': {PBehavior.COMMENTS: comment}} result = self._update_pbehavior(pbehavior_id, query) if not result: result = self._update_pbehavior( pbehavior_id, {'$set': {PBehavior.COMMENTS: []}} ) if not result: return None result = self._update_pbehavior(pbehavior_id, query) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return comment_id return None def update_pbehavior_comment(self, pbehavior_id, _id, **kwargs): """ Update the comment record. :param str pbehavior_id: pbehavior id :param str_id: comment id :param dict kwargs: values comment fields """ pbehavior = self.get( pbehavior_id, query={PBehavior.COMMENTS: {'$elemMatch': {'_id': _id}}} ) if not pbehavior: return None _comments = pbehavior[PBehavior.COMMENTS] if not _comments: return None comment = Comment(**_comments[0]) comment.update(**kwargs) result = self.pb_storage._update( spec={'_id': pbehavior_id, 'comments._id': _id}, document={'$set': {'comments.$': comment.to_dict()}}, multi=False, cache=False ) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return comment.to_dict() return None def delete_pbehavior_comment(self, pbehavior_id, _id): """ Delete comment record. :param str pbehavior_id: pbehavior id :param str _id: comment id """ result = self.pb_storage._update( spec={'_id': pbehavior_id}, document={'$pull': {PBehavior.COMMENTS: {'_id': _id}}}, multi=False, cache=False ) return self._check_response(result) def get_pbehaviors(self, entity_id): """ Return all pbehaviors related to an entity_id, sorted by descending tstart. :param str entity_id: Id for which behaviors have to be returned :return: pbehaviors, with name, tstart, tstop, rrule and enabled keys :rtype: list of dict """ res = list( self.pb_storage._backend.find( {PBehavior.EIDS: {'$in': [entity_id]}}, sort=[(PBehavior.TSTART, DESCENDING)] ) ) return res def compute_pbehaviors_filters(self): """ Compute all filters and update eids attributes. """ pbehaviors = self.pb_storage.get_elements( query={PBehavior.FILTER: {'$exists': True}} ) for pbehavior in pbehaviors: query = loads(pbehavior[PBehavior.FILTER]) if not isinstance(query, dict): self.logger.error('compute_pbehaviors_filters(): filter is ' 'not a dict !\n{}'.format(query)) continue entities = self.context.ent_storage.get_elements( query=query ) pbehavior[PBehavior.EIDS] = [e['_id'] for e in entities] self.pb_storage.put_element(element=pbehavior) def _check_active_simple_pbehavior(self, timestamp, pbh): """ Check if a pbehavior without a rrule is active at the given time. :param int timestamp: the number a second this 1970/01/01 00:00:00 :param dict pbehavior: a pbehavior as a dict. :return bool: True if the boolean is active, false otherwise """ if pbh[PBehavior.TSTART] <= timestamp <= pbh[PBehavior.TSTOP]: return True return False @staticmethod def __convert_timestamp(timestamp, timezone): """Convert a pbehavior timestamp defined in the timezone to a datetime in the same timezone. :param timestamp:""" return datetime.fromtimestamp(timestamp, tz.gettz(timezone)) def _check_active_reccuring_pbehavior(self, timestamp, pbehavior): """ Check if a pbehavior with a rrule is active at the given time. :param int timestamp: the number a second this 1970/01/01 00:00:00 :param dict pbehavior: a pbehavior as a dict. :return bool: True if the boolean is active, false otherwise :raise ValueError: if the pbehavior.exdate is invalid. Or if the date of an occurence of the pbehavior is not a valid date. """ tz_name = pbehavior.get(PBehavior.TIMEZONE, self.default_tz) rec_set = rrule.rruleset() # convert the timestamp to a datetime in the pbehavior's timezone now = self.__convert_timestamp(timestamp, tz_name) start = self.__convert_timestamp(pbehavior[PBehavior.TSTART], tz_name) stop = self.__convert_timestamp(pbehavior[PBehavior.TSTOP], tz_name) if PBehavior.EXDATE in pbehavior and\ isinstance(pbehavior[PBehavior.EXDATE], list): for date in pbehavior[PBehavior.EXDATE]: exdate = self.__convert_timestamp(date, tz_name) rec_set.exdate(exdate) duration = stop - start # pbehavior duration rec_set.rrule(rrule.rrulestr(pbehavior[PBehavior.RRULE], dtstart=start)) rec_start = rec_set.before(now) self.logger.debug("Recurence start : {}".format(rec_start)) # No recurrence found if rec_start is None: return False self.logger.debug("Timestamp : {}".format(now)) self.logger.debug("Recurence stop : {}".format(rec_start + duration)) if rec_start <= now <= rec_start + duration: return True return False def check_active_pbehavior(self, timestamp, pbehavior): """ Check if a pbehavior is active at the given time. :param int timestamp: the number a second this 1970/01/01 00:00:00 :param dict pbehavior: a pbehavior as a dict. :return bool: True if the boolean is active, false otherwise :raise ValueError: if the pbehavior.exdate is invalid. Or if the date of an occurence of the pbehavior is not a valid date. """ if PBehavior.RRULE not in pbehavior or\ pbehavior[PBehavior.RRULE] is None or\ pbehavior[PBehavior.RRULE] == "": return self._check_active_simple_pbehavior(timestamp, pbehavior) else: if PBehavior.EXDATE not in pbehavior: pbehavior[PBehavior.EXDATE] = [] return self._check_active_reccuring_pbehavior(timestamp, pbehavior) def check_pbehaviors(self, entity_id, list_in, list_out): """ !!!! DEPRECATED !!!! :param str entity_id: :param list list_in: list of pbehavior names :param list list_out: list of pbehavior names :returns: bool if the entity_id is currently in list_in arg and out list_out arg """ return (self._check_pbehavior(entity_id, list_in) and not self._check_pbehavior(entity_id, list_out)) def _check_pbehavior(self, entity_id, pb_names): """ :param str entity_id: :param list pb_names: list of pbehavior names :returns: bool if the entity_id is currently in pb_names arg """ self.logger.critical("_check_pbehavior is DEPRECATED !!!!") try: entity = self.context.get_entities_by_id(entity_id)[0] except Exception: self.logger.error('Unable to check_behavior on {} entity_id' .format(entity_id)) return None event = self.context.get_event(entity) pbehaviors = self.pb_storage.get_elements( query={ PBehavior.NAME: {'$in': pb_names}, PBehavior.EIDS: {'$in': [entity_id]} } ) names = [] fromts = datetime.fromtimestamp for pbehavior in pbehaviors: tstart = pbehavior[PBehavior.TSTART] tstop = pbehavior[PBehavior.TSTOP] if not isinstance(tstart, (int, float)): self.logger.error('Cannot parse tstart value: {}' .format(pbehavior)) continue if not isinstance(tstop, (int, float)): self.logger.error('Cannot parse tstop value: {}' .format(pbehavior)) continue tstart = fromts(tstart) tstop = fromts(tstop) dt_list = [tstart, tstop] if pbehavior['rrule'] is not None: dt_list = list( rrule.rrulestr(pbehavior['rrule'], dtstart=tstart).between( tstart, tstop, inc=True ) ) if (len(dt_list) >= 2 and fromts(event['timestamp']) >= dt_list[0] and fromts(event['timestamp']) <= dt_list[-1]): names.append(pbehavior[PBehavior.NAME]) result = set(pb_names).isdisjoint(set(names)) return not result @staticmethod def _check_response(response): ack = True if 'ok' in response and response['ok'] == 1 else False return { 'acknowledged': ack, 'deletedCount': response['n'] } def get_active_pbehaviors(self, eids): """ Return a list of active pbehaviors linked to some entites. :param list eids: the desired entities id :returns: list of pbehaviors """ result = [] for eid in eids: pbhs = self.get_pbehaviors(eid) result = result + [x for x in pbhs if self._check_pbehavior( eid, [x['name']] )] return result def get_all_active_pbehaviors(self): """ Return all pbehaviors currently active using self.check_active_pbehavior """ now = int(time()) query = {} ret_val = list(self.pb_storage.get_elements(query=query)) results = [] for pb in ret_val: try: if self.check_active_pbehavior(now, pb): results.append(pb) except ValueError as exept: self.logger.exception("Can't check if the pbehavior is active.") return results def get_active_pbehaviors_from_type(self, types=None): """ Return pbehaviors currently active, with a specific type, using self.check_active_pbehavior """ if types is None: types = [] now = int(time()) query = {PBehavior.TYPE: {'$in': types}} ret_val = list(self.pb_storage.get_elements(query=query)) results = [] for pb in ret_val: if self.check_active_pbehavior(now, pb): results.append(pb) return results def get_varying_pbehavior_list(self): """ get_varying_pbehavior_list :returns: list of PBehavior id activated since last check :rtype: list """ active_pbehaviors = self.get_all_active_pbehaviors() active_pbehaviors_ids = set() for active_pb in active_pbehaviors: active_pbehaviors_ids.add(active_pb['_id']) varying_pbs = active_pbehaviors_ids.symmetric_difference(self.currently_active_pb) self.currently_active_pb = active_pbehaviors_ids return list(varying_pbs) def launch_update_watcher(self, watcher_manager): """ launch_update_watcher update watcher when a pbehavior is active :param object watcher_manager: watcher manager :returns: number of watcher updated retype: int """ new_pbs = self.get_varying_pbehavior_list() new_pbs_full = list(self.pb_storage._backend.find( {'_id': {'$in': new_pbs}} )) merged_eids = [] for pbehaviour in new_pbs_full: merged_eids = merged_eids + pbehaviour['eids'] watchers_ids = set() for watcher in self.get_wacher_on_entities(merged_eids): watchers_ids.add(watcher['_id']) for watcher_id in watchers_ids: watcher_manager.compute_state(watcher_id) return len(list(watchers_ids)) def get_wacher_on_entities(self, entities_ids): """ get_wacher_on_entities. :param entities_ids: entity id :returns: list of watchers :rtype: list """ query = { '$and': [ {'depends': {'$in': entities_ids}}, {'type': 'watcher'} ] } watchers = self.context.get_entities(query=query) return watchers @staticmethod def get_active_intervals(after, before, pbehavior): """ Return all the time intervals between after and before during which the pbehavior was active. The intervals are returned as a list of tuples (start, end), ordered chronologically. start and end are UTC timestamps, and are always between after and before. :param int after: a UTC timestamp :param int before: a UTC timestamp :param Dict[str, Any] pbehavior: :rtype: List[Tuple[int, int]] """ rrule_str = pbehavior[PBehavior.RRULE] tstart = pbehavior[PBehavior.TSTART] tstop = pbehavior[PBehavior.TSTOP] if not isinstance(tstart, (int, float)): return if not isinstance(tstop, (int, float)): return # Convert the timestamps to datetimes tz = pytz.UTC dttstart = datetime.utcfromtimestamp(tstart).replace(tzinfo=tz) dttstop = datetime.utcfromtimestamp(tstop).replace(tzinfo=tz) delta = dttstop - dttstart dtafter = datetime.utcfromtimestamp(after).replace(tzinfo=tz) dtbefore = datetime.utcfromtimestamp(before).replace(tzinfo=tz) if not rrule_str: # The only interval where the pbehavior is active is # [dttstart, dttstop]. Ensure that it is included in # [after, before], and convert the datetimes to timestamps. if dttstart < dtafter: dttstart = dtafter if dttstop > dtbefore: dttstop = dtbefore yield ( timegm(dttstart.timetuple()), timegm(dttstop.timetuple()) ) else: # Get all the intervals that intersect with the [after, before] # interval. interval_starts = rrule.rrulestr(rrule_str, dtstart=dttstart).between( dtafter - delta, dtbefore, inc=False) for interval_start in interval_starts: interval_end = interval_start + delta # Ensure that the interval is included in [after, before], and # datetimes to timestamps. if interval_start < dtafter: interval_start = dtafter if interval_end > dtbefore: interval_end = dtbefore yield ( timegm(interval_start.timetuple()), timegm(interval_end.timetuple()) ) def get_intervals_with_pbehaviors_by_eid(self, after, before, entity_id): """ Yields intervals between after and before with a boolean indicating if a pbehavior affects the entity during this interval. The intervals are returned as a list of tuples (start, end, pbehavior), ordered chronologically. start and end are UTC timestamps, and are always between after and before, pbehavior is a boolean indicating if a pbehavior affects the entity during this interval. None of the intervals overlap. :param int after: a UTC timestamp :param int before: a UTC timestamp :param str entity_id: the id of the entity :rtype: Iterator[Tuple[int, int, bool]] """ return self.get_intervals_with_pbehaviors( after, before, self.get_pbehaviors(entity_id)) def get_intervals_with_pbehaviors(self, after, before, pbehaviors): """ Yields intervals between after and before with a boolean indicating if one of the pbehaviors is active during this interval. The intervals are returned as a list of tuples (start, end, pbehavior), ordered chronologically. start and end are UTC timestamps, and are always between after and before, pbehavior is a boolean indicating if a pbehavior affects the entity during this interval. None of the intervals overlap. :param int after: a UTC timestamp :param int before: a UTC timestamp :param List[Dict[str, Any]] pbehaviors: a list of pbehabiors :rtype: Iterator[Tuple[int, int, bool]] """ intervals = [] # Get all the intervals where a pbehavior is active for pbehavior in pbehaviors: for interval in self.get_active_intervals(after, before, pbehavior): intervals.append(interval) if not intervals: yield (after, before, False) return # Order them chronologically (by start date) intervals.sort(key=lambda a: a[0]) # Yield the first interval without any active pbehavior merged_interval_start, merged_interval_end = intervals[0] yield ( after, merged_interval_start, False ) # At this point intervals is a list of intervals where a pbehavior is # active, ordered by start date. Some of those intervals may be # overlapping. This merges the overlapping intervals. for interval_start, interval_end in intervals[1:]: if interval_end < merged_interval_end: # The interval is included in the merged interval, skip it. continue if interval_start > merged_interval_end: # Since the interval starts after the end of the merged # interval, they cannot be merged. Yield the merged interval, # and move to the new one. yield ( merged_interval_start, merged_interval_end, True ) yield ( merged_interval_end, interval_start, False ) merged_interval_start = interval_start merged_interval_end = interval_end yield ( merged_interval_start, merged_interval_end, True ) yield ( merged_interval_end, before, False ) def get_enabled_pbehaviors(self): """ Yields all the enabled pbehaviors. :rtype: Iterator[Dict[str, Any]] """ return self.pb_storage._backend.find({ PBehavior.ENABLED: True })
class PBehaviorManager(object): """ PBehavior manager class. """ PB_STORAGE_URI = 'mongodb-default-pbehavior://' LOG_PATH = 'var/log/pbehaviormanager.log' LOG_NAME = 'pbehaviormanager' _UPDATE_FLAG = 'updatedExisting' __TYPE_ERR = "id_ must be a list of string or a string" @classmethod def provide_default_basics(cls): """ Provide the default configuration and logger objects for PBehaviorManager. Do not use those defaults for tests. :return: config, logger, storage :rtype: Union[dict, logging.Logger, canopsis.storage.core.Storage] """ logger = Logger.get(cls.LOG_NAME, cls.LOG_PATH) pb_storage = Middleware.get_middleware_by_uri(cls.PB_STORAGE_URI) return logger, pb_storage def __init__(self, logger, pb_storage): """ :param dict config: configuration :param pb_storage: PBehavior Storage object """ super(PBehaviorManager, self).__init__() kwargs = {"logger": logger} self.context = singleton_per_scope(ContextGraph, kwargs=kwargs) self.logger = logger self.pb_storage = pb_storage self.pb_store = MongoCollection( MongoStore.get_default().get_collection('default_pbehavior')) self.currently_active_pb = set() def get(self, _id, query=None): """Get pbehavior by id. :param str id: pbehavior id :param dict query: filtering options """ return self.pb_storage.get_elements(ids=_id, query=query) def create(self, name, filter, author, tstart, tstop, rrule='', enabled=True, comments=None, connector='canopsis', connector_name='canopsis', type_=PBehavior.DEFAULT_TYPE, reason=''): """ Method creates pbehavior record :param str name: filtering options :param dict filter: a mongo filter that match entities from canopsis context :param str author: the name of the user/app that has generated the pbehavior :param timestamp tstart: timestamp that correspond to the start of the pbehavior :param timestamp tstop: timestamp that correspond to the end of the pbehavior :param str rrule: reccurent rule that is compliant with rrule spec :param bool enabled: boolean to know if pbhevior is enabled or disabled :param list of dict comments: a list of comments made by users :param str connector: a string representing the type of connector that has generated the pbehavior :param str connector_name: a string representing the name of connector that has generated the pbehavior :param str type_: associated type_ for this pbh :param str reason: associated reason for this pbh :raises ValueError: invalid RRULE :return: created element eid :rtype: str """ if enabled in [True, "True", "true"]: enabled = True elif enabled in [False, "False", "false"]: enabled = False else: raise ValueError("The enabled value does not match a boolean") check_valid_rrule(rrule) if comments is not None: for comment in comments: if "author" in comment: if not isinstance(comment["author"], string_types): raise ValueError("The author field must be an string") else: raise ValueError("The author field is missing") if "message" in comment: if not isinstance(comment["message"], string_types): raise ValueError("The message field must be an string") else: raise ValueError("The message field is missing") pb_kwargs = { 'name': name, 'filter': filter, 'author': author, 'tstart': tstart, 'tstop': tstop, 'rrule': rrule, 'enabled': enabled, 'comments': comments, 'connector': connector, 'connector_name': connector_name, PBehavior.TYPE: type_, 'reason': reason } if PBehavior.EIDS not in pb_kwargs: pb_kwargs[PBehavior.EIDS] = [] data = PBehavior(**pb_kwargs) if not data.comments or not isinstance(data.comments, list): data.update(comments=[]) else: for comment in data.comments: comment.update({'_id': str(uuid4())}) result = self.pb_storage.put_element(element=data.to_dict()) return result def get_pbehaviors_by_eid(self, id_): """Retreive from database every pbehavior that contains the given id_ in the PBehavior.EIDS field. :param list,str: the id(s) as a str or a list of string :returns: a list of pbehavior, with the isActive key in pbehavior is active when queried. :rtype: list """ if not isinstance(id_, (list, string_types)): raise TypeError(self.__TYPE_ERR) if isinstance(id_, list): for element in id_: if not isinstance(element, string_types): raise TypeError(self.__TYPE_ERR) else: id_ = [id_] cursor = self.pb_storage.get_elements( query={PBehavior.EIDS: { "$in": id_ }}) pbehaviors = [] now = int(time()) for pb in cursor: if pb['tstart'] <= now and (pb['tstop'] is None or pb['tstop'] >= now): pb['isActive'] = True else: pb['isActive'] = False pbehaviors.append(pb) return pbehaviors def read(self, _id=None): """Get pbehavior or list pbehaviors. :param str _id: pbehavior id, _id may be equal to None """ result = self.get(_id) return result if _id else list(result) def update(self, _id, **kwargs): """ Update pbehavior record :param str _id: pbehavior id :param dict kwargs: values pbehavior fields. If a field is None, it will **not** be updated. :raises ValueError: invalid RRULE or no pbehavior with given _id """ pb_value = self.get(_id) if pb_value is None: raise ValueError("The id does not match any pebahvior") check_valid_rrule(kwargs.get('rrule', '')) pbehavior = PBehavior(**self.get(_id)) new_data = {k: v for k, v in kwargs.items() if v is not None} pbehavior.update(**new_data) result = self.pb_storage.put_element(element=new_data, _id=_id) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return pbehavior.to_dict() return None def upsert(self, pbehavior): """ Creates or update the given pbehavior. This function uses MongoStore/MongoCollection instead of Storage. :param canopsis.models.pbehavior.PBehavior pbehavior: :rtype: bool, dict :returns: success, update result """ r = self.pb_store.update({'_id': pbehavior._id}, pbehavior.to_dict(), upsert=True) if r.get('updatedExisting', False) and r.get('nModified') == 1: return True, r elif r.get('updatedExisting', None) is False and r.get( 'nModified') == 0 and r.get('ok') == 1.0: return True, r else: return False, r def delete(self, _id=None, _filter=None): """ Delete pbehavior record :param str _id: pbehavior id """ result = self.pb_storage.remove_elements(ids=_id, _filter=_filter) return self._check_response(result) def _update_pbehavior(self, pbehavior_id, query): result = self.pb_storage._update(spec={'_id': pbehavior_id}, document=query, multi=False, cache=False) return result def create_pbehavior_comment(self, pbehavior_id, author, message): """ Сreate comment for pbehavior. :param str pbehavior_id: pbehavior id :param str author: author of the comment :param str message: text of the comment """ comment_id = str(uuid4()) comment = { Comment.ID: comment_id, Comment.AUTHOR: author, Comment.TS: timegm(datetime.utcnow().timetuple()), Comment.MESSAGE: message } query = {'$addToSet': {PBehavior.COMMENTS: comment}} result = self._update_pbehavior(pbehavior_id, query) if not result: result = self._update_pbehavior(pbehavior_id, {'$set': { PBehavior.COMMENTS: [] }}) if not result: return None result = self._update_pbehavior(pbehavior_id, query) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return comment_id return None def update_pbehavior_comment(self, pbehavior_id, _id, **kwargs): """ Update the comment record. :param str pbehavior_id: pbehavior id :param str_id: comment id :param dict kwargs: values comment fields """ pbehavior = self.get( pbehavior_id, query={PBehavior.COMMENTS: { '$elemMatch': { '_id': _id } }}) if not pbehavior: return None _comments = pbehavior[PBehavior.COMMENTS] if not _comments: return None comment = Comment(**_comments[0]) comment.update(**kwargs) result = self.pb_storage._update( spec={ '_id': pbehavior_id, 'comments._id': _id }, document={'$set': { 'comments.$': comment.to_dict() }}, multi=False, cache=False) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return comment.to_dict() return None def delete_pbehavior_comment(self, pbehavior_id, _id): """ Delete comment record. :param str pbehavior_id: pbehavior id :param str _id: comment id """ result = self.pb_storage._update( spec={'_id': pbehavior_id}, document={'$pull': { PBehavior.COMMENTS: { '_id': _id } }}, multi=False, cache=False) return self._check_response(result) def get_pbehaviors(self, entity_id): """ Return all pbehaviors related to an entity_id, sorted by descending tstart. :param str entity_id: Id for which behaviors have to be returned :return: pbehaviors, with name, tstart, tstop, rrule and enabled keys :rtype: list of dict """ res = list( self.pb_storage._backend.find( {PBehavior.EIDS: { '$in': [entity_id] }}, sort=[(PBehavior.TSTART, DESCENDING)])) return res def compute_pbehaviors_filters(self): """ Compute all filters and update eids attributes. """ pbehaviors = self.pb_storage.get_elements( query={PBehavior.FILTER: { '$exists': True }}) for pbehavior in pbehaviors: query = loads(pbehavior[PBehavior.FILTER]) if not isinstance(query, dict): self.logger.error('compute_pbehaviors_filters(): filter is ' 'not a dict !\n{}'.format(query)) continue entities = self.context.ent_storage.get_elements(query=query) pbehavior[PBehavior.EIDS] = [e['_id'] for e in entities] self.pb_storage.put_element(element=pbehavior) @staticmethod def check_active_pbehavior(timestamp, pbehavior): """ For a given pbehavior (as dict) check that the given timestamp is active using either: the rrule, if any, from the pbehavior + tstart and tstop to define start and stop times. tstart and tstop alone if no rrule is available. All dates and times are computed using UTC timezone, so check that your timestamp was exported using UTC. :param int timestamp: timestamp to check :param dict pbehavior: the pbehavior :rtype bool: :returns: pbehavior is currently active or not """ fromts = datetime.utcfromtimestamp tstart = pbehavior[PBehavior.TSTART] tstop = pbehavior[PBehavior.TSTOP] if not isinstance(tstart, (int, float)): return False if not isinstance(tstop, (int, float)): return False tz = pytz.UTC dtts = fromts(timestamp).replace(tzinfo=tz) dttstart = fromts(tstart).replace(tzinfo=tz) dttstop = fromts(tstop).replace(tzinfo=tz) dt_list = [dttstart, dttstop] rrule = pbehavior['rrule'] if rrule: # compute the minimal date from which to start generating # dates from the rrule. # a complementary date (missing_date) is computed and added # at index 0 of the generated dt_list to ensure we manage # dates at boundaries. dt_tstart_date = dtts.date() dt_tstart_time = dttstart.time().replace(tzinfo=tz) dt_dtstart = datetime.combine(dt_tstart_date, dt_tstart_time) # dates in dt_list at 0 and 1 indexes can be equal, so we generate # three dates to ensure [1] - [2] will give a non-zero timedelta # object. dt_list = list( rrulestr(rrule, dtstart=dt_dtstart).xafter(dttstart, count=3, inc=True)) # compute the "missing dates": dates before the rrule started to # generate dates so we can check for a pbehavior in the past. multiply = 1 while True: missing_date = dt_list[0] - multiply * (dt_list[-1] - dt_list[-2]) dt_list.insert(0, missing_date) if missing_date < dtts: break multiply += 1 delta = dttstop - dttstart for dt in sorted(dt_list): if dtts >= dt and dtts <= dt + delta: return True return False else: if dtts >= dttstart and dtts <= dttstop: return True return False return False def check_pbehaviors(self, entity_id, list_in, list_out): """ :param str entity_id: :param list list_in: list of pbehavior names :param list list_out: list of pbehavior names :returns: bool if the entity_id is currently in list_in arg and out list_out arg """ return (self._check_pbehavior(entity_id, list_in) and not self._check_pbehavior(entity_id, list_out)) def _check_pbehavior(self, entity_id, pb_names): """ :param str entity_id: :param list pb_names: list of pbehavior names :returns: bool if the entity_id is currently in pb_names arg """ try: entity = self.context.get_entities_by_id(entity_id)[0] except Exception: self.logger.error( 'Unable to check_behavior on {} entity_id'.format(entity_id)) return None event = self.context.get_event(entity) pbehaviors = self.pb_storage.get_elements( query={ PBehavior.NAME: { '$in': pb_names }, PBehavior.EIDS: { '$in': [entity_id] } }) names = [] fromts = datetime.fromtimestamp for pbehavior in pbehaviors: tstart = pbehavior[PBehavior.TSTART] tstop = pbehavior[PBehavior.TSTOP] if not isinstance(tstart, (int, float)): self.logger.error( 'Cannot parse tstart value: {}'.format(pbehavior)) continue if not isinstance(tstop, (int, float)): self.logger.error( 'Cannot parse tstop value: {}'.format(pbehavior)) continue tstart = fromts(tstart) tstop = fromts(tstop) dt_list = [tstart, tstop] if pbehavior['rrule'] is not None: dt_list = list( rrulestr(pbehavior['rrule'], dtstart=tstart).between(tstart, tstop, inc=True)) if (len(dt_list) >= 2 and fromts(event['timestamp']) >= dt_list[0] and fromts(event['timestamp']) <= dt_list[-1]): names.append(pbehavior[PBehavior.NAME]) result = set(pb_names).isdisjoint(set(names)) return not result @staticmethod def _check_response(response): ack = True if 'ok' in response and response['ok'] == 1 else False return {'acknowledged': ack, 'deletedCount': response['n']} def get_active_pbehaviors(self, eids): """ Return a list of active pbehaviors linked to some entites. :param list eids: the desired entities id :returns: list of pbehaviors """ result = [] for eid in eids: pbhs = self.get_pbehaviors(eid) result = result + [ x for x in pbhs if self._check_pbehavior(eid, [x['name']]) ] return result def get_all_active_pbehaviors(self): """ Return all pbehaviors currently active using self.check_active_pbehavior """ now = int(time()) query = {} ret_val = list(self.pb_storage.get_elements(query=query)) results = [] for pb in ret_val: if self.check_active_pbehavior(now, pb): results.append(pb) return results def get_active_pbehaviors_from_type(self, types=None): """ Return pbehaviors currently active, with a specific type, using self.check_active_pbehavior """ if types is None: types = [] now = int(time()) query = {PBehavior.TYPE: {'$in': types}} ret_val = list(self.pb_storage.get_elements(query=query)) results = [] for pb in ret_val: if self.check_active_pbehavior(now, pb): results.append(pb) return results def get_varying_pbehavior_list(self): """ get_varying_pbehavior_list :returns: list of PBehavior id activated since last check :rtype: list """ active_pbehaviors = self.get_all_active_pbehaviors() active_pbehaviors_ids = set() for active_pb in active_pbehaviors: active_pbehaviors_ids.add(active_pb['_id']) varying_pbs = active_pbehaviors_ids.symmetric_difference( self.currently_active_pb) self.currently_active_pb = active_pbehaviors_ids return list(varying_pbs) def launch_update_watcher(self, watcher_manager): """ launch_update_watcher update watcher when a pbehavior is active :param object watcher_manager: watcher manager :returns: number of watcher updated retype: int """ new_pbs = self.get_varying_pbehavior_list() new_pbs_full = list( self.pb_storage._backend.find({'_id': { '$in': new_pbs }})) merged_eids = [] for pbehaviour in new_pbs_full: merged_eids = merged_eids + pbehaviour['eids'] watchers_ids = set() for watcher in self.get_wacher_on_entities(merged_eids): watchers_ids.add(watcher['_id']) for watcher_id in watchers_ids: watcher_manager.compute_state(watcher_id) return len(list(watchers_ids)) def get_wacher_on_entities(self, entities_ids): """ get_wacher_on_entities. :param entities_ids: entity id :returns: list of watchers :rtype: list """ query = { '$and': [{ 'depends': { '$in': entities_ids } }, { 'type': 'watcher' }] } watchers = self.context.get_entities(query=query) return watchers @staticmethod def get_active_intervals(after, before, pbehavior): """ Return all the time intervals between after and before during which the pbehavior was active. The intervals are returned as a list of tuples (start, end), ordered chronologically. start and end are UTC timestamps, and are always between after and before. :param int after: a UTC timestamp :param int before: a UTC timestamp :param Dict[str, Any] pbehavior: :rtype: List[Tuple[int, int]] """ rrule = pbehavior[PBehavior.RRULE] tstart = pbehavior[PBehavior.TSTART] tstop = pbehavior[PBehavior.TSTOP] if not isinstance(tstart, (int, float)): return if not isinstance(tstop, (int, float)): return # Convert the timestamps to datetimes tz = pytz.UTC dttstart = datetime.utcfromtimestamp(tstart).replace(tzinfo=tz) dttstop = datetime.utcfromtimestamp(tstop).replace(tzinfo=tz) delta = dttstop - dttstart dtafter = datetime.utcfromtimestamp(after).replace(tzinfo=tz) dtbefore = datetime.utcfromtimestamp(before).replace(tzinfo=tz) if not rrule: # The only interval where the pbehavior is active is # [dttstart, dttstop]. Ensure that it is included in # [after, before], and convert the datetimes to timestamps. if dttstart < dtafter: dttstart = dtafter if dttstop > dtbefore: dttstop = dtbefore yield (timegm(dttstart.timetuple()), timegm(dttstop.timetuple())) else: # Get all the intervals that intersect with the [after, before] # interval. interval_starts = rrulestr(rrule, dtstart=dttstart).between( dtafter - delta, dtbefore, inc=False) for interval_start in interval_starts: interval_end = interval_start + delta # Ensure that the interval is included in [after, before], and # datetimes to timestamps. if interval_start < dtafter: interval_start = dtafter if interval_end > dtbefore: interval_end = dtbefore yield (timegm(interval_start.timetuple()), timegm(interval_end.timetuple())) def get_intervals_with_pbehaviors(self, after, before, entity_id): """ Yields intervals between after and before with a boolean indicating if a pbehavior affects the entity during this interval. The intervals are returned as a list of tuples (start, end, pbehavior), ordered chronologically. start and end are UTC timestamps, and are always between after and before, pbehavior is a boolean indicating if a pbehavior affects the entity during this interval. None of the intervals overlap. :param int after: a UTC timestamp :param int before: a UTC timestamp :param str entity_id: the id of the entity :rtype: Iterator[Tuple[int, int, bool]] """ intervals = [] # Get all the intervals where a pbehavior is active pbehaviors = self.get_pbehaviors(entity_id) for pbehavior in pbehaviors: for interval in self.get_active_intervals(after, before, pbehavior): intervals.append(interval) if not intervals: yield (after, before, False) return # Order them chronologically (by start date) intervals.sort(key=lambda a: a[0]) # Yield the first interval without any active pbehavior merged_interval_start, merged_interval_end = intervals[0] yield (after, merged_interval_start, False) # At this point intervals is a list of intervals where a pbehavior is # active, ordered by start date. Some of those intervals may be # overlapping. This merges the overlapping intervals. for interval_start, interval_end in intervals[1:]: if interval_end < merged_interval_end: # The interval is included in the merged interval, skip it. continue if interval_start > merged_interval_end: # Since the interval starts after the end of the merged # interval, they cannot be merged. Yield the merged interval, # and move to the new one. yield (merged_interval_start, merged_interval_end, True) yield (merged_interval_end, interval_start, False) merged_interval_start = interval_start merged_interval_end = interval_end yield (merged_interval_start, merged_interval_end, True) yield (merged_interval_end, before, False)
class TestMongoCollection(unittest.TestCase): def setUp(self): output = StringIO() self.logger = Logger.get('test', output, OutputStream) self.storage = Middleware.get_middleware_by_uri( 'storage-default-testmongocollection://') self.collection = MongoCollection(collection=self.storage._backend, logger=self.logger) self.id_ = 'testid' def tearDown(self): """Teardown""" self.collection.remove() def test_insert(self): res = self.collection.insert(document={'_id': self.id_}) self.assertEqual(res, self.id_) res2 = self.collection.insert(document={'up': 'down'}) self.assertTrue(isinstance(res, string_types)) self.assertNotEqual(res, res2) def test_update(self): res = self.collection.update(query={'_id': self.id_}, document={'strange': 'charm'}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 0) res = self.collection.update(query={'_id': self.id_}, document={'yin': 'yang'}, upsert=True) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 1) res = self.collection.find_one(self.id_) self.assertEqual(res['yin'], 'yang') self.assertTrue('strange' not in res) def test_remove(self): res = self.collection.insert(document={ '_id': self.id_, 'top': 'bottom' }) self.assertIsNotNone(res) res = self.collection.remove(query={'_id': self.id_}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 1) # Deleting non-existing object doesn't throw error res = self.collection.remove(query={}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 0) def test_find(self): res = self.collection.insert(document={'_id': self.id_, 'yin': 'yang'}) self.assertIsNotNone(res) res = self.collection.find(query={'_id': self.id_}) self.assertTrue(isinstance(res, Cursor)) res = list(res) self.assertTrue(isinstance(res, list)) self.assertEqual(res[0]['yin'], 'yang') def test_find_one(self): res = self.collection.insert(document={'_id': self.id_, 'up': 'down'}) self.assertIsNotNone(res) res = self.collection.insert(document={'strange': 'charm'}) self.assertIsNotNone(res) res = self.collection.find_one(query={'_id': self.id_}) self.assertTrue(isinstance(res, dict)) self.assertEqual(res['up'], 'down') def test_is_successfull(self): dico = {'ok': 1.0, 'n': 2} self.assertTrue(MongoCollection.is_successfull(dico)) dico = {'ok': 666.667, 'n': 1} self.assertFalse(MongoCollection.is_successfull(dico)) dico = {'n': 2} self.assertFalse(MongoCollection.is_successfull(dico))
class RuleManager(object): """ Manager for event filter rules. """ def __init__(self, logger): self.logger = logger self.rule_collection = MongoCollection( MongoStore.get_default().get_collection(RULE_COLLECTION)) def get_by_id(self, rule_id): """ Get an event filter rule given its id. :param str rule_id: the id of the rule. :rtype: Dict[str, Any] """ return self.rule_collection.find_one({ RuleField.id: rule_id }) def create(self, rule): """ Create a new rule and return its id. :param Dict[str, Any] rule: :rtype: str :raises: InvalidRuleError if the rule is invalid. CollectionError if the creation fails. """ rule_id = str(uuid4()) rule[RuleField.id] = rule_id self.validate(rule_id, rule) self.rule_collection.insert(rule) return rule_id def remove_with_id(self, rule_id): """ Remove a rule given its id. :param str rule_id: the id of the rule. CollectionError if the creation fails. """ self.rule_collection.remove({ RuleField.id: rule_id }) def list(self): """ Return a list of all the rules. :rtype: List[Dict[str, Any]] """ return list(self.rule_collection.find({})) def update(self, rule_id, rule): """ Update a rule given its id. :param str rule_id: the id of the rule. :param Dict[str, Any] rule: :raises: InvalidRuleError if the rule is invalid. CollectionError if the creation fails. """ self.validate(rule_id, rule) self.rule_collection.update({ RuleField.id: rule_id }, rule, upsert=False) def validate(self, rule_id, rule): """ Check that the rule is valid. The pattern and external_data fields are not validated by this method. :param Dict[str, Any] view: :raises: InvalidRuleError if it is invalid. """ # Validate id field if rule.get(RuleField.id, rule_id) != rule_id: raise InvalidRuleError( 'The {0} field should not be modified.'.format(RuleField.id)) # Check that there are no unexpected fields in the rule unexpected_fields = set(rule.keys()).difference(RuleField.values) if unexpected_fields: raise InvalidRuleError( 'Unexpected fields: {0}.'.format(', '.join(unexpected_fields))) # Validate the type field if RuleField.type not in rule: raise InvalidRuleError( 'The {0} field is required.'.format(RuleField.type)) if rule.get(RuleField.type) not in RuleType.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.type, ', '.join(RuleType.values))) # Validate the priority field if not isinstance(rule.get(RuleField.priority, 0), int): raise InvalidRuleError( 'The {0} field should be an integer.'.format( RuleField.priority)) # Validate the enabled field if not isinstance(rule.get(RuleField.enabled, True), bool): raise InvalidRuleError( 'The {0} field should be a boolean.'.format( RuleField.enabled)) if rule.get(RuleField.type) != RuleType.enrichment: # Check that the enrichment fields are not defined for # non-enrichment rules. unexpected_fields = set(rule.keys()).intersection( ENRICHMENT_FIELDS) if unexpected_fields: raise InvalidRuleError( 'The following fields should only be defined for ' 'enrichment rules: {0}.'.format( ', '.join(unexpected_fields))) else: # Validate the actions field of the enrichment rules. if RuleField.actions not in rule: raise InvalidRuleError( 'The {0} field is required for enrichment rules.'.format( RuleField.actions)) if not isinstance(rule.get(RuleField.actions), list): raise InvalidRuleError( 'The {0} field should be a list.'.format( RuleField.actions)) if not rule.get(RuleField.actions): raise InvalidRuleError( 'The {0} field should contain at least one action.'.format( RuleField.actions)) # Validate the on_success field of the enrichment rules. outcome = rule.get(RuleField.on_success) if outcome and outcome not in RuleOutcome.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.on_success, ', '.join(RuleOutcome.values))) # Validate the on_failure field of the enrichment rules. outcome = rule.get(RuleField.on_failure) if outcome and outcome not in RuleOutcome.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.on_failure, ', '.join(RuleOutcome.values)))
class engine(Engine): etype = 'event_filter' def __init__(self, *args, **kargs): super(engine, self).__init__(*args, **kargs) self.mg_store = MongoStore.get_default() self.collection = MongoCollection( self.mg_store.get_collection("object")) self.name = kargs['name'] self.drop_event_count = 0 self.pass_event_count = 0 self.__load_rules() def pre_run(self): self.beat() def a_override(self, event, action): """Override a field from event or add a new one if it does not have one. """ afield = action.get('field', None) avalue = action.get('value', None) # This must be a hard check because value can be a boolean or a null # integer if afield is None or avalue is None: self.logger.error( "Malformed action ('field' and 'value' required): {}".format( action)) return False if afield not in event: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True # afield is in event if not isinstance(avalue, list): if isinstance(event[afield], list): self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) event[afield].append(avalue) else: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True else: # operation field is supported only for list values op = action.get('operation', 'append') if op == 'override': self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True elif op == 'append': self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) if isinstance(event[afield], list): event[afield] += avalue else: event[afield] = [event[afield]] + avalue return True else: self.logger.error( "Operation '{}' unsupported (action '{}')".format( op, action)) return False def a_remove(self, event, action): """Remove an event from a field in event or the whole field if no element is specified. """ akey = action.get('key', None) aelement = action.get('element', None) del_met = action.get('met', 0) if akey: if aelement: if del_met: for i, met in enumerate(event[akey]): if met['name'] == aelement: del event[akey][i] break elif isinstance(event[akey], dict): del event[akey][aelement] elif isinstance(event[akey], list): del event[akey][event[akey].index(aelement)] self.logger.debug(u" + {}: Removed: '{}' from '{}'".format( event['rk'], aelement, akey)) else: del event[akey] self.logger.debug(u" + {}: Removed: '{}'".format( event['rk'], akey)) return True else: self.logger.error( u"Action malformed (needs 'key' and/or 'element'): {}".format( action)) return False def a_modify(self, event, action, name): """ Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ derogated = False atype = action.get('type') actionMap = {'override': self.a_override, 'remove': self.a_remove} if atype in actionMap: derogated = actionMap[atype](event, action) else: self.logger.warning(u"Unknown action '{}'".format(atype)) # If the event was derogated, fill some informations if derogated: self.logger.debug(u"Event changed by rule '{}'".format(name)) return None def a_drop(self, event, action, name): """ Drop the event. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event dropped by rule '{}'".format(name)) self.drop_event_count += 1 return DROP def a_pass(self, event, action, name): """Pass the event to the next queue. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event passed by rule '{}'".format(name)) self.pass_event_count += 1 return event def a_route(self, event, action, name): """ Change the route to which an event will be sent Args: event: map of the event to be modified action: map of type action name: of the rule Returns: ``None`` """ if "route" in action: self.next_amqp_queues = [action["route"]] self.logger.debug(u"Event re-routed by rule '{}'".format(name)) else: self.logger.error( u"Action malformed (needs 'route'): {}".format(action)) return None def a_exec_job(self, event, action, name): records = self.collection.find({ 'crecord_type': 'job', '_id': action['job'] }) for record in records: job = copy.deepcopy(record) job['context'] = event try: self.work_amqp_publisher.direct_event(job, 'Engine_scheduler') except Exception as e: self.logger.exception("Unable to send job") time.sleep(1) return True def a_snooze(self, event, action, name): """ Snooze event checks :param dict event: event to be snoozed :param dict action: action :param str name: name of the rule :returns: True if a snooze has been sent, False otherwise :rtype: boolean """ if event.get('event_type') == 'snooze': return False # Only check events can trigger an auto-snooze if event.get('event_type') != 'check': return False # A check OK cannot trigger an auto-snooze if event.get('state') == 0: return False # Alerts manager caching if not hasattr(self, 'am'): self.am = Alerts(*Alerts.provide_default_basics()) # Context manager caching if not hasattr(self, 'cm'): self.cm = ContextGraph(self.logger) entity_id = self.cm.get_id(event) current_alarm = self.am.get_current_alarm(entity_id) if current_alarm is None: snooze = { 'connector': event.get('connector', ''), 'connector_name': event.get('connector_name', ''), 'source_type': event.get('source_type', ''), 'component': event.get('component', ''), 'event_type': 'snooze', 'duration': action['duration'], 'author': 'event_filter', 'output': 'Auto snooze generated by rule "{}"'.format(name), 'timestamp': int(time.time()) } if event.get('resource', ''): snooze['resource'] = event['resource'] try: self.work_amqp_publisher.direct_event(snooze, 'Engine_event_filter') except Exception as e: self.logger.exception("Unable to send snooze event") return True return False def a_baseline(self, event, actions, name): """a_baseline :param event: :param action: baseline conf in event filter :param name: """ event['baseline_name'] = actions['baseline_name'] event['check_frequency'] = actions['check_frequency'] try: self.work_amqp_publisher.direct_event(event, 'Engine_baseline') except Exception as e: self.logger.exception("Unable to send baseline event") def apply_actions(self, event, actions): pass_event = False actionMap = { 'drop': self.a_drop, 'pass': self.a_pass, 'override': self.a_modify, 'remove': self.a_modify, 'execjob': self.a_exec_job, 'route': self.a_route, 'snooze': self.a_snooze, 'baseline': self.a_baseline } for name, action in actions: if action['type'] in actionMap: ret = actionMap[action['type'].lower()](event, action, name) if ret: pass_event = True else: self.logger.warning(u"Unknown action '{}'".format(action)) return pass_event def work(self, event, *xargs, **kwargs): rk = get_routingkey(event) default_action = self.configuration.get('default_action', 'pass') # list of supported actions rules = self.configuration.get('rules', []) to_apply = [] self.logger.debug(u'event {}'.format(event)) # When list configuration then check black and # white lists depending on json configuration for filterItem in rules: actions = filterItem.get('actions') name = filterItem.get('name', 'no_name') self.logger.debug(u'rule {}'.format(filterItem)) self.logger.debug(u'filter is {}'.format(filterItem['mfilter'])) # Try filter rules on current event if filterItem['mfilter'] and check(filterItem['mfilter'], event): self.logger.debug(u'Event: {}, filter matches'.format( event.get('rk', event))) if 'pbehaviors' in filterItem: pbehaviors = filterItem.get('pbehaviors', {}) list_in = pbehaviors.get('in', []) list_out = pbehaviors.get('out', []) if list_in or list_out: pbm = singleton_per_scope(PBehaviorManager) cm = singleton_per_scope(ContextGraph) entity = cm.get_entity(event) entity_id = cm.get_entity_id(entity) result = pbm.check_pbehaviors(entity_id, list_in, list_out) if not result: break for action in actions: if action['type'].lower() == 'drop': self.apply_actions(event, to_apply) return self.a_drop(event, None, name) to_apply.append((name, action)) if filterItem.get('break', 0): self.logger.debug( u' + Filter {} broke the next filters processing'. format(filterItem.get('name', 'filter'))) break if len(to_apply): if self.apply_actions(event, to_apply): self.logger.debug(u'Event before sent to next engine: %s' % event) event['rk'] = event['_id'] = get_routingkey(event) return event # No rules matched if default_action == 'drop': self.logger.debug("Event '%s' dropped by default action" % (rk)) self.drop_event_count += 1 return DROP self.logger.debug("Event '%s' passed by default action" % (rk)) self.pass_event_count += 1 self.logger.debug(u'Event before sent to next engine: %s' % event) event['rk'] = event['_id'] = get_routingkey(event) return event def __load_rules(self): tmp_rules = [] records = self.collection.find({ 'crecord_type': 'filter', 'enable': True }) records.sort('priority', 1) for record in records: record_dump = copy.deepcopy(record) self.set_loaded(record_dump) try: record_dump["mfilter"] = loads(record_dump["mfilter"]) except Exception: self.logger.info(u'Invalid mfilter {}, filter {}'.format( record_dump['mfilter'], record_dump['name'], )) self.logger.debug(u'Loading record_dump:') self.logger.debug(record_dump) tmp_rules.append(record_dump) self.configuration = { 'rules': tmp_rules, 'default_action': self.find_default_action() } def beat(self, *args, **kargs): """ Configuration reload for realtime ui changes handling """ self.logger.debug(u'Reload configuration rules') self.__load_rules() self.logger.debug('Loaded {} rules'.format( len(self.configuration['rules']))) self.send_stat_event() def set_loaded(self, record): if 'run_once' in record and not record['run_once']: self.collection.update({"_id": record['_id']}, {"$set": { 'run_once': True }}) self.logger.info('record {} has been run once'.format( record['_id'])) def send_stat_event(self): """ Send AMQP Event for drop and pass metrics """ message_dropped = '{} event dropped since {}'.format( self.drop_event_count, self.beat_interval) message_passed = '{} event passed since {}'.format( self.pass_event_count, self.beat_interval) event = forger(connector='Engine', connector_name='engine', event_type='check', source_type='resource', resource=self.amqp_queue + '_data', state=0, state_type=1, output=message_dropped, perf_data_array=[{ 'metric': 'pass_event', 'value': self.pass_event_count, 'type': 'GAUGE' }, { 'metric': 'drop_event', 'value': self.drop_event_count, 'type': 'GAUGE' }]) self.logger.debug(message_dropped) self.logger.debug(message_passed) try: self.beat_amqp_publisher.canopsis_event(event) except Exception as e: self.logger.exception("Unable to send stat event") self.drop_event_count = 0 self.pass_event_count = 0 def find_default_action(self): """Find the default action stored and returns it, else assume it default action is pass. """ records = self.collection.find_one({'crecord_type': 'defaultrule'}) if records: return records[0]["action"] self.logger.debug( "No default action found. Assuming default action is pass") return 'pass'
class ViewAdapter(object): """ Adapter for the view collection. """ def __init__(self, logger): self.logger = logger self.view_collection = MongoCollection( MongoStore.get_default().get_collection(VIEWS_COLLECTION)) self.group_collection = MongoCollection( MongoStore.get_default().get_collection(GROUPS_COLLECTION)) def get_by_id(self, view_id): """ Get a view given its id. :param str view_id: the id of the view. """ return self.view_collection.find_one({ViewField.id: view_id}) def create(self, view): """ Create a new view and return its id. :param Dict[str, Any] view: :rtype: str """ view_id = str(uuid4()) view[ViewField.id] = view_id self.validate(view_id, view) self.view_collection.insert(view) return view_id def update(self, view_id, view): """ Update a view given its id. :param str view_id: the id of the view. :param Dict[str, Any] view: """ self.validate(view_id, view) self.view_collection.update({ViewField.id: view_id}, view, upsert=False) def remove_with_id(self, view_id): """ Remove a view given its id. :param str view_id: the id of the view. """ self.view_collection.remove({ViewField.id: view_id}) def list(self, name=None, title=None): """ Return a list of views, optionally filtered by name or title. :param str name: :param str title: :rtype: List[Dict[str, Any]] :raises: InvalidFilterError """ view_filter = {} if name is not None and title is not None: raise InvalidFilterError( 'Cannot filter both by name and by title.') if name is not None: view_filter[ViewField.name] = name if title is not None: view_filter[ViewField.title] = title views = self.view_collection.find(view_filter) groups = self.group_collection.find({}) response_groups = {} for group in groups: group_id = group[GroupField.id] del group[GroupField.id] group[GroupField.views] = [] response_groups[group_id] = group for view in views: try: group_id = view[ViewField.group_id] except KeyError: # This should never happen as long as the collections are only # modified using the API self.logger.exception( 'The view {0} is missing the group_id field.'.format( view[ViewField.id])) continue try: response_groups[group_id][GroupField.views].append(view) except KeyError: # This should never happen as long as the collections are only # modified using the API self.logger.exception('No group with id: {0}'.format(group_id)) return {ViewResponseField.groups: response_groups} def validate(self, view_id, view): """ Check that the view is valid, return InvalidViewError if it is not. :param Dict[str, Any] view: """ # Validate id field if view.get(ViewField.id, view_id) != view_id: raise InvalidViewError( 'The {0} field should not be modified'.format(ViewField.id)) # Validate group_id field if ViewField.group_id not in view: raise InvalidViewError('The view should have a group_id field.') group_id = view[ViewField.group_id] group = self.group_collection.find_one({GroupField.id: group_id}) if not group: raise InvalidViewError('No group with id: {0}'.format(group_id)) # Validate name field if ViewField.name not in view: raise InvalidViewError('The view should have a name field.') view_name = view[ViewField.name] same_name_view = self.view_collection.find_one({ ViewField.id: { '$ne': view_id }, ViewField.name: view_name }) if same_name_view: raise InvalidViewError( 'There is already a view with the name: {0}'.format(view_name)) # Validate title field if ViewField.title not in view: raise InvalidViewError('The view should have a title field.')
class GroupAdapter(object): """ Adapter for the group collection. """ def __init__(self, logger): self.logger = logger self.group_collection = MongoCollection( MongoStore.get_default().get_collection(GROUPS_COLLECTION)) self.view_collection = MongoCollection( MongoStore.get_default().get_collection(VIEWS_COLLECTION)) def get_by_id(self, group_id, name=None, title=None): """ Get a group given its id. :param str group_id: the id of the group. :param str name: :param str title: :rtype: Dict[str, Any] """ group = self.group_collection.find_one({GroupField.id: group_id}) if group: group[GroupField.views] = self.get_views(group_id, name, title) return group def get_views(self, group_id, name=None, title=None): """ Returns the list of views of a group, optionally filtered by name or title. :param str group_id: :param str name: :param str title: :rtype: List[Dict[str, Any]] :raises: InvalidFilterError """ view_filter = {ViewField.group_id: group_id} if name is not None and title is not None: raise InvalidFilterError( 'Cannot filter both by name and by title.') if name is not None: view_filter[ViewField.name] = name if title is not None: view_filter[ViewField.title] = title return list(self.view_collection.find(view_filter)) def is_empty(self, group_id): """ Return True if a group is empty. :param str group_id: :rtype: bool """ return self.view_collection.find({ ViewField.group_id: group_id }).limit(1).count() == 0 def create(self, group): """ Create a new group. :param Dict group: :rtype: str :raises: InvalidGroupError """ group_id = str(uuid4()) group[GroupField.id] = group_id self.validate(group_id, group) self.group_collection.insert(group) return group_id def update(self, group_id, group): """ Update a group given its id. :param str group_id: :param Dict group: :raises: InvalidGroupError """ self.validate(group_id, group) self.group_collection.update({GroupField.id: group_id}, group, upsert=False) def remove_with_id(self, group_id): """ Remove a group given its id. :param str group_id: :raises: NonEmptyGroupError """ if not self.is_empty(group_id): raise NonEmptyGroupError() self.group_collection.remove({GroupField.id: group_id}) def list(self, name=None): """ Return a list of groups, optionally filtered by name. :param str name: :rtype: List[Dict[str, Any]] """ group_filter = {} if name is not None: group_filter[GroupField.name] = name return list(self.group_collection.find(group_filter)) def validate(self, group_id, group): """ Check that the gorup is valid, return InvalidGroupError if it is not. :param Dict[str, Any] view: :raises: InvalidGroupError """ if group.get(GroupField.id, group_id) != group_id: raise InvalidGroupError( 'The {0} field should not be modified'.format(GroupField.id)) if GroupField.name not in group: raise InvalidGroupError('The group should have a name field.') group_name = group[GroupField.name] same_name_group = self.group_collection.find_one({ GroupField.id: { '$ne': group_id }, GroupField.name: group_name }) if same_name_group: raise InvalidGroupError( 'There is already a group with the name: {0}'.format( group_name)) def exists(self, group_id): """ Return True if a group exists. :param str group_id: :rtype: bool """ group = self.group_collection.find_one({GroupField.id: group_id}) return group is not None
class engine(Engine): etype = 'event_filter' def __init__(self, *args, **kargs): super(engine, self).__init__(*args, **kargs) self.mg_store = MongoStore.get_default() self.collection = MongoCollection(self.mg_store.get_collection("object")) self.name = kargs['name'] self.drop_event_count = 0 self.pass_event_count = 0 self.__load_rules() def pre_run(self): self.beat() def a_override(self, event, action): """Override a field from event or add a new one if it does not have one. """ afield = action.get('field', None) avalue = action.get('value', None) # This must be a hard check because value can be a boolean or a null # integer if afield is None or avalue is None: self.logger.error( "Malformed action ('field' and 'value' required): {}".format( action ) ) return False if afield not in event: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True # afield is in event if not isinstance(avalue, list): if isinstance(event[afield], list): self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) event[afield].append(avalue) else: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True else: # operation field is supported only for list values op = action.get('operation', 'append') if op == 'override': self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True elif op == 'append': self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) if isinstance(event[afield], list): event[afield] += avalue else: event[afield] = [event[afield]] + avalue return True else: self.logger.error( "Operation '{}' unsupported (action '{}')".format( op, action ) ) return False def a_remove(self, event, action): """Remove an event from a field in event or the whole field if no element is specified. """ akey = action.get('key', None) aelement = action.get('element', None) del_met = action.get('met', 0) if akey: if aelement: if del_met: for i, met in enumerate(event[akey]): if met['name'] == aelement: del event[akey][i] break elif isinstance(event[akey], dict): del event[akey][aelement] elif isinstance(event[akey], list): del event[akey][event[akey].index(aelement)] self.logger.debug(u" + {}: Removed: '{}' from '{}'".format( event['rk'], aelement, akey)) else: del event[akey] self.logger.debug(u" + {}: Removed: '{}'".format( event['rk'], akey)) return True else: self.logger.error( u"Action malformed (needs 'key' and/or 'element'): {}".format( action)) return False def a_modify(self, event, action, name): """ Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ derogated = False atype = action.get('type') actionMap = { 'override': self.a_override, 'remove': self.a_remove } if atype in actionMap: derogated = actionMap[atype](event, action) else: self.logger.warning(u"Unknown action '{}'".format(atype)) # If the event was derogated, fill some informations if derogated: self.logger.debug(u"Event changed by rule '{}'".format(name)) return None def a_drop(self, event, action, name): """ Drop the event. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event dropped by rule '{}'".format(name)) self.drop_event_count += 1 return DROP def a_pass(self, event, action, name): """Pass the event to the next queue. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event passed by rule '{}'".format(name)) self.pass_event_count += 1 return event def a_route(self, event, action, name): """ Change the route to which an event will be sent Args: event: map of the event to be modified action: map of type action name: of the rule Returns: ``None`` """ if "route" in action: self.next_amqp_queues = [action["route"]] self.logger.debug(u"Event re-routed by rule '{}'".format(name)) else: self.logger.error( u"Action malformed (needs 'route'): {}".format(action)) return None def a_exec_job(self, event, action, name): records = self.collection.find( {'crecord_type': 'job', '_id': action['job']} ) for record in records: job = copy.deepcopy(record) job['context'] = event try: self.work_amqp_publisher.direct_event(job, 'Engine_scheduler') except Exception as e: self.logger.exception("Unable to send job") time.sleep(1) return True def a_snooze(self, event, action, name): """ Snooze event checks :param dict event: event to be snoozed :param dict action: action :param str name: name of the rule :returns: True if a snooze has been sent, False otherwise :rtype: boolean """ if event.get('event_type') == 'snooze': return False # Only check events can trigger an auto-snooze if event.get('event_type') != 'check': return False # A check OK cannot trigger an auto-snooze if event.get('state') == 0: return False # Alerts manager caching if not hasattr(self, 'am'): self.am = Alerts(*Alerts.provide_default_basics()) # Context manager caching if not hasattr(self, 'cm'): self.cm = ContextGraph(self.logger) entity_id = self.cm.get_id(event) current_alarm = self.am.get_current_alarm(entity_id) if current_alarm is None: snooze = { 'connector': event.get('connector', ''), 'connector_name': event.get('connector_name', ''), 'source_type': event.get('source_type', ''), 'component': event.get('component', ''), 'event_type': 'snooze', 'duration': action['duration'], 'author': 'event_filter', 'output': 'Auto snooze generated by rule "{}"'.format(name), 'timestamp': int(time.time()) } if event.get('resource', ''): snooze['resource'] = event['resource'] try: self.work_amqp_publisher.direct_event( snooze, 'Engine_event_filter') except Exception as e: self.logger.exception("Unable to send snooze event") return True return False def a_baseline(self, event, actions, name): """a_baseline :param event: :param action: baseline conf in event filter :param name: """ event['baseline_name'] = actions['baseline_name'] event['check_frequency'] = actions['check_frequency'] try: self.work_amqp_publisher.direct_event( event, 'Engine_baseline') except Exception as e: self.logger.exception("Unable to send baseline event") def apply_actions(self, event, actions): pass_event = False actionMap = { 'drop': self.a_drop, 'pass': self.a_pass, 'override': self.a_modify, 'remove': self.a_modify, 'execjob': self.a_exec_job, 'route': self.a_route, 'snooze': self.a_snooze, 'baseline': self.a_baseline } for name, action in actions: if action['type'] in actionMap: ret = actionMap[action['type'].lower()](event, action, name) if ret: pass_event = True else: self.logger.warning(u"Unknown action '{}'".format(action)) return pass_event def work(self, event, *xargs, **kwargs): rk = get_routingkey(event) default_action = self.configuration.get('default_action', 'pass') # list of supported actions rules = self.configuration.get('rules', []) to_apply = [] self.logger.debug(u'event {}'.format(event)) # When list configuration then check black and # white lists depending on json configuration for filterItem in rules: actions = filterItem.get('actions') name = filterItem.get('name', 'no_name') self.logger.debug(u'rule {}'.format(filterItem)) self.logger.debug(u'filter is {}'.format(filterItem['mfilter'])) # Try filter rules on current event if filterItem['mfilter'] and check(filterItem['mfilter'], event): self.logger.debug( u'Event: {}, filter matches'.format(event.get('rk', event)) ) if 'pbehaviors' in filterItem: pbehaviors = filterItem.get('pbehaviors', {}) list_in = pbehaviors.get('in', []) list_out = pbehaviors.get('out', []) if list_in or list_out: pbm = singleton_per_scope(PBehaviorManager) cm = singleton_per_scope(ContextGraph) entity = cm.get_entity(event) entity_id = cm.get_entity_id(entity) result = pbm.check_pbehaviors( entity_id, list_in, list_out ) if not result: break for action in actions: if action['type'].lower() == 'drop': self.apply_actions(event, to_apply) return self.a_drop(event, None, name) to_apply.append((name, action)) if filterItem.get('break', 0): self.logger.debug( u' + Filter {} broke the next filters processing' .format( filterItem.get('name', 'filter') ) ) break if len(to_apply): if self.apply_actions(event, to_apply): self.logger.debug( u'Event before sent to next engine: %s' % event ) event['rk'] = event['_id'] = get_routingkey(event) return event # No rules matched if default_action == 'drop': self.logger.debug("Event '%s' dropped by default action" % (rk)) self.drop_event_count += 1 return DROP self.logger.debug("Event '%s' passed by default action" % (rk)) self.pass_event_count += 1 self.logger.debug(u'Event before sent to next engine: %s' % event) event['rk'] = event['_id'] = get_routingkey(event) return event def __load_rules(self): tmp_rules = [] records = self.collection.find( {'crecord_type': 'filter', 'enable': True}) records.sort('priority', 1) for record in records: record_dump = copy.deepcopy(record) self.set_loaded(record_dump) try: record_dump["mfilter"] = loads(record_dump["mfilter"]) except Exception: self.logger.info(u'Invalid mfilter {}, filter {}'.format( record_dump['mfilter'], record_dump['name'], )) self.logger.debug(u'Loading record_dump:') self.logger.debug(record_dump) tmp_rules.append(record_dump) self.configuration = { 'rules': tmp_rules, 'default_action': self.find_default_action() } def beat(self, *args, **kargs): """ Configuration reload for realtime ui changes handling """ self.logger.debug(u'Reload configuration rules') self.__load_rules() self.logger.debug( 'Loaded {} rules'.format(len(self.configuration['rules'])) ) self.send_stat_event() def set_loaded(self, record): if 'run_once' in record and not record['run_once']: self.collection.update({"_id": record['_id']}, {"$set": {'run_once': True}}) self.logger.info( 'record {} has been run once'.format(record['_id']) ) def send_stat_event(self): """ Send AMQP Event for drop and pass metrics """ message_dropped = '{} event dropped since {}'.format( self.drop_event_count, self.beat_interval ) message_passed = '{} event passed since {}'.format( self.pass_event_count, self.beat_interval ) event = forger( connector='Engine', connector_name='engine', event_type='check', source_type='resource', resource=self.amqp_queue + '_data', state=0, state_type=1, output=message_dropped, perf_data_array=[ {'metric': 'pass_event', 'value': self.pass_event_count, 'type': 'GAUGE'}, {'metric': 'drop_event', 'value': self.drop_event_count, 'type': 'GAUGE'} ] ) self.logger.debug(message_dropped) self.logger.debug(message_passed) try: self.beat_amqp_publisher.canopsis_event(event) except Exception as e: self.logger.exception("Unable to send stat event") self.drop_event_count = 0 self.pass_event_count = 0 def find_default_action(self): """Find the default action stored and returns it, else assume it default action is pass. """ records = self.collection.find_one({'crecord_type': 'defaultrule'}) if records: return records[0]["action"] self.logger.debug( "No default action found. Assuming default action is pass" ) return 'pass'