class CanopsisVersionManager(object):
"""
Canopsis version manager abstraction.
"""
COLLECTION = "configuration"
VERSION_FIELD = "version"
__DOCUMENT_ID = "canopsis_version"
def __init__(self, collection):
"""
:param collection: `pymongo.collection.Collection` object.
"""
self.__collection = MongoCollection(collection)
def find_canopsis_version_document(self):
"""
Find Canopsis version document.
:returns: Canopsis version document or None if not found.
:raises: (`pymongo.errors.PyMongoError`, ).
"""
return self.__collection.find_one({
'_id': self.__DOCUMENT_ID
})
def put_canopsis_version_document(self, version):
"""
Put Canopsis version document (upsert).
:param version: `str` Canopsis version.
:raises: (`canopsis.common.collection.CollectionError`, ).
"""
self.__collection.update(
{
'_id': self.__DOCUMENT_ID
},
{
'_id': self.__DOCUMENT_ID,
self.VERSION_FIELD: version
},
upsert=True
)
class CanopsisVersionManager(object):
"""
Canopsis version manager abstraction.
"""
COLLECTION = "configuration"
EDITION_FIELD = "edition"
STACK_FIELD = "stack"
VERSION_FIELD = "version"
__DOCUMENT_ID = "canopsis_version"
def __init__(self, collection):
"""
:param collection: `pymongo.collection.Collection` object.
"""
self.__collection = MongoCollection(collection)
def find_canopsis_document(self):
"""
Find Canopsis version document.
:returns: Canopsis version document or None if not found.
:raises: (`pymongo.errors.PyMongoError`, ).
"""
return self.__collection.find_one({'_id': self.__DOCUMENT_ID})
def put_canopsis_document(self, edition, stack, version):
"""
Put Canopsis version document (upsert).
:param version: `str` Canopsis version.
:raises: (`canopsis.common.collection.CollectionError`, ).
"""
document = {}
if edition is not None:
document[self.EDITION_FIELD] = edition
if stack is not None:
document[self.STACK_FIELD] = stack
if version is not None:
document[self.VERSION_FIELD] = version
if len(document) > 0:
resp = self.__collection.update({'_id': self.__DOCUMENT_ID},
{'$set': document},
upsert=True)
return self.__collection.is_successfull(resp)
return True
class PBehaviorManager(object):
"""
PBehavior manager class.
"""
PB_STORAGE_URI = 'mongodb-default-pbehavior://'
LOG_PATH = 'var/log/pbehaviormanager.log'
LOG_NAME = 'pbehaviormanager'
CONF_PATH = 'etc/pbehavior/manager.conf'
PBH_CAT = "PBEHAVIOR"
_UPDATE_FLAG = 'updatedExisting'
__TYPE_ERR = "id_ must be a list of string or a string"
@classmethod
def provide_default_basics(cls):
"""
Provide the default configuration and logger objects
for PBehaviorManager.
Do not use those defaults for tests.
:return: config, logger, storage
:rtype: Union[dict, logging.Logger, canopsis.storage.core.Storage]
"""
logger = Logger.get(cls.LOG_NAME, cls.LOG_PATH)
pb_storage = Middleware.get_middleware_by_uri(cls.PB_STORAGE_URI)
config = Configuration.load(PBehaviorManager.CONF_PATH, Ini)
return config, logger, pb_storage
def __init__(self, config, logger, pb_storage):
"""
:param dict config: configuration
:param pb_storage: PBehavior Storage object
"""
super(PBehaviorManager, self).__init__()
kwargs = {"logger": logger}
self.context = singleton_per_scope(ContextGraph, kwargs=kwargs)
self.logger = logger
self.pb_storage = pb_storage
self.config = config
self.config_data = self.config.get(self.PBH_CAT, {})
self.default_tz = self.config_data.get("default_timezone",
"Europe/Paris")
# this line allow us to raise an exception pytz.UnknownTimeZoneError,
# if the timezone defined in the pbehabior configuration file is wrong
pytz.timezone(self.default_tz)
self.pb_store = MongoCollection(MongoStore.get_default().get_collection('default_pbehavior'))
self.currently_active_pb = set()
def get(self, _id, query=None):
"""Get pbehavior by id.
:param str id: pbehavior id
:param dict query: filtering options
"""
return self.pb_storage.get_elements(ids=_id, query=query)
def create(
self,
name, filter, author,
tstart, tstop, rrule='',
enabled=True, comments=None,
connector='canopsis', connector_name='canopsis',
type_=PBehavior.DEFAULT_TYPE, reason='', timezone=None,
exdate=None):
"""
Method creates pbehavior record
:param str name: filtering options
:param dict filter: a mongo filter that match entities from canopsis
context
:param str author: the name of the user/app that has generated the
pbehavior
:param timestamp tstart: timestamp that correspond to the start of the
pbehavior
:param timestamp tstop: timestamp that correspond to the end of the
pbehavior
:param str rrule: reccurent rule that is compliant with rrule spec
:param bool enabled: boolean to know if pbhevior is enabled or disabled
:param list of dict comments: a list of comments made by users
:param str connector: a string representing the type of connector that
has generated the pbehavior
:param str connector_name: a string representing the name of connector
that has generated the pbehavior
:param str type_: associated type_ for this pbh
:param str reason: associated reason for this pbh
:param str timezone: the timezone of the new pbehabior. If no timezone
are given, use the default one. See the pbehavior documentation
for more information.
:param list of str| str exdate: a list of string representation of a date
following this pattern "YYYY/MM/DD HH:MM:00 TIMEZONE". The hour use the
24 hours clock system and the timezone is the name of the timezone. The
month, the day of the month, the hour, the minute and second are
zero-padded.
:raises ValueError: invalid RRULE
:raises pytz.UnknownTimeZoneError: invalid timezone
:return: created element eid
:rtype: str
"""
if timezone is None:
timezone = self.default_tz
if exdate is None:
exdate = []
# this line allow us to raise an exception pytz.UnknownTimeZoneError,
# if the timezone defined in the pbehabior configuration file is wrong
pytz.timezone(timezone)
if enabled in [True, "True", "true"]:
enabled = True
elif enabled in [False, "False", "false"]:
enabled = False
else:
raise ValueError("The enabled value does not match a boolean")
if not isinstance(exdate, list):
exdate = [exdate]
check_valid_rrule(rrule)
if comments is not None:
for comment in comments:
if "author" in comment:
if not isinstance(comment["author"], string_types):
raise ValueError("The author field must be an string")
else:
raise ValueError("The author field is missing")
if "message" in comment:
if not isinstance(comment["message"], string_types):
raise ValueError("The message field must be an string")
else:
raise ValueError("The message field is missing")
pb_kwargs = {
PBehavior.NAME: name,
PBehavior.FILTER: filter,
PBehavior.AUTHOR: author,
PBehavior.TSTART: tstart,
PBehavior.TSTOP: tstop,
PBehavior.RRULE: rrule,
PBehavior.ENABLED: enabled,
PBehavior.COMMENTS: comments,
PBehavior.CONNECTOR: connector,
PBehavior.CONNECTOR_NAME: connector_name,
PBehavior.TYPE: type_,
PBehavior.REASON: reason,
PBehavior.TIMEZONE: timezone,
PBehavior.EXDATE: exdate,
PBehavior.EIDS: []
}
data = PBehavior(**pb_kwargs)
if not data.comments or not isinstance(data.comments, list):
data.update(comments=[])
else:
for comment in data.comments:
comment.update({'_id': str(uuid4())})
result = self.pb_storage.put_element(element=data.to_dict())
return result
def get_pbehaviors_by_eid(self, id_):
"""Retreive from database every pbehavior that contains
the given id_ in the PBehavior.EIDS field.
:param list,str: the id(s) as a str or a list of string
:returns: a list of pbehavior, with the isActive key in pbehavior is
active when queried.
:rtype: list
"""
if not isinstance(id_, (list, string_types)):
raise TypeError(self.__TYPE_ERR)
if isinstance(id_, list):
for element in id_:
if not isinstance(element, string_types):
raise TypeError(self.__TYPE_ERR)
else:
id_ = [id_]
cursor = self.pb_storage.get_elements(
query={PBehavior.EIDS: {"$in": id_}}
)
pbehaviors = []
now = int(time())
for pb in cursor:
if pb['tstart'] <= now and (pb['tstop'] is None or pb['tstop'] >= now):
pb['isActive'] = True
else:
pb['isActive'] = False
pbehaviors.append(pb)
return pbehaviors
def read(self, _id=None):
"""Get pbehavior or list pbehaviors.
:param str _id: pbehavior id, _id may be equal to None
"""
result = self.get(_id)
return result if _id else list(result)
def update(self, _id, **kwargs):
"""
Update pbehavior record
:param str _id: pbehavior id
:param dict kwargs: values pbehavior fields. If a field is None, it will
**not** be updated.
:raises ValueError: invalid RRULE or no pbehavior with given _id
"""
pb_value = self.get(_id)
if pb_value is None:
raise ValueError("The id does not match any pebahvior")
check_valid_rrule(kwargs.get('rrule', ''))
pbehavior = PBehavior(**self.get(_id))
new_data = {k: v for k, v in kwargs.items() if v is not None}
pbehavior.update(**new_data)
result = self.pb_storage.put_element(
element=new_data, _id=_id
)
if (PBehaviorManager._UPDATE_FLAG in result and
result[PBehaviorManager._UPDATE_FLAG]):
return pbehavior.to_dict()
return None
def upsert(self, pbehavior):
"""
Creates or update the given pbehavior.
This function uses MongoStore/MongoCollection instead of Storage.
:param canopsis.models.pbehavior.PBehavior pbehavior:
:rtype: bool, dict
:returns: success, update result
"""
r = self.pb_store.update({'_id': pbehavior._id}, pbehavior.to_dict(), upsert=True)
if r.get('updatedExisting', False) and r.get('nModified') == 1:
return True, r
elif r.get('updatedExisting', None) is False and r.get('nModified') == 0 and r.get('ok') == 1.0:
return True, r
else:
return False, r
def delete(self, _id=None, _filter=None):
"""
Delete pbehavior record
:param str _id: pbehavior id
"""
result = self.pb_storage.remove_elements(
ids=_id, _filter=_filter
)
return self._check_response(result)
def _update_pbehavior(self, pbehavior_id, query):
result = self.pb_storage._update(
spec={'_id': pbehavior_id},
document=query,
multi=False, cache=False
)
return result
def create_pbehavior_comment(self, pbehavior_id, author, message):
"""
Сreate comment for pbehavior.
:param str pbehavior_id: pbehavior id
:param str author: author of the comment
:param str message: text of the comment
"""
comment_id = str(uuid4())
comment = {
Comment.ID: comment_id,
Comment.AUTHOR: author,
Comment.TS: timegm(datetime.utcnow().timetuple()),
Comment.MESSAGE: message
}
query = {'$addToSet': {PBehavior.COMMENTS: comment}}
result = self._update_pbehavior(pbehavior_id, query)
if not result:
result = self._update_pbehavior(
pbehavior_id, {'$set': {PBehavior.COMMENTS: []}}
)
if not result:
return None
result = self._update_pbehavior(pbehavior_id, query)
if (PBehaviorManager._UPDATE_FLAG in result and
result[PBehaviorManager._UPDATE_FLAG]):
return comment_id
return None
def update_pbehavior_comment(self, pbehavior_id, _id, **kwargs):
"""
Update the comment record.
:param str pbehavior_id: pbehavior id
:param str_id: comment id
:param dict kwargs: values comment fields
"""
pbehavior = self.get(
pbehavior_id,
query={PBehavior.COMMENTS: {'$elemMatch': {'_id': _id}}}
)
if not pbehavior:
return None
_comments = pbehavior[PBehavior.COMMENTS]
if not _comments:
return None
comment = Comment(**_comments[0])
comment.update(**kwargs)
result = self.pb_storage._update(
spec={'_id': pbehavior_id, 'comments._id': _id},
document={'$set': {'comments.$': comment.to_dict()}},
multi=False, cache=False
)
if (PBehaviorManager._UPDATE_FLAG in result and
result[PBehaviorManager._UPDATE_FLAG]):
return comment.to_dict()
return None
def delete_pbehavior_comment(self, pbehavior_id, _id):
"""
Delete comment record.
:param str pbehavior_id: pbehavior id
:param str _id: comment id
"""
result = self.pb_storage._update(
spec={'_id': pbehavior_id},
document={'$pull': {PBehavior.COMMENTS: {'_id': _id}}},
multi=False, cache=False
)
return self._check_response(result)
def get_pbehaviors(self, entity_id):
"""
Return all pbehaviors related to an entity_id, sorted by descending
tstart.
:param str entity_id: Id for which behaviors have to be returned
:return: pbehaviors, with name, tstart, tstop, rrule and enabled keys
:rtype: list of dict
"""
res = list(
self.pb_storage._backend.find(
{PBehavior.EIDS: {'$in': [entity_id]}},
sort=[(PBehavior.TSTART, DESCENDING)]
)
)
return res
def compute_pbehaviors_filters(self):
"""
Compute all filters and update eids attributes.
"""
pbehaviors = self.pb_storage.get_elements(
query={PBehavior.FILTER: {'$exists': True}}
)
for pbehavior in pbehaviors:
query = loads(pbehavior[PBehavior.FILTER])
if not isinstance(query, dict):
self.logger.error('compute_pbehaviors_filters(): filter is '
'not a dict !\n{}'.format(query))
continue
entities = self.context.ent_storage.get_elements(
query=query
)
pbehavior[PBehavior.EIDS] = [e['_id'] for e in entities]
self.pb_storage.put_element(element=pbehavior)
def _check_active_simple_pbehavior(self, timestamp, pbh):
""" Check if a pbehavior without a rrule is active at the given time.
:param int timestamp: the number a second this 1970/01/01 00:00:00
:param dict pbehavior: a pbehavior as a dict.
:return bool: True if the boolean is active, false otherwise
"""
if pbh[PBehavior.TSTART] <= timestamp <= pbh[PBehavior.TSTOP]:
return True
return False
@staticmethod
def __convert_timestamp(timestamp, timezone):
"""Convert a pbehavior timestamp defined in the timezone to a datetime
in the same timezone.
:param timestamp:"""
return datetime.fromtimestamp(timestamp, tz.gettz(timezone))
def _check_active_reccuring_pbehavior(self, timestamp, pbehavior):
""" Check if a pbehavior with a rrule is active at the given time.
:param int timestamp: the number a second this 1970/01/01 00:00:00
:param dict pbehavior: a pbehavior as a dict.
:return bool: True if the boolean is active, false otherwise
:raise ValueError: if the pbehavior.exdate is invalid. Or if the
date of an occurence of the pbehavior is not a valid date.
"""
tz_name = pbehavior.get(PBehavior.TIMEZONE, self.default_tz)
rec_set = rrule.rruleset()
# convert the timestamp to a datetime in the pbehavior's timezone
now = self.__convert_timestamp(timestamp, tz_name)
start = self.__convert_timestamp(pbehavior[PBehavior.TSTART], tz_name)
stop = self.__convert_timestamp(pbehavior[PBehavior.TSTOP], tz_name)
if PBehavior.EXDATE in pbehavior and\
isinstance(pbehavior[PBehavior.EXDATE], list):
for date in pbehavior[PBehavior.EXDATE]:
exdate = self.__convert_timestamp(date, tz_name)
rec_set.exdate(exdate)
duration = stop - start # pbehavior duration
rec_set.rrule(rrule.rrulestr(pbehavior[PBehavior.RRULE],
dtstart=start))
rec_start = rec_set.before(now)
self.logger.debug("Recurence start : {}".format(rec_start))
# No recurrence found
if rec_start is None:
return False
self.logger.debug("Timestamp : {}".format(now))
self.logger.debug("Recurence stop : {}".format(rec_start + duration))
if rec_start <= now <= rec_start + duration:
return True
return False
def check_active_pbehavior(self, timestamp, pbehavior):
""" Check if a pbehavior is active at the given time.
:param int timestamp: the number a second this 1970/01/01 00:00:00
:param dict pbehavior: a pbehavior as a dict.
:return bool: True if the boolean is active, false otherwise
:raise ValueError: if the pbehavior.exdate is invalid. Or if the
date of an occurence of the pbehavior is not a valid date.
"""
if PBehavior.RRULE not in pbehavior or\
pbehavior[PBehavior.RRULE] is None or\
pbehavior[PBehavior.RRULE] == "":
return self._check_active_simple_pbehavior(timestamp, pbehavior)
else:
if PBehavior.EXDATE not in pbehavior:
pbehavior[PBehavior.EXDATE] = []
return self._check_active_reccuring_pbehavior(timestamp, pbehavior)
def check_pbehaviors(self, entity_id, list_in, list_out):
"""
!!!! DEPRECATED !!!!
:param str entity_id:
:param list list_in: list of pbehavior names
:param list list_out: list of pbehavior names
:returns: bool if the entity_id is currently in list_in arg and out list_out arg
"""
return (self._check_pbehavior(entity_id, list_in) and
not self._check_pbehavior(entity_id, list_out))
def _check_pbehavior(self, entity_id, pb_names):
"""
:param str entity_id:
:param list pb_names: list of pbehavior names
:returns: bool if the entity_id is currently in pb_names arg
"""
self.logger.critical("_check_pbehavior is DEPRECATED !!!!")
try:
entity = self.context.get_entities_by_id(entity_id)[0]
except Exception:
self.logger.error('Unable to check_behavior on {} entity_id'
.format(entity_id))
return None
event = self.context.get_event(entity)
pbehaviors = self.pb_storage.get_elements(
query={
PBehavior.NAME: {'$in': pb_names},
PBehavior.EIDS: {'$in': [entity_id]}
}
)
names = []
fromts = datetime.fromtimestamp
for pbehavior in pbehaviors:
tstart = pbehavior[PBehavior.TSTART]
tstop = pbehavior[PBehavior.TSTOP]
if not isinstance(tstart, (int, float)):
self.logger.error('Cannot parse tstart value: {}'
.format(pbehavior))
continue
if not isinstance(tstop, (int, float)):
self.logger.error('Cannot parse tstop value: {}'
.format(pbehavior))
continue
tstart = fromts(tstart)
tstop = fromts(tstop)
dt_list = [tstart, tstop]
if pbehavior['rrule'] is not None:
dt_list = list(
rrule.rrulestr(pbehavior['rrule'], dtstart=tstart).between(
tstart, tstop, inc=True
)
)
if (len(dt_list) >= 2
and fromts(event['timestamp']) >= dt_list[0]
and fromts(event['timestamp']) <= dt_list[-1]):
names.append(pbehavior[PBehavior.NAME])
result = set(pb_names).isdisjoint(set(names))
return not result
@staticmethod
def _check_response(response):
ack = True if 'ok' in response and response['ok'] == 1 else False
return {
'acknowledged': ack,
'deletedCount': response['n']
}
def get_active_pbehaviors(self, eids):
"""
Return a list of active pbehaviors linked to some entites.
:param list eids: the desired entities id
:returns: list of pbehaviors
"""
result = []
for eid in eids:
pbhs = self.get_pbehaviors(eid)
result = result + [x for x in pbhs if self._check_pbehavior(
eid, [x['name']]
)]
return result
def get_all_active_pbehaviors(self):
"""
Return all pbehaviors currently active using
self.check_active_pbehavior
"""
now = int(time())
query = {}
ret_val = list(self.pb_storage.get_elements(query=query))
results = []
for pb in ret_val:
try:
if self.check_active_pbehavior(now, pb):
results.append(pb)
except ValueError as exept:
self.logger.exception("Can't check if the pbehavior is active.")
return results
def get_active_pbehaviors_from_type(self, types=None):
"""
Return pbehaviors currently active, with a specific type,
using self.check_active_pbehavior
"""
if types is None:
types = []
now = int(time())
query = {PBehavior.TYPE: {'$in': types}}
ret_val = list(self.pb_storage.get_elements(query=query))
results = []
for pb in ret_val:
if self.check_active_pbehavior(now, pb):
results.append(pb)
return results
def get_varying_pbehavior_list(self):
"""
get_varying_pbehavior_list
:returns: list of PBehavior id activated since last check
:rtype: list
"""
active_pbehaviors = self.get_all_active_pbehaviors()
active_pbehaviors_ids = set()
for active_pb in active_pbehaviors:
active_pbehaviors_ids.add(active_pb['_id'])
varying_pbs = active_pbehaviors_ids.symmetric_difference(self.currently_active_pb)
self.currently_active_pb = active_pbehaviors_ids
return list(varying_pbs)
def launch_update_watcher(self, watcher_manager):
"""
launch_update_watcher update watcher when a pbehavior is active
:param object watcher_manager: watcher manager
:returns: number of watcher updated
retype: int
"""
new_pbs = self.get_varying_pbehavior_list()
new_pbs_full = list(self.pb_storage._backend.find(
{'_id': {'$in': new_pbs}}
))
merged_eids = []
for pbehaviour in new_pbs_full:
merged_eids = merged_eids + pbehaviour['eids']
watchers_ids = set()
for watcher in self.get_wacher_on_entities(merged_eids):
watchers_ids.add(watcher['_id'])
for watcher_id in watchers_ids:
watcher_manager.compute_state(watcher_id)
return len(list(watchers_ids))
def get_wacher_on_entities(self, entities_ids):
"""
get_wacher_on_entities.
:param entities_ids: entity id
:returns: list of watchers
:rtype: list
"""
query = {
'$and': [
{'depends': {'$in': entities_ids}},
{'type': 'watcher'}
]
}
watchers = self.context.get_entities(query=query)
return watchers
@staticmethod
def get_active_intervals(after, before, pbehavior):
"""
Return all the time intervals between after and before during which the
pbehavior was active.
The intervals are returned as a list of tuples (start, end), ordered
chronologically. start and end are UTC timestamps, and are always
between after and before.
:param int after: a UTC timestamp
:param int before: a UTC timestamp
:param Dict[str, Any] pbehavior:
:rtype: List[Tuple[int, int]]
"""
rrule_str = pbehavior[PBehavior.RRULE]
tstart = pbehavior[PBehavior.TSTART]
tstop = pbehavior[PBehavior.TSTOP]
if not isinstance(tstart, (int, float)):
return
if not isinstance(tstop, (int, float)):
return
# Convert the timestamps to datetimes
tz = pytz.UTC
dttstart = datetime.utcfromtimestamp(tstart).replace(tzinfo=tz)
dttstop = datetime.utcfromtimestamp(tstop).replace(tzinfo=tz)
delta = dttstop - dttstart
dtafter = datetime.utcfromtimestamp(after).replace(tzinfo=tz)
dtbefore = datetime.utcfromtimestamp(before).replace(tzinfo=tz)
if not rrule_str:
# The only interval where the pbehavior is active is
# [dttstart, dttstop]. Ensure that it is included in
# [after, before], and convert the datetimes to timestamps.
if dttstart < dtafter:
dttstart = dtafter
if dttstop > dtbefore:
dttstop = dtbefore
yield (
timegm(dttstart.timetuple()),
timegm(dttstop.timetuple())
)
else:
# Get all the intervals that intersect with the [after, before]
# interval.
interval_starts = rrule.rrulestr(rrule_str, dtstart=dttstart).between(
dtafter - delta, dtbefore, inc=False)
for interval_start in interval_starts:
interval_end = interval_start + delta
# Ensure that the interval is included in [after, before], and
# datetimes to timestamps.
if interval_start < dtafter:
interval_start = dtafter
if interval_end > dtbefore:
interval_end = dtbefore
yield (
timegm(interval_start.timetuple()),
timegm(interval_end.timetuple())
)
def get_intervals_with_pbehaviors_by_eid(self, after, before, entity_id):
"""
Yields intervals between after and before with a boolean indicating if
a pbehavior affects the entity during this interval.
The intervals are returned as a list of tuples (start, end, pbehavior),
ordered chronologically. start and end are UTC timestamps, and are
always between after and before, pbehavior is a boolean indicating if a
pbehavior affects the entity during this interval. None of the
intervals overlap.
:param int after: a UTC timestamp
:param int before: a UTC timestamp
:param str entity_id: the id of the entity
:rtype: Iterator[Tuple[int, int, bool]]
"""
return self.get_intervals_with_pbehaviors(
after, before, self.get_pbehaviors(entity_id))
def get_intervals_with_pbehaviors(self, after, before, pbehaviors):
"""
Yields intervals between after and before with a boolean indicating if
one of the pbehaviors is active during this interval.
The intervals are returned as a list of tuples (start, end, pbehavior),
ordered chronologically. start and end are UTC timestamps, and are
always between after and before, pbehavior is a boolean indicating if a
pbehavior affects the entity during this interval. None of the
intervals overlap.
:param int after: a UTC timestamp
:param int before: a UTC timestamp
:param List[Dict[str, Any]] pbehaviors: a list of pbehabiors
:rtype: Iterator[Tuple[int, int, bool]]
"""
intervals = []
# Get all the intervals where a pbehavior is active
for pbehavior in pbehaviors:
for interval in self.get_active_intervals(after, before, pbehavior):
intervals.append(interval)
if not intervals:
yield (after, before, False)
return
# Order them chronologically (by start date)
intervals.sort(key=lambda a: a[0])
# Yield the first interval without any active pbehavior
merged_interval_start, merged_interval_end = intervals[0]
yield (
after,
merged_interval_start,
False
)
# At this point intervals is a list of intervals where a pbehavior is
# active, ordered by start date. Some of those intervals may be
# overlapping. This merges the overlapping intervals.
for interval_start, interval_end in intervals[1:]:
if interval_end < merged_interval_end:
# The interval is included in the merged interval, skip it.
continue
if interval_start > merged_interval_end:
# Since the interval starts after the end of the merged
# interval, they cannot be merged. Yield the merged interval,
# and move to the new one.
yield (
merged_interval_start,
merged_interval_end,
True
)
yield (
merged_interval_end,
interval_start,
False
)
merged_interval_start = interval_start
merged_interval_end = interval_end
yield (
merged_interval_start,
merged_interval_end,
True
)
yield (
merged_interval_end,
before,
False
)
def get_enabled_pbehaviors(self):
"""
Yields all the enabled pbehaviors.
:rtype: Iterator[Dict[str, Any]]
"""
return self.pb_storage._backend.find({
PBehavior.ENABLED: True
})
class PBehaviorManager(object):
"""
PBehavior manager class.
"""
PB_STORAGE_URI = 'mongodb-default-pbehavior://'
LOG_PATH = 'var/log/pbehaviormanager.log'
LOG_NAME = 'pbehaviormanager'
_UPDATE_FLAG = 'updatedExisting'
__TYPE_ERR = "id_ must be a list of string or a string"
@classmethod
def provide_default_basics(cls):
"""
Provide the default configuration and logger objects
for PBehaviorManager.
Do not use those defaults for tests.
:return: config, logger, storage
:rtype: Union[dict, logging.Logger, canopsis.storage.core.Storage]
"""
logger = Logger.get(cls.LOG_NAME, cls.LOG_PATH)
pb_storage = Middleware.get_middleware_by_uri(cls.PB_STORAGE_URI)
return logger, pb_storage
def __init__(self, logger, pb_storage):
"""
:param dict config: configuration
:param pb_storage: PBehavior Storage object
"""
super(PBehaviorManager, self).__init__()
kwargs = {"logger": logger}
self.context = singleton_per_scope(ContextGraph, kwargs=kwargs)
self.logger = logger
self.pb_storage = pb_storage
self.pb_store = MongoCollection(
MongoStore.get_default().get_collection('default_pbehavior'))
self.currently_active_pb = set()
def get(self, _id, query=None):
"""Get pbehavior by id.
:param str id: pbehavior id
:param dict query: filtering options
"""
return self.pb_storage.get_elements(ids=_id, query=query)
def create(self,
name,
filter,
author,
tstart,
tstop,
rrule='',
enabled=True,
comments=None,
connector='canopsis',
connector_name='canopsis',
type_=PBehavior.DEFAULT_TYPE,
reason=''):
"""
Method creates pbehavior record
:param str name: filtering options
:param dict filter: a mongo filter that match entities from canopsis
context
:param str author: the name of the user/app that has generated the
pbehavior
:param timestamp tstart: timestamp that correspond to the start of the
pbehavior
:param timestamp tstop: timestamp that correspond to the end of the
pbehavior
:param str rrule: reccurent rule that is compliant with rrule spec
:param bool enabled: boolean to know if pbhevior is enabled or disabled
:param list of dict comments: a list of comments made by users
:param str connector: a string representing the type of connector that
has generated the pbehavior
:param str connector_name: a string representing the name of connector
that has generated the pbehavior
:param str type_: associated type_ for this pbh
:param str reason: associated reason for this pbh
:raises ValueError: invalid RRULE
:return: created element eid
:rtype: str
"""
if enabled in [True, "True", "true"]:
enabled = True
elif enabled in [False, "False", "false"]:
enabled = False
else:
raise ValueError("The enabled value does not match a boolean")
check_valid_rrule(rrule)
if comments is not None:
for comment in comments:
if "author" in comment:
if not isinstance(comment["author"], string_types):
raise ValueError("The author field must be an string")
else:
raise ValueError("The author field is missing")
if "message" in comment:
if not isinstance(comment["message"], string_types):
raise ValueError("The message field must be an string")
else:
raise ValueError("The message field is missing")
pb_kwargs = {
'name': name,
'filter': filter,
'author': author,
'tstart': tstart,
'tstop': tstop,
'rrule': rrule,
'enabled': enabled,
'comments': comments,
'connector': connector,
'connector_name': connector_name,
PBehavior.TYPE: type_,
'reason': reason
}
if PBehavior.EIDS not in pb_kwargs:
pb_kwargs[PBehavior.EIDS] = []
data = PBehavior(**pb_kwargs)
if not data.comments or not isinstance(data.comments, list):
data.update(comments=[])
else:
for comment in data.comments:
comment.update({'_id': str(uuid4())})
result = self.pb_storage.put_element(element=data.to_dict())
return result
def get_pbehaviors_by_eid(self, id_):
"""Retreive from database every pbehavior that contains
the given id_ in the PBehavior.EIDS field.
:param list,str: the id(s) as a str or a list of string
:returns: a list of pbehavior, with the isActive key in pbehavior is
active when queried.
:rtype: list
"""
if not isinstance(id_, (list, string_types)):
raise TypeError(self.__TYPE_ERR)
if isinstance(id_, list):
for element in id_:
if not isinstance(element, string_types):
raise TypeError(self.__TYPE_ERR)
else:
id_ = [id_]
cursor = self.pb_storage.get_elements(
query={PBehavior.EIDS: {
"$in": id_
}})
pbehaviors = []
now = int(time())
for pb in cursor:
if pb['tstart'] <= now and (pb['tstop'] is None
or pb['tstop'] >= now):
pb['isActive'] = True
else:
pb['isActive'] = False
pbehaviors.append(pb)
return pbehaviors
def read(self, _id=None):
"""Get pbehavior or list pbehaviors.
:param str _id: pbehavior id, _id may be equal to None
"""
result = self.get(_id)
return result if _id else list(result)
def update(self, _id, **kwargs):
"""
Update pbehavior record
:param str _id: pbehavior id
:param dict kwargs: values pbehavior fields. If a field is None, it will
**not** be updated.
:raises ValueError: invalid RRULE or no pbehavior with given _id
"""
pb_value = self.get(_id)
if pb_value is None:
raise ValueError("The id does not match any pebahvior")
check_valid_rrule(kwargs.get('rrule', ''))
pbehavior = PBehavior(**self.get(_id))
new_data = {k: v for k, v in kwargs.items() if v is not None}
pbehavior.update(**new_data)
result = self.pb_storage.put_element(element=new_data, _id=_id)
if (PBehaviorManager._UPDATE_FLAG in result
and result[PBehaviorManager._UPDATE_FLAG]):
return pbehavior.to_dict()
return None
def upsert(self, pbehavior):
"""
Creates or update the given pbehavior.
This function uses MongoStore/MongoCollection instead of Storage.
:param canopsis.models.pbehavior.PBehavior pbehavior:
:rtype: bool, dict
:returns: success, update result
"""
r = self.pb_store.update({'_id': pbehavior._id},
pbehavior.to_dict(),
upsert=True)
if r.get('updatedExisting', False) and r.get('nModified') == 1:
return True, r
elif r.get('updatedExisting', None) is False and r.get(
'nModified') == 0 and r.get('ok') == 1.0:
return True, r
else:
return False, r
def delete(self, _id=None, _filter=None):
"""
Delete pbehavior record
:param str _id: pbehavior id
"""
result = self.pb_storage.remove_elements(ids=_id, _filter=_filter)
return self._check_response(result)
def _update_pbehavior(self, pbehavior_id, query):
result = self.pb_storage._update(spec={'_id': pbehavior_id},
document=query,
multi=False,
cache=False)
return result
def create_pbehavior_comment(self, pbehavior_id, author, message):
"""
Сreate comment for pbehavior.
:param str pbehavior_id: pbehavior id
:param str author: author of the comment
:param str message: text of the comment
"""
comment_id = str(uuid4())
comment = {
Comment.ID: comment_id,
Comment.AUTHOR: author,
Comment.TS: timegm(datetime.utcnow().timetuple()),
Comment.MESSAGE: message
}
query = {'$addToSet': {PBehavior.COMMENTS: comment}}
result = self._update_pbehavior(pbehavior_id, query)
if not result:
result = self._update_pbehavior(pbehavior_id,
{'$set': {
PBehavior.COMMENTS: []
}})
if not result:
return None
result = self._update_pbehavior(pbehavior_id, query)
if (PBehaviorManager._UPDATE_FLAG in result
and result[PBehaviorManager._UPDATE_FLAG]):
return comment_id
return None
def update_pbehavior_comment(self, pbehavior_id, _id, **kwargs):
"""
Update the comment record.
:param str pbehavior_id: pbehavior id
:param str_id: comment id
:param dict kwargs: values comment fields
"""
pbehavior = self.get(
pbehavior_id,
query={PBehavior.COMMENTS: {
'$elemMatch': {
'_id': _id
}
}})
if not pbehavior:
return None
_comments = pbehavior[PBehavior.COMMENTS]
if not _comments:
return None
comment = Comment(**_comments[0])
comment.update(**kwargs)
result = self.pb_storage._update(
spec={
'_id': pbehavior_id,
'comments._id': _id
},
document={'$set': {
'comments.$': comment.to_dict()
}},
multi=False,
cache=False)
if (PBehaviorManager._UPDATE_FLAG in result
and result[PBehaviorManager._UPDATE_FLAG]):
return comment.to_dict()
return None
def delete_pbehavior_comment(self, pbehavior_id, _id):
"""
Delete comment record.
:param str pbehavior_id: pbehavior id
:param str _id: comment id
"""
result = self.pb_storage._update(
spec={'_id': pbehavior_id},
document={'$pull': {
PBehavior.COMMENTS: {
'_id': _id
}
}},
multi=False,
cache=False)
return self._check_response(result)
def get_pbehaviors(self, entity_id):
"""
Return all pbehaviors related to an entity_id, sorted by descending
tstart.
:param str entity_id: Id for which behaviors have to be returned
:return: pbehaviors, with name, tstart, tstop, rrule and enabled keys
:rtype: list of dict
"""
res = list(
self.pb_storage._backend.find(
{PBehavior.EIDS: {
'$in': [entity_id]
}},
sort=[(PBehavior.TSTART, DESCENDING)]))
return res
def compute_pbehaviors_filters(self):
"""
Compute all filters and update eids attributes.
"""
pbehaviors = self.pb_storage.get_elements(
query={PBehavior.FILTER: {
'$exists': True
}})
for pbehavior in pbehaviors:
query = loads(pbehavior[PBehavior.FILTER])
if not isinstance(query, dict):
self.logger.error('compute_pbehaviors_filters(): filter is '
'not a dict !\n{}'.format(query))
continue
entities = self.context.ent_storage.get_elements(query=query)
pbehavior[PBehavior.EIDS] = [e['_id'] for e in entities]
self.pb_storage.put_element(element=pbehavior)
@staticmethod
def check_active_pbehavior(timestamp, pbehavior):
"""
For a given pbehavior (as dict) check that the given timestamp is active
using either:
the rrule, if any, from the pbehavior + tstart and tstop to define
start and stop times.
tstart and tstop alone if no rrule is available.
All dates and times are computed using UTC timezone, so check that your
timestamp was exported using UTC.
:param int timestamp: timestamp to check
:param dict pbehavior: the pbehavior
:rtype bool:
:returns: pbehavior is currently active or not
"""
fromts = datetime.utcfromtimestamp
tstart = pbehavior[PBehavior.TSTART]
tstop = pbehavior[PBehavior.TSTOP]
if not isinstance(tstart, (int, float)):
return False
if not isinstance(tstop, (int, float)):
return False
tz = pytz.UTC
dtts = fromts(timestamp).replace(tzinfo=tz)
dttstart = fromts(tstart).replace(tzinfo=tz)
dttstop = fromts(tstop).replace(tzinfo=tz)
dt_list = [dttstart, dttstop]
rrule = pbehavior['rrule']
if rrule:
# compute the minimal date from which to start generating
# dates from the rrule.
# a complementary date (missing_date) is computed and added
# at index 0 of the generated dt_list to ensure we manage
# dates at boundaries.
dt_tstart_date = dtts.date()
dt_tstart_time = dttstart.time().replace(tzinfo=tz)
dt_dtstart = datetime.combine(dt_tstart_date, dt_tstart_time)
# dates in dt_list at 0 and 1 indexes can be equal, so we generate
# three dates to ensure [1] - [2] will give a non-zero timedelta
# object.
dt_list = list(
rrulestr(rrule, dtstart=dt_dtstart).xafter(dttstart,
count=3,
inc=True))
# compute the "missing dates": dates before the rrule started to
# generate dates so we can check for a pbehavior in the past.
multiply = 1
while True:
missing_date = dt_list[0] - multiply * (dt_list[-1] -
dt_list[-2])
dt_list.insert(0, missing_date)
if missing_date < dtts:
break
multiply += 1
delta = dttstop - dttstart
for dt in sorted(dt_list):
if dtts >= dt and dtts <= dt + delta:
return True
return False
else:
if dtts >= dttstart and dtts <= dttstop:
return True
return False
return False
def check_pbehaviors(self, entity_id, list_in, list_out):
"""
:param str entity_id:
:param list list_in: list of pbehavior names
:param list list_out: list of pbehavior names
:returns: bool if the entity_id is currently in list_in arg and out list_out arg
"""
return (self._check_pbehavior(entity_id, list_in)
and not self._check_pbehavior(entity_id, list_out))
def _check_pbehavior(self, entity_id, pb_names):
"""
:param str entity_id:
:param list pb_names: list of pbehavior names
:returns: bool if the entity_id is currently in pb_names arg
"""
try:
entity = self.context.get_entities_by_id(entity_id)[0]
except Exception:
self.logger.error(
'Unable to check_behavior on {} entity_id'.format(entity_id))
return None
event = self.context.get_event(entity)
pbehaviors = self.pb_storage.get_elements(
query={
PBehavior.NAME: {
'$in': pb_names
},
PBehavior.EIDS: {
'$in': [entity_id]
}
})
names = []
fromts = datetime.fromtimestamp
for pbehavior in pbehaviors:
tstart = pbehavior[PBehavior.TSTART]
tstop = pbehavior[PBehavior.TSTOP]
if not isinstance(tstart, (int, float)):
self.logger.error(
'Cannot parse tstart value: {}'.format(pbehavior))
continue
if not isinstance(tstop, (int, float)):
self.logger.error(
'Cannot parse tstop value: {}'.format(pbehavior))
continue
tstart = fromts(tstart)
tstop = fromts(tstop)
dt_list = [tstart, tstop]
if pbehavior['rrule'] is not None:
dt_list = list(
rrulestr(pbehavior['rrule'],
dtstart=tstart).between(tstart, tstop, inc=True))
if (len(dt_list) >= 2 and fromts(event['timestamp']) >= dt_list[0]
and fromts(event['timestamp']) <= dt_list[-1]):
names.append(pbehavior[PBehavior.NAME])
result = set(pb_names).isdisjoint(set(names))
return not result
@staticmethod
def _check_response(response):
ack = True if 'ok' in response and response['ok'] == 1 else False
return {'acknowledged': ack, 'deletedCount': response['n']}
def get_active_pbehaviors(self, eids):
"""
Return a list of active pbehaviors linked to some entites.
:param list eids: the desired entities id
:returns: list of pbehaviors
"""
result = []
for eid in eids:
pbhs = self.get_pbehaviors(eid)
result = result + [
x for x in pbhs if self._check_pbehavior(eid, [x['name']])
]
return result
def get_all_active_pbehaviors(self):
"""
Return all pbehaviors currently active using
self.check_active_pbehavior
"""
now = int(time())
query = {}
ret_val = list(self.pb_storage.get_elements(query=query))
results = []
for pb in ret_val:
if self.check_active_pbehavior(now, pb):
results.append(pb)
return results
def get_active_pbehaviors_from_type(self, types=None):
"""
Return pbehaviors currently active, with a specific type,
using self.check_active_pbehavior
"""
if types is None:
types = []
now = int(time())
query = {PBehavior.TYPE: {'$in': types}}
ret_val = list(self.pb_storage.get_elements(query=query))
results = []
for pb in ret_val:
if self.check_active_pbehavior(now, pb):
results.append(pb)
return results
def get_varying_pbehavior_list(self):
"""
get_varying_pbehavior_list
:returns: list of PBehavior id activated since last check
:rtype: list
"""
active_pbehaviors = self.get_all_active_pbehaviors()
active_pbehaviors_ids = set()
for active_pb in active_pbehaviors:
active_pbehaviors_ids.add(active_pb['_id'])
varying_pbs = active_pbehaviors_ids.symmetric_difference(
self.currently_active_pb)
self.currently_active_pb = active_pbehaviors_ids
return list(varying_pbs)
def launch_update_watcher(self, watcher_manager):
"""
launch_update_watcher update watcher when a pbehavior is active
:param object watcher_manager: watcher manager
:returns: number of watcher updated
retype: int
"""
new_pbs = self.get_varying_pbehavior_list()
new_pbs_full = list(
self.pb_storage._backend.find({'_id': {
'$in': new_pbs
}}))
merged_eids = []
for pbehaviour in new_pbs_full:
merged_eids = merged_eids + pbehaviour['eids']
watchers_ids = set()
for watcher in self.get_wacher_on_entities(merged_eids):
watchers_ids.add(watcher['_id'])
for watcher_id in watchers_ids:
watcher_manager.compute_state(watcher_id)
return len(list(watchers_ids))
def get_wacher_on_entities(self, entities_ids):
"""
get_wacher_on_entities.
:param entities_ids: entity id
:returns: list of watchers
:rtype: list
"""
query = {
'$and': [{
'depends': {
'$in': entities_ids
}
}, {
'type': 'watcher'
}]
}
watchers = self.context.get_entities(query=query)
return watchers
@staticmethod
def get_active_intervals(after, before, pbehavior):
"""
Return all the time intervals between after and before during which the
pbehavior was active.
The intervals are returned as a list of tuples (start, end), ordered
chronologically. start and end are UTC timestamps, and are always
between after and before.
:param int after: a UTC timestamp
:param int before: a UTC timestamp
:param Dict[str, Any] pbehavior:
:rtype: List[Tuple[int, int]]
"""
rrule = pbehavior[PBehavior.RRULE]
tstart = pbehavior[PBehavior.TSTART]
tstop = pbehavior[PBehavior.TSTOP]
if not isinstance(tstart, (int, float)):
return
if not isinstance(tstop, (int, float)):
return
# Convert the timestamps to datetimes
tz = pytz.UTC
dttstart = datetime.utcfromtimestamp(tstart).replace(tzinfo=tz)
dttstop = datetime.utcfromtimestamp(tstop).replace(tzinfo=tz)
delta = dttstop - dttstart
dtafter = datetime.utcfromtimestamp(after).replace(tzinfo=tz)
dtbefore = datetime.utcfromtimestamp(before).replace(tzinfo=tz)
if not rrule:
# The only interval where the pbehavior is active is
# [dttstart, dttstop]. Ensure that it is included in
# [after, before], and convert the datetimes to timestamps.
if dttstart < dtafter:
dttstart = dtafter
if dttstop > dtbefore:
dttstop = dtbefore
yield (timegm(dttstart.timetuple()), timegm(dttstop.timetuple()))
else:
# Get all the intervals that intersect with the [after, before]
# interval.
interval_starts = rrulestr(rrule, dtstart=dttstart).between(
dtafter - delta, dtbefore, inc=False)
for interval_start in interval_starts:
interval_end = interval_start + delta
# Ensure that the interval is included in [after, before], and
# datetimes to timestamps.
if interval_start < dtafter:
interval_start = dtafter
if interval_end > dtbefore:
interval_end = dtbefore
yield (timegm(interval_start.timetuple()),
timegm(interval_end.timetuple()))
def get_intervals_with_pbehaviors(self, after, before, entity_id):
"""
Yields intervals between after and before with a boolean indicating if
a pbehavior affects the entity during this interval.
The intervals are returned as a list of tuples (start, end, pbehavior),
ordered chronologically. start and end are UTC timestamps, and are
always between after and before, pbehavior is a boolean indicating if a
pbehavior affects the entity during this interval. None of the
intervals overlap.
:param int after: a UTC timestamp
:param int before: a UTC timestamp
:param str entity_id: the id of the entity
:rtype: Iterator[Tuple[int, int, bool]]
"""
intervals = []
# Get all the intervals where a pbehavior is active
pbehaviors = self.get_pbehaviors(entity_id)
for pbehavior in pbehaviors:
for interval in self.get_active_intervals(after, before,
pbehavior):
intervals.append(interval)
if not intervals:
yield (after, before, False)
return
# Order them chronologically (by start date)
intervals.sort(key=lambda a: a[0])
# Yield the first interval without any active pbehavior
merged_interval_start, merged_interval_end = intervals[0]
yield (after, merged_interval_start, False)
# At this point intervals is a list of intervals where a pbehavior is
# active, ordered by start date. Some of those intervals may be
# overlapping. This merges the overlapping intervals.
for interval_start, interval_end in intervals[1:]:
if interval_end < merged_interval_end:
# The interval is included in the merged interval, skip it.
continue
if interval_start > merged_interval_end:
# Since the interval starts after the end of the merged
# interval, they cannot be merged. Yield the merged interval,
# and move to the new one.
yield (merged_interval_start, merged_interval_end, True)
yield (merged_interval_end, interval_start, False)
merged_interval_start = interval_start
merged_interval_end = interval_end
yield (merged_interval_start, merged_interval_end, True)
yield (merged_interval_end, before, False)
class TestMongoCollection(unittest.TestCase):
def setUp(self):
output = StringIO()
self.logger = Logger.get('test', output, OutputStream)
self.storage = Middleware.get_middleware_by_uri(
'storage-default-testmongocollection://')
self.collection = MongoCollection(collection=self.storage._backend,
logger=self.logger)
self.id_ = 'testid'
def tearDown(self):
"""Teardown"""
self.collection.remove()
def test_insert(self):
res = self.collection.insert(document={'_id': self.id_})
self.assertEqual(res, self.id_)
res2 = self.collection.insert(document={'up': 'down'})
self.assertTrue(isinstance(res, string_types))
self.assertNotEqual(res, res2)
def test_update(self):
res = self.collection.update(query={'_id': self.id_},
document={'strange': 'charm'})
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 0)
res = self.collection.update(query={'_id': self.id_},
document={'yin': 'yang'},
upsert=True)
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 1)
res = self.collection.find_one(self.id_)
self.assertEqual(res['yin'], 'yang')
self.assertTrue('strange' not in res)
def test_remove(self):
res = self.collection.insert(document={
'_id': self.id_,
'top': 'bottom'
})
self.assertIsNotNone(res)
res = self.collection.remove(query={'_id': self.id_})
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 1)
# Deleting non-existing object doesn't throw error
res = self.collection.remove(query={})
self.assertTrue(MongoCollection.is_successfull(res))
self.assertEqual(res['n'], 0)
def test_find(self):
res = self.collection.insert(document={'_id': self.id_, 'yin': 'yang'})
self.assertIsNotNone(res)
res = self.collection.find(query={'_id': self.id_})
self.assertTrue(isinstance(res, Cursor))
res = list(res)
self.assertTrue(isinstance(res, list))
self.assertEqual(res[0]['yin'], 'yang')
def test_find_one(self):
res = self.collection.insert(document={'_id': self.id_, 'up': 'down'})
self.assertIsNotNone(res)
res = self.collection.insert(document={'strange': 'charm'})
self.assertIsNotNone(res)
res = self.collection.find_one(query={'_id': self.id_})
self.assertTrue(isinstance(res, dict))
self.assertEqual(res['up'], 'down')
def test_is_successfull(self):
dico = {'ok': 1.0, 'n': 2}
self.assertTrue(MongoCollection.is_successfull(dico))
dico = {'ok': 666.667, 'n': 1}
self.assertFalse(MongoCollection.is_successfull(dico))
dico = {'n': 2}
self.assertFalse(MongoCollection.is_successfull(dico))
class RuleManager(object):
"""
Manager for event filter rules.
"""
def __init__(self, logger):
self.logger = logger
self.rule_collection = MongoCollection(
MongoStore.get_default().get_collection(RULE_COLLECTION))
def get_by_id(self, rule_id):
"""
Get an event filter rule given its id.
:param str rule_id: the id of the rule.
:rtype: Dict[str, Any]
"""
return self.rule_collection.find_one({
RuleField.id: rule_id
})
def create(self, rule):
"""
Create a new rule and return its id.
:param Dict[str, Any] rule:
:rtype: str
:raises: InvalidRuleError if the rule is invalid. CollectionError if
the creation fails.
"""
rule_id = str(uuid4())
rule[RuleField.id] = rule_id
self.validate(rule_id, rule)
self.rule_collection.insert(rule)
return rule_id
def remove_with_id(self, rule_id):
"""
Remove a rule given its id.
:param str rule_id: the id of the rule. CollectionError if the
creation fails.
"""
self.rule_collection.remove({
RuleField.id: rule_id
})
def list(self):
"""
Return a list of all the rules.
:rtype: List[Dict[str, Any]]
"""
return list(self.rule_collection.find({}))
def update(self, rule_id, rule):
"""
Update a rule given its id.
:param str rule_id: the id of the rule.
:param Dict[str, Any] rule:
:raises: InvalidRuleError if the rule is invalid. CollectionError if
the creation fails.
"""
self.validate(rule_id, rule)
self.rule_collection.update({
RuleField.id: rule_id
}, rule, upsert=False)
def validate(self, rule_id, rule):
"""
Check that the rule is valid.
The pattern and external_data fields are not validated by this method.
:param Dict[str, Any] view:
:raises: InvalidRuleError if it is invalid.
"""
# Validate id field
if rule.get(RuleField.id, rule_id) != rule_id:
raise InvalidRuleError(
'The {0} field should not be modified.'.format(RuleField.id))
# Check that there are no unexpected fields in the rule
unexpected_fields = set(rule.keys()).difference(RuleField.values)
if unexpected_fields:
raise InvalidRuleError(
'Unexpected fields: {0}.'.format(', '.join(unexpected_fields)))
# Validate the type field
if RuleField.type not in rule:
raise InvalidRuleError(
'The {0} field is required.'.format(RuleField.type))
if rule.get(RuleField.type) not in RuleType.values:
raise InvalidRuleError(
'The {0} field should be one of: {1}.'.format(
RuleField.type,
', '.join(RuleType.values)))
# Validate the priority field
if not isinstance(rule.get(RuleField.priority, 0), int):
raise InvalidRuleError(
'The {0} field should be an integer.'.format(
RuleField.priority))
# Validate the enabled field
if not isinstance(rule.get(RuleField.enabled, True), bool):
raise InvalidRuleError(
'The {0} field should be a boolean.'.format(
RuleField.enabled))
if rule.get(RuleField.type) != RuleType.enrichment:
# Check that the enrichment fields are not defined for
# non-enrichment rules.
unexpected_fields = set(rule.keys()).intersection(
ENRICHMENT_FIELDS)
if unexpected_fields:
raise InvalidRuleError(
'The following fields should only be defined for '
'enrichment rules: {0}.'.format(
', '.join(unexpected_fields)))
else:
# Validate the actions field of the enrichment rules.
if RuleField.actions not in rule:
raise InvalidRuleError(
'The {0} field is required for enrichment rules.'.format(
RuleField.actions))
if not isinstance(rule.get(RuleField.actions), list):
raise InvalidRuleError(
'The {0} field should be a list.'.format(
RuleField.actions))
if not rule.get(RuleField.actions):
raise InvalidRuleError(
'The {0} field should contain at least one action.'.format(
RuleField.actions))
# Validate the on_success field of the enrichment rules.
outcome = rule.get(RuleField.on_success)
if outcome and outcome not in RuleOutcome.values:
raise InvalidRuleError(
'The {0} field should be one of: {1}.'.format(
RuleField.on_success,
', '.join(RuleOutcome.values)))
# Validate the on_failure field of the enrichment rules.
outcome = rule.get(RuleField.on_failure)
if outcome and outcome not in RuleOutcome.values:
raise InvalidRuleError(
'The {0} field should be one of: {1}.'.format(
RuleField.on_failure,
', '.join(RuleOutcome.values)))
class engine(Engine):
etype = 'event_filter'
def __init__(self, *args, **kargs):
super(engine, self).__init__(*args, **kargs)
self.mg_store = MongoStore.get_default()
self.collection = MongoCollection(
self.mg_store.get_collection("object"))
self.name = kargs['name']
self.drop_event_count = 0
self.pass_event_count = 0
self.__load_rules()
def pre_run(self):
self.beat()
def a_override(self, event, action):
"""Override a field from event or add a new one if it does not have
one.
"""
afield = action.get('field', None)
avalue = action.get('value', None)
# This must be a hard check because value can be a boolean or a null
# integer
if afield is None or avalue is None:
self.logger.error(
"Malformed action ('field' and 'value' required): {}".format(
action))
return False
if afield not in event:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
# afield is in event
if not isinstance(avalue, list):
if isinstance(event[afield], list):
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
event[afield].append(avalue)
else:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
else:
# operation field is supported only for list values
op = action.get('operation', 'append')
if op == 'override':
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
elif op == 'append':
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
if isinstance(event[afield], list):
event[afield] += avalue
else:
event[afield] = [event[afield]] + avalue
return True
else:
self.logger.error(
"Operation '{}' unsupported (action '{}')".format(
op, action))
return False
def a_remove(self, event, action):
"""Remove an event from a field in event or the whole field if no
element is specified.
"""
akey = action.get('key', None)
aelement = action.get('element', None)
del_met = action.get('met', 0)
if akey:
if aelement:
if del_met:
for i, met in enumerate(event[akey]):
if met['name'] == aelement:
del event[akey][i]
break
elif isinstance(event[akey], dict):
del event[akey][aelement]
elif isinstance(event[akey], list):
del event[akey][event[akey].index(aelement)]
self.logger.debug(u" + {}: Removed: '{}' from '{}'".format(
event['rk'], aelement, akey))
else:
del event[akey]
self.logger.debug(u" + {}: Removed: '{}'".format(
event['rk'], akey))
return True
else:
self.logger.error(
u"Action malformed (needs 'key' and/or 'element'): {}".format(
action))
return False
def a_modify(self, event, action, name):
"""
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
derogated = False
atype = action.get('type')
actionMap = {'override': self.a_override, 'remove': self.a_remove}
if atype in actionMap:
derogated = actionMap[atype](event, action)
else:
self.logger.warning(u"Unknown action '{}'".format(atype))
# If the event was derogated, fill some informations
if derogated:
self.logger.debug(u"Event changed by rule '{}'".format(name))
return None
def a_drop(self, event, action, name):
""" Drop the event.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event dropped by rule '{}'".format(name))
self.drop_event_count += 1
return DROP
def a_pass(self, event, action, name):
"""Pass the event to the next queue.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event passed by rule '{}'".format(name))
self.pass_event_count += 1
return event
def a_route(self, event, action, name):
"""
Change the route to which an event will be sent
Args:
event: map of the event to be modified
action: map of type action
name: of the rule
Returns:
``None``
"""
if "route" in action:
self.next_amqp_queues = [action["route"]]
self.logger.debug(u"Event re-routed by rule '{}'".format(name))
else:
self.logger.error(
u"Action malformed (needs 'route'): {}".format(action))
return None
def a_exec_job(self, event, action, name):
records = self.collection.find({
'crecord_type': 'job',
'_id': action['job']
})
for record in records:
job = copy.deepcopy(record)
job['context'] = event
try:
self.work_amqp_publisher.direct_event(job, 'Engine_scheduler')
except Exception as e:
self.logger.exception("Unable to send job")
time.sleep(1)
return True
def a_snooze(self, event, action, name):
"""
Snooze event checks
:param dict event: event to be snoozed
:param dict action: action
:param str name: name of the rule
:returns: True if a snooze has been sent, False otherwise
:rtype: boolean
"""
if event.get('event_type') == 'snooze':
return False
# Only check events can trigger an auto-snooze
if event.get('event_type') != 'check':
return False
# A check OK cannot trigger an auto-snooze
if event.get('state') == 0:
return False
# Alerts manager caching
if not hasattr(self, 'am'):
self.am = Alerts(*Alerts.provide_default_basics())
# Context manager caching
if not hasattr(self, 'cm'):
self.cm = ContextGraph(self.logger)
entity_id = self.cm.get_id(event)
current_alarm = self.am.get_current_alarm(entity_id)
if current_alarm is None:
snooze = {
'connector': event.get('connector', ''),
'connector_name': event.get('connector_name', ''),
'source_type': event.get('source_type', ''),
'component': event.get('component', ''),
'event_type': 'snooze',
'duration': action['duration'],
'author': 'event_filter',
'output': 'Auto snooze generated by rule "{}"'.format(name),
'timestamp': int(time.time())
}
if event.get('resource', ''):
snooze['resource'] = event['resource']
try:
self.work_amqp_publisher.direct_event(snooze,
'Engine_event_filter')
except Exception as e:
self.logger.exception("Unable to send snooze event")
return True
return False
def a_baseline(self, event, actions, name):
"""a_baseline
:param event:
:param action: baseline conf in event filter
:param name:
"""
event['baseline_name'] = actions['baseline_name']
event['check_frequency'] = actions['check_frequency']
try:
self.work_amqp_publisher.direct_event(event, 'Engine_baseline')
except Exception as e:
self.logger.exception("Unable to send baseline event")
def apply_actions(self, event, actions):
pass_event = False
actionMap = {
'drop': self.a_drop,
'pass': self.a_pass,
'override': self.a_modify,
'remove': self.a_modify,
'execjob': self.a_exec_job,
'route': self.a_route,
'snooze': self.a_snooze,
'baseline': self.a_baseline
}
for name, action in actions:
if action['type'] in actionMap:
ret = actionMap[action['type'].lower()](event, action, name)
if ret:
pass_event = True
else:
self.logger.warning(u"Unknown action '{}'".format(action))
return pass_event
def work(self, event, *xargs, **kwargs):
rk = get_routingkey(event)
default_action = self.configuration.get('default_action', 'pass')
# list of supported actions
rules = self.configuration.get('rules', [])
to_apply = []
self.logger.debug(u'event {}'.format(event))
# When list configuration then check black and
# white lists depending on json configuration
for filterItem in rules:
actions = filterItem.get('actions')
name = filterItem.get('name', 'no_name')
self.logger.debug(u'rule {}'.format(filterItem))
self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
# Try filter rules on current event
if filterItem['mfilter'] and check(filterItem['mfilter'], event):
self.logger.debug(u'Event: {}, filter matches'.format(
event.get('rk', event)))
if 'pbehaviors' in filterItem:
pbehaviors = filterItem.get('pbehaviors', {})
list_in = pbehaviors.get('in', [])
list_out = pbehaviors.get('out', [])
if list_in or list_out:
pbm = singleton_per_scope(PBehaviorManager)
cm = singleton_per_scope(ContextGraph)
entity = cm.get_entity(event)
entity_id = cm.get_entity_id(entity)
result = pbm.check_pbehaviors(entity_id, list_in,
list_out)
if not result:
break
for action in actions:
if action['type'].lower() == 'drop':
self.apply_actions(event, to_apply)
return self.a_drop(event, None, name)
to_apply.append((name, action))
if filterItem.get('break', 0):
self.logger.debug(
u' + Filter {} broke the next filters processing'.
format(filterItem.get('name', 'filter')))
break
if len(to_apply):
if self.apply_actions(event, to_apply):
self.logger.debug(u'Event before sent to next engine: %s' %
event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
# No rules matched
if default_action == 'drop':
self.logger.debug("Event '%s' dropped by default action" % (rk))
self.drop_event_count += 1
return DROP
self.logger.debug("Event '%s' passed by default action" % (rk))
self.pass_event_count += 1
self.logger.debug(u'Event before sent to next engine: %s' % event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
def __load_rules(self):
tmp_rules = []
records = self.collection.find({
'crecord_type': 'filter',
'enable': True
})
records.sort('priority', 1)
for record in records:
record_dump = copy.deepcopy(record)
self.set_loaded(record_dump)
try:
record_dump["mfilter"] = loads(record_dump["mfilter"])
except Exception:
self.logger.info(u'Invalid mfilter {}, filter {}'.format(
record_dump['mfilter'],
record_dump['name'],
))
self.logger.debug(u'Loading record_dump:')
self.logger.debug(record_dump)
tmp_rules.append(record_dump)
self.configuration = {
'rules': tmp_rules,
'default_action': self.find_default_action()
}
def beat(self, *args, **kargs):
""" Configuration reload for realtime ui changes handling """
self.logger.debug(u'Reload configuration rules')
self.__load_rules()
self.logger.debug('Loaded {} rules'.format(
len(self.configuration['rules'])))
self.send_stat_event()
def set_loaded(self, record):
if 'run_once' in record and not record['run_once']:
self.collection.update({"_id": record['_id']},
{"$set": {
'run_once': True
}})
self.logger.info('record {} has been run once'.format(
record['_id']))
def send_stat_event(self):
""" Send AMQP Event for drop and pass metrics """
message_dropped = '{} event dropped since {}'.format(
self.drop_event_count, self.beat_interval)
message_passed = '{} event passed since {}'.format(
self.pass_event_count, self.beat_interval)
event = forger(connector='Engine',
connector_name='engine',
event_type='check',
source_type='resource',
resource=self.amqp_queue + '_data',
state=0,
state_type=1,
output=message_dropped,
perf_data_array=[{
'metric': 'pass_event',
'value': self.pass_event_count,
'type': 'GAUGE'
}, {
'metric': 'drop_event',
'value': self.drop_event_count,
'type': 'GAUGE'
}])
self.logger.debug(message_dropped)
self.logger.debug(message_passed)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send stat event")
self.drop_event_count = 0
self.pass_event_count = 0
def find_default_action(self):
"""Find the default action stored and returns it, else assume it
default action is pass.
"""
records = self.collection.find_one({'crecord_type': 'defaultrule'})
if records:
return records[0]["action"]
self.logger.debug(
"No default action found. Assuming default action is pass")
return 'pass'
class ViewAdapter(object):
"""
Adapter for the view collection.
"""
def __init__(self, logger):
self.logger = logger
self.view_collection = MongoCollection(
MongoStore.get_default().get_collection(VIEWS_COLLECTION))
self.group_collection = MongoCollection(
MongoStore.get_default().get_collection(GROUPS_COLLECTION))
def get_by_id(self, view_id):
"""
Get a view given its id.
:param str view_id: the id of the view.
"""
return self.view_collection.find_one({ViewField.id: view_id})
def create(self, view):
"""
Create a new view and return its id.
:param Dict[str, Any] view:
:rtype: str
"""
view_id = str(uuid4())
view[ViewField.id] = view_id
self.validate(view_id, view)
self.view_collection.insert(view)
return view_id
def update(self, view_id, view):
"""
Update a view given its id.
:param str view_id: the id of the view.
:param Dict[str, Any] view:
"""
self.validate(view_id, view)
self.view_collection.update({ViewField.id: view_id},
view,
upsert=False)
def remove_with_id(self, view_id):
"""
Remove a view given its id.
:param str view_id: the id of the view.
"""
self.view_collection.remove({ViewField.id: view_id})
def list(self, name=None, title=None):
"""
Return a list of views, optionally filtered by name or title.
:param str name:
:param str title:
:rtype: List[Dict[str, Any]]
:raises: InvalidFilterError
"""
view_filter = {}
if name is not None and title is not None:
raise InvalidFilterError(
'Cannot filter both by name and by title.')
if name is not None:
view_filter[ViewField.name] = name
if title is not None:
view_filter[ViewField.title] = title
views = self.view_collection.find(view_filter)
groups = self.group_collection.find({})
response_groups = {}
for group in groups:
group_id = group[GroupField.id]
del group[GroupField.id]
group[GroupField.views] = []
response_groups[group_id] = group
for view in views:
try:
group_id = view[ViewField.group_id]
except KeyError:
# This should never happen as long as the collections are only
# modified using the API
self.logger.exception(
'The view {0} is missing the group_id field.'.format(
view[ViewField.id]))
continue
try:
response_groups[group_id][GroupField.views].append(view)
except KeyError:
# This should never happen as long as the collections are only
# modified using the API
self.logger.exception('No group with id: {0}'.format(group_id))
return {ViewResponseField.groups: response_groups}
def validate(self, view_id, view):
"""
Check that the view is valid, return InvalidViewError if it is not.
:param Dict[str, Any] view:
"""
# Validate id field
if view.get(ViewField.id, view_id) != view_id:
raise InvalidViewError(
'The {0} field should not be modified'.format(ViewField.id))
# Validate group_id field
if ViewField.group_id not in view:
raise InvalidViewError('The view should have a group_id field.')
group_id = view[ViewField.group_id]
group = self.group_collection.find_one({GroupField.id: group_id})
if not group:
raise InvalidViewError('No group with id: {0}'.format(group_id))
# Validate name field
if ViewField.name not in view:
raise InvalidViewError('The view should have a name field.')
view_name = view[ViewField.name]
same_name_view = self.view_collection.find_one({
ViewField.id: {
'$ne': view_id
},
ViewField.name:
view_name
})
if same_name_view:
raise InvalidViewError(
'There is already a view with the name: {0}'.format(view_name))
# Validate title field
if ViewField.title not in view:
raise InvalidViewError('The view should have a title field.')
class GroupAdapter(object):
"""
Adapter for the group collection.
"""
def __init__(self, logger):
self.logger = logger
self.group_collection = MongoCollection(
MongoStore.get_default().get_collection(GROUPS_COLLECTION))
self.view_collection = MongoCollection(
MongoStore.get_default().get_collection(VIEWS_COLLECTION))
def get_by_id(self, group_id, name=None, title=None):
"""
Get a group given its id.
:param str group_id: the id of the group.
:param str name:
:param str title:
:rtype: Dict[str, Any]
"""
group = self.group_collection.find_one({GroupField.id: group_id})
if group:
group[GroupField.views] = self.get_views(group_id, name, title)
return group
def get_views(self, group_id, name=None, title=None):
"""
Returns the list of views of a group, optionally filtered by name or
title.
:param str group_id:
:param str name:
:param str title:
:rtype: List[Dict[str, Any]]
:raises: InvalidFilterError
"""
view_filter = {ViewField.group_id: group_id}
if name is not None and title is not None:
raise InvalidFilterError(
'Cannot filter both by name and by title.')
if name is not None:
view_filter[ViewField.name] = name
if title is not None:
view_filter[ViewField.title] = title
return list(self.view_collection.find(view_filter))
def is_empty(self, group_id):
"""
Return True if a group is empty.
:param str group_id:
:rtype: bool
"""
return self.view_collection.find({
ViewField.group_id: group_id
}).limit(1).count() == 0
def create(self, group):
"""
Create a new group.
:param Dict group:
:rtype: str
:raises: InvalidGroupError
"""
group_id = str(uuid4())
group[GroupField.id] = group_id
self.validate(group_id, group)
self.group_collection.insert(group)
return group_id
def update(self, group_id, group):
"""
Update a group given its id.
:param str group_id:
:param Dict group:
:raises: InvalidGroupError
"""
self.validate(group_id, group)
self.group_collection.update({GroupField.id: group_id},
group,
upsert=False)
def remove_with_id(self, group_id):
"""
Remove a group given its id.
:param str group_id:
:raises: NonEmptyGroupError
"""
if not self.is_empty(group_id):
raise NonEmptyGroupError()
self.group_collection.remove({GroupField.id: group_id})
def list(self, name=None):
"""
Return a list of groups, optionally filtered by name.
:param str name:
:rtype: List[Dict[str, Any]]
"""
group_filter = {}
if name is not None:
group_filter[GroupField.name] = name
return list(self.group_collection.find(group_filter))
def validate(self, group_id, group):
"""
Check that the gorup is valid, return InvalidGroupError if it is not.
:param Dict[str, Any] view:
:raises: InvalidGroupError
"""
if group.get(GroupField.id, group_id) != group_id:
raise InvalidGroupError(
'The {0} field should not be modified'.format(GroupField.id))
if GroupField.name not in group:
raise InvalidGroupError('The group should have a name field.')
group_name = group[GroupField.name]
same_name_group = self.group_collection.find_one({
GroupField.id: {
'$ne': group_id
},
GroupField.name:
group_name
})
if same_name_group:
raise InvalidGroupError(
'There is already a group with the name: {0}'.format(
group_name))
def exists(self, group_id):
"""
Return True if a group exists.
:param str group_id:
:rtype: bool
"""
group = self.group_collection.find_one({GroupField.id: group_id})
return group is not None
class engine(Engine):
etype = 'event_filter'
def __init__(self, *args, **kargs):
super(engine, self).__init__(*args, **kargs)
self.mg_store = MongoStore.get_default()
self.collection = MongoCollection(self.mg_store.get_collection("object"))
self.name = kargs['name']
self.drop_event_count = 0
self.pass_event_count = 0
self.__load_rules()
def pre_run(self):
self.beat()
def a_override(self, event, action):
"""Override a field from event or add a new one if it does not have
one.
"""
afield = action.get('field', None)
avalue = action.get('value', None)
# This must be a hard check because value can be a boolean or a null
# integer
if afield is None or avalue is None:
self.logger.error(
"Malformed action ('field' and 'value' required): {}".format(
action
)
)
return False
if afield not in event:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
# afield is in event
if not isinstance(avalue, list):
if isinstance(event[afield], list):
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
event[afield].append(avalue)
else:
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
else:
# operation field is supported only for list values
op = action.get('operation', 'append')
if op == 'override':
self.logger.debug("Overriding: '{}' -> '{}'".format(
afield, avalue))
event[afield] = avalue
return True
elif op == 'append':
self.logger.debug("Appending: '{}' to '{}'".format(
avalue, afield))
if isinstance(event[afield], list):
event[afield] += avalue
else:
event[afield] = [event[afield]] + avalue
return True
else:
self.logger.error(
"Operation '{}' unsupported (action '{}')".format(
op, action
)
)
return False
def a_remove(self, event, action):
"""Remove an event from a field in event or the whole field if no
element is specified.
"""
akey = action.get('key', None)
aelement = action.get('element', None)
del_met = action.get('met', 0)
if akey:
if aelement:
if del_met:
for i, met in enumerate(event[akey]):
if met['name'] == aelement:
del event[akey][i]
break
elif isinstance(event[akey], dict):
del event[akey][aelement]
elif isinstance(event[akey], list):
del event[akey][event[akey].index(aelement)]
self.logger.debug(u" + {}: Removed: '{}' from '{}'".format(
event['rk'],
aelement,
akey))
else:
del event[akey]
self.logger.debug(u" + {}: Removed: '{}'".format(
event['rk'],
akey))
return True
else:
self.logger.error(
u"Action malformed (needs 'key' and/or 'element'): {}".format(
action))
return False
def a_modify(self, event, action, name):
"""
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
derogated = False
atype = action.get('type')
actionMap = {
'override': self.a_override,
'remove': self.a_remove
}
if atype in actionMap:
derogated = actionMap[atype](event, action)
else:
self.logger.warning(u"Unknown action '{}'".format(atype))
# If the event was derogated, fill some informations
if derogated:
self.logger.debug(u"Event changed by rule '{}'".format(name))
return None
def a_drop(self, event, action, name):
""" Drop the event.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event dropped by rule '{}'".format(name))
self.drop_event_count += 1
return DROP
def a_pass(self, event, action, name):
"""Pass the event to the next queue.
Args:
event map of the event to be modified
action map of type action
_name of the rule
Returns:
``None``
"""
self.logger.debug(u"Event passed by rule '{}'".format(name))
self.pass_event_count += 1
return event
def a_route(self, event, action, name):
"""
Change the route to which an event will be sent
Args:
event: map of the event to be modified
action: map of type action
name: of the rule
Returns:
``None``
"""
if "route" in action:
self.next_amqp_queues = [action["route"]]
self.logger.debug(u"Event re-routed by rule '{}'".format(name))
else:
self.logger.error(
u"Action malformed (needs 'route'): {}".format(action))
return None
def a_exec_job(self, event, action, name):
records = self.collection.find(
{'crecord_type': 'job', '_id': action['job']}
)
for record in records:
job = copy.deepcopy(record)
job['context'] = event
try:
self.work_amqp_publisher.direct_event(job, 'Engine_scheduler')
except Exception as e:
self.logger.exception("Unable to send job")
time.sleep(1)
return True
def a_snooze(self, event, action, name):
"""
Snooze event checks
:param dict event: event to be snoozed
:param dict action: action
:param str name: name of the rule
:returns: True if a snooze has been sent, False otherwise
:rtype: boolean
"""
if event.get('event_type') == 'snooze':
return False
# Only check events can trigger an auto-snooze
if event.get('event_type') != 'check':
return False
# A check OK cannot trigger an auto-snooze
if event.get('state') == 0:
return False
# Alerts manager caching
if not hasattr(self, 'am'):
self.am = Alerts(*Alerts.provide_default_basics())
# Context manager caching
if not hasattr(self, 'cm'):
self.cm = ContextGraph(self.logger)
entity_id = self.cm.get_id(event)
current_alarm = self.am.get_current_alarm(entity_id)
if current_alarm is None:
snooze = {
'connector': event.get('connector', ''),
'connector_name': event.get('connector_name', ''),
'source_type': event.get('source_type', ''),
'component': event.get('component', ''),
'event_type': 'snooze',
'duration': action['duration'],
'author': 'event_filter',
'output': 'Auto snooze generated by rule "{}"'.format(name),
'timestamp': int(time.time())
}
if event.get('resource', ''):
snooze['resource'] = event['resource']
try:
self.work_amqp_publisher.direct_event(
snooze, 'Engine_event_filter')
except Exception as e:
self.logger.exception("Unable to send snooze event")
return True
return False
def a_baseline(self, event, actions, name):
"""a_baseline
:param event:
:param action: baseline conf in event filter
:param name:
"""
event['baseline_name'] = actions['baseline_name']
event['check_frequency'] = actions['check_frequency']
try:
self.work_amqp_publisher.direct_event(
event, 'Engine_baseline')
except Exception as e:
self.logger.exception("Unable to send baseline event")
def apply_actions(self, event, actions):
pass_event = False
actionMap = {
'drop': self.a_drop,
'pass': self.a_pass,
'override': self.a_modify,
'remove': self.a_modify,
'execjob': self.a_exec_job,
'route': self.a_route,
'snooze': self.a_snooze,
'baseline': self.a_baseline
}
for name, action in actions:
if action['type'] in actionMap:
ret = actionMap[action['type'].lower()](event, action, name)
if ret:
pass_event = True
else:
self.logger.warning(u"Unknown action '{}'".format(action))
return pass_event
def work(self, event, *xargs, **kwargs):
rk = get_routingkey(event)
default_action = self.configuration.get('default_action', 'pass')
# list of supported actions
rules = self.configuration.get('rules', [])
to_apply = []
self.logger.debug(u'event {}'.format(event))
# When list configuration then check black and
# white lists depending on json configuration
for filterItem in rules:
actions = filterItem.get('actions')
name = filterItem.get('name', 'no_name')
self.logger.debug(u'rule {}'.format(filterItem))
self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
# Try filter rules on current event
if filterItem['mfilter'] and check(filterItem['mfilter'], event):
self.logger.debug(
u'Event: {}, filter matches'.format(event.get('rk', event))
)
if 'pbehaviors' in filterItem:
pbehaviors = filterItem.get('pbehaviors', {})
list_in = pbehaviors.get('in', [])
list_out = pbehaviors.get('out', [])
if list_in or list_out:
pbm = singleton_per_scope(PBehaviorManager)
cm = singleton_per_scope(ContextGraph)
entity = cm.get_entity(event)
entity_id = cm.get_entity_id(entity)
result = pbm.check_pbehaviors(
entity_id, list_in, list_out
)
if not result:
break
for action in actions:
if action['type'].lower() == 'drop':
self.apply_actions(event, to_apply)
return self.a_drop(event, None, name)
to_apply.append((name, action))
if filterItem.get('break', 0):
self.logger.debug(
u' + Filter {} broke the next filters processing'
.format(
filterItem.get('name', 'filter')
)
)
break
if len(to_apply):
if self.apply_actions(event, to_apply):
self.logger.debug(
u'Event before sent to next engine: %s' % event
)
event['rk'] = event['_id'] = get_routingkey(event)
return event
# No rules matched
if default_action == 'drop':
self.logger.debug("Event '%s' dropped by default action" % (rk))
self.drop_event_count += 1
return DROP
self.logger.debug("Event '%s' passed by default action" % (rk))
self.pass_event_count += 1
self.logger.debug(u'Event before sent to next engine: %s' % event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
def __load_rules(self):
tmp_rules = []
records = self.collection.find(
{'crecord_type': 'filter', 'enable': True})
records.sort('priority', 1)
for record in records:
record_dump = copy.deepcopy(record)
self.set_loaded(record_dump)
try:
record_dump["mfilter"] = loads(record_dump["mfilter"])
except Exception:
self.logger.info(u'Invalid mfilter {}, filter {}'.format(
record_dump['mfilter'],
record_dump['name'],
))
self.logger.debug(u'Loading record_dump:')
self.logger.debug(record_dump)
tmp_rules.append(record_dump)
self.configuration = {
'rules': tmp_rules,
'default_action': self.find_default_action()
}
def beat(self, *args, **kargs):
""" Configuration reload for realtime ui changes handling """
self.logger.debug(u'Reload configuration rules')
self.__load_rules()
self.logger.debug(
'Loaded {} rules'.format(len(self.configuration['rules']))
)
self.send_stat_event()
def set_loaded(self, record):
if 'run_once' in record and not record['run_once']:
self.collection.update({"_id": record['_id']}, {"$set": {'run_once': True}})
self.logger.info(
'record {} has been run once'.format(record['_id'])
)
def send_stat_event(self):
""" Send AMQP Event for drop and pass metrics """
message_dropped = '{} event dropped since {}'.format(
self.drop_event_count,
self.beat_interval
)
message_passed = '{} event passed since {}'.format(
self.pass_event_count,
self.beat_interval
)
event = forger(
connector='Engine',
connector_name='engine',
event_type='check',
source_type='resource',
resource=self.amqp_queue + '_data',
state=0,
state_type=1,
output=message_dropped,
perf_data_array=[
{'metric': 'pass_event',
'value': self.pass_event_count,
'type': 'GAUGE'},
{'metric': 'drop_event',
'value': self.drop_event_count,
'type': 'GAUGE'}
]
)
self.logger.debug(message_dropped)
self.logger.debug(message_passed)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send stat event")
self.drop_event_count = 0
self.pass_event_count = 0
def find_default_action(self):
"""Find the default action stored and returns it, else assume it
default action is pass.
"""
records = self.collection.find_one({'crecord_type': 'defaultrule'})
if records:
return records[0]["action"]
self.logger.debug(
"No default action found. Assuming default action is pass"
)
return 'pass'
