def __init__(self, logger, config, storage, pbehavior_manager): """ :param logger: a logger object :param config: a confng instance :param storage: a storage instance :param pbehavior_manager: a pbehavior manager instance """ self.logger = logger self.config = config self.alarm_storage = storage self.alarm_collection = MongoCollection(self.alarm_storage._backend) self.pbehavior_manager = pbehavior_manager self.pbh_filter = None category = self.config.get(self.CATEGORY, {}) self.expiration = int(category.get('expiration', DEFAULT_EXPIRATION)) self.opened_truncate = cfg_to_bool(category.get('opened_truncate', DEFAULT_OPENED_TRUNC)) self.opened_limit = int(category.get('opened_limit', DEFAULT_OPENED_LIMIT)) self.resolved_truncate = cfg_to_bool(category.get('resolved_truncate', DEFAULT_RESOLVED_TRUNC)) self.resolved_limit = int(category.get('resolved_limit', DEFAULT_RESOLVED_LIMIT)) self.count_cache = {} self.grammar = join(root_path, self.GRAMMAR_FILE) self.has_active_pbh = None
def setUp(self): output = StringIO() self.logger = Logger.get('test', output, OutputStream) store = MongoStore.get_default() self.collection = store.get_collection(name='default_test') self.mongo_collection = MongoCollection(collection=self.collection, logger=self.logger) # Cleanup self.tearDown() self.manager = ActionManager(logger=self.logger, mongo_collection=self.mongo_collection) self.id_ = 'testid' self.action = { "_id": self.id_, "hook": None, "type": "pbehavior", "fields": ["Resource"], "regex": ".*wine.*", "parameters": { "author": "Matho", "name": "Salammbo", "reason": "Madness", "type": "Mercenary War", "rrule": "" }, "delay": "" }
def fill(self, init=None, yes=False, reinit_auth=False): self.__put_canopsis_document() tools = [] for module in self.modules: try: migrationcls = lookup(module) except ImportError as err: self.logger.error( 'Impossible to load module "{0}": {1}'.format(module, err)) continue migrationtool = migrationcls() migrationtool.logger.addHandler(self.loghandler) tools.append(migrationtool) coll = None if init is None: store = MongoStore.get_default() store.authenticate() coll = MongoCollection(store.get_collection(self.FLAG_COLLECTION)) data = coll.find_one({"_id": self.FLAG_COLLECTION}) if data is None: print("Database not intialized. Initializing...") init = True else: print("Database already intialized. Updating...") init = False if init is None and reinit_auth is False: data = { "_id": "initialized", "at": str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")) } print("The canopsis initialization flag did not exist in the " "database. So canopsinit will (re?)initialized the " "database. Meaning, it may delete some important data " "from canopsis database. If you still want to initialize " "the database, call the same command with the " "`--authorize-reinit` flag. Or if you do not want to " "initialize the database, add the document `{0}` in the {1} " "collections.".format(data, self.FLAG_COLLECTION)) exit(1) for tool in tools: if init: tool.init(yes=yes) else: tool.update(yes=yes) if init is True: coll.insert({ "_id": self.FLAG_COLLECTION, "at": str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")) })
class DynamicInfosManagerTest(unittest.TestCase): def setUp(self): mongo = MongoStore.get_default() collection = mongo.get_collection("test_dynamic_infos") self.dynamic_coll = MongoCollection(collection) logger = Logger.get('test_dynamic_infos', None, output_cls=OutputNull) self.dynamic_infos = DynamicInfosManager( logger=logger, mongo_collection=self.dynamic_coll) self.dynamic_coll.drop() self.dynamic_infos_doc = { "_id": "rule2", "name": "Test", "author": "billy", "creation_date": 1576260000, "last_modified_date": 1576260000, "description": "Freedom !", "infos": [{ "name": "info", "value": "value" }, { "name": "info2", "value": "value2" }], "entity_patterns": [{ "_id": "cpu/billys-laptop" }], "alarm_patterns": [{ "v": { "state": { "val": 3 } } }] } def tearDown(self): self.dynamic_coll.drop() def test_count(self): rule = DynamicInfosRule.new_from_dict(self.dynamic_infos_doc, "test_author", 1583301306) self.dynamic_infos.create(rule) count = self.dynamic_infos.count() self.assertEqual(count, 1) count = self.dynamic_infos.count(search="test_author", search_fields=["author"]) self.assertEqual(count, 1) count = self.dynamic_infos.count(search="test_author", search_fields=["description"]) self.assertEqual(count, 0) count = self.dynamic_infos.count(search="Test", search_fields=["name"]) self.assertEqual(count, 1)
def __init__(self, logger, collection): """ :param `~.logger.Logger` logger: object. :param `~.common.collection.MongoCollection` collection: object. """ self.__logger = logger self.__collection = MongoCollection(collection)
def setUp(self): self.storage = Middleware.get_middleware_by_uri( 'mongodb-default-testsession://') self.collection = MongoCollection(self.storage._backend) self.manager = Session(collection=self.collection) self.user = '******'
def test_is_successfull(self): dico = {'ok': 1.0, 'n': 2} self.assertTrue(MongoCollection.is_successfull(dico)) dico = {'ok': 666.667, 'n': 1} self.assertFalse(MongoCollection.is_successfull(dico)) dico = {'n': 2} self.assertFalse(MongoCollection.is_successfull(dico))
class TestActionManager(unittest.TestCase): def setUp(self): output = StringIO() self.logger = Logger.get('test', output, OutputStream) store = MongoStore.get_default() self.collection = store.get_collection(name='default_test') self.mongo_collection = MongoCollection(collection=self.collection, logger=self.logger) # Cleanup self.tearDown() self.manager = ActionManager(logger=self.logger, mongo_collection=self.mongo_collection) self.id_ = 'testid' self.action = { "_id": self.id_, "type": "pbehavior", "fields": ["Resource"], "regex": ".*wine.*", "parameters": { "author": "Matho", "name": "Salammbo", "reason": "Madness", "type": "Mercenary War", "rrule": "" } } def tearDown(self): """Teardown""" self.mongo_collection.remove({}) def test_crud(self): res = self.manager.create(action=self.action) self.assertTrue(res) res = self.manager.get_id(self.id_) self.assertIsNotNone(res) self.assertDictEqual(res.to_dict(), self.action) action2 = self.action.copy() action2[Action.FIELDS] = ['Component'] res = self.manager.update_id(id_=self.id_, action=action2) self.assertTrue(res) res = self.manager.get_id(self.id_) self.assertIsNotNone(res) self.assertDictEqual(res.to_dict(), action2) res = self.manager.delete_id(id_=self.id_) self.assertTrue(res) res = self.manager.get_id(self.id_) self.assertIsNone(res)
def __init__(self, *args, **kargs): super(engine, self).__init__(*args, **kargs) self.mg_store = MongoStore.get_default() self.collection = MongoCollection( self.mg_store.get_collection("object")) self.name = kargs['name'] self.drop_event_count = 0 self.pass_event_count = 0 self.__load_rules()
class CanopsisVersionManager(object): """ Canopsis version manager abstraction. """ COLLECTION = "configuration" EDITION_FIELD = "edition" STACK_FIELD = "stack" VERSION_FIELD = "version" __DOCUMENT_ID = "canopsis_version" def __init__(self, collection): """ :param collection: `pymongo.collection.Collection` object. """ self.__collection = MongoCollection(collection) def find_canopsis_document(self): """ Find Canopsis version document. :returns: Canopsis version document or None if not found. :raises: (`pymongo.errors.PyMongoError`, ). """ return self.__collection.find_one({'_id': self.__DOCUMENT_ID}) def put_canopsis_document(self, edition, stack, version): """ Put Canopsis version document (upsert). :param version: `str` Canopsis version. :raises: (`canopsis.common.collection.CollectionError`, ). """ document = {} if edition is not None: document[self.EDITION_FIELD] = edition if stack is not None: document[self.STACK_FIELD] = stack if version is not None: document[self.VERSION_FIELD] = version if len(document) > 0: resp = self.__collection.update({'_id': self.__DOCUMENT_ID}, {'$set': document}, upsert=True) return self.__collection.is_successfull(resp) return True
class SessionManagerTest(TestCase): def setUp(self): self.storage = Middleware.get_middleware_by_uri( 'mongodb-default-testsession://' ) self.collection = MongoCollection(self.storage._backend) self.manager = Session(collection=self.collection) self.user = '******' def tearDown(self): self.collection.remove() def test_keep_alive(self): self.manager.session_start(self.user) sleep(1) got = self.manager.keep_alive(self.user) session = self.collection.find_one({'_id': self.user}) self.assertTrue(isinstance(session, dict)) self.assertEqual(got, session['last_check']) def test_session_start(self): got = self.manager.session_start(self.user) session = self.collection.find_one({'_id': self.user}) self.assertTrue(isinstance(session, dict)) self.assertTrue(session['active']) self.assertEqual(got, session['session_start']) def test_session_start_already_started(self): self.test_session_start() got = self.manager.session_start(self.user) self.assertTrue(got is None) def test_is_session_active(self): self.assertFalse(self.manager.is_session_active(self.user)) self.manager.session_start(self.user) self.assertTrue(self.manager.is_session_active(self.user)) def test_sessions_close(self): got = self.manager.session_start(self.user) self.manager.alive_session_duration = 0 self.assertTrue(got is not None) sessions = self.manager.sessions_close() self.assertTrue(len(sessions) > 0) self.assertEqual(got, sessions[0]['last_check'])
def setUp(self): output = StringIO() self.logger = Logger.get('test', output, OutputStream) self.storage = Middleware.get_middleware_by_uri( 'storage-default-testmongocollection://') self.collection = MongoCollection(collection=self.storage._backend, logger=self.logger) self.id_ = 'testid'
class SessionManagerTest(TestCase): def setUp(self): self.storage = Middleware.get_middleware_by_uri( 'mongodb-default-testsession://') self.collection = MongoCollection(self.storage._backend) self.manager = Session(collection=self.collection) self.user = '******' def tearDown(self): self.collection.remove() def test_keep_alive(self): self.manager.session_start(self.user) sleep(1) got = self.manager.keep_alive(self.user) session = self.collection.find_one({'_id': self.user}) self.assertTrue(isinstance(session, dict)) self.assertEqual(got, session['last_check']) def test_session_start(self): got = self.manager.session_start(self.user) session = self.collection.find_one({'_id': self.user}) self.assertTrue(isinstance(session, dict)) self.assertTrue(session['active']) self.assertEqual(got, session['session_start']) def test_session_start_already_started(self): self.test_session_start() got = self.manager.session_start(self.user) self.assertTrue(got is None) def test_is_session_active(self): self.assertFalse(self.manager.is_session_active(self.user)) self.manager.session_start(self.user) self.assertTrue(self.manager.is_session_active(self.user)) def test_sessions_close(self): got = self.manager.session_start(self.user) self.manager.alive_session_duration = 0 self.assertTrue(got is not None) sessions = self.manager.sessions_close() self.assertTrue(len(sessions) > 0) self.assertEqual(got, sessions[0]['last_check'])
def __init__(self, logger, pb_storage): """ :param dict config: configuration :param pb_storage: PBehavior Storage object """ super(PBehaviorManager, self).__init__() kwargs = {"logger": logger} self.context = singleton_per_scope(ContextGraph, kwargs=kwargs) self.logger = logger self.pb_storage = pb_storage self.pb_store = MongoCollection( MongoStore.get_default().get_collection('default_pbehavior')) self.currently_active_pb = set()
def test_remove(self): res = self.collection.insert(document={ '_id': self.id_, 'top': 'bottom' }) self.assertIsNotNone(res) res = self.collection.remove(query={'_id': self.id_}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 1) # Deleting non-existing object doesn't throw error res = self.collection.remove(query={}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 0)
def test_update(self): res = self.collection.update(query={'_id': self.id_}, document={'strange': 'charm'}) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 0) res = self.collection.update(query={'_id': self.id_}, document={'yin': 'yang'}, upsert=True) self.assertTrue(MongoCollection.is_successfull(res)) self.assertEqual(res['n'], 1) res = self.collection.find_one(self.id_) self.assertEqual(res['yin'], 'yang') self.assertTrue('strange' not in res)
def setUp(self): output = StringIO() self.logger = Logger.get('test', output, OutputStream) store = MongoStore.get_default() self.collection = store.get_collection(name='default_test') self.mongo_collection = MongoCollection( collection=self.collection, logger=self.logger ) # Cleanup self.tearDown() self.manager = ActionManager( logger=self.logger, mongo_collection=self.mongo_collection ) self.id_ = 'testid' self.action = { "_id": self.id_, "type": "pbehavior", "fields": ["Resource"], "regex": ".*wine.*", "parameters": { "author": "Matho", "name": "Salammbo", "reason": "Madness", "type": "Mercenary War", "rrule": "" } }
def setUp(self): super(TestReader, self).setUp() mongo = MongoStore.get_default() collection = mongo.get_collection("default_testpbehavior") pb_coll = MongoCollection(collection) self.logger = Logger.get('alertsreader', '/tmp/null') conf = Configuration.load(PBehaviorManager.CONF_PATH, Ini) self.pbehavior_manager = PBehaviorManager(config=conf, logger=self.logger, pb_collection=pb_coll) self.reader = AlertsReader(config=conf, logger=self.logger, storage=self.manager.alerts_storage, pbehavior_manager=self.pbehavior_manager) self.reader._alarm_fields = { 'properties': { 'connector': { 'stored_name': 'v.ctr' }, 'component': { 'stored_name': 'v.cpt' }, 'entity_id': { 'stored_name': 'd' } } }
class CanopsisVersionManager(object): """ Canopsis version manager abstraction. """ COLLECTION = "configuration" VERSION_FIELD = "version" __DOCUMENT_ID = "canopsis_version" def __init__(self, collection): """ :param collection: `pymongo.collection.Collection` object. """ self.__collection = MongoCollection(collection) def find_canopsis_version_document(self): """ Find Canopsis version document. :returns: Canopsis version document or None if not found. :raises: (`pymongo.errors.PyMongoError`, ). """ return self.__collection.find_one({ '_id': self.__DOCUMENT_ID }) def put_canopsis_version_document(self, version): """ Put Canopsis version document (upsert). :param version: `str` Canopsis version. :raises: (`canopsis.common.collection.CollectionError`, ). """ self.__collection.update( { '_id': self.__DOCUMENT_ID }, { '_id': self.__DOCUMENT_ID, self.VERSION_FIELD: version }, upsert=True )
def setUp(self): self.storage = Middleware.get_middleware_by_uri( 'mongodb-default-testsession://') self.collection = MongoCollection(self.storage._backend) self.manager = Session(collection=self.collection) self.user = '******' self.id_beaker_session = 'cm9vdF8xNTc2MDY1MzY2' self.path = [ "view/da7ac9b9-db1c-4435-a1f2-edb4d6be4db8", "view-tab_edd5855b-54f1-4c51-9550-d88c2da60768" ] self.path_bis = [ "view/da7ac9b9-db1c-4435-a1f2-edb4d6be4db8", "view-tab_edd5855b-54f1-4c51-azerty" ]
def __init__(self, *args, **kargs): super(engine, self).__init__(*args, **kargs) self.mg_store = MongoStore.get_default() self.collection = MongoCollection(self.mg_store.get_collection("object")) self.name = kargs['name'] self.drop_event_count = 0 self.pass_event_count = 0 self.__load_rules()
def setUp(self): self.storage = Middleware.get_middleware_by_uri( 'mongodb-default-testsession://' ) self.collection = MongoCollection(self.storage._backend) self.manager = Session(collection=self.collection) self.user = '******'
def provide_default_basics(cls): """ Provide mongo collection. ! Do not use in tests ! :rtype: `~.common.collection.MongoCollection`. """ store = MongoStore.get_default() return (MongoCollection(store.get_collection(cls.COLLECTION)), )
def default_collection(cls): """ Returns the default collection for the manager. ! Do not use in tests ! :rtype: canopsis.common.collection.MongoCollection """ store = MongoStore.get_default() collection = store.get_collection(name=cls.COLLECTION) return MongoCollection(collection)
def __init__(self, config, logger, pb_storage): """ :param dict config: configuration :param pb_storage: PBehavior Storage object """ super(PBehaviorManager, self).__init__() kwargs = {"logger": logger} self.context = singleton_per_scope(ContextGraph, kwargs=kwargs) self.logger = logger self.pb_storage = pb_storage self.config = config self.config_data = self.config.get(self.PBH_CAT, {}) self.default_tz = self.config_data.get("default_timezone", "Europe/Paris") # this line allow us to raise an exception pytz.UnknownTimeZoneError, # if the timezone defined in the pbehabior configuration file is wrong pytz.timezone(self.default_tz) self.pb_store = MongoCollection( MongoStore.get_default().get_collection('default_pbehavior')) self.currently_active_pb = set()
def setUp(self): mongo = MongoStore.get_default() collection = mongo.get_collection("test_dynamic_infos") self.dynamic_coll = MongoCollection(collection) logger = Logger.get('test_dynamic_infos', None, output_cls=OutputNull) self.dynamic_infos = DynamicInfosManager( logger=logger, mongo_collection=self.dynamic_coll) self.dynamic_coll.drop() self.dynamic_infos_doc = { "_id": "rule2", "name": "Test", "author": "billy", "creation_date": 1576260000, "last_modified_date": 1576260000, "description": "Freedom !", "infos": [{ "name": "info", "value": "value" }, { "name": "info2", "value": "value2" }], "entity_patterns": [{ "_id": "cpu/billys-laptop" }], "alarm_patterns": [{ "v": { "state": { "val": 3 } } }] }
def check_db(self): """ Check if database service is available. :rtype: ServiceState """ existing_cols = self.db_store.client.collection_names() for collection_name in self.CHECK_COLLECTIONS: # Existence test if collection_name not in existing_cols: msg = 'Missing collection {}'.format(collection_name) return ServiceState(message=msg) # Read test collection = self.db_store.get_collection(name=collection_name) mongo_collection = MongoCollection(collection) try: mongo_collection.find({}, limit=1) except Exception as exc: return ServiceState(message='Find error: {}'.format(exc)) return ServiceState()
def provide_default_basics(cls, logger): """ Returns the default collection for the manager. ! Do not use in tests ! :rtype: (canopsis.common.collection.MongoCollection, canopsis.common.amqp.AmqpPublisher) """ store = MongoStore.get_default() collection = store.get_collection(name=cls.COLLECTION) amqp_pub = AmqpPublisher(get_default_amqp_conn(), logger) return (MongoCollection(collection), amqp_pub)
def provide_default_basics(cls): """ Provide logger, config, storages... ! Do not use in tests ! :rtype: Union[logging.Logger, canopsis.common.collection.MongoCollection] """ store = MongoStore.get_default() collection = store.get_collection(name=cls.COLLECTION) return (Logger.get('action', cls.LOG_PATH), MongoCollection(collection))
def provide_default_basics(cls): """Provide logger and collection. ! Do not use in tests ! :rtype: Tuple[logging.Logger, canopsis.common.collection.MongoCollection] """ logger = Logger.get('dynamic_infos', cls.LOG_PATH) store = MongoStore.get_default() collection = store.get_collection(name=cls.COLLECTION) mongo_collection = MongoCollection(collection) return (logger, mongo_collection)
def provide_default_basics(cls): """ Provide logger, config, storages... ! Do not use in tests ! :rtype: Union[logging.Logger, canopsis.common.collection.MongoCollection] """ logger = Logger.get('ticketapi', cls.LOG_PATH) store = MongoStore.get_default() collection = store.get_collection(name=cls.ACTION_COLLECTION) mongo_collection = MongoCollection(collection) return (logger, mongo_collection)
def setUp(self): mongo = MongoStore.get_default() collection = mongo.get_collection("default_testpbehavior") pb_coll = MongoCollection(collection) entities_storage = Middleware.get_middleware_by_uri( 'storage-default-testentities://') logger = Logger.get('test_pb', None, output_cls=OutputNull) conf = Configuration.load(PBehaviorManager.CONF_PATH, Ini) self.pbm = PBehaviorManager(config=conf, logger=logger, pb_collection=pb_coll) self.context = ContextGraph(logger) self.context.ent_storage = entities_storage self.pbm.context = self.context
def get_backend(self, namespace=None): self.check_connected() if not namespace: namespace = self.namespace try: backend = self.backend[namespace] self.logger.debug("Use %s collection" % namespace) return backend except Exception: self.backend[namespace] = MongoCollection(self.conn.get_collection(namespace)) self.logger.debug("Connected to %s collection." % namespace) return self.backend[namespace]
def provide_default_basics(cls): """ Provide the default configuration and logger objects for MetaAlarmRuleManager. Do not use those defaults for tests. :return: config, logger, storage :rtype: Union[dict, logging.Logger, canopsis.storage.core.Storage] """ logger = Logger.get('metaalarmrule', cls.LOG_PATH) mongo = MongoStore.get_default() collection = mongo.get_collection(cls.MA_RULE_COLLECTION) mongo_collection = MongoCollection(collection) return logger, mongo_collection
def __init__(self, config, logger, pb_storage): """ :param dict config: configuration :param pb_storage: PBehavior Storage object """ super(PBehaviorManager, self).__init__() kwargs = {"logger": logger} self.context = singleton_per_scope(ContextGraph, kwargs=kwargs) self.logger = logger self.pb_storage = pb_storage self.config = config self.config_data = self.config.get(self.PBH_CAT, {}) self.default_tz = self.config_data.get("default_timezone", "Europe/Paris") # this line allow us to raise an exception pytz.UnknownTimeZoneError, # if the timezone defined in the pbehabior configuration file is wrong pytz.timezone(self.default_tz) self.pb_store = MongoCollection(MongoStore.get_default().get_collection('default_pbehavior')) self.currently_active_pb = set()
class TestActionManager(unittest.TestCase): def setUp(self): output = StringIO() self.logger = Logger.get('test', output, OutputStream) store = MongoStore.get_default() self.collection = store.get_collection(name='default_test') self.mongo_collection = MongoCollection( collection=self.collection, logger=self.logger ) # Cleanup self.tearDown() self.manager = ActionManager( logger=self.logger, mongo_collection=self.mongo_collection ) self.id_ = 'testid' self.action = { "_id": self.id_, "type": "pbehavior", "fields": ["Resource"], "regex": ".*wine.*", "parameters": { "author": "Matho", "name": "Salammbo", "reason": "Madness", "type": "Mercenary War", "rrule": "" } } def tearDown(self): """Teardown""" self.mongo_collection.remove({}) def test_crud(self): res = self.manager.create(action=self.action) self.assertTrue(res) res = self.manager.get_id(self.id_) self.assertIsNotNone(res) self.assertDictEqual(res.to_dict(), self.action) action2 = self.action.copy() action2[Action.FIELDS] = ['Component'] res = self.manager.update_id(id_=self.id_, action=action2) self.assertTrue(res) res = self.manager.get_id(self.id_) self.assertIsNotNone(res) self.assertDictEqual(res.to_dict(), action2) res = self.manager.delete_id(id_=self.id_) self.assertTrue(res) res = self.manager.get_id(self.id_) self.assertIsNone(res)
class PBehaviorManager(object): """ PBehavior manager class. """ PB_STORAGE_URI = 'mongodb-default-pbehavior://' LOG_PATH = 'var/log/pbehaviormanager.log' LOG_NAME = 'pbehaviormanager' CONF_PATH = 'etc/pbehavior/manager.conf' PBH_CAT = "PBEHAVIOR" _UPDATE_FLAG = 'updatedExisting' __TYPE_ERR = "id_ must be a list of string or a string" @classmethod def provide_default_basics(cls): """ Provide the default configuration and logger objects for PBehaviorManager. Do not use those defaults for tests. :return: config, logger, storage :rtype: Union[dict, logging.Logger, canopsis.storage.core.Storage] """ logger = Logger.get(cls.LOG_NAME, cls.LOG_PATH) pb_storage = Middleware.get_middleware_by_uri(cls.PB_STORAGE_URI) config = Configuration.load(PBehaviorManager.CONF_PATH, Ini) return config, logger, pb_storage def __init__(self, config, logger, pb_storage): """ :param dict config: configuration :param pb_storage: PBehavior Storage object """ super(PBehaviorManager, self).__init__() kwargs = {"logger": logger} self.context = singleton_per_scope(ContextGraph, kwargs=kwargs) self.logger = logger self.pb_storage = pb_storage self.config = config self.config_data = self.config.get(self.PBH_CAT, {}) self.default_tz = self.config_data.get("default_timezone", "Europe/Paris") # this line allow us to raise an exception pytz.UnknownTimeZoneError, # if the timezone defined in the pbehabior configuration file is wrong pytz.timezone(self.default_tz) self.pb_store = MongoCollection(MongoStore.get_default().get_collection('default_pbehavior')) self.currently_active_pb = set() def get(self, _id, query=None): """Get pbehavior by id. :param str id: pbehavior id :param dict query: filtering options """ return self.pb_storage.get_elements(ids=_id, query=query) def create( self, name, filter, author, tstart, tstop, rrule='', enabled=True, comments=None, connector='canopsis', connector_name='canopsis', type_=PBehavior.DEFAULT_TYPE, reason='', timezone=None, exdate=None): """ Method creates pbehavior record :param str name: filtering options :param dict filter: a mongo filter that match entities from canopsis context :param str author: the name of the user/app that has generated the pbehavior :param timestamp tstart: timestamp that correspond to the start of the pbehavior :param timestamp tstop: timestamp that correspond to the end of the pbehavior :param str rrule: reccurent rule that is compliant with rrule spec :param bool enabled: boolean to know if pbhevior is enabled or disabled :param list of dict comments: a list of comments made by users :param str connector: a string representing the type of connector that has generated the pbehavior :param str connector_name: a string representing the name of connector that has generated the pbehavior :param str type_: associated type_ for this pbh :param str reason: associated reason for this pbh :param str timezone: the timezone of the new pbehabior. If no timezone are given, use the default one. See the pbehavior documentation for more information. :param list of str| str exdate: a list of string representation of a date following this pattern "YYYY/MM/DD HH:MM:00 TIMEZONE". The hour use the 24 hours clock system and the timezone is the name of the timezone. The month, the day of the month, the hour, the minute and second are zero-padded. :raises ValueError: invalid RRULE :raises pytz.UnknownTimeZoneError: invalid timezone :return: created element eid :rtype: str """ if timezone is None: timezone = self.default_tz if exdate is None: exdate = [] # this line allow us to raise an exception pytz.UnknownTimeZoneError, # if the timezone defined in the pbehabior configuration file is wrong pytz.timezone(timezone) if enabled in [True, "True", "true"]: enabled = True elif enabled in [False, "False", "false"]: enabled = False else: raise ValueError("The enabled value does not match a boolean") if not isinstance(exdate, list): exdate = [exdate] check_valid_rrule(rrule) if comments is not None: for comment in comments: if "author" in comment: if not isinstance(comment["author"], string_types): raise ValueError("The author field must be an string") else: raise ValueError("The author field is missing") if "message" in comment: if not isinstance(comment["message"], string_types): raise ValueError("The message field must be an string") else: raise ValueError("The message field is missing") pb_kwargs = { PBehavior.NAME: name, PBehavior.FILTER: filter, PBehavior.AUTHOR: author, PBehavior.TSTART: tstart, PBehavior.TSTOP: tstop, PBehavior.RRULE: rrule, PBehavior.ENABLED: enabled, PBehavior.COMMENTS: comments, PBehavior.CONNECTOR: connector, PBehavior.CONNECTOR_NAME: connector_name, PBehavior.TYPE: type_, PBehavior.REASON: reason, PBehavior.TIMEZONE: timezone, PBehavior.EXDATE: exdate, PBehavior.EIDS: [] } data = PBehavior(**pb_kwargs) if not data.comments or not isinstance(data.comments, list): data.update(comments=[]) else: for comment in data.comments: comment.update({'_id': str(uuid4())}) result = self.pb_storage.put_element(element=data.to_dict()) return result def get_pbehaviors_by_eid(self, id_): """Retreive from database every pbehavior that contains the given id_ in the PBehavior.EIDS field. :param list,str: the id(s) as a str or a list of string :returns: a list of pbehavior, with the isActive key in pbehavior is active when queried. :rtype: list """ if not isinstance(id_, (list, string_types)): raise TypeError(self.__TYPE_ERR) if isinstance(id_, list): for element in id_: if not isinstance(element, string_types): raise TypeError(self.__TYPE_ERR) else: id_ = [id_] cursor = self.pb_storage.get_elements( query={PBehavior.EIDS: {"$in": id_}} ) pbehaviors = [] now = int(time()) for pb in cursor: if pb['tstart'] <= now and (pb['tstop'] is None or pb['tstop'] >= now): pb['isActive'] = True else: pb['isActive'] = False pbehaviors.append(pb) return pbehaviors def read(self, _id=None): """Get pbehavior or list pbehaviors. :param str _id: pbehavior id, _id may be equal to None """ result = self.get(_id) return result if _id else list(result) def update(self, _id, **kwargs): """ Update pbehavior record :param str _id: pbehavior id :param dict kwargs: values pbehavior fields. If a field is None, it will **not** be updated. :raises ValueError: invalid RRULE or no pbehavior with given _id """ pb_value = self.get(_id) if pb_value is None: raise ValueError("The id does not match any pebahvior") check_valid_rrule(kwargs.get('rrule', '')) pbehavior = PBehavior(**self.get(_id)) new_data = {k: v for k, v in kwargs.items() if v is not None} pbehavior.update(**new_data) result = self.pb_storage.put_element( element=new_data, _id=_id ) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return pbehavior.to_dict() return None def upsert(self, pbehavior): """ Creates or update the given pbehavior. This function uses MongoStore/MongoCollection instead of Storage. :param canopsis.models.pbehavior.PBehavior pbehavior: :rtype: bool, dict :returns: success, update result """ r = self.pb_store.update({'_id': pbehavior._id}, pbehavior.to_dict(), upsert=True) if r.get('updatedExisting', False) and r.get('nModified') == 1: return True, r elif r.get('updatedExisting', None) is False and r.get('nModified') == 0 and r.get('ok') == 1.0: return True, r else: return False, r def delete(self, _id=None, _filter=None): """ Delete pbehavior record :param str _id: pbehavior id """ result = self.pb_storage.remove_elements( ids=_id, _filter=_filter ) return self._check_response(result) def _update_pbehavior(self, pbehavior_id, query): result = self.pb_storage._update( spec={'_id': pbehavior_id}, document=query, multi=False, cache=False ) return result def create_pbehavior_comment(self, pbehavior_id, author, message): """ Сreate comment for pbehavior. :param str pbehavior_id: pbehavior id :param str author: author of the comment :param str message: text of the comment """ comment_id = str(uuid4()) comment = { Comment.ID: comment_id, Comment.AUTHOR: author, Comment.TS: timegm(datetime.utcnow().timetuple()), Comment.MESSAGE: message } query = {'$addToSet': {PBehavior.COMMENTS: comment}} result = self._update_pbehavior(pbehavior_id, query) if not result: result = self._update_pbehavior( pbehavior_id, {'$set': {PBehavior.COMMENTS: []}} ) if not result: return None result = self._update_pbehavior(pbehavior_id, query) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return comment_id return None def update_pbehavior_comment(self, pbehavior_id, _id, **kwargs): """ Update the comment record. :param str pbehavior_id: pbehavior id :param str_id: comment id :param dict kwargs: values comment fields """ pbehavior = self.get( pbehavior_id, query={PBehavior.COMMENTS: {'$elemMatch': {'_id': _id}}} ) if not pbehavior: return None _comments = pbehavior[PBehavior.COMMENTS] if not _comments: return None comment = Comment(**_comments[0]) comment.update(**kwargs) result = self.pb_storage._update( spec={'_id': pbehavior_id, 'comments._id': _id}, document={'$set': {'comments.$': comment.to_dict()}}, multi=False, cache=False ) if (PBehaviorManager._UPDATE_FLAG in result and result[PBehaviorManager._UPDATE_FLAG]): return comment.to_dict() return None def delete_pbehavior_comment(self, pbehavior_id, _id): """ Delete comment record. :param str pbehavior_id: pbehavior id :param str _id: comment id """ result = self.pb_storage._update( spec={'_id': pbehavior_id}, document={'$pull': {PBehavior.COMMENTS: {'_id': _id}}}, multi=False, cache=False ) return self._check_response(result) def get_pbehaviors(self, entity_id): """ Return all pbehaviors related to an entity_id, sorted by descending tstart. :param str entity_id: Id for which behaviors have to be returned :return: pbehaviors, with name, tstart, tstop, rrule and enabled keys :rtype: list of dict """ res = list( self.pb_storage._backend.find( {PBehavior.EIDS: {'$in': [entity_id]}}, sort=[(PBehavior.TSTART, DESCENDING)] ) ) return res def compute_pbehaviors_filters(self): """ Compute all filters and update eids attributes. """ pbehaviors = self.pb_storage.get_elements( query={PBehavior.FILTER: {'$exists': True}} ) for pbehavior in pbehaviors: query = loads(pbehavior[PBehavior.FILTER]) if not isinstance(query, dict): self.logger.error('compute_pbehaviors_filters(): filter is ' 'not a dict !\n{}'.format(query)) continue entities = self.context.ent_storage.get_elements( query=query ) pbehavior[PBehavior.EIDS] = [e['_id'] for e in entities] self.pb_storage.put_element(element=pbehavior) def _check_active_simple_pbehavior(self, timestamp, pbh): """ Check if a pbehavior without a rrule is active at the given time. :param int timestamp: the number a second this 1970/01/01 00:00:00 :param dict pbehavior: a pbehavior as a dict. :return bool: True if the boolean is active, false otherwise """ if pbh[PBehavior.TSTART] <= timestamp <= pbh[PBehavior.TSTOP]: return True return False @staticmethod def __convert_timestamp(timestamp, timezone): """Convert a pbehavior timestamp defined in the timezone to a datetime in the same timezone. :param timestamp:""" return datetime.fromtimestamp(timestamp, tz.gettz(timezone)) def _check_active_reccuring_pbehavior(self, timestamp, pbehavior): """ Check if a pbehavior with a rrule is active at the given time. :param int timestamp: the number a second this 1970/01/01 00:00:00 :param dict pbehavior: a pbehavior as a dict. :return bool: True if the boolean is active, false otherwise :raise ValueError: if the pbehavior.exdate is invalid. Or if the date of an occurence of the pbehavior is not a valid date. """ tz_name = pbehavior.get(PBehavior.TIMEZONE, self.default_tz) rec_set = rrule.rruleset() # convert the timestamp to a datetime in the pbehavior's timezone now = self.__convert_timestamp(timestamp, tz_name) start = self.__convert_timestamp(pbehavior[PBehavior.TSTART], tz_name) stop = self.__convert_timestamp(pbehavior[PBehavior.TSTOP], tz_name) if PBehavior.EXDATE in pbehavior and\ isinstance(pbehavior[PBehavior.EXDATE], list): for date in pbehavior[PBehavior.EXDATE]: exdate = self.__convert_timestamp(date, tz_name) rec_set.exdate(exdate) duration = stop - start # pbehavior duration rec_set.rrule(rrule.rrulestr(pbehavior[PBehavior.RRULE], dtstart=start)) rec_start = rec_set.before(now) self.logger.debug("Recurence start : {}".format(rec_start)) # No recurrence found if rec_start is None: return False self.logger.debug("Timestamp : {}".format(now)) self.logger.debug("Recurence stop : {}".format(rec_start + duration)) if rec_start <= now <= rec_start + duration: return True return False def check_active_pbehavior(self, timestamp, pbehavior): """ Check if a pbehavior is active at the given time. :param int timestamp: the number a second this 1970/01/01 00:00:00 :param dict pbehavior: a pbehavior as a dict. :return bool: True if the boolean is active, false otherwise :raise ValueError: if the pbehavior.exdate is invalid. Or if the date of an occurence of the pbehavior is not a valid date. """ if PBehavior.RRULE not in pbehavior or\ pbehavior[PBehavior.RRULE] is None or\ pbehavior[PBehavior.RRULE] == "": return self._check_active_simple_pbehavior(timestamp, pbehavior) else: if PBehavior.EXDATE not in pbehavior: pbehavior[PBehavior.EXDATE] = [] return self._check_active_reccuring_pbehavior(timestamp, pbehavior) def check_pbehaviors(self, entity_id, list_in, list_out): """ !!!! DEPRECATED !!!! :param str entity_id: :param list list_in: list of pbehavior names :param list list_out: list of pbehavior names :returns: bool if the entity_id is currently in list_in arg and out list_out arg """ return (self._check_pbehavior(entity_id, list_in) and not self._check_pbehavior(entity_id, list_out)) def _check_pbehavior(self, entity_id, pb_names): """ :param str entity_id: :param list pb_names: list of pbehavior names :returns: bool if the entity_id is currently in pb_names arg """ self.logger.critical("_check_pbehavior is DEPRECATED !!!!") try: entity = self.context.get_entities_by_id(entity_id)[0] except Exception: self.logger.error('Unable to check_behavior on {} entity_id' .format(entity_id)) return None event = self.context.get_event(entity) pbehaviors = self.pb_storage.get_elements( query={ PBehavior.NAME: {'$in': pb_names}, PBehavior.EIDS: {'$in': [entity_id]} } ) names = [] fromts = datetime.fromtimestamp for pbehavior in pbehaviors: tstart = pbehavior[PBehavior.TSTART] tstop = pbehavior[PBehavior.TSTOP] if not isinstance(tstart, (int, float)): self.logger.error('Cannot parse tstart value: {}' .format(pbehavior)) continue if not isinstance(tstop, (int, float)): self.logger.error('Cannot parse tstop value: {}' .format(pbehavior)) continue tstart = fromts(tstart) tstop = fromts(tstop) dt_list = [tstart, tstop] if pbehavior['rrule'] is not None: dt_list = list( rrule.rrulestr(pbehavior['rrule'], dtstart=tstart).between( tstart, tstop, inc=True ) ) if (len(dt_list) >= 2 and fromts(event['timestamp']) >= dt_list[0] and fromts(event['timestamp']) <= dt_list[-1]): names.append(pbehavior[PBehavior.NAME]) result = set(pb_names).isdisjoint(set(names)) return not result @staticmethod def _check_response(response): ack = True if 'ok' in response and response['ok'] == 1 else False return { 'acknowledged': ack, 'deletedCount': response['n'] } def get_active_pbehaviors(self, eids): """ Return a list of active pbehaviors linked to some entites. :param list eids: the desired entities id :returns: list of pbehaviors """ result = [] for eid in eids: pbhs = self.get_pbehaviors(eid) result = result + [x for x in pbhs if self._check_pbehavior( eid, [x['name']] )] return result def get_all_active_pbehaviors(self): """ Return all pbehaviors currently active using self.check_active_pbehavior """ now = int(time()) query = {} ret_val = list(self.pb_storage.get_elements(query=query)) results = [] for pb in ret_val: try: if self.check_active_pbehavior(now, pb): results.append(pb) except ValueError as exept: self.logger.exception("Can't check if the pbehavior is active.") return results def get_active_pbehaviors_from_type(self, types=None): """ Return pbehaviors currently active, with a specific type, using self.check_active_pbehavior """ if types is None: types = [] now = int(time()) query = {PBehavior.TYPE: {'$in': types}} ret_val = list(self.pb_storage.get_elements(query=query)) results = [] for pb in ret_val: if self.check_active_pbehavior(now, pb): results.append(pb) return results def get_varying_pbehavior_list(self): """ get_varying_pbehavior_list :returns: list of PBehavior id activated since last check :rtype: list """ active_pbehaviors = self.get_all_active_pbehaviors() active_pbehaviors_ids = set() for active_pb in active_pbehaviors: active_pbehaviors_ids.add(active_pb['_id']) varying_pbs = active_pbehaviors_ids.symmetric_difference(self.currently_active_pb) self.currently_active_pb = active_pbehaviors_ids return list(varying_pbs) def launch_update_watcher(self, watcher_manager): """ launch_update_watcher update watcher when a pbehavior is active :param object watcher_manager: watcher manager :returns: number of watcher updated retype: int """ new_pbs = self.get_varying_pbehavior_list() new_pbs_full = list(self.pb_storage._backend.find( {'_id': {'$in': new_pbs}} )) merged_eids = [] for pbehaviour in new_pbs_full: merged_eids = merged_eids + pbehaviour['eids'] watchers_ids = set() for watcher in self.get_wacher_on_entities(merged_eids): watchers_ids.add(watcher['_id']) for watcher_id in watchers_ids: watcher_manager.compute_state(watcher_id) return len(list(watchers_ids)) def get_wacher_on_entities(self, entities_ids): """ get_wacher_on_entities. :param entities_ids: entity id :returns: list of watchers :rtype: list """ query = { '$and': [ {'depends': {'$in': entities_ids}}, {'type': 'watcher'} ] } watchers = self.context.get_entities(query=query) return watchers @staticmethod def get_active_intervals(after, before, pbehavior): """ Return all the time intervals between after and before during which the pbehavior was active. The intervals are returned as a list of tuples (start, end), ordered chronologically. start and end are UTC timestamps, and are always between after and before. :param int after: a UTC timestamp :param int before: a UTC timestamp :param Dict[str, Any] pbehavior: :rtype: List[Tuple[int, int]] """ rrule_str = pbehavior[PBehavior.RRULE] tstart = pbehavior[PBehavior.TSTART] tstop = pbehavior[PBehavior.TSTOP] if not isinstance(tstart, (int, float)): return if not isinstance(tstop, (int, float)): return # Convert the timestamps to datetimes tz = pytz.UTC dttstart = datetime.utcfromtimestamp(tstart).replace(tzinfo=tz) dttstop = datetime.utcfromtimestamp(tstop).replace(tzinfo=tz) delta = dttstop - dttstart dtafter = datetime.utcfromtimestamp(after).replace(tzinfo=tz) dtbefore = datetime.utcfromtimestamp(before).replace(tzinfo=tz) if not rrule_str: # The only interval where the pbehavior is active is # [dttstart, dttstop]. Ensure that it is included in # [after, before], and convert the datetimes to timestamps. if dttstart < dtafter: dttstart = dtafter if dttstop > dtbefore: dttstop = dtbefore yield ( timegm(dttstart.timetuple()), timegm(dttstop.timetuple()) ) else: # Get all the intervals that intersect with the [after, before] # interval. interval_starts = rrule.rrulestr(rrule_str, dtstart=dttstart).between( dtafter - delta, dtbefore, inc=False) for interval_start in interval_starts: interval_end = interval_start + delta # Ensure that the interval is included in [after, before], and # datetimes to timestamps. if interval_start < dtafter: interval_start = dtafter if interval_end > dtbefore: interval_end = dtbefore yield ( timegm(interval_start.timetuple()), timegm(interval_end.timetuple()) ) def get_intervals_with_pbehaviors_by_eid(self, after, before, entity_id): """ Yields intervals between after and before with a boolean indicating if a pbehavior affects the entity during this interval. The intervals are returned as a list of tuples (start, end, pbehavior), ordered chronologically. start and end are UTC timestamps, and are always between after and before, pbehavior is a boolean indicating if a pbehavior affects the entity during this interval. None of the intervals overlap. :param int after: a UTC timestamp :param int before: a UTC timestamp :param str entity_id: the id of the entity :rtype: Iterator[Tuple[int, int, bool]] """ return self.get_intervals_with_pbehaviors( after, before, self.get_pbehaviors(entity_id)) def get_intervals_with_pbehaviors(self, after, before, pbehaviors): """ Yields intervals between after and before with a boolean indicating if one of the pbehaviors is active during this interval. The intervals are returned as a list of tuples (start, end, pbehavior), ordered chronologically. start and end are UTC timestamps, and are always between after and before, pbehavior is a boolean indicating if a pbehavior affects the entity during this interval. None of the intervals overlap. :param int after: a UTC timestamp :param int before: a UTC timestamp :param List[Dict[str, Any]] pbehaviors: a list of pbehabiors :rtype: Iterator[Tuple[int, int, bool]] """ intervals = [] # Get all the intervals where a pbehavior is active for pbehavior in pbehaviors: for interval in self.get_active_intervals(after, before, pbehavior): intervals.append(interval) if not intervals: yield (after, before, False) return # Order them chronologically (by start date) intervals.sort(key=lambda a: a[0]) # Yield the first interval without any active pbehavior merged_interval_start, merged_interval_end = intervals[0] yield ( after, merged_interval_start, False ) # At this point intervals is a list of intervals where a pbehavior is # active, ordered by start date. Some of those intervals may be # overlapping. This merges the overlapping intervals. for interval_start, interval_end in intervals[1:]: if interval_end < merged_interval_end: # The interval is included in the merged interval, skip it. continue if interval_start > merged_interval_end: # Since the interval starts after the end of the merged # interval, they cannot be merged. Yield the merged interval, # and move to the new one. yield ( merged_interval_start, merged_interval_end, True ) yield ( merged_interval_end, interval_start, False ) merged_interval_start = interval_start merged_interval_end = interval_end yield ( merged_interval_start, merged_interval_end, True ) yield ( merged_interval_end, before, False ) def get_enabled_pbehaviors(self): """ Yields all the enabled pbehaviors. :rtype: Iterator[Dict[str, Any]] """ return self.pb_storage._backend.find({ PBehavior.ENABLED: True })
def fill(self, init=None, yes=False, reinit_auth=False): self.__put_canopsis_version_document() tools = [] for module in self.modules: try: migrationcls = lookup(module) except ImportError as err: self.logger.error( 'Impossible to load module "{0}": {1}'.format( module, err ) ) continue migrationtool = migrationcls() migrationtool.logger.addHandler(self.loghandler) tools.append(migrationtool) coll = None if init is None: store = MongoStore.get_default() store.authenticate() coll = MongoCollection(store.get_collection(self.FLAG_COLLECTION)) data = coll.find_one({"_id": self.FLAG_COLLECTION}) if data is None: print("Database not intialized. Initializing...") init = True else: print("Database already intialized. Updating...") init = False if init is None and reinit_auth is False: data = { "_id": "initialized", "at": str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")) } print("The canopsis initialization flag did not exist in the " "database. So canopsinit will (re?)initialized the " "database. Meaning, it may delete some important data " "from canopsis database. If you still want to initialize " "the database, call the same command with the " "`--authorize-reinit` flag. Or if you do not want to " "initialize the database, add the document `{0}` in the {1} " "collections.".format(data, self.FLAG_COLLECTION)) exit(1) for tool in tools: if init: tool.init(yes=yes) else: tool.update(yes=yes) if init is True: coll.insert({"_id": self.FLAG_COLLECTION, "at": str(time.strftime( "%a, %d %b %Y %H:%M:%S +0000"))})
class AlertsReader(object): """ Alarm cycle managment. Used to retrieve events related to alarms in a TimedStorage. """ LOG_PATH = 'var/log/alertsreader.log' CONF_PATH = 'etc/alerts/manager.conf' CATEGORY = 'COUNT_CACHE' GRAMMAR_FILE = 'etc/alerts/search/grammar.bnf' DEFAULT_ACTIVE_COLUMNS = ["v.component", "v.connector", "v.resource", "v.connector_name"] def __init__(self, logger, config, storage, pbehavior_manager): """ :param logger: a logger object :param config: a confng instance :param storage: a storage instance :param pbehavior_manager: a pbehavior manager instance """ self.logger = logger self.config = config self.alarm_storage = storage self.alarm_collection = MongoCollection(self.alarm_storage._backend) self.pbehavior_manager = pbehavior_manager self.pbh_filter = None category = self.config.get(self.CATEGORY, {}) self.expiration = int(category.get('expiration', DEFAULT_EXPIRATION)) self.opened_truncate = cfg_to_bool(category.get('opened_truncate', DEFAULT_OPENED_TRUNC)) self.opened_limit = int(category.get('opened_limit', DEFAULT_OPENED_LIMIT)) self.resolved_truncate = cfg_to_bool(category.get('resolved_truncate', DEFAULT_RESOLVED_TRUNC)) self.resolved_limit = int(category.get('resolved_limit', DEFAULT_RESOLVED_LIMIT)) self.count_cache = {} self.grammar = join(root_path, self.GRAMMAR_FILE) self.has_active_pbh = None @classmethod def provide_default_basics(cls): """ Provide logger, config, storages... ! Do not use in tests ! :rtype: Union[logging.Logger, canospis.confng.simpleconf.Configuration, canopsis.storage.core.Storage, canopsis.pbehavior.manager.PBehaviorManager] """ logger = Logger.get('alertsreader', cls.LOG_PATH) conf = Configuration.load(Alerts.CONF_PATH, Ini) alerts_storage = Middleware.get_middleware_by_uri( Alerts.ALERTS_STORAGE_URI ) pbm = PBehaviorManager(*PBehaviorManager.provide_default_basics()) return (logger, conf, alerts_storage, pbm) @property def alarm_fields(self): """ alarm_field parameter """ if not hasattr(self, '_alarm_fields'): self._alarm_fields = get_schema('alarm_fields') return self._alarm_fields def _translate_key(self, key): if key in self.alarm_fields['properties']: return self.alarm_fields['properties'][key]['stored_name'] return key def _translate_filter(self, filter_): """ Translate a mongo filter key names. Input keys are UI column names and output keys are corresponding keys in the alarm collection. :param dict filter_: Mongo filter written by an user :return: Mongo filter usable in a query :rtype: dict """ if isinstance(filter_, list): for i, fil in enumerate(filter_): filter_[i] = self._translate_filter(fil) elif isinstance(filter_, dict): for key, value in filter_.items(): new_value = self._translate_filter(value) filter_[key] = new_value new_key = self._translate_key(key) filter_[new_key] = filter_.pop(key) return filter_ def _translate_sort(self, key, dir_): """ Translate sort parameters. :param str key: UI column name to sort :param str dir_: Direction ('ASC' or 'DESC') :return: Key usable in a sort operation and translated direction for pymongo :rtype: tuple :raises ValueError: If dir_ is not 'ASC' nor 'DESC' """ if dir_ not in ['ASC', 'DESC']: raise ValueError( 'Sort direction must be "ASC" or "DESC" (got "{}")'.format( dir_ ) ) tkey = self._translate_key(key) tdir = 1 if dir_ == 'ASC' else -1 return tkey, tdir def _get_time_filter(self, opened, resolved, tstart, tstop): """ Transform opened, resolved, tstart and tstop parameters into a mongo filter. This filter is specific to alarms collection. :param bool opened: If True, select opened alarms :param bool resolved: If True, select resolved alarms :param tstart: Timestamp :param tstop: Timestamp :type tstart: int or None :type tstop: int or None :return: Specific mongo filter or None if opened and resolved are False :rtype: dict or None """ if opened and resolved: if tstart is None and tstop is None: return {} return { '$or': [ self._get_opened_time_filter(tstart, tstop), self._get_resolved_time_filter(tstart, tstop) ] } if opened: return self._get_opened_time_filter(tstart, tstop) if resolved: return self._get_resolved_time_filter(tstart, tstop) return None @staticmethod def _get_opened_time_filter(tstart, tstop): """ Get a specific mongo filter. :param tstart: Timestamp :param tstop: Timestamp :type tstart: int or None :type tstop: int or None :return: Mongo filter :rtype: dict """ if tstop is not None and tstart is not None: return { 'v.resolved': None, 't': {'$lte': tstop, "$gte": tstart} } if tstop is not None: return { 'v.resolved': None, 't': {'$lte': tstop} } elif tstart is not None: return { 'v.resolved': None, 't': {'$lte': tstart} } return {'v.resolved': None} @staticmethod def _get_resolved_time_filter(tstart, tstop): """ Get a specific mongo filter. :param tstart: Timestamp :param tstop: Timestamp :type tstart: int or None :type tstop: int or None :return: Specific mongo filter :rtype: dict """ if tstart is not None and tstop is not None: return { 'v.resolved': {'$ne': None}, 't': {'$gte': tstart, '$lte': tstop} } elif tstart is not None: return {'v.resolved': {'$ne': None, '$gte': tstart}} elif tstop is not None: return { 'v.resolved': {'$ne': None}, 't': {'$lte': tstop} } return {'v.resolved': {'$ne': None}} @classmethod def __convert_to_bool(cls, value): """Take a string and return the corresponding boolean. This method is case insensitive. Raise a a ValueError if the string can not be parsed. :param str value: a string containing the following value true, false : return bool: True or false""" if isinstance(value, bool): return value if value.lower() == "true": return True if value.lower() == "false": return False msg_err = "Can not convert {} to a boolean. true or false (case insensitive)" raise ValueError(msg_err.format(value)) def _filter_list(self, filter_): for item in filter_: self._filter(item) def _filter_dict(self, filter_): for key in filter_: if key == "has_active_pb": self.has_active_pbh = self.__convert_to_bool(filter_[key]) del filter_[key] return else: self._filter(filter_[key]) def _filter(self, filter_): if isinstance(filter_, dict): self._filter_dict(filter_) elif isinstance(filter_, list): self._filter_list(filter_) def parse_filter(self, filter_): """Set self.has_active_pbh true if the filter contain a active_pb key set to true or false if it set to false. If the key is not present or set to None, None. This method store the first occurrence. :param dict alarms: a filter from the brick listalarm """ if filter_ is not None: self._filter(filter_) def interpret_search(self, search): """ Parse a search expression to return a mongo filter and a search scope. :param str search: Search expression :return: Scope ('this' or 'all') and filter (dict) :rtype: tuple :raises ValueError: If search is not grammatically correct """ if not search: return ('this', {}) return interpret(search, grammar_file=self.grammar) def _lookup(self, alarms, lookups): """ Add extra keys to a list of alarms. :param list alarms: List of alarms as dict :param list lookups: List of extra keys to add. :return: Alarms with extra keys :rtype: list """ for lookup in lookups: task = get_task( 'alerts.lookup.{}'.format(lookup), cacheonly=True ) if task is None: raise ValueError('Unknown lookup "{}"'.format(lookup)) for alarm in alarms: alarm = task(self, alarm) return alarms def _get_final_filter( self, view_filter, time_filter, search, active_columns ): """ Computes the real filter: The view filter and time filter are always part of the final filter, if not empty. In the search matches the BNF grammar, it is appended to the final filter. Otherwise, regex on columns is made. All filters are aggregated with $and. { '$and': [ view_filter, time_filter, bnf_filter | column_filter ] } :param view_filter dict: the filter given by the canopsis view. :param time_filter dict: hehe. dunno. :param search str: text to search in columns, or a BNF valid search as defined by the grammar in etc/search/grammar.bnf The BNF grammar is tried first, if the string does not comply with the grammar, column search is used instead. :param active_columns list[str]: list of columns to search in. in a column ends with '.' it will be ignored. The 'd' column is always added. """ final_filter = {'$and': []} t_view_filter = self._translate_filter(view_filter) # add the view filter if not empty if view_filter not in [None, {}]: final_filter['$and'].append(t_view_filter) if time_filter not in [None, {}]: final_filter['$and'].append(time_filter) # try grammar search try: _, bnf_search_filter = self.interpret_search(search) bnf_search_filter = self._translate_filter(bnf_search_filter) except ValueError: bnf_search_filter = None if bnf_search_filter is not None: final_filter['$and'].append(bnf_search_filter) else: escaped_search = re.escape(str(search)) column_filter = {'$or': []} for column in active_columns: column_filter['$or'].append( { column: { '$regex': '.*{}.*'.format(escaped_search), '$options': 'i' } } ) column_filter['$or'].append( { 'd': { '$regex': '.*{}.*'.format(escaped_search), '$options': 'i' } } ) final_filter['$and'].append(column_filter) return final_filter def add_pbh_filter(self, pipeline, filter_): """Add to the aggregation pipeline the stages to filter the alarm with their pbehavior. :param list pipeline: the aggregation pipeline :param dict filter_: the filter received from the front.""" self.parse_filter(filter_) pipeline.append({"$lookup": { "from": "default_pbehavior", "localField": "d", "foreignField": "eids", "as": "pbehaviors"}}) if self.has_active_pbh is not None: tnow = int(time()) stage = { "$project": { "pbehaviors": { "$filter": { "as": "pbh", "input": "$pbehaviors", "cond": { "$and": [ {"$lte": ["$$pbh.tstart", tnow]}, {"$gte": ["$$pbh.tstop", tnow]} ] } } }, "_id": 1, "v": 1, "d": 1, "t": 1, "entity": 1 } } pipeline.append(stage) pbh_filter = {"$match": {"pbehaviors": None}} if self.has_active_pbh is True: pbh_filter["$match"]["pbehaviors"] = {"$ne": []} if self.has_active_pbh is False: pbh_filter["$match"]["pbehaviors"] = {"$eq": []} pipeline.append(pbh_filter) self.has_active_pbh = None def _build_aggregate_pipeline(self, final_filter, sort_key, sort_dir, with_steps, filter_): """ :param dict final_filter: the filter sent by the front page :param str sort_key: Name of the column to sort. If the value ends with a dot '.', sort_key is replaced with 'v.last_update_date'. :param str sort_dir: Either "ASC" or "DESC" :param bool with_steps: True if you want alarm steps in your alarm. :param dict filter_: Mongo filter :returns: List of steps used in mongo aggregation :rtype: list """ pipeline = [ { "$lookup": { "from": "default_entities", "localField": "d", "foreignField": "_id", "as": "entity" } }, { "$unwind": { "path": "$entity", "preserveNullAndEmptyArrays": True, } }, { "$match": {"$or": [ {"entity.enabled": True}, { "entity": {"$exists": False}} ]} }, { "$match": final_filter }, { "$sort": { sort_key: sort_dir } } ] if not with_steps: pipeline.insert(0, {"$project": {"v.steps": False}}) self.add_pbh_filter(pipeline, filter_) return pipeline def _search_aggregate(self, skip, limit, pipeline): """ :param int skip: Number of alarms to skip (pagination) :param int limit: Maximum number of alarms to return :param list pipeline: list of steps in mongo aggregate command :returns: Dict containing alarms, the list of alarms returned by mongo and truncated, a boolean true when there's still paginated data after these :rtype: dict """ aggregate_pipeline = pipeline[:] aggregate_pipeline.append({ "$skip": skip }) if limit is not None: aggregate_pipeline.append({"$limit": limit}) result = self.alarm_collection.aggregate( aggregate_pipeline, allowDiskUse=True, cursor={} ) alarms = list(result) truncated = len(alarms) > limit res = { 'alarms': alarms, 'truncated': truncated, } return res def _offset_aggregate(self, results, skip, limit, filters, pipeline): """ :param dict results: the results from previous sets :param int skip: Number of alarms to skip (pagination) :param int limit: Maximum number of alarms to return :param list filters: list of functions to apply on alarms :param list pipeline: list of steps in mongo aggregate command :returns: Three values: - results is the dict containing alarms, truncated, first and last - skip is the updated (next) value of skip, depending on limit - truncated_by is the number of useless data removed depending on the filters :rtype: dict, int, int """ tmp_res = self._search_aggregate(skip, limit, pipeline) pre_filter_len = len(tmp_res['alarms']) # no results, all good if tmp_res['alarms']: results['truncated'] |= tmp_res['truncated'] # filter useless data for afilter in filters: tmp_res['alarms'] = afilter(tmp_res['alarms']) results['alarms'].extend(tmp_res['alarms']) skip += limit truncated_by = pre_filter_len - len(tmp_res['alarms']) return results, skip, truncated_by def _loop_aggregate(self, skip, limit, filters, post_sort, total, sort_key, sort_dir, api_limit, pipeline): """ :param int skip: Number of alarms to skip (pagination) :param int limit: Maximum number of alarms to return :param list filters: list of functions to apply on alarms :param int total: Total numer of alarms :param dict final_filter: the filter sent by the front page :param str sort_key: Name of the column to sort. If the value ends with a dot '.', sort_key is replaced with 'v.last_update_date'. :param str sort_dir: Either "ASC" or "DESC" :param bool with_steps: True if you want alarm steps in your alarm. :param dict filter_: Mongo filter :param int apt_limit: A hard limit for when hide_resources is active :param list pipeline: list of steps in mongo aggregate command :returns: Dict containing alarms, truncated, first and last :rtype: dict """ len_alarms = 0 results = { 'alarms': [], 'total': total, 'truncated': False, 'first': 1+skip, 'last': 0 # To be changed } while len(results['alarms']) < api_limit: results, skip, truncated_by = self._offset_aggregate(results, skip, limit, filters, pipeline) len_alarms = len(results['alarms']) # premature break in case we do not have any filter that could # modify the real count. # this condition cannot be embedded in while <cond> because the # loop needs to be ran at least one time. if not filters: break # filters did not filtered any thing: we don't need to loop # again, even if we don't have enough results. elif filters and truncated_by == 0: break if post_sort: results['alarms'] = self._aggregate_post_sort( results['alarms'], sort_key, sort_dir ) if len_alarms > api_limit: results['alarms'] = results['alarms'][0:api_limit] results['last'] = results['first']-1+len(results['alarms']) return results def get( self, tstart=None, tstop=None, opened=True, resolved=False, lookups=None, filter_=None, search='', sort_key='opened', sort_dir='DESC', skip=0, limit=None, with_steps=False, natural_search=False, active_columns=None, hide_resources=False ): """ Return filtered, sorted and paginated alarms. :param tstart: Beginning timestamp of requested period :param tstop: End timestamp of requested period :type tstart: int or None :type tstop: int or None :param bool opened: If True, consider alarms that are currently opened :param bool resolved: If True, consider alarms that have been resolved :param list lookups: List of extra columns to compute for each returned alarm. Extra columns are "pbehaviors". :param dict filter_: Mongo filter :param str search: Search expression in custom DSL :param str sort_key: Name of the column to sort. If the value ends with a dot '.', sort_key is replaced with 'v.last_update_date'. :param str sort_dir: Either "ASC" or "DESC" :param int skip: Number of alarms to skip (pagination) :param int limit: Maximum number of alarms to return :param bool with_steps: True if you want alarm steps in your alarm. :param bool natural_search: True if you want to use a natural search :param list active_columns: the list of alarms columns on which to apply the natural search filter. :param bool hide_resources: hide resources' alarms if the component has an alarm :returns: List of sorted alarms + pagination informations :rtype: dict """ if sort_key == 'v.duration': sort_key = 'v.creation_date' elif sort_key == 'v.current_state_duration': sort_key = 'v.state.t' if lookups is None: lookups = [] if filter_ is None: filter_ = {} if active_columns is None: active_columns = self.DEFAULT_ACTIVE_COLUMNS time_filter = self._get_time_filter( opened=opened, resolved=resolved, tstart=tstart, tstop=tstop ) if time_filter is None: return {'alarms': [], 'total': 0, 'first': 0, 'last': 0} sort_key, sort_dir = self._translate_sort(sort_key, sort_dir) final_filter = self._get_final_filter( filter_, time_filter, search, active_columns ) if sort_key[-1] == '.': sort_key = 'v.last_update_date' pipeline = self._build_aggregate_pipeline(final_filter, sort_key, sort_dir, with_steps, filter_) count_pipeline = pipeline[:] count_pipeline.append({ "$count": "count" }) try: total = list(self.alarm_collection.aggregate(count_pipeline, allowDiskUse=True, cursor={}))[0]['count'] except IndexError: total = 0 if limit is None: limit = total # truncate results if more than required api_limit = limit # get a little bit more results so we may avoid querying the database # more than once. if hide_resources: limit = limit * 2 hide_resources &= rconn.exists('featureflag:hide_resources') filters = [] post_sort = False if hide_resources: post_sort = True filters.append(self._hide_resources) result = self._loop_aggregate(skip, limit, filters, post_sort, total, sort_key, sort_dir, api_limit, pipeline) return result @staticmethod def _aggregate_post_sort(alarms, sort_key, sort_dir): return sorted( alarms, key=lambda k: get_sub_key(k, sort_key), reverse=(sort_dir == -1) ) @staticmethod def _hide_resources(alarms): """ Reads alarm_hideresources_resource:<connector>/<connector_name>/<resource>/<component>:drop key from redis. if such key exists then the alarm is removed from the result set. """ filtered_alarms = [] for alarm in alarms: if alarm['v'].get('resource', '') == '': filtered_alarms.append(alarm) continue drop_id = 'alarm_hideresources_resource:{}/{}/{}/{}:drop'.format( alarm['v'].get('connector'), alarm['v'].get('connector_name'), alarm['v'].get('resource'), alarm['v'].get('component'), ) drop_value = rconn.get(drop_id) to_drop = False try: to_drop = drop_value is not None except (TypeError, ValueError): pass if not to_drop: filtered_alarms.append(alarm) return filtered_alarms def count_alarms_by_period( self, start, stop, subperiod=None, limit=100, query=None, ): """ Count alarms that have been opened during (stop - start) period. :param start: Beginning timestamp of period :type start: int :param stop: End timestamp of period :type stop: int :param subperiod: Cut (stop - start) in ``subperiod`` subperiods. :type subperiod: dict :param limit: Counts cannot exceed this value :type limit: int :param query: Custom mongodb filter for alarms :type query: dict :return: List in which each item contains an interval and the related count :rtype: list """ if subperiod is None: subperiod = {'day': 1} if query is None: query = {} intervals = Interval.get_intervals_by_period(start, stop, subperiod) results = [] for date in intervals: count = self.alarm_storage.count( data_ids=None, timewindow=TimeWindow(start=date['begin'], stop=date['end']), window_start_bind=True, _filter=query, ) results.append( { 'date': date, 'count': limit if count > limit else count, } ) return results
class engine(Engine): etype = 'event_filter' def __init__(self, *args, **kargs): super(engine, self).__init__(*args, **kargs) self.mg_store = MongoStore.get_default() self.collection = MongoCollection(self.mg_store.get_collection("object")) self.name = kargs['name'] self.drop_event_count = 0 self.pass_event_count = 0 self.__load_rules() def pre_run(self): self.beat() def a_override(self, event, action): """Override a field from event or add a new one if it does not have one. """ afield = action.get('field', None) avalue = action.get('value', None) # This must be a hard check because value can be a boolean or a null # integer if afield is None or avalue is None: self.logger.error( "Malformed action ('field' and 'value' required): {}".format( action ) ) return False if afield not in event: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True # afield is in event if not isinstance(avalue, list): if isinstance(event[afield], list): self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) event[afield].append(avalue) else: self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True else: # operation field is supported only for list values op = action.get('operation', 'append') if op == 'override': self.logger.debug("Overriding: '{}' -> '{}'".format( afield, avalue)) event[afield] = avalue return True elif op == 'append': self.logger.debug("Appending: '{}' to '{}'".format( avalue, afield)) if isinstance(event[afield], list): event[afield] += avalue else: event[afield] = [event[afield]] + avalue return True else: self.logger.error( "Operation '{}' unsupported (action '{}')".format( op, action ) ) return False def a_remove(self, event, action): """Remove an event from a field in event or the whole field if no element is specified. """ akey = action.get('key', None) aelement = action.get('element', None) del_met = action.get('met', 0) if akey: if aelement: if del_met: for i, met in enumerate(event[akey]): if met['name'] == aelement: del event[akey][i] break elif isinstance(event[akey], dict): del event[akey][aelement] elif isinstance(event[akey], list): del event[akey][event[akey].index(aelement)] self.logger.debug(u" + {}: Removed: '{}' from '{}'".format( event['rk'], aelement, akey)) else: del event[akey] self.logger.debug(u" + {}: Removed: '{}'".format( event['rk'], akey)) return True else: self.logger.error( u"Action malformed (needs 'key' and/or 'element'): {}".format( action)) return False def a_modify(self, event, action, name): """ Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ derogated = False atype = action.get('type') actionMap = { 'override': self.a_override, 'remove': self.a_remove } if atype in actionMap: derogated = actionMap[atype](event, action) else: self.logger.warning(u"Unknown action '{}'".format(atype)) # If the event was derogated, fill some informations if derogated: self.logger.debug(u"Event changed by rule '{}'".format(name)) return None def a_drop(self, event, action, name): """ Drop the event. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event dropped by rule '{}'".format(name)) self.drop_event_count += 1 return DROP def a_pass(self, event, action, name): """Pass the event to the next queue. Args: event map of the event to be modified action map of type action _name of the rule Returns: ``None`` """ self.logger.debug(u"Event passed by rule '{}'".format(name)) self.pass_event_count += 1 return event def a_route(self, event, action, name): """ Change the route to which an event will be sent Args: event: map of the event to be modified action: map of type action name: of the rule Returns: ``None`` """ if "route" in action: self.next_amqp_queues = [action["route"]] self.logger.debug(u"Event re-routed by rule '{}'".format(name)) else: self.logger.error( u"Action malformed (needs 'route'): {}".format(action)) return None def a_exec_job(self, event, action, name): records = self.collection.find( {'crecord_type': 'job', '_id': action['job']} ) for record in records: job = copy.deepcopy(record) job['context'] = event try: self.work_amqp_publisher.direct_event(job, 'Engine_scheduler') except Exception as e: self.logger.exception("Unable to send job") time.sleep(1) return True def a_snooze(self, event, action, name): """ Snooze event checks :param dict event: event to be snoozed :param dict action: action :param str name: name of the rule :returns: True if a snooze has been sent, False otherwise :rtype: boolean """ if event.get('event_type') == 'snooze': return False # Only check events can trigger an auto-snooze if event.get('event_type') != 'check': return False # A check OK cannot trigger an auto-snooze if event.get('state') == 0: return False # Alerts manager caching if not hasattr(self, 'am'): self.am = Alerts(*Alerts.provide_default_basics()) # Context manager caching if not hasattr(self, 'cm'): self.cm = ContextGraph(self.logger) entity_id = self.cm.get_id(event) current_alarm = self.am.get_current_alarm(entity_id) if current_alarm is None: snooze = { 'connector': event.get('connector', ''), 'connector_name': event.get('connector_name', ''), 'source_type': event.get('source_type', ''), 'component': event.get('component', ''), 'event_type': 'snooze', 'duration': action['duration'], 'author': 'event_filter', 'output': 'Auto snooze generated by rule "{}"'.format(name), 'timestamp': int(time.time()) } if event.get('resource', ''): snooze['resource'] = event['resource'] try: self.work_amqp_publisher.direct_event( snooze, 'Engine_event_filter') except Exception as e: self.logger.exception("Unable to send snooze event") return True return False def a_baseline(self, event, actions, name): """a_baseline :param event: :param action: baseline conf in event filter :param name: """ event['baseline_name'] = actions['baseline_name'] event['check_frequency'] = actions['check_frequency'] try: self.work_amqp_publisher.direct_event( event, 'Engine_baseline') except Exception as e: self.logger.exception("Unable to send baseline event") def apply_actions(self, event, actions): pass_event = False actionMap = { 'drop': self.a_drop, 'pass': self.a_pass, 'override': self.a_modify, 'remove': self.a_modify, 'execjob': self.a_exec_job, 'route': self.a_route, 'snooze': self.a_snooze, 'baseline': self.a_baseline } for name, action in actions: if action['type'] in actionMap: ret = actionMap[action['type'].lower()](event, action, name) if ret: pass_event = True else: self.logger.warning(u"Unknown action '{}'".format(action)) return pass_event def work(self, event, *xargs, **kwargs): rk = get_routingkey(event) default_action = self.configuration.get('default_action', 'pass') # list of supported actions rules = self.configuration.get('rules', []) to_apply = [] self.logger.debug(u'event {}'.format(event)) # When list configuration then check black and # white lists depending on json configuration for filterItem in rules: actions = filterItem.get('actions') name = filterItem.get('name', 'no_name') self.logger.debug(u'rule {}'.format(filterItem)) self.logger.debug(u'filter is {}'.format(filterItem['mfilter'])) # Try filter rules on current event if filterItem['mfilter'] and check(filterItem['mfilter'], event): self.logger.debug( u'Event: {}, filter matches'.format(event.get('rk', event)) ) if 'pbehaviors' in filterItem: pbehaviors = filterItem.get('pbehaviors', {}) list_in = pbehaviors.get('in', []) list_out = pbehaviors.get('out', []) if list_in or list_out: pbm = singleton_per_scope(PBehaviorManager) cm = singleton_per_scope(ContextGraph) entity = cm.get_entity(event) entity_id = cm.get_entity_id(entity) result = pbm.check_pbehaviors( entity_id, list_in, list_out ) if not result: break for action in actions: if action['type'].lower() == 'drop': self.apply_actions(event, to_apply) return self.a_drop(event, None, name) to_apply.append((name, action)) if filterItem.get('break', 0): self.logger.debug( u' + Filter {} broke the next filters processing' .format( filterItem.get('name', 'filter') ) ) break if len(to_apply): if self.apply_actions(event, to_apply): self.logger.debug( u'Event before sent to next engine: %s' % event ) event['rk'] = event['_id'] = get_routingkey(event) return event # No rules matched if default_action == 'drop': self.logger.debug("Event '%s' dropped by default action" % (rk)) self.drop_event_count += 1 return DROP self.logger.debug("Event '%s' passed by default action" % (rk)) self.pass_event_count += 1 self.logger.debug(u'Event before sent to next engine: %s' % event) event['rk'] = event['_id'] = get_routingkey(event) return event def __load_rules(self): tmp_rules = [] records = self.collection.find( {'crecord_type': 'filter', 'enable': True}) records.sort('priority', 1) for record in records: record_dump = copy.deepcopy(record) self.set_loaded(record_dump) try: record_dump["mfilter"] = loads(record_dump["mfilter"]) except Exception: self.logger.info(u'Invalid mfilter {}, filter {}'.format( record_dump['mfilter'], record_dump['name'], )) self.logger.debug(u'Loading record_dump:') self.logger.debug(record_dump) tmp_rules.append(record_dump) self.configuration = { 'rules': tmp_rules, 'default_action': self.find_default_action() } def beat(self, *args, **kargs): """ Configuration reload for realtime ui changes handling """ self.logger.debug(u'Reload configuration rules') self.__load_rules() self.logger.debug( 'Loaded {} rules'.format(len(self.configuration['rules'])) ) self.send_stat_event() def set_loaded(self, record): if 'run_once' in record and not record['run_once']: self.collection.update({"_id": record['_id']}, {"$set": {'run_once': True}}) self.logger.info( 'record {} has been run once'.format(record['_id']) ) def send_stat_event(self): """ Send AMQP Event for drop and pass metrics """ message_dropped = '{} event dropped since {}'.format( self.drop_event_count, self.beat_interval ) message_passed = '{} event passed since {}'.format( self.pass_event_count, self.beat_interval ) event = forger( connector='Engine', connector_name='engine', event_type='check', source_type='resource', resource=self.amqp_queue + '_data', state=0, state_type=1, output=message_dropped, perf_data_array=[ {'metric': 'pass_event', 'value': self.pass_event_count, 'type': 'GAUGE'}, {'metric': 'drop_event', 'value': self.drop_event_count, 'type': 'GAUGE'} ] ) self.logger.debug(message_dropped) self.logger.debug(message_passed) try: self.beat_amqp_publisher.canopsis_event(event) except Exception as e: self.logger.exception("Unable to send stat event") self.drop_event_count = 0 self.pass_event_count = 0 def find_default_action(self): """Find the default action stored and returns it, else assume it default action is pass. """ records = self.collection.find_one({'crecord_type': 'defaultrule'}) if records: return records[0]["action"] self.logger.debug( "No default action found. Assuming default action is pass" ) return 'pass'
class RuleManager(object): """ Manager for event filter rules. """ def __init__(self, logger): self.logger = logger self.rule_collection = MongoCollection( MongoStore.get_default().get_collection(RULE_COLLECTION)) def get_by_id(self, rule_id): """ Get an event filter rule given its id. :param str rule_id: the id of the rule. :rtype: Dict[str, Any] """ return self.rule_collection.find_one({ RuleField.id: rule_id }) def create(self, rule): """ Create a new rule and return its id. :param Dict[str, Any] rule: :rtype: str :raises: InvalidRuleError if the rule is invalid. CollectionError if the creation fails. """ rule_id = str(uuid4()) rule[RuleField.id] = rule_id self.validate(rule_id, rule) self.rule_collection.insert(rule) return rule_id def remove_with_id(self, rule_id): """ Remove a rule given its id. :param str rule_id: the id of the rule. CollectionError if the creation fails. """ self.rule_collection.remove({ RuleField.id: rule_id }) def list(self): """ Return a list of all the rules. :rtype: List[Dict[str, Any]] """ return list(self.rule_collection.find({})) def update(self, rule_id, rule): """ Update a rule given its id. :param str rule_id: the id of the rule. :param Dict[str, Any] rule: :raises: InvalidRuleError if the rule is invalid. CollectionError if the creation fails. """ self.validate(rule_id, rule) self.rule_collection.update({ RuleField.id: rule_id }, rule, upsert=False) def validate(self, rule_id, rule): """ Check that the rule is valid. The pattern and external_data fields are not validated by this method. :param Dict[str, Any] view: :raises: InvalidRuleError if it is invalid. """ # Validate id field if rule.get(RuleField.id, rule_id) != rule_id: raise InvalidRuleError( 'The {0} field should not be modified.'.format(RuleField.id)) # Check that there are no unexpected fields in the rule unexpected_fields = set(rule.keys()).difference(RuleField.values) if unexpected_fields: raise InvalidRuleError( 'Unexpected fields: {0}.'.format(', '.join(unexpected_fields))) # Validate the type field if RuleField.type not in rule: raise InvalidRuleError( 'The {0} field is required.'.format(RuleField.type)) if rule.get(RuleField.type) not in RuleType.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.type, ', '.join(RuleType.values))) # Validate the priority field if not isinstance(rule.get(RuleField.priority, 0), int): raise InvalidRuleError( 'The {0} field should be an integer.'.format( RuleField.priority)) # Validate the enabled field if not isinstance(rule.get(RuleField.enabled, True), bool): raise InvalidRuleError( 'The {0} field should be a boolean.'.format( RuleField.enabled)) if rule.get(RuleField.type) != RuleType.enrichment: # Check that the enrichment fields are not defined for # non-enrichment rules. unexpected_fields = set(rule.keys()).intersection( ENRICHMENT_FIELDS) if unexpected_fields: raise InvalidRuleError( 'The following fields should only be defined for ' 'enrichment rules: {0}.'.format( ', '.join(unexpected_fields))) else: # Validate the actions field of the enrichment rules. if RuleField.actions not in rule: raise InvalidRuleError( 'The {0} field is required for enrichment rules.'.format( RuleField.actions)) if not isinstance(rule.get(RuleField.actions), list): raise InvalidRuleError( 'The {0} field should be a list.'.format( RuleField.actions)) if not rule.get(RuleField.actions): raise InvalidRuleError( 'The {0} field should contain at least one action.'.format( RuleField.actions)) # Validate the on_success field of the enrichment rules. outcome = rule.get(RuleField.on_success) if outcome and outcome not in RuleOutcome.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.on_success, ', '.join(RuleOutcome.values))) # Validate the on_failure field of the enrichment rules. outcome = rule.get(RuleField.on_failure) if outcome and outcome not in RuleOutcome.values: raise InvalidRuleError( 'The {0} field should be one of: {1}.'.format( RuleField.on_failure, ', '.join(RuleOutcome.values)))
def __init__(self, logger): self.logger = logger self.rule_collection = MongoCollection( MongoStore.get_default().get_collection(RULE_COLLECTION))