Example #1
0
    def test_05_in_nin(self):
        filter1 = {'timestamp': {'$in': [0, 5, 6, 1378713357]}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$nin': [0, 5, 6]}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)
Example #2
0
    def test_02_exists(self):
        filter1 = {'timestamp': {'$exists': True}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$exists': False}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #3
0
    def test_01_simple(self):
        filter1 = {'connector': 'Engine'}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': 'cengidddddne'}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #4
0
    def test_01_simple(self):
        filter1 = {'connector': 'Engine'}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': 'cengidddddne'}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #5
0
    def test_07_all(self):
        filter1 = {'connector': {'$all': ['Engine']}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$all': ['Engine', 'cEngine']}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #6
0
    def test_05_in_nin(self):
        filter1 = {'timestamp': {'$in': [0, 5, 6, 1378713357]}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$nin': [0, 5, 6]}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)
Example #7
0
    def test_02_exists(self):
        filter1 = {'timestamp': {'$exists': True}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$exists': False}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #8
0
    def test_07_all(self):
        filter1 = {'connector': {'$all': ['Engine']}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$all': ['Engine', 'cEngine']}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #9
0
    def test_08_regex(self):
        filter1 = {'connector': {'$regex': '.ngInE'}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$regex': '.ngInE', '$options': 'i'}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$regex': '..ngine', '$options': 'i'}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #10
0
    def test_08_regex(self):
        filter1 = {'connector': {'$regex': '.ngInE'}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$regex': '.ngInE', '$options': 'i'}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$regex': '..ngine', '$options': 'i'}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #11
0
    def test_04_gt_gte(self):
        filter1 = {'timestamp': {'$gt': 1378713357}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$gte': 1378713357}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$gt': 137871335}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)
Example #12
0
    def test_04_gt_gte(self):
        filter1 = {'timestamp': {'$gt': 1378713357}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$gte': 1378713357}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$gt': 137871335}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)
Example #13
0
    def get_metrics(self, regex, metrics=None):
        """
        Get metrics with regular expression from storage or existing set.

        :param regex: "co:regex re:regex me:regex" filter
        :type regex: str

        :param metrics: If not specified, get from storage (optional)
        :type metrics: list

        :returns: list of matching metrics
        """

        mfilter = build_filter_from_regex(regex)

        if metrics is None:
            return self[Serie.CONTEXT_MANAGER].find(
                _type='metric',
                _filter=mfilter
            )

        else:
            result = [
                metric
                for metric in metrics
                if check(mfilter, metric)
            ]

            return result
Example #14
0
    def test_03_eq(self):
        filter1 = {'connector': {'$eq': 'Engine'}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'$or': [{'state': {'$eq': 0}}]}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$eq': 'Enginessssss'}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$eq': 1378713357}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)
Example #15
0
    def conditions(self, event, derogation):
        conditions_json = derogation.get('conditions', None)

        try:
            conditions = loads(conditions_json)

        except ValueError:
            self.logger.error(
                "Invalid conditions field in '%s': %s" % (
                    derogation['_id'], conditions_json
                )
            )
            self.logger.debug(derogation)

            return False

        if conditions:
            _check = check(conditions, event)

            self.logger.debug(" + 'conditions' check is %s" % _check)

            return _check

        else:
            return True
Example #16
0
    def test_03_eq(self):
        filter1 = {'connector': {'$eq': 'Engine'}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'$or': [{'state': {'$eq': 0}}]}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$eq': 'Enginessssss'}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'timestamp': {'$eq': 1378713357}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)
Example #17
0
    def work(self, event, *args, **kargs):
        """
        Each event comming to tag work method is beeing tagged with many
        informations. Those tags aim to display tags into UI to enhence
        information search.
        """

        event['tags'] = event.get('tags', [])

        event = self.add_tag(event, 'connector_name')
        event = self.add_tag(event, 'event_type')
        event = self.add_tag(event, 'source_type')
        event = self.add_tag(event, 'component')
        event = self.add_tag(event, 'resource')

        # Adds tag to event if selector crecord matches current event.
        self.logger.debug(u'Will process selector tag on event {} '.format(
            event['rk']
        ))

        for selector in self.selectors:

            add_tag = False
            cfilter = False
            self.logger.debug('Filter {}: type {}'.format(
                selector.mfilter,
                type(selector.mfilter))
            )

            if selector.mfilter:
                cfilter = check(selector.mfilter, event)

            if 'rk' in event:
                if event['rk'] not in selector.exclude_ids and (
                        event['rk'] in selector.include_ids or cfilter):
                    add_tag = True
            elif cfilter:
                add_tag = True

            if add_tag:
                self.logger.debug(
                    'Will write tag to event: %s' % (selector.tags))
                for tag in selector.tags:
                    if tag not in event['tags']:
                        event['tags'].append(tag)

        # Tag with dynamic tags
        sels = self.selByRk.get(event['rk'], [])

        for sel in sels:
            event = self.add_tag(event, value=sel)

        return event
Example #18
0
    def match(self, event):
        """
        Get filters names which match the event.

        :param event: Event to check
        :type event: dict

        :returns: filters names as list
        """

        storage = self[MetricProducer.FILTER_STORAGE]
        matches = [
            evfilter['crecord_name']
            for evfilter in storage.find_elements()
            if check(evfilter.get('filter', None) or {}, event)
        ]

        return matches
Example #19
0
    def find(
        self,
        names=None,
        meta=None,
        sort=None,
        limit=-1,
        skip=0,
        with_meta=False
    ):
        if names is not None:
            names = ensure_iterable(names)

        result = []

        for _, _, files in os.walk(self.uri):
            for filename in files:
                # ignore meta files
                if not filename.endswith(UnixFileStorage.META_EXT):
                    if names is not None and filename not in names:
                        continue

                    if meta is not None:
                        metadata = self.get_meta(filename)

                        if not check(meta, metadata):
                            continue

                    result.append(filename)

        result = result[skip:limit]
        result = [open(filename) for filename in result]

        if with_meta:
            result = [(f, self.get_meta(f.name)) for f in result]

        if sort is not None:
            raise NotImplementedError('sort is not yet supported')

        return result
Example #20
0
    def work(self, event, *xargs, **kwargs):

        rk = get_routingkey(event)
        default_action = self.configuration.get('default_action', 'pass')

        # list of supported actions

        rules = self.configuration.get('rules', [])
        to_apply = []

        self.logger.debug(u'event {}'.format(event))

        # When list configuration then check black and
        # white lists depending on json configuration
        for filterItem in rules:
            actions = filterItem.get('actions')
            name = filterItem.get('name', 'no_name')

            self.logger.debug(u'rule {}'.format(filterItem))
            self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
            # Try filter rules on current event
            if filterItem['mfilter'] and check(filterItem['mfilter'], event):
                self.logger.debug(
                    u'Event: {}, filter matches'.format(event.get('rk', event))
                )

                if 'pbehaviors' in filterItem:
                    pbehaviors = filterItem.get('pbehaviors', {})
                    list_in = pbehaviors.get('in', [])
                    list_out = pbehaviors.get('out', [])

                    if list_in or list_out:
                        pbm = singleton_per_scope(PBehaviorManager)
                        cm = singleton_per_scope(ContextGraph)
                        entity = cm.get_entity(event)
                        entity_id = cm.get_entity_id(entity)

                        result = pbm.check_pbehaviors(
                            entity_id, list_in, list_out
                        )

                        if not result:
                            break

                for action in actions:
                    if action['type'].lower() == 'drop':
                        self.apply_actions(event, to_apply)
                        return self.a_drop(event, None, name)
                    to_apply.append((name, action))

                if filterItem.get('break', 0):
                    self.logger.debug(
                        u' + Filter {} broke the next filters processing'
                        .format(
                            filterItem.get('name', 'filter')
                        )
                    )
                    break

        if len(to_apply):
            if self.apply_actions(event, to_apply):
                self.logger.debug(
                    u'Event before sent to next engine: %s' % event
                )
                event['rk'] = event['_id'] = get_routingkey(event)
                return event

        # No rules matched
        if default_action == 'drop':
            self.logger.debug("Event '%s' dropped by default action" % (rk))
            self.drop_event_count += 1
            return DROP

        self.logger.debug("Event '%s' passed by default action" % (rk))
        self.pass_event_count += 1

        self.logger.debug(u'Event before sent to next engine: %s' % event)
        event['rk'] = event['_id'] = get_routingkey(event)
        return event
Example #21
0
    def work(self, event, *xargs, **kwargs):

        rk = get_routingkey(event)
        default_action = self.configuration.get('default_action', 'pass')

        # list of actions supported

        rules = self.configuration.get('rules', [])
        to_apply = []

        self.logger.debug(u'event {}'.format(event))

        # When list configuration then check black and
        # white lists depending on json configuration
        for filterItem in rules:
            actions = filterItem.get('actions')
            name = filterItem.get('name', 'no_name')

            self.logger.debug(u'rule {}'.format(filterItem))
            self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
            # Try filter rules on current event
            if filterItem['mfilter'] and check(filterItem['mfilter'], event):

                self.logger.debug(
                    u'Event: {}, filter matches'.format(event['rk'])
                )

                for action in actions:
                    if action['type'].lower() == 'drop':
                        self.apply_actions(event, to_apply)
                        return self.a_drop(event, None, name)
                    to_apply.append((name, action))

                if filterItem.get('break', 0):
                    self.logger.debug(
                        u' + Filter {} broke the next filters processing'
                        .format(
                            filterItem.get('name', 'filter')
                        )
                    )
                    break

        if len(to_apply):
            if self.apply_actions(event, to_apply):
                self.logger.debug(
                    u'Event before sent to next engine: %s' % event
                )
                event['rk'] = event['_id'] = get_routingkey(event)
                return event

        # No rules matched
        if default_action == 'drop':
            self.logger.debug("Event '%s' dropped by default action" % (rk))
            self.drop_event_count += 1
            return DROP

        self.logger.debug("Event '%s' passed by default action" % (rk))
        self.pass_event_count += 1

        self.logger.debug(u'Event before sent to next engine: %s' % event)
        event['rk'] = event['_id'] = get_routingkey(event)
        return event
Example #22
0
    def work(self, event, *xargs, **kwargs):

        rk = get_routingkey(event)
        default_action = self.configuration.get('default_action', 'pass')

        # list of supported actions

        rules = self.configuration.get('rules', [])
        to_apply = []

        self.logger.debug(u'event {}'.format(event))

        # When list configuration then check black and
        # white lists depending on json configuration
        for filterItem in rules:
            actions = filterItem.get('actions')
            name = filterItem.get('name', 'no_name')

            self.logger.debug(u'rule {}'.format(filterItem))
            self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
            # Try filter rules on current event
            if filterItem['mfilter'] and check(filterItem['mfilter'], event):
                self.logger.debug(u'Event: {}, filter matches'.format(
                    event.get('rk', event)))

                if 'pbehaviors' in filterItem:
                    pbehaviors = filterItem.get('pbehaviors', {})
                    list_in = pbehaviors.get('in', [])
                    list_out = pbehaviors.get('out', [])

                    if list_in or list_out:
                        pbm = singleton_per_scope(PBehaviorManager)
                        cm = singleton_per_scope(ContextGraph)
                        entity = cm.get_entity(event)
                        entity_id = cm.get_entity_id(entity)

                        result = pbm.check_pbehaviors(entity_id, list_in,
                                                      list_out)

                        if not result:
                            break

                for action in actions:
                    if action['type'].lower() == 'drop':
                        self.apply_actions(event, to_apply)
                        return self.a_drop(event, None, name)
                    to_apply.append((name, action))

                if filterItem.get('break', 0):
                    self.logger.debug(
                        u' + Filter {} broke the next filters processing'.
                        format(filterItem.get('name', 'filter')))
                    break

        if len(to_apply):
            if self.apply_actions(event, to_apply):
                self.logger.debug(u'Event before sent to next engine: %s' %
                                  event)
                event['rk'] = event['_id'] = get_routingkey(event)
                return event

        # No rules matched
        if default_action == 'drop':
            self.logger.debug("Event '%s' dropped by default action" % (rk))
            self.drop_event_count += 1
            return DROP

        self.logger.debug("Event '%s' passed by default action" % (rk))
        self.pass_event_count += 1

        self.logger.debug(u'Event before sent to next engine: %s' % event)
        event['rk'] = event['_id'] = get_routingkey(event)
        return event
Example #23
0
    def test_06_complex(self):
        filter1 = {'timestamp': {'$gt': 0, '$lt': 2378713357}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {
            '$and': [{
                'timestamp': {
                    '$gt': 0
                }
            }, {
                'timestamp': {
                    '$lt': 2378713357
                }
            }]
        }
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {
            'connector': {
                '$eq': 'Engine'
            },
            'timestamp': {
                '$gt': 137871335
            }
        }
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$not': {'$eq': 'cccenngine'}}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$not': {'$eq': 'Engine'}}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {
            '$nor': [{
                'connector': {
                    '$eq': 'cEngine'
                }
            }, {
                'connector': {
                    '$eq': 'ccEngine'
                }
            }]
        }
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {
            '$nor': [{
                'connector': {
                    '$eq': 'Engine'
                }
            }, {
                'connector': {
                    '$eq': 'ccEngine'
                }
            }]
        }
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': 'Engine', 'event_type': 'check'}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {
            '$and': [{
                'connector': 'Engine'
            }, {
                'event_type': 'check'
            }, {
                'event_type': 'check'
            }]
        }
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {
            '$or': [{
                'connector': 'cenginddddde'
            }, {
                'event_type': 'check'
            }, {
                'event_type': 'checkkkkk'
            }]
        }
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {
            '$or': [{
                '$and': [{
                    'connector': 'cenginddddde'
                }, {
                    'event_type': 'check'
                }]
            }, {
                'event_type': 'checkkkkk'
            }]
        }
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)
Example #24
0
    def test_06_complex(self):
        filter1 = {'timestamp': {'$gt': 0, '$lt': 2378713357}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'$and': [{'timestamp': {'$gt': 0}}, {'timestamp': {'$lt': 2378713357}}]}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$eq': 'Engine'}, 'timestamp': {'$gt': 137871335}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$not': {'$eq': 'cccenngine'}}}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': {'$not': {'$eq': 'Engine'}}}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'$nor': [{'connector': {'$eq': 'cEngine'}}, {'connector': {'$eq': 'ccEngine'}}]}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'$nor': [{'connector': {'$eq': 'Engine'}}, {'connector': {'$eq': 'ccEngine'}}]}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)

        filter1 = {'connector': 'Engine', 'event_type': 'check'}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'$and': [{'connector': 'Engine'}, {'event_type': 'check'}, {'event_type': 'check'}]}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'$or': [{'connector': 'cenginddddde'}, {'event_type': 'check'},  {'event_type': 'checkkkkk'}]}
        match = check(filter1, event)
        self.assertTrue(match, msg='Filter: %s' % filter1)

        filter1 = {'$or': [{'$and': [{'connector': 'cenginddddde'}, {'event_type': 'check'}]},  {'event_type': 'checkkkkk'}]}
        match = check(filter1, event)
        self.assertFalse(match, msg='Filter: %s' % filter1)