def test_05_in_nin(self):
filter1 = {'timestamp': {'$in': [0, 5, 6, 1378713357]}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$nin': [0, 5, 6]}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
def test_02_exists(self):
filter1 = {'timestamp': {'$exists': True}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$exists': False}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_01_simple(self):
filter1 = {'connector': 'Engine'}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': 'cengidddddne'}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_01_simple(self):
filter1 = {'connector': 'Engine'}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': 'cengidddddne'}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_07_all(self):
filter1 = {'connector': {'$all': ['Engine']}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$all': ['Engine', 'cEngine']}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_05_in_nin(self):
filter1 = {'timestamp': {'$in': [0, 5, 6, 1378713357]}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$nin': [0, 5, 6]}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
def test_02_exists(self):
filter1 = {'timestamp': {'$exists': True}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$exists': False}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_07_all(self):
filter1 = {'connector': {'$all': ['Engine']}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$all': ['Engine', 'cEngine']}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_08_regex(self):
filter1 = {'connector': {'$regex': '.ngInE'}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$regex': '.ngInE', '$options': 'i'}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$regex': '..ngine', '$options': 'i'}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_08_regex(self):
filter1 = {'connector': {'$regex': '.ngInE'}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$regex': '.ngInE', '$options': 'i'}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$regex': '..ngine', '$options': 'i'}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_04_gt_gte(self):
filter1 = {'timestamp': {'$gt': 1378713357}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$gte': 1378713357}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$gt': 137871335}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
def test_04_gt_gte(self):
filter1 = {'timestamp': {'$gt': 1378713357}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$gte': 1378713357}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$gt': 137871335}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
def get_metrics(self, regex, metrics=None):
"""
Get metrics with regular expression from storage or existing set.
:param regex: "co:regex re:regex me:regex" filter
:type regex: str
:param metrics: If not specified, get from storage (optional)
:type metrics: list
:returns: list of matching metrics
"""
mfilter = build_filter_from_regex(regex)
if metrics is None:
return self[Serie.CONTEXT_MANAGER].find(
_type='metric',
_filter=mfilter
)
else:
result = [
metric
for metric in metrics
if check(mfilter, metric)
]
return result
def test_03_eq(self):
filter1 = {'connector': {'$eq': 'Engine'}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'$or': [{'state': {'$eq': 0}}]}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$eq': 'Enginessssss'}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$eq': 1378713357}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
def conditions(self, event, derogation):
conditions_json = derogation.get('conditions', None)
try:
conditions = loads(conditions_json)
except ValueError:
self.logger.error(
"Invalid conditions field in '%s': %s" % (
derogation['_id'], conditions_json
)
)
self.logger.debug(derogation)
return False
if conditions:
_check = check(conditions, event)
self.logger.debug(" + 'conditions' check is %s" % _check)
return _check
else:
return True
def test_03_eq(self):
filter1 = {'connector': {'$eq': 'Engine'}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'$or': [{'state': {'$eq': 0}}]}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$eq': 'Enginessssss'}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'timestamp': {'$eq': 1378713357}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
def work(self, event, *args, **kargs):
"""
Each event comming to tag work method is beeing tagged with many
informations. Those tags aim to display tags into UI to enhence
information search.
"""
event['tags'] = event.get('tags', [])
event = self.add_tag(event, 'connector_name')
event = self.add_tag(event, 'event_type')
event = self.add_tag(event, 'source_type')
event = self.add_tag(event, 'component')
event = self.add_tag(event, 'resource')
# Adds tag to event if selector crecord matches current event.
self.logger.debug(u'Will process selector tag on event {} '.format(
event['rk']
))
for selector in self.selectors:
add_tag = False
cfilter = False
self.logger.debug('Filter {}: type {}'.format(
selector.mfilter,
type(selector.mfilter))
)
if selector.mfilter:
cfilter = check(selector.mfilter, event)
if 'rk' in event:
if event['rk'] not in selector.exclude_ids and (
event['rk'] in selector.include_ids or cfilter):
add_tag = True
elif cfilter:
add_tag = True
if add_tag:
self.logger.debug(
'Will write tag to event: %s' % (selector.tags))
for tag in selector.tags:
if tag not in event['tags']:
event['tags'].append(tag)
# Tag with dynamic tags
sels = self.selByRk.get(event['rk'], [])
for sel in sels:
event = self.add_tag(event, value=sel)
return event
def match(self, event):
"""
Get filters names which match the event.
:param event: Event to check
:type event: dict
:returns: filters names as list
"""
storage = self[MetricProducer.FILTER_STORAGE]
matches = [
evfilter['crecord_name']
for evfilter in storage.find_elements()
if check(evfilter.get('filter', None) or {}, event)
]
return matches
def find(
self,
names=None,
meta=None,
sort=None,
limit=-1,
skip=0,
with_meta=False
):
if names is not None:
names = ensure_iterable(names)
result = []
for _, _, files in os.walk(self.uri):
for filename in files:
# ignore meta files
if not filename.endswith(UnixFileStorage.META_EXT):
if names is not None and filename not in names:
continue
if meta is not None:
metadata = self.get_meta(filename)
if not check(meta, metadata):
continue
result.append(filename)
result = result[skip:limit]
result = [open(filename) for filename in result]
if with_meta:
result = [(f, self.get_meta(f.name)) for f in result]
if sort is not None:
raise NotImplementedError('sort is not yet supported')
return result
def work(self, event, *xargs, **kwargs):
rk = get_routingkey(event)
default_action = self.configuration.get('default_action', 'pass')
# list of supported actions
rules = self.configuration.get('rules', [])
to_apply = []
self.logger.debug(u'event {}'.format(event))
# When list configuration then check black and
# white lists depending on json configuration
for filterItem in rules:
actions = filterItem.get('actions')
name = filterItem.get('name', 'no_name')
self.logger.debug(u'rule {}'.format(filterItem))
self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
# Try filter rules on current event
if filterItem['mfilter'] and check(filterItem['mfilter'], event):
self.logger.debug(
u'Event: {}, filter matches'.format(event.get('rk', event))
)
if 'pbehaviors' in filterItem:
pbehaviors = filterItem.get('pbehaviors', {})
list_in = pbehaviors.get('in', [])
list_out = pbehaviors.get('out', [])
if list_in or list_out:
pbm = singleton_per_scope(PBehaviorManager)
cm = singleton_per_scope(ContextGraph)
entity = cm.get_entity(event)
entity_id = cm.get_entity_id(entity)
result = pbm.check_pbehaviors(
entity_id, list_in, list_out
)
if not result:
break
for action in actions:
if action['type'].lower() == 'drop':
self.apply_actions(event, to_apply)
return self.a_drop(event, None, name)
to_apply.append((name, action))
if filterItem.get('break', 0):
self.logger.debug(
u' + Filter {} broke the next filters processing'
.format(
filterItem.get('name', 'filter')
)
)
break
if len(to_apply):
if self.apply_actions(event, to_apply):
self.logger.debug(
u'Event before sent to next engine: %s' % event
)
event['rk'] = event['_id'] = get_routingkey(event)
return event
# No rules matched
if default_action == 'drop':
self.logger.debug("Event '%s' dropped by default action" % (rk))
self.drop_event_count += 1
return DROP
self.logger.debug("Event '%s' passed by default action" % (rk))
self.pass_event_count += 1
self.logger.debug(u'Event before sent to next engine: %s' % event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
def work(self, event, *xargs, **kwargs):
rk = get_routingkey(event)
default_action = self.configuration.get('default_action', 'pass')
# list of actions supported
rules = self.configuration.get('rules', [])
to_apply = []
self.logger.debug(u'event {}'.format(event))
# When list configuration then check black and
# white lists depending on json configuration
for filterItem in rules:
actions = filterItem.get('actions')
name = filterItem.get('name', 'no_name')
self.logger.debug(u'rule {}'.format(filterItem))
self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
# Try filter rules on current event
if filterItem['mfilter'] and check(filterItem['mfilter'], event):
self.logger.debug(
u'Event: {}, filter matches'.format(event['rk'])
)
for action in actions:
if action['type'].lower() == 'drop':
self.apply_actions(event, to_apply)
return self.a_drop(event, None, name)
to_apply.append((name, action))
if filterItem.get('break', 0):
self.logger.debug(
u' + Filter {} broke the next filters processing'
.format(
filterItem.get('name', 'filter')
)
)
break
if len(to_apply):
if self.apply_actions(event, to_apply):
self.logger.debug(
u'Event before sent to next engine: %s' % event
)
event['rk'] = event['_id'] = get_routingkey(event)
return event
# No rules matched
if default_action == 'drop':
self.logger.debug("Event '%s' dropped by default action" % (rk))
self.drop_event_count += 1
return DROP
self.logger.debug("Event '%s' passed by default action" % (rk))
self.pass_event_count += 1
self.logger.debug(u'Event before sent to next engine: %s' % event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
def work(self, event, *xargs, **kwargs):
rk = get_routingkey(event)
default_action = self.configuration.get('default_action', 'pass')
# list of supported actions
rules = self.configuration.get('rules', [])
to_apply = []
self.logger.debug(u'event {}'.format(event))
# When list configuration then check black and
# white lists depending on json configuration
for filterItem in rules:
actions = filterItem.get('actions')
name = filterItem.get('name', 'no_name')
self.logger.debug(u'rule {}'.format(filterItem))
self.logger.debug(u'filter is {}'.format(filterItem['mfilter']))
# Try filter rules on current event
if filterItem['mfilter'] and check(filterItem['mfilter'], event):
self.logger.debug(u'Event: {}, filter matches'.format(
event.get('rk', event)))
if 'pbehaviors' in filterItem:
pbehaviors = filterItem.get('pbehaviors', {})
list_in = pbehaviors.get('in', [])
list_out = pbehaviors.get('out', [])
if list_in or list_out:
pbm = singleton_per_scope(PBehaviorManager)
cm = singleton_per_scope(ContextGraph)
entity = cm.get_entity(event)
entity_id = cm.get_entity_id(entity)
result = pbm.check_pbehaviors(entity_id, list_in,
list_out)
if not result:
break
for action in actions:
if action['type'].lower() == 'drop':
self.apply_actions(event, to_apply)
return self.a_drop(event, None, name)
to_apply.append((name, action))
if filterItem.get('break', 0):
self.logger.debug(
u' + Filter {} broke the next filters processing'.
format(filterItem.get('name', 'filter')))
break
if len(to_apply):
if self.apply_actions(event, to_apply):
self.logger.debug(u'Event before sent to next engine: %s' %
event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
# No rules matched
if default_action == 'drop':
self.logger.debug("Event '%s' dropped by default action" % (rk))
self.drop_event_count += 1
return DROP
self.logger.debug("Event '%s' passed by default action" % (rk))
self.pass_event_count += 1
self.logger.debug(u'Event before sent to next engine: %s' % event)
event['rk'] = event['_id'] = get_routingkey(event)
return event
def test_06_complex(self):
filter1 = {'timestamp': {'$gt': 0, '$lt': 2378713357}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {
'$and': [{
'timestamp': {
'$gt': 0
}
}, {
'timestamp': {
'$lt': 2378713357
}
}]
}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {
'connector': {
'$eq': 'Engine'
},
'timestamp': {
'$gt': 137871335
}
}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$not': {'$eq': 'cccenngine'}}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$not': {'$eq': 'Engine'}}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {
'$nor': [{
'connector': {
'$eq': 'cEngine'
}
}, {
'connector': {
'$eq': 'ccEngine'
}
}]
}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {
'$nor': [{
'connector': {
'$eq': 'Engine'
}
}, {
'connector': {
'$eq': 'ccEngine'
}
}]
}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'connector': 'Engine', 'event_type': 'check'}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {
'$and': [{
'connector': 'Engine'
}, {
'event_type': 'check'
}, {
'event_type': 'check'
}]
}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {
'$or': [{
'connector': 'cenginddddde'
}, {
'event_type': 'check'
}, {
'event_type': 'checkkkkk'
}]
}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {
'$or': [{
'$and': [{
'connector': 'cenginddddde'
}, {
'event_type': 'check'
}]
}, {
'event_type': 'checkkkkk'
}]
}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
def test_06_complex(self):
filter1 = {'timestamp': {'$gt': 0, '$lt': 2378713357}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'$and': [{'timestamp': {'$gt': 0}}, {'timestamp': {'$lt': 2378713357}}]}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$eq': 'Engine'}, 'timestamp': {'$gt': 137871335}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$not': {'$eq': 'cccenngine'}}}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'connector': {'$not': {'$eq': 'Engine'}}}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'$nor': [{'connector': {'$eq': 'cEngine'}}, {'connector': {'$eq': 'ccEngine'}}]}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'$nor': [{'connector': {'$eq': 'Engine'}}, {'connector': {'$eq': 'ccEngine'}}]}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)
filter1 = {'connector': 'Engine', 'event_type': 'check'}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'$and': [{'connector': 'Engine'}, {'event_type': 'check'}, {'event_type': 'check'}]}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'$or': [{'connector': 'cenginddddde'}, {'event_type': 'check'}, {'event_type': 'checkkkkk'}]}
match = check(filter1, event)
self.assertTrue(match, msg='Filter: %s' % filter1)
filter1 = {'$or': [{'$and': [{'connector': 'cenginddddde'}, {'event_type': 'check'}]}, {'event_type': 'checkkkkk'}]}
match = check(filter1, event)
self.assertFalse(match, msg='Filter: %s' % filter1)