def test_new_term_with_terms():
rules = {'fields': ['a'],
'timestamp_field': '@timestamp',
'es_host': 'example.com', 'es_port': 10, 'index': 'logstash', 'query_key': 'a',
'window_step_size': {'days': 2}}
mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
{'key': 'key2', 'doc_count': 5}]}}}}
with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
rule = NewTermsRule(rules)
# Only 15 queries because of custom step size
assert rule.es.search.call_count == 15
# Key1 and key2 shouldn't cause a match
terms = {ts_now(): [{'key': 'key1', 'doc_count': 1},
{'key': 'key2', 'doc_count': 1}]}
rule.add_terms_data(terms)
assert rule.matches == []
# Key3 causes an alert for field a
terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
rule.add_terms_data(terms)
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'a'
assert rule.matches[0]['a'] == 'key3'
rule.matches = []
# Key3 doesn't cause another alert
terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
rule.add_terms_data(terms)
assert rule.matches == []
def test_new_term_with_terms():
rules = {'fields': ['a'],
'timestamp_field': '@timestamp',
'es_host': 'example.com', 'es_port': 10, 'index': 'logstash', 'query_key': 'a'}
mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
{'key': 'key2', 'doc_count': 5}]}}}}
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 1
# Key1 and key2 shouldn't cause a match
terms = {ts_now(): [{'key': 'key1', 'doc_count': 1},
{'key': 'key2', 'doc_count': 1}]}
rule.add_terms_data(terms)
assert rule.matches == []
# Key3 causes an alert for field a
terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
rule.add_terms_data(terms)
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'a'
assert rule.matches[0]['a'] == 'key3'
rule.matches = []
# Key3 doesn't cause another alert
terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
rule.add_terms_data(terms)
assert rule.matches == []
def test_new_term():
rules = {
'fields': ['a', 'b'],
'timestamp_field': '@timestamp',
'es_host': 'example.com',
'es_port': 10,
'index': 'logstash'
}
mock_res = {
'aggregations': {
'filtered': {
'values': {
'buckets': [{
'key': 'key1',
'doc_count': 1
}, {
'key': 'key2',
'doc_count': 5
}]
}
}
}
}
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 2
# Key1 and key2 shouldn't cause a match
rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2'}])
assert rule.matches == []
# Neither will missing values
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert rule.matches == []
# Key3 causes an alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'b'
assert rule.matches[0]['b'] == 'key3'
rule.matches = []
# Key3 doesn't cause another alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert rule.matches == []
# Missing_field
rules['alert_on_missing_field'] = True
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert len(rule.matches) == 1
assert rule.matches[0]['missing_field'] == 'b'
def test_new_term():
rules = {
"fields": ["a", "b"],
"timestamp_field": "@timestamp",
"es_host": "example.com",
"es_port": 10,
"index": "logstash",
}
mock_res = {
"aggregations": {
"filtered": {"values": {"buckets": [{"key": "key1", "doc_count": 1}, {"key": "key2", "doc_count": 5}]}}
}
}
with mock.patch("elastalert.ruletypes.Elasticsearch") as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 2
# Key1 and key2 shouldn't cause a match
rule.add_data([{"@timestamp": ts_now(), "a": "key1", "b": "key2"}])
assert rule.matches == []
# Neither will missing values
rule.add_data([{"@timestamp": ts_now(), "a": "key2"}])
assert rule.matches == []
# Key3 causes an alert for field b
rule.add_data([{"@timestamp": ts_now(), "a": "key2", "b": "key3"}])
assert len(rule.matches) == 1
assert rule.matches[0]["new_field"] == "b"
assert rule.matches[0]["b"] == "key3"
rule.matches = []
# Key3 doesn't cause another alert for field b
rule.add_data([{"@timestamp": ts_now(), "a": "key2", "b": "key3"}])
assert rule.matches == []
# Missing_field
rules["alert_on_missing_field"] = True
with mock.patch("elastalert.ruletypes.Elasticsearch") as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
rule.add_data([{"@timestamp": ts_now(), "a": "key2"}])
assert len(rule.matches) == 1
assert rule.matches[0]["missing_field"] == "b"
def test_new_term_nested_field():
rules = {
'fields': ['a', 'b.c'],
'timestamp_field': '@timestamp',
'es_host': 'example.com',
'es_port': 10,
'index': 'logstash',
'ts_to_dt': ts_to_dt,
'dt_to_ts': dt_to_ts
}
mock_res = {
'aggregations': {
'filtered': {
'values': {
'buckets': [{
'key': 'key1',
'doc_count': 1
}, {
'key': 'key2',
'doc_count': 5
}]
}
}
}
}
with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
mock_es.return_value.info.return_value = {
'version': {
'number': '2.x.x'
}
}
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 60
# Key3 causes an alert for nested field b.c
rule.add_data([{'@timestamp': ts_now(), 'b': {'c': 'key3'}}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'b.c'
assert rule.matches[0]['b']['c'] == 'key3'
rule.matches = []
def test_new_term():
rules = {'fields': ['a', 'b'],
'timestamp_field': '@timestamp',
'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}
mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
{'key': 'key2', 'doc_count': 5}]}}}}
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 2
# Key1 and key2 shouldn't cause a match
rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2'}])
assert rule.matches == []
# Neither will missing values
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert rule.matches == []
# Key3 causes an alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'b'
assert rule.matches[0]['b'] == 'key3'
rule.matches = []
# Key3 doesn't cause another alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert rule.matches == []
# Missing_field
rules['alert_on_missing_field'] = True
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert len(rule.matches) == 1
assert rule.matches[0]['missing_field'] == 'b'
def test_new_term_nested_field():
rules = {'fields': ['a', 'b.c'],
'timestamp_field': '@timestamp',
'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}
mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
{'key': 'key2', 'doc_count': 5}]}}}}
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 2
# Key3 causes an alert for nested field b.c
rule.add_data([{'@timestamp': ts_now(), 'b': {'c': 'key3'}}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'b.c'
assert rule.matches[0]['b']['c'] == 'key3'
rule.matches = []
def test_new_term_with_composite_fields():
rules = {
'fields': [['a', 'b', 'c'], ['d', 'e.f']],
'timestamp_field': '@timestamp',
'es_host': 'example.com',
'es_port': 10,
'index': 'logstash'
}
mock_res = {
'aggregations': {
'filtered': {
'values': {
'buckets': [{
'key': 'key1',
'doc_count': 5,
'values': {
'buckets': [{
'key': 'key2',
'doc_count': 5,
'values': {
'buckets': [
{
'key': 'key3',
'doc_count': 3,
},
{
'key': 'key4',
'doc_count': 2,
},
]
}
}]
}
}]
}
}
}
}
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 2
# key3 already exists, and thus shouldn't cause a match
rule.add_data([{
'@timestamp': ts_now(),
'a': 'key1',
'b': 'key2',
'c': 'key3'
}])
assert rule.matches == []
# key5 causes an alert for composite field [a, b, c]
rule.add_data([{
'@timestamp': ts_now(),
'a': 'key1',
'b': 'key2',
'c': 'key5'
}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == ('a', 'b', 'c')
assert rule.matches[0]['a'] == 'key1'
assert rule.matches[0]['b'] == 'key2'
assert rule.matches[0]['c'] == 'key5'
rule.matches = []
# New values in other fields that are not part of the composite key should not cause an alert
rule.add_data([{
'@timestamp': ts_now(),
'a': 'key1',
'b': 'key2',
'c': 'key4',
'd': 'unrelated_value'
}])
assert len(rule.matches) == 0
rule.matches = []
# Verify nested fields work properly
# Key6 causes an alert for nested field e.f
rule.add_data([{'@timestamp': ts_now(), 'd': 'key4', 'e': {'f': 'key6'}}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == ('d', 'e.f')
assert rule.matches[0]['d'] == 'key4'
assert rule.matches[0]['e']['f'] == 'key6'
rule.matches = []
# Missing_fields
rules['alert_on_missing_field'] = True
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert len(rule.matches) == 2
# This means that any one of the three n composite fields were not present
assert rule.matches[0]['missing_field'] == ('a', 'b', 'c')
assert rule.matches[1]['missing_field'] == ('d', 'e.f')
def test_new_term():
rules = {
'fields': ['a', 'b'],
'timestamp_field': '@timestamp',
'es_host': 'example.com',
'es_port': 10,
'index': 'logstash'
}
mock_res = {
'aggregations': {
'filtered': {
'values': {
'buckets': [{
'key': 'key1',
'doc_count': 1
}, {
'key': 'key2',
'doc_count': 5
}]
}
}
}
}
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
call_args = []
# search is called with a mutable dict containing timestamps, this is required to test
def record_args(*args, **kwargs):
call_args.append((copy.deepcopy(args), copy.deepcopy(kwargs)))
return mock_res
mock_es.return_value.search.side_effect = record_args
rule = NewTermsRule(rules)
# 30 day default range, 1 day default step, times 2 fields
assert rule.es.search.call_count == 60
# Assert that all calls have the proper ordering of time ranges
old_ts = '2010-01-01T00:00:00Z'
old_field = ''
for call in call_args:
field = call[1]['body']['aggs']['filtered']['aggs']['values']['terms'][
'field']
if old_field != field:
old_field = field
old_ts = '2010-01-01T00:00:00Z'
gte = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0][
'range']['@timestamp']['gte']
assert gte > old_ts
lt = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0][
'range']['@timestamp']['lt']
assert lt > gte
old_ts = gte
# Key1 and key2 shouldn't cause a match
rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2'}])
assert rule.matches == []
# Neither will missing values
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert rule.matches == []
# Key3 causes an alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'b'
assert rule.matches[0]['b'] == 'key3'
rule.matches = []
# Key3 doesn't cause another alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert rule.matches == []
# Missing_field
rules['alert_on_missing_field'] = True
with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
rule = NewTermsRule(rules)
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert len(rule.matches) == 1
assert rule.matches[0]['missing_field'] == 'b'
def test_new_term_with_composite_fields():
rules = {'fields': [['a', 'b', 'c'], ['d', 'e.f']],
'timestamp_field': '@timestamp',
'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}
mock_res = {
'aggregations': {
'filtered': {
'values': {
'buckets': [
{
'key': 'key1',
'doc_count': 5,
'values': {
'buckets': [
{
'key': 'key2',
'doc_count': 5,
'values': {
'buckets': [
{
'key': 'key3',
'doc_count': 3,
},
{
'key': 'key4',
'doc_count': 2,
},
]
}
}
]
}
}
]
}
}
}
}
with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
rule = NewTermsRule(rules)
assert rule.es.search.call_count == 60
# key3 already exists, and thus shouldn't cause a match
rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key3'}])
assert rule.matches == []
# key5 causes an alert for composite field [a, b, c]
rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key5'}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == ('a', 'b', 'c')
assert rule.matches[0]['a'] == 'key1'
assert rule.matches[0]['b'] == 'key2'
assert rule.matches[0]['c'] == 'key5'
rule.matches = []
# New values in other fields that are not part of the composite key should not cause an alert
rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key4', 'd': 'unrelated_value'}])
assert len(rule.matches) == 0
rule.matches = []
# Verify nested fields work properly
# Key6 causes an alert for nested field e.f
rule.add_data([{'@timestamp': ts_now(), 'd': 'key4', 'e': {'f': 'key6'}}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == ('d', 'e.f')
assert rule.matches[0]['d'] == 'key4'
assert rule.matches[0]['e']['f'] == 'key6'
rule.matches = []
# Missing_fields
rules['alert_on_missing_field'] = True
with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
rule = NewTermsRule(rules)
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert len(rule.matches) == 2
# This means that any one of the three n composite fields were not present
assert rule.matches[0]['missing_field'] == ('a', 'b', 'c')
assert rule.matches[1]['missing_field'] == ('d', 'e.f')
def test_new_term():
rules = {'fields': ['a', 'b'],
'timestamp_field': '@timestamp',
'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}
mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
{'key': 'key2', 'doc_count': 5}]}}}}
with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
call_args = []
# search is called with a mutable dict containing timestamps, this is required to test
def record_args(*args, **kwargs):
call_args.append((copy.deepcopy(args), copy.deepcopy(kwargs)))
return mock_res
mock_es.return_value.search.side_effect = record_args
rule = NewTermsRule(rules)
# 30 day default range, 1 day default step, times 2 fields
assert rule.es.search.call_count == 60
# Assert that all calls have the proper ordering of time ranges
old_ts = '2010-01-01T00:00:00Z'
old_field = ''
for call in call_args:
field = call[1]['body']['aggs']['filtered']['aggs']['values']['terms']['field']
if old_field != field:
old_field = field
old_ts = '2010-01-01T00:00:00Z'
gte = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gte']
assert gte > old_ts
lt = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lt']
assert lt > gte
old_ts = gte
# Key1 and key2 shouldn't cause a match
rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2'}])
assert rule.matches == []
# Neither will missing values
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert rule.matches == []
# Key3 causes an alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert len(rule.matches) == 1
assert rule.matches[0]['new_field'] == 'b'
assert rule.matches[0]['b'] == 'key3'
rule.matches = []
# Key3 doesn't cause another alert for field b
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
assert rule.matches == []
# Missing_field
rules['alert_on_missing_field'] = True
with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
mock_es.return_value = mock.Mock()
mock_es.return_value.search.return_value = mock_res
mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
rule = NewTermsRule(rules)
rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
assert len(rule.matches) == 1
assert rule.matches[0]['missing_field'] == 'b'