def test_dt_to_ts(caplog):
caplog.set_level(logging.WARNING)
dt_to_ts('a')
user, level, message = caplog.record_tuples[0]
assert 'elastalert' == user
assert logging.WARNING == level
assert 'Expected datetime, got' in message
def test_agg_cron(ea):
ea.max_aggregation = 1337
hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']
hits = generate_hits(hits_timestamps)
ea.current_es.search.return_value = hits
alerttime1 = dt_to_ts(ts_to_dt('2014-09-26T12:46:00'))
alerttime2 = dt_to_ts(ts_to_dt('2014-09-26T13:04:00'))
with mock.patch('elastalert.elastalert.Elasticsearch'):
with mock.patch('elastalert.elastalert.croniter.get_next') as mock_ts:
# Aggregate first two, query over full range
mock_ts.side_effect = [dt_to_unix(ts_to_dt('2014-09-26T12:46:00')), dt_to_unix(ts_to_dt('2014-09-26T13:04:00'))]
ea.rules[0]['aggregation'] = {'schedule': '*/5 * * * *'}
ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
ea.run_rule(ea.rules[0], END, START)
# Assert that the three matches were added to elasticsearch
call1 = ea.writeback_es.create.call_args_list[0][1]['body']
call2 = ea.writeback_es.create.call_args_list[1][1]['body']
call3 = ea.writeback_es.create.call_args_list[2][1]['body']
assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
assert not call1['alert_sent']
assert 'aggregate_id' not in call1
assert call1['alert_time'] == alerttime1
assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
assert not call2['alert_sent']
assert call2['aggregate_id'] == 'ABCD'
assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
assert call3['alert_time'] == alerttime2
assert not call3['alert_sent']
assert 'aggregate_id' not in call3
def run_and_assert_segmented_queries(ea, start, end, segment_size):
with mock.patch.object(ea, 'run_query') as mock_run_query:
ea.run_rule(ea.rules[0], end, start)
original_end, original_start = end, start
for call_args in mock_run_query.call_args_list:
end = min(start + segment_size, original_end)
assert call_args[0][1:3] == (start, end)
start += segment_size
# Assert elastalert_status was created for the entire time range
assert ea.writeback_es.create.call_args_list[-1][1]['body']['starttime'] == dt_to_ts(original_start)
assert ea.writeback_es.create.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(original_end)
def test_agg(ea):
hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'
alerttime1 = dt_to_ts(ts_to_dt(hit1) + datetime.timedelta(minutes=10))
hits = generate_hits([hit1, hit2, hit3])
ea.current_es.search.return_value = hits
with mock.patch('elastalert.elastalert.Elasticsearch'):
# Aggregate first two, query over full range
ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
ea.rules[0]['type'].matches = [{'@timestamp': hit1},
{'@timestamp': hit2},
{'@timestamp': hit3}]
ea.run_rule(ea.rules[0], END, START)
# Assert that the three matches were added to elasticsearch
call1 = ea.writeback_es.create.call_args_list[0][1]['body']
call2 = ea.writeback_es.create.call_args_list[1][1]['body']
call3 = ea.writeback_es.create.call_args_list[2][1]['body']
assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
assert not call1['alert_sent']
assert 'aggregate_id' not in call1
assert call1['alert_time'] == alerttime1
assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
assert not call2['alert_sent']
assert call2['aggregate_id'] == 'ABCD'
assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
assert not call3['alert_sent']
assert 'aggregate_id' not in call3
# First call - Find all pending alerts
# Second call - Find matches with agg_id == 'ABCD'
# Third call - Find matches with agg_id == 'CDEF'
ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1},
{'_id': 'BCDE', '_source': call2},
{'_id': 'CDEF', '_source': call3}]}},
{'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}},
{'hits': {'hits': []}}]
ea.send_pending_alerts()
assert_alerts(ea, [[dt_to_ts(hit1), dt_to_ts(hit2)], [dt_to_ts(hit3)]])
call1 = ea.writeback_es.search.call_args_list[6][1]['body']
call2 = ea.writeback_es.search.call_args_list[7][1]['body']
call3 = ea.writeback_es.search.call_args_list[8][1]['body']
assert 'alert_time' in call1['filter']['range']
assert call2['query']['query_string']['query'] == 'aggregate_id:ABCD'
assert call3['query']['query_string']['query'] == 'aggregate_id:CDEF'
def test_agg_not_matchtime(ea):
ea.max_aggregation = 1337
hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']
match_time = ts_to_dt('2014-09-26T12:55:00Z')
hits = generate_hits(hits_timestamps)
ea.current_es.search.return_value = hits
with mock.patch('elastalert.elastalert.elasticsearch_client'):
with mock.patch('elastalert.elastalert.ts_now', return_value=match_time):
ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
ea.run_rule(ea.rules[0], END, START)
# Assert that the three matches were added to Elasticsearch
call1 = ea.writeback_es.index.call_args_list[0][1]['body']
call2 = ea.writeback_es.index.call_args_list[1][1]['body']
call3 = ea.writeback_es.index.call_args_list[2][1]['body']
assert call1['match_body']['@timestamp'] == '2014-09-26T12:34:45'
assert not call1['alert_sent']
assert 'aggregate_id' not in call1
assert call1['alert_time'] == dt_to_ts(match_time + datetime.timedelta(minutes=10))
assert call2['match_body']['@timestamp'] == '2014-09-26T12:40:45'
assert not call2['alert_sent']
assert call2['aggregate_id'] == 'ABCD'
assert call3['match_body']['@timestamp'] == '2014-09-26T12:47:45'
assert not call3['alert_sent']
assert call3['aggregate_id'] == 'ABCD'
def test_silence_query_key(ea):
# Silence test rule for 4 hours
ea.args.rule = 'test_rule.yaml' # Not a real name, just has to be set
ea.args.silence = 'hours=4'
ea.silence('anytest.qlo')
# Don't alert even with a match
match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
ea.rules[0]['type'].matches = match
ea.rules[0]['query_key'] = 'username'
with mock.patch('elastalert.elastalert.elasticsearch_client'):
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 0
# If there is a new record with a different value for the query_key, we should get an alert
match = [{'@timestamp': '2014-11-17T00:00:01', 'username': '******'}]
ea.rules[0]['type'].matches = match
with mock.patch('elastalert.elastalert.elasticsearch_client'):
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 1
# Mock ts_now() to +5 hours, alert on match
match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
ea.rules[0]['type'].matches = match
with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
with mock.patch('elastalert.elastalert.elasticsearch_client'):
# Converted twice to add tzinfo
mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 2
def test_silence_query_key(ea):
# Silence test rule for 4 hours
ea.args.rule = 'test_rule.yaml' # Not a real name, just has to be set
ea.args.silence = 'hours=4'
ea.silence()
# Don't alert even with a match
match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
ea.rules[0]['type'].matches = match
ea.rules[0]['query_key'] = 'username'
with mock.patch('elastalert.elastalert.Elasticsearch'):
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 0
# Mock ts_now() to +5 hours, alert on match
match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
ea.rules[0]['type'].matches = match
with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
with mock.patch('elastalert.elastalert.Elasticsearch'):
# Converted twice to add tzinfo
mock_ts.return_value = ts_to_dt(
dt_to_ts(datetime.datetime.utcnow() +
datetime.timedelta(hours=5)))
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 1
def test_realert(ea):
hits = ['2014-09-26T12:35:%sZ' % (x) for x in range(60)]
matches = [{'@timestamp': x} for x in hits]
ea.current_es.search.return_value = hits
with mock.patch('elastalert.elastalert.Elasticsearch'):
ea.rules[0]['realert'] = datetime.timedelta(seconds=50)
ea.rules[0]['type'].matches = matches
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 1
# Doesn't alert again
matches = [{'@timestamp': x} for x in hits]
with mock.patch('elastalert.elastalert.Elasticsearch'):
ea.run_rule(ea.rules[0], END, START)
ea.rules[0]['type'].matches = matches
assert ea.rules[0]['alert'][0].alert.call_count == 1
# mock ts_now() to past the realert time
matches = [{'@timestamp': hits[0]}]
with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
with mock.patch('elastalert.elastalert.Elasticsearch'):
# mock_ts is converted twice to add tzinfo
mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(minutes=10)))
ea.rules[0]['type'].matches = matches
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 2
def test_queries_with_rule_buffertime(ea):
ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=53)
hits = generate_hits([START_TIMESTAMP])
mock_es = mock.Mock()
mock_es.search.return_value = hits
with mock.patch('elastalert.elastalert.Elasticsearch') as mock_es_init:
mock_es_init.return_value = mock_es
ea.run_rule(ea.rules[0], END, START)
# Assert that es.search is run against every run_every timeframe between START and END
end = END_TIMESTAMP
start = START
query = {
'filter': {
'bool': {
'must': [{
'range': {
'@timestamp': {
'to': END_TIMESTAMP,
'from': START_TIMESTAMP
}
}
}]
}
},
'sort': [{
'@timestamp': {
'order': 'asc'
}
}]
}
while END - start > ea.rules[0]['buffer_time']:
end = start + ea.run_every
query['filter']['bool']['must'][0]['range']['@timestamp'][
'to'] = dt_to_ts(end)
query['filter']['bool']['must'][0]['range']['@timestamp'][
'from'] = dt_to_ts(start)
start = start + ea.run_every
ea.current_es.search.assert_any_call(body=query,
size=ea.max_query_size,
index='idx',
ignore_unavailable=True,
_source_include=['@timestamp'])
# Assert that num_hits correctly summed every result
assert ea.num_hits == ea.current_es.search.call_count
def test_agg(ea):
hits_timestamps = ["2014-09-26T12:34:45", "2014-09-26T12:40:45", "2014-09-26T12:47:45"]
alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10))
hits = generate_hits(hits_timestamps)
ea.current_es.search.return_value = hits
with mock.patch("elastalert.elastalert.Elasticsearch"):
# Aggregate first two, query over full range
ea.rules[0]["aggregation"] = datetime.timedelta(minutes=10)
ea.rules[0]["type"].matches = [{"@timestamp": h} for h in hits_timestamps]
ea.run_rule(ea.rules[0], END, START)
# Assert that the three matches were added to elasticsearch
call1 = ea.writeback_es.create.call_args_list[0][1]["body"]
call2 = ea.writeback_es.create.call_args_list[1][1]["body"]
call3 = ea.writeback_es.create.call_args_list[2][1]["body"]
assert call1["match_body"] == {"@timestamp": "2014-09-26T12:34:45"}
assert not call1["alert_sent"]
assert "aggregate_id" not in call1
assert call1["alert_time"] == alerttime1
assert call2["match_body"] == {"@timestamp": "2014-09-26T12:40:45"}
assert not call2["alert_sent"]
assert call2["aggregate_id"] == "ABCD"
assert call3["match_body"] == {"@timestamp": "2014-09-26T12:47:45"}
assert not call3["alert_sent"]
assert "aggregate_id" not in call3
# First call - Find all pending alerts
# Second call - Find matches with agg_id == 'ABCD'
# Third call - Find matches with agg_id == 'CDEF'
ea.writeback_es.search.side_effect = [
{
"hits": {
"hits": [
{"_id": "ABCD", "_source": call1},
{"_id": "BCDE", "_source": call2},
{"_id": "CDEF", "_source": call3},
]
}
},
{"hits": {"hits": [{"_id": "BCDE", "_source": call2}]}},
{"hits": {"hits": []}},
]
ea.send_pending_alerts()
assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]])
call1 = ea.writeback_es.search.call_args_list[6][1]["body"]
call2 = ea.writeback_es.search.call_args_list[7][1]["body"]
call3 = ea.writeback_es.search.call_args_list[8][1]["body"]
assert "alert_time" in call1["filter"]["range"]
assert call2["query"]["query_string"]["query"] == "aggregate_id:ABCD"
assert call3["query"]["query_string"]["query"] == "aggregate_id:CDEF"
def get_match_str(self, match):
ts = match[self.rules['timestamp_field']]
lt = self.rules.get('use_local_time')
key = match.get('key', 'all')
message = 'An abnormally low number of events occurred around %s.\n' % (
pretty_ts(ts, lt))
message += 'Between %s and %s, there were less than %s events.\n\n' % (
pretty_ts(dt_to_ts(ts_to_dt(ts) - self.timeframe(key)),
lt), pretty_ts(ts, lt), self.rules['threshold'])
message = json.dumps(match)
return message
def get_match_str(self, match):
lt = self.rules.get('use_local_time')
match_ts = lookup_es_key(match, self.ts_field)
key = match.get('key', 'all')
starttime = pretty_ts(
dt_to_ts(ts_to_dt(match_ts) - self.timeframe(key)), lt)
endtime = pretty_ts(match_ts, lt)
message = 'At least %d events occurred between %s and %s\n\n' % (
self.rules['num_events'], starttime, endtime)
message = json.dumps(match)
return message
def test_agg(ea):
ea.max_aggregation = 1337
hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']
alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10))
hits = generate_hits(hits_timestamps)
ea.current_es.search.return_value = hits
with mock.patch('elastalert.elastalert.Elasticsearch'):
# Aggregate first two, query over full range
ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
ea.run_rule(ea.rules[0], END, START)
# Assert that the three matches were added to elasticsearch
call1 = ea.writeback_es.create.call_args_list[0][1]['body']
call2 = ea.writeback_es.create.call_args_list[1][1]['body']
call3 = ea.writeback_es.create.call_args_list[2][1]['body']
assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
assert not call1['alert_sent']
assert 'aggregate_id' not in call1
assert call1['alert_time'] == alerttime1
assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
assert not call2['alert_sent']
assert call2['aggregate_id'] == 'ABCD'
assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
assert not call3['alert_sent']
assert 'aggregate_id' not in call3
# First call - Find all pending alerts
# Second call - Find matches with agg_id == 'ABCD'
# Third call - Find matches with agg_id == 'CDEF'
ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1},
{'_id': 'BCDE', '_source': call2},
{'_id': 'CDEF', '_source': call3}]}},
{'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}},
{'hits': {'hits': []}}]
with mock.patch('elastalert.elastalert.Elasticsearch') as mock_es:
ea.send_pending_alerts()
# Assert that current_es was refreshed from the aggregate rules
assert mock_es.called_with(host='', port='')
assert mock_es.call_count == 2
assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]])
call1 = ea.writeback_es.search.call_args_list[6][1]['body']
call2 = ea.writeback_es.search.call_args_list[7][1]['body']
call3 = ea.writeback_es.search.call_args_list[8][1]['body']
assert 'alert_time' in call1['filter']['range']
assert call2['query']['query_string']['query'] == 'aggregate_id:ABCD'
assert call3['query']['query_string']['query'] == 'aggregate_id:CDEF'
assert ea.writeback_es.search.call_args_list[7][1]['size'] == 1337
def test_queries_with_rule_buffertime(ea):
ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=53)
mock_es = mock.Mock()
mock_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP])
with mock.patch('elastalert.elastalert.Elasticsearch') as mock_es_init:
mock_es_init.return_value = mock_es
ea.run_rule(ea.rules[0], END, START)
# Assert that es.search is run against every run_every timeframe between START and END
end = END_TIMESTAMP
start = START
query = {'filter': {'bool': {'must': [{'range': {'@timestamp': {'to': END_TIMESTAMP, 'from': START_TIMESTAMP}}}]}},
'sort': [{'@timestamp': {'order': 'asc'}}]}
while END - start > ea.rules[0]['buffer_time']:
end = start + ea.run_every
query['filter']['bool']['must'][0]['range']['@timestamp']['to'] = dt_to_ts(end)
query['filter']['bool']['must'][0]['range']['@timestamp']['from'] = dt_to_ts(start)
start = start + ea.run_every
ea.current_es.search.assert_any_call(body=query, size=ea.max_query_size, index='idx', ignore_unavailable=True, _source_include=['@timestamp'])
# Assert that num_hits correctly summed every result
assert ea.num_hits == ea.current_es.search.call_count
def process(self, match):
self.local_time = self.rule.get('local_time_feild', 'local_time')
self.local_starttime = self.rule.get('local_starttime_feild',
'local_starttime')
self.local_endtime = self.rule.get('local_endtime_feild',
'local_endtime')
self.ts_field = self.rule.get('timestamp_field', '@timestamp')
lt = self.rule.get('use_local_time', "False")
match_ts = match[self.ts_field]
match[self.local_time] = pretty_ts(match_ts, lt)
match[self.local_starttime] = pretty_ts(
dt_to_ts(ts_to_dt(match_ts) - self.rule['timeframe']), lt)
match[self.local_endtime] = match[self.local_time]
def test_count(ea):
ea.rules[0]['use_count_query'] = True
ea.rules[0]['doc_type'] = 'doctype'
with mock.patch('elastalert.elastalert.Elasticsearch'):
ea.run_rule(ea.rules[0], END, START)
# Assert that es.count is run against every run_every timeframe between START and END
start = START
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'range': {
'@timestamp': {
'lte': END_TIMESTAMP,
'gt': START_TIMESTAMP
}
}
}]
}
}
}
}
}
while END - start > ea.run_every:
end = start + ea.run_every
query['query']['filtered']['filter']['bool']['must'][0]['range'][
'@timestamp']['lte'] = dt_to_ts(end)
query['query']['filtered']['filter']['bool']['must'][0]['range'][
'@timestamp']['gt'] = dt_to_ts(start)
start = start + ea.run_every
ea.current_es.count.assert_any_call(body=query,
doc_type='doctype',
index='idx',
ignore_unavailable=True)
def test_silence(ea):
# Silence test rule for 4 hours
ea.args.rule = 'test_rule.yaml' # Not a real name, just has to be set
ea.args.silence = 'hours=4'
ea.silence()
# Don't alert even with a match
match = [{'@timestamp': '2014-11-17T00:00:00'}]
ea.rules[0]['type'].matches = match
with mock.patch('elastalert.elastalert.Elasticsearch'):
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 0
# Mock ts_now() to +5 hours, alert on match
match = [{'@timestamp': '2014-11-17T00:00:00'}]
ea.rules[0]['type'].matches = match
with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
with mock.patch('elastalert.elastalert.Elasticsearch'):
# Converted twice to add tzinfo
mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]['alert'][0].alert.call_count == 1
def test_silence_query_key(ea):
# Silence test rule for 4 hours
ea.args.rule = "test_rule.yaml" # Not a real name, just has to be set
ea.args.silence = "hours=4"
ea.silence()
# Don't alert even with a match
match = [{"@timestamp": "2014-11-17T00:00:00", "username": "******"}]
ea.rules[0]["type"].matches = match
ea.rules[0]["query_key"] = "username"
with mock.patch("elastalert.elastalert.Elasticsearch"):
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]["alert"][0].alert.call_count == 0
# Mock ts_now() to +5 hours, alert on match
match = [{"@timestamp": "2014-11-17T00:00:00", "username": "******"}]
ea.rules[0]["type"].matches = match
with mock.patch("elastalert.elastalert.ts_now") as mock_ts:
with mock.patch("elastalert.elastalert.Elasticsearch"):
# Converted twice to add tzinfo
mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
ea.run_rule(ea.rules[0], END, START)
assert ea.rules[0]["alert"][0].alert.call_count == 1
def test_count(ea):
ea.rules[0]['use_count_query'] = True
ea.rules[0]['doc_type'] = 'doctype'
with mock.patch('elastalert.elastalert.Elasticsearch'):
ea.run_rule(ea.rules[0], END, START)
# Assert that es.count is run against every run_every timeframe between START and END
start = START
query = {'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}}
while END - start > ea.run_every:
end = start + ea.run_every
query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end)
query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start)
start = start + ea.run_every
ea.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True)
def test_count(ea):
ea.rules[0]["use_count_query"] = True
ea.rules[0]["doc_type"] = "doctype"
with mock.patch("elastalert.elastalert.Elasticsearch"):
ea.run_rule(ea.rules[0], END, START)
# Assert that es.count is run against every run_every timeframe between START and END
start = START
query = {
"query": {
"filtered": {
"filter": {
"bool": {"must": [{"range": {"@timestamp": {"to": END_TIMESTAMP, "from": START_TIMESTAMP}}}]}
}
}
}
}
while END - start > ea.run_every:
end = start + ea.run_every
query["query"]["filtered"]["filter"]["bool"]["must"][0]["range"]["@timestamp"]["to"] = dt_to_ts(end)
query["query"]["filtered"]["filter"]["bool"]["must"][0]["range"]["@timestamp"]["from"] = dt_to_ts(start)
start = start + ea.run_every
ea.current_es.count.assert_any_call(body=query, doc_type="doctype", index="idx", ignore_unavailable=True)
def test_agg(ea):
hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'
alerttime1 = dt_to_ts(ts_to_dt(hit1) + datetime.timedelta(minutes=10))
hits = generate_hits([hit1, hit2, hit3])
ea.current_es.search.return_value = hits
with mock.patch('elastalert.elastalert.Elasticsearch'):
# Aggregate first two, query over full range
ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
ea.rules[0]['type'].matches = [{
'@timestamp': hit1
}, {
'@timestamp': hit2
}, {
'@timestamp': hit3
}]
ea.run_rule(ea.rules[0], END, START)
# Assert that the three matches were added to elasticsearch
call1 = ea.writeback_es.create.call_args_list[0][1]['body']
call2 = ea.writeback_es.create.call_args_list[1][1]['body']
call3 = ea.writeback_es.create.call_args_list[2][1]['body']
assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
assert not call1['alert_sent']
assert 'aggregate_id' not in call1
assert call1['alert_time'] == alerttime1
assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
assert not call2['alert_sent']
assert call2['aggregate_id'] == 'ABCD'
assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
assert not call3['alert_sent']
assert 'aggregate_id' not in call3
# First call - Find all pending alerts
# Second call - Find matches with agg_id == 'ABCD'
# Third call - Find matches with agg_id == 'CDEF'
ea.writeback_es.search.side_effect = [{
'hits': {
'hits': [{
'_id': 'ABCD',
'_source': call1
}, {
'_id': 'BCDE',
'_source': call2
}, {
'_id': 'CDEF',
'_source': call3
}]
}
}, {
'hits': {
'hits': [{
'_id': 'BCDE',
'_source': call2
}]
}
}, {
'hits': {
'hits': []
}
}]
ea.send_pending_alerts()
assert_alerts(ea, [[dt_to_ts(hit1), dt_to_ts(hit2)], [dt_to_ts(hit3)]])
call1 = ea.writeback_es.search.call_args_list[6][1]['body']
call2 = ea.writeback_es.search.call_args_list[7][1]['body']
call3 = ea.writeback_es.search.call_args_list[8][1]['body']
assert 'alert_time' in call1['filter']['range']
assert call2['query']['query_string']['query'] == 'aggregate_id:ABCD'
assert call3['query']['query_string']['query'] == 'aggregate_id:CDEF'
def test_agg_with_aggregation_key(ea):
ea.max_aggregation = 1337
hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:43:45']
alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10))
alerttime2 = dt_to_ts(ts_to_dt(hits_timestamps[1]) + datetime.timedelta(minutes=10))
hits = generate_hits(hits_timestamps)
ea.current_es.search.return_value = hits
with mock.patch('elastalert.elastalert.elasticsearch_client'):
ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
# Hit1 and Hit3 should be aggregated together, since they have same query_key value
ea.rules[0]['type'].matches[0]['key'] = 'Key Value 1'
ea.rules[0]['type'].matches[1]['key'] = 'Key Value 2'
ea.rules[0]['type'].matches[2]['key'] = 'Key Value 1'
ea.rules[0]['aggregation_key'] = 'key'
ea.run_rule(ea.rules[0], END, START)
# Assert that the three matches were added to elasticsearch
call1 = ea.writeback_es.create.call_args_list[0][1]['body']
call2 = ea.writeback_es.create.call_args_list[1][1]['body']
call3 = ea.writeback_es.create.call_args_list[2][1]['body']
assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45', 'key': 'Key Value 1'}
assert not call1['alert_sent']
assert 'aggregate_id' not in call1
assert 'aggregate_key' in call1
assert call1['aggregate_key'] == 'Key Value 1'
assert call1['alert_time'] == alerttime1
assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45', 'key': 'Key Value 2'}
assert not call2['alert_sent']
assert 'aggregate_id' not in call2
assert 'aggregate_key' in call2
assert call2['aggregate_key'] == 'Key Value 2'
assert call2['alert_time'] == alerttime2
assert call3['match_body'] == {'@timestamp': '2014-09-26T12:43:45', 'key': 'Key Value 1', 'key': 'Key Value 1'}
assert not call3['alert_sent']
# Call3 should have it's aggregate_id set to call1's _id
# It should also have the same alert_time as call1
assert call3['aggregate_id'] == 'ABCD'
assert 'aggregate_key' in call3
assert call3['aggregate_key'] == 'Key Value 1'
assert call3['alert_time'] == alerttime1
# First call - Find all pending alerts (only entries without agg_id)
# Second call - Find matches with agg_id == 'ABCD'
# Third call - Find matches with agg_id == 'CDEF'
ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1},
{'_id': 'CDEF', '_source': call2}]}},
{'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}},
{'hits': {'total': 0, 'hits': []}}]
with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:
ea.send_pending_alerts()
# Assert that current_es was refreshed from the aggregate rules
assert mock_es.called_with(host='', port='')
assert mock_es.call_count == 2
assert_alerts(ea, [[hits_timestamps[0], hits_timestamps[2]], [hits_timestamps[1]]])
call1 = ea.writeback_es.search.call_args_list[7][1]['body']
call2 = ea.writeback_es.search.call_args_list[8][1]['body']
call3 = ea.writeback_es.search.call_args_list[9][1]['body']
call4 = ea.writeback_es.search.call_args_list[10][1]['body']
assert 'alert_time' in call2['filter']['range']
assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD'
assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF'
assert ea.writeback_es.search.call_args_list[9][1]['size'] == 1337