Пример #1
0
def test_dt_to_ts(caplog):
    caplog.set_level(logging.WARNING)
    dt_to_ts('a')
    user, level, message = caplog.record_tuples[0]
    assert 'elastalert' == user
    assert logging.WARNING == level
    assert 'Expected datetime, got' in message
Пример #2
0
def test_agg_cron(ea):
    ea.max_aggregation = 1337
    hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']
    hits = generate_hits(hits_timestamps)
    ea.current_es.search.return_value = hits
    alerttime1 = dt_to_ts(ts_to_dt('2014-09-26T12:46:00'))
    alerttime2 = dt_to_ts(ts_to_dt('2014-09-26T13:04:00'))

    with mock.patch('elastalert.elastalert.Elasticsearch'):
        with mock.patch('elastalert.elastalert.croniter.get_next') as mock_ts:
            # Aggregate first two, query over full range
            mock_ts.side_effect = [dt_to_unix(ts_to_dt('2014-09-26T12:46:00')), dt_to_unix(ts_to_dt('2014-09-26T13:04:00'))]
            ea.rules[0]['aggregation'] = {'schedule': '*/5 * * * *'}
            ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
            ea.run_rule(ea.rules[0], END, START)

    # Assert that the three matches were added to elasticsearch
    call1 = ea.writeback_es.create.call_args_list[0][1]['body']
    call2 = ea.writeback_es.create.call_args_list[1][1]['body']
    call3 = ea.writeback_es.create.call_args_list[2][1]['body']

    assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
    assert not call1['alert_sent']
    assert 'aggregate_id' not in call1
    assert call1['alert_time'] == alerttime1

    assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
    assert not call2['alert_sent']
    assert call2['aggregate_id'] == 'ABCD'

    assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
    assert call3['alert_time'] == alerttime2
    assert not call3['alert_sent']
    assert 'aggregate_id' not in call3
Пример #3
0
def run_and_assert_segmented_queries(ea, start, end, segment_size):
    with mock.patch.object(ea, 'run_query') as mock_run_query:
        ea.run_rule(ea.rules[0], end, start)
        original_end, original_start = end, start
        for call_args in mock_run_query.call_args_list:
            end = min(start + segment_size, original_end)
            assert call_args[0][1:3] == (start, end)
            start += segment_size

        # Assert elastalert_status was created for the entire time range
        assert ea.writeback_es.create.call_args_list[-1][1]['body']['starttime'] == dt_to_ts(original_start)
        assert ea.writeback_es.create.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(original_end)
Пример #4
0
def test_agg(ea):
    hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'
    alerttime1 = dt_to_ts(ts_to_dt(hit1) + datetime.timedelta(minutes=10))
    hits = generate_hits([hit1, hit2, hit3])
    ea.current_es.search.return_value = hits
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        # Aggregate first two, query over full range
        ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
        ea.rules[0]['type'].matches = [{'@timestamp': hit1},
                                       {'@timestamp': hit2},
                                       {'@timestamp': hit3}]
        ea.run_rule(ea.rules[0], END, START)

    # Assert that the three matches were added to elasticsearch
    call1 = ea.writeback_es.create.call_args_list[0][1]['body']
    call2 = ea.writeback_es.create.call_args_list[1][1]['body']
    call3 = ea.writeback_es.create.call_args_list[2][1]['body']

    assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
    assert not call1['alert_sent']
    assert 'aggregate_id' not in call1
    assert call1['alert_time'] == alerttime1

    assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
    assert not call2['alert_sent']
    assert call2['aggregate_id'] == 'ABCD'

    assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
    assert not call3['alert_sent']
    assert 'aggregate_id' not in call3

    # First call - Find all pending alerts
    # Second call - Find matches with agg_id == 'ABCD'
    # Third call - Find matches with agg_id == 'CDEF'
    ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1},
                                                             {'_id': 'BCDE', '_source': call2},
                                                             {'_id': 'CDEF', '_source': call3}]}},
                                          {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}},
                                          {'hits': {'hits': []}}]
    ea.send_pending_alerts()
    assert_alerts(ea, [[dt_to_ts(hit1), dt_to_ts(hit2)], [dt_to_ts(hit3)]])

    call1 = ea.writeback_es.search.call_args_list[6][1]['body']
    call2 = ea.writeback_es.search.call_args_list[7][1]['body']
    call3 = ea.writeback_es.search.call_args_list[8][1]['body']

    assert 'alert_time' in call1['filter']['range']
    assert call2['query']['query_string']['query'] == 'aggregate_id:ABCD'
    assert call3['query']['query_string']['query'] == 'aggregate_id:CDEF'
Пример #5
0
def test_agg_not_matchtime(ea):
    ea.max_aggregation = 1337
    hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']
    match_time = ts_to_dt('2014-09-26T12:55:00Z')
    hits = generate_hits(hits_timestamps)
    ea.current_es.search.return_value = hits
    with mock.patch('elastalert.elastalert.elasticsearch_client'):
        with mock.patch('elastalert.elastalert.ts_now', return_value=match_time):
            ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
            ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
            ea.run_rule(ea.rules[0], END, START)

    # Assert that the three matches were added to Elasticsearch
    call1 = ea.writeback_es.index.call_args_list[0][1]['body']
    call2 = ea.writeback_es.index.call_args_list[1][1]['body']
    call3 = ea.writeback_es.index.call_args_list[2][1]['body']
    assert call1['match_body']['@timestamp'] == '2014-09-26T12:34:45'
    assert not call1['alert_sent']
    assert 'aggregate_id' not in call1
    assert call1['alert_time'] == dt_to_ts(match_time + datetime.timedelta(minutes=10))

    assert call2['match_body']['@timestamp'] == '2014-09-26T12:40:45'
    assert not call2['alert_sent']
    assert call2['aggregate_id'] == 'ABCD'

    assert call3['match_body']['@timestamp'] == '2014-09-26T12:47:45'
    assert not call3['alert_sent']
    assert call3['aggregate_id'] == 'ABCD'
Пример #6
0
def test_silence_query_key(ea):
    # Silence test rule for 4 hours
    ea.args.rule = 'test_rule.yaml'  # Not a real name, just has to be set
    ea.args.silence = 'hours=4'
    ea.silence('anytest.qlo')

    # Don't alert even with a match
    match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
    ea.rules[0]['type'].matches = match
    ea.rules[0]['query_key'] = 'username'
    with mock.patch('elastalert.elastalert.elasticsearch_client'):
        ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]['alert'][0].alert.call_count == 0

    # If there is a new record with a different value for the query_key, we should get an alert
    match = [{'@timestamp': '2014-11-17T00:00:01', 'username': '******'}]
    ea.rules[0]['type'].matches = match
    with mock.patch('elastalert.elastalert.elasticsearch_client'):
        ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]['alert'][0].alert.call_count == 1

    # Mock ts_now() to +5 hours, alert on match
    match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
    ea.rules[0]['type'].matches = match
    with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
        with mock.patch('elastalert.elastalert.elasticsearch_client'):
            # Converted twice to add tzinfo
            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
            ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]['alert'][0].alert.call_count == 2
Пример #7
0
def test_silence_query_key(ea):
    # Silence test rule for 4 hours
    ea.args.rule = 'test_rule.yaml'  # Not a real name, just has to be set
    ea.args.silence = 'hours=4'
    ea.silence()

    # Don't alert even with a match
    match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
    ea.rules[0]['type'].matches = match
    ea.rules[0]['query_key'] = 'username'
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]['alert'][0].alert.call_count == 0

    # Mock ts_now() to +5 hours, alert on match
    match = [{'@timestamp': '2014-11-17T00:00:00', 'username': '******'}]
    ea.rules[0]['type'].matches = match
    with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
        with mock.patch('elastalert.elastalert.Elasticsearch'):
            # Converted twice to add tzinfo
            mock_ts.return_value = ts_to_dt(
                dt_to_ts(datetime.datetime.utcnow() +
                         datetime.timedelta(hours=5)))
            ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]['alert'][0].alert.call_count == 1
Пример #8
0
def test_realert(ea):
    hits = ['2014-09-26T12:35:%sZ' % (x) for x in range(60)]
    matches = [{'@timestamp': x} for x in hits]
    ea.current_es.search.return_value = hits
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        ea.rules[0]['realert'] = datetime.timedelta(seconds=50)
        ea.rules[0]['type'].matches = matches
        ea.run_rule(ea.rules[0], END, START)
        assert ea.rules[0]['alert'][0].alert.call_count == 1

    # Doesn't alert again
    matches = [{'@timestamp': x} for x in hits]
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        ea.run_rule(ea.rules[0], END, START)
        ea.rules[0]['type'].matches = matches
        assert ea.rules[0]['alert'][0].alert.call_count == 1

    # mock ts_now() to past the realert time
    matches = [{'@timestamp': hits[0]}]
    with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
        with mock.patch('elastalert.elastalert.Elasticsearch'):
            # mock_ts is converted twice to add tzinfo
            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(minutes=10)))
            ea.rules[0]['type'].matches = matches
            ea.run_rule(ea.rules[0], END, START)
            assert ea.rules[0]['alert'][0].alert.call_count == 2
Пример #9
0
def test_queries_with_rule_buffertime(ea):
    ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=53)
    hits = generate_hits([START_TIMESTAMP])
    mock_es = mock.Mock()
    mock_es.search.return_value = hits
    with mock.patch('elastalert.elastalert.Elasticsearch') as mock_es_init:
        mock_es_init.return_value = mock_es
        ea.run_rule(ea.rules[0], END, START)

    # Assert that es.search is run against every run_every timeframe between START and END
    end = END_TIMESTAMP
    start = START
    query = {
        'filter': {
            'bool': {
                'must': [{
                    'range': {
                        '@timestamp': {
                            'to': END_TIMESTAMP,
                            'from': START_TIMESTAMP
                        }
                    }
                }]
            }
        },
        'sort': [{
            '@timestamp': {
                'order': 'asc'
            }
        }]
    }
    while END - start > ea.rules[0]['buffer_time']:
        end = start + ea.run_every
        query['filter']['bool']['must'][0]['range']['@timestamp'][
            'to'] = dt_to_ts(end)
        query['filter']['bool']['must'][0]['range']['@timestamp'][
            'from'] = dt_to_ts(start)
        start = start + ea.run_every
        ea.current_es.search.assert_any_call(body=query,
                                             size=ea.max_query_size,
                                             index='idx',
                                             ignore_unavailable=True,
                                             _source_include=['@timestamp'])

    # Assert that num_hits correctly summed every result
    assert ea.num_hits == ea.current_es.search.call_count
Пример #10
0
def test_agg(ea):
    hits_timestamps = ["2014-09-26T12:34:45", "2014-09-26T12:40:45", "2014-09-26T12:47:45"]
    alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10))
    hits = generate_hits(hits_timestamps)
    ea.current_es.search.return_value = hits
    with mock.patch("elastalert.elastalert.Elasticsearch"):
        # Aggregate first two, query over full range
        ea.rules[0]["aggregation"] = datetime.timedelta(minutes=10)
        ea.rules[0]["type"].matches = [{"@timestamp": h} for h in hits_timestamps]
        ea.run_rule(ea.rules[0], END, START)

    # Assert that the three matches were added to elasticsearch
    call1 = ea.writeback_es.create.call_args_list[0][1]["body"]
    call2 = ea.writeback_es.create.call_args_list[1][1]["body"]
    call3 = ea.writeback_es.create.call_args_list[2][1]["body"]

    assert call1["match_body"] == {"@timestamp": "2014-09-26T12:34:45"}
    assert not call1["alert_sent"]
    assert "aggregate_id" not in call1
    assert call1["alert_time"] == alerttime1

    assert call2["match_body"] == {"@timestamp": "2014-09-26T12:40:45"}
    assert not call2["alert_sent"]
    assert call2["aggregate_id"] == "ABCD"

    assert call3["match_body"] == {"@timestamp": "2014-09-26T12:47:45"}
    assert not call3["alert_sent"]
    assert "aggregate_id" not in call3

    # First call - Find all pending alerts
    # Second call - Find matches with agg_id == 'ABCD'
    # Third call - Find matches with agg_id == 'CDEF'
    ea.writeback_es.search.side_effect = [
        {
            "hits": {
                "hits": [
                    {"_id": "ABCD", "_source": call1},
                    {"_id": "BCDE", "_source": call2},
                    {"_id": "CDEF", "_source": call3},
                ]
            }
        },
        {"hits": {"hits": [{"_id": "BCDE", "_source": call2}]}},
        {"hits": {"hits": []}},
    ]
    ea.send_pending_alerts()
    assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]])

    call1 = ea.writeback_es.search.call_args_list[6][1]["body"]
    call2 = ea.writeback_es.search.call_args_list[7][1]["body"]
    call3 = ea.writeback_es.search.call_args_list[8][1]["body"]

    assert "alert_time" in call1["filter"]["range"]
    assert call2["query"]["query_string"]["query"] == "aggregate_id:ABCD"
    assert call3["query"]["query_string"]["query"] == "aggregate_id:CDEF"
Пример #11
0
 def get_match_str(self, match):
     ts = match[self.rules['timestamp_field']]
     lt = self.rules.get('use_local_time')
     key = match.get('key', 'all')
     message = 'An abnormally low number of events occurred around %s.\n' % (
         pretty_ts(ts, lt))
     message += 'Between %s and %s, there were less than %s events.\n\n' % (
         pretty_ts(dt_to_ts(ts_to_dt(ts) - self.timeframe(key)),
                   lt), pretty_ts(ts, lt), self.rules['threshold'])
     message = json.dumps(match)
     return message
Пример #12
0
 def get_match_str(self, match):
     lt = self.rules.get('use_local_time')
     match_ts = lookup_es_key(match, self.ts_field)
     key = match.get('key', 'all')
     starttime = pretty_ts(
         dt_to_ts(ts_to_dt(match_ts) - self.timeframe(key)), lt)
     endtime = pretty_ts(match_ts, lt)
     message = 'At least %d events occurred between %s and %s\n\n' % (
         self.rules['num_events'], starttime, endtime)
     message = json.dumps(match)
     return message
Пример #13
0
def test_agg(ea):
    ea.max_aggregation = 1337
    hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']
    alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10))
    hits = generate_hits(hits_timestamps)
    ea.current_es.search.return_value = hits
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        # Aggregate first two, query over full range
        ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
        ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
        ea.run_rule(ea.rules[0], END, START)

    # Assert that the three matches were added to elasticsearch
    call1 = ea.writeback_es.create.call_args_list[0][1]['body']
    call2 = ea.writeback_es.create.call_args_list[1][1]['body']
    call3 = ea.writeback_es.create.call_args_list[2][1]['body']

    assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
    assert not call1['alert_sent']
    assert 'aggregate_id' not in call1
    assert call1['alert_time'] == alerttime1

    assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
    assert not call2['alert_sent']
    assert call2['aggregate_id'] == 'ABCD'

    assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
    assert not call3['alert_sent']
    assert 'aggregate_id' not in call3

    # First call - Find all pending alerts
    # Second call - Find matches with agg_id == 'ABCD'
    # Third call - Find matches with agg_id == 'CDEF'
    ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1},
                                                             {'_id': 'BCDE', '_source': call2},
                                                             {'_id': 'CDEF', '_source': call3}]}},
                                          {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}},
                                          {'hits': {'hits': []}}]

    with mock.patch('elastalert.elastalert.Elasticsearch') as mock_es:
        ea.send_pending_alerts()
        # Assert that current_es was refreshed from the aggregate rules
        assert mock_es.called_with(host='', port='')
        assert mock_es.call_count == 2
    assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]])

    call1 = ea.writeback_es.search.call_args_list[6][1]['body']
    call2 = ea.writeback_es.search.call_args_list[7][1]['body']
    call3 = ea.writeback_es.search.call_args_list[8][1]['body']

    assert 'alert_time' in call1['filter']['range']
    assert call2['query']['query_string']['query'] == 'aggregate_id:ABCD'
    assert call3['query']['query_string']['query'] == 'aggregate_id:CDEF'
    assert ea.writeback_es.search.call_args_list[7][1]['size'] == 1337
Пример #14
0
def test_queries_with_rule_buffertime(ea):
    ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=53)
    mock_es = mock.Mock()
    mock_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP])
    with mock.patch('elastalert.elastalert.Elasticsearch') as mock_es_init:
        mock_es_init.return_value = mock_es
        ea.run_rule(ea.rules[0], END, START)

    # Assert that es.search is run against every run_every timeframe between START and END
    end = END_TIMESTAMP
    start = START
    query = {'filter': {'bool': {'must': [{'range': {'@timestamp': {'to': END_TIMESTAMP, 'from': START_TIMESTAMP}}}]}},
             'sort': [{'@timestamp': {'order': 'asc'}}]}
    while END - start > ea.rules[0]['buffer_time']:
        end = start + ea.run_every
        query['filter']['bool']['must'][0]['range']['@timestamp']['to'] = dt_to_ts(end)
        query['filter']['bool']['must'][0]['range']['@timestamp']['from'] = dt_to_ts(start)
        start = start + ea.run_every
        ea.current_es.search.assert_any_call(body=query, size=ea.max_query_size, index='idx', ignore_unavailable=True, _source_include=['@timestamp'])

    # Assert that num_hits correctly summed every result
    assert ea.num_hits == ea.current_es.search.call_count
Пример #15
0
    def process(self, match):
        self.local_time = self.rule.get('local_time_feild', 'local_time')
        self.local_starttime = self.rule.get('local_starttime_feild',
                                             'local_starttime')
        self.local_endtime = self.rule.get('local_endtime_feild',
                                           'local_endtime')
        self.ts_field = self.rule.get('timestamp_field', '@timestamp')
        lt = self.rule.get('use_local_time', "False")

        match_ts = match[self.ts_field]
        match[self.local_time] = pretty_ts(match_ts, lt)
        match[self.local_starttime] = pretty_ts(
            dt_to_ts(ts_to_dt(match_ts) - self.rule['timeframe']), lt)
        match[self.local_endtime] = match[self.local_time]
Пример #16
0
def test_count(ea):
    ea.rules[0]['use_count_query'] = True
    ea.rules[0]['doc_type'] = 'doctype'
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        ea.run_rule(ea.rules[0], END, START)

    # Assert that es.count is run against every run_every timeframe between START and END
    start = START
    query = {
        'query': {
            'filtered': {
                'filter': {
                    'bool': {
                        'must': [{
                            'range': {
                                '@timestamp': {
                                    'lte': END_TIMESTAMP,
                                    'gt': START_TIMESTAMP
                                }
                            }
                        }]
                    }
                }
            }
        }
    }
    while END - start > ea.run_every:
        end = start + ea.run_every
        query['query']['filtered']['filter']['bool']['must'][0]['range'][
            '@timestamp']['lte'] = dt_to_ts(end)
        query['query']['filtered']['filter']['bool']['must'][0]['range'][
            '@timestamp']['gt'] = dt_to_ts(start)
        start = start + ea.run_every
        ea.current_es.count.assert_any_call(body=query,
                                            doc_type='doctype',
                                            index='idx',
                                            ignore_unavailable=True)
Пример #17
0
def test_silence(ea):
    # Silence test rule for 4 hours
    ea.args.rule = 'test_rule.yaml'  # Not a real name, just has to be set
    ea.args.silence = 'hours=4'
    ea.silence()

    # Don't alert even with a match
    match = [{'@timestamp': '2014-11-17T00:00:00'}]
    ea.rules[0]['type'].matches = match
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]['alert'][0].alert.call_count == 0

    # Mock ts_now() to +5 hours, alert on match
    match = [{'@timestamp': '2014-11-17T00:00:00'}]
    ea.rules[0]['type'].matches = match
    with mock.patch('elastalert.elastalert.ts_now') as mock_ts:
        with mock.patch('elastalert.elastalert.Elasticsearch'):
            # Converted twice to add tzinfo
            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
            ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]['alert'][0].alert.call_count == 1
Пример #18
0
def test_silence_query_key(ea):
    # Silence test rule for 4 hours
    ea.args.rule = "test_rule.yaml"  # Not a real name, just has to be set
    ea.args.silence = "hours=4"
    ea.silence()

    # Don't alert even with a match
    match = [{"@timestamp": "2014-11-17T00:00:00", "username": "******"}]
    ea.rules[0]["type"].matches = match
    ea.rules[0]["query_key"] = "username"
    with mock.patch("elastalert.elastalert.Elasticsearch"):
        ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]["alert"][0].alert.call_count == 0

    # Mock ts_now() to +5 hours, alert on match
    match = [{"@timestamp": "2014-11-17T00:00:00", "username": "******"}]
    ea.rules[0]["type"].matches = match
    with mock.patch("elastalert.elastalert.ts_now") as mock_ts:
        with mock.patch("elastalert.elastalert.Elasticsearch"):
            # Converted twice to add tzinfo
            mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))
            ea.run_rule(ea.rules[0], END, START)
    assert ea.rules[0]["alert"][0].alert.call_count == 1
Пример #19
0
def test_count(ea):
    ea.rules[0]['use_count_query'] = True
    ea.rules[0]['doc_type'] = 'doctype'
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        ea.run_rule(ea.rules[0], END, START)

    # Assert that es.count is run against every run_every timeframe between START and END
    start = START
    query = {'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}}
    while END - start > ea.run_every:
        end = start + ea.run_every
        query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end)
        query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start)
        start = start + ea.run_every
        ea.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True)
Пример #20
0
def test_count(ea):
    ea.rules[0]["use_count_query"] = True
    ea.rules[0]["doc_type"] = "doctype"
    with mock.patch("elastalert.elastalert.Elasticsearch"):
        ea.run_rule(ea.rules[0], END, START)

    # Assert that es.count is run against every run_every timeframe between START and END
    start = START
    query = {
        "query": {
            "filtered": {
                "filter": {
                    "bool": {"must": [{"range": {"@timestamp": {"to": END_TIMESTAMP, "from": START_TIMESTAMP}}}]}
                }
            }
        }
    }
    while END - start > ea.run_every:
        end = start + ea.run_every
        query["query"]["filtered"]["filter"]["bool"]["must"][0]["range"]["@timestamp"]["to"] = dt_to_ts(end)
        query["query"]["filtered"]["filter"]["bool"]["must"][0]["range"]["@timestamp"]["from"] = dt_to_ts(start)
        start = start + ea.run_every
        ea.current_es.count.assert_any_call(body=query, doc_type="doctype", index="idx", ignore_unavailable=True)
Пример #21
0
def test_agg(ea):
    hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'
    alerttime1 = dt_to_ts(ts_to_dt(hit1) + datetime.timedelta(minutes=10))
    hits = generate_hits([hit1, hit2, hit3])
    ea.current_es.search.return_value = hits
    with mock.patch('elastalert.elastalert.Elasticsearch'):
        # Aggregate first two, query over full range
        ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
        ea.rules[0]['type'].matches = [{
            '@timestamp': hit1
        }, {
            '@timestamp': hit2
        }, {
            '@timestamp': hit3
        }]
        ea.run_rule(ea.rules[0], END, START)

    # Assert that the three matches were added to elasticsearch
    call1 = ea.writeback_es.create.call_args_list[0][1]['body']
    call2 = ea.writeback_es.create.call_args_list[1][1]['body']
    call3 = ea.writeback_es.create.call_args_list[2][1]['body']

    assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45'}
    assert not call1['alert_sent']
    assert 'aggregate_id' not in call1
    assert call1['alert_time'] == alerttime1

    assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45'}
    assert not call2['alert_sent']
    assert call2['aggregate_id'] == 'ABCD'

    assert call3['match_body'] == {'@timestamp': '2014-09-26T12:47:45'}
    assert not call3['alert_sent']
    assert 'aggregate_id' not in call3

    # First call - Find all pending alerts
    # Second call - Find matches with agg_id == 'ABCD'
    # Third call - Find matches with agg_id == 'CDEF'
    ea.writeback_es.search.side_effect = [{
        'hits': {
            'hits': [{
                '_id': 'ABCD',
                '_source': call1
            }, {
                '_id': 'BCDE',
                '_source': call2
            }, {
                '_id': 'CDEF',
                '_source': call3
            }]
        }
    }, {
        'hits': {
            'hits': [{
                '_id': 'BCDE',
                '_source': call2
            }]
        }
    }, {
        'hits': {
            'hits': []
        }
    }]
    ea.send_pending_alerts()
    assert_alerts(ea, [[dt_to_ts(hit1), dt_to_ts(hit2)], [dt_to_ts(hit3)]])

    call1 = ea.writeback_es.search.call_args_list[6][1]['body']
    call2 = ea.writeback_es.search.call_args_list[7][1]['body']
    call3 = ea.writeback_es.search.call_args_list[8][1]['body']

    assert 'alert_time' in call1['filter']['range']
    assert call2['query']['query_string']['query'] == 'aggregate_id:ABCD'
    assert call3['query']['query_string']['query'] == 'aggregate_id:CDEF'
Пример #22
0
def test_agg_with_aggregation_key(ea):
    ea.max_aggregation = 1337
    hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:43:45']
    alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10))
    alerttime2 = dt_to_ts(ts_to_dt(hits_timestamps[1]) + datetime.timedelta(minutes=10))
    hits = generate_hits(hits_timestamps)
    ea.current_es.search.return_value = hits
    with mock.patch('elastalert.elastalert.elasticsearch_client'):
        ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)
        ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]
        # Hit1 and Hit3 should be aggregated together, since they have same query_key value
        ea.rules[0]['type'].matches[0]['key'] = 'Key Value 1'
        ea.rules[0]['type'].matches[1]['key'] = 'Key Value 2'
        ea.rules[0]['type'].matches[2]['key'] = 'Key Value 1'
        ea.rules[0]['aggregation_key'] = 'key'
        ea.run_rule(ea.rules[0], END, START)

    # Assert that the three matches were added to elasticsearch
    call1 = ea.writeback_es.create.call_args_list[0][1]['body']
    call2 = ea.writeback_es.create.call_args_list[1][1]['body']
    call3 = ea.writeback_es.create.call_args_list[2][1]['body']
    assert call1['match_body'] == {'@timestamp': '2014-09-26T12:34:45', 'key': 'Key Value 1'}
    assert not call1['alert_sent']
    assert 'aggregate_id' not in call1
    assert 'aggregate_key' in call1
    assert call1['aggregate_key'] == 'Key Value 1'
    assert call1['alert_time'] == alerttime1

    assert call2['match_body'] == {'@timestamp': '2014-09-26T12:40:45', 'key': 'Key Value 2'}
    assert not call2['alert_sent']
    assert 'aggregate_id' not in call2
    assert 'aggregate_key' in call2
    assert call2['aggregate_key'] == 'Key Value 2'
    assert call2['alert_time'] == alerttime2

    assert call3['match_body'] == {'@timestamp': '2014-09-26T12:43:45', 'key': 'Key Value 1', 'key': 'Key Value 1'}
    assert not call3['alert_sent']
    # Call3 should have it's aggregate_id set to call1's _id
    # It should also have the same alert_time as call1
    assert call3['aggregate_id'] == 'ABCD'
    assert 'aggregate_key' in call3
    assert call3['aggregate_key'] == 'Key Value 1'
    assert call3['alert_time'] == alerttime1

    # First call - Find all pending alerts (only entries without agg_id)
    # Second call - Find matches with agg_id == 'ABCD'
    # Third call - Find matches with agg_id == 'CDEF'
    ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1},
                                                             {'_id': 'CDEF', '_source': call2}]}},
                                          {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}},
                                          {'hits': {'total': 0, 'hits': []}}]

    with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:
        ea.send_pending_alerts()
        # Assert that current_es was refreshed from the aggregate rules
        assert mock_es.called_with(host='', port='')
        assert mock_es.call_count == 2
    assert_alerts(ea, [[hits_timestamps[0], hits_timestamps[2]], [hits_timestamps[1]]])

    call1 = ea.writeback_es.search.call_args_list[7][1]['body']
    call2 = ea.writeback_es.search.call_args_list[8][1]['body']
    call3 = ea.writeback_es.search.call_args_list[9][1]['body']
    call4 = ea.writeback_es.search.call_args_list[10][1]['body']

    assert 'alert_time' in call2['filter']['range']
    assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD'
    assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF'
    assert ea.writeback_es.search.call_args_list[9][1]['size'] == 1337