def test_spike_query_key():
events = hits(100, timestamp_field='ts', username='******')
# Constant rate, doesn't match
rules = {
'threshold_ref': 10,
'spike_height': 2,
'timeframe': datetime.timedelta(seconds=10),
'spike_type': 'both',
'use_count_query': False,
'timestamp_field': 'ts',
'query_key': 'username'
}
rule = SpikeRule(rules)
rule.add_data(events)
assert len(rule.matches) == 0
# Double the rate of events, but with a different usename
events_bob = hits(100, timestamp_field='ts', username='******')
events2 = events[:50]
for num in range(50, 99):
events2.append(events_bob[num])
events2.append(events[num])
rule = SpikeRule(rules)
rule.add_data(events2)
assert len(rule.matches) == 0
# Double the rate of events, with the same username
events2 = events[:50]
for num in range(50, 99):
events2.append(events_bob[num])
events2.append(events[num])
events2.append(events[num])
rule = SpikeRule(rules)
rule.add_data(events2)
assert len(rule.matches) == 1
def test_spike_count():
rules = {
'threshold_ref': 10,
'spike_height': 2,
'timeframe': datetime.timedelta(seconds=10),
'spike_type': 'both',
'timestamp_field': '@timestamp'
}
rule = SpikeRule(rules)
# Double rate of events at 20 seconds
rule.add_count_data({ts_to_dt('2014-09-26T00:00:00'): 10})
assert len(rule.matches) == 0
rule.add_count_data({ts_to_dt('2014-09-26T00:00:10'): 10})
assert len(rule.matches) == 0
rule.add_count_data({ts_to_dt('2014-09-26T00:00:20'): 20})
assert len(rule.matches) == 1
# Downward spike
rule = SpikeRule(rules)
rule.add_count_data({ts_to_dt('2014-09-26T00:00:00'): 10})
assert len(rule.matches) == 0
rule.add_count_data({ts_to_dt('2014-09-26T00:00:10'): 10})
assert len(rule.matches) == 0
rule.add_count_data({ts_to_dt('2014-09-26T00:00:20'): 0})
assert len(rule.matches) == 1
def test_spike_terms():
rules = {'threshold_ref': 5,
'spike_height': 2,
'timeframe': datetime.timedelta(minutes=10),
'spike_type': 'both',
'use_count_query': False,
'timestamp_field': 'ts',
'query_key': 'username',
'use_term_query': True}
terms1 = {ts_to_dt('2014-01-01T00:01:00Z'): [{'key': 'userA', 'doc_count': 10},
{'key': 'userB', 'doc_count': 5}]}
terms2 = {ts_to_dt('2014-01-01T00:10:00Z'): [{'key': 'userA', 'doc_count': 22},
{'key': 'userB', 'doc_count': 5}]}
terms3 = {ts_to_dt('2014-01-01T00:25:00Z'): [{'key': 'userA', 'doc_count': 25},
{'key': 'userB', 'doc_count': 27}]}
terms4 = {ts_to_dt('2014-01-01T00:27:00Z'): [{'key': 'userA', 'doc_count': 10},
{'key': 'userB', 'doc_count': 12},
{'key': 'userC', 'doc_count': 100}]}
terms5 = {ts_to_dt('2014-01-01T00:30:00Z'): [{'key': 'userD', 'doc_count': 100},
{'key': 'userC', 'doc_count': 100}]}
rule = SpikeRule(rules)
# Initial input
rule.add_terms_data(terms1)
assert len(rule.matches) == 0
# No spike for UserA because windows not filled
rule.add_terms_data(terms2)
assert len(rule.matches) == 0
# Spike for userB only
rule.add_terms_data(terms3)
assert len(rule.matches) == 1
assert rule.matches[0].get('username') == 'userB'
# Test no alert for new user over threshold
rules.pop('threshold_ref')
rules['threshold_cur'] = 50
rule = SpikeRule(rules)
rule.add_terms_data(terms1)
rule.add_terms_data(terms2)
rule.add_terms_data(terms3)
rule.add_terms_data(terms4)
assert len(rule.matches) == 0
# Test alert_on_new_data
rules['alert_on_new_data'] = True
rule = SpikeRule(rules)
rule.add_terms_data(terms1)
rule.add_terms_data(terms2)
rule.add_terms_data(terms3)
rule.add_terms_data(terms4)
assert len(rule.matches) == 1
# Test that another alert doesn't fire immediately for userC but it does for userD
rule.matches = []
rule.add_terms_data(terms5)
assert len(rule.matches) == 1
assert rule.matches[0]['username'] == 'userD'
def test_spike_deep_key():
rules = {'threshold_ref': 10,
'spike_height': 2,
'timeframe': datetime.timedelta(seconds=10),
'spike_type': 'both',
'timestamp_field': '@timestamp',
'query_key': 'foo.bar.baz'}
rule = SpikeRule(rules)
rule.add_data([{'@timestamp': ts_to_dt('2015'), 'foo': {'bar': {'baz': 'LOL'}}}])
assert 'LOL' in rule.cur_windows
def test_spike_terms_query_key_alert_on_new_data():
rules = {'spike_height': 1.5,
'timeframe': datetime.timedelta(minutes=10),
'spike_type': 'both',
'use_count_query': False,
'timestamp_field': 'ts',
'query_key': 'username',
'use_term_query': True,
'alert_on_new_data': True}
terms1 = {ts_to_dt('2014-01-01T00:01:00Z'): [{'key': 'userA', 'doc_count': 10}]}
terms2 = {ts_to_dt('2014-01-01T00:06:00Z'): [{'key': 'userA', 'doc_count': 10}]}
terms3 = {ts_to_dt('2014-01-01T00:11:00Z'): [{'key': 'userA', 'doc_count': 10}]}
terms4 = {ts_to_dt('2014-01-01T00:21:00Z'): [{'key': 'userA', 'doc_count': 20}]}
terms5 = {ts_to_dt('2014-01-01T00:26:00Z'): [{'key': 'userA', 'doc_count': 20}]}
terms6 = {ts_to_dt('2014-01-01T00:31:00Z'): [{'key': 'userA', 'doc_count': 20}]}
terms7 = {ts_to_dt('2014-01-01T00:36:00Z'): [{'key': 'userA', 'doc_count': 20}]}
terms8 = {ts_to_dt('2014-01-01T00:41:00Z'): [{'key': 'userA', 'doc_count': 20}]}
rule = SpikeRule(rules)
# Initial input
rule.add_terms_data(terms1)
assert len(rule.matches) == 0
# No spike for UserA because windows not filled
rule.add_terms_data(terms2)
assert len(rule.matches) == 0
rule.add_terms_data(terms3)
assert len(rule.matches) == 0
rule.add_terms_data(terms4)
assert len(rule.matches) == 0
# Spike
rule.add_terms_data(terms5)
assert len(rule.matches) == 1
rule.matches[:] = []
# There will be no more spikes since all terms have the same doc_count
rule.add_terms_data(terms6)
assert len(rule.matches) == 0
rule.add_terms_data(terms7)
assert len(rule.matches) == 0
rule.add_terms_data(terms8)
assert len(rule.matches) == 0
def test_spike():
# Events are 1 per second
events = hits(100, timestamp_field='ts')
# Constant rate, doesn't match
rules = {
'threshold_ref': 10,
'spike_height': 2,
'timeframe': datetime.timedelta(seconds=10),
'spike_type': 'both',
'use_count_query': False,
'timestamp_field': 'ts'
}
rule = SpikeRule(rules)
rule.add_data(events)
assert len(rule.matches) == 0
# Double the rate of events after [50:]
events2 = events[:50]
for event in events[50:]:
events2.append(event)
events2.append(
{'ts': event['ts'] + datetime.timedelta(milliseconds=1)})
rules['spike_type'] = 'up'
rule = SpikeRule(rules)
rule.add_data(events2)
assert len(rule.matches) == 1
# Doesn't match
rules['spike_height'] = 3
rule = SpikeRule(rules)
rule.add_data(events2)
assert len(rule.matches) == 0
# Downward spike
events = events[:50] + events[75:]
rules['spike_type'] = 'down'
rule = SpikeRule(rules)
rule.add_data(events)
assert len(rule.matches) == 1
# Doesn't meet threshold_ref
# When ref hits 11, cur is only 20
rules['spike_height'] = 2
rules['threshold_ref'] = 11
rules['spike_type'] = 'up'
rule = SpikeRule(rules)
rule.add_data(events2)
assert len(rule.matches) == 0
# Doesn't meet threshold_cur
# Maximum rate of events is 20 per 10 seconds
rules['threshold_ref'] = 10
rules['threshold_cur'] = 30
rule = SpikeRule(rules)
rule.add_data(events2)
assert len(rule.matches) == 0
# Alert on new data
# (At least 25 events occur before 30 seconds has elapsed)
rules.pop('threshold_ref')
rules['timeframe'] = datetime.timedelta(seconds=30)
rules['threshold_cur'] = 25
rules['spike_height'] = 2
rules['alert_on_new_data'] = True
rule = SpikeRule(rules)
rule.add_data(events2)
assert len(rule.matches) == 1
