def get_match_str(self, match):
ts = match[self.rules['timestamp_field']]
lt = self.rules.get('use_local_time')
message = 'An abnormally low number of events occurred around %s.\n' % (pretty_ts(ts, lt))
message += 'Between %s and %s, there were less than %s events.\n\n' % (pretty_ts(dt_to_ts(ts_to_dt(ts) - self.rules['timeframe']), lt),
pretty_ts(ts, lt),
self.rules['threshold'])
return message
def get_match_str(self, match):
lt = self.rules.get('use_local_time')
starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']), lt)
endtime = pretty_ts(match[self.ts_field], lt)
message = 'At least %d events occurred between %s and %s\n\n' % (self.rules['num_events'],
starttime,
endtime)
return message
def start(self):
""" Periodically go through each rule and run it """
starttime = self.args.start
if starttime:
try:
starttime = ts_to_dt(starttime)
except (TypeError, ValueError):
self.handle_error("%s is not a valid ISO 8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (starttime))
exit(1)
while True:
# If writeback_es errored, it's disabled until the next query cycle
if not self.writeback_es:
self.writeback_es = self.new_elasticsearch(self.es_conn_config)
self.send_pending_alerts()
next_run = datetime.datetime.utcnow() + self.run_every
for rule in self.rules:
# Set endtime based on the rule's delay
delay = rule.get('query_delay')
if hasattr(self.args, 'end') and self.args.end:
endtime = ts_to_dt(self.args.end)
elif delay:
endtime = ts_now() - delay
else:
endtime = ts_now()
try:
num_matches = self.run_rule(rule, endtime, starttime)
except EAException as e:
self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']})
else:
old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time'))
logging.info("Ran %s from %s to %s: %s query hits, %s matches,"
" %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')),
self.num_hits, num_matches, self.alerts_sent))
self.alerts_sent = 0
self.remove_old_events(rule)
if next_run < datetime.datetime.utcnow():
# We were processing for longer than our refresh interval
# This can happen if --start was specified with a large time period
# or if we are running too slow to process events in real time.
logging.warning("Querying from %s to %s took longer than %s!" % (old_starttime, endtime, self.run_every))
continue
# Only force starttime once
starttime = None
if not self.args.pin_rules:
self.load_rule_changes()
# Wait before querying again
sleep_for = (next_run - datetime.datetime.utcnow()).seconds
logging.info("Sleeping for %s seconds" % (sleep_for))
time.sleep(sleep_for)
def get_match_str(self, match):
lt = self.rules.get('use_local_time')
match_ts = lookup_es_key(match, self.ts_field)
starttime = pretty_ts(
dt_to_ts(ts_to_dt(match_ts) - self.rules['timeframe']), lt)
endtime = pretty_ts(match_ts, lt)
message = 'At least %d events occurred between %s and %s\n\n' % (
self.rules['num_events'], starttime, endtime)
return message
def get_match_str(self, match):
ts = match[self.rules['timestamp_field']]
lt = self.rules.get('use_local_time')
message = 'An abnormally low number of events occurred around %s.\n' % (
pretty_ts(ts, lt))
message += 'Between %s and %s, there were less than %s events.\n\n' % (
pretty_ts(dt_to_ts(ts_to_dt(ts) - self.rules['timeframe']),
lt), pretty_ts(ts, lt), self.rules['threshold'])
return message
def get_match_str(self, match):
lt = self.rules.get('use_local_time')
starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']), lt)
endtime = pretty_ts(match[self.ts_field], lt)
message = ('A maximum of %d unique %s(s) occurred since last alert or '
'between %s and %s\n\n' % (self.rules['max_cardinality'],
self.rules['cardinality_field'],
starttime, endtime))
return message
def get_match_str(self, match):
lt = self.rules.get("use_local_time")
starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules["timeframe"]), lt)
endtime = pretty_ts(match[self.ts_field], lt)
message = "A maximum of %d unique %s(s) occurred since last alert or " "between %s and %s\n\n" % (
self.rules["max_cardinality"],
self.rules["cardinality_field"],
starttime,
endtime,
)
return message
def get_match_str(self, match):
lt = self.rules.get('use_local_time')
starttime = pretty_ts(
dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']),
lt)
endtime = pretty_ts(match[self.ts_field], lt)
message = ('A maximum of %d unique %s(s) occurred since last alert or '
'between %s and %s\n\n' %
(self.rules['max_cardinality'],
self.rules['cardinality_field'], starttime, endtime))
return message
def get_match_str(self, match):
message = 'An abnormal number (%d) of events occurred around %s.\n' % (
match['spike_count'],
pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time'))
)
message += 'Preceding that time, there were only %d events within %s\n\n' % (match['reference_count'], self.rules['timeframe'])
return message
def comment_on_ticket(self, ticket, match):
text = unicode(JiraFormattedMatchString(self.rule, match))
timestamp = pretty_ts(
lookup_es_key(match, self.rule['timestamp_field']))
comment = "This alert was triggered again at %s\n%s" % (timestamp,
text)
self.client.add_comment(ticket, comment)
def get_match_str(self, match):
message = 'An abnormal number (%d) of events occurred around %s.\n' % (
match['spike_count'],
pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time'))
)
message += 'Preceding that time, there were only %d events within %s\n\n' % (match['reference_count'], self.rules['timeframe'])
return message
def get_hits(self, rule, starttime, endtime, index):
""" Query elasticsearch for the given rule and return the results.
:param rule: The rule configuration.
:param starttime: The earliest time to query.
:param endtime: The latest time to query.
:return: A list of hits, bounded by self.max_query_size.
"""
query = self.get_query(rule['filter'], starttime, endtime, timestamp_field=rule['timestamp_field'])
try:
res = self.current_es.search(index=index, size=self.max_query_size, body=query, _source_include=rule['include'], ignore_unavailable=True)
except ElasticsearchException as e:
# Elasticsearch sometimes gives us GIGANTIC error messages
# (so big that they will fill the entire terminal buffer)
if len(str(e)) > 1024:
e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
self.handle_error('Error running query: %s' % (e), {'rule': rule['name']})
return None
hits = res['hits']['hits']
self.num_hits += len(hits)
lt = rule.get('use_local_time')
logging.info("Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(hits)))
self.replace_ts(hits, rule)
# Record doc_type for use in get_top_counts
if 'doc_type' not in rule and len(hits):
rule['doc_type'] = hits[0]['_type']
return hits
def get_hits_count(self, rule, starttime, endtime, index):
""" Query elasticsearch for the count of results and returns a list of timestamps
equal to the endtime. This allows the results to be passed to rules which expect
an object for each hit.
:param rule: The rule configuration dictionary.
:param starttime: The earliest time to query.
:param endtime: The latest time to query.
:return: A dictionary mapping timestamps to number of hits for that time period.
"""
query = self.get_query(rule['filter'], starttime, endtime, timestamp_field=rule['timestamp_field'], sort=False)
query = {'query': {'filtered': query}}
try:
res = self.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True)
except ElasticsearchException as e:
# Elasticsearch sometimes gives us GIGANTIC error messages
# (so big that they will fill the entire terminal buffer)
if len(str(e)) > 1024:
e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
self.handle_error('Error running count query: %s' % (e), {'rule': rule['name']})
return None
self.num_hits += res['count']
lt = rule.get('use_local_time')
logging.info("Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count']))
return {endtime: res['count']}
def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None):
rule_filter = copy.copy(rule['filter'])
if qk:
filter_key = rule['query_key']
if rule.get('raw_count_keys', True) and not rule['query_key'].endswith('.raw'):
filter_key += '.raw'
rule_filter.extend([{'term': {filter_key: qk}}])
base_query = self.get_query(rule_filter, starttime, endtime, timestamp_field=rule['timestamp_field'], sort=False)
query = self.get_terms_query(base_query, rule.get('terms_size', 5), key)
try:
res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, search_type='count', ignore_unavailable=True)
except ElasticsearchException as e:
# Elasticsearch sometimes gives us GIGANTIC error messages
# (so big that they will fill the entire terminal buffer)
if len(str(e)) > 1024:
e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
self.handle_error('Error running query: %s' % (e), {'rule': rule['name']})
return None
buckets = res['aggregations']['filtered']['counts']['buckets']
self.num_hits += len(buckets)
lt = rule.get('use_local_time')
logging.info('Queried rule %s from %s to %s: %s buckets' % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(buckets)))
return {endtime: buckets}
def get_match_str(self, match):
lt = self.rules.get('use_local_time')
match_ts = lookup_es_key(match, self.ts_field)
starttime = pretty_ts(dt_to_ts(ts_to_dt(match_ts) - self.rules['timeframe']), lt)
message = 'At least %d(%d) events occurred between %s and %s\n\n' % (self.rules['num_events'],
match['count'],
starttime,
endtime)
return message
def run_all_rules(self):
""" Run each rule one time """
# If writeback_es errored, it's disabled until the next query cycle
if not self.writeback_es:
self.writeback_es = self.new_elasticsearch(self.es_conn_config)
self.send_pending_alerts()
next_run = datetime.datetime.utcnow() + self.run_every
for rule in self.rules:
# Set endtime based on the rule's delay
delay = rule.get('query_delay')
if hasattr(self.args, 'end') and self.args.end:
endtime = ts_to_dt(self.args.end)
elif delay:
endtime = ts_now() - delay
else:
endtime = ts_now()
try:
num_matches = self.run_rule(rule, endtime, self.starttime)
except EAException as e:
self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']})
else:
old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time'))
logging.info("Ran %s from %s to %s: %s query hits, %s matches,"
" %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')),
self.num_hits, num_matches, self.alerts_sent))
self.alerts_sent = 0
self.remove_old_events(rule)
if next_run < datetime.datetime.utcnow():
# We were processing for longer than our refresh interval
# This can happen if --start was specified with a large time period
# or if we are running too slow to process events in real time.
logging.warning("Querying from %s to %s took longer than %s!" % (old_starttime, endtime, self.run_every))
# Only force starttime once
self.starttime = None
if not self.args.pin_rules:
self.load_rule_changes()
def get_match_str(self, match):
message = 'An abnormal value of %d occurred around %s for %s:%s.\n' % (
match['spike_value'],
pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time')),
self.rules['metric_agg_type'],
self.rules['metric_agg_key'],
)
message += 'Preceding that time, there were only %d events within %s\n\n' % (match['reference_value'], self.rules['timeframe'])
return message
def get_match_str(self, match):
message = "An abnormal number (%d) of events occurred around %s.\n" % (
match["spike_count"],
pretty_ts(match[self.rules["timestamp_field"]], self.rules.get("use_local_time")),
)
message += "Preceding that time, there were only %d events within %s\n\n" % (
match["reference_count"],
self.rules["timeframe"],
)
return message
def get_hits(self, rule, starttime, endtime, index):
""" Query elasticsearch for the given rule and return the results.
:param rule: The rule configuration.
:param starttime: The earliest time to query.
:param endtime: The latest time to query.
:return: A list of hits, bounded by self.max_query_size.
"""
query = self.get_query(rule['filter'],
starttime,
endtime,
timestamp_field=rule['timestamp_field'])
try:
res = self.current_es.search(index=index,
size=self.max_query_size,
body=query,
_source_include=rule['include'],
ignore_unavailable=True)
except ElasticsearchException as e:
# Elasticsearch sometimes gives us GIGANTIC error messages
# (so big that they will fill the entire terminal buffer)
if len(str(e)) > 1024:
e = str(e)[:1024] + '... (%d characters removed)' % (
len(str(e)) - 1024)
self.handle_error('Error running query: %s' % (e),
{'rule': rule['name']})
return None
hits = res['hits']['hits']
self.num_hits += len(hits)
lt = rule.get('use_local_time')
logging.info("Queried rule %s from %s to %s: %s hits" %
(rule['name'], pretty_ts(
starttime, lt), pretty_ts(endtime, lt), len(hits)))
self.replace_ts(hits, rule)
# Record doc_type for use in get_top_counts
if 'doc_type' not in rule and len(hits):
rule['doc_type'] = hits[0]['_type']
return hits
def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None):
rule_filter = copy.copy(rule['filter'])
if qk:
filter_key = rule['query_key']
if rule.get('raw_count_keys',
True) and not rule['query_key'].endswith('.raw'):
filter_key += '.raw'
rule_filter.extend([{'term': {filter_key: qk}}])
base_query = self.get_query(rule_filter,
starttime,
endtime,
timestamp_field=rule['timestamp_field'],
sort=False)
query = self.get_terms_query(base_query, rule.get('terms_size', 5),
key)
try:
res = self.current_es.search(index=index,
doc_type=rule['doc_type'],
body=query,
search_type='count',
ignore_unavailable=True)
except ElasticsearchException as e:
# Elasticsearch sometimes gives us GIGANTIC error messages
# (so big that they will fill the entire terminal buffer)
if len(str(e)) > 1024:
e = str(e)[:1024] + '... (%d characters removed)' % (
len(str(e)) - 1024)
self.handle_error('Error running query: %s' % (e),
{'rule': rule['name']})
return None
buckets = res['aggregations']['filtered']['counts']['buckets']
self.num_hits += len(buckets)
lt = rule.get('use_local_time')
logging.info('Queried rule %s from %s to %s: %s buckets' %
(rule['name'], pretty_ts(
starttime, lt), pretty_ts(endtime, lt), len(buckets)))
return {endtime: buckets}
def get_hits_count(self, rule, starttime, endtime, index):
""" Query elasticsearch for the count of results and returns a list of timestamps
equal to the endtime. This allows the results to be passed to rules which expect
an object for each hit.
:param rule: The rule configuration dictionary.
:param starttime: The earliest time to query.
:param endtime: The latest time to query.
:return: A dictionary mapping timestamps to number of hits for that time period.
"""
query = self.get_query(rule['filter'],
starttime,
endtime,
timestamp_field=rule['timestamp_field'],
sort=False)
query = {'query': {'filtered': query}}
try:
res = self.current_es.count(index=index,
doc_type=rule['doc_type'],
body=query,
ignore_unavailable=True)
except ElasticsearchException as e:
# Elasticsearch sometimes gives us GIGANTIC error messages
# (so big that they will fill the entire terminal buffer)
if len(str(e)) > 1024:
e = str(e)[:1024] + '... (%d characters removed)' % (
len(str(e)) - 1024)
self.handle_error('Error running count query: %s' % (e),
{'rule': rule['name']})
return None
self.num_hits += res['count']
lt = rule.get('use_local_time')
logging.info("Queried rule %s from %s to %s: %s hits" %
(rule['name'], pretty_ts(
starttime, lt), pretty_ts(endtime, lt), res['count']))
return {endtime: res['count']}
def create_default_title(self, matches, for_search=False):
# If there is a query_key, use that in the title
if 'query_key' in self.rule and self.rule['query_key'] in matches[0]:
title = 'ElastAlert: %s matched %s' % (matches[0][self.rule['query_key']], self.rule['name'])
else:
title = 'ElastAlert: %s' % (self.rule['name'])
if for_search:
return title
title += ' - %s' % (pretty_ts(matches[0][self.rule['timestamp_field']], self.rule.get('use_local_time')))
# Add count for spikes
count = matches[0].get('spike_count')
if count:
title += ' - %s+ events' % (count)
return title
def get_match_str(self, match):
ts = match[self.rules['timestamp_field']]
lt = self.rules.get('use_local_time')
try:
match_value = self.match_value[-1][:5]
except:
match_value = []
message = "Between %s and %s\n" % (pretty_ts(dt_to_ts(ts_to_dt(ts) - self.rules['timeframe']), lt), pretty_ts(ts, lt))
message += "%s(%s) %s %s\nmatch value:\n\t%s...\n\n" % (
self.rules['stat'],
self.rules['stat_field'],
self.rules['stat_type'],
self.rules['threshold'],
'\n\t'.join(match_value)
)
return message
def create_default_title(self, matches, for_search=False):
# If there is a query_key, use that in the title
if "query_key" in self.rule and self.rule["query_key"] in matches[0]:
title = "ElastAlert: %s matched %s" % (matches[0][self.rule["query_key"]], self.rule["name"])
else:
title = "ElastAlert: %s" % (self.rule["name"])
if for_search:
return title
title += " - %s" % (pretty_ts(matches[0][self.rule["timestamp_field"]], self.rule.get("use_local_time")))
# Add count for spikes
count = matches[0].get("spike_count")
if count:
title += " - %s+ events" % (count)
return title
def comment_on_ticket(self, ticket, match):
text = unicode(JiraFormattedMatchString(self.rule, match))
timestamp = pretty_ts(lookup_es_key(match, self.rule['timestamp_field']))
comment = "This alert was triggered again at %s\n%s" % (timestamp, text)
self.client.add_comment(ticket, comment)
def start(self):
""" Periodically go through each rule and run it """
starttime = self.args.start
if starttime:
try:
starttime = ts_to_dt(starttime)
except (TypeError, ValueError):
self.handle_error(
"%s is not a valid ISO 8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)"
% (starttime))
exit(1)
while True:
# If writeback_es errored, it's disabled until the next query cycle
if not self.writeback_es:
self.writeback_es = Elasticsearch(host=self.es_host,
port=self.es_port)
self.send_pending_alerts()
next_run = datetime.datetime.utcnow() + self.run_every
for rule in self.rules:
# Set endtime based on the rule's delay
delay = rule.get('query_delay')
if hasattr(self.args, 'end') and self.args.end:
endtime = ts_to_dt(self.args.end)
elif delay:
endtime = ts_now() - delay
else:
endtime = ts_now()
try:
num_matches = self.run_rule(rule, endtime, starttime)
except EAException as e:
self.handle_error(
"Error running rule %s: %s" % (rule['name'], e),
{'rule': rule['name']})
else:
old_starttime = pretty_ts(rule.get('original_starttime'),
rule.get('use_local_time'))
logging.info(
"Ran %s from %s to %s: %s query hits, %s matches,"
" %s alerts sent" %
(rule['name'], old_starttime,
pretty_ts(endtime, rule.get('use_local_time')),
self.num_hits, num_matches, self.alerts_sent))
self.alerts_sent = 0
self.remove_old_events(rule)
if next_run < datetime.datetime.utcnow():
# We were processing for longer than our refresh interval
# This can happen if --start was specified with a large time period
# or if we are running too slow to process events in real time.
logging.warning("Querying from %s to %s took longer than %s!" %
(old_starttime, endtime, self.run_every))
continue
# Only force starttime once
starttime = None
if not self.args.pin_rules:
self.load_rule_changes()
# Wait before querying again
sleep_for = (next_run - datetime.datetime.utcnow()).seconds
logging.info("Sleeping for %s seconds" % (sleep_for))
time.sleep(sleep_for)
def process(self, match):
match['@timestamp'] = pretty_ts(match['@timestamp'])
def comment_on_ticket(self, ticket, match):
text = str(JiraFormattedMatchString(self.rule, match))
timestamp = pretty_ts(match[self.rule["timestamp_field"]])
comment = "This alert was triggered again at %s\n%s" % (timestamp, text)
self.client.add_comment(ticket, comment)
def comment_on_ticket(self, ticket, match):
text = basic_match_string(self.rule, match)
timestamp = pretty_ts(match[self.rule['timestamp_field']])
comment = "This alert was triggered again at %s\n%s" % (timestamp, text)
self.client.add_comment(ticket, comment)
def process(self, match):
match['@timestamp'] = pretty_ts(match['@timestamp'])
match['starttime'] = self.rule['starttime']
def create_default_title(self, matches):
subject = '%s: %d matches found - %s' % \
(self.rule['name'], len(matches),
pretty_ts(ts_to_dt(self.pipeline['alert_time'])))
return subject
def comment_on_ticket(self, ticket, match):
text = basic_match_string(self.rule, match)
timestamp = pretty_ts(match[self.rule['timestamp_field']])
comment = "This alert was triggered again at %s\n%s" % (timestamp,
text)
self.client.add_comment(ticket, comment)