def create_custom_title(self, matches):
# Assume rule['alert_subject'] to be a jinja templated string. See Alerter.create_title()
subject = self.rule['alert_subject']
es_conn_conf = util.build_es_conn_config(self.rule)
env = {
'rule':
self.rule,
'matches':
matches,
'pipeline':
self.pipeline,
'jira_server':
self.pipeline['jira_server'] if
(self.pipeline and 'jira_server' in self.pipeline) else None,
'jira_ticket':
self.pipeline['jira_ticket'] if
(self.pipeline and 'jira_ticket' in self.pipeline) else None,
'es':
util.elasticsearch_client(es_conn_conf),
'util':
util,
'datetime':
datetime,
}
return Environment().from_string(subject).render(**env)
def _get_es_client(conf):
"""
Created for testing reasons. Making it easier for us to mock this method.
:param conf:
:return:
"""
return elasticsearch_client(conf)
def get_all_terms(self, args):
""" Performs a terms aggregation for each field to get every existing term. """
self.es = elasticsearch_client(self.rules)
window_size = datetime.timedelta(**self.rules.get('terms_window_size', {'days': 30}))
field_name = {"field": "", "size": 2147483647} # Integer.MAX_VALUE
query_template = {"aggs": {"values": {"terms": field_name}}}
if args and args.start:
end = ts_to_dt(args.start)
else:
end = ts_now()
start = end - window_size
step = datetime.timedelta(**self.rules.get('window_step_size', {'days': 1}))
for field in self.fields:
tmp_start = start
tmp_end = min(start + step, end)
time_filter = {self.rules['timestamp_field']: {'lt': dt_to_ts(tmp_end), 'gte': dt_to_ts(tmp_start)}}
query_template['filter'] = {'bool': {'must': [{'range': time_filter}]}}
query = {'aggs': {'filtered': query_template}}
# For composite keys, we will need to perform sub-aggregations
if type(field) == list:
self.seen_values.setdefault(tuple(field), [])
level = query_template['aggs']
# Iterate on each part of the composite key and add a sub aggs clause to the elastic search query
for i, sub_field in enumerate(field):
level['values']['terms']['field'] = add_raw_postfix(sub_field)
if i < len(field) - 1:
# If we have more fields after the current one, then set up the next nested structure
level['values']['aggs'] = {'values': {'terms': copy.deepcopy(field_name)}}
level = level['values']['aggs']
else:
self.seen_values.setdefault(field, [])
# For non-composite keys, only a single agg is needed
field_name['field'] = add_raw_postfix(field)
# Query the entire time range in small chunks
while tmp_start < end:
if self.rules.get('use_strftime_index'):
index = format_index(self.rules['index'], tmp_start, tmp_end)
else:
index = self.rules['index']
res = self.es.search(body=query, index=index, ignore_unavailable=True, timeout='50s')
if 'aggregations' in res:
buckets = res['aggregations']['filtered']['values']['buckets']
if type(field) == list:
# For composite keys, make the lookup based on all fields
# Make it a tuple since it can be hashed and used in dictionary lookups
for bucket in buckets:
# We need to walk down the hierarchy and obtain the value at each level
self.seen_values[tuple(field)] += self.flatten_aggregation_hierarchy(bucket)
else:
keys = [bucket['key'] for bucket in buckets]
self.seen_values[field] += keys
else:
self.seen_values.setdefault(field, [])
if tmp_start == tmp_end:
break
tmp_start = tmp_end
tmp_end = min(tmp_start + step, end)
time_filter[self.rules['timestamp_field']] = {'lt': dt_to_ts(tmp_end), 'gte': dt_to_ts(tmp_start)}
for key, values in self.seen_values.iteritems():
if not values:
if type(key) == tuple:
# If we don't have any results, it could either be because of the absence of any baseline data
# OR it may be because the composite key contained a non-primitive type. Either way, give the
# end-users a heads up to help them debug what might be going on.
elastalert_logger.warning((
'No results were found from all sub-aggregations. This can either indicate that there is '
'no baseline data OR that a non-primitive field was used in a composite key.'
))
else:
elastalert_logger.info('Found no values for %s' % (field))
continue
self.seen_values[key] = list(set(values))
elastalert_logger.info('Found %s unique values for %s' % (len(values), key))
def alert(self, matches):
if False:
print "*********** STACK ***********"
import traceback
traceback.print_stack()
print "*********** MATCHES ***********"
print matches
print "*********** RULES ************"
print self.rule
if 'template_text_content' in self.rule:
text = self.rule['template_text_content']
elif 'template_text_file' in self.rule:
with open(self.rule['template_text_file'], 'r') as f:
text = f.read()
else:
text = '{{ matches|length }} items found'
if 'template_html_content' in self.rule:
html = self.rule['template_html_content']
elif 'template_html_file' in self.rule:
with open(self.rule['template_html_file'], 'r') as f:
html = f.read()
else:
html = '{{ matches|length }} items found'
if 'email_from_field' in self.rule:
recipient = lookup_es_key(matches[0],
self.rule['email_from_field'])
if isinstance(recipient, basestring):
if '@' in recipient:
to_addr = [recipient]
elif 'email_add_domain' in self.rule:
to_addr = [recipient + self.rule['email_add_domain']]
elif isinstance(recipient, list):
to_addr = recipient
if 'email_add_domain' in self.rule:
to_addr = [
name + self.rule['email_add_domain']
for name in to_addr
]
es_conn_conf = util.build_es_conn_config(self.rule)
env = {
'rule':
self.rule,
'matches':
matches,
'pipeline':
self.pipeline,
'jira_server':
self.pipeline['jira_server'] if
(self.pipeline and 'jira_server' in self.pipeline) else None,
'jira_ticket':
self.pipeline['jira_ticket'] if
(self.pipeline and 'jira_ticket' in self.pipeline) else None,
'es':
util.elasticsearch_client(es_conn_conf),
'json':
json,
'util':
util,
'datetime':
datetime,
'kibana_context_args':
kibana4_context_args,
}
text = Environment().from_string(text).render(**env)
html = Environment().from_string(html).render(**env)
messageRoot = MIMEMultipart('related')
messageRoot['Subject'] = _encode_str(self.create_title(matches))
messageRoot['From'] = _encode_str(self.from_addr)
if 'email_from_field' in self.rule:
messageRoot['To'] = _convert_to_strings(to_addr)
else:
messageRoot['To'] = _convert_to_strings(self.rule['email'])
if self.rule.get('email_reply_to'):
messageRoot['Reply-To'] = _convert_to_strings(
self.rule.get('email_reply_to'))
messageRoot.preamble = 'This is a multi-part message in MIME format.'
# Encapsulate the plain and HTML versions of the message body in an
# 'alternative' part, so message agents can decide which they want to display.
msgAlternative = MIMEMultipart('alternative')
msgAlternative.attach(MIMEText(_encode_str(text), 'plain'))
msgAlternative.attach(MIMEText(_encode_str(html), 'html'))
messageRoot.attach(msgAlternative)
try:
if self.smtp_ssl:
if self.smtp_port:
self.smtp = SMTP_SSL(self.smtp_host, self.smtp_port)
else:
self.smtp = SMTP_SSL(self.smtp_host)
else:
if self.smtp_port:
self.smtp = SMTP(self.smtp_host, self.smtp_port)
else:
self.smtp = SMTP(self.smtp_host)
self.smtp.ehlo()
if self.smtp.has_extn('STARTTLS'):
self.smtp.starttls()
if 'smtp_auth_file' in self.rule:
self.smtp.login(self.user, self.password)
except (SMTPException, error) as e:
raise EAException("Error connecting to SMTP host: %s" % (e))
except SMTPAuthenticationError as e:
raise EAException("SMTP username/password rejected: %s" % (e))
self.smtp.sendmail(messageRoot['From'], self.rule['email'],
messageRoot.as_string())
self.smtp.close()
elastalert_logger.info("Sent email to %s for rule: %s" %
(self.rule['email'], self.rule['name']))
