Esempio n. 1
0
 def get_match_str(self, match):
     ts = match[self.rules['timestamp_field']]
     lt = self.rules.get('use_local_time')
     message = 'An abnormally low number of events occurred around %s.\n' % (pretty_ts(ts, lt))
     message += 'Between %s and %s, there were less than %s events.\n\n' % (pretty_ts(dt_to_ts(ts_to_dt(ts) - self.rules['timeframe']), lt),
                                                                            pretty_ts(ts, lt),
                                                                            self.rules['threshold'])
     return message
Esempio n. 2
0
 def get_match_str(self, match):
     lt = self.rules.get('use_local_time')
     starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']), lt)
     endtime = pretty_ts(match[self.ts_field], lt)
     message = 'At least %d events occurred between %s and %s\n\n' % (self.rules['num_events'],
                                                                      starttime,
                                                                      endtime)
     return message
Esempio n. 3
0
    def start(self):
        """ Periodically go through each rule and run it """
        starttime = self.args.start
        if starttime:
            try:
                starttime = ts_to_dt(starttime)
            except (TypeError, ValueError):
                self.handle_error("%s is not a valid ISO 8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (starttime))
                exit(1)
        while True:
            # If writeback_es errored, it's disabled until the next query cycle
            if not self.writeback_es:
                self.writeback_es = self.new_elasticsearch(self.es_conn_config)

            self.send_pending_alerts()

            next_run = datetime.datetime.utcnow() + self.run_every

            for rule in self.rules:
                # Set endtime based on the rule's delay
                delay = rule.get('query_delay')
                if hasattr(self.args, 'end') and self.args.end:
                    endtime = ts_to_dt(self.args.end)
                elif delay:
                    endtime = ts_now() - delay
                else:
                    endtime = ts_now()

                try:
                    num_matches = self.run_rule(rule, endtime, starttime)
                except EAException as e:
                    self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']})
                else:
                    old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time'))
                    logging.info("Ran %s from %s to %s: %s query hits, %s matches,"
                                 " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')),
                                                      self.num_hits, num_matches, self.alerts_sent))
                    self.alerts_sent = 0

                self.remove_old_events(rule)

            if next_run < datetime.datetime.utcnow():
                # We were processing for longer than our refresh interval
                # This can happen if --start was specified with a large time period
                # or if we are running too slow to process events in real time.
                logging.warning("Querying from %s to %s took longer than %s!" % (old_starttime, endtime, self.run_every))
                continue

            # Only force starttime once
            starttime = None

            if not self.args.pin_rules:
                self.load_rule_changes()

            # Wait before querying again
            sleep_for = (next_run - datetime.datetime.utcnow()).seconds
            logging.info("Sleeping for %s seconds" % (sleep_for))
            time.sleep(sleep_for)
Esempio n. 4
0
 def get_match_str(self, match):
     lt = self.rules.get('use_local_time')
     match_ts = lookup_es_key(match, self.ts_field)
     starttime = pretty_ts(
         dt_to_ts(ts_to_dt(match_ts) - self.rules['timeframe']), lt)
     endtime = pretty_ts(match_ts, lt)
     message = 'At least %d events occurred between %s and %s\n\n' % (
         self.rules['num_events'], starttime, endtime)
     return message
Esempio n. 5
0
 def get_match_str(self, match):
     ts = match[self.rules['timestamp_field']]
     lt = self.rules.get('use_local_time')
     message = 'An abnormally low number of events occurred around %s.\n' % (
         pretty_ts(ts, lt))
     message += 'Between %s and %s, there were less than %s events.\n\n' % (
         pretty_ts(dt_to_ts(ts_to_dt(ts) - self.rules['timeframe']),
                   lt), pretty_ts(ts, lt), self.rules['threshold'])
     return message
Esempio n. 6
0
 def get_match_str(self, match):
     lt = self.rules.get('use_local_time')
     starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']), lt)
     endtime = pretty_ts(match[self.ts_field], lt)
     message = ('A maximum of %d unique %s(s) occurred since last alert or '
                'between %s and %s\n\n' % (self.rules['max_cardinality'],
                                           self.rules['cardinality_field'],
                                           starttime, endtime))
     return message
Esempio n. 7
0
 def get_match_str(self, match):
     lt = self.rules.get("use_local_time")
     starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules["timeframe"]), lt)
     endtime = pretty_ts(match[self.ts_field], lt)
     message = "A maximum of %d unique %s(s) occurred since last alert or " "between %s and %s\n\n" % (
         self.rules["max_cardinality"],
         self.rules["cardinality_field"],
         starttime,
         endtime,
     )
     return message
Esempio n. 8
0
 def get_match_str(self, match):
     lt = self.rules.get('use_local_time')
     starttime = pretty_ts(
         dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']),
         lt)
     endtime = pretty_ts(match[self.ts_field], lt)
     message = ('A maximum of %d unique %s(s) occurred since last alert or '
                'between %s and %s\n\n' %
                (self.rules['max_cardinality'],
                 self.rules['cardinality_field'], starttime, endtime))
     return message
Esempio n. 9
0
 def get_match_str(self, match):
     message = 'An abnormal number (%d) of events occurred around %s.\n' % (
         match['spike_count'],
         pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time'))
     )
     message += 'Preceding that time, there were only %d events within %s\n\n' % (match['reference_count'], self.rules['timeframe'])
     return message
Esempio n. 10
0
 def comment_on_ticket(self, ticket, match):
     text = unicode(JiraFormattedMatchString(self.rule, match))
     timestamp = pretty_ts(
         lookup_es_key(match, self.rule['timestamp_field']))
     comment = "This alert was triggered again at %s\n%s" % (timestamp,
                                                             text)
     self.client.add_comment(ticket, comment)
Esempio n. 11
0
 def get_match_str(self, match):
     message = 'An abnormal number (%d) of events occurred around %s.\n' % (
         match['spike_count'],
         pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time'))
     )
     message += 'Preceding that time, there were only %d events within %s\n\n' % (match['reference_count'], self.rules['timeframe'])
     return message
Esempio n. 12
0
    def get_hits(self, rule, starttime, endtime, index):
        """ Query elasticsearch for the given rule and return the results.

        :param rule: The rule configuration.
        :param starttime: The earliest time to query.
        :param endtime: The latest time to query.
        :return: A list of hits, bounded by self.max_query_size.
        """
        query = self.get_query(rule['filter'], starttime, endtime, timestamp_field=rule['timestamp_field'])
        try:
            res = self.current_es.search(index=index, size=self.max_query_size, body=query, _source_include=rule['include'], ignore_unavailable=True)
        except ElasticsearchException as e:
            # Elasticsearch sometimes gives us GIGANTIC error messages
            # (so big that they will fill the entire terminal buffer)
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
            self.handle_error('Error running query: %s' % (e), {'rule': rule['name']})
            return None

        hits = res['hits']['hits']
        self.num_hits += len(hits)
        lt = rule.get('use_local_time')
        logging.info("Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(hits)))
        self.replace_ts(hits, rule)

        # Record doc_type for use in get_top_counts
        if 'doc_type' not in rule and len(hits):
            rule['doc_type'] = hits[0]['_type']
        return hits
Esempio n. 13
0
    def get_hits_count(self, rule, starttime, endtime, index):
        """ Query elasticsearch for the count of results and returns a list of timestamps
        equal to the endtime. This allows the results to be passed to rules which expect
        an object for each hit.

        :param rule: The rule configuration dictionary.
        :param starttime: The earliest time to query.
        :param endtime: The latest time to query.
        :return: A dictionary mapping timestamps to number of hits for that time period.
        """
        query = self.get_query(rule['filter'], starttime, endtime, timestamp_field=rule['timestamp_field'], sort=False)
        query = {'query': {'filtered': query}}

        try:
            res = self.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True)
        except ElasticsearchException as e:
            # Elasticsearch sometimes gives us GIGANTIC error messages
            # (so big that they will fill the entire terminal buffer)
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
            self.handle_error('Error running count query: %s' % (e), {'rule': rule['name']})
            return None

        self.num_hits += res['count']
        lt = rule.get('use_local_time')
        logging.info("Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count']))
        return {endtime: res['count']}
Esempio n. 14
0
    def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None):
        rule_filter = copy.copy(rule['filter'])
        if qk:
            filter_key = rule['query_key']
            if rule.get('raw_count_keys', True) and not rule['query_key'].endswith('.raw'):
                filter_key += '.raw'
            rule_filter.extend([{'term': {filter_key: qk}}])
        base_query = self.get_query(rule_filter, starttime, endtime, timestamp_field=rule['timestamp_field'], sort=False)
        query = self.get_terms_query(base_query, rule.get('terms_size', 5), key)

        try:
            res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, search_type='count', ignore_unavailable=True)
        except ElasticsearchException as e:
            # Elasticsearch sometimes gives us GIGANTIC error messages
            # (so big that they will fill the entire terminal buffer)
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)
            self.handle_error('Error running query: %s' % (e), {'rule': rule['name']})
            return None

        buckets = res['aggregations']['filtered']['counts']['buckets']
        self.num_hits += len(buckets)
        lt = rule.get('use_local_time')
        logging.info('Queried rule %s from %s to %s: %s buckets' % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(buckets)))
        return {endtime: buckets}
Esempio n. 15
0
 def get_match_str(self, match):
     lt = self.rules.get('use_local_time')
     match_ts = lookup_es_key(match, self.ts_field)
     starttime = pretty_ts(dt_to_ts(ts_to_dt(match_ts) - self.rules['timeframe']), lt)
     message = 'At least %d(%d) events occurred between %s and %s\n\n' % (self.rules['num_events'],
                                                                      match['count'],
                                                                      starttime,
                                                                      endtime)
     return message
Esempio n. 16
0
    def run_all_rules(self):
        """ Run each rule one time """
        # If writeback_es errored, it's disabled until the next query cycle
        if not self.writeback_es:
            self.writeback_es = self.new_elasticsearch(self.es_conn_config)

        self.send_pending_alerts()

        next_run = datetime.datetime.utcnow() + self.run_every

        for rule in self.rules:
            # Set endtime based on the rule's delay
            delay = rule.get('query_delay')
            if hasattr(self.args, 'end') and self.args.end:
                endtime = ts_to_dt(self.args.end)
            elif delay:
                endtime = ts_now() - delay
            else:
                endtime = ts_now()

            try:
                num_matches = self.run_rule(rule, endtime, self.starttime)
            except EAException as e:
                self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']})
            else:
                old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time'))
                logging.info("Ran %s from %s to %s: %s query hits, %s matches,"
                             " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')),
                                                  self.num_hits, num_matches, self.alerts_sent))
                self.alerts_sent = 0

            self.remove_old_events(rule)

        if next_run < datetime.datetime.utcnow():
            # We were processing for longer than our refresh interval
            # This can happen if --start was specified with a large time period
            # or if we are running too slow to process events in real time.
            logging.warning("Querying from %s to %s took longer than %s!" % (old_starttime, endtime, self.run_every))

        # Only force starttime once
        self.starttime = None

        if not self.args.pin_rules:
            self.load_rule_changes()
Esempio n. 17
0
    def get_match_str(self, match):
        message = 'An abnormal value of %d occurred around %s for %s:%s.\n' % (
            match['spike_value'],
            pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time')),
            self.rules['metric_agg_type'],
            self.rules['metric_agg_key'],
        )
        message += 'Preceding that time, there were only %d events within %s\n\n' % (match['reference_value'], self.rules['timeframe'])

        return message
Esempio n. 18
0
 def get_match_str(self, match):
     message = "An abnormal number (%d) of events occurred around %s.\n" % (
         match["spike_count"],
         pretty_ts(match[self.rules["timestamp_field"]], self.rules.get("use_local_time")),
     )
     message += "Preceding that time, there were only %d events within %s\n\n" % (
         match["reference_count"],
         self.rules["timeframe"],
     )
     return message
Esempio n. 19
0
    def get_hits(self, rule, starttime, endtime, index):
        """ Query elasticsearch for the given rule and return the results.

        :param rule: The rule configuration.
        :param starttime: The earliest time to query.
        :param endtime: The latest time to query.
        :return: A list of hits, bounded by self.max_query_size.
        """
        query = self.get_query(rule['filter'],
                               starttime,
                               endtime,
                               timestamp_field=rule['timestamp_field'])
        try:
            res = self.current_es.search(index=index,
                                         size=self.max_query_size,
                                         body=query,
                                         _source_include=rule['include'],
                                         ignore_unavailable=True)
        except ElasticsearchException as e:
            # Elasticsearch sometimes gives us GIGANTIC error messages
            # (so big that they will fill the entire terminal buffer)
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (
                    len(str(e)) - 1024)
            self.handle_error('Error running query: %s' % (e),
                              {'rule': rule['name']})
            return None

        hits = res['hits']['hits']
        self.num_hits += len(hits)
        lt = rule.get('use_local_time')
        logging.info("Queried rule %s from %s to %s: %s hits" %
                     (rule['name'], pretty_ts(
                         starttime, lt), pretty_ts(endtime, lt), len(hits)))
        self.replace_ts(hits, rule)

        # Record doc_type for use in get_top_counts
        if 'doc_type' not in rule and len(hits):
            rule['doc_type'] = hits[0]['_type']
        return hits
Esempio n. 20
0
    def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None):
        rule_filter = copy.copy(rule['filter'])
        if qk:
            filter_key = rule['query_key']
            if rule.get('raw_count_keys',
                        True) and not rule['query_key'].endswith('.raw'):
                filter_key += '.raw'
            rule_filter.extend([{'term': {filter_key: qk}}])
        base_query = self.get_query(rule_filter,
                                    starttime,
                                    endtime,
                                    timestamp_field=rule['timestamp_field'],
                                    sort=False)
        query = self.get_terms_query(base_query, rule.get('terms_size', 5),
                                     key)

        try:
            res = self.current_es.search(index=index,
                                         doc_type=rule['doc_type'],
                                         body=query,
                                         search_type='count',
                                         ignore_unavailable=True)
        except ElasticsearchException as e:
            # Elasticsearch sometimes gives us GIGANTIC error messages
            # (so big that they will fill the entire terminal buffer)
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (
                    len(str(e)) - 1024)
            self.handle_error('Error running query: %s' % (e),
                              {'rule': rule['name']})
            return None

        buckets = res['aggregations']['filtered']['counts']['buckets']
        self.num_hits += len(buckets)
        lt = rule.get('use_local_time')
        logging.info('Queried rule %s from %s to %s: %s buckets' %
                     (rule['name'], pretty_ts(
                         starttime, lt), pretty_ts(endtime, lt), len(buckets)))
        return {endtime: buckets}
Esempio n. 21
0
    def get_hits_count(self, rule, starttime, endtime, index):
        """ Query elasticsearch for the count of results and returns a list of timestamps
        equal to the endtime. This allows the results to be passed to rules which expect
        an object for each hit.

        :param rule: The rule configuration dictionary.
        :param starttime: The earliest time to query.
        :param endtime: The latest time to query.
        :return: A dictionary mapping timestamps to number of hits for that time period.
        """
        query = self.get_query(rule['filter'],
                               starttime,
                               endtime,
                               timestamp_field=rule['timestamp_field'],
                               sort=False)
        query = {'query': {'filtered': query}}

        try:
            res = self.current_es.count(index=index,
                                        doc_type=rule['doc_type'],
                                        body=query,
                                        ignore_unavailable=True)
        except ElasticsearchException as e:
            # Elasticsearch sometimes gives us GIGANTIC error messages
            # (so big that they will fill the entire terminal buffer)
            if len(str(e)) > 1024:
                e = str(e)[:1024] + '... (%d characters removed)' % (
                    len(str(e)) - 1024)
            self.handle_error('Error running count query: %s' % (e),
                              {'rule': rule['name']})
            return None

        self.num_hits += res['count']
        lt = rule.get('use_local_time')
        logging.info("Queried rule %s from %s to %s: %s hits" %
                     (rule['name'], pretty_ts(
                         starttime, lt), pretty_ts(endtime, lt), res['count']))
        return {endtime: res['count']}
Esempio n. 22
0
    def create_default_title(self, matches, for_search=False):
        # If there is a query_key, use that in the title
        if 'query_key' in self.rule and self.rule['query_key'] in matches[0]:
            title = 'ElastAlert: %s matched %s' % (matches[0][self.rule['query_key']], self.rule['name'])
        else:
            title = 'ElastAlert: %s' % (self.rule['name'])

        if for_search:
            return title

        title += ' - %s' % (pretty_ts(matches[0][self.rule['timestamp_field']], self.rule.get('use_local_time')))

        # Add count for spikes
        count = matches[0].get('spike_count')
        if count:
            title += ' - %s+ events' % (count)

        return title
Esempio n. 23
0
    def get_match_str(self, match):
        ts = match[self.rules['timestamp_field']]
        lt = self.rules.get('use_local_time')

        try:
            match_value = self.match_value[-1][:5]
        except:
            match_value = []

        message =  "Between %s and %s\n" % (pretty_ts(dt_to_ts(ts_to_dt(ts) - self.rules['timeframe']), lt), pretty_ts(ts, lt))
        message += "%s(%s) %s %s\nmatch value:\n\t%s...\n\n" % (
                self.rules['stat'],
                self.rules['stat_field'],
                self.rules['stat_type'],
                self.rules['threshold'],
                '\n\t'.join(match_value)
                ) 
        return message
Esempio n. 24
0
    def create_default_title(self, matches, for_search=False):
        # If there is a query_key, use that in the title
        if "query_key" in self.rule and self.rule["query_key"] in matches[0]:
            title = "ElastAlert: %s matched %s" % (matches[0][self.rule["query_key"]], self.rule["name"])
        else:
            title = "ElastAlert: %s" % (self.rule["name"])

        if for_search:
            return title

        title += " - %s" % (pretty_ts(matches[0][self.rule["timestamp_field"]], self.rule.get("use_local_time")))

        # Add count for spikes
        count = matches[0].get("spike_count")
        if count:
            title += " - %s+ events" % (count)

        return title
Esempio n. 25
0
 def comment_on_ticket(self, ticket, match):
     text = unicode(JiraFormattedMatchString(self.rule, match))
     timestamp = pretty_ts(lookup_es_key(match, self.rule['timestamp_field']))
     comment = "This alert was triggered again at %s\n%s" % (timestamp, text)
     self.client.add_comment(ticket, comment)
Esempio n. 26
0
    def start(self):
        """ Periodically go through each rule and run it """
        starttime = self.args.start
        if starttime:
            try:
                starttime = ts_to_dt(starttime)
            except (TypeError, ValueError):
                self.handle_error(
                    "%s is not a valid ISO 8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)"
                    % (starttime))
                exit(1)
        while True:
            # If writeback_es errored, it's disabled until the next query cycle
            if not self.writeback_es:
                self.writeback_es = Elasticsearch(host=self.es_host,
                                                  port=self.es_port)

            self.send_pending_alerts()

            next_run = datetime.datetime.utcnow() + self.run_every

            for rule in self.rules:
                # Set endtime based on the rule's delay
                delay = rule.get('query_delay')
                if hasattr(self.args, 'end') and self.args.end:
                    endtime = ts_to_dt(self.args.end)
                elif delay:
                    endtime = ts_now() - delay
                else:
                    endtime = ts_now()

                try:
                    num_matches = self.run_rule(rule, endtime, starttime)
                except EAException as e:
                    self.handle_error(
                        "Error running rule %s: %s" % (rule['name'], e),
                        {'rule': rule['name']})
                else:
                    old_starttime = pretty_ts(rule.get('original_starttime'),
                                              rule.get('use_local_time'))
                    logging.info(
                        "Ran %s from %s to %s: %s query hits, %s matches,"
                        " %s alerts sent" %
                        (rule['name'], old_starttime,
                         pretty_ts(endtime, rule.get('use_local_time')),
                         self.num_hits, num_matches, self.alerts_sent))
                    self.alerts_sent = 0

                self.remove_old_events(rule)

            if next_run < datetime.datetime.utcnow():
                # We were processing for longer than our refresh interval
                # This can happen if --start was specified with a large time period
                # or if we are running too slow to process events in real time.
                logging.warning("Querying from %s to %s took longer than %s!" %
                                (old_starttime, endtime, self.run_every))
                continue

            # Only force starttime once
            starttime = None

            if not self.args.pin_rules:
                self.load_rule_changes()

            # Wait before querying again
            sleep_for = (next_run - datetime.datetime.utcnow()).seconds
            logging.info("Sleeping for %s seconds" % (sleep_for))
            time.sleep(sleep_for)
Esempio n. 27
0
 def process(self, match):
     match['@timestamp'] = pretty_ts(match['@timestamp'])
Esempio n. 28
0
 def comment_on_ticket(self, ticket, match):
     text = str(JiraFormattedMatchString(self.rule, match))
     timestamp = pretty_ts(match[self.rule["timestamp_field"]])
     comment = "This alert was triggered again at %s\n%s" % (timestamp, text)
     self.client.add_comment(ticket, comment)
Esempio n. 29
0
 def comment_on_ticket(self, ticket, match):
     text = basic_match_string(self.rule, match)
     timestamp = pretty_ts(match[self.rule['timestamp_field']])
     comment = "This alert was triggered again at %s\n%s" % (timestamp, text)
     self.client.add_comment(ticket, comment)
Esempio n. 30
0
 def process(self, match):
     match['@timestamp'] = pretty_ts(match['@timestamp'])
     match['starttime'] = self.rule['starttime']
Esempio n. 31
0
    def create_default_title(self, matches):
        subject = '%s: %d matches found - %s' % \
                      (self.rule['name'], len(matches),
                      pretty_ts(ts_to_dt(self.pipeline['alert_time'])))

        return subject
Esempio n. 32
0
 def comment_on_ticket(self, ticket, match):
     text = basic_match_string(self.rule, match)
     timestamp = pretty_ts(match[self.rule['timestamp_field']])
     comment = "This alert was triggered again at %s\n%s" % (timestamp,
                                                             text)
     self.client.add_comment(ticket, comment)