def __init__( self, influxdbstg=None, *args, **kwargs ): super(Stats, self).__init__(*args, **kwargs) if influxdbstg is not None: self.influxdbstg = influxdbstg else: self.influxdbstg = InfluxDBStorage()
class Stats(MiddlewareRegistry): """ Stats management """ def __init__( self, influxdbstg=None, *args, **kwargs ): super(Stats, self).__init__(*args, **kwargs) if influxdbstg is not None: self.influxdbstg = influxdbstg else: self.influxdbstg = InfluxDBStorage() def get_event_stats(self, tstart, tstop, tags={}): """ Get event related stats :param int tstart: Start timestamp :param int tstop: End timestamp :param dict tags: Query formatting tags :returns: Stats or an empty dict if tags={} :rtype: dict Example: tags = {'domain': ['d1', 'd2'], 'perimeter': ['p1']} return = { 'domain': [ { 'name': 'd1', 'stats_count': { 'alarms_new': 12, 'alarms_ack': 3, 'alarms_solved_ack': 10, 'alarms_solved_without_ack': 10 }, 'stats_delay': { 'ack_delay_min' : 3, 'ack_delay_max' : 10, 'ack_delay_avg' : 5, 'ack_solved_delay_min' : 3, 'ack_solved_delay_max' : 10, 'ack_solved_delay_avg' : 5, 'alarm_solved_delay_min' : 3, 'alarm_solved_delay_max' : 10, 'alarm_solved_delay_avg' : 5 } }, { 'name': 'd1', [...] }, { 'name': '__total__', [...] } ], 'perimeter': [ { 'name': 'p1', [...] }, { 'name': '__total__', [...] ], '__total__': { 'stats_count': {...}, 'stats_delay': {...} } } """ self.logger.info("Retrieving event based stats") result = {} time_where = 'time >= {}s AND time <= {}s'.format(tstart, tstop) tags_where_total = '' for tag_group, tag_names in tags.items(): tags_where_total += '{} =~ {} OR '.format( tag_group, self._influx_or_regex(tag_names) ) else: # Remove trailing ' OR ' tags_where_total = tags_where_total[:-4] if tags_where_total: where_total = '{} AND ({})'.format(time_where, tags_where_total) else: where_total = time_where g_alarm_new_total = self._influx_query( metric_id='alarm_opened_count', aggregations='count', condition=where_total ) g_ack_new_total = self._influx_query( metric_id='alarm_ack_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where_total ) g_ack_solved_total = self._influx_query( metric_id='alarm_ack_solved_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where_total ) g_alarm_solved_total = self._influx_query( metric_id='alarm_solved_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where_total ) al_new_cnt_tot = self._get_stats( result_set=g_alarm_new_total, column='count') ack_new_cnt_tot = self._get_stats( result_set=g_ack_new_total, column='count') ack_new_min_tot = self._get_stats( result_set=g_ack_new_total, column='min') ack_new_avg_tot = self._get_stats( result_set=g_ack_new_total, column='mean') ack_new_max_tot = self._get_stats( result_set=g_ack_new_total, column='max') ack_sol_cnt_tot = self._get_stats( result_set=g_ack_solved_total, column='count') ack_sol_min_tot = self._get_stats( result_set=g_ack_solved_total, column='min') ack_sol_avg_tot = self._get_stats( result_set=g_ack_solved_total, column='mean') ack_sol_max_tot = self._get_stats( result_set=g_ack_solved_total, column='max') al_sol_cnt_tot = self._get_stats( result_set=g_alarm_solved_total, column='count') al_sol_min_tot = self._get_stats( result_set=g_alarm_solved_total, column='min') al_sol_avg_tot = self._get_stats( result_set=g_alarm_solved_total, column='mean') al_sol_max_tot = self._get_stats( result_set=g_alarm_solved_total, column='max') no_ack_sol_tot = al_sol_cnt_tot - ack_sol_cnt_tot result['__total__'] = { 'stats_count': { 'alarms_new': al_new_cnt_tot, 'alarms_ack': ack_new_cnt_tot, 'alarms_solved_ack': ack_sol_cnt_tot, 'alarms_solved_without_ack': no_ack_sol_tot }, 'stats_delay': { 'ack_delay_min': ack_new_min_tot, 'ack_delay_avg': int(ack_new_avg_tot), 'ack_delay_max': ack_new_max_tot, 'ack_solved_delay_min': ack_sol_min_tot, 'ack_solved_delay_avg': int(ack_sol_avg_tot), 'ack_solved_delay_max': ack_sol_max_tot, 'alarm_solved_delay_min': al_sol_min_tot, 'alarm_solved_delay_avg': int(al_sol_avg_tot), 'alarm_solved_delay_max': al_sol_max_tot } } for tag_group, tag_names in tags.items(): self.logger.debug( "Retrieving stats for tag id '{}'".format(tag_group)) result[tag_group] = [] tag_regex = self._influx_or_regex(tag_names) where = '{} AND {} =~ {}'.format(time_where, tag_group, tag_regex) g_alarm_new = self._influx_query( metric_id='alarm_opened_count', aggregations='count', condition=where, groupby=tag_group ) g_ack_new = self._influx_query( metric_id='alarm_ack_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where, groupby=tag_group ) g_ack_solved = self._influx_query( metric_id='alarm_ack_solved_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where, groupby=tag_group ) g_alarm_solved = self._influx_query( metric_id='alarm_solved_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where, groupby=tag_group ) for tag_name in tag_names: self.logger.debug( "Retrieving stats for tag value '{}'".format(tag_name)) al_new_cnt = self._get_stats( result_set=g_alarm_new, tags={tag_group: tag_name}, column='count' ) ack_new_cnt = self._get_stats( result_set=g_ack_new, tags={tag_group: tag_name}, column='count' ) ack_new_min = self._get_stats( result_set=g_ack_new, tags={tag_group: tag_name}, column='min' ) ack_new_avg = self._get_stats( result_set=g_ack_new, tags={tag_group: tag_name}, column='mean' ) ack_new_max = self._get_stats( result_set=g_ack_new, tags={tag_group: tag_name}, column='max' ) ack_sol_cnt = self._get_stats( result_set=g_ack_solved, tags={tag_group: tag_name}, column='count' ) ack_sol_min = self._get_stats( result_set=g_ack_solved, tags={tag_group: tag_name}, column='min' ) ack_sol_avg = self._get_stats( result_set=g_ack_solved, tags={tag_group: tag_name}, column='mean' ) ack_sol_max = self._get_stats( result_set=g_ack_solved, tags={tag_group: tag_name}, column='max' ) al_sol_cnt = self._get_stats( result_set=g_alarm_solved, tags={tag_group: tag_name}, column='count' ) al_sol_min = self._get_stats( result_set=g_alarm_solved, tags={tag_group: tag_name}, column='min' ) al_sol_avg = self._get_stats( result_set=g_alarm_solved, tags={tag_group: tag_name}, column='mean' ) al_sol_max = self._get_stats( result_set=g_alarm_solved, tags={tag_group: tag_name}, column='max' ) no_ack_sol = al_sol_cnt - ack_sol_cnt stats = { 'name': tag_name, 'stats_count': { 'alarms_new': al_new_cnt, 'alarms_ack': ack_new_cnt, 'alarms_solved_ack': ack_sol_cnt, 'alarms_solved_without_ack': no_ack_sol }, 'stats_delay': { 'ack_delay_min': ack_new_min, 'ack_delay_avg': int(ack_new_avg), 'ack_delay_max': ack_new_max, 'ack_solved_delay_min': ack_sol_min, 'ack_solved_delay_avg': int(ack_sol_avg), 'ack_solved_delay_max': ack_sol_max, 'alarm_solved_delay_min': al_sol_min, 'alarm_solved_delay_avg': int(al_sol_avg), 'alarm_solved_delay_max': al_sol_max } } result[tag_group].append(stats) # Insert total stats for tag_group at the end of the list g_alarm_new_total = self._influx_query( metric_id='alarm_opened_count', aggregations='count', condition=where ) g_ack_new_total = self._influx_query( metric_id='alarm_ack_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where ) g_ack_solved_total = self._influx_query( metric_id='alarm_ack_solved_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where ) g_alarm_solved_total = self._influx_query( metric_id='alarm_solved_delay', aggregations=['count', 'min', 'mean', 'max'], condition=where ) al_new_cnt_tot = self._get_stats( result_set=g_alarm_new_total, column='count') ack_new_cnt_tot = self._get_stats( result_set=g_ack_new_total, column='count') ack_new_min_tot = self._get_stats( result_set=g_ack_new_total, column='min') ack_new_avg_tot = self._get_stats( result_set=g_ack_new_total, column='mean') ack_new_max_tot = self._get_stats( result_set=g_ack_new_total, column='max') ack_sol_cnt_tot = self._get_stats( result_set=g_ack_solved_total, column='count') ack_sol_min_tot = self._get_stats( result_set=g_ack_solved_total, column='min') ack_sol_avg_tot = self._get_stats( result_set=g_ack_solved_total, column='mean') ack_sol_max_tot = self._get_stats( result_set=g_ack_solved_total, column='max') al_sol_cnt_tot = self._get_stats( result_set=g_alarm_solved_total, column='count') al_sol_min_tot = self._get_stats( result_set=g_alarm_solved_total, column='min') al_sol_avg_tot = self._get_stats( result_set=g_alarm_solved_total, column='mean') al_sol_max_tot = self._get_stats( result_set=g_alarm_solved_total, column='max') no_ack_sol_tot = al_sol_cnt_tot - ack_sol_cnt_tot stats = { 'name': '__total__', 'stats_count': { 'alarms_new': al_new_cnt_tot, 'alarms_ack': ack_new_cnt_tot, 'alarms_solved_ack': ack_sol_cnt_tot, 'alarms_solved_without_ack': no_ack_sol_tot }, 'stats_delay': { 'ack_delay_min': ack_new_min_tot, 'ack_delay_avg': int(ack_new_avg_tot), 'ack_delay_max': ack_new_max_tot, 'ack_solved_delay_min': ack_sol_min_tot, 'ack_solved_delay_avg': int(ack_sol_avg_tot), 'ack_solved_delay_max': ack_sol_max_tot, 'alarm_solved_delay_min': al_sol_min_tot, 'alarm_solved_delay_avg': int(al_sol_avg_tot), 'alarm_solved_delay_max': al_sol_max_tot } } result[tag_group].append(stats) self.logger.debug("Event based stats have succesfully been retrieved") return result def get_user_stats(self, tstart, tstop, users=[], tags={}): """ Get user related stats :param int tstart: Start timestamp :param int tstop: End timestamp :param list users: Users whose stats want to be retrieved :param dict tags: Groups stats by value :returns: Stats or an empty list if users=[] :rtype: list Example: users = ['u1', 'u2'] tags = {'domain': ['d1', 'd2'], 'perimeter': ['p1']} result = [ { 'author': 'MMA', 'ack': { 'total': 12, 'delay_min': 3, 'delay_avg': 5, 'delay_max': 10, }, 'session': { 'duration_min': 12, 'duration_avg': 50, 'duration_max': 300, }, 'tags': { 'domain': [ { 'name': 'd1', 'ack_total': 3 }, { 'name': 'd2', 'ack_total': 4 } ], 'perimeter': [ { 'name': 'p1', 'ack_total': 7 } ] } } ] """ self.logger.info("Retrieving user based stats") result = [] if not users: return result time_where = 'time >= {}s AND time <= {}s'.format(tstart, tstop) user_regex = self._influx_or_regex(users) user_where = 'component =~ {}'.format(user_regex) tags_where = '' for tag_group, tag_names in tags.items(): tags_where += '{} =~ {} OR '.format( tag_group, self._influx_or_regex(tag_names) ) else: # Remove trailing ' OR ' tags_where = tags_where[:-4] if tags_where: ack_new_where = '{} AND {} AND ({})'.format( time_where, user_where, tags_where ) else: ack_new_where = '{} AND {}'.format( time_where, user_where ) g_ack_new = self._influx_query( metric_id='alarm_ack_delay', aggregations=['count', 'min', 'mean', 'max'], condition=ack_new_where, groupby='component' ) duration_where = '{} AND {}'.format(time_where, user_where) g_duration = self._influx_query( metric_id='session_duration', aggregations=['min', 'mean', 'max'], condition=duration_where, groupby='component' ) for user in users: self.logger.debug("Retrieving stats for user '{}'".format(user)) ack_new_cnt = self._get_stats( result_set=g_ack_new, tags={'component': user}, column='count') ack_new_min = self._get_stats( result_set=g_ack_new, tags={'component': user}, column='min') ack_new_avg = self._get_stats( result_set=g_ack_new, tags={'component': user}, column='mean') ack_new_max = self._get_stats( result_set=g_ack_new, tags={'component': user}, column='max') duration_min = self._get_stats( result_set=g_duration, tags={'component': user}, column='min') duration_avg = self._get_stats( result_set=g_duration, tags={'component': user}, column='mean') duration_max = self._get_stats( result_set=g_duration, tags={'component': user}, column='max') stats = { 'author': user, 'ack': { 'total': ack_new_cnt, 'delay_min': ack_new_min, 'delay_avg': int(ack_new_avg), 'delay_max': ack_new_max }, 'session': { 'duration_min': int(duration_min), 'duration_avg': int(duration_avg), 'duration_max': int(duration_max) }, 'tags': {} } result.append(stats) for tag_group, tag_names in tags.items(): self.logger.debug( "Retrieving stats for tag id '{}'".format(tag_group) ) tag_regex = self._influx_or_regex(tag_names) tag_where = '{} AND {} =~ {}'.format( user_where, tag_group, tag_regex) g_ack_new = self._influx_query( metric_id='alarm_ack_delay', aggregations='count', condition=tag_where, groupby=['component', tag_group] ) for tag_name in tag_names: self.logger.debug( "Retrieving stats for tag value '{}'".format(tag_name) ) for i in range(len(users)): ack_new = self._get_stats( result_set=g_ack_new, tags={'component': users[i], tag_group: tag_name}, column='count' ) result[i]['tags'].setdefault(tag_group, []) result[i]['tags'][tag_group].append( { 'name': tag_name, 'ack_total': ack_new } ) self.logger.debug("User based stats have succesfully been retrieved") return result def _influx_or_regex(self, items): """ Transform a list of strings in an influx regex with an or filter between each element. :param list items: List of strings :return: Ready to use regex :rtype: str """ regex = '/^(' for item in items: regex = '{}{}|'.format(regex, escape(item)) # Truncate regex for the last '|' regex = '{})$/'.format(regex[:-1]) return regex def _influx_query( self, metric_id, aggregations=[], condition=None, groupby=[] ): query = 'select' if not aggregations: query = '{} value'.format(query) else: for aggr in ensure_iterable(aggregations): query = '{} {}(value),'.format(query, aggr) else: # Remove last ',' query = query[:-1] query = '{} from {}'.format(query, metric_id) if condition is not None: query = '{} where {}'.format(query, condition) if groupby: query = '{} group by'.format(query) for gb in ensure_iterable(groupby): query = '{} {},'.format(query, gb) # Remove last ',' else: query = query[:-1] self.logger.debug(u'Processing query `{}`'.format(query)) result = self.influxdbstg.raw_query(query) # If something went wrong (bad metric_id, bad aggregation...), # InfluxDBStorage will just return None if result is None: raise ValueError( 'Query `{}` failed : unable to retrieve stats'.format(query) ) return result def _get_stats(self, result_set, tags={}, column=None): self.logger.debug( "Retrieving stats on column {} for tags '{}'".format( column, tags ) ) try: stats = next( result_set.get_points(tags=tags) ) # If there is no entry for tag name, stats cannot be iterated on except StopIteration: return 0 if column is None: return stats else: # stats[column] might be None if requested period does not have # any data. if stats[column] is not None: return stats[column] else: return 0