def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of models.Statistics instance containing meter statistics described by the query parameters. The filter must have a meter value set. """ if (groupby and set(groupby) - set(['user_id', 'project_id', 'resource_id', 'source'])): raise NotImplementedError("Unable to group by these fields") if aggregate: raise NotImplementedError('Selectable aggregates not implemented') q = pymongo_base.make_query_from_filter(sample_filter) if period: if sample_filter.start: period_start = sample_filter.start else: period_start = self.db.meter.find(limit=1, sort=[('timestamp', pymongo.ASCENDING) ])[0]['timestamp'] if groupby: sort_keys = ['counter_name'] + groupby + ['timestamp'] else: sort_keys = ['counter_name', 'timestamp'] sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir='asc') meters = self.db.meter.find(q, sort=sort_instructions) def _group_key(meter): # the method to define a key for groupby call key = {} for y in sort_keys: if y == 'timestamp' and period: key[y] = ( timeutils.delta_seconds(period_start, meter[y]) // period) elif y != 'timestamp': key[y] = meter[y] return key def _to_offset(periods): return { 'days': (periods * period) // self.SECONDS_IN_A_DAY, 'seconds': (periods * period) % self.SECONDS_IN_A_DAY } for key, grouped_meters in itertools.groupby(meters, key=_group_key): stat = models.Statistics(unit=None, min=sys.maxint, max=-sys.maxint, avg=0, sum=0, count=0, period=0, period_start=0, period_end=0, duration=0, duration_start=0, duration_end=0, groupby=None) for meter in grouped_meters: stat.unit = meter.get('counter_unit', '') m_volume = meter.get('counter_volume') if stat.min > m_volume: stat.min = m_volume if stat.max < m_volume: stat.max = m_volume stat.sum += m_volume stat.count += 1 if stat.duration_start == 0: stat.duration_start = meter['timestamp'] stat.duration_end = meter['timestamp'] if groupby and not stat.groupby: stat.groupby = {} for group_key in groupby: stat.groupby[group_key] = meter[group_key] stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end) stat.avg = stat.sum / stat.count if period: stat.period = period periods = key.get('timestamp') stat.period_start = period_start + \ datetime.timedelta(**(_to_offset(periods))) stat.period_end = period_start + \ datetime.timedelta(**(_to_offset(periods + 1))) else: stat.period_start = stat.duration_start stat.period_end = stat.duration_end yield stat
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of models.Statistics instance containing meter statistics described by the query parameters. The filter must have a meter value set. """ if (groupby and set(groupby) - set(['user_id', 'project_id', 'resource_id', 'source'])): raise NotImplementedError("Unable to group by these fields") q = pymongo_base.make_query_from_filter(sample_filter) if period: if sample_filter.start: period_start = sample_filter.start else: period_start = self.db.meter.find( limit=1, sort=[('timestamp', pymongo.ASCENDING)])[0]['timestamp'] period_start = int(calendar.timegm(period_start.utctimetuple())) map_params = {'period': period, 'period_first': period_start, 'groupby_fields': json.dumps(groupby)} if groupby: map_fragment = self.MAP_STATS_PERIOD_GROUPBY else: map_fragment = self.MAP_STATS_PERIOD else: if groupby: map_params = {'groupby_fields': json.dumps(groupby)} map_fragment = self.MAP_STATS_GROUPBY else: map_params = dict() map_fragment = self.MAP_STATS sub = self._aggregate_param map_params['aggregate_initial_val'] = sub('emit_initial', aggregate) map_params['aggregate_body_val'] = sub('emit_body', aggregate) map_stats = map_fragment % map_params reduce_params = dict( aggregate_initial_val=sub('reduce_initial', aggregate), aggregate_body_val=sub('reduce_body', aggregate), aggregate_computation_val=sub('reduce_computation', aggregate) ) reduce_stats = self.REDUCE_STATS % reduce_params finalize_params = dict(aggregate_val=sub('finalize', aggregate)) finalize_stats = self.FINALIZE_STATS % finalize_params results = self.db.meter.map_reduce( map_stats, reduce_stats, {'inline': 1}, finalize=finalize_stats, query=q, ) # FIXME(terriyu) Fix get_meter_statistics() so we don't use sorted() # to return the results return sorted( (self._stats_result_to_model(r['value'], groupby, aggregate) for r in results['results']), key=operator.attrgetter('period_start'))
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of models.Statistics instance containing meter statistics described by the query parameters. The filter must have a meter value set. """ if groupby and set(groupby) - set(["user_id", "project_id", "resource_id", "source"]): raise NotImplementedError("Unable to group by these fields") if aggregate: msg = _("Selectable aggregates not implemented") raise NotImplementedError(msg) q = pymongo_base.make_query_from_filter(sample_filter) if period: if sample_filter.start: period_start = sample_filter.start else: period_start = self.db.meter.find(limit=1, sort=[("timestamp", pymongo.ASCENDING)])[0]["timestamp"] if groupby: sort_keys = ["counter_name"] + groupby + ["timestamp"] else: sort_keys = ["counter_name", "timestamp"] sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir="asc") meters = self.db.meter.find(q, sort=sort_instructions) def _group_key(meter): # the method to define a key for groupby call key = {} for y in sort_keys: if y == "timestamp" and period: key[y] = timeutils.delta_seconds(period_start, meter[y]) // period elif y != "timestamp": key[y] = meter[y] return key def _to_offset(periods): return { "days": (periods * period) // self.SECONDS_IN_A_DAY, "seconds": (periods * period) % self.SECONDS_IN_A_DAY, } for key, grouped_meters in itertools.groupby(meters, key=_group_key): stat = models.Statistics( unit=None, min=sys.maxint, max=-sys.maxint, avg=0, sum=0, count=0, period=0, period_start=0, period_end=0, duration=0, duration_start=0, duration_end=0, groupby=None, ) for meter in grouped_meters: stat.unit = meter.get("counter_unit", "") m_volume = meter.get("counter_volume") if stat.min > m_volume: stat.min = m_volume if stat.max < m_volume: stat.max = m_volume stat.sum += m_volume stat.count += 1 if stat.duration_start == 0: stat.duration_start = meter["timestamp"] stat.duration_end = meter["timestamp"] if groupby and not stat.groupby: stat.groupby = {} for group_key in groupby: stat.groupby[group_key] = meter[group_key] stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end) stat.avg = stat.sum / stat.count if period: stat.period = period periods = key.get("timestamp") stat.period_start = period_start + datetime.timedelta(**(_to_offset(periods))) stat.period_end = period_start + datetime.timedelta(**(_to_offset(periods + 1))) else: stat.period_start = stat.duration_start stat.period_end = stat.duration_end yield stat