def get_meter_statistics(self,
sample_filter,
period=None,
groupby=None,
aggregate=None):
"""Return an iterable of models.Statistics instance containing meter
statistics described by the query parameters.
The filter must have a meter value set.
"""
if (groupby and set(groupby) -
set(['user_id', 'project_id', 'resource_id', 'source'])):
raise NotImplementedError("Unable to group by these fields")
if aggregate:
raise NotImplementedError('Selectable aggregates not implemented')
q = pymongo_base.make_query_from_filter(sample_filter)
if period:
if sample_filter.start:
period_start = sample_filter.start
else:
period_start = self.db.meter.find(limit=1,
sort=[('timestamp',
pymongo.ASCENDING)
])[0]['timestamp']
if groupby:
sort_keys = ['counter_name'] + groupby + ['timestamp']
else:
sort_keys = ['counter_name', 'timestamp']
sort_instructions = self._build_sort_instructions(sort_keys=sort_keys,
sort_dir='asc')
meters = self.db.meter.find(q, sort=sort_instructions)
def _group_key(meter):
# the method to define a key for groupby call
key = {}
for y in sort_keys:
if y == 'timestamp' and period:
key[y] = (
timeutils.delta_seconds(period_start, meter[y]) //
period)
elif y != 'timestamp':
key[y] = meter[y]
return key
def _to_offset(periods):
return {
'days': (periods * period) // self.SECONDS_IN_A_DAY,
'seconds': (periods * period) % self.SECONDS_IN_A_DAY
}
for key, grouped_meters in itertools.groupby(meters, key=_group_key):
stat = models.Statistics(unit=None,
min=sys.maxint,
max=-sys.maxint,
avg=0,
sum=0,
count=0,
period=0,
period_start=0,
period_end=0,
duration=0,
duration_start=0,
duration_end=0,
groupby=None)
for meter in grouped_meters:
stat.unit = meter.get('counter_unit', '')
m_volume = meter.get('counter_volume')
if stat.min > m_volume:
stat.min = m_volume
if stat.max < m_volume:
stat.max = m_volume
stat.sum += m_volume
stat.count += 1
if stat.duration_start == 0:
stat.duration_start = meter['timestamp']
stat.duration_end = meter['timestamp']
if groupby and not stat.groupby:
stat.groupby = {}
for group_key in groupby:
stat.groupby[group_key] = meter[group_key]
stat.duration = timeutils.delta_seconds(stat.duration_start,
stat.duration_end)
stat.avg = stat.sum / stat.count
if period:
stat.period = period
periods = key.get('timestamp')
stat.period_start = period_start + \
datetime.timedelta(**(_to_offset(periods)))
stat.period_end = period_start + \
datetime.timedelta(**(_to_offset(periods + 1)))
else:
stat.period_start = stat.duration_start
stat.period_end = stat.duration_end
yield stat
def get_meter_statistics(self, sample_filter, period=None, groupby=None,
aggregate=None):
"""Return an iterable of models.Statistics instance containing meter
statistics described by the query parameters.
The filter must have a meter value set.
"""
if (groupby and
set(groupby) - set(['user_id', 'project_id',
'resource_id', 'source'])):
raise NotImplementedError("Unable to group by these fields")
q = pymongo_base.make_query_from_filter(sample_filter)
if period:
if sample_filter.start:
period_start = sample_filter.start
else:
period_start = self.db.meter.find(
limit=1, sort=[('timestamp',
pymongo.ASCENDING)])[0]['timestamp']
period_start = int(calendar.timegm(period_start.utctimetuple()))
map_params = {'period': period,
'period_first': period_start,
'groupby_fields': json.dumps(groupby)}
if groupby:
map_fragment = self.MAP_STATS_PERIOD_GROUPBY
else:
map_fragment = self.MAP_STATS_PERIOD
else:
if groupby:
map_params = {'groupby_fields': json.dumps(groupby)}
map_fragment = self.MAP_STATS_GROUPBY
else:
map_params = dict()
map_fragment = self.MAP_STATS
sub = self._aggregate_param
map_params['aggregate_initial_val'] = sub('emit_initial', aggregate)
map_params['aggregate_body_val'] = sub('emit_body', aggregate)
map_stats = map_fragment % map_params
reduce_params = dict(
aggregate_initial_val=sub('reduce_initial', aggregate),
aggregate_body_val=sub('reduce_body', aggregate),
aggregate_computation_val=sub('reduce_computation', aggregate)
)
reduce_stats = self.REDUCE_STATS % reduce_params
finalize_params = dict(aggregate_val=sub('finalize', aggregate))
finalize_stats = self.FINALIZE_STATS % finalize_params
results = self.db.meter.map_reduce(
map_stats,
reduce_stats,
{'inline': 1},
finalize=finalize_stats,
query=q,
)
# FIXME(terriyu) Fix get_meter_statistics() so we don't use sorted()
# to return the results
return sorted(
(self._stats_result_to_model(r['value'], groupby, aggregate)
for r in results['results']),
key=operator.attrgetter('period_start'))
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None):
"""Return an iterable of models.Statistics instance containing meter
statistics described by the query parameters.
The filter must have a meter value set.
"""
if groupby and set(groupby) - set(["user_id", "project_id", "resource_id", "source"]):
raise NotImplementedError("Unable to group by these fields")
if aggregate:
msg = _("Selectable aggregates not implemented")
raise NotImplementedError(msg)
q = pymongo_base.make_query_from_filter(sample_filter)
if period:
if sample_filter.start:
period_start = sample_filter.start
else:
period_start = self.db.meter.find(limit=1, sort=[("timestamp", pymongo.ASCENDING)])[0]["timestamp"]
if groupby:
sort_keys = ["counter_name"] + groupby + ["timestamp"]
else:
sort_keys = ["counter_name", "timestamp"]
sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir="asc")
meters = self.db.meter.find(q, sort=sort_instructions)
def _group_key(meter):
# the method to define a key for groupby call
key = {}
for y in sort_keys:
if y == "timestamp" and period:
key[y] = timeutils.delta_seconds(period_start, meter[y]) // period
elif y != "timestamp":
key[y] = meter[y]
return key
def _to_offset(periods):
return {
"days": (periods * period) // self.SECONDS_IN_A_DAY,
"seconds": (periods * period) % self.SECONDS_IN_A_DAY,
}
for key, grouped_meters in itertools.groupby(meters, key=_group_key):
stat = models.Statistics(
unit=None,
min=sys.maxint,
max=-sys.maxint,
avg=0,
sum=0,
count=0,
period=0,
period_start=0,
period_end=0,
duration=0,
duration_start=0,
duration_end=0,
groupby=None,
)
for meter in grouped_meters:
stat.unit = meter.get("counter_unit", "")
m_volume = meter.get("counter_volume")
if stat.min > m_volume:
stat.min = m_volume
if stat.max < m_volume:
stat.max = m_volume
stat.sum += m_volume
stat.count += 1
if stat.duration_start == 0:
stat.duration_start = meter["timestamp"]
stat.duration_end = meter["timestamp"]
if groupby and not stat.groupby:
stat.groupby = {}
for group_key in groupby:
stat.groupby[group_key] = meter[group_key]
stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end)
stat.avg = stat.sum / stat.count
if period:
stat.period = period
periods = key.get("timestamp")
stat.period_start = period_start + datetime.timedelta(**(_to_offset(periods)))
stat.period_end = period_start + datetime.timedelta(**(_to_offset(periods + 1)))
else:
stat.period_start = stat.duration_start
stat.period_end = stat.duration_end
yield stat
