def _is_master(self, interval):
"""Determine if the current partition is the master."""
now = timeutils.utcnow()
if timeutils.delta_seconds(self.start, now) < interval * 2:
LOG.debug(_('%s still warming up') % self.this)
return False
is_master = True
for partition, last_heard in self.reports.items():
delta = timeutils.delta_seconds(last_heard, now)
LOG.debug(
_('last heard from %(report)s %(delta)s seconds ago') %
dict(report=partition, delta=delta))
if delta > interval * 2:
del self.reports[partition]
self._record_oldest(partition, stale=True)
LOG.debug(
_('%(this)s detects stale evaluator: %(stale)s') %
dict(this=self.this, stale=partition))
self.presence_changed = True
elif partition < self.this:
is_master = False
LOG.info(
_('%(this)s sees older potential master: %(older)s') %
dict(this=self.this, older=partition))
LOG.info(
_('%(this)s is master?: %(is_master)s') %
dict(this=self.this, is_master=is_master))
return is_master
def get_meter_statistics(self, event_filter, period=None):
"""Return a dictionary containing meter statistics.
described by the query parameters.
The filter must have a meter value set.
{ 'min':
'max':
'avg':
'sum':
'count':
'period':
'period_start':
'period_end':
'duration':
'duration_start':
'duration_end':
}
"""
res = self._make_stats_query(event_filter).all()[0]
if not period:
return [self._stats_result_to_dict(res, 0, res.tsmin, res.tsmax)]
query = self._make_stats_query(event_filter)
# HACK(jd) This is an awful method to compute stats by period, but
# since we're trying to be SQL agnostic we have to write portable
# code, so here it is, admire! We're going to do one request to get
# stats by period. We would like to use GROUP BY, but there's no
# portable way to manipulate timestamp in SQL, so we can't.
results = []
for i in range(
int(
math.ceil(
timeutils.delta_seconds(event_filter.start or res.tsmin, event_filter.end or res.tsmax)
/ float(period)
)
)
):
period_start = event_filter.start + datetime.timedelta(seconds=i * period)
period_end = period_start + datetime.timedelta(seconds=period)
q = query.filter(Meter.timestamp >= period_start)
q = q.filter(Meter.timestamp < period_end)
results.append(
self._stats_result_to_dict(
result=q.all()[0],
period=int(timeutils.delta_seconds(period_start, period_end)),
period_start=period_start,
period_end=period_end,
)
)
return results
def _stats_result_to_dict(result, period_start, period_end):
return {'count': result.count,
'min': result.min,
'max': result.max,
'avg': result.avg,
'sum': result.sum,
'duration_start': result.tsmin,
'duration_end': result.tsmax,
'duration': timeutils.delta_seconds(result.tsmin,
result.tsmax),
'period': timeutils.delta_seconds(period_start, period_end),
'period_start': period_start,
'period_end': period_end}
def get_interval(event_filter, period, groupby, aggregate):
self.assertIsNotNone(event_filter.start)
self.assertIsNotNone(event_filter.end)
if event_filter.start > end or event_filter.end < start:
return []
duration_start = max(event_filter.start, start)
duration_end = min(event_filter.end, end)
duration = timeutils.delta_seconds(duration_start, duration_end)
return [
models.Statistics(
unit='',
min=0,
max=0,
avg=0,
sum=0,
count=0,
period=None,
period_start=None,
period_end=None,
duration=duration,
duration_start=duration_start,
duration_end=duration_end,
groupby=None,
)
]
def get_meter_statistics(self, event_filter, period=None):
"""Return an iterable of api_models.Statistics instances containing
meter statistics described by the query parameters.
The filter must have a meter value set.
"""
if not period or not event_filter.start or not event_filter.end:
res = self._make_stats_query(event_filter).all()[0]
if not period:
yield self._stats_result_to_model(res, 0, res.tsmin, res.tsmax)
return
query = self._make_stats_query(event_filter)
# HACK(jd) This is an awful method to compute stats by period, but
# since we're trying to be SQL agnostic we have to write portable
# code, so here it is, admire! We're going to do one request to get
# stats by period. We would like to use GROUP BY, but there's no
# portable way to manipulate timestamp in SQL, so we can't.
for period_start, period_end in base.iter_period(
event_filter.start or res.tsmin, event_filter.end or res.tsmax,
period):
q = query.filter(Meter.timestamp >= period_start)
q = q.filter(Meter.timestamp < period_end)
r = q.all()[0]
# Don't return results that didn't have any event
if r.count:
yield self._stats_result_to_model(
result=r,
period=int(
timeutils.delta_seconds(period_start, period_end)),
period_start=period_start,
period_end=period_end,
)
def handle_sample(self, context, counter, source):
"""Handle a sample, converting if necessary."""
LOG.debug('handling counter %s', (counter,))
key = counter.name + counter.resource_id
prev = self.cache.get(key)
timestamp = timeutils.parse_isotime(counter.timestamp)
self.cache[key] = (counter.volume, timestamp)
if prev:
prev_volume = prev[0]
prev_timestamp = prev[1]
time_delta = timeutils.delta_seconds(prev_timestamp, timestamp)
# we only allow negative deltas for noncumulative counters, whereas
# for cumulative we assume that a reset has occurred in the interim
# so that the current volume gives a lower bound on growth
volume_delta = (counter.volume - prev_volume
if (prev_volume <= counter.volume or
counter.type != ceilocounter.TYPE_CUMULATIVE)
else counter.volume)
rate_of_change = ((1.0 * volume_delta / time_delta)
if time_delta else 0.0)
transformed = self._convert(counter, rate_of_change)
LOG.debug(_('converted to: %s') % (transformed,))
counter = self._keep(counter, transformed)
elif self.replace:
LOG.warn(_('dropping counter with no predecessor: %s') % counter)
counter = None
return counter
def get_interval(event_filter, period, groupby, aggregate):
assert event_filter.start
assert event_filter.end
if (event_filter.start > end or event_filter.end < start):
return []
duration_start = max(event_filter.start, start)
duration_end = min(event_filter.end, end)
duration = timeutils.delta_seconds(duration_start, duration_end)
return [
models.Statistics(
unit='',
min=0,
max=0,
avg=0,
sum=0,
count=0,
period=None,
period_start=None,
period_end=None,
duration=duration,
duration_start=duration_start,
duration_end=duration_end,
groupby=None,
)
]
def _update_duration(self, start_timestamp, end_timestamp):
# "Clamp" the timestamps we return to the original time
# range, excluding the offset.
if (start_timestamp and self.duration_start
and self.duration_start < start_timestamp):
self.duration_start = start_timestamp
LOG.debug('clamping min timestamp to range')
if (end_timestamp and self.duration_end
and self.duration_end > end_timestamp):
self.duration_end = end_timestamp
LOG.debug('clamping max timestamp to range')
# If we got valid timestamps back, compute a duration in minutes.
#
# If the min > max after clamping then we know the
# timestamps on the samples fell outside of the time
# range we care about for the query, so treat them as
# "invalid."
#
# If the timestamps are invalid, return None as a
# sentinal indicating that there is something "funny"
# about the range.
if (self.duration_start and self.duration_end
and self.duration_start <= self.duration_end):
self.duration = timeutils.delta_seconds(self.duration_start,
self.duration_end)
else:
self.duration_start = self.duration_end = self.duration = None
def _update_duration(self, start_timestamp, end_timestamp):
# "Clamp" the timestamps we return to the original time
# range, excluding the offset.
if (start_timestamp and
self.duration_start and
self.duration_start < start_timestamp):
self.duration_start = start_timestamp
LOG.debug('clamping min timestamp to range')
if (end_timestamp and
self.duration_end and
self.duration_end > end_timestamp):
self.duration_end = end_timestamp
LOG.debug('clamping max timestamp to range')
# If we got valid timestamps back, compute a duration in minutes.
#
# If the min > max after clamping then we know the
# timestamps on the samples fell outside of the time
# range we care about for the query, so treat them as
# "invalid."
#
# If the timestamps are invalid, return None as a
# sentinal indicating that there is something "funny"
# about the range.
if (self.duration_start and
self.duration_end and
self.duration_start <= self.duration_end):
self.duration = timeutils.delta_seconds(self.duration_start,
self.duration_end)
else:
self.duration_start = self.duration_end = self.duration = None
def handle_sample(self, context, s):
"""Handle a sample, converting if necessary."""
LOG.debug(_('handling sample %s'), (s,))
key = s.name + s.resource_id
prev = self.cache.get(key)
timestamp = timeutils.parse_isotime(s.timestamp)
self.cache[key] = (s.volume, timestamp)
if prev:
prev_volume = prev[0]
prev_timestamp = prev[1]
time_delta = timeutils.delta_seconds(prev_timestamp, timestamp)
# we only allow negative deltas for noncumulative samples, whereas
# for cumulative we assume that a reset has occurred in the interim
# so that the current volume gives a lower bound on growth
volume_delta = (s.volume - prev_volume
if (prev_volume <= s.volume or
s.type != sample.TYPE_CUMULATIVE)
else s.volume)
rate_of_change = ((1.0 * volume_delta / time_delta)
if time_delta else 0.0)
s = self._convert(s, rate_of_change)
LOG.debug(_('converted to: %s'), (s,))
else:
LOG.warn(_('dropping sample with no predecessor: %s'),
(s,))
s = None
return s
def get_interval(ignore_self, event_filter, period, groupby):
assert event_filter.start
assert event_filter.end
if (event_filter.start > end or event_filter.end < start):
return []
duration_start = max(event_filter.start, start)
duration_end = min(event_filter.end, end)
duration = timeutils.delta_seconds(duration_start, duration_end)
return [
models.Statistics(
unit='',
min=0,
max=0,
avg=0,
sum=0,
count=0,
period=None,
period_start=None,
period_end=None,
duration=duration,
duration_start=duration_start,
duration_end=duration_end,
groupby=None,
)
]
def get_meter_statistics(self, sample_filter, period=None):
"""Return an iterable of api_models.Statistics instances containing
meter statistics described by the query parameters.
The filter must have a meter value set.
"""
if not period or not sample_filter.start or not sample_filter.end:
res = self._make_stats_query(sample_filter).all()[0]
if not period:
yield self._stats_result_to_model(res, 0, res.tsmin, res.tsmax)
return
query = self._make_stats_query(sample_filter)
# HACK(jd) This is an awful method to compute stats by period, but
# since we're trying to be SQL agnostic we have to write portable
# code, so here it is, admire! We're going to do one request to get
# stats by period. We would like to use GROUP BY, but there's no
# portable way to manipulate timestamp in SQL, so we can't.
for period_start, period_end in base.iter_period(
sample_filter.start or res.tsmin, sample_filter.end or res.tsmax, period
):
q = query.filter(Meter.timestamp >= period_start)
q = q.filter(Meter.timestamp < period_end)
r = q.all()[0]
# Don't return results that didn't have any data.
if r.count:
yield self._stats_result_to_model(
result=r,
period=int(timeutils.delta_seconds(period_start, period_end)),
period_start=period_start,
period_end=period_end,
)
def compute_duration_by_resource(resource, meter):
"""Return the earliest timestamp, last timestamp,
and duration for the resource and meter.
:param resource: The ID of the resource.
:param meter: The name of the meter.
:param start_timestamp: ISO-formatted string of the
earliest timestamp to return.
:param end_timestamp: ISO-formatted string of the
latest timestamp to return.
:param search_offset: Number of minutes before
and after start and end timestamps to query.
"""
q_ts = _get_query_timestamps(flask.request.args)
start_timestamp = q_ts['start_timestamp']
end_timestamp = q_ts['end_timestamp']
# Query the database for the interval of timestamps
# within the desired range.
f = storage.SampleFilter(
meter=meter,
project=acl.get_limited_to_project(flask.request.headers),
resource=resource,
start=q_ts['query_start'],
end=q_ts['query_end'],
)
stats = flask.request.storage_conn.get_meter_statistics(f)
min_ts, max_ts = stats.duration_start, stats.duration_end
# "Clamp" the timestamps we return to the original time
# range, excluding the offset.
LOG.debug('start_timestamp %s, end_timestamp %s, min_ts %s, max_ts %s',
start_timestamp, end_timestamp, min_ts, max_ts)
if start_timestamp and min_ts and min_ts < start_timestamp:
min_ts = start_timestamp
LOG.debug('clamping min timestamp to range')
if end_timestamp and max_ts and max_ts > end_timestamp:
max_ts = end_timestamp
LOG.debug('clamping max timestamp to range')
# If we got valid timestamps back, compute a duration in minutes.
#
# If the min > max after clamping then we know the
# timestamps on the samples fell outside of the time
# range we care about for the query, so treat them as
# "invalid."
#
# If the timestamps are invalid, return None as a
# sentinal indicating that there is something "funny"
# about the range.
if min_ts and max_ts and (min_ts <= max_ts):
duration = timeutils.delta_seconds(min_ts, max_ts)
else:
min_ts = max_ts = duration = None
return flask.jsonify(
start_timestamp=min_ts,
end_timestamp=max_ts,
duration=duration,
)
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_LW('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_LE('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def handle_sample(self, context, counter):
"""Handle a sample, converting if necessary."""
LOG.debug('handling counter %s', (counter, ))
key = counter.name + counter.resource_id
prev = self.cache.get(key)
timestamp = timeutils.parse_isotime(counter.timestamp)
self.cache[key] = (counter.volume, timestamp)
if prev:
prev_volume = prev[0]
prev_timestamp = prev[1]
time_delta = timeutils.delta_seconds(prev_timestamp, timestamp)
# we only allow negative deltas for noncumulative counters, whereas
# for cumulative we assume that a reset has occurred in the interim
# so that the current volume gives a lower bound on growth
volume_delta = (counter.volume - prev_volume if
(prev_volume <= counter.volume
or counter.type != sample.TYPE_CUMULATIVE) else
counter.volume)
rate_of_change = ((1.0 * volume_delta /
time_delta) if time_delta else 0.0)
counter = self._convert(counter, rate_of_change)
LOG.debug(_('converted to: %s') % (counter, ))
else:
LOG.warn(
_('dropping counter with no predecessor: %s') % (counter, ))
counter = None
return counter
def compute_duration_by_resource(resource, meter):
"""Return the earliest timestamp, last timestamp,
and duration for the resource and meter.
:param resource: The ID of the resource.
:param meter: The name of the meter.
:param start_timestamp: ISO-formatted string of the
earliest timestamp to return.
:param end_timestamp: ISO-formatted string of the
latest timestamp to return.
:param search_offset: Number of minutes before
and after start and end timestamps to query.
"""
q_ts = _get_query_timestamps(flask.request.args)
start_timestamp = q_ts['start_timestamp']
end_timestamp = q_ts['end_timestamp']
# Query the database for the interval of timestamps
# within the desired range.
f = storage.EventFilter(
meter=meter,
project=acl.get_limited_to_project(flask.request.headers),
resource=resource,
start=q_ts['query_start'],
end=q_ts['query_end'],
)
stats = flask.request.storage_conn.get_meter_statistics(f)
min_ts, max_ts = stats.duration_start, stats.duration_end
# "Clamp" the timestamps we return to the original time
# range, excluding the offset.
LOG.debug('start_timestamp %s, end_timestamp %s, min_ts %s, max_ts %s',
start_timestamp, end_timestamp, min_ts, max_ts)
if start_timestamp and min_ts and min_ts < start_timestamp:
min_ts = start_timestamp
LOG.debug('clamping min timestamp to range')
if end_timestamp and max_ts and max_ts > end_timestamp:
max_ts = end_timestamp
LOG.debug('clamping max timestamp to range')
# If we got valid timestamps back, compute a duration in minutes.
#
# If the min > max after clamping then we know the
# timestamps on the samples fell outside of the time
# range we care about for the query, so treat them as
# "invalid."
#
# If the timestamps are invalid, return None as a
# sentinal indicating that there is something "funny"
# about the range.
if min_ts and max_ts and (min_ts <= max_ts):
duration = timeutils.delta_seconds(min_ts, max_ts)
else:
min_ts = max_ts = duration = None
return flask.jsonify(start_timestamp=min_ts,
end_timestamp=max_ts,
duration=duration,
)
def assertTimestampEqual(self, first, second, msg=None):
"""Checks that two timestamps are equals.
This relies on assertAlmostEqual to avoid rounding problem, and only
checks up the first microsecond values.
"""
return self.assertAlmostEqual(timeutils.delta_seconds(first, second), 0.0, places=5)
def _group_key(meter):
# the method to define a key for groupby call
key = {}
for y in sort_keys:
if y == "timestamp" and period:
key[y] = timeutils.delta_seconds(period_start, meter[y]) // period
elif y != "timestamp":
key[y] = meter[y]
return key
def get_meter_statistics(self, sample_filter, period=None, groupby=None,
aggregate=None):
"""Return an iterable of api_models.Statistics instances containing
meter statistics described by the query parameters.
The filter must have a meter value set.
"""
if groupby:
for group in groupby:
if group not in ['user_id', 'project_id', 'resource_id']:
raise NotImplementedError('Unable to group by '
'these fields')
if not period:
for res in self._make_stats_query(sample_filter,
groupby,
aggregate):
if res.count:
yield self._stats_result_to_model(res, 0,
res.tsmin, res.tsmax,
groupby,
aggregate)
return
if not sample_filter.start or not sample_filter.end:
res = self._make_stats_query(sample_filter,
None,
aggregate).first()
if not res:
# NOTE(liusheng):The 'res' may be NoneType, because no
# sample has found with sample filter(s).
return
query = self._make_stats_query(sample_filter, groupby, aggregate)
# HACK(jd) This is an awful method to compute stats by period, but
# since we're trying to be SQL agnostic we have to write portable
# code, so here it is, admire! We're going to do one request to get
# stats by period. We would like to use GROUP BY, but there's no
# portable way to manipulate timestamp in SQL, so we can't.
for period_start, period_end in base.iter_period(
sample_filter.start or res.tsmin,
sample_filter.end or res.tsmax,
period):
q = query.filter(models.Sample.timestamp >= period_start)
q = q.filter(models.Sample.timestamp < period_end)
for r in q.all():
if r.count:
yield self._stats_result_to_model(
result=r,
period=int(timeutils.delta_seconds(period_start,
period_end)),
period_start=period_start,
period_end=period_end,
groupby=groupby,
aggregate=aggregate
)
def _group_key(meter):
# the method to define a key for groupby call
key = {}
for y in sort_keys:
if y == 'timestamp' and period:
key[y] = (timeutils.delta_seconds(period_start,
meter[y]) // period)
elif y != 'timestamp':
key[y] = meter[y]
return key
def assertTimestampEqual(self, first, second, msg=None):
"""Checks that two timestamps are equals.
This relies on assertAlmostEqual to avoid rounding problem, and only
checks up the first microsecond values.
"""
return self.assertAlmostEqual(timeutils.delta_seconds(first, second),
0.0,
places=5)
def get_meter_statistics(self,
sample_filter,
period=None,
groupby=None,
aggregate=None):
"""Return an iterable of api_models.Statistics instances containing
meter statistics described by the query parameters.
The filter must have a meter value set.
"""
if groupby:
for group in groupby:
if group not in ['user_id', 'project_id', 'resource_id']:
raise NotImplementedError('Unable to group by '
'these fields')
if not period:
for res in self._make_stats_query(sample_filter, groupby,
aggregate):
if res.count:
yield self._stats_result_to_model(res, 0, res.tsmin,
res.tsmax, groupby,
aggregate)
return
if not sample_filter.start or not sample_filter.end:
res = self._make_stats_query(sample_filter, None,
aggregate).first()
if not res:
# NOTE(liusheng):The 'res' may be NoneType, because no
# sample has found with sample filter(s).
return
query = self._make_stats_query(sample_filter, groupby, aggregate)
# HACK(jd) This is an awful method to compute stats by period, but
# since we're trying to be SQL agnostic we have to write portable
# code, so here it is, admire! We're going to do one request to get
# stats by period. We would like to use GROUP BY, but there's no
# portable way to manipulate timestamp in SQL, so we can't.
for period_start, period_end in base.iter_period(
sample_filter.start or res.tsmin, sample_filter.end
or res.tsmax, period):
q = query.filter(models.Sample.timestamp >= period_start)
q = q.filter(models.Sample.timestamp < period_end)
for r in q.all():
if r.count:
yield self._stats_result_to_model(
result=r,
period=int(
timeutils.delta_seconds(period_start, period_end)),
period_start=period_start,
period_end=period_end,
groupby=groupby,
aggregate=aggregate)
def _is_master(self, interval):
"""Determine if the current partition is the master."""
now = timeutils.utcnow()
if timeutils.delta_seconds(self.start, now) < interval * 2:
LOG.debug(_("%s still warming up") % self.this)
return False
is_master = True
for partition, last_heard in self.reports.items():
delta = timeutils.delta_seconds(last_heard, now)
LOG.debug(_("last heard from %(report)s %(delta)s seconds ago") % dict(report=partition, delta=delta))
if delta > interval * 2:
del self.reports[partition]
self._record_oldest(partition, stale=True)
LOG.debug(_("%(this)s detects stale evaluator: %(stale)s") % dict(this=self.this, stale=partition))
self.presence_changed = True
elif partition < self.this:
is_master = False
LOG.info(_("%(this)s sees older potential master: %(older)s") % dict(this=self.this, older=partition))
LOG.info(_("%(this)s is master?: %(is_master)s") % dict(this=self.this, is_master=is_master))
return is_master
def _record_poll_time(self):
"""Method records current time as the poll time.
:return: time in seconds since the last poll time was recorded
"""
current_time = timeutils.utcnow()
duration = None
if hasattr(self, "_last_poll_time"):
duration = timeutils.delta_seconds(self._last_poll_time, current_time)
self._last_poll_time = current_time
return duration
def _record_poll_time(self):
"""Method records current time as the poll time.
:return: time in seconds since the last poll time was recorded
"""
current_time = timeutils.utcnow()
duration = None
if hasattr(self, '_last_poll_time'):
duration = timeutils.delta_seconds(self._last_poll_time,
current_time)
self._last_poll_time = current_time
return duration
def _stats_result_to_dict(result, period, period_start, period_end):
return {
"count": result.count,
"min": result.min,
"max": result.max,
"avg": result.avg,
"sum": result.sum,
"duration_start": result.tsmin,
"duration_end": result.tsmax,
"duration": timeutils.delta_seconds(result.tsmin, result.tsmax),
"period": period,
"period_start": period_start,
"period_end": period_end,
}
def test_extract_when(self):
now = timeutils.utcnow()
modified = now + datetime.timedelta(minutes=1)
timeutils.set_time_override(now)
body = {"timestamp": str(modified)}
when = service.CollectorService._extract_when(body)
self.assertAlmostEqual(timeutils.delta_seconds(modified, when), 0.0,
places=5)
body = {"_context_timestamp": str(modified)}
when = service.CollectorService._extract_when(body)
self.assertAlmostEqual(timeutils.delta_seconds(modified, when), 0.0,
places=5)
then = now + datetime.timedelta(hours=1)
body = {"timestamp": str(modified), "_context_timestamp": str(then)}
when = service.CollectorService._extract_when(body)
self.assertAlmostEqual(timeutils.delta_seconds(modified, when), 0.0,
places=5)
when = service.CollectorService._extract_when({})
self.assertAlmostEqual(timeutils.delta_seconds(now, when), 0.0,
places=5)
def _stats_result_to_dict(result, period, period_start, period_end):
return {'count': int(result.count),
'min': result.min,
'max': result.max,
'avg': result.avg,
'sum': result.sum,
'duration_start': result.tsmin,
'duration_end': result.tsmax,
'duration': (timeutils.delta_seconds(result.tsmin,
result.tsmax)
if result.tsmin and result.tsmax
else None),
'period': period,
'period_start': period_start,
'period_end': period_end}
def iter_period(start, end, period):
"""Split a time from start to end in periods of a number of seconds. This
function yield the (start, end) time for each period composing the time
passed as argument.
:param start: When the period set start.
:param end: When the period end starts.
:param period: The duration of the period.
"""
period_start = start
increment = datetime.timedelta(seconds=period)
for i in moves.xrange(int(math.ceil(timeutils.delta_seconds(start, end) / float(period)))):
next_start = period_start + increment
yield (period_start, next_start)
period_start = next_start
def _stats_result_to_model(result, period, period_start,
period_end, groupby, aggregate):
stats_args = Connection._stats_result_aggregates(result, aggregate)
stats_args['unit'] = result.unit
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
stats_args['duration'] = duration
stats_args['duration_start'] = result.tsmin
stats_args['duration_end'] = result.tsmax
stats_args['period'] = period
stats_args['period_start'] = period_start
stats_args['period_end'] = period_end
stats_args['groupby'] = (dict(
(g, getattr(result, g)) for g in groupby) if groupby else None)
return api_models.Statistics(**stats_args)
def _stats_result_to_model(result, period, period_start, period_end,
groupby, aggregate):
stats_args = Connection._stats_result_aggregates(result, aggregate)
stats_args['unit'] = result.unit
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
stats_args['duration'] = duration
stats_args['duration_start'] = result.tsmin
stats_args['duration_end'] = result.tsmax
stats_args['period'] = period
stats_args['period_start'] = period_start
stats_args['period_end'] = period_end
stats_args['groupby'] = (dict(
(g, getattr(result, g)) for g in groupby) if groupby else None)
return api_models.Statistics(**stats_args)
def _stats_result_to_model(result, period, period_start, period_end):
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
return api_models.Statistics(
count=int(result.count),
min=result.min,
max=result.max,
avg=result.avg,
sum=result.sum,
duration_start=result.tsmin,
duration_end=result.tsmax,
duration=duration,
period=period,
period_start=period_start,
period_end=period_end,
)
def _stats_result_to_model(result, period, period_start, period_end):
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
return api_models.Statistics(
count=int(result.count),
min=result.min,
max=result.max,
avg=result.avg,
sum=result.sum,
duration_start=result.tsmin,
duration_end=result.tsmax,
duration=duration,
period=period,
period_start=period_start,
period_end=period_end,
)
def iter_period(start, end, period):
"""Split a time from start to end in periods of a number of seconds. This
function yield the (start, end) time for each period composing the time
passed as argument.
:param start: When the period set start.
:param end: When the period end starts.
:param period: The duration of the period.
"""
period_start = start
increment = datetime.timedelta(seconds=period)
for i in xrange(
int(math.ceil(timeutils.delta_seconds(start, end) /
float(period)))):
next_start = period_start + increment
yield (period_start, next_start)
period_start = next_start
def test_notification_service(self, fake_publisher_cls):
fake_publisher_cls.return_value = self.publisher
self.srv.start()
notifier = messaging.get_notifier(self.transport,
"compute.vagrant-precise")
notifier.info(context.RequestContext(), 'compute.instance.create.end',
TEST_NOTICE_PAYLOAD)
start = timeutils.utcnow()
while timeutils.delta_seconds(start, timeutils.utcnow()) < 600:
if len(self.publisher.samples) >= self.expected_samples:
break
eventlet.sleep(0)
self.srv.stop()
resources = list(set(s.resource_id for s in self.publisher.samples))
self.assertEqual(self.expected_samples, len(self.publisher.samples))
self.assertEqual(["9f9d01b9-4a58-4271-9e27-398b21ab20d1"], resources)
def test_notification_service(self, fake_publisher_cls):
fake_publisher_cls.return_value = self.publisher
self.srv.start()
notifier = messaging.get_notifier(self.transport,
"compute.vagrant-precise")
notifier.info(context.RequestContext(), 'compute.instance.create.end',
TEST_NOTICE_PAYLOAD)
start = timeutils.utcnow()
while timeutils.delta_seconds(start, timeutils.utcnow()) < 600:
if len(self.publisher.samples) >= self.expected_samples:
break
eventlet.sleep(0)
self.srv.stop()
resources = list(set(s.resource_id for s in self.publisher.samples))
self.assertEqual(self.expected_samples, len(self.publisher.samples))
self.assertEqual(["9f9d01b9-4a58-4271-9e27-398b21ab20d1"], resources)
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone, e:
self.stop()
done.send(e.retvalue)
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(
_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone, e:
self.stop()
done.send(e.retvalue)
def _update_meter_stats(stat, meter):
"""Do the stats calculation on a requested time bucket in stats dict
:param stats: dict where aggregated stats are kept
:param index: time bucket index in stats
:param meter: meter record as returned from HBase
:param start_time: query start time
:param period: length of the time bucket
"""
vol = int(meter["f:counter_volume"])
ts = timeutils.parse_strtime(meter["f:timestamp"])
stat.unit = meter["f:counter_unit"]
stat.min = min(vol, stat.min or vol)
stat.max = max(vol, stat.max)
stat.sum = vol + (stat.sum or 0)
stat.count += 1
stat.avg = stat.sum / float(stat.count)
stat.duration_start = min(ts, stat.duration_start or ts)
stat.duration_end = max(ts, stat.duration_end or ts)
stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end)
def _stats_result_to_model(result, period, period_start, period_end,
groupby):
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
return api_models.Statistics(
unit=result.unit,
count=int(result.count),
min=result.min,
max=result.max,
avg=result.avg,
sum=result.sum,
duration_start=result.tsmin,
duration_end=result.tsmax,
duration=duration,
period=period,
period_start=period_start,
period_end=period_end,
groupby=(dict((g, getattr(result, g))
for g in groupby) if groupby else None))
def _stats_result_to_model(result, period, period_start,
period_end, groupby):
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
return api_models.Statistics(
unit=result.unit,
count=int(result.count),
min=result.min,
max=result.max,
avg=result.avg,
sum=result.sum,
duration_start=result.tsmin,
duration_end=result.tsmax,
duration=duration,
period=period,
period_start=period_start,
period_end=period_end,
groupby=(dict((g, getattr(result, g)) for g in groupby)
if groupby else None)
)
def _update_meter_stats(self, stat, meter):
"""Do the stats calculation on a requested time bucket in stats dict
:param stats: dict where aggregated stats are kept
:param index: time bucket index in stats
:param meter: meter record as returned from HBase
:param start_time: query start time
:param period: length of the time bucket
"""
vol = int(meter['f:counter_volume'])
ts = timeutils.parse_strtime(meter['f:timestamp'])
stat['min'] = min(vol, stat['min'] or vol)
stat['max'] = max(vol, stat['max'])
stat['sum'] = vol + (stat['sum'] or 0)
stat['count'] += 1
stat['avg'] = (stat['sum'] / float(stat['count']))
stat['duration_start'] = min(ts, stat['duration_start'] or ts)
stat['duration_end'] = max(ts, stat['duration_end'] or ts)
stat['duration'] = \
timeutils.delta_seconds(stat['duration_start'],
stat['duration_end'])
def _update_meter_stats(self, stat, meter):
"""Do the stats calculation on a requested time bucket in stats dict
:param stats: dict where aggregated stats are kept
:param index: time bucket index in stats
:param meter: meter record as returned from HBase
:param start_time: query start time
:param period: length of the time bucket
"""
vol = int(meter['f:counter_volume'])
ts = timeutils.parse_strtime(meter['f:timestamp'])
stat.min = min(vol, stat.min or vol)
stat.max = max(vol, stat.max)
stat.sum = vol + (stat.sum or 0)
stat.count += 1
stat.avg = (stat.sum / float(stat.count))
stat.duration_start = min(ts, stat.duration_start or ts)
stat.duration_end = max(ts, stat.duration_end or ts)
stat.duration = \
timeutils.delta_seconds(stat.duration_start,
stat.duration_end)
def get_interval(ignore_self, event_filter, period):
assert event_filter.start
assert event_filter.end
if (event_filter.start > end or event_filter.end < start):
return []
duration_start = max(event_filter.start, start)
duration_end = min(event_filter.end, end)
duration = timeutils.delta_seconds(duration_start, duration_end)
return [
models.Statistics(
min=0,
max=0,
avg=0,
sum=0,
count=0,
period=None,
period_start=None,
period_end=None,
duration=duration,
duration_start=duration_start,
duration_end=duration_end,
)
]
def get_meter_statistics(self, sample_filter, period=None, groupby=None,
aggregate=None):
"""Return an iterable of models.Statistics instances containing meter
statistics described by the query parameters.
The filter must have a meter value set.
.. note::
Due to HBase limitations the aggregations are implemented
in the driver itself, therefore this method will be quite slow
because of all the Thrift traffic it is going to create.
"""
if groupby:
raise NotImplementedError("Group by not implemented.")
if aggregate:
raise NotImplementedError('Selectable aggregates not implemented')
with self.conn_pool.connection() as conn:
meter_table = conn.table(self.METER_TABLE)
q, start, stop = make_sample_query_from_filter(sample_filter)
meters = map(deserialize_entry, list(meter for (ignored, meter) in
meter_table.scan(filter=q, row_start=start,
row_stop=stop)))
if sample_filter.start:
start_time = sample_filter.start
elif meters:
start_time = meters[-1][0]['timestamp']
else:
start_time = None
if sample_filter.end:
end_time = sample_filter.end
elif meters:
end_time = meters[0][0]['timestamp']
else:
end_time = None
results = []
if not period:
period = 0
period_start = start_time
period_end = end_time
# As our HBase meters are stored as newest-first, we need to iterate
# in the reverse order
for meter in meters[::-1]:
ts = meter[0]['timestamp']
if period:
offset = int(timeutils.delta_seconds(
start_time, ts) / period) * period
period_start = start_time + datetime.timedelta(0, offset)
if not results or not results[-1].period_start == \
period_start:
if period:
period_end = period_start + datetime.timedelta(
0, period)
results.append(
models.Statistics(unit='',
count=0,
min=0,
max=0,
avg=0,
sum=0,
period=period,
period_start=period_start,
period_end=period_end,
duration=None,
duration_start=None,
duration_end=None,
groupby=None)
)
self._update_meter_stats(results[-1], meter[0])
return results
def get_meter_statistics(self,
sample_filter,
period=None,
groupby=None,
aggregate=None):
"""Return an iterable of models.Statistics instances containing meter
statistics described by the query parameters.
The filter must have a meter value set.
.. note::
Due to HBase limitations the aggregations are implemented
in the driver itself, therefore this method will be quite slow
because of all the Thrift traffic it is going to create.
"""
if groupby:
raise NotImplementedError("Group by not implemented.")
if aggregate:
raise NotImplementedError('Selectable aggregates not implemented')
with self.conn_pool.connection() as conn:
meter_table = conn.table(self.METER_TABLE)
q, start, stop = make_sample_query_from_filter(sample_filter)
meters = map(
deserialize_entry,
list(meter for (ignored, meter) in meter_table.scan(
filter=q, row_start=start, row_stop=stop)))
if sample_filter.start:
start_time = sample_filter.start
elif meters:
start_time = meters[-1][0]['timestamp']
else:
start_time = None
if sample_filter.end:
end_time = sample_filter.end
elif meters:
end_time = meters[0][0]['timestamp']
else:
end_time = None
results = []
if not period:
period = 0
period_start = start_time
period_end = end_time
# As our HBase meters are stored as newest-first, we need to iterate
# in the reverse order
for meter in meters[::-1]:
ts = meter[0]['timestamp']
if period:
offset = int(
timeutils.delta_seconds(start_time, ts) / period) * period
period_start = start_time + datetime.timedelta(0, offset)
if not results or not results[-1].period_start == \
period_start:
if period:
period_end = period_start + datetime.timedelta(0, period)
results.append(
models.Statistics(unit='',
count=0,
min=0,
max=0,
avg=0,
sum=0,
period=period,
period_start=period_start,
period_end=period_end,
duration=None,
duration_start=None,
duration_end=None,
groupby=None))
self._update_meter_stats(results[-1], meter[0])
return results
def get_meter_statistics(self, sample_filter, period=None, groupby=None):
"""Return an iterable of models.Statistics instance containing meter
statistics described by the query parameters.
The filter must have a meter value set.
"""
if groupby and set(groupby) - set(["user_id", "project_id", "resource_id", "source"]):
raise NotImplementedError("Unable to group by these fields")
q = make_query_from_filter(sample_filter)
if period:
if sample_filter.start:
period_start = sample_filter.start
else:
period_start = self.db.meter.find(limit=1, sort=[("timestamp", pymongo.ASCENDING)])[0]["timestamp"]
if groupby:
sort_keys = ["counter_name"] + groupby + ["timestamp"]
else:
sort_keys = ["counter_name", "timestamp"]
sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir="asc")
meters = self.db.meter.find(q, sort=sort_instructions)
def _group_key(meter):
# the method to define a key for groupby call
key = {}
for y in sort_keys:
if y == "timestamp" and period:
key[y] = timeutils.delta_seconds(period_start, meter[y]) // period
elif y != "timestamp":
key[y] = meter[y]
return key
def _to_offset(periods):
return {
"days": (periods * period) // self.SECONDS_IN_A_DAY,
"seconds": (periods * period) % self.SECONDS_IN_A_DAY,
}
for key, grouped_meters in itertools.groupby(meters, key=_group_key):
stat = models.Statistics(None, sys.maxint, -sys.maxint, 0, 0, 0, 0, 0, 0, 0, 0, 0, None)
for meter in grouped_meters:
stat.unit = meter.get("counter_unit", "")
m_volume = meter.get("counter_volume")
if stat.min > m_volume:
stat.min = m_volume
if stat.max < m_volume:
stat.max = m_volume
stat.sum += m_volume
stat.count += 1
if stat.duration_start == 0:
stat.duration_start = meter["timestamp"]
stat.duration_end = meter["timestamp"]
if groupby and not stat.groupby:
stat.groupby = {}
for group_key in groupby:
stat.groupby[group_key] = meter[group_key]
stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end)
stat.avg = stat.sum / stat.count
if period:
stat.period = period
periods = key.get("timestamp")
stat.period_start = period_start + datetime.timedelta(**(_to_offset(periods)))
stat.period_end = period_start + datetime.timedelta(**(_to_offset(periods + 1)))
else:
stat.period_start = stat.duration_start
stat.period_end = stat.duration_end
yield stat
def get_meter_statistics(self, sample_filter, period=None):
"""Return an iterable of models.Statistics instances containing meter
statistics described by the query parameters.
The filter must have a meter value set.
.. note::
Due to HBase limitations the aggregations are implemented
in the driver itself, therefore this method will be quite slow
because of all the Thrift traffic it is going to create.
"""
q, start, stop = make_query_from_filter(sample_filter)
meters = list(meter for (ignored, meter) in
self.meter.scan(filter=q,
row_start=start,
row_stop=stop)
)
if sample_filter.start:
start_time = sample_filter.start
elif meters:
start_time = timeutils.parse_strtime(meters[-1]['f:timestamp'])
else:
start_time = None
if sample_filter.end:
end_time = sample_filter.end
elif meters:
end_time = timeutils.parse_strtime(meters[0]['f:timestamp'])
else:
end_time = None
results = []
if not period:
period = 0
period_start = start_time
period_end = end_time
# As our HBase meters are stored as newest-first, we need to iterate
# in the reverse order
for meter in meters[::-1]:
ts = timeutils.parse_strtime(meter['f:timestamp'])
if period:
offset = int(timeutils.delta_seconds(
start_time, ts) / period) * period
period_start = start_time + datetime.timedelta(0, offset)
if not len(results) or not results[-1].period_start == \
period_start:
if period:
period_end = period_start + datetime.timedelta(
0, period)
results.append(
models.Statistics(count=0,
min=0,
max=0,
avg=0,
sum=0,
period=period,
period_start=period_start,
period_end=period_end,
duration=None,
duration_start=None,
duration_end=None)
)
self._update_meter_stats(results[-1], meter)
return results
def get_meter_statistics(self, sample_filter, period=None, groupby=None,
aggregate=None):
"""Return an iterable of models.Statistics instance.
Items are containing meter statistics described by the query
parameters. The filter must have a meter value set.
"""
if (groupby and
set(groupby) - set(['user_id', 'project_id',
'resource_id', 'source'])):
raise NotImplementedError("Unable to group by these fields")
if aggregate:
raise NotImplementedError('Selectable aggregates not implemented')
q = pymongo_utils.make_query_from_filter(sample_filter)
if period:
if sample_filter.start:
period_start = sample_filter.start
else:
period_start = self.db.meter.find(
limit=1, sort=[('timestamp',
pymongo.ASCENDING)])[0]['timestamp']
if groupby:
sort_keys = ['counter_name'] + groupby + ['timestamp']
else:
sort_keys = ['counter_name', 'timestamp']
sort_instructions = self._build_sort_instructions(sort_keys=sort_keys,
sort_dir='asc')
meters = self.db.meter.find(q, sort=sort_instructions)
def _group_key(meter):
# the method to define a key for groupby call
key = {}
for y in sort_keys:
if y == 'timestamp' and period:
key[y] = (timeutils.delta_seconds(period_start,
meter[y]) // period)
elif y != 'timestamp':
key[y] = meter[y]
return key
def _to_offset(periods):
return {'days': (periods * period) // self.SECONDS_IN_A_DAY,
'seconds': (periods * period) % self.SECONDS_IN_A_DAY}
for key, grouped_meters in itertools.groupby(meters, key=_group_key):
stat = models.Statistics(unit=None,
min=sys.maxint, max=-sys.maxint,
avg=0, sum=0, count=0,
period=0, period_start=0, period_end=0,
duration=0, duration_start=0,
duration_end=0, groupby=None)
for meter in grouped_meters:
stat.unit = meter.get('counter_unit', '')
m_volume = meter.get('counter_volume')
if stat.min > m_volume:
stat.min = m_volume
if stat.max < m_volume:
stat.max = m_volume
stat.sum += m_volume
stat.count += 1
if stat.duration_start == 0:
stat.duration_start = meter['timestamp']
stat.duration_end = meter['timestamp']
if groupby and not stat.groupby:
stat.groupby = {}
for group_key in groupby:
stat.groupby[group_key] = meter[group_key]
stat.duration = timeutils.delta_seconds(stat.duration_start,
stat.duration_end)
stat.avg = stat.sum / stat.count
if period:
stat.period = period
periods = key.get('timestamp')
stat.period_start = (period_start +
datetime.
timedelta(**(_to_offset(periods))))
stat.period_end = (period_start +
datetime.
timedelta(**(_to_offset(periods + 1))))
else:
stat.period_start = stat.duration_start
stat.period_end = stat.duration_end
yield stat
def get_meter_statistics(self, sample_filter, period=None):
"""Return an iterable of models.Statistics instances containing meter
statistics described by the query parameters.
The filter must have a meter value set.
.. note::
Due to HBase limitations the aggregations are implemented
in the driver itself, therefore this method will be quite slow
because of all the Thrift traffic it is going to create.
"""
q, start, stop = make_query_from_filter(sample_filter)
meters = list(meter for (ignored, meter) in self.meter.scan(
filter=q, row_start=start, row_stop=stop))
if sample_filter.start:
start_time = sample_filter.start
elif meters:
start_time = timeutils.parse_strtime(meters[-1]['f:timestamp'])
else:
start_time = None
if sample_filter.end:
end_time = sample_filter.end
elif meters:
end_time = timeutils.parse_strtime(meters[0]['f:timestamp'])
else:
end_time = None
results = []
if not period:
period = 0
period_start = start_time
period_end = end_time
# As our HBase meters are stored as newest-first, we need to iterate
# in the reverse order
for meter in meters[::-1]:
ts = timeutils.parse_strtime(meter['f:timestamp'])
if period:
offset = int(
timeutils.delta_seconds(start_time, ts) / period) * period
period_start = start_time + datetime.timedelta(0, offset)
if not len(results) or not results[-1].period_start == \
period_start:
if period:
period_end = period_start + datetime.timedelta(0, period)
results.append(
models.Statistics(count=0,
min=0,
max=0,
avg=0,
sum=0,
period=period,
period_start=period_start,
period_end=period_end,
duration=None,
duration_start=None,
duration_end=None))
self._update_meter_stats(results[-1], meter)
return results
