def _is_master(self, interval): """Determine if the current partition is the master.""" now = timeutils.utcnow() if timeutils.delta_seconds(self.start, now) < interval * 2: LOG.debug(_('%s still warming up') % self.this) return False is_master = True for partition, last_heard in self.reports.items(): delta = timeutils.delta_seconds(last_heard, now) LOG.debug( _('last heard from %(report)s %(delta)s seconds ago') % dict(report=partition, delta=delta)) if delta > interval * 2: del self.reports[partition] self._record_oldest(partition, stale=True) LOG.debug( _('%(this)s detects stale evaluator: %(stale)s') % dict(this=self.this, stale=partition)) self.presence_changed = True elif partition < self.this: is_master = False LOG.info( _('%(this)s sees older potential master: %(older)s') % dict(this=self.this, older=partition)) LOG.info( _('%(this)s is master?: %(is_master)s') % dict(this=self.this, is_master=is_master)) return is_master
def get_meter_statistics(self, event_filter, period=None): """Return a dictionary containing meter statistics. described by the query parameters. The filter must have a meter value set. { 'min': 'max': 'avg': 'sum': 'count': 'period': 'period_start': 'period_end': 'duration': 'duration_start': 'duration_end': } """ res = self._make_stats_query(event_filter).all()[0] if not period: return [self._stats_result_to_dict(res, 0, res.tsmin, res.tsmax)] query = self._make_stats_query(event_filter) # HACK(jd) This is an awful method to compute stats by period, but # since we're trying to be SQL agnostic we have to write portable # code, so here it is, admire! We're going to do one request to get # stats by period. We would like to use GROUP BY, but there's no # portable way to manipulate timestamp in SQL, so we can't. results = [] for i in range( int( math.ceil( timeutils.delta_seconds(event_filter.start or res.tsmin, event_filter.end or res.tsmax) / float(period) ) ) ): period_start = event_filter.start + datetime.timedelta(seconds=i * period) period_end = period_start + datetime.timedelta(seconds=period) q = query.filter(Meter.timestamp >= period_start) q = q.filter(Meter.timestamp < period_end) results.append( self._stats_result_to_dict( result=q.all()[0], period=int(timeutils.delta_seconds(period_start, period_end)), period_start=period_start, period_end=period_end, ) ) return results
def _stats_result_to_dict(result, period_start, period_end): return {'count': result.count, 'min': result.min, 'max': result.max, 'avg': result.avg, 'sum': result.sum, 'duration_start': result.tsmin, 'duration_end': result.tsmax, 'duration': timeutils.delta_seconds(result.tsmin, result.tsmax), 'period': timeutils.delta_seconds(period_start, period_end), 'period_start': period_start, 'period_end': period_end}
def get_interval(event_filter, period, groupby, aggregate): self.assertIsNotNone(event_filter.start) self.assertIsNotNone(event_filter.end) if event_filter.start > end or event_filter.end < start: return [] duration_start = max(event_filter.start, start) duration_end = min(event_filter.end, end) duration = timeutils.delta_seconds(duration_start, duration_end) return [ models.Statistics( unit='', min=0, max=0, avg=0, sum=0, count=0, period=None, period_start=None, period_end=None, duration=duration, duration_start=duration_start, duration_end=duration_end, groupby=None, ) ]
def get_meter_statistics(self, event_filter, period=None): """Return an iterable of api_models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. """ if not period or not event_filter.start or not event_filter.end: res = self._make_stats_query(event_filter).all()[0] if not period: yield self._stats_result_to_model(res, 0, res.tsmin, res.tsmax) return query = self._make_stats_query(event_filter) # HACK(jd) This is an awful method to compute stats by period, but # since we're trying to be SQL agnostic we have to write portable # code, so here it is, admire! We're going to do one request to get # stats by period. We would like to use GROUP BY, but there's no # portable way to manipulate timestamp in SQL, so we can't. for period_start, period_end in base.iter_period( event_filter.start or res.tsmin, event_filter.end or res.tsmax, period): q = query.filter(Meter.timestamp >= period_start) q = q.filter(Meter.timestamp < period_end) r = q.all()[0] # Don't return results that didn't have any event if r.count: yield self._stats_result_to_model( result=r, period=int( timeutils.delta_seconds(period_start, period_end)), period_start=period_start, period_end=period_end, )
def handle_sample(self, context, counter, source): """Handle a sample, converting if necessary.""" LOG.debug('handling counter %s', (counter,)) key = counter.name + counter.resource_id prev = self.cache.get(key) timestamp = timeutils.parse_isotime(counter.timestamp) self.cache[key] = (counter.volume, timestamp) if prev: prev_volume = prev[0] prev_timestamp = prev[1] time_delta = timeutils.delta_seconds(prev_timestamp, timestamp) # we only allow negative deltas for noncumulative counters, whereas # for cumulative we assume that a reset has occurred in the interim # so that the current volume gives a lower bound on growth volume_delta = (counter.volume - prev_volume if (prev_volume <= counter.volume or counter.type != ceilocounter.TYPE_CUMULATIVE) else counter.volume) rate_of_change = ((1.0 * volume_delta / time_delta) if time_delta else 0.0) transformed = self._convert(counter, rate_of_change) LOG.debug(_('converted to: %s') % (transformed,)) counter = self._keep(counter, transformed) elif self.replace: LOG.warn(_('dropping counter with no predecessor: %s') % counter) counter = None return counter
def get_interval(event_filter, period, groupby, aggregate): assert event_filter.start assert event_filter.end if (event_filter.start > end or event_filter.end < start): return [] duration_start = max(event_filter.start, start) duration_end = min(event_filter.end, end) duration = timeutils.delta_seconds(duration_start, duration_end) return [ models.Statistics( unit='', min=0, max=0, avg=0, sum=0, count=0, period=None, period_start=None, period_end=None, duration=duration, duration_start=duration_start, duration_end=duration_end, groupby=None, ) ]
def _update_duration(self, start_timestamp, end_timestamp): # "Clamp" the timestamps we return to the original time # range, excluding the offset. if (start_timestamp and self.duration_start and self.duration_start < start_timestamp): self.duration_start = start_timestamp LOG.debug('clamping min timestamp to range') if (end_timestamp and self.duration_end and self.duration_end > end_timestamp): self.duration_end = end_timestamp LOG.debug('clamping max timestamp to range') # If we got valid timestamps back, compute a duration in minutes. # # If the min > max after clamping then we know the # timestamps on the samples fell outside of the time # range we care about for the query, so treat them as # "invalid." # # If the timestamps are invalid, return None as a # sentinal indicating that there is something "funny" # about the range. if (self.duration_start and self.duration_end and self.duration_start <= self.duration_end): self.duration = timeutils.delta_seconds(self.duration_start, self.duration_end) else: self.duration_start = self.duration_end = self.duration = None
def _update_duration(self, start_timestamp, end_timestamp): # "Clamp" the timestamps we return to the original time # range, excluding the offset. if (start_timestamp and self.duration_start and self.duration_start < start_timestamp): self.duration_start = start_timestamp LOG.debug('clamping min timestamp to range') if (end_timestamp and self.duration_end and self.duration_end > end_timestamp): self.duration_end = end_timestamp LOG.debug('clamping max timestamp to range') # If we got valid timestamps back, compute a duration in minutes. # # If the min > max after clamping then we know the # timestamps on the samples fell outside of the time # range we care about for the query, so treat them as # "invalid." # # If the timestamps are invalid, return None as a # sentinal indicating that there is something "funny" # about the range. if (self.duration_start and self.duration_end and self.duration_start <= self.duration_end): self.duration = timeutils.delta_seconds(self.duration_start, self.duration_end) else: self.duration_start = self.duration_end = self.duration = None
def handle_sample(self, context, s): """Handle a sample, converting if necessary.""" LOG.debug(_('handling sample %s'), (s,)) key = s.name + s.resource_id prev = self.cache.get(key) timestamp = timeutils.parse_isotime(s.timestamp) self.cache[key] = (s.volume, timestamp) if prev: prev_volume = prev[0] prev_timestamp = prev[1] time_delta = timeutils.delta_seconds(prev_timestamp, timestamp) # we only allow negative deltas for noncumulative samples, whereas # for cumulative we assume that a reset has occurred in the interim # so that the current volume gives a lower bound on growth volume_delta = (s.volume - prev_volume if (prev_volume <= s.volume or s.type != sample.TYPE_CUMULATIVE) else s.volume) rate_of_change = ((1.0 * volume_delta / time_delta) if time_delta else 0.0) s = self._convert(s, rate_of_change) LOG.debug(_('converted to: %s'), (s,)) else: LOG.warn(_('dropping sample with no predecessor: %s'), (s,)) s = None return s
def get_interval(ignore_self, event_filter, period, groupby): assert event_filter.start assert event_filter.end if (event_filter.start > end or event_filter.end < start): return [] duration_start = max(event_filter.start, start) duration_end = min(event_filter.end, end) duration = timeutils.delta_seconds(duration_start, duration_end) return [ models.Statistics( unit='', min=0, max=0, avg=0, sum=0, count=0, period=None, period_start=None, period_end=None, duration=duration, duration_start=duration_start, duration_end=duration_end, groupby=None, ) ]
def get_meter_statistics(self, sample_filter, period=None): """Return an iterable of api_models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. """ if not period or not sample_filter.start or not sample_filter.end: res = self._make_stats_query(sample_filter).all()[0] if not period: yield self._stats_result_to_model(res, 0, res.tsmin, res.tsmax) return query = self._make_stats_query(sample_filter) # HACK(jd) This is an awful method to compute stats by period, but # since we're trying to be SQL agnostic we have to write portable # code, so here it is, admire! We're going to do one request to get # stats by period. We would like to use GROUP BY, but there's no # portable way to manipulate timestamp in SQL, so we can't. for period_start, period_end in base.iter_period( sample_filter.start or res.tsmin, sample_filter.end or res.tsmax, period ): q = query.filter(Meter.timestamp >= period_start) q = q.filter(Meter.timestamp < period_end) r = q.all()[0] # Don't return results that didn't have any data. if r.count: yield self._stats_result_to_model( result=r, period=int(timeutils.delta_seconds(period_start, period_end)), period_start=period_start, period_end=period_end, )
def compute_duration_by_resource(resource, meter): """Return the earliest timestamp, last timestamp, and duration for the resource and meter. :param resource: The ID of the resource. :param meter: The name of the meter. :param start_timestamp: ISO-formatted string of the earliest timestamp to return. :param end_timestamp: ISO-formatted string of the latest timestamp to return. :param search_offset: Number of minutes before and after start and end timestamps to query. """ q_ts = _get_query_timestamps(flask.request.args) start_timestamp = q_ts['start_timestamp'] end_timestamp = q_ts['end_timestamp'] # Query the database for the interval of timestamps # within the desired range. f = storage.SampleFilter( meter=meter, project=acl.get_limited_to_project(flask.request.headers), resource=resource, start=q_ts['query_start'], end=q_ts['query_end'], ) stats = flask.request.storage_conn.get_meter_statistics(f) min_ts, max_ts = stats.duration_start, stats.duration_end # "Clamp" the timestamps we return to the original time # range, excluding the offset. LOG.debug('start_timestamp %s, end_timestamp %s, min_ts %s, max_ts %s', start_timestamp, end_timestamp, min_ts, max_ts) if start_timestamp and min_ts and min_ts < start_timestamp: min_ts = start_timestamp LOG.debug('clamping min timestamp to range') if end_timestamp and max_ts and max_ts > end_timestamp: max_ts = end_timestamp LOG.debug('clamping max timestamp to range') # If we got valid timestamps back, compute a duration in minutes. # # If the min > max after clamping then we know the # timestamps on the samples fell outside of the time # range we care about for the query, so treat them as # "invalid." # # If the timestamps are invalid, return None as a # sentinal indicating that there is something "funny" # about the range. if min_ts and max_ts and (min_ts <= max_ts): duration = timeutils.delta_seconds(min_ts, max_ts) else: min_ts = max_ts = duration = None return flask.jsonify( start_timestamp=min_ts, end_timestamp=max_ts, duration=duration, )
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_LW('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_LE('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def handle_sample(self, context, counter): """Handle a sample, converting if necessary.""" LOG.debug('handling counter %s', (counter, )) key = counter.name + counter.resource_id prev = self.cache.get(key) timestamp = timeutils.parse_isotime(counter.timestamp) self.cache[key] = (counter.volume, timestamp) if prev: prev_volume = prev[0] prev_timestamp = prev[1] time_delta = timeutils.delta_seconds(prev_timestamp, timestamp) # we only allow negative deltas for noncumulative counters, whereas # for cumulative we assume that a reset has occurred in the interim # so that the current volume gives a lower bound on growth volume_delta = (counter.volume - prev_volume if (prev_volume <= counter.volume or counter.type != sample.TYPE_CUMULATIVE) else counter.volume) rate_of_change = ((1.0 * volume_delta / time_delta) if time_delta else 0.0) counter = self._convert(counter, rate_of_change) LOG.debug(_('converted to: %s') % (counter, )) else: LOG.warn( _('dropping counter with no predecessor: %s') % (counter, )) counter = None return counter
def compute_duration_by_resource(resource, meter): """Return the earliest timestamp, last timestamp, and duration for the resource and meter. :param resource: The ID of the resource. :param meter: The name of the meter. :param start_timestamp: ISO-formatted string of the earliest timestamp to return. :param end_timestamp: ISO-formatted string of the latest timestamp to return. :param search_offset: Number of minutes before and after start and end timestamps to query. """ q_ts = _get_query_timestamps(flask.request.args) start_timestamp = q_ts['start_timestamp'] end_timestamp = q_ts['end_timestamp'] # Query the database for the interval of timestamps # within the desired range. f = storage.EventFilter( meter=meter, project=acl.get_limited_to_project(flask.request.headers), resource=resource, start=q_ts['query_start'], end=q_ts['query_end'], ) stats = flask.request.storage_conn.get_meter_statistics(f) min_ts, max_ts = stats.duration_start, stats.duration_end # "Clamp" the timestamps we return to the original time # range, excluding the offset. LOG.debug('start_timestamp %s, end_timestamp %s, min_ts %s, max_ts %s', start_timestamp, end_timestamp, min_ts, max_ts) if start_timestamp and min_ts and min_ts < start_timestamp: min_ts = start_timestamp LOG.debug('clamping min timestamp to range') if end_timestamp and max_ts and max_ts > end_timestamp: max_ts = end_timestamp LOG.debug('clamping max timestamp to range') # If we got valid timestamps back, compute a duration in minutes. # # If the min > max after clamping then we know the # timestamps on the samples fell outside of the time # range we care about for the query, so treat them as # "invalid." # # If the timestamps are invalid, return None as a # sentinal indicating that there is something "funny" # about the range. if min_ts and max_ts and (min_ts <= max_ts): duration = timeutils.delta_seconds(min_ts, max_ts) else: min_ts = max_ts = duration = None return flask.jsonify(start_timestamp=min_ts, end_timestamp=max_ts, duration=duration, )
def assertTimestampEqual(self, first, second, msg=None): """Checks that two timestamps are equals. This relies on assertAlmostEqual to avoid rounding problem, and only checks up the first microsecond values. """ return self.assertAlmostEqual(timeutils.delta_seconds(first, second), 0.0, places=5)
def _group_key(meter): # the method to define a key for groupby call key = {} for y in sort_keys: if y == "timestamp" and period: key[y] = timeutils.delta_seconds(period_start, meter[y]) // period elif y != "timestamp": key[y] = meter[y] return key
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of api_models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. """ if groupby: for group in groupby: if group not in ['user_id', 'project_id', 'resource_id']: raise NotImplementedError('Unable to group by ' 'these fields') if not period: for res in self._make_stats_query(sample_filter, groupby, aggregate): if res.count: yield self._stats_result_to_model(res, 0, res.tsmin, res.tsmax, groupby, aggregate) return if not sample_filter.start or not sample_filter.end: res = self._make_stats_query(sample_filter, None, aggregate).first() if not res: # NOTE(liusheng):The 'res' may be NoneType, because no # sample has found with sample filter(s). return query = self._make_stats_query(sample_filter, groupby, aggregate) # HACK(jd) This is an awful method to compute stats by period, but # since we're trying to be SQL agnostic we have to write portable # code, so here it is, admire! We're going to do one request to get # stats by period. We would like to use GROUP BY, but there's no # portable way to manipulate timestamp in SQL, so we can't. for period_start, period_end in base.iter_period( sample_filter.start or res.tsmin, sample_filter.end or res.tsmax, period): q = query.filter(models.Sample.timestamp >= period_start) q = q.filter(models.Sample.timestamp < period_end) for r in q.all(): if r.count: yield self._stats_result_to_model( result=r, period=int(timeutils.delta_seconds(period_start, period_end)), period_start=period_start, period_end=period_end, groupby=groupby, aggregate=aggregate )
def _group_key(meter): # the method to define a key for groupby call key = {} for y in sort_keys: if y == 'timestamp' and period: key[y] = (timeutils.delta_seconds(period_start, meter[y]) // period) elif y != 'timestamp': key[y] = meter[y] return key
def assertTimestampEqual(self, first, second, msg=None): """Checks that two timestamps are equals. This relies on assertAlmostEqual to avoid rounding problem, and only checks up the first microsecond values. """ return self.assertAlmostEqual(timeutils.delta_seconds(first, second), 0.0, places=5)
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of api_models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. """ if groupby: for group in groupby: if group not in ['user_id', 'project_id', 'resource_id']: raise NotImplementedError('Unable to group by ' 'these fields') if not period: for res in self._make_stats_query(sample_filter, groupby, aggregate): if res.count: yield self._stats_result_to_model(res, 0, res.tsmin, res.tsmax, groupby, aggregate) return if not sample_filter.start or not sample_filter.end: res = self._make_stats_query(sample_filter, None, aggregate).first() if not res: # NOTE(liusheng):The 'res' may be NoneType, because no # sample has found with sample filter(s). return query = self._make_stats_query(sample_filter, groupby, aggregate) # HACK(jd) This is an awful method to compute stats by period, but # since we're trying to be SQL agnostic we have to write portable # code, so here it is, admire! We're going to do one request to get # stats by period. We would like to use GROUP BY, but there's no # portable way to manipulate timestamp in SQL, so we can't. for period_start, period_end in base.iter_period( sample_filter.start or res.tsmin, sample_filter.end or res.tsmax, period): q = query.filter(models.Sample.timestamp >= period_start) q = q.filter(models.Sample.timestamp < period_end) for r in q.all(): if r.count: yield self._stats_result_to_model( result=r, period=int( timeutils.delta_seconds(period_start, period_end)), period_start=period_start, period_end=period_end, groupby=groupby, aggregate=aggregate)
def _is_master(self, interval): """Determine if the current partition is the master.""" now = timeutils.utcnow() if timeutils.delta_seconds(self.start, now) < interval * 2: LOG.debug(_("%s still warming up") % self.this) return False is_master = True for partition, last_heard in self.reports.items(): delta = timeutils.delta_seconds(last_heard, now) LOG.debug(_("last heard from %(report)s %(delta)s seconds ago") % dict(report=partition, delta=delta)) if delta > interval * 2: del self.reports[partition] self._record_oldest(partition, stale=True) LOG.debug(_("%(this)s detects stale evaluator: %(stale)s") % dict(this=self.this, stale=partition)) self.presence_changed = True elif partition < self.this: is_master = False LOG.info(_("%(this)s sees older potential master: %(older)s") % dict(this=self.this, older=partition)) LOG.info(_("%(this)s is master?: %(is_master)s") % dict(this=self.this, is_master=is_master)) return is_master
def _record_poll_time(self): """Method records current time as the poll time. :return: time in seconds since the last poll time was recorded """ current_time = timeutils.utcnow() duration = None if hasattr(self, "_last_poll_time"): duration = timeutils.delta_seconds(self._last_poll_time, current_time) self._last_poll_time = current_time return duration
def _record_poll_time(self): """Method records current time as the poll time. :return: time in seconds since the last poll time was recorded """ current_time = timeutils.utcnow() duration = None if hasattr(self, '_last_poll_time'): duration = timeutils.delta_seconds(self._last_poll_time, current_time) self._last_poll_time = current_time return duration
def _stats_result_to_dict(result, period, period_start, period_end): return { "count": result.count, "min": result.min, "max": result.max, "avg": result.avg, "sum": result.sum, "duration_start": result.tsmin, "duration_end": result.tsmax, "duration": timeutils.delta_seconds(result.tsmin, result.tsmax), "period": period, "period_start": period_start, "period_end": period_end, }
def test_extract_when(self): now = timeutils.utcnow() modified = now + datetime.timedelta(minutes=1) timeutils.set_time_override(now) body = {"timestamp": str(modified)} when = service.CollectorService._extract_when(body) self.assertAlmostEqual(timeutils.delta_seconds(modified, when), 0.0, places=5) body = {"_context_timestamp": str(modified)} when = service.CollectorService._extract_when(body) self.assertAlmostEqual(timeutils.delta_seconds(modified, when), 0.0, places=5) then = now + datetime.timedelta(hours=1) body = {"timestamp": str(modified), "_context_timestamp": str(then)} when = service.CollectorService._extract_when(body) self.assertAlmostEqual(timeutils.delta_seconds(modified, when), 0.0, places=5) when = service.CollectorService._extract_when({}) self.assertAlmostEqual(timeutils.delta_seconds(now, when), 0.0, places=5)
def _stats_result_to_dict(result, period, period_start, period_end): return {'count': int(result.count), 'min': result.min, 'max': result.max, 'avg': result.avg, 'sum': result.sum, 'duration_start': result.tsmin, 'duration_end': result.tsmax, 'duration': (timeutils.delta_seconds(result.tsmin, result.tsmax) if result.tsmin and result.tsmax else None), 'period': period, 'period_start': period_start, 'period_end': period_end}
def iter_period(start, end, period): """Split a time from start to end in periods of a number of seconds. This function yield the (start, end) time for each period composing the time passed as argument. :param start: When the period set start. :param end: When the period end starts. :param period: The duration of the period. """ period_start = start increment = datetime.timedelta(seconds=period) for i in moves.xrange(int(math.ceil(timeutils.delta_seconds(start, end) / float(period)))): next_start = period_start + increment yield (period_start, next_start) period_start = next_start
def _stats_result_to_model(result, period, period_start, period_end, groupby, aggregate): stats_args = Connection._stats_result_aggregates(result, aggregate) stats_args['unit'] = result.unit duration = (timeutils.delta_seconds(result.tsmin, result.tsmax) if result.tsmin is not None and result.tsmax is not None else None) stats_args['duration'] = duration stats_args['duration_start'] = result.tsmin stats_args['duration_end'] = result.tsmax stats_args['period'] = period stats_args['period_start'] = period_start stats_args['period_end'] = period_end stats_args['groupby'] = (dict( (g, getattr(result, g)) for g in groupby) if groupby else None) return api_models.Statistics(**stats_args)
def _stats_result_to_model(result, period, period_start, period_end, groupby, aggregate): stats_args = Connection._stats_result_aggregates(result, aggregate) stats_args['unit'] = result.unit duration = (timeutils.delta_seconds(result.tsmin, result.tsmax) if result.tsmin is not None and result.tsmax is not None else None) stats_args['duration'] = duration stats_args['duration_start'] = result.tsmin stats_args['duration_end'] = result.tsmax stats_args['period'] = period stats_args['period_start'] = period_start stats_args['period_end'] = period_end stats_args['groupby'] = (dict( (g, getattr(result, g)) for g in groupby) if groupby else None) return api_models.Statistics(**stats_args)
def _stats_result_to_model(result, period, period_start, period_end): duration = (timeutils.delta_seconds(result.tsmin, result.tsmax) if result.tsmin is not None and result.tsmax is not None else None) return api_models.Statistics( count=int(result.count), min=result.min, max=result.max, avg=result.avg, sum=result.sum, duration_start=result.tsmin, duration_end=result.tsmax, duration=duration, period=period, period_start=period_start, period_end=period_end, )
def _stats_result_to_model(result, period, period_start, period_end): duration = (timeutils.delta_seconds(result.tsmin, result.tsmax) if result.tsmin is not None and result.tsmax is not None else None) return api_models.Statistics( count=int(result.count), min=result.min, max=result.max, avg=result.avg, sum=result.sum, duration_start=result.tsmin, duration_end=result.tsmax, duration=duration, period=period, period_start=period_start, period_end=period_end, )
def iter_period(start, end, period): """Split a time from start to end in periods of a number of seconds. This function yield the (start, end) time for each period composing the time passed as argument. :param start: When the period set start. :param end: When the period end starts. :param period: The duration of the period. """ period_start = start increment = datetime.timedelta(seconds=period) for i in xrange( int(math.ceil(timeutils.delta_seconds(start, end) / float(period)))): next_start = period_start + increment yield (period_start, next_start) period_start = next_start
def test_notification_service(self, fake_publisher_cls): fake_publisher_cls.return_value = self.publisher self.srv.start() notifier = messaging.get_notifier(self.transport, "compute.vagrant-precise") notifier.info(context.RequestContext(), 'compute.instance.create.end', TEST_NOTICE_PAYLOAD) start = timeutils.utcnow() while timeutils.delta_seconds(start, timeutils.utcnow()) < 600: if len(self.publisher.samples) >= self.expected_samples: break eventlet.sleep(0) self.srv.stop() resources = list(set(s.resource_id for s in self.publisher.samples)) self.assertEqual(self.expected_samples, len(self.publisher.samples)) self.assertEqual(["9f9d01b9-4a58-4271-9e27-398b21ab20d1"], resources)
def test_notification_service(self, fake_publisher_cls): fake_publisher_cls.return_value = self.publisher self.srv.start() notifier = messaging.get_notifier(self.transport, "compute.vagrant-precise") notifier.info(context.RequestContext(), 'compute.instance.create.end', TEST_NOTICE_PAYLOAD) start = timeutils.utcnow() while timeutils.delta_seconds(start, timeutils.utcnow()) < 600: if len(self.publisher.samples) >= self.expected_samples: break eventlet.sleep(0) self.srv.stop() resources = list(set(s.resource_id for s in self.publisher.samples)) self.assertEqual(self.expected_samples, len(self.publisher.samples)) self.assertEqual(["9f9d01b9-4a58-4271-9e27-398b21ab20d1"], resources)
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone, e: self.stop() done.send(e.retvalue)
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn( _('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone, e: self.stop() done.send(e.retvalue)
def _update_meter_stats(stat, meter): """Do the stats calculation on a requested time bucket in stats dict :param stats: dict where aggregated stats are kept :param index: time bucket index in stats :param meter: meter record as returned from HBase :param start_time: query start time :param period: length of the time bucket """ vol = int(meter["f:counter_volume"]) ts = timeutils.parse_strtime(meter["f:timestamp"]) stat.unit = meter["f:counter_unit"] stat.min = min(vol, stat.min or vol) stat.max = max(vol, stat.max) stat.sum = vol + (stat.sum or 0) stat.count += 1 stat.avg = stat.sum / float(stat.count) stat.duration_start = min(ts, stat.duration_start or ts) stat.duration_end = max(ts, stat.duration_end or ts) stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end)
def _stats_result_to_model(result, period, period_start, period_end, groupby): duration = (timeutils.delta_seconds(result.tsmin, result.tsmax) if result.tsmin is not None and result.tsmax is not None else None) return api_models.Statistics( unit=result.unit, count=int(result.count), min=result.min, max=result.max, avg=result.avg, sum=result.sum, duration_start=result.tsmin, duration_end=result.tsmax, duration=duration, period=period, period_start=period_start, period_end=period_end, groupby=(dict((g, getattr(result, g)) for g in groupby) if groupby else None))
def _stats_result_to_model(result, period, period_start, period_end, groupby): duration = (timeutils.delta_seconds(result.tsmin, result.tsmax) if result.tsmin is not None and result.tsmax is not None else None) return api_models.Statistics( unit=result.unit, count=int(result.count), min=result.min, max=result.max, avg=result.avg, sum=result.sum, duration_start=result.tsmin, duration_end=result.tsmax, duration=duration, period=period, period_start=period_start, period_end=period_end, groupby=(dict((g, getattr(result, g)) for g in groupby) if groupby else None) )
def _update_meter_stats(self, stat, meter): """Do the stats calculation on a requested time bucket in stats dict :param stats: dict where aggregated stats are kept :param index: time bucket index in stats :param meter: meter record as returned from HBase :param start_time: query start time :param period: length of the time bucket """ vol = int(meter['f:counter_volume']) ts = timeutils.parse_strtime(meter['f:timestamp']) stat['min'] = min(vol, stat['min'] or vol) stat['max'] = max(vol, stat['max']) stat['sum'] = vol + (stat['sum'] or 0) stat['count'] += 1 stat['avg'] = (stat['sum'] / float(stat['count'])) stat['duration_start'] = min(ts, stat['duration_start'] or ts) stat['duration_end'] = max(ts, stat['duration_end'] or ts) stat['duration'] = \ timeutils.delta_seconds(stat['duration_start'], stat['duration_end'])
def _update_meter_stats(self, stat, meter): """Do the stats calculation on a requested time bucket in stats dict :param stats: dict where aggregated stats are kept :param index: time bucket index in stats :param meter: meter record as returned from HBase :param start_time: query start time :param period: length of the time bucket """ vol = int(meter['f:counter_volume']) ts = timeutils.parse_strtime(meter['f:timestamp']) stat.min = min(vol, stat.min or vol) stat.max = max(vol, stat.max) stat.sum = vol + (stat.sum or 0) stat.count += 1 stat.avg = (stat.sum / float(stat.count)) stat.duration_start = min(ts, stat.duration_start or ts) stat.duration_end = max(ts, stat.duration_end or ts) stat.duration = \ timeutils.delta_seconds(stat.duration_start, stat.duration_end)
def get_interval(ignore_self, event_filter, period): assert event_filter.start assert event_filter.end if (event_filter.start > end or event_filter.end < start): return [] duration_start = max(event_filter.start, start) duration_end = min(event_filter.end, end) duration = timeutils.delta_seconds(duration_start, duration_end) return [ models.Statistics( min=0, max=0, avg=0, sum=0, count=0, period=None, period_start=None, period_end=None, duration=duration, duration_start=duration_start, duration_end=duration_end, ) ]
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. .. note:: Due to HBase limitations the aggregations are implemented in the driver itself, therefore this method will be quite slow because of all the Thrift traffic it is going to create. """ if groupby: raise NotImplementedError("Group by not implemented.") if aggregate: raise NotImplementedError('Selectable aggregates not implemented') with self.conn_pool.connection() as conn: meter_table = conn.table(self.METER_TABLE) q, start, stop = make_sample_query_from_filter(sample_filter) meters = map(deserialize_entry, list(meter for (ignored, meter) in meter_table.scan(filter=q, row_start=start, row_stop=stop))) if sample_filter.start: start_time = sample_filter.start elif meters: start_time = meters[-1][0]['timestamp'] else: start_time = None if sample_filter.end: end_time = sample_filter.end elif meters: end_time = meters[0][0]['timestamp'] else: end_time = None results = [] if not period: period = 0 period_start = start_time period_end = end_time # As our HBase meters are stored as newest-first, we need to iterate # in the reverse order for meter in meters[::-1]: ts = meter[0]['timestamp'] if period: offset = int(timeutils.delta_seconds( start_time, ts) / period) * period period_start = start_time + datetime.timedelta(0, offset) if not results or not results[-1].period_start == \ period_start: if period: period_end = period_start + datetime.timedelta( 0, period) results.append( models.Statistics(unit='', count=0, min=0, max=0, avg=0, sum=0, period=period, period_start=period_start, period_end=period_end, duration=None, duration_start=None, duration_end=None, groupby=None) ) self._update_meter_stats(results[-1], meter[0]) return results
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. .. note:: Due to HBase limitations the aggregations are implemented in the driver itself, therefore this method will be quite slow because of all the Thrift traffic it is going to create. """ if groupby: raise NotImplementedError("Group by not implemented.") if aggregate: raise NotImplementedError('Selectable aggregates not implemented') with self.conn_pool.connection() as conn: meter_table = conn.table(self.METER_TABLE) q, start, stop = make_sample_query_from_filter(sample_filter) meters = map( deserialize_entry, list(meter for (ignored, meter) in meter_table.scan( filter=q, row_start=start, row_stop=stop))) if sample_filter.start: start_time = sample_filter.start elif meters: start_time = meters[-1][0]['timestamp'] else: start_time = None if sample_filter.end: end_time = sample_filter.end elif meters: end_time = meters[0][0]['timestamp'] else: end_time = None results = [] if not period: period = 0 period_start = start_time period_end = end_time # As our HBase meters are stored as newest-first, we need to iterate # in the reverse order for meter in meters[::-1]: ts = meter[0]['timestamp'] if period: offset = int( timeutils.delta_seconds(start_time, ts) / period) * period period_start = start_time + datetime.timedelta(0, offset) if not results or not results[-1].period_start == \ period_start: if period: period_end = period_start + datetime.timedelta(0, period) results.append( models.Statistics(unit='', count=0, min=0, max=0, avg=0, sum=0, period=period, period_start=period_start, period_end=period_end, duration=None, duration_start=None, duration_end=None, groupby=None)) self._update_meter_stats(results[-1], meter[0]) return results
def get_meter_statistics(self, sample_filter, period=None, groupby=None): """Return an iterable of models.Statistics instance containing meter statistics described by the query parameters. The filter must have a meter value set. """ if groupby and set(groupby) - set(["user_id", "project_id", "resource_id", "source"]): raise NotImplementedError("Unable to group by these fields") q = make_query_from_filter(sample_filter) if period: if sample_filter.start: period_start = sample_filter.start else: period_start = self.db.meter.find(limit=1, sort=[("timestamp", pymongo.ASCENDING)])[0]["timestamp"] if groupby: sort_keys = ["counter_name"] + groupby + ["timestamp"] else: sort_keys = ["counter_name", "timestamp"] sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir="asc") meters = self.db.meter.find(q, sort=sort_instructions) def _group_key(meter): # the method to define a key for groupby call key = {} for y in sort_keys: if y == "timestamp" and period: key[y] = timeutils.delta_seconds(period_start, meter[y]) // period elif y != "timestamp": key[y] = meter[y] return key def _to_offset(periods): return { "days": (periods * period) // self.SECONDS_IN_A_DAY, "seconds": (periods * period) % self.SECONDS_IN_A_DAY, } for key, grouped_meters in itertools.groupby(meters, key=_group_key): stat = models.Statistics(None, sys.maxint, -sys.maxint, 0, 0, 0, 0, 0, 0, 0, 0, 0, None) for meter in grouped_meters: stat.unit = meter.get("counter_unit", "") m_volume = meter.get("counter_volume") if stat.min > m_volume: stat.min = m_volume if stat.max < m_volume: stat.max = m_volume stat.sum += m_volume stat.count += 1 if stat.duration_start == 0: stat.duration_start = meter["timestamp"] stat.duration_end = meter["timestamp"] if groupby and not stat.groupby: stat.groupby = {} for group_key in groupby: stat.groupby[group_key] = meter[group_key] stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end) stat.avg = stat.sum / stat.count if period: stat.period = period periods = key.get("timestamp") stat.period_start = period_start + datetime.timedelta(**(_to_offset(periods))) stat.period_end = period_start + datetime.timedelta(**(_to_offset(periods + 1))) else: stat.period_start = stat.duration_start stat.period_end = stat.duration_end yield stat
def get_meter_statistics(self, sample_filter, period=None): """Return an iterable of models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. .. note:: Due to HBase limitations the aggregations are implemented in the driver itself, therefore this method will be quite slow because of all the Thrift traffic it is going to create. """ q, start, stop = make_query_from_filter(sample_filter) meters = list(meter for (ignored, meter) in self.meter.scan(filter=q, row_start=start, row_stop=stop) ) if sample_filter.start: start_time = sample_filter.start elif meters: start_time = timeutils.parse_strtime(meters[-1]['f:timestamp']) else: start_time = None if sample_filter.end: end_time = sample_filter.end elif meters: end_time = timeutils.parse_strtime(meters[0]['f:timestamp']) else: end_time = None results = [] if not period: period = 0 period_start = start_time period_end = end_time # As our HBase meters are stored as newest-first, we need to iterate # in the reverse order for meter in meters[::-1]: ts = timeutils.parse_strtime(meter['f:timestamp']) if period: offset = int(timeutils.delta_seconds( start_time, ts) / period) * period period_start = start_time + datetime.timedelta(0, offset) if not len(results) or not results[-1].period_start == \ period_start: if period: period_end = period_start + datetime.timedelta( 0, period) results.append( models.Statistics(count=0, min=0, max=0, avg=0, sum=0, period=period, period_start=period_start, period_end=period_end, duration=None, duration_start=None, duration_end=None) ) self._update_meter_stats(results[-1], meter) return results
def get_meter_statistics(self, sample_filter, period=None, groupby=None, aggregate=None): """Return an iterable of models.Statistics instance. Items are containing meter statistics described by the query parameters. The filter must have a meter value set. """ if (groupby and set(groupby) - set(['user_id', 'project_id', 'resource_id', 'source'])): raise NotImplementedError("Unable to group by these fields") if aggregate: raise NotImplementedError('Selectable aggregates not implemented') q = pymongo_utils.make_query_from_filter(sample_filter) if period: if sample_filter.start: period_start = sample_filter.start else: period_start = self.db.meter.find( limit=1, sort=[('timestamp', pymongo.ASCENDING)])[0]['timestamp'] if groupby: sort_keys = ['counter_name'] + groupby + ['timestamp'] else: sort_keys = ['counter_name', 'timestamp'] sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir='asc') meters = self.db.meter.find(q, sort=sort_instructions) def _group_key(meter): # the method to define a key for groupby call key = {} for y in sort_keys: if y == 'timestamp' and period: key[y] = (timeutils.delta_seconds(period_start, meter[y]) // period) elif y != 'timestamp': key[y] = meter[y] return key def _to_offset(periods): return {'days': (periods * period) // self.SECONDS_IN_A_DAY, 'seconds': (periods * period) % self.SECONDS_IN_A_DAY} for key, grouped_meters in itertools.groupby(meters, key=_group_key): stat = models.Statistics(unit=None, min=sys.maxint, max=-sys.maxint, avg=0, sum=0, count=0, period=0, period_start=0, period_end=0, duration=0, duration_start=0, duration_end=0, groupby=None) for meter in grouped_meters: stat.unit = meter.get('counter_unit', '') m_volume = meter.get('counter_volume') if stat.min > m_volume: stat.min = m_volume if stat.max < m_volume: stat.max = m_volume stat.sum += m_volume stat.count += 1 if stat.duration_start == 0: stat.duration_start = meter['timestamp'] stat.duration_end = meter['timestamp'] if groupby and not stat.groupby: stat.groupby = {} for group_key in groupby: stat.groupby[group_key] = meter[group_key] stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end) stat.avg = stat.sum / stat.count if period: stat.period = period periods = key.get('timestamp') stat.period_start = (period_start + datetime. timedelta(**(_to_offset(periods)))) stat.period_end = (period_start + datetime. timedelta(**(_to_offset(periods + 1)))) else: stat.period_start = stat.duration_start stat.period_end = stat.duration_end yield stat
def get_meter_statistics(self, sample_filter, period=None): """Return an iterable of models.Statistics instances containing meter statistics described by the query parameters. The filter must have a meter value set. .. note:: Due to HBase limitations the aggregations are implemented in the driver itself, therefore this method will be quite slow because of all the Thrift traffic it is going to create. """ q, start, stop = make_query_from_filter(sample_filter) meters = list(meter for (ignored, meter) in self.meter.scan( filter=q, row_start=start, row_stop=stop)) if sample_filter.start: start_time = sample_filter.start elif meters: start_time = timeutils.parse_strtime(meters[-1]['f:timestamp']) else: start_time = None if sample_filter.end: end_time = sample_filter.end elif meters: end_time = timeutils.parse_strtime(meters[0]['f:timestamp']) else: end_time = None results = [] if not period: period = 0 period_start = start_time period_end = end_time # As our HBase meters are stored as newest-first, we need to iterate # in the reverse order for meter in meters[::-1]: ts = timeutils.parse_strtime(meter['f:timestamp']) if period: offset = int( timeutils.delta_seconds(start_time, ts) / period) * period period_start = start_time + datetime.timedelta(0, offset) if not len(results) or not results[-1].period_start == \ period_start: if period: period_end = period_start + datetime.timedelta(0, period) results.append( models.Statistics(count=0, min=0, max=0, avg=0, sum=0, period=period, period_start=period_start, period_end=period_end, duration=None, duration_start=None, duration_end=None)) self._update_meter_stats(results[-1], meter) return results