def _get_query_timestamps(args={}): # Determine the desired range, if any, from the # GET arguments. Set up the query range using # the specified offset. # [query_start ... start_timestamp ... end_timestamp ... query_end] search_offset = int(args.get('search_offset', 0)) start_timestamp = args.get('start_timestamp') if start_timestamp: start_timestamp = timeutils.parse_isotime(start_timestamp) start_timestamp = start_timestamp.replace(tzinfo=None) query_start = (start_timestamp - datetime.timedelta(minutes=search_offset)) else: query_start = None end_timestamp = args.get('end_timestamp') if end_timestamp: end_timestamp = timeutils.parse_isotime(end_timestamp) end_timestamp = end_timestamp.replace(tzinfo=None) query_end = end_timestamp + datetime.timedelta(minutes=search_offset) else: query_end = None return dict(query_start=query_start, query_end=query_end, start_timestamp=start_timestamp, end_timestamp=end_timestamp, search_offset=search_offset, )
def statistics(self, q=[], period=None): """Computes the statistics of the samples in the time range given. :param q: Filter rules for the data to be returned. :param period: Returned result will be an array of statistics for a period long of that number of seconds. """ kwargs = _query_to_kwargs(q, storage.EventFilter.__init__) kwargs['meter'] = self._id f = storage.EventFilter(**kwargs) computed = pecan.request.storage_conn.get_meter_statistics(f, period) LOG.debug('computed value coming from %r', pecan.request.storage_conn) # Find the original timestamp in the query to use for clamping # the duration returned in the statistics. start = end = None for i in q: if i.field == 'timestamp' and i.op in ('lt', 'le'): end = timeutils.parse_isotime(i.value).replace(tzinfo=None) elif i.field == 'timestamp' and i.op in ('gt', 'ge'): start = timeutils.parse_isotime(i.value).replace(tzinfo=None) return [ Statistics(start_timestamp=start, end_timestamp=end, **c.as_dict()) for c in computed ]
def statistics(self, q=[], period=None): """Computes the statistics of the samples in the time range given. :param q: Filter rules for the data to be returned. :param period: Returned result will be an array of statistics for a period long of that number of seconds. """ kwargs = _query_to_kwargs(q, storage.SampleFilter.__init__) kwargs['meter'] = self._id f = storage.SampleFilter(**kwargs) computed = pecan.request.storage_conn.get_meter_statistics(f, period) LOG.debug('computed value coming from %r', pecan.request.storage_conn) # Find the original timestamp in the query to use for clamping # the duration returned in the statistics. start = end = None for i in q: if i.field == 'timestamp' and i.op in ('lt', 'le'): end = timeutils.parse_isotime(i.value).replace(tzinfo=None) elif i.field == 'timestamp' and i.op in ('gt', 'ge'): start = timeutils.parse_isotime(i.value).replace(tzinfo=None) return [Statistics(start_timestamp=start, end_timestamp=end, **c.as_dict()) for c in computed]
def statistics(self, q=[], groupby=[], period=None): """Computes the statistics of the samples in the time range given. :param q: Filter rules for the data to be returned. :param groupby: Fields for group by aggregation :param period: Returned result will be an array of statistics for a period long of that number of seconds. """ if period and period < 0: error = _("Period must be positive.") pecan.response.translatable_error = error raise wsme.exc.ClientSideError(error) kwargs = _query_to_kwargs(q, storage.SampleFilter.__init__) kwargs['meter'] = self._id f = storage.SampleFilter(**kwargs) g = _validate_groupby_fields(groupby) computed = pecan.request.storage_conn.get_meter_statistics(f, period, g) LOG.debug('computed value coming from %r', pecan.request.storage_conn) # Find the original timestamp in the query to use for clamping # the duration returned in the statistics. start = end = None for i in q: if i.field == 'timestamp' and i.op in ('lt', 'le'): end = timeutils.parse_isotime(i.value).replace(tzinfo=None) elif i.field == 'timestamp' and i.op in ('gt', 'ge'): start = timeutils.parse_isotime(i.value).replace(tzinfo=None) return [Statistics(start_timestamp=start, end_timestamp=end, **c.as_dict()) for c in computed]
def sanitize_timestamp(timestamp): """Return a naive utc datetime object.""" if not timestamp: return timestamp if not isinstance(timestamp, datetime.datetime): timestamp = timeutils.parse_isotime(timestamp) return timeutils.normalize_time(timestamp)
def record_metering_data(self, data): # We may have receive only one counter on the wire if not isinstance(data, list): data = [data] for meter in data: LOG.debug( _('metering data %(counter_name)s ' 'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s') % ({ 'counter_name': meter['counter_name'], 'resource_id': meter['resource_id'], 'timestamp': meter.get('timestamp', 'NO TIMESTAMP'), 'counter_volume': meter['counter_volume'] })) if publisher_utils.verify_signature( meter, self.conf.publisher.metering_secret): try: # Convert the timestamp to a datetime instance. # Storage engines are responsible for converting # that value to something they can store. if meter.get('timestamp'): ts = timeutils.parse_isotime(meter['timestamp']) meter['timestamp'] = timeutils.normalize_time(ts) self.storage_conn.record_metering_data(meter) except Exception as err: LOG.exception(_('Failed to record metering data: %s'), err) else: LOG.warning( _('message signature invalid, discarding message: %r'), meter)
def record_metering_data(self, context, data): """This method is triggered when metering data is cast from an agent. """ #LOG.info('metering data: %r', data) LOG.info('metering data %s for %s @ %s: %s', data['counter_name'], data['resource_id'], data.get('timestamp', 'NO TIMESTAMP'), data['counter_volume']) if not meter.verify_signature(data, cfg.CONF.metering_secret): LOG.warning('message signature invalid, discarding message: %r', data) else: try: # Convert the timestamp to a datetime instance. # Storage engines are responsible for converting # that value to something they can store. if data.get('timestamp'): data['timestamp'] = timeutils.parse_isotime( data['timestamp'], ) self.storage_conn.record_metering_data(data) except Exception as err: LOG.error('Failed to record metering data: %s', err) LOG.exception(err)
def _list_resources(source=None, user=None, project=None, start_timestamp=None, end_timestamp=None): """Return a list of resource identifiers. """ if start_timestamp: start_timestamp = timeutils.parse_isotime(start_timestamp) if end_timestamp: end_timestamp = timeutils.parse_isotime(end_timestamp) resources = flask.request.storage_conn.get_resources( source=source, user=user, project=project, start_timestamp=start_timestamp, end_timestamp=end_timestamp, ) return flask.jsonify(resources=list(resources))
def handle_sample(self, context, counter, source): """Handle a sample, converting if necessary.""" LOG.debug('handling counter %s', (counter,)) key = counter.name + counter.resource_id prev = self.cache.get(key) timestamp = timeutils.parse_isotime(counter.timestamp) self.cache[key] = (counter.volume, timestamp) if prev: prev_volume = prev[0] prev_timestamp = prev[1] time_delta = timeutils.delta_seconds(prev_timestamp, timestamp) # we only allow negative deltas for noncumulative counters, whereas # for cumulative we assume that a reset has occurred in the interim # so that the current volume gives a lower bound on growth volume_delta = (counter.volume - prev_volume if (prev_volume <= counter.volume or counter.type != ceilocounter.TYPE_CUMULATIVE) else counter.volume) rate_of_change = ((1.0 * volume_delta / time_delta) if time_delta else 0.0) transformed = self._convert(counter, rate_of_change) LOG.debug(_('converted to: %s') % (transformed,)) counter = self._keep(counter, transformed) elif self.replace: LOG.warn(_('dropping counter with no predecessor: %s') % counter) counter = None return counter
def record_metering_data(self, data): # We may have receive only one counter on the wire if not isinstance(data, list): data = [data] for meter in data: LOG.debug(_( 'metering data %(counter_name)s ' 'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s') % ({'counter_name': meter['counter_name'], 'resource_id': meter['resource_id'], 'timestamp': meter.get('timestamp', 'NO TIMESTAMP'), 'counter_volume': meter['counter_volume']})) if publisher_utils.verify_signature( meter, self.conf.publisher.metering_secret): try: # Convert the timestamp to a datetime instance. # Storage engines are responsible for converting # that value to something they can store. if meter.get('timestamp'): ts = timeutils.parse_isotime(meter['timestamp']) meter['timestamp'] = timeutils.normalize_time(ts) self.storage_conn.record_metering_data(meter) except Exception as err: LOG.exception(_('Failed to record metering data: %s'), err) else: LOG.warning(_( 'message signature invalid, discarding message: %r'), meter)
def handle_sample(self, context, counter): """Handle a sample, converting if necessary.""" LOG.debug('handling counter %s', (counter, )) key = counter.name + counter.resource_id prev = self.cache.get(key) timestamp = timeutils.parse_isotime(counter.timestamp) self.cache[key] = (counter.volume, timestamp) if prev: prev_volume = prev[0] prev_timestamp = prev[1] time_delta = timeutils.delta_seconds(prev_timestamp, timestamp) # we only allow negative deltas for noncumulative counters, whereas # for cumulative we assume that a reset has occurred in the interim # so that the current volume gives a lower bound on growth volume_delta = (counter.volume - prev_volume if (prev_volume <= counter.volume or counter.type != sample.TYPE_CUMULATIVE) else counter.volume) rate_of_change = ((1.0 * volume_delta / time_delta) if time_delta else 0.0) counter = self._convert(counter, rate_of_change) LOG.debug(_('converted to: %s') % (counter, )) else: LOG.warn( _('dropping counter with no predecessor: %s') % (counter, )) counter = None return counter
def handle_sample(self, context, s): """Handle a sample, converting if necessary.""" LOG.debug(_('handling sample %s'), (s,)) key = s.name + s.resource_id prev = self.cache.get(key) timestamp = timeutils.parse_isotime(s.timestamp) self.cache[key] = (s.volume, timestamp) if prev: prev_volume = prev[0] prev_timestamp = prev[1] time_delta = timeutils.delta_seconds(prev_timestamp, timestamp) # we only allow negative deltas for noncumulative samples, whereas # for cumulative we assume that a reset has occurred in the interim # so that the current volume gives a lower bound on growth volume_delta = (s.volume - prev_volume if (prev_volume <= s.volume or s.type != sample.TYPE_CUMULATIVE) else s.volume) rate_of_change = ((1.0 * volume_delta / time_delta) if time_delta else 0.0) s = self._convert(s, rate_of_change) LOG.debug(_('converted to: %s'), (s,)) else: LOG.warn(_('dropping sample with no predecessor: %s'), (s,)) s = None return s
def record_metering_data(self, context, data): """This method is triggered when metering data is cast from an agent. """ # We may have receive only one counter on the wire if not isinstance(data, list): data = [data] for meter in data: LOG.info('metering data %s for %s @ %s: %s', meter['counter_name'], meter['resource_id'], meter.get('timestamp', 'NO TIMESTAMP'), meter['counter_volume']) if meter_api.verify_signature(meter, cfg.CONF.metering_secret): try: # Convert the timestamp to a datetime instance. # Storage engines are responsible for converting # that value to something they can store. if meter.get('timestamp'): ts = timeutils.parse_isotime(meter['timestamp']) meter['timestamp'] = timeutils.normalize_time(ts) self.storage_conn.record_metering_data(meter) except Exception as err: LOG.error('Failed to record metering data: %s', err) LOG.exception(err) else: LOG.warning( 'message signature invalid, discarding message: %r', meter)
def test_multiple_samples(self): """Send multiple samples. The usecase here is to reduce the chatter and send the counters at a slower cadence. """ samples = [] for x in range(6): dt = datetime.datetime(2012, 8, 27, x, 0, tzinfo=None) s = { 'counter_name': 'apples', 'counter_type': 'gauge', 'counter_unit': 'instance', 'counter_volume': float(x * 3), 'source': 'evil', 'timestamp': dt.isoformat(), 'resource_id': 'bd9431c1-8d69-4ad3-803a-8d4a6b89fd36', 'project_id': '35b17138-b364-4e6a-a131-8f3099c5be68', 'user_id': 'efd87807-12d2-4b38-9c70-5f5c2ac427ff', 'resource_metadata': { 'name1': str(x), 'name2': str(x + 4) } } samples.append(s) data = self.post_json('/meters/apples/', samples) for x, s in enumerate(samples): # source is modified to include the project_id. s['source'] = '%s:%s' % (s['project_id'], s['source']) # Ignore message id that is randomly generated s['message_id'] = data.json[x]['message_id'] # remove tzinfo to compare generated timestamp # with the provided one c = data.json[x] timestamp = timeutils.parse_isotime(c['timestamp']) c['timestamp'] = timestamp.replace(tzinfo=None).isoformat() # do the same on the pipeline msg = self.published[0][x] timestamp = timeutils.parse_isotime(msg['timestamp']) msg['timestamp'] = timestamp.replace(tzinfo=None).isoformat() self.assertEqual(s, c) self.assertEqual(s, self.published[0][x])
def _extract_when(body): """Extract the generated datetime from the notification. """ when = body.get('timestamp', body.get('_context_timestamp')) if when: return timeutils.normalize_time(timeutils.parse_isotime(when)) return timeutils.utcnow()
def convert_value(cls, trait_type, value): if trait_type is cls.INT_TYPE: return int(value) if trait_type is cls.FLOAT_TYPE: return float(value) if trait_type is cls.DATETIME_TYPE: return timeutils.normalize_time(timeutils.parse_isotime(value)) return str(value)
def test_multiple_samples(self): """Send multiple samples. The usecase here is to reduce the chatter and send the counters at a slower cadence. """ samples = [] for x in range(6): dt = datetime.datetime(2012, 8, 27, x, 0, tzinfo=None) s = {'counter_name': 'apples', 'counter_type': 'gauge', 'counter_unit': 'instance', 'counter_volume': float(x * 3), 'source': 'evil', 'timestamp': dt.isoformat(), 'resource_id': 'bd9431c1-8d69-4ad3-803a-8d4a6b89fd36', 'project_id': '35b17138-b364-4e6a-a131-8f3099c5be68', 'user_id': 'efd87807-12d2-4b38-9c70-5f5c2ac427ff', 'resource_metadata': {'name1': str(x), 'name2': str(x + 4)}} samples.append(s) data = self.post_json('/meters/apples/', samples) for x, s in enumerate(samples): # source is modified to include the project_id. s['source'] = '%s:%s' % (s['project_id'], s['source']) # Ignore message id that is randomly generated s['message_id'] = data.json[x]['message_id'] # remove tzinfo to compare generated timestamp # with the provided one c = data.json[x] timestamp = timeutils.parse_isotime(c['timestamp']) c['timestamp'] = timestamp.replace(tzinfo=None).isoformat() # do the same on the pipeline msg = self.published[0][x] timestamp = timeutils.parse_isotime(msg['timestamp']) msg['timestamp'] = timestamp.replace(tzinfo=None).isoformat() self.assertEqual(s, c) self.assertEqual(s, self.published[0][x])
def statistics(self, q=[]): """Computes the statistics of the meter events in the time range given. """ kwargs = _query_to_kwargs(q, storage.EventFilter.__init__) kwargs['meter'] = self._id f = storage.EventFilter(**kwargs) computed = request.storage_conn.get_meter_statistics(f) # Find the original timestamp in the query to use for clamping # the duration returned in the statistics. start = end = None for i in q: if i.field == 'timestamp' and i.op in ('lt', 'le'): end = timeutils.parse_isotime(i.value).replace(tzinfo=None) elif i.field == 'timestamp' and i.op in ('gt', 'ge'): start = timeutils.parse_isotime(i.value).replace(tzinfo=None) stat = Statistics(start_timestamp=start, end_timestamp=end, **computed) return stat
def _get_query_timestamps(args={}): """Return any optional timestamp information in the request. Determine the desired range, if any, from the GET arguments. Set up the query range using the specified offset. [query_start ... start_timestamp ... end_timestamp ... query_end] Returns a dictionary containing: query_start: First timestamp to use for query start_timestamp: start_timestamp parameter from request query_end: Final timestamp to use for query end_timestamp: end_timestamp parameter from request search_offset: search_offset parameter from request """ search_offset = int(args.get('search_offset', 0)) start_timestamp = args.get('start_timestamp') if start_timestamp: start_timestamp = timeutils.parse_isotime(start_timestamp) start_timestamp = start_timestamp.replace(tzinfo=None) query_start = (start_timestamp - datetime.timedelta(minutes=search_offset)) else: query_start = None end_timestamp = args.get('end_timestamp') if end_timestamp: end_timestamp = timeutils.parse_isotime(end_timestamp) end_timestamp = end_timestamp.replace(tzinfo=None) query_end = end_timestamp + datetime.timedelta(minutes=search_offset) else: query_end = None return { 'query_start': query_start, 'query_end': query_end, 'start_timestamp': start_timestamp, 'end_timestamp': end_timestamp, 'search_offset': search_offset, }
def _get_query_timestamps(args={}): """Return any optional timestamp information in the request. Determine the desired range, if any, from the GET arguments. Set up the query range using the specified offset. [query_start ... start_timestamp ... end_timestamp ... query_end] Returns a dictionary containing: query_start: First timestamp to use for query start_timestamp: start_timestamp parameter from request query_end: Final timestamp to use for query end_timestamp: end_timestamp parameter from request search_offset: search_offset parameter from request """ search_offset = int(args.get('search_offset', 0)) start_timestamp = args.get('start_timestamp') if start_timestamp: start_timestamp = timeutils.parse_isotime(start_timestamp) start_timestamp = start_timestamp.replace(tzinfo=None) query_start = (start_timestamp - datetime.timedelta(minutes=search_offset)) else: query_start = None end_timestamp = args.get('end_timestamp') if end_timestamp: end_timestamp = timeutils.parse_isotime(end_timestamp) end_timestamp = end_timestamp.replace(tzinfo=None) query_end = end_timestamp + datetime.timedelta(minutes=search_offset) else: query_end = None return {'query_start': query_start, 'query_end': query_end, 'start_timestamp': start_timestamp, 'end_timestamp': end_timestamp, 'search_offset': search_offset, }
def test_query_with_isotime(self): date_time = datetime.datetime(2012, 7, 2, 10, 41) isotime = date_time.isoformat() data = self.post_json(self.url, params={"filter": '{">=": {"timestamp": "' + isotime + '"}}'}) self.assertEqual(1, len(data.json)) for sample in data.json: result_time = timeutils.parse_isotime(sample["timestamp"]) result_time = result_time.replace(tzinfo=None) self.assertTrue(result_time >= date_time)
def test_filter_with_isotime(self): date_time = datetime.datetime(2013, 1, 1) isotime = date_time.isoformat() data = self.post_json(self.url, params={"filter": '{">": {"timestamp":"' + isotime + '"}}'}) self.assertEqual(4, len(data.json)) for history in data.json: result_time = timeutils.parse_isotime(history["timestamp"]) result_time = result_time.replace(tzinfo=None) self.assertTrue(result_time > date_time)
def _extract_when(body): """Extract the generated datetime from the notification.""" # NOTE: I am keeping the logic the same as it was in the collector, # However, *ALL* notifications should have a 'timestamp' field, it's # part of the notification envelope spec. If this was put here because # some openstack project is generating notifications without a # timestamp, then that needs to be filed as a bug with the offending # project (mdragon) when = body.get("timestamp", body.get("_context_timestamp")) if when: return timeutils.normalize_time(timeutils.parse_isotime(when)) return timeutils.utcnow()
def _extract_when(body): """Extract the generated datetime from the notification.""" # NOTE: I am keeping the logic the same as it was in the collector, # However, *ALL* notifications should have a 'timestamp' field, it's # part of the notification envelope spec. If this was put here because # some openstack project is generating notifications without a # timestamp, then that needs to be filed as a bug with the offending # project (mdragon) when = body.get('timestamp', body.get('_context_timestamp')) if when: return timeutils.normalize_time(timeutils.parse_isotime(when)) return timeutils.utcnow()
def test_filter_with_isotime(self): date_time = datetime.datetime(2013, 1, 1) isotime = date_time.isoformat() data = self.post_json( self.url, params={"filter": '{">": {"timestamp":"' + isotime + '"}}'}) self.assertEqual(4, len(data.json)) for history in data.json: result_time = timeutils.parse_isotime(history['timestamp']) result_time = result_time.replace(tzinfo=None) self.assertTrue(result_time > date_time)
def test_query_with_isotime(self): date_time = datetime.datetime(2012, 7, 2, 10, 41) isotime = date_time.isoformat() data = self.post_json( self.url, params={"filter": '{">=": {"timestamp": "' + isotime + '"}}'}) self.assertEqual(2, len(data.json)) for sample_item in data.json: result_time = timeutils.parse_isotime(sample_item['timestamp']) result_time = result_time.replace(tzinfo=None) self.assertTrue(result_time >= date_time)
def test_filter_with_isotime_state_timestamp(self): date_time = datetime.datetime(2013, 1, 1) isotime = date_time.isoformat() data = self.post_json(self.alarm_url, params={"filter": '{">": {"state_timestamp": "' + isotime + '"}}'}) self.assertEqual(6, len(data.json)) for alarm in data.json: result_time = timeutils.parse_isotime(alarm['state_timestamp']) result_time = result_time.replace(tzinfo=None) self.assertTrue(result_time > date_time)
def __init__(self, counter_volume=None, resource_metadata={}, timestamp=None, **kwds): if counter_volume is not None: counter_volume = float(counter_volume) resource_metadata = _flatten_metadata(resource_metadata) # this is to make it easier for clients to pass a timestamp in if timestamp and isinstance(timestamp, basestring): timestamp = timeutils.parse_isotime(timestamp) super(Sample, self).__init__( counter_volume=counter_volume, resource_metadata=resource_metadata, timestamp=timestamp, **kwds ) # Seems the mandatory option doesn't work so do it manually for m in ("counter_volume", "counter_unit", "counter_name", "counter_type", "resource_id"): if getattr(self, m) in (wsme.Unset, None): raise wsme.exc.MissingArgument(m) if self.resource_metadata in (wtypes.Unset, None): self.resource_metadata = {}
def test_multiple_samples(self): ''' send multiple samples. The usecase here is to reduce the chatter and send the counters at a slower cadence. ''' samples = [] stamps = [] for x in range(6): dt = datetime.datetime(2012, 8, 27, x, 0, tzinfo=None) stamps.append(dt) s = { 'counter_name': 'apples', 'counter_type': 'gauge', 'counter_unit': 'instance', 'counter_volume': float(x * 3), 'source': 'evil', 'timestamp': dt.isoformat(), 'resource_id': 'bd9431c1-8d69-4ad3-803a-8d4a6b89fd36', 'project_id': '35b17138-b364-4e6a-a131-8f3099c5be68', 'user_id': 'efd87807-12d2-4b38-9c70-5f5c2ac427ff', 'resource_metadata': { 'name1': str(x), 'name2': str(x + 4) } } samples.append(s) data = self.post_json('/meters/apples/', samples) # source is modified to include the project_id. for x in range(6): for (k, v) in samples[x].iteritems(): if k == 'timestamp': timestamp = timeutils.parse_isotime(data.json[x][k]) self.assertEquals(stamps[x].replace(tzinfo=None), timestamp.replace(tzinfo=None)) elif k == 'source': self.assertEquals( data.json[x][k], '%s:%s' % (samples[x]['project_id'], samples[x]['source'])) else: self.assertEquals(v, data.json[x][k])
def __init__(self, counter_volume=None, resource_metadata={}, timestamp=None, **kwds): if counter_volume is not None: counter_volume = float(counter_volume) resource_metadata = _flatten_metadata(resource_metadata) # this is to make it easier for clients to pass a timestamp in if timestamp and isinstance(timestamp, basestring): timestamp = timeutils.parse_isotime(timestamp) super(Sample, self).__init__(counter_volume=counter_volume, resource_metadata=resource_metadata, timestamp=timestamp, **kwds) # Seems the mandatory option doesn't work so do it manually for m in ('counter_volume', 'counter_unit', 'counter_name', 'counter_type', 'resource_id'): if getattr(self, m) in (wsme.Unset, None): raise wsme.exc.MissingArgument(m) if self.resource_metadata in (wtypes.Unset, None): self.resource_metadata = {}
def handle_sample(self, context, sample_): if not self.initial_timestamp: self.initial_timestamp = timeutils.parse_isotime(sample_.timestamp) self.aggregated_samples += 1 key = self._get_unique_key(sample_) self.counts[key] += 1 if key not in self.samples: self.samples[key] = self._convert(sample_) if self.merged_attribute_policy[ 'resource_metadata'] == 'drop': self.samples[key].resource_metadata = {} else: if sample_.type == sample.TYPE_CUMULATIVE: self.samples[key].volume = self._scale(sample_) else: self.samples[key].volume += self._scale(sample_) for field in self.merged_attribute_policy: if self.merged_attribute_policy[field] == 'last': setattr(self.samples[key], field, getattr(sample_, field))
def test_multiple_samples(self): ''' send multiple samples. The usecase here is to reduce the chatter and send the counters at a slower cadence. ''' samples = [] stamps = [] for x in range(6): dt = datetime.datetime(2012, 8, 27, x, 0, tzinfo=None) stamps.append(dt) s = {'counter_name': 'apples', 'counter_type': 'gauge', 'counter_unit': 'instance', 'counter_volume': float(x * 3), 'source': 'evil', 'timestamp': dt.isoformat(), 'resource_id': 'bd9431c1-8d69-4ad3-803a-8d4a6b89fd36', 'project_id': '35b17138-b364-4e6a-a131-8f3099c5be68', 'user_id': 'efd87807-12d2-4b38-9c70-5f5c2ac427ff', 'resource_metadata': {'name1': str(x), 'name2': str(x + 4)}} samples.append(s) data = self.post_json('/meters/apples/', samples) # source is modified to include the project_id. for x in range(6): for (k, v) in samples[x].iteritems(): if k == 'timestamp': timestamp = timeutils.parse_isotime(data.json[x][k]) self.assertEquals(stamps[x].replace(tzinfo=None), timestamp.replace(tzinfo=None)) elif k == 'source': self.assertEquals(data.json[x][k], '%s:%s' % (samples[x]['project_id'], samples[x]['source'])) else: self.assertEquals(v, data.json[x][k])
def record_metering_data(self, context, data): """This method is triggered when metering data is cast from an agent. """ #LOG.info('metering data: %r', data) LOG.info('metering data %s for %s @ %s: %s', data['counter_name'], data['resource_id'], data.get('timestamp', 'NO TIMESTAMP'), data['counter_volume']) if not meter.verify_signature(data): LOG.warning('message signature invalid, discarding message: %r', data) else: try: # Convert the timestamp to a datetime instance. # Storage engines are responsible for converting # that value to something they can store. if data.get('timestamp'): data['timestamp'] = timeutils.parse_isotime( data['timestamp'], ) self.storage_conn.record_metering_data(data) except Exception as err: LOG.error('Failed to record metering data: %s', err) LOG.exception(err)
def main(): cfg.CONF([], project='ceilometer') parser = argparse.ArgumentParser(description='generate metering data', ) parser.add_argument( '--interval', default=10, type=int, help='the period between events, in minutes', ) parser.add_argument( '--start', default=31, help='the number of days in the past to start timestamps', ) parser.add_argument( '--end', default=2, help='the number of days into the future to continue timestamps', ) parser.add_argument( '--type', choices=('gauge', 'cumulative'), default='gauge', help='counter type', ) parser.add_argument( '--unit', default=None, help='counter unit', ) parser.add_argument( '--project', help='project id of owner', ) parser.add_argument( '--user', help='user id of owner', ) parser.add_argument( 'resource', help='the resource id for the meter data', ) parser.add_argument( 'counter', help='the counter name for the meter data', ) parser.add_argument( 'volume', help='the amount to attach to the meter', type=int, default=1, ) args = parser.parse_args() # Set up logging to use the console console = logging.StreamHandler(sys.stderr) console.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s') console.setFormatter(formatter) root_logger = logging.getLogger('') root_logger.addHandler(console) root_logger.setLevel(logging.DEBUG) # Connect to the metering database conn = storage.get_connection(cfg.CONF) # Find the user and/or project for a real resource if not (args.user or args.project): for r in conn.get_resources(): if r['resource_id'] == args.resource: args.user = r['user_id'] args.project = r['project_id'] break # Compute start and end timestamps for the # new data. timestamp = timeutils.parse_isotime(args.start) end = timeutils.parse_isotime(args.end) increment = datetime.timedelta(minutes=args.interval) # Generate events n = 0 while timestamp <= end: c = sample.Sample( name=args.counter, type=args.type, unit=args.unit, volume=args.volume, user_id=args.user, project_id=args.project, resource_id=args.resource, timestamp=timestamp, resource_metadata={}, source='artificial', ) data = rpc.meter_message_from_counter( c, cfg.CONF.publisher_rpc.metering_secret) conn.record_metering_data(data) n += 1 timestamp = timestamp + increment print 'Added %d new events' % n return 0
def _assert_times_match(self, actual, expected): actual = timeutils.parse_isotime(actual).replace(tzinfo=None) self.assertEqual(actual, expected)
def compute_duration_by_resource(resource, meter): """Return the earliest timestamp, last timestamp, and duration for the resource and meter. :param resource: The ID of the resource. :param meter: The name of the meter. :param start_timestamp: ISO-formatted string of the earliest timestamp to return. :param end_timestamp: ISO-formatted string of the latest timestamp to return. :param search_offset: Number of minutes before and after start and end timestamps to query. """ # Determine the desired range, if any, from the # GET arguments. Set up the query range using # the specified offset. # [query_start ... start_timestamp ... end_timestamp ... query_end] search_offset = int(flask.request.args.get('search_offset', 0)) start_timestamp = flask.request.args.get('start_timestamp') if start_timestamp: start_timestamp = timeutils.parse_isotime(start_timestamp) start_timestamp = start_timestamp.replace(tzinfo=None) query_start = (start_timestamp - datetime.timedelta(minutes=search_offset)) else: query_start = None end_timestamp = flask.request.args.get('end_timestamp') if end_timestamp: end_timestamp = timeutils.parse_isotime(end_timestamp) end_timestamp = end_timestamp.replace(tzinfo=None) query_end = end_timestamp + datetime.timedelta(minutes=search_offset) else: query_end = None # Query the database for the interval of timestamps # within the desired range. f = storage.EventFilter(meter=meter, resource=resource, start=query_start, end=query_end, ) min_ts, max_ts = flask.request.storage_conn.get_event_interval(f) # "Clamp" the timestamps we return to the original time # range, excluding the offset. LOG.debug('start_timestamp %s, end_timestamp %s, min_ts %s, max_ts %s', start_timestamp, end_timestamp, min_ts, max_ts) if start_timestamp and min_ts and min_ts < start_timestamp: min_ts = start_timestamp LOG.debug('clamping min timestamp to range') if end_timestamp and max_ts and max_ts > end_timestamp: max_ts = end_timestamp LOG.debug('clamping max timestamp to range') # If we got valid timestamps back, compute a duration in minutes. # # If the min > max after clamping then we know the # timestamps on the events fell outside of the time # range we care about for the query, so treat them as # "invalid." # # If the timestamps are invalid, return None as a # sentinal indicating that there is something "funny" # about the range. if min_ts and max_ts and (min_ts <= max_ts): # Can't use timedelta.total_seconds() because # it is not available in Python 2.6. diff = max_ts - min_ts duration = (diff.seconds + (diff.days * 24 * 60 ** 2)) / 60 else: min_ts = max_ts = duration = None return flask.jsonify(start_timestamp=min_ts, end_timestamp=max_ts, duration=duration, )
def main(): cfg.CONF([], project="ceilometer") parser = argparse.ArgumentParser(description="generate metering data") parser.add_argument("--interval", default=10, type=int, help="the period between events, in minutes") parser.add_argument("--start", default=31, help="the number of days in the past to start timestamps") parser.add_argument("--end", default=2, help="the number of days into the future to continue timestamps") parser.add_argument("--type", choices=("gauge", "cumulative"), default="gauge", help="counter type") parser.add_argument("--unit", default=None, help="counter unit") parser.add_argument("--project", help="project id of owner") parser.add_argument("--user", help="user id of owner") parser.add_argument("resource", help="the resource id for the meter data") parser.add_argument("counter", help="the counter name for the meter data") parser.add_argument("volume", help="the amount to attach to the meter", type=int, default=1) args = parser.parse_args() # Set up logging to use the console console = logging.StreamHandler(sys.stderr) console.setLevel(logging.DEBUG) formatter = logging.Formatter("%(message)s") console.setFormatter(formatter) root_logger = logging.getLogger("") root_logger.addHandler(console) root_logger.setLevel(logging.DEBUG) # Connect to the metering database conn = storage.get_connection(cfg.CONF) # Find the user and/or project for a real resource if not (args.user or args.project): for r in conn.get_resources(): if r["resource_id"] == args.resource: args.user = r["user_id"] args.project = r["project_id"] break # Compute start and end timestamps for the # new data. timestamp = timeutils.parse_isotime(args.start) end = timeutils.parse_isotime(args.end) increment = datetime.timedelta(minutes=args.interval) # Generate events n = 0 while timestamp <= end: c = counter.Counter( name=args.counter, type=args.type, unit=args.unit, volume=args.volume, user_id=args.user, project_id=args.project, resource_id=args.resource, timestamp=timestamp, resource_metadata={}, ) data = meter.meter_message_from_counter(c, cfg.CONF.metering_secret, "artificial") conn.record_metering_data(data) n += 1 timestamp = timestamp + increment print "Added %d new events" % n return 0
def _assert_times_match(self, actual, expected): actual = timeutils.parse_isotime(actual).replace(tzinfo=None) assert actual == expected
def compute_duration_by_resource(resource, meter): """Return the earliest timestamp, last timestamp, and duration for the resource and meter. :param resource: The ID of the resource. :param meter: The name of the meter. :param start_timestamp: ISO-formatted string of the earliest timestamp to return. :param end_timestamp: ISO-formatted string of the latest timestamp to return. :param search_offset: Number of minutes before and after start and end timestamps to query. """ # Determine the desired range, if any, from the # GET arguments. Set up the query range using # the specified offset. # [query_start ... start_timestamp ... end_timestamp ... query_end] search_offset = int(flask.request.args.get('search_offset', 0)) start_timestamp = flask.request.args.get('start_timestamp') if start_timestamp: start_timestamp = timeutils.parse_isotime(start_timestamp) start_timestamp = start_timestamp.replace(tzinfo=None) query_start = (start_timestamp - datetime.timedelta(minutes=search_offset)) else: query_start = None end_timestamp = flask.request.args.get('end_timestamp') if end_timestamp: end_timestamp = timeutils.parse_isotime(end_timestamp) end_timestamp = end_timestamp.replace(tzinfo=None) query_end = end_timestamp + datetime.timedelta(minutes=search_offset) else: query_end = None # Query the database for the interval of timestamps # within the desired range. f = storage.EventFilter( meter=meter, resource=resource, start=query_start, end=query_end, ) min_ts, max_ts = flask.request.storage_conn.get_event_interval(f) # "Clamp" the timestamps we return to the original time # range, excluding the offset. LOG.debug('start_timestamp %s, end_timestamp %s, min_ts %s, max_ts %s', start_timestamp, end_timestamp, min_ts, max_ts) if start_timestamp and min_ts and min_ts < start_timestamp: min_ts = start_timestamp LOG.debug('clamping min timestamp to range') if end_timestamp and max_ts and max_ts > end_timestamp: max_ts = end_timestamp LOG.debug('clamping max timestamp to range') # If we got valid timestamps back, compute a duration in minutes. # # If the min > max after clamping then we know the # timestamps on the events fell outside of the time # range we care about for the query, so treat them as # "invalid." # # If the timestamps are invalid, return None as a # sentinal indicating that there is something "funny" # about the range. if min_ts and max_ts and (min_ts <= max_ts): # Can't use timedelta.total_seconds() because # it is not available in Python 2.6. diff = max_ts - min_ts duration = (diff.seconds + (diff.days * 24 * 60**2)) / 60 else: min_ts = max_ts = duration = None return flask.jsonify( start_timestamp=min_ts, end_timestamp=max_ts, duration=duration, )
def _assert_times_match(self, actual, expected): if actual: actual = timeutils.parse_isotime(actual) actual = actual.replace(tzinfo=None) assert actual == expected
def _assert_times_match(self, actual, expected): if actual: actual = timeutils.parse_isotime(actual) actual = actual.replace(tzinfo=None) self.assertEqual(expected, actual)
def main(): cfg.CONF([], project='ceilometer') parser = argparse.ArgumentParser( description='generate metering data', ) parser.add_argument( '--interval', default=10, type=int, help='the period between events, in minutes', ) parser.add_argument( '--start', default=31, help='the number of days in the past to start timestamps', ) parser.add_argument( '--end', default=2, help='the number of days into the future to continue timestamps', ) parser.add_argument( '--type', choices=('gauge', 'cumulative'), default='gauge', help='counter type', ) parser.add_argument( '--unit', default=None, help='counter unit', ) parser.add_argument( '--project', help='project id of owner', ) parser.add_argument( '--user', help='user id of owner', ) parser.add_argument( 'resource', help='the resource id for the meter data', ) parser.add_argument( 'counter', help='the counter name for the meter data', ) parser.add_argument( 'volume', help='the amount to attach to the meter', type=int, default=1, ) args = parser.parse_args() # Set up logging to use the console console = logging.StreamHandler(sys.stderr) console.setLevel(logging.DEBUG) formatter = logging.Formatter('%(message)s') console.setFormatter(formatter) root_logger = logging.getLogger('') root_logger.addHandler(console) root_logger.setLevel(logging.DEBUG) # Connect to the metering database conn = storage.get_connection(cfg.CONF) # Find the user and/or project for a real resource if not (args.user or args.project): for r in conn.get_resources(): if r['resource_id'] == args.resource: args.user = r['user_id'] args.project = r['project_id'] break # Compute start and end timestamps for the # new data. timestamp = timeutils.parse_isotime(args.start) end = timeutils.parse_isotime(args.end) increment = datetime.timedelta(minutes=args.interval) # Generate events n = 0 while timestamp <= end: c = sample.Sample(name=args.counter, type=args.type, unit=args.unit, volume=args.volume, user_id=args.user, project_id=args.project, resource_id=args.resource, timestamp=timestamp, resource_metadata={}, source='artificial', ) data = utils.meter_message_from_counter( c, cfg.CONF.publisher.metering_secret) conn.record_metering_data(data) n += 1 timestamp = timestamp + increment print('Added %d new events' % n) return 0