def show_resources(db, args): if args: users = args else: users = sorted(db.get_users()) for u in users: print u for resource in db.get_resources(user=u): print ' %(resource_id)s %(timestamp)s' % resource for k, v in sorted(resource['metadata'].iteritems()): print ' %-10s : %s' % (k, v) for meter in resource['meter']: # FIXME(dhellmann): Need a way to tell whether to use # max() or sum() by meter name without hard-coding. if meter['counter_name'] in ['cpu', 'disk']: totals = db.get_volume_max(storage.EventFilter( user=u, meter=meter['counter_name'], resource=resource['resource_id'], )) else: totals = db.get_volume_sum(storage.EventFilter( user=u, meter=meter['counter_name'], resource=resource['resource_id'], )) print ' %s (%s): %s' % \ (meter['counter_name'], meter['counter_type'], totals.next()['value'])
def test_storage_can_handle_large_values(self): f = storage.EventFilter(meter='dummyBigCounter', ) results = list(self.conn.get_samples(f)) self.assertEqual(results[0].counter_volume, 3372036854775807) f = storage.EventFilter(meter='dummySmallCounter', ) results = list(self.conn.get_samples(f)) self.assertEqual(results[0].counter_volume, -3372036854775807)
def get_one(self, message_id): """Return a single event with the given message id. :param message_id: Message ID of the Event to be returned """ rbac.enforce("events:show", pecan.request) filters = _build_rbac_query_filters() t_filter = filters['t_filter'] admin_proj = filters['admin_proj'] event_filter = storage.EventFilter(traits_filter=t_filter, admin_proj=admin_proj, message_id=message_id) events = [ event for event in pecan.request.event_storage_conn.get_events( event_filter) ] if not events: raise base.EntityNotFound(_("Event"), message_id) if len(events) > 1: LOG.error( _("More than one event with " "id %s returned from storage driver") % message_id) event = events[0] return Event(message_id=event.message_id, event_type=event.event_type, generated=event.generated, traits=event.traits, raw=event.raw)
def statistics(self, q=[], period=None): """Computes the statistics of the samples in the time range given. :param q: Filter rules for the data to be returned. :param period: Returned result will be an array of statistics for a period long of that number of seconds. """ kwargs = _query_to_kwargs(q, storage.EventFilter.__init__) kwargs['meter'] = self._id f = storage.EventFilter(**kwargs) computed = pecan.request.storage_conn.get_meter_statistics(f, period) LOG.debug('computed value coming from %r', pecan.request.storage_conn) # Find the original timestamp in the query to use for clamping # the duration returned in the statistics. start = end = None for i in q: if i.field == 'timestamp' and i.op in ('lt', 'le'): end = timeutils.parse_isotime(i.value).replace(tzinfo=None) elif i.field == 'timestamp' and i.op in ('gt', 'ge'): start = timeutils.parse_isotime(i.value).replace(tzinfo=None) return [ Statistics(start_timestamp=start, end_timestamp=end, **c.as_dict()) for c in computed ]
def test_by_user_period_start_end(self): f = storage.EventFilter( user='******', meter='volume.size', start='2012-09-25T10:28:00', end='2012-09-25T11:28:00', ) results = list(self.conn.get_meter_statistics(f, period=1800)) self.assertEqual(len(results), 1) r = results[0] self.assertEqual(r.period_start, datetime.datetime(2012, 9, 25, 10, 28)) self.assertEqual(r.count, 1) self.assertEqual(r.avg, 8) self.assertEqual(r.min, 8) self.assertEqual(r.max, 8) self.assertEqual(r.sum, 8) self.assertEqual(r.period, 1800) self.assertEqual(r.period_end, r.period_start + datetime.timedelta(seconds=1800)) self.assertEqual(r.duration, 0) self.assertEqual(r.duration_start, datetime.datetime(2012, 9, 25, 10, 30)) self.assertEqual(r.duration_end, datetime.datetime(2012, 9, 25, 10, 30))
def test_no_period_in_query(self): f = storage.EventFilter( user='******', meter='volume.size', ) results = list(self.conn.get_meter_statistics(f))[0] assert results.period == 0
def _event_query_to_event_filter(q): evt_model_filter = { 'event_type': None, 'message_id': None, 'start_timestamp': None, 'end_timestamp': None } traits_filter = [] for i in q: if not i.op: i.op = 'eq' elif i.op not in base.operation_kind: error = _("operator {} is incorrect").format(i.op) raise base.ClientSideError(error) if i.field in evt_model_filter: evt_model_filter[i.field] = i.value else: trait_type = i.type or 'string' traits_filter.append({ "key": i.field, trait_type: i._get_value_as_type(), "op": i.op }) return storage.EventFilter(traits_filter=traits_filter, **evt_model_filter)
def test_get_raw_events_by_resource(self): f = storage.EventFilter(user='******', resource='resource-id') results = list(self.conn.get_raw_events(f)) assert results meter = results[0] assert meter is not None assert meter == self.msg
def get_one(self, message_id): """Return a single event with the given message id. :param message_id: Message ID of the Event to be returned """ event_filter = storage.EventFilter(message_id=message_id) events = [ event for event in pecan.request.event_storage_conn.get_events( event_filter) ] if not events: raise base.EntityNotFound(_("Event"), message_id) if len(events) > 1: LOG.error( _("More than one event with " "id %s returned from storage driver") % message_id) event = events[0] return Event(message_id=event.message_id, event_type=event.event_type, generated=event.generated, traits=event.traits, raw=event.raw)
def test_get_raw_events_by_meter2(self): f = storage.EventFilter( user='******', meter='instance', ) results = list(self.conn.get_raw_events(f)) assert results
def setUp(self): super(TestGetEventInterval, self).setUp() # NOTE(dhellmann): mim requires spidermonkey to implement the # map-reduce functions, so if we can't import it then just # skip these tests unless we aren't using mim. try: import spidermonkey except: if isinstance(self.conn.conn, mim.Connection): raise skip.SkipTest('requires spidermonkey') # Create events relative to the range and pretend # that the intervening events exist. self.start = datetime.datetime(2012, 8, 28, 0, 0) self.end = datetime.datetime(2012, 8, 29, 0, 0) self.early1 = self.start - datetime.timedelta(minutes=20) self.early2 = self.start - datetime.timedelta(minutes=10) self.middle1 = self.start + datetime.timedelta(minutes=10) self.middle2 = self.end - datetime.timedelta(minutes=10) self.late1 = self.end + datetime.timedelta(minutes=10) self.late2 = self.end + datetime.timedelta(minutes=20) self._filter = storage.EventFilter( resource='resource-id', meter='instance', start=self.start, end=self.end, )
def _list_samples(meter, project=None, resource=None, source=None, user=None): """Return a list of raw samples. Note: the API talks about "events" these are equivelent to samples. but we still need to return the samples within the "events" dict to maintain API compatibilty. """ q_ts = _get_query_timestamps(flask.request.args) f = storage.EventFilter( user=user, project=project, source=source, meter=meter, resource=resource, start=q_ts['start_timestamp'], end=q_ts['end_timestamp'], metaquery=_get_metaquery(flask.request.args), ) events = flask.request.storage_conn.get_samples(f) jsonified = flask.jsonify(events=[e.as_dict() for e in events]) if request_wants_html(): return flask.templating.render_template('list_event.html', user=user, project=project, source=source, meter=meter, resource=resource, events=jsonified) return jsonified
def test_get_raw_events_by_meter(self): f = storage.EventFilter( user='******', meter='no-such-meter', ) results = list(self.conn.get_raw_events(f)) assert not results
def _event_query_to_event_filter(q): evt_model_filter = { 'event_type': None, 'message_id': None, 'start_timestamp': None, 'end_timestamp': None } filters = _build_rbac_query_filters() traits_filter = filters['t_filter'] admin_proj = filters['admin_proj'] for i in q: if not i.op: i.op = 'eq' elif i.op not in base.operation_kind: error = (_('Operator %(operator)s is not supported. The supported' ' operators are: %(supported)s') % {'operator': i.op, 'supported': base.operation_kind}) raise base.ClientSideError(error) if i.field in evt_model_filter: if i.op != 'eq': error = (_('Operator %(operator)s is not supported. Only' ' equality operator is available for field' ' %(field)s') % {'operator': i.op, 'field': i.field}) raise base.ClientSideError(error) evt_model_filter[i.field] = i.value else: trait_type = i.type or 'string' traits_filter.append({"key": i.field, trait_type: i._get_value_as_type(), "op": i.op}) return storage.EventFilter(traits_filter=traits_filter, admin_proj=admin_proj, **evt_model_filter)
def _event_query_to_event_filter(q): evt_model_filter = { 'event_type': None, 'message_id': None, 'start_timestamp': None, 'end_timestamp': None } traits_filter = [] for i in q: if not i.op: i.op = 'eq' elif i.op not in base.operation_kind: error = (_('operator %(operator)s is not supported. the supported' ' operators are: %(supported)s') % { 'operator': i.op, 'supported': base.operation_kind }) raise base.ClientSideError(error) if i.field in evt_model_filter: evt_model_filter[i.field] = i.value else: trait_type = i.type or 'string' traits_filter.append({ "key": i.field, trait_type: i._get_value_as_type(), "op": i.op }) return storage.EventFilter(traits_filter=traits_filter, **evt_model_filter)
def compute_duration_by_resource(resource, meter): """Return the earliest timestamp, last timestamp, and duration for the resource and meter. :param resource: The ID of the resource. :param meter: The name of the meter. :param start_timestamp: ISO-formatted string of the earliest timestamp to return. :param end_timestamp: ISO-formatted string of the latest timestamp to return. :param search_offset: Number of minutes before and after start and end timestamps to query. """ q_ts = _get_query_timestamps(flask.request.args) start_timestamp = q_ts['start_timestamp'] end_timestamp = q_ts['end_timestamp'] # Query the database for the interval of timestamps # within the desired range. f = storage.EventFilter( meter=meter, project=acl.get_limited_to_project(flask.request.headers), resource=resource, start=q_ts['query_start'], end=q_ts['query_end'], ) stats = flask.request.storage_conn.get_meter_statistics(f) min_ts, max_ts = stats.duration_start, stats.duration_end # "Clamp" the timestamps we return to the original time # range, excluding the offset. LOG.debug('start_timestamp %s, end_timestamp %s, min_ts %s, max_ts %s', start_timestamp, end_timestamp, min_ts, max_ts) if start_timestamp and min_ts and min_ts < start_timestamp: min_ts = start_timestamp LOG.debug('clamping min timestamp to range') if end_timestamp and max_ts and max_ts > end_timestamp: max_ts = end_timestamp LOG.debug('clamping max timestamp to range') # If we got valid timestamps back, compute a duration in minutes. # # If the min > max after clamping then we know the # timestamps on the samples fell outside of the time # range we care about for the query, so treat them as # "invalid." # # If the timestamps are invalid, return None as a # sentinal indicating that there is something "funny" # about the range. if min_ts and max_ts and (min_ts <= max_ts): duration = timeutils.delta_seconds(min_ts, max_ts) else: min_ts = max_ts = duration = None return flask.jsonify(start_timestamp=min_ts, end_timestamp=max_ts, duration=duration, )
def test_get_samples_by_start_time(self): f = storage.EventFilter( user='******', start=datetime.datetime(2012, 7, 2, 10, 41), ) results = list(self.conn.get_samples(f)) assert len(results) == 1 assert results[0].timestamp == datetime.datetime(2012, 7, 2, 10, 41)
def test_get_raw_events_by_end_time(self): f = storage.EventFilter( user='******', end=datetime.datetime(2012, 7, 2, 10, 41), ) results = list(self.conn.get_raw_events(f)) length = len(results) assert length == 1 assert results[0]['timestamp'] == datetime.datetime(2012, 7, 2, 10, 40)
def test_get_raw_events_by_both_times(self): f = storage.EventFilter( start=datetime.datetime(2012, 7, 2, 10, 42), end=datetime.datetime(2012, 7, 2, 10, 43), ) results = list(self.conn.get_raw_events(f)) length = len(results) assert length == 1 assert results[0]['timestamp'] == datetime.datetime(2012, 7, 2, 10, 42)
def test_get_event_trait_filter(self): trait_filters = {'key': 'trait_B', 't_int': 101} event_filter = storage.EventFilter(self.start, self.end, traits=trait_filters) events = self.conn.get_events(event_filter) self.assertEquals(1, len(events)) self.assertEquals(events[0].event_name, "Bar") self.assertEquals(4, len(events[0].traits))
def show_total_resources(db, args): if args: users = args else: users = sorted(db.get_users()) for u in users: print u for meter in ['disk', 'cpu', 'instance']: if meter in ['cpu', 'disk']: total = db.get_volume_max(storage.EventFilter( user=u, meter=meter, )) else: total = db.get_volume_sum(storage.EventFilter( user=u, meter=meter, )) for t in total: print ' ', meter, t['resource_id'], t['value']
def show_raw(db, args): fmt = ' %(timestamp)s %(counter_name)10s %(counter_volume)s' for u in sorted(db.get_users()): print u for resource in db.get_resources(user=u): print ' ', resource['resource_id'] for event in db.get_raw_events(storage.EventFilter( user=u, resource=resource['resource_id'], )): print fmt % event
def test_one_resource(self): f = storage.EventFilter( user='******', meter='instance', resource='resource-id', ) results = list(self.conn.get_volume_sum(f)) assert results counts = dict((r['resource_id'], r['value']) for r in results) assert counts['resource-id'] == 1 assert set(counts.keys()) == set(['resource-id'])
def get_all(self, q=[]): """Return samples for the meter. :param q: Filter rules for the data to be returned. """ kwargs = _query_to_kwargs(q, storage.EventFilter.__init__) kwargs['meter'] = self._id f = storage.EventFilter(**kwargs) return [ Sample.from_db_model(e) for e in pecan.request.storage_conn.get_samples(f) ]
def test_get_samples_by_metaquery(self): q = {'metadata.display_name': 'test-server'} f = storage.EventFilter(metaquery=q) got_not_imp = False try: results = list(self.conn.get_samples(f)) assert results for meter in results: assert meter.as_dict() in self.msgs except NotImplementedError: got_not_imp = True self.assertTrue(got_not_imp)
def test_by_project(self): f = storage.EventFilter( project='project-id', meter='instance', ) results = list(self.conn.get_volume_sum(f)) assert results counts = dict((r['resource_id'], r['value']) for r in results) assert counts['resource-id'] == 1 assert counts['resource-id-alternate'] == 2 assert set(counts.keys()) == set( ['resource-id', 'resource-id-alternate'])
def test_simple_get(self): event_filter = storage.EventFilter(self.start, self.end) events = self.conn.get_events(event_filter) self.assertEquals(3, len(events)) start_time = None for i, name in enumerate(["Foo", "Bar", "Zoo"]): self.assertEquals(events[i].event_name, name) self.assertEquals(4, len(events[i].traits)) # Ensure sorted results ... if start_time is not None: # Python 2.6 has no assertLess :( self.assertTrue(start_time < events[i].generated) start_time = events[i].generated
def test_by_project(self): f = storage.EventFilter( meter='volume.size', resource='resource-id', start='2012-09-25T11:30:00', end='2012-09-25T11:32:00', ) results = list(self.conn.get_meter_statistics(f))[0] self.assertEqual(results.duration, 0) assert results.count == 1 assert results.min == 6 assert results.max == 6 assert results.sum == 6 assert results.avg == 6
def test_one_resource(self): f = storage.EventFilter( user='******', meter='volume.size', ) results = list(self.conn.get_meter_statistics(f))[0] self.assertEqual(results.duration, (datetime.datetime(2012, 9, 25, 12, 32) - datetime.datetime(2012, 9, 25, 10, 30)).seconds) assert results.count == 3 assert results.min == 5 assert results.max == 7 assert results.sum == 18 assert results.avg == 6
def show_total_resources(db, args): if args: users = args else: users = sorted(db.get_users()) for u in users: print u for meter in ['disk', 'cpu', 'instance']: stats = db.get_statistics(storage.EventFilter( user=u, meter=meter, )) if meter in ['cpu', 'disk']: total = stats['max'] else: total = stats['sum'] print ' ', meter, total