def test_handle_sort_key(self): sort_keys_meter = base._handle_sort_key('meter', 'foo') self.assertEqual(['foo', 'user_id', 'project_id'], sort_keys_meter) sort_keys_resource = base._handle_sort_key('resource', 'project_id') self.assertEqual(['project_id', 'user_id', 'timestamp'], sort_keys_resource)
def test_handle_sort_key(self): sort_keys_alarm = base._handle_sort_key('alarm') self.assertEqual(sort_keys_alarm, ['name', 'user_id', 'project_id']) sort_keys_meter = base._handle_sort_key('meter', 'foo') self.assertEqual(sort_keys_meter, ['foo', 'user_id', 'project_id']) sort_keys_resource = base._handle_sort_key('resource', 'project_id') self.assertEqual(sort_keys_resource, ['project_id', 'user_id'])
def _get_floating_resources(self, query, metaquery, resource): """Return an iterable of models.Resource instances unconstrained by timestamp. :param query: project/user/source query :param metaquery: dict with metadata to match on. :param resource: resource filter. """ if resource is not None: query['_id'] = resource query.update(dict((k, v) for (k, v) in metaquery.iteritems())) keys = base._handle_sort_key('resource') sort_keys = ['last_sample_timestamp' if i == 'timestamp' else i for i in keys] sort_instructions = self._build_sort_instructions(sort_keys)[0] for r in self.db.resource.find(query, sort=sort_instructions): yield models.Resource( resource_id=r['_id'], user_id=r['user_id'], project_id=r['project_id'], first_sample_timestamp=r.get('first_sample_timestamp', self._GENESIS), last_sample_timestamp=r.get('last_sample_timestamp', self._APOCALYPSE), source=r['source'], metadata=r['metadata'])
def _get_floating_resources(self, query, metaquery, resource): """Return an iterable of models.Resource instances Items are unconstrained by timestamp. :param query: project/user/source query :param metaquery: dict with metadata to match on. :param resource: resource filter. """ if resource is not None: query["_id"] = resource query.update(dict((k, v) for (k, v) in six.iteritems(metaquery))) keys = base._handle_sort_key("resource") sort_keys = ["last_sample_timestamp" if i == "timestamp" else i for i in keys] sort_instructions = self._build_sort_instructions(sort_keys)[0] for r in self.db.resource.find(query, sort=sort_instructions): yield models.Resource( resource_id=r["_id"], user_id=r["user_id"], project_id=r["project_id"], first_sample_timestamp=r.get("first_sample_timestamp", self._GENESIS), last_sample_timestamp=r.get("last_sample_timestamp", self._APOCALYPSE), source=r["source"], metadata=r["metadata"], )
def _get_time_constrained_resources( self, query, start_timestamp, start_timestamp_op, end_timestamp, end_timestamp_op, metaquery, resource ): """Return an iterable of models.Resource instances Items are constrained by sample timestamp. :param query: project/user/source query :param start_timestamp: modified timestamp start range. :param start_timestamp_op: start time operator, like gt, ge. :param end_timestamp: modified timestamp end range. :param end_timestamp_op: end time operator, like lt, le. :param metaquery: dict with metadata to match on. :param resource: resource filter. """ if resource is not None: query["resource_id"] = resource # Add resource_ prefix so it matches the field in the db query.update(dict(("resource_" + k, v) for (k, v) in six.iteritems(metaquery))) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = pymongo_utils.make_timestamp_range( start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op ) if ts_range: query["timestamp"] = ts_range sort_keys = base._handle_sort_key("resource") sort_instructions = self._build_sort_instructions(sort_keys)[0] # use a unique collection name for the results collection, # as result post-sorting (as oppposed to reduce pre-sorting) # is not possible on an inline M-R out = "resource_list_%s" % uuid.uuid4() self.db.meter.map_reduce( self.MAP_RESOURCES, self.REDUCE_RESOURCES, out=out, sort={"resource_id": 1}, query=query ) try: for r in self.db[out].find(sort=sort_instructions): resource = r["value"] yield models.Resource( resource_id=r["_id"], user_id=resource["user_id"], project_id=resource["project_id"], first_sample_timestamp=resource["first_timestamp"], last_sample_timestamp=resource["last_timestamp"], source=resource["source"], metadata=resource["metadata"], ) finally: self.db[out].drop()
def get_meters(self, user=None, project=None, resource=None, source=None, metaquery={}, limit=None, marker_pairs=None, sort_key=None, sort_dir=None): """Return an iterable of models.Meter instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param resource: Optional resource filter. :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param limit: Number of documents should be returned. :param marker_pairs: Attribute-value pairs to identify the last item of the previous page. :param sort_key: Attribute by which results be sorted. :param sort_dir: Direction with which results be sorted(asc, desc). """ q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if resource is not None: q['_id'] = resource if source is not None: q['source'] = source q.update(metaquery) marker = self._get_marker(self.db.resource, marker_pairs=marker_pairs) sort_keys = base._handle_sort_key('meter', sort_key) for r in self.paginate_query(q, self.db.resource, limit=limit, marker=marker, sort_keys=sort_keys, sort_dir=sort_dir): for r_meter in r['meter']: yield models.Meter( name=r_meter['counter_name'], type=r_meter['counter_type'], # Return empty string if 'counter_unit' is not valid for # backward compatibility. unit=r_meter.get('counter_unit', ''), resource_id=r['_id'], project_id=r['project_id'], source=r['source'], user_id=r['user_id'], )
def get_alarms(self, name=None, user=None, project=None, enabled=True, alarm_id=None, limit=None, marker_pairs=None, sort_key=None, sort_dir=None): """Yields a lists of alarms that match filters :param name: The Alarm name. :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param enabled: Optional boolean to list disable alarm. :param alarm_id: Optional alarm_id to return one alarm. :param limit: Number of rows should be returned. :param marker_pairs: Attribute-value pairs to identify the last item of the previous page. :param sort_key: Attribute by which results be sorted. :param sort_dir: Direction with which results be sorted(asc, desc). """ q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if name is not None: q['name'] = name if enabled is not None: q['enabled'] = enabled if alarm_id is not None: q['alarm_id'] = alarm_id marker = self._get_marker(self.db.alarm, marker_pairs=marker_pairs) sort_keys = base._handle_sort_key('alarm', sort_key) for alarm in self.paginate_query(q, self.db.alarm, limit=limit, marker=marker, sort_keys=sort_keys, sort_dir=sort_dir): a = {} a.update(alarm) del a['_id'] a['matching_metadata'] = \ self._decode_matching_metadata(a['matching_metadata']) yield models.Alarm(**a)
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None, pagination=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise NotImplementedError(_('Pagination not implemented')) q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict(('resource_' + k, v) for (k, v) in metaquery.iteritems())) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource') sort_instructions = self._build_sort_instructions(sort_keys)[0] aggregate = self.db.meter.aggregate([ {"$match": q}, {"$sort": dict(sort_instructions)}, {"$group": { "_id": "$resource_id", "user_id": {"$first": "$user_id"}, "project_id": {"$first": "$project_id"}, "source": {"$first": "$source"}, "first_sample_timestamp": {"$min": "$timestamp"}, "last_sample_timestamp": {"$max": "$timestamp"}, "metadata": {"$first": "$resource_metadata"}, "meters_name": {"$push": "$counter_name"}, "meters_type": {"$push": "$counter_type"}, "meters_unit": {"$push": "$counter_unit"}, }}, ]) for result in aggregate['result']: yield models.Resource( resource_id=result['_id'], user_id=result['user_id'], project_id=result['project_id'], first_sample_timestamp=result['first_sample_timestamp'], last_sample_timestamp=result['last_sample_timestamp'], source=result['source'], metadata=result['metadata'], meter=[ models.ResourceMeter( counter_name=m_n, counter_type=m_t, counter_unit=m_u, ) for m_n, m_u, m_t in zip(result['meters_name'], result['meters_unit'], result['meters_type']) ], )
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery=None, resource=None, pagination=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise ceilometer.NotImplementedError('Pagination not implemented') metaquery = metaquery or {} q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update( dict(('resource_' + k, v) for (k, v) in six.iteritems(metaquery))) if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = pymongo_utils.make_timestamp_range( start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource', 'timestamp') sort_keys.insert(0, 'resource_id') sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir='desc') resource = lambda x: x['resource_id'] meters = self.db.meter.find(q, sort=sort_instructions) for resource_id, r_meters in itertools.groupby(meters, key=resource): # Because we have to know first/last timestamp, and we need a full # list of references to the resource's meters, we need a tuple # here. r_meters = tuple(r_meters) latest_meter = r_meters[0] last_ts = latest_meter['timestamp'] first_ts = r_meters[-1]['timestamp'] yield models.Resource(resource_id=latest_meter['resource_id'], project_id=latest_meter['project_id'], first_sample_timestamp=first_ts, last_sample_timestamp=last_ts, source=latest_meter['source'], user_id=latest_meter['user_id'], metadata=latest_meter['resource_metadata'])
def _get_time_constrained_resources(self, query, start_timestamp, start_timestamp_op, end_timestamp, end_timestamp_op, metaquery, resource): """Return an iterable of models.Resource instances constrained by sample timestamp. :param query: project/user/source query :param start_timestamp: modified timestamp start range. :param start_timestamp_op: start time operator, like gt, ge. :param end_timestamp: modified timestamp end range. :param end_timestamp_op: end time operator, like lt, le. :param metaquery: dict with metadata to match on. :param resource: resource filter. """ if resource is not None: query['resource_id'] = resource # Add resource_ prefix so it matches the field in the db query.update(dict(('resource_' + k, v) for (k, v) in metaquery.iteritems())) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = pymongo_base.make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: query['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource') sort_instructions = self._build_sort_instructions(sort_keys)[0] # use a unique collection name for the results collection, # as result post-sorting (as oppposed to reduce pre-sorting) # is not possible on an inline M-R out = 'resource_list_%s' % uuid.uuid4() self.db.meter.map_reduce(self.MAP_RESOURCES, self.REDUCE_RESOURCES, out=out, sort={'resource_id': 1}, query=query) try: for r in self.db[out].find(sort=sort_instructions): resource = r['value'] yield models.Resource( resource_id=r['_id'], user_id=resource['user_id'], project_id=resource['project_id'], first_sample_timestamp=resource['first_timestamp'], last_sample_timestamp=resource['last_timestamp'], source=resource['source'], metadata=resource['metadata']) finally: self.db[out].drop()
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None, pagination=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise NotImplementedError(_('Pagination not implemented')) q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict( ('resource_' + k, v) for (k, v) in metaquery.iteritems())) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource') sort_instructions = self._build_sort_instructions(sort_keys)[0] # use a unique collection name for the results collection, # as result post-sorting (as oppposed to reduce pre-sorting) # is not possible on an inline M-R out = 'resource_list_%s' % uuid.uuid4() self.db.meter.map_reduce(self.MAP_RESOURCES, self.REDUCE_RESOURCES, out=out, sort={'resource_id': 1}, query=q) try: for r in self.db[out].find(sort=sort_instructions): resource = r['value'] yield models.Resource( resource_id=r['_id'], user_id=resource['user_id'], project_id=resource['project_id'], first_sample_timestamp=resource['first_timestamp'], last_sample_timestamp=resource['last_timestamp'], source=resource['source'], metadata=resource['metadata']) finally: self.db[out].drop()
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None, limit=None, marker_pairs=None, sort_key=None, sort_dir=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param limit: Number of documents should be returned. :param marker_pairs: Attribute-value pairs to identify the last item of the previous page. :param sort_key: Attribute by which results be sorted. :param sort_dir: Direction with which results be sorted(asc, desc). """ q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict(('resource_' + k, v) for (k, v) in metaquery.iteritems())) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range marker = self._get_marker(self.db.resource, marker_pairs=marker_pairs) sort_keys = base._handle_sort_key('resource', sort_key) # FIXME(jd): We should use self.db.meter.group() and not use the # resource collection, but that's not supported by MIM, so it's not # easily testable yet. Since it was bugged before anyway, it's still # better for now. resource_ids = self.db.meter.find(q).distinct('resource_id') q = {'_id': {'$in': resource_ids}} for resource in self.paginate_query(q, self.db.resource, limit=limit, marker=marker, sort_keys=sort_keys, sort_dir=sort_dir): yield models.Resource( resource_id=resource['_id'], project_id=resource['project_id'], source=resource['source'], user_id=resource['user_id'], metadata=resource['metadata'], meter=[ models.ResourceMeter( counter_name=meter['counter_name'], counter_type=meter['counter_type'], counter_unit=meter.get('counter_unit', ''), ) for meter in resource['meter'] ], )
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None, limit=None, marker_pairs=None, sort_key=None, sort_dir=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param limit: Number of documents should be returned. :param marker_pairs: Attribute-value pairs to identify the last item of the previous page. :param sort_key: Attribute by which results be sorted. :param sort_dir: Direction with which results be sorted(asc, desc). """ if marker_pairs: raise NotImplementedError( "Cannot use marker pairs in resource listing, not implemented") q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict( ('resource_' + k, v) for (k, v) in metaquery.iteritems())) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource', sort_key) sort_instruction, query = self._build_paginate_query( None, sort_keys, sort_dir) q.update(query) aggregate = self.db.meter.aggregate([ { "$match": q }, { "$sort": dict(sort_instruction) }, { "$group": { "_id": "$resource_id", "user_id": { "$first": "$user_id" }, "project_id": { "$first": "$project_id" }, "source": { "$first": "$source" }, "first_sample_timestamp": { "$min": "$timestamp" }, "last_sample_timestamp": { "$max": "$timestamp" }, "metadata": { "$first": "$resource_metadata" }, "meters_name": { "$push": "$counter_name" }, "meters_type": { "$push": "$counter_type" }, "meters_unit": { "$push": "$counter_unit" }, } }, ]) for result in aggregate['result']: if limit is not None: if limit == 0: break limit -= 1 yield models.Resource( resource_id=result['_id'], user_id=result['user_id'], project_id=result['project_id'], first_sample_timestamp=result['first_sample_timestamp'], last_sample_timestamp=result['last_sample_timestamp'], source=result['source'], metadata=result['metadata'], meter=[ models.ResourceMeter( counter_name=m_n, counter_type=m_t, counter_unit=m_u, ) for m_n, m_u, m_t in zip(result['meters_name'], result['meters_unit'], result['meters_type']) ], )
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None, limit=None, marker_pairs=None, sort_key=None, sort_dir=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param limit: Number of documents should be returned. :param marker_pairs: Attribute-value pairs to identify the last item of the previous page. :param sort_key: Attribute by which results be sorted. :param sort_dir: Direction with which results be sorted(asc, desc). """ if marker_pairs is not None: raise NotImplementedError( "Cannot use marker pairs in resource listing") q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict(('resource_' + k, v) for (k, v) in metaquery.iteritems())) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource', sort_key) sort_instruction, query = self._build_paginate_query(None, sort_keys, sort_dir) q.update(query) aggregate = self.db.meter.aggregate([ {"$match": q}, {"$sort": dict(sort_instruction)}, {"$group": { "_id": "$resource_id", "user_id": {"$first": "$user_id"}, "project_id": {"$first": "$project_id"}, "source": {"$first": "$source"}, "first_sample_timestamp": {"$min": "$timestamp"}, "last_sample_timestamp": {"$max": "$timestamp"}, "metadata": {"$first": "$resource_metadata"}, "meters_name": {"$push": "$counter_name"}, "meters_type": {"$push": "$counter_type"}, "meters_unit": {"$push": "$counter_unit"}, }}, ]) for result in aggregate['result']: if limit is not None: if limit == 0: break limit -= 1 yield models.Resource( resource_id=result['_id'], user_id=result['user_id'], project_id=result['project_id'], first_sample_timestamp=result['first_sample_timestamp'], last_sample_timestamp=result['last_sample_timestamp'], source=result['source'], metadata=result['metadata'], meter=[ models.ResourceMeter( counter_name=m_n, counter_type=m_t, counter_unit=m_u, ) for m_n, m_u, m_t in zip(result['meters_name'], result['meters_unit'], result['meters_type']) ], )
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery=None, resource=None, pagination=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise ceilometer.NotImplementedError('Pagination not implemented') metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {} q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict(('resource_' + k, v) for (k, v) in six.iteritems(metaquery))) if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = pymongo_utils.make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource', 'timestamp') sort_keys.insert(0, 'resource_id') sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir='desc') resource = lambda x: x['resource_id'] meters = self.db.meter.find(q, sort=sort_instructions) for resource_id, r_meters in itertools.groupby(meters, key=resource): # Because we have to know first/last timestamp, and we need a full # list of references to the resource's meters, we need a tuple # here. r_meters = tuple(r_meters) latest_meter = r_meters[0] last_ts = latest_meter['timestamp'] first_ts = r_meters[-1]['timestamp'] yield models.Resource(resource_id=latest_meter['resource_id'], project_id=latest_meter['project_id'], first_sample_timestamp=first_ts, last_sample_timestamp=last_ts, source=latest_meter['source'], user_id=latest_meter['user_id'], metadata=pymongo_utils.unquote_keys( latest_meter['resource_metadata']))
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None, pagination=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise NotImplementedError(_('Pagination not implemented')) q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict(('resource_' + k, v) for (k, v) in metaquery.iteritems())) # FIXME(dhellmann): This may not perform very well, # but doing any better will require changing the database # schema and that will need more thought than I have time # to put into it today. if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource') sort_instructions = self._build_sort_instructions(sort_keys)[0] # use a unique collection name for the results collection, # as result post-sorting (as oppposed to reduce pre-sorting) # is not possible on an inline M-R out = 'resource_list_%s' % uuid.uuid4() self.db.meter.map_reduce(self.MAP_RESOURCES, self.REDUCE_RESOURCES, out=out, sort={'resource_id': 1}, query=q) try: for r in self.db[out].find(sort=sort_instructions): resource = r['value'] yield models.Resource( resource_id=r['_id'], user_id=resource['user_id'], project_id=resource['project_id'], first_sample_timestamp=resource['first_timestamp'], last_sample_timestamp=resource['last_timestamp'], source=resource['source'], metadata=resource['metadata']) finally: self.db[out].drop()
def get_resources( self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery={}, resource=None, pagination=None, ): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise NotImplementedError(_("Pagination not implemented")) q = {} if user is not None: q["user_id"] = user if project is not None: q["project_id"] = project if source is not None: q["source"] = source if resource is not None: q["resource_id"] = resource # Add resource_ prefix so it matches the field in the db q.update(dict(("resource_" + k, v) for (k, v) in metaquery.iteritems())) if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q["timestamp"] = ts_range sort_keys = base._handle_sort_key("resource", "timestamp") sort_keys.insert(0, "resource_id") sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir="desc") resource = lambda x: x["resource_id"] meters = self.db.meter.find(q, sort=sort_instructions) for resource_id, r_meters in itertools.groupby(meters, key=resource): resource_meters = [] # Because we have to know first/last timestamp, and we need a full # list of references to the resource's meters, we need a tuple # here. r_meters = tuple(r_meters) for meter in r_meters: resource_meters.append( models.ResourceMeter( counter_name=meter["counter_name"], counter_type=meter["counter_type"], counter_unit=meter.get("counter_unit", ""), ) ) latest_meter = r_meters[0] last_ts = latest_meter["timestamp"] first_ts = r_meters[-1]["timestamp"] yield models.Resource( resource_id=latest_meter["resource_id"], project_id=latest_meter["project_id"], first_sample_timestamp=first_ts, last_sample_timestamp=last_ts, source=latest_meter["source"], user_id=latest_meter["user_id"], metadata=latest_meter["resource_metadata"], meter=resource_meters, )