def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery=None, resource=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. """ metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {} query = {} if user is not None: query['user_id'] = user if project is not None: query['project_id'] = project if source is not None: query['source'] = source if start_timestamp or end_timestamp: return self._get_time_constrained_resources(query, start_timestamp, start_timestamp_op, end_timestamp, end_timestamp_op, metaquery, resource) else: return self._get_floating_resources(query, metaquery, resource)
def get_meters(self, user=None, project=None, resource=None, source=None, metaquery=None, limit=None): """Return an iterable of models.Meter instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param resource: Optional resource filter. :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param limit: Maximum number of results to return. """ if limit == 0: return metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {} q = {} if user == 'None': q['user_id'] = None elif user is not None: q['user_id'] = user if project == 'None': q['project_id'] = None elif project is not None: q['project_id'] = project if resource == 'None': q['_id'] = None elif resource is not None: q['_id'] = resource if source is not None: q['source'] = source q.update(metaquery) count = 0 for r in self.db.resource.find(q): for r_meter in r['meter']: if limit and count >= limit: return else: count += 1 yield models.Meter( name=r_meter['counter_name'], type=r_meter['counter_type'], # Return empty string if 'counter_unit' is not valid for # backward compatibility. unit=r_meter.get('counter_unit', ''), resource_id=r['_id'], project_id=r['project_id'], source=r['source'], user_id=r['user_id'], )
def make_query(index, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery=None, resource=None, limit=None): """Make filter query for the resource-list and meter-list requests.""" q_args = {} filters = [] if limit is not None: q_args['size'] = limit q_args['index'] = index if source is not None: filters.append({'term': {'source': source}}) if resource is not None: filters.append({'term': {'_id': resource}}) if user is not None: filters.append({'term': {'user_id': user}}) if project is not None: filters.append({'term': {'project_id': project}}) if start_timestamp or end_timestamp: ts_filter = {} st_op = start_timestamp_op or 'gte' et_op = end_timestamp_op or 'lt' if start_timestamp: ts_filter[st_op] = start_timestamp if end_timestamp: ts_filter[et_op] = end_timestamp filters.append({'range': {'last_sample_timestamp': ts_filter}}) if metaquery is not None: metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) for key, value in six.iteritems(metaquery): filters.append({'term': {key: value}}) q_args['body'] = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': filters } } } } } return q_args
def sample_to_resource(data): data = copy.deepcopy(data) data['counter_name'] = quote(data['counter_name']) data['resource_metadata'] = pymongo_utils.improve_keys( data.pop('resource_metadata')) return { 'script': ('ctx._source.meters += meter;' 'ctx._source.user_id = user_id;' 'ctx._source.project_id = project_id;' 'ctx._source.source = source;' 'ctx._source.metadata = ' 'ctx._source.last_sample_timestamp <= timestamp ? ' 'metadata : ctx._source.metadata;' 'ctx._source.last_sample_timestamp = ' 'ctx._source.last_sample_timestamp < timestamp ?' 'timestamp : ctx._source.last_sample_timestamp;' 'ctx._source.first_sample_timestamp = ' 'ctx._source.first_sample_timestamp > timestamp ?' 'timestamp : ctx._source.first_sample_timestamp;'), 'params': { 'meter': { data['counter_name']: { 'type': data['counter_type'], 'unit': data['counter_unit'] } }, 'metadata': data['resource_metadata'], 'timestamp': data['timestamp'], 'user_id': data.get('user_id', ''), 'project_id': data.get('project_id', ''), 'source': data.get('source', ''), }, 'upsert': { "first_sample_timestamp": data['timestamp'], "last_sample_timestamp": data['timestamp'], "project_id": data['project_id'], "user_id": data['user_id'], "source": data.get('source', ''), "metadata": data['resource_metadata'], "meters": { data['counter_name']: { 'type': data['counter_type'], 'unit': data['counter_unit'] } } } }
def get_meters(self, user=None, project=None, resource=None, source=None, metaquery=None, pagination=None): """Return an iterable of models.Meter instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param resource: Optional resource filter. :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param pagination: Optional pagination query. """ if pagination: raise ceilometer.NotImplementedError('Pagination not implemented') metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {} q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if resource is not None: q['_id'] = resource if source is not None: q['source'] = source q.update(metaquery) for r in self.db.resource.find(q): for r_meter in r['meter']: yield models.Meter( name=r_meter['counter_name'], type=r_meter['counter_type'], # Return empty string if 'counter_unit' is not valid for # backward compatibility. unit=r_meter.get('counter_unit', ''), resource_id=r['_id'], project_id=r['project_id'], source=r['source'], user_id=r['user_id'], )
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ # Record the updated resource metadata data = copy.deepcopy(data) data['resource_metadata'] = pymongo_utils.improve_keys( data.pop('resource_metadata')) self.db.resource.update( {'_id': data['resource_id']}, { '$set': { 'project_id': data['project_id'], 'user_id': data['user_id'] or 'null', 'metadata': data['resource_metadata'], 'source': data['source'], }, '$addToSet': { 'meter': { 'counter_name': data['counter_name'], 'counter_type': data['counter_type'], 'counter_unit': data['counter_unit'], }, }, }, upsert=True, ) # Record the raw data for the meter. Use a copy so we do not # modify a data structure owned by our caller (the driver adds # a new key '_id'). record = copy.copy(data) record['recorded_at'] = timeutils.utcnow() # Make sure that the data does have field _id which db2 wont add # automatically. if record.get('_id') is None: record['_id'] = str(bson.objectid.ObjectId()) self.db.meter.insert(record)
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ # Record the updated resource metadata data = copy.deepcopy(data) data["resource_metadata"] = pymongo_utils.improve_keys(data.pop("resource_metadata")) self.db.resource.update( {"_id": data["resource_id"]}, { "$set": { "project_id": data["project_id"], "user_id": data["user_id"] or "null", "metadata": data["resource_metadata"], "source": data["source"], }, "$addToSet": { "meter": { "counter_name": data["counter_name"], "counter_type": data["counter_type"], "counter_unit": data["counter_unit"], } }, }, upsert=True, ) # Record the raw data for the meter. Use a copy so we do not # modify a data structure owned by our caller (the driver adds # a new key '_id'). record = copy.copy(data) record["recorded_at"] = timeutils.utcnow() # Make sure that the data does have field _id which db2 wont add # automatically. if record.get("_id") is None: record["_id"] = str(bson.objectid.ObjectId()) self.db.meter.insert(record)
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ # Record the updated resource metadata data = copy.deepcopy(data) data['resource_metadata'] = pymongo_utils.improve_keys( data.pop('resource_metadata')) self.db.resource.update( {'_id': data['resource_id']}, {'$set': {'project_id': data['project_id'], 'user_id': data['user_id'] or 'null', 'metadata': data['resource_metadata'], 'source': data['source'], }, '$addToSet': {'meter': {'counter_name': data['counter_name'], 'counter_type': data['counter_type'], 'counter_unit': data['counter_unit'], }, }, }, upsert=True, ) # Record the raw data for the meter. Use a copy so we do not # modify a data structure owned by our caller (the driver adds # a new key '_id'). record = copy.copy(data) record['recorded_at'] = timeutils.utcnow() # Make sure that the data does have field _id which db2 wont add # automatically. if record.get('_id') is None: record['_id'] = str(bson.objectid.ObjectId()) self.db.meter.insert(record)
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery=None, resource=None, pagination=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise ceilometer.NotImplementedError('Pagination not implemented') metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {} q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update(dict(('resource_' + k, v) for (k, v) in six.iteritems(metaquery))) if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = pymongo_utils.make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource', 'timestamp') sort_keys.insert(0, 'resource_id') sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir='desc') resource = lambda x: x['resource_id'] meters = self.db.meter.find(q, sort=sort_instructions) for resource_id, r_meters in itertools.groupby(meters, key=resource): # Because we have to know first/last timestamp, and we need a full # list of references to the resource's meters, we need a tuple # here. r_meters = tuple(r_meters) latest_meter = r_meters[0] last_ts = latest_meter['timestamp'] first_ts = r_meters[-1]['timestamp'] yield models.Resource(resource_id=latest_meter['resource_id'], project_id=latest_meter['project_id'], first_sample_timestamp=first_ts, last_sample_timestamp=last_ts, source=latest_meter['source'], user_id=latest_meter['user_id'], metadata=pymongo_utils.unquote_keys( latest_meter['resource_metadata']))
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ # Record the updated resource metadata - we use $setOnInsert to # unconditionally insert sample timestamps and resource metadata # (in the update case, this must be conditional on the sample not # being out-of-order) data = copy.deepcopy(data) data['resource_metadata'] = pymongo_utils.improve_keys( data.pop('resource_metadata')) resource = self.db.resource.find_one_and_update( {'_id': data['resource_id']}, { '$set': { 'project_id': data['project_id'], 'user_id': data['user_id'], 'source': data['source'], }, '$setOnInsert': { 'metadata': data['resource_metadata'], 'first_sample_timestamp': data['timestamp'], 'last_sample_timestamp': data['timestamp'], }, '$addToSet': { 'meter': { 'counter_name': data['counter_name'], 'counter_type': data['counter_type'], 'counter_unit': data['counter_unit'], }, }, }, upsert=True, return_document=pymongo.ReturnDocument.AFTER, ) # only update last sample timestamp if actually later (the usual # in-order case) last_sample_timestamp = resource.get('last_sample_timestamp') if (last_sample_timestamp is None or last_sample_timestamp <= data['timestamp']): self.db.resource.update_one({'_id': data['resource_id']}, { '$set': { 'metadata': data['resource_metadata'], 'last_sample_timestamp': data['timestamp'] } }) # only update first sample timestamp if actually earlier (the unusual # out-of-order case) # NOTE: a null first sample timestamp is not updated as this indicates # a pre-existing resource document dating from before we started # recording these timestamps in the resource collection first_sample_timestamp = resource.get('first_sample_timestamp') if (first_sample_timestamp is not None and first_sample_timestamp > data['timestamp']): self.db.resource.update_one( {'_id': data['resource_id']}, {'$set': { 'first_sample_timestamp': data['timestamp'] }}) # Record the raw data for the meter. Use a copy so we do not # modify a data structure owned by our caller (the driver adds # a new key '_id'). record = copy.copy(data) record['recorded_at'] = timeutils.utcnow() self.db.meter.insert_one(record)
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ # Record the updated resource metadata - we use $setOnInsert to # unconditionally insert sample timestamps and resource metadata # (in the update case, this must be conditional on the sample not # being out-of-order) data = copy.deepcopy(data) data['resource_metadata'] = pymongo_utils.improve_keys( data.pop('resource_metadata')) resource = self.db.resource.find_one_and_update( {'_id': data['resource_id']}, {'$set': {'project_id': data['project_id'], 'user_id': data['user_id'], 'source': data['source'], }, '$setOnInsert': {'metadata': data['resource_metadata'], 'first_sample_timestamp': data['timestamp'], 'last_sample_timestamp': data['timestamp'], }, '$addToSet': {'meter': {'counter_name': data['counter_name'], 'counter_type': data['counter_type'], 'counter_unit': data['counter_unit'], }, }, }, upsert=True, return_document=pymongo.ReturnDocument.AFTER, ) # only update last sample timestamp if actually later (the usual # in-order case) last_sample_timestamp = resource.get('last_sample_timestamp') if (last_sample_timestamp is None or last_sample_timestamp <= data['timestamp']): self.db.resource.update_one( {'_id': data['resource_id']}, {'$set': {'metadata': data['resource_metadata'], 'last_sample_timestamp': data['timestamp']}} ) # only update first sample timestamp if actually earlier (the unusual # out-of-order case) # NOTE: a null first sample timestamp is not updated as this indicates # a pre-existing resource document dating from before we started # recording these timestamps in the resource collection first_sample_timestamp = resource.get('first_sample_timestamp') if (first_sample_timestamp is not None and first_sample_timestamp > data['timestamp']): self.db.resource.update_one( {'_id': data['resource_id']}, {'$set': {'first_sample_timestamp': data['timestamp']}} ) # Record the raw data for the meter. Use a copy so we do not # modify a data structure owned by our caller (the driver adds # a new key '_id'). record = copy.copy(data) record['recorded_at'] = timeutils.utcnow() self.db.meter.insert_one(record)
def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery=None, resource=None, pagination=None): """Return an iterable of models.Resource instances :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param pagination: Optional pagination query. """ if pagination: raise ceilometer.NotImplementedError('Pagination not implemented') metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {} q = {} if user is not None: q['user_id'] = user if project is not None: q['project_id'] = project if source is not None: q['source'] = source if resource is not None: q['resource_id'] = resource # Add resource_ prefix so it matches the field in the db q.update( dict(('resource_' + k, v) for (k, v) in six.iteritems(metaquery))) if start_timestamp or end_timestamp: # Look for resources matching the above criteria and with # samples in the time range we care about, then change the # resource query to return just those resources by id. ts_range = pymongo_utils.make_timestamp_range( start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op) if ts_range: q['timestamp'] = ts_range sort_keys = base._handle_sort_key('resource', 'timestamp') sort_keys.insert(0, 'resource_id') sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir='desc') resource = lambda x: x['resource_id'] meters = self.db.meter.find(q, sort=sort_instructions) for resource_id, r_meters in itertools.groupby(meters, key=resource): # Because we have to know first/last timestamp, and we need a full # list of references to the resource's meters, we need a tuple # here. r_meters = tuple(r_meters) latest_meter = r_meters[0] last_ts = latest_meter['timestamp'] first_ts = r_meters[-1]['timestamp'] yield models.Resource(resource_id=latest_meter['resource_id'], project_id=latest_meter['project_id'], first_sample_timestamp=first_ts, last_sample_timestamp=last_ts, source=latest_meter['source'], user_id=latest_meter['user_id'], metadata=pymongo_utils.unquote_keys( latest_meter['resource_metadata']))
def record_metering_data_batch(self, samples): """Record the metering data in batch. :param samples: a list of samples dict. """ # Record the updated resource metadata - we use $setOnInsert to # unconditionally insert sample timestamps and resource metadata # (in the update case, this must be conditional on the sample not # being out-of-order) # We must not store this samples = copy.deepcopy(samples) for sample in samples: sample.pop("monotonic_time", None) sorted_samples = sorted(copy.deepcopy(samples), key=lambda s: (s['resource_id'], s['timestamp'])) res_grouped_samples = itertools.groupby( sorted_samples, key=operator.itemgetter('resource_id')) samples_to_update_resource = [] for resource_id, g_samples in res_grouped_samples: g_samples = list(g_samples) g_samples[-1]['meter'] = [{ 'counter_name': s['counter_name'], 'counter_type': s['counter_type'], 'counter_unit': s['counter_unit'], } for s in g_samples] g_samples[-1]['last_sample_timestamp'] = g_samples[-1]['timestamp'] g_samples[-1]['first_sample_timestamp'] = g_samples[0]['timestamp'] samples_to_update_resource.append(g_samples[-1]) for sample in samples_to_update_resource: sample['resource_metadata'] = pymongo_utils.improve_keys( sample.pop('resource_metadata')) resource = self.db.resource.find_one_and_update( {'_id': sample['resource_id']}, { '$set': { 'project_id': sample['project_id'], 'user_id': sample['user_id'], 'source': sample['source'], }, '$setOnInsert': { 'metadata': sample['resource_metadata'], 'first_sample_timestamp': sample['timestamp'], 'last_sample_timestamp': sample['timestamp'], }, '$addToSet': { 'meter': { '$each': sample['meter'] }, }, }, upsert=True, return_document=pymongo.ReturnDocument.AFTER, ) # only update last sample timestamp if actually later (the usual # in-order case) last_sample_timestamp = resource.get('last_sample_timestamp') if (last_sample_timestamp is None or last_sample_timestamp <= sample['last_sample_timestamp']): self.db.resource.update_one({'_id': sample['resource_id']}, { '$set': { 'metadata': sample['resource_metadata'], 'last_sample_timestamp': sample['last_sample_timestamp'] } }) # only update first sample timestamp if actually earlier ( # the unusual out-of-order case) # NOTE: a null first sample timestamp is not updated as this # indicates a pre-existing resource document dating from before # we started recording these timestamps in the resource collection first_sample_timestamp = resource.get('first_sample_timestamp') if (first_sample_timestamp is not None and first_sample_timestamp > sample['first_sample_timestamp']): self.db.resource.update_one({'_id': sample['resource_id']}, { '$set': { 'first_sample_timestamp': sample['first_sample_timestamp'] } }) # Record the raw data for the meter. Use a copy so we do not # modify a data structure owned by our caller (the driver adds # a new key '_id'). record = copy.deepcopy(samples) for s in record: s['recorded_at'] = timeutils.utcnow() s['resource_metadata'] = pymongo_utils.improve_keys( s.pop('resource_metadata')) self.db.meter.insert_many(record)
def record_metering_data_batch(self, samples): """Record the metering data in batch. :param samples: a list of samples dict. """ # Record the updated resource metadata - we use $setOnInsert to # unconditionally insert sample timestamps and resource metadata # (in the update case, this must be conditional on the sample not # being out-of-order) sorted_samples = sorted( copy.deepcopy(samples), key=lambda s: (s['resource_id'], s['timestamp'])) res_grouped_samples = itertools.groupby( sorted_samples, key=operator.itemgetter('resource_id')) samples_to_update_resource = [] for resource_id, g_samples in res_grouped_samples: g_samples = list(g_samples) g_samples[-1]['meter'] = [{'counter_name': s['counter_name'], 'counter_type': s['counter_type'], 'counter_unit': s['counter_unit'], } for s in g_samples] g_samples[-1]['last_sample_timestamp'] = g_samples[-1]['timestamp'] g_samples[-1]['first_sample_timestamp'] = g_samples[0]['timestamp'] samples_to_update_resource.append(g_samples[-1]) for sample in samples_to_update_resource: sample['resource_metadata'] = pymongo_utils.improve_keys( sample.pop('resource_metadata')) resource = self.db.resource.find_one_and_update( {'_id': sample['resource_id']}, {'$set': {'project_id': sample['project_id'], 'user_id': sample['user_id'], 'source': sample['source'], }, '$setOnInsert': { 'metadata': sample['resource_metadata'], 'first_sample_timestamp': sample['timestamp'], 'last_sample_timestamp': sample['timestamp'], }, '$addToSet': { 'meter': {'$each': sample['meter']}, }, }, upsert=True, return_document=pymongo.ReturnDocument.AFTER, ) # only update last sample timestamp if actually later (the usual # in-order case) last_sample_timestamp = resource.get('last_sample_timestamp') if (last_sample_timestamp is None or last_sample_timestamp <= sample['last_sample_timestamp']): self.db.resource.update_one( {'_id': sample['resource_id']}, {'$set': {'metadata': sample['resource_metadata'], 'last_sample_timestamp': sample['last_sample_timestamp']}} ) # only update first sample timestamp if actually earlier ( # the unusual out-of-order case) # NOTE: a null first sample timestamp is not updated as this # indicates a pre-existing resource document dating from before # we started recording these timestamps in the resource collection first_sample_timestamp = resource.get('first_sample_timestamp') if (first_sample_timestamp is not None and first_sample_timestamp > sample['first_sample_timestamp']): self.db.resource.update_one( {'_id': sample['resource_id']}, {'$set': {'first_sample_timestamp': sample['first_sample_timestamp']}} ) # Record the raw data for the meter. Use a copy so we do not # modify a data structure owned by our caller (the driver adds # a new key '_id'). record = copy.deepcopy(samples) for s in record: s['recorded_at'] = timeutils.utcnow() s['resource_metadata'] = pymongo_utils.improve_keys( s.pop('resource_metadata')) self.db.meter.insert_many(record)