Ejemplo n.º 1
0
    def test_handle_sort_key(self):
        sort_keys_meter = base._handle_sort_key('meter', 'foo')
        self.assertEqual(['foo', 'user_id', 'project_id'], sort_keys_meter)

        sort_keys_resource = base._handle_sort_key('resource', 'project_id')
        self.assertEqual(['project_id', 'user_id', 'timestamp'],
                         sort_keys_resource)
Ejemplo n.º 2
0
    def test_handle_sort_key(self):
        sort_keys_alarm = base._handle_sort_key('alarm')
        self.assertEqual(sort_keys_alarm, ['name', 'user_id', 'project_id'])

        sort_keys_meter = base._handle_sort_key('meter', 'foo')
        self.assertEqual(sort_keys_meter, ['foo', 'user_id', 'project_id'])

        sort_keys_resource = base._handle_sort_key('resource', 'project_id')
        self.assertEqual(sort_keys_resource, ['project_id', 'user_id'])
Ejemplo n.º 3
0
    def test_handle_sort_key(self):
        sort_keys_alarm = base._handle_sort_key('alarm')
        self.assertEqual(sort_keys_alarm, ['name', 'user_id', 'project_id'])

        sort_keys_meter = base._handle_sort_key('meter', 'foo')
        self.assertEqual(sort_keys_meter, ['foo', 'user_id', 'project_id'])

        sort_keys_resource = base._handle_sort_key('resource', 'project_id')
        self.assertEqual(sort_keys_resource, ['project_id', 'user_id'])
Ejemplo n.º 4
0
    def _get_floating_resources(self, query, metaquery, resource):
        """Return an iterable of models.Resource instances unconstrained
           by timestamp.

        :param query: project/user/source query
        :param metaquery: dict with metadata to match on.
        :param resource: resource filter.
        """
        if resource is not None:
            query['_id'] = resource

        query.update(dict((k, v)
                          for (k, v) in metaquery.iteritems()))

        keys = base._handle_sort_key('resource')
        sort_keys = ['last_sample_timestamp' if i == 'timestamp' else i
                     for i in keys]
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        for r in self.db.resource.find(query, sort=sort_instructions):
            yield models.Resource(
                resource_id=r['_id'],
                user_id=r['user_id'],
                project_id=r['project_id'],
                first_sample_timestamp=r.get('first_sample_timestamp',
                                             self._GENESIS),
                last_sample_timestamp=r.get('last_sample_timestamp',
                                            self._APOCALYPSE),
                source=r['source'],
                metadata=r['metadata'])
Ejemplo n.º 5
0
    def _get_floating_resources(self, query, metaquery, resource):
        """Return an iterable of models.Resource instances

        Items are unconstrained by timestamp.
        :param query: project/user/source query
        :param metaquery: dict with metadata to match on.
        :param resource: resource filter.
        """
        if resource is not None:
            query["_id"] = resource

        query.update(dict((k, v) for (k, v) in six.iteritems(metaquery)))

        keys = base._handle_sort_key("resource")
        sort_keys = ["last_sample_timestamp" if i == "timestamp" else i for i in keys]
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        for r in self.db.resource.find(query, sort=sort_instructions):
            yield models.Resource(
                resource_id=r["_id"],
                user_id=r["user_id"],
                project_id=r["project_id"],
                first_sample_timestamp=r.get("first_sample_timestamp", self._GENESIS),
                last_sample_timestamp=r.get("last_sample_timestamp", self._APOCALYPSE),
                source=r["source"],
                metadata=r["metadata"],
            )
Ejemplo n.º 6
0
    def _get_time_constrained_resources(
        self, query, start_timestamp, start_timestamp_op, end_timestamp, end_timestamp_op, metaquery, resource
    ):
        """Return an iterable of models.Resource instances

        Items are constrained by sample timestamp.
        :param query: project/user/source query
        :param start_timestamp: modified timestamp start range.
        :param start_timestamp_op: start time operator, like gt, ge.
        :param end_timestamp: modified timestamp end range.
        :param end_timestamp_op: end time operator, like lt, le.
        :param metaquery: dict with metadata to match on.
        :param resource: resource filter.
        """
        if resource is not None:
            query["resource_id"] = resource

        # Add resource_ prefix so it matches the field in the db
        query.update(dict(("resource_" + k, v) for (k, v) in six.iteritems(metaquery)))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        # Look for resources matching the above criteria and with
        # samples in the time range we care about, then change the
        # resource query to return just those resources by id.
        ts_range = pymongo_utils.make_timestamp_range(
            start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op
        )
        if ts_range:
            query["timestamp"] = ts_range

        sort_keys = base._handle_sort_key("resource")
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        # use a unique collection name for the results collection,
        # as result post-sorting (as oppposed to reduce pre-sorting)
        # is not possible on an inline M-R
        out = "resource_list_%s" % uuid.uuid4()
        self.db.meter.map_reduce(
            self.MAP_RESOURCES, self.REDUCE_RESOURCES, out=out, sort={"resource_id": 1}, query=query
        )

        try:
            for r in self.db[out].find(sort=sort_instructions):
                resource = r["value"]
                yield models.Resource(
                    resource_id=r["_id"],
                    user_id=resource["user_id"],
                    project_id=resource["project_id"],
                    first_sample_timestamp=resource["first_timestamp"],
                    last_sample_timestamp=resource["last_timestamp"],
                    source=resource["source"],
                    metadata=resource["metadata"],
                )
        finally:
            self.db[out].drop()
Ejemplo n.º 7
0
    def get_meters(self,
                   user=None,
                   project=None,
                   resource=None,
                   source=None,
                   metaquery={},
                   limit=None,
                   marker_pairs=None,
                   sort_key=None,
                   sort_dir=None):
        """Return an iterable of models.Meter instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param resource: Optional resource filter.
        :param source: Optional source filter.
        :param metaquery: Optional dict with metadata to match on.
        :param limit: Number of documents should be returned.
        :param marker_pairs: Attribute-value pairs to identify the last item of
                             the previous page.
        :param sort_key: Attribute by which results be sorted.
        :param sort_dir: Direction with which results be sorted(asc, desc).
        """
        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if resource is not None:
            q['_id'] = resource
        if source is not None:
            q['source'] = source
        q.update(metaquery)

        marker = self._get_marker(self.db.resource, marker_pairs=marker_pairs)
        sort_keys = base._handle_sort_key('meter', sort_key)

        for r in self.paginate_query(q,
                                     self.db.resource,
                                     limit=limit,
                                     marker=marker,
                                     sort_keys=sort_keys,
                                     sort_dir=sort_dir):
            for r_meter in r['meter']:
                yield models.Meter(
                    name=r_meter['counter_name'],
                    type=r_meter['counter_type'],
                    # Return empty string if 'counter_unit' is not valid for
                    # backward compatibility.
                    unit=r_meter.get('counter_unit', ''),
                    resource_id=r['_id'],
                    project_id=r['project_id'],
                    source=r['source'],
                    user_id=r['user_id'],
                )
Ejemplo n.º 8
0
    def get_alarms(self,
                   name=None,
                   user=None,
                   project=None,
                   enabled=True,
                   alarm_id=None,
                   limit=None,
                   marker_pairs=None,
                   sort_key=None,
                   sort_dir=None):
        """Yields a lists of alarms that match filters
        :param name: The Alarm name.
        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param enabled: Optional boolean to list disable alarm.
        :param alarm_id: Optional alarm_id to return one alarm.
        :param limit: Number of rows should be returned.
        :param marker_pairs: Attribute-value pairs to identify the last item of
                            the previous page.
        :param sort_key: Attribute by which results be sorted.
        :param sort_dir: Direction with which results be sorted(asc, desc).
        """
        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if name is not None:
            q['name'] = name
        if enabled is not None:
            q['enabled'] = enabled
        if alarm_id is not None:
            q['alarm_id'] = alarm_id

        marker = self._get_marker(self.db.alarm, marker_pairs=marker_pairs)
        sort_keys = base._handle_sort_key('alarm', sort_key)

        for alarm in self.paginate_query(q,
                                         self.db.alarm,
                                         limit=limit,
                                         marker=marker,
                                         sort_keys=sort_keys,
                                         sort_dir=sort_dir):
            a = {}
            a.update(alarm)
            del a['_id']
            a['matching_metadata'] = \
                self._decode_matching_metadata(a['matching_metadata'])
            yield models.Alarm(**a)
Ejemplo n.º 9
0
    def get_meters(self, user=None, project=None, resource=None, source=None,
                   metaquery={}, limit=None, marker_pairs=None, sort_key=None,
                   sort_dir=None):
        """Return an iterable of models.Meter instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param resource: Optional resource filter.
        :param source: Optional source filter.
        :param metaquery: Optional dict with metadata to match on.
        :param limit: Number of documents should be returned.
        :param marker_pairs: Attribute-value pairs to identify the last item of
                             the previous page.
        :param sort_key: Attribute by which results be sorted.
        :param sort_dir: Direction with which results be sorted(asc, desc).
        """
        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if resource is not None:
            q['_id'] = resource
        if source is not None:
            q['source'] = source
        q.update(metaquery)

        marker = self._get_marker(self.db.resource, marker_pairs=marker_pairs)
        sort_keys = base._handle_sort_key('meter', sort_key)

        for r in self.paginate_query(q, self.db.resource, limit=limit,
                                     marker=marker,
                                     sort_keys=sort_keys, sort_dir=sort_dir):
            for r_meter in r['meter']:
                yield models.Meter(
                    name=r_meter['counter_name'],
                    type=r_meter['counter_type'],
                    # Return empty string if 'counter_unit' is not valid for
                    # backward compatibility.
                    unit=r_meter.get('counter_unit', ''),
                    resource_id=r['_id'],
                    project_id=r['project_id'],
                    source=r['source'],
                    user_id=r['user_id'],
                )
Ejemplo n.º 10
0
    def get_alarms(self, name=None, user=None,
                   project=None, enabled=True, alarm_id=None, limit=None,
                   marker_pairs=None, sort_key=None, sort_dir=None):
        """Yields a lists of alarms that match filters
        :param name: The Alarm name.
        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param enabled: Optional boolean to list disable alarm.
        :param alarm_id: Optional alarm_id to return one alarm.
        :param limit: Number of rows should be returned.
        :param marker_pairs: Attribute-value pairs to identify the last item of
                            the previous page.
        :param sort_key: Attribute by which results be sorted.
        :param sort_dir: Direction with which results be sorted(asc, desc).
        """
        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if name is not None:
            q['name'] = name
        if enabled is not None:
            q['enabled'] = enabled
        if alarm_id is not None:
            q['alarm_id'] = alarm_id

        marker = self._get_marker(self.db.alarm, marker_pairs=marker_pairs)
        sort_keys = base._handle_sort_key('alarm', sort_key)

        for alarm in self.paginate_query(q, self.db.alarm, limit=limit,
                                         marker=marker, sort_keys=sort_keys,
                                         sort_dir=sort_dir):
            a = {}
            a.update(alarm)
            del a['_id']
            a['matching_metadata'] = \
                self._decode_matching_metadata(a['matching_metadata'])
            yield models.Alarm(**a)
Ejemplo n.º 11
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery={}, resource=None, pagination=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise NotImplementedError(_('Pagination not implemented'))

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(('resource_' + k, v)
                      for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp,
                                            start_timestamp_op,
                                            end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource')
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        aggregate = self.db.meter.aggregate([
            {"$match": q},
            {"$sort": dict(sort_instructions)},
            {"$group": {
                "_id": "$resource_id",
                "user_id": {"$first": "$user_id"},
                "project_id": {"$first": "$project_id"},
                "source": {"$first": "$source"},
                "first_sample_timestamp": {"$min": "$timestamp"},
                "last_sample_timestamp": {"$max": "$timestamp"},
                "metadata": {"$first": "$resource_metadata"},
                "meters_name": {"$push": "$counter_name"},
                "meters_type": {"$push": "$counter_type"},
                "meters_unit": {"$push": "$counter_unit"},
            }},
        ])

        for result in aggregate['result']:
            yield models.Resource(
                resource_id=result['_id'],
                user_id=result['user_id'],
                project_id=result['project_id'],
                first_sample_timestamp=result['first_sample_timestamp'],
                last_sample_timestamp=result['last_sample_timestamp'],
                source=result['source'],
                metadata=result['metadata'],
                meter=[
                    models.ResourceMeter(
                        counter_name=m_n,
                        counter_type=m_t,
                        counter_unit=m_u,
                    )
                    for m_n, m_u, m_t in zip(result['meters_name'],
                                             result['meters_unit'],
                                             result['meters_type'])
                ],
            )
Ejemplo n.º 12
0
    def get_resources(self,
                      user=None,
                      project=None,
                      source=None,
                      start_timestamp=None,
                      start_timestamp_op=None,
                      end_timestamp=None,
                      end_timestamp_op=None,
                      metaquery=None,
                      resource=None,
                      pagination=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise ceilometer.NotImplementedError('Pagination not implemented')

        metaquery = metaquery or {}

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(
            dict(('resource_' + k, v) for (k, v) in six.iteritems(metaquery)))

        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = pymongo_utils.make_timestamp_range(
                start_timestamp, end_timestamp, start_timestamp_op,
                end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource', 'timestamp')
        sort_keys.insert(0, 'resource_id')
        sort_instructions = self._build_sort_instructions(sort_keys=sort_keys,
                                                          sort_dir='desc')
        resource = lambda x: x['resource_id']
        meters = self.db.meter.find(q, sort=sort_instructions)
        for resource_id, r_meters in itertools.groupby(meters, key=resource):
            # Because we have to know first/last timestamp, and we need a full
            # list of references to the resource's meters, we need a tuple
            # here.
            r_meters = tuple(r_meters)
            latest_meter = r_meters[0]
            last_ts = latest_meter['timestamp']
            first_ts = r_meters[-1]['timestamp']

            yield models.Resource(resource_id=latest_meter['resource_id'],
                                  project_id=latest_meter['project_id'],
                                  first_sample_timestamp=first_ts,
                                  last_sample_timestamp=last_ts,
                                  source=latest_meter['source'],
                                  user_id=latest_meter['user_id'],
                                  metadata=latest_meter['resource_metadata'])
Ejemplo n.º 13
0
    def _get_time_constrained_resources(self, query,
                                        start_timestamp, start_timestamp_op,
                                        end_timestamp, end_timestamp_op,
                                        metaquery, resource):
        """Return an iterable of models.Resource instances constrained
           by sample timestamp.

        :param query: project/user/source query
        :param start_timestamp: modified timestamp start range.
        :param start_timestamp_op: start time operator, like gt, ge.
        :param end_timestamp: modified timestamp end range.
        :param end_timestamp_op: end time operator, like lt, le.
        :param metaquery: dict with metadata to match on.
        :param resource: resource filter.
        """
        if resource is not None:
            query['resource_id'] = resource

        # Add resource_ prefix so it matches the field in the db
        query.update(dict(('resource_' + k, v)
                          for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        # Look for resources matching the above criteria and with
        # samples in the time range we care about, then change the
        # resource query to return just those resources by id.
        ts_range = pymongo_base.make_timestamp_range(start_timestamp,
                                                     end_timestamp,
                                                     start_timestamp_op,
                                                     end_timestamp_op)
        if ts_range:
            query['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource')
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        # use a unique collection name for the results collection,
        # as result post-sorting (as oppposed to reduce pre-sorting)
        # is not possible on an inline M-R
        out = 'resource_list_%s' % uuid.uuid4()
        self.db.meter.map_reduce(self.MAP_RESOURCES,
                                 self.REDUCE_RESOURCES,
                                 out=out,
                                 sort={'resource_id': 1},
                                 query=query)

        try:
            for r in self.db[out].find(sort=sort_instructions):
                resource = r['value']
                yield models.Resource(
                    resource_id=r['_id'],
                    user_id=resource['user_id'],
                    project_id=resource['project_id'],
                    first_sample_timestamp=resource['first_timestamp'],
                    last_sample_timestamp=resource['last_timestamp'],
                    source=resource['source'],
                    metadata=resource['metadata'])
        finally:
            self.db[out].drop()
Ejemplo n.º 14
0
    def get_resources(self,
                      user=None,
                      project=None,
                      source=None,
                      start_timestamp=None,
                      start_timestamp_op=None,
                      end_timestamp=None,
                      end_timestamp_op=None,
                      metaquery={},
                      resource=None,
                      pagination=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise NotImplementedError(_('Pagination not implemented'))

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(
            ('resource_' + k, v) for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp,
                                            start_timestamp_op,
                                            end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource')
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        # use a unique collection name for the results collection,
        # as result post-sorting (as oppposed to reduce pre-sorting)
        # is not possible on an inline M-R
        out = 'resource_list_%s' % uuid.uuid4()
        self.db.meter.map_reduce(self.MAP_RESOURCES,
                                 self.REDUCE_RESOURCES,
                                 out=out,
                                 sort={'resource_id': 1},
                                 query=q)

        try:
            for r in self.db[out].find(sort=sort_instructions):
                resource = r['value']
                yield models.Resource(
                    resource_id=r['_id'],
                    user_id=resource['user_id'],
                    project_id=resource['project_id'],
                    first_sample_timestamp=resource['first_timestamp'],
                    last_sample_timestamp=resource['last_timestamp'],
                    source=resource['source'],
                    metadata=resource['metadata'])
        finally:
            self.db[out].drop()
Ejemplo n.º 15
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery={}, resource=None, pagination=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise NotImplementedError(_('Pagination not implemented'))

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(('resource_' + k, v)
                      for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp,
                                            start_timestamp_op,
                                            end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource')
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        aggregate = self.db.meter.aggregate([
            {"$match": q},
            {"$sort": dict(sort_instructions)},
            {"$group": {
                "_id": "$resource_id",
                "user_id": {"$first": "$user_id"},
                "project_id": {"$first": "$project_id"},
                "source": {"$first": "$source"},
                "first_sample_timestamp": {"$min": "$timestamp"},
                "last_sample_timestamp": {"$max": "$timestamp"},
                "metadata": {"$first": "$resource_metadata"},
                "meters_name": {"$push": "$counter_name"},
                "meters_type": {"$push": "$counter_type"},
                "meters_unit": {"$push": "$counter_unit"},
            }},
        ])

        for result in aggregate['result']:
            yield models.Resource(
                resource_id=result['_id'],
                user_id=result['user_id'],
                project_id=result['project_id'],
                first_sample_timestamp=result['first_sample_timestamp'],
                last_sample_timestamp=result['last_sample_timestamp'],
                source=result['source'],
                metadata=result['metadata'],
                meter=[
                    models.ResourceMeter(
                        counter_name=m_n,
                        counter_type=m_t,
                        counter_unit=m_u,
                    )
                    for m_n, m_u, m_t in zip(result['meters_name'],
                                             result['meters_unit'],
                                             result['meters_type'])
                ],
            )
Ejemplo n.º 16
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery={}, resource=None, limit=None,
                      marker_pairs=None, sort_key=None, sort_dir=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param limit: Number of documents should be returned.
        :param marker_pairs: Attribute-value pairs to identify the last item of
                            the previous page.
        :param sort_key: Attribute by which results be sorted.
        :param sort_dir: Direction with which results be sorted(asc, desc).
        """
        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(('resource_' + k, v)
                      for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp,
                                            start_timestamp_op,
                                            end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        marker = self._get_marker(self.db.resource, marker_pairs=marker_pairs)
        sort_keys = base._handle_sort_key('resource', sort_key)

        # FIXME(jd): We should use self.db.meter.group() and not use the
        # resource collection, but that's not supported by MIM, so it's not
        # easily testable yet. Since it was bugged before anyway, it's still
        # better for now.
        resource_ids = self.db.meter.find(q).distinct('resource_id')
        q = {'_id': {'$in': resource_ids}}
        for resource in self.paginate_query(q, self.db.resource, limit=limit,
                                            marker=marker, sort_keys=sort_keys,
                                            sort_dir=sort_dir):
            yield models.Resource(
                resource_id=resource['_id'],
                project_id=resource['project_id'],
                source=resource['source'],
                user_id=resource['user_id'],
                metadata=resource['metadata'],
                meter=[
                    models.ResourceMeter(
                        counter_name=meter['counter_name'],
                        counter_type=meter['counter_type'],
                        counter_unit=meter.get('counter_unit', ''),
                    )
                    for meter in resource['meter']
                ],
            )
Ejemplo n.º 17
0
    def get_resources(self,
                      user=None,
                      project=None,
                      source=None,
                      start_timestamp=None,
                      start_timestamp_op=None,
                      end_timestamp=None,
                      end_timestamp_op=None,
                      metaquery={},
                      resource=None,
                      limit=None,
                      marker_pairs=None,
                      sort_key=None,
                      sort_dir=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param limit: Number of documents should be returned.
        :param marker_pairs: Attribute-value pairs to identify the last item of
                            the previous page.
        :param sort_key: Attribute by which results be sorted.
        :param sort_dir: Direction with which results be sorted(asc, desc).
        """

        if marker_pairs:
            raise NotImplementedError(
                "Cannot use marker pairs in resource listing, not implemented")

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(
            ('resource_' + k, v) for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp,
                                            start_timestamp_op,
                                            end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource', sort_key)
        sort_instruction, query = self._build_paginate_query(
            None, sort_keys, sort_dir)
        q.update(query)

        aggregate = self.db.meter.aggregate([
            {
                "$match": q
            },
            {
                "$sort": dict(sort_instruction)
            },
            {
                "$group": {
                    "_id": "$resource_id",
                    "user_id": {
                        "$first": "$user_id"
                    },
                    "project_id": {
                        "$first": "$project_id"
                    },
                    "source": {
                        "$first": "$source"
                    },
                    "first_sample_timestamp": {
                        "$min": "$timestamp"
                    },
                    "last_sample_timestamp": {
                        "$max": "$timestamp"
                    },
                    "metadata": {
                        "$first": "$resource_metadata"
                    },
                    "meters_name": {
                        "$push": "$counter_name"
                    },
                    "meters_type": {
                        "$push": "$counter_type"
                    },
                    "meters_unit": {
                        "$push": "$counter_unit"
                    },
                }
            },
        ])

        for result in aggregate['result']:
            if limit is not None:
                if limit == 0:
                    break
                limit -= 1
            yield models.Resource(
                resource_id=result['_id'],
                user_id=result['user_id'],
                project_id=result['project_id'],
                first_sample_timestamp=result['first_sample_timestamp'],
                last_sample_timestamp=result['last_sample_timestamp'],
                source=result['source'],
                metadata=result['metadata'],
                meter=[
                    models.ResourceMeter(
                        counter_name=m_n,
                        counter_type=m_t,
                        counter_unit=m_u,
                    ) for m_n, m_u, m_t in
                    zip(result['meters_name'], result['meters_unit'],
                        result['meters_type'])
                ],
            )
Ejemplo n.º 18
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery={}, resource=None, limit=None,
                      marker_pairs=None, sort_key=None, sort_dir=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param limit: Number of documents should be returned.
        :param marker_pairs: Attribute-value pairs to identify the last item of
                            the previous page.
        :param sort_key: Attribute by which results be sorted.
        :param sort_dir: Direction with which results be sorted(asc, desc).
        """

        if marker_pairs is not None:
            raise NotImplementedError(
                "Cannot use marker pairs in resource listing")

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(('resource_' + k, v)
                      for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp,
                                            start_timestamp_op,
                                            end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource', sort_key)
        sort_instruction, query = self._build_paginate_query(None,
                                                             sort_keys,
                                                             sort_dir)
        q.update(query)

        aggregate = self.db.meter.aggregate([
            {"$match": q},
            {"$sort": dict(sort_instruction)},
            {"$group": {
                "_id": "$resource_id",
                "user_id": {"$first": "$user_id"},
                "project_id": {"$first": "$project_id"},
                "source": {"$first": "$source"},
                "first_sample_timestamp": {"$min": "$timestamp"},
                "last_sample_timestamp": {"$max": "$timestamp"},
                "metadata": {"$first": "$resource_metadata"},
                "meters_name": {"$push": "$counter_name"},
                "meters_type": {"$push": "$counter_type"},
                "meters_unit": {"$push": "$counter_unit"},
            }},
        ])

        for result in aggregate['result']:
            if limit is not None:
                if limit == 0:
                    break
                limit -= 1
            yield models.Resource(
                resource_id=result['_id'],
                user_id=result['user_id'],
                project_id=result['project_id'],
                first_sample_timestamp=result['first_sample_timestamp'],
                last_sample_timestamp=result['last_sample_timestamp'],
                source=result['source'],
                metadata=result['metadata'],
                meter=[
                    models.ResourceMeter(
                        counter_name=m_n,
                        counter_type=m_t,
                        counter_unit=m_u,
                    )
                    for m_n, m_u, m_t in zip(result['meters_name'],
                                             result['meters_unit'],
                                             result['meters_type'])
                ],
            )
Ejemplo n.º 19
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery=None, resource=None, pagination=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise ceilometer.NotImplementedError('Pagination not implemented')

        metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {}

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(('resource_' + k, v)
                      for (k, v) in six.iteritems(metaquery)))

        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = pymongo_utils.make_timestamp_range(start_timestamp,
                                                          end_timestamp,
                                                          start_timestamp_op,
                                                          end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource', 'timestamp')
        sort_keys.insert(0, 'resource_id')
        sort_instructions = self._build_sort_instructions(sort_keys=sort_keys,
                                                          sort_dir='desc')
        resource = lambda x: x['resource_id']
        meters = self.db.meter.find(q, sort=sort_instructions)
        for resource_id, r_meters in itertools.groupby(meters, key=resource):
            # Because we have to know first/last timestamp, and we need a full
            # list of references to the resource's meters, we need a tuple
            # here.
            r_meters = tuple(r_meters)
            latest_meter = r_meters[0]
            last_ts = latest_meter['timestamp']
            first_ts = r_meters[-1]['timestamp']

            yield models.Resource(resource_id=latest_meter['resource_id'],
                                  project_id=latest_meter['project_id'],
                                  first_sample_timestamp=first_ts,
                                  last_sample_timestamp=last_ts,
                                  source=latest_meter['source'],
                                  user_id=latest_meter['user_id'],
                                  metadata=pymongo_utils.unquote_keys(
                latest_meter['resource_metadata']))
Ejemplo n.º 20
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery={}, resource=None, pagination=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise NotImplementedError(_('Pagination not implemented'))

        q = {}
        if user is not None:
            q['user_id'] = user
        if project is not None:
            q['project_id'] = project
        if source is not None:
            q['source'] = source
        if resource is not None:
            q['resource_id'] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(('resource_' + k, v)
                      for (k, v) in metaquery.iteritems()))

        # FIXME(dhellmann): This may not perform very well,
        # but doing any better will require changing the database
        # schema and that will need more thought than I have time
        # to put into it today.
        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp,
                                            start_timestamp_op,
                                            end_timestamp_op)
            if ts_range:
                q['timestamp'] = ts_range

        sort_keys = base._handle_sort_key('resource')
        sort_instructions = self._build_sort_instructions(sort_keys)[0]

        # use a unique collection name for the results collection,
        # as result post-sorting (as oppposed to reduce pre-sorting)
        # is not possible on an inline M-R
        out = 'resource_list_%s' % uuid.uuid4()
        self.db.meter.map_reduce(self.MAP_RESOURCES,
                                 self.REDUCE_RESOURCES,
                                 out=out,
                                 sort={'resource_id': 1},
                                 query=q)

        try:
            for r in self.db[out].find(sort=sort_instructions):
                resource = r['value']
                yield models.Resource(
                    resource_id=r['_id'],
                    user_id=resource['user_id'],
                    project_id=resource['project_id'],
                    first_sample_timestamp=resource['first_timestamp'],
                    last_sample_timestamp=resource['last_timestamp'],
                    source=resource['source'],
                    metadata=resource['metadata'])
        finally:
            self.db[out].drop()
Ejemplo n.º 21
0
    def get_resources(
        self,
        user=None,
        project=None,
        source=None,
        start_timestamp=None,
        start_timestamp_op=None,
        end_timestamp=None,
        end_timestamp_op=None,
        metaquery={},
        resource=None,
        pagination=None,
    ):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like gt, ge.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param pagination: Optional pagination query.
        """
        if pagination:
            raise NotImplementedError(_("Pagination not implemented"))

        q = {}
        if user is not None:
            q["user_id"] = user
        if project is not None:
            q["project_id"] = project
        if source is not None:
            q["source"] = source
        if resource is not None:
            q["resource_id"] = resource
        # Add resource_ prefix so it matches the field in the db
        q.update(dict(("resource_" + k, v) for (k, v) in metaquery.iteritems()))

        if start_timestamp or end_timestamp:
            # Look for resources matching the above criteria and with
            # samples in the time range we care about, then change the
            # resource query to return just those resources by id.
            ts_range = make_timestamp_range(start_timestamp, end_timestamp, start_timestamp_op, end_timestamp_op)
            if ts_range:
                q["timestamp"] = ts_range

        sort_keys = base._handle_sort_key("resource", "timestamp")
        sort_keys.insert(0, "resource_id")
        sort_instructions = self._build_sort_instructions(sort_keys=sort_keys, sort_dir="desc")
        resource = lambda x: x["resource_id"]
        meters = self.db.meter.find(q, sort=sort_instructions)
        for resource_id, r_meters in itertools.groupby(meters, key=resource):
            resource_meters = []
            # Because we have to know first/last timestamp, and we need a full
            # list of references to the resource's meters, we need a tuple
            # here.
            r_meters = tuple(r_meters)
            for meter in r_meters:
                resource_meters.append(
                    models.ResourceMeter(
                        counter_name=meter["counter_name"],
                        counter_type=meter["counter_type"],
                        counter_unit=meter.get("counter_unit", ""),
                    )
                )
            latest_meter = r_meters[0]
            last_ts = latest_meter["timestamp"]
            first_ts = r_meters[-1]["timestamp"]

            yield models.Resource(
                resource_id=latest_meter["resource_id"],
                project_id=latest_meter["project_id"],
                first_sample_timestamp=first_ts,
                last_sample_timestamp=last_ts,
                source=latest_meter["source"],
                user_id=latest_meter["user_id"],
                metadata=latest_meter["resource_metadata"],
                meter=resource_meters,
            )