Exemplo n.º 1
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery=None, resource=None, limit=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like ge, gt.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param limit: Maximum number of results to return.
        """
        if limit == 0:
            return
        q = hbase_utils.make_query(metaquery=metaquery, user_id=user,
                                   project_id=project,
                                   resource_id=resource, source=source)
        q = hbase_utils.make_meter_query_for_resource(start_timestamp,
                                                      start_timestamp_op,
                                                      end_timestamp,
                                                      end_timestamp_op,
                                                      source, q)
        with self.conn_pool.connection() as conn:
            resource_table = conn.table(self.RESOURCE_TABLE)
            LOG.debug("Query Resource table: %s", q)
            for resource_id, data in resource_table.scan(filter=q,
                                                         limit=limit):
                f_res, meters, md = hbase_utils.deserialize_entry(
                    data)
                resource_id = hbase_utils.encode_unicode(resource_id)
                # Unfortunately happybase doesn't keep ordered result from
                # HBase. So that's why it's needed to find min and max
                # manually
                first_ts = min(meters, key=operator.itemgetter(1))[1]
                last_ts = max(meters, key=operator.itemgetter(1))[1]
                source = meters[0][0][1]
                # If we use QualifierFilter then HBase returns only
                # qualifiers filtered by. It will not return the whole entry.
                # That's why if we need to ask additional qualifiers manually.
                if 'project_id' not in f_res and 'user_id' not in f_res:
                    row = resource_table.row(
                        resource_id, columns=['f:project_id', 'f:user_id',
                                              'f:resource_metadata'])
                    f_res, _m, md = hbase_utils.deserialize_entry(row)
                yield models.Resource(
                    resource_id=resource_id,
                    first_sample_timestamp=first_ts,
                    last_sample_timestamp=last_ts,
                    project_id=f_res['project_id'],
                    source=source,
                    user_id=f_res['user_id'],
                    metadata=md)
Exemplo n.º 2
0
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery=None, resource=None, limit=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like ge, gt.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param limit: Maximum number of results to return.
        """
        if limit == 0:
            return
        q = hbase_utils.make_query(metaquery=metaquery, user_id=user,
                                   project_id=project,
                                   resource_id=resource, source=source)
        q = hbase_utils.make_meter_query_for_resource(start_timestamp,
                                                      start_timestamp_op,
                                                      end_timestamp,
                                                      end_timestamp_op,
                                                      source, q)
        with self.conn_pool.connection() as conn:
            resource_table = conn.table(self.RESOURCE_TABLE)
            LOG.debug("Query Resource table: %s", q)
            for resource_id, data in resource_table.scan(filter=q,
                                                         limit=limit):
                f_res, sources, meters, md = hbase_utils.deserialize_entry(
                    data)
                resource_id = hbase_utils.encode_unicode(resource_id)
                # Unfortunately happybase doesn't keep ordered result from
                # HBase. So that's why it's needed to find min and max
                # manually
                first_ts = min(meters, key=operator.itemgetter(1))[1]
                last_ts = max(meters, key=operator.itemgetter(1))[1]
                source = meters[0][0][1]
                # If we use QualifierFilter then HBase returnes only
                # qualifiers filtered by. It will not return the whole entry.
                # That's why if we need to ask additional qualifiers manually.
                if 'project_id' not in f_res and 'user_id' not in f_res:
                    row = resource_table.row(
                        resource_id, columns=['f:project_id', 'f:user_id',
                                              'f:resource_metadata'])
                    f_res, _s, _m, md = hbase_utils.deserialize_entry(row)
                yield models.Resource(
                    resource_id=resource_id,
                    first_sample_timestamp=first_ts,
                    last_sample_timestamp=last_ts,
                    project_id=f_res['project_id'],
                    source=source,
                    user_id=f_res['user_id'],
                    metadata=md)
Exemplo n.º 3
0
    def record_metering_data(self, data):
        """Write the data to the backend storage system.

        :param data: a dictionary such as returned by
          ceilometer.publisher.utils.meter_message_from_counter
        """

        # We must not record thing.
        data.pop("monotonic_time", None)

        with self.conn_pool.connection() as conn:
            resource_table = conn.table(self.RESOURCE_TABLE)
            meter_table = conn.table(self.METER_TABLE)

            resource_metadata = data.get('resource_metadata', {})
            # Determine the name of new meter
            rts = hbase_utils.timestamp(data['timestamp'])
            new_meter = hbase_utils.prepare_key(rts, data['source'],
                                                data['counter_name'],
                                                data['counter_type'],
                                                data['counter_unit'])

            # TODO(nprivalova): try not to store resource_id
            resource = hbase_utils.serialize_entry(
                **{
                    'source': data['source'],
                    'meter': {
                        new_meter: data['timestamp']
                    },
                    'resource_metadata': resource_metadata,
                    'resource_id': data['resource_id'],
                    'project_id': data['project_id'],
                    'user_id': data['user_id']
                })
            # Here we put entry in HBase with our own timestamp. This is needed
            # when samples arrive out-of-order
            # If we use timestamp=data['timestamp'] the newest data will be
            # automatically 'on the top'. It is needed to keep metadata
            # up-to-date: metadata from newest samples is considered as actual.
            ts = int(time.mktime(data['timestamp'].timetuple()) * 1000)
            resource_table.put(hbase_utils.encode_unicode(data['resource_id']),
                               resource, ts)

            # Rowkey consists of reversed timestamp, meter and a
            # message uuid for purposes of uniqueness
            row = hbase_utils.prepare_key(data['counter_name'], rts,
                                          data['message_id'])
            record = hbase_utils.serialize_entry(
                data, **{
                    'source': data['source'],
                    'rts': rts,
                    'message': data,
                    'recorded_at': timeutils.utcnow()
                })
            meter_table.put(row, record)
Exemplo n.º 4
0
    def record_metering_data(self, data):
        """Write the data to the backend storage system.

        :param data: a dictionary such as returned by
          ceilometer.meter.meter_message_from_counter
        """
        with self.conn_pool.connection() as conn:
            resource_table = conn.table(self.RESOURCE_TABLE)
            meter_table = conn.table(self.METER_TABLE)

            resource_metadata = data.get("resource_metadata", {})
            # Determine the name of new meter
            rts = hbase_utils.timestamp(data["timestamp"])
            new_meter = hbase_utils.prepare_key(
                rts, data["source"], data["counter_name"], data["counter_type"], data["counter_unit"]
            )

            # TODO(nprivalova): try not to store resource_id
            resource = hbase_utils.serialize_entry(
                **{
                    "source": data["source"],
                    "meter": {new_meter: data["timestamp"]},
                    "resource_metadata": resource_metadata,
                    "resource_id": data["resource_id"],
                    "project_id": data["project_id"],
                    "user_id": data["user_id"],
                }
            )
            # Here we put entry in HBase with our own timestamp. This is needed
            # when samples arrive out-of-order
            # If we use timestamp=data['timestamp'] the newest data will be
            # automatically 'on the top'. It is needed to keep metadata
            # up-to-date: metadata from newest samples is considered as actual.
            ts = int(time.mktime(data["timestamp"].timetuple()) * 1000)
            resource_table.put(hbase_utils.encode_unicode(data["resource_id"]), resource, ts)

            # Rowkey consists of reversed timestamp, meter and a
            # message uuid for purposes of uniqueness
            row = hbase_utils.prepare_key(data["counter_name"], rts, data["message_id"])
            record = hbase_utils.serialize_entry(
                data, **{"source": data["source"], "rts": rts, "message": data, "recorded_at": timeutils.utcnow()}
            )
            meter_table.put(row, record)