Example #1
    def get_resources(self, user=None, project=None, source=None,
                      start_timestamp=None, start_timestamp_op=None,
                      end_timestamp=None, end_timestamp_op=None,
                      metaquery=None, resource=None, limit=None):
        """Return an iterable of models.Resource instances

        :param user: Optional ID for user that owns the resource.
        :param project: Optional ID for project that owns the resource.
        :param source: Optional source filter.
        :param start_timestamp: Optional modified timestamp start range.
        :param start_timestamp_op: Optional start time operator, like ge, gt.
        :param end_timestamp: Optional modified timestamp end range.
        :param end_timestamp_op: Optional end time operator, like lt, le.
        :param metaquery: Optional dict with metadata to match on.
        :param resource: Optional resource filter.
        :param limit: Maximum number of results to return.
        if limit == 0:
        q = hbase_utils.make_query(metaquery=metaquery, user_id=user,
                                   resource_id=resource, source=source)
        q = hbase_utils.make_meter_query_for_resource(start_timestamp,
                                                      source, q)
        with self.conn_pool.connection() as conn:
            resource_table = conn.table(self.RESOURCE_TABLE)
            LOG.debug("Query Resource table: %s", q)
            for resource_id, data in resource_table.scan(filter=q,
                f_res, meters, md = hbase_utils.deserialize_entry(
                resource_id = hbase_utils.encode_unicode(resource_id)
                # Unfortunately happybase doesn't keep ordered result from
                # HBase. So that's why it's needed to find min and max
                # manually
                first_ts = min(meters, key=operator.itemgetter(1))[1]
                last_ts = max(meters, key=operator.itemgetter(1))[1]
                source = meters[0][0][1]
                # If we use QualifierFilter then HBase returns only
                # qualifiers filtered by. It will not return the whole entry.
                # That's why if we need to ask additional qualifiers manually.
                if 'project_id' not in f_res and 'user_id' not in f_res:
                    row = resource_table.row(
                        resource_id, columns=['f:project_id', 'f:user_id',
                    f_res, _m, md = hbase_utils.deserialize_entry(row)
                yield models.Resource(
Example #2
    def record_metering_data(self, data):
        """Write the data to the backend storage system.

        :param data: a dictionary such as returned by
        with self.conn_pool.connection() as conn:
            resource_table = conn.table(self.RESOURCE_TABLE)
            meter_table = conn.table(self.METER_TABLE)

            resource_metadata = data.get("resource_metadata", {})
            # Determine the name of new meter
            rts = hbase_utils.timestamp(data["timestamp"])
            new_meter = hbase_utils.prepare_key(
                rts, data["source"], data["counter_name"], data["counter_type"], data["counter_unit"]

            # TODO(nprivalova): try not to store resource_id
            resource = hbase_utils.serialize_entry(
                    "source": data["source"],
                    "meter": {new_meter: data["timestamp"]},
                    "resource_metadata": resource_metadata,
                    "resource_id": data["resource_id"],
                    "project_id": data["project_id"],
                    "user_id": data["user_id"],
            # Here we put entry in HBase with our own timestamp. This is needed
            # when samples arrive out-of-order
            # If we use timestamp=data['timestamp'] the newest data will be
            # automatically 'on the top'. It is needed to keep metadata
            # up-to-date: metadata from newest samples is considered as actual.
            ts = int(time.mktime(data["timestamp"].timetuple()) * 1000)
            resource_table.put(hbase_utils.encode_unicode(data["resource_id"]), resource, ts)

            # Rowkey consists of reversed timestamp, meter and a
            # message uuid for purposes of uniqueness
            row = hbase_utils.prepare_key(data["counter_name"], rts, data["message_id"])
            record = hbase_utils.serialize_entry(
                data, **{"source": data["source"], "rts": rts, "message": data, "recorded_at": timeutils.utcnow()}
            meter_table.put(row, record)