def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ with self.conn_pool.connection() as conn: resource_table = conn.table(self.RESOURCE_TABLE) meter_table = conn.table(self.METER_TABLE) resource_metadata = data.get('resource_metadata', {}) # Determine the name of new meter rts = hbase_utils.timestamp(data['timestamp']) new_meter = hbase_utils.format_meter_reference( data['counter_name'], data['counter_type'], data['counter_unit'], rts, data['source']) # TODO(nprivalova): try not to store resource_id resource = hbase_utils.serialize_entry( **{ 'source': data['source'], 'meter': { new_meter: data['timestamp'] }, 'resource_metadata': resource_metadata, 'resource_id': data['resource_id'], 'project_id': data['project_id'], 'user_id': data['user_id'] }) # Here we put entry in HBase with our own timestamp. This is needed # when samples arrive out-of-order # If we use timestamp=data['timestamp'] the newest data will be # automatically 'on the top'. It is needed to keep metadata # up-to-date: metadata from newest samples is considered as actual. ts = int(time.mktime(data['timestamp'].timetuple()) * 1000) resource_table.put(data['resource_id'], resource, ts) # TODO(nprivalova): improve uniqueness # Rowkey consists of reversed timestamp, meter and an md5 of # user+resource+project for purposes of uniqueness m = hashlib.md5() m.update( "%s%s%s" % (data['user_id'], data['resource_id'], data['project_id'])) row = "%s_%d_%s" % (data['counter_name'], rts, m.hexdigest()) record = hbase_utils.serialize_entry( data, **{ 'source': data['source'], 'rts': rts, 'message': data, 'recorded_at': timeutils.utcnow() }) meter_table.put(row, record)
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ with self.conn_pool.connection() as conn: resource_table = conn.table(self.RESOURCE_TABLE) meter_table = conn.table(self.METER_TABLE) resource_metadata = data.get("resource_metadata", {}) # Determine the name of new meter rts = hbase_utils.timestamp(data["timestamp"]) new_meter = hbase_utils.format_meter_reference( data["counter_name"], data["counter_type"], data["counter_unit"], rts, data["source"] ) # TODO(nprivalova): try not to store resource_id resource = hbase_utils.serialize_entry( **{ "source": data["source"], "meter": {new_meter: data["timestamp"]}, "resource_metadata": resource_metadata, "resource_id": data["resource_id"], "project_id": data["project_id"], "user_id": data["user_id"], } ) # Here we put entry in HBase with our own timestamp. This is needed # when samples arrive out-of-order # If we use timestamp=data['timestamp'] the newest data will be # automatically 'on the top'. It is needed to keep metadata # up-to-date: metadata from newest samples is considered as actual. ts = int(time.mktime(data["timestamp"].timetuple()) * 1000) resource_table.put(data["resource_id"], resource, ts) # TODO(nprivalova): improve uniqueness # Rowkey consists of reversed timestamp, meter and an md5 of # user+resource+project for purposes of uniqueness m = hashlib.md5() m.update("%s%s%s" % (data["user_id"], data["resource_id"], data["project_id"])) row = "%s_%d_%s" % (data["counter_name"], rts, m.hexdigest()) record = hbase_utils.serialize_entry( data, **{"source": data["source"], "rts": rts, "message": data, "recorded_at": timeutils.utcnow()} ) meter_table.put(row, record)
def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter """ with self.conn_pool.connection() as conn: resource_table = conn.table(self.RESOURCE_TABLE) meter_table = conn.table(self.METER_TABLE) resource_metadata = data.get('resource_metadata', {}) # Determine the name of new meter rts = hbase_utils.timestamp(data['timestamp']) new_meter = hbase_utils.format_meter_reference( data['counter_name'], data['counter_type'], data['counter_unit'], rts, data['source']) # TODO(nprivalova): try not to store resource_id resource = hbase_utils.serialize_entry(**{ 'source': data['source'], 'meter': {new_meter: data['timestamp']}, 'resource_metadata': resource_metadata, 'resource_id': data['resource_id'], 'project_id': data['project_id'], 'user_id': data['user_id']}) # Here we put entry in HBase with our own timestamp. This is needed # when samples arrive out-of-order # If we use timestamp=data['timestamp'] the newest data will be # automatically 'on the top'. It is needed to keep metadata # up-to-date: metadata from newest samples is considered as actual. ts = int(time.mktime(data['timestamp'].timetuple()) * 1000) resource_table.put(data['resource_id'], resource, ts) # Rowkey consists of reversed timestamp, meter and a # message signature for purposes of uniqueness row = "%s_%d_%s" % (data['counter_name'], rts, data['message_signature']) record = hbase_utils.serialize_entry( data, **{'source': data['source'], 'rts': rts, 'message': data, 'recorded_at': timeutils.utcnow()}) meter_table.put(row, record)