def record_metering_data(self, data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.publisher.utils.meter_message_from_counter
"""
# We must not record thing.
data.pop("monotonic_time", None)
with self.conn_pool.connection() as conn:
resource_table = conn.table(self.RESOURCE_TABLE)
meter_table = conn.table(self.METER_TABLE)
resource_metadata = data.get('resource_metadata', {})
# Determine the name of new meter
rts = hbase_utils.timestamp(data['timestamp'])
new_meter = hbase_utils.prepare_key(rts, data['source'],
data['counter_name'],
data['counter_type'],
data['counter_unit'])
# TODO(nprivalova): try not to store resource_id
resource = hbase_utils.serialize_entry(
**{
'source': data['source'],
'meter': {
new_meter: data['timestamp']
},
'resource_metadata': resource_metadata,
'resource_id': data['resource_id'],
'project_id': data['project_id'],
'user_id': data['user_id']
})
# Here we put entry in HBase with our own timestamp. This is needed
# when samples arrive out-of-order
# If we use timestamp=data['timestamp'] the newest data will be
# automatically 'on the top'. It is needed to keep metadata
# up-to-date: metadata from newest samples is considered as actual.
ts = int(time.mktime(data['timestamp'].timetuple()) * 1000)
resource_table.put(hbase_utils.encode_unicode(data['resource_id']),
resource, ts)
# Rowkey consists of reversed timestamp, meter and a
# message uuid for purposes of uniqueness
row = hbase_utils.prepare_key(data['counter_name'], rts,
data['message_id'])
record = hbase_utils.serialize_entry(
data, **{
'source': data['source'],
'rts': rts,
'message': data,
'recorded_at': timeutils.utcnow()
})
meter_table.put(row, record)
def record_metering_data(self, data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.meter.meter_message_from_counter
"""
with self.conn_pool.connection() as conn:
resource_table = conn.table(self.RESOURCE_TABLE)
meter_table = conn.table(self.METER_TABLE)
resource_metadata = data.get('resource_metadata', {})
# Determine the name of new meter
rts = hbase_utils.timestamp(data['timestamp'])
new_meter = hbase_utils.format_meter_reference(
data['counter_name'], data['counter_type'],
data['counter_unit'], rts, data['source'])
# TODO(nprivalova): try not to store resource_id
resource = hbase_utils.serialize_entry(
**{
'source': data['source'],
'meter': {
new_meter: data['timestamp']
},
'resource_metadata': resource_metadata,
'resource_id': data['resource_id'],
'project_id': data['project_id'],
'user_id': data['user_id']
})
# Here we put entry in HBase with our own timestamp. This is needed
# when samples arrive out-of-order
# If we use timestamp=data['timestamp'] the newest data will be
# automatically 'on the top'. It is needed to keep metadata
# up-to-date: metadata from newest samples is considered as actual.
ts = int(time.mktime(data['timestamp'].timetuple()) * 1000)
resource_table.put(data['resource_id'], resource, ts)
# TODO(nprivalova): improve uniqueness
# Rowkey consists of reversed timestamp, meter and an md5 of
# user+resource+project for purposes of uniqueness
m = hashlib.md5()
m.update(
"%s%s%s" %
(data['user_id'], data['resource_id'], data['project_id']))
row = "%s_%d_%s" % (data['counter_name'], rts, m.hexdigest())
record = hbase_utils.serialize_entry(
data, **{
'source': data['source'],
'rts': rts,
'message': data,
'recorded_at': timeutils.utcnow()
})
meter_table.put(row, record)
def record_events(self, event_models):
"""Write the events to Hbase.
:param event_models: a list of models.Event objects.
"""
error = None
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
for event_model in event_models:
# Row key consists of timestamp and message_id from
# models.Event or purposes of storage event sorted by
# timestamp in the database.
ts = event_model.generated
row = hbase_utils.prepare_key(hbase_utils.timestamp(ts, reverse=False), event_model.message_id)
event_type = event_model.event_type
traits = {}
if event_model.traits:
for trait in event_model.traits:
key = hbase_utils.prepare_key(trait.name, trait.dtype)
traits[key] = trait.value
record = hbase_utils.serialize_entry(traits, event_type=event_type, timestamp=ts, raw=event_model.raw)
try:
events_table.put(row, record)
except Exception as ex:
LOG.exception(_LE("Failed to record event: %s") % ex)
error = ex
if error:
raise error
def record_events(self, event_models):
"""Write the events to Hbase.
:param event_models: a list of models.Event objects.
:return problem_events: a list of events that could not be saved in a
(reason, event) tuple. From the reasons that are enumerated in
storage.models.Event only the UNKNOWN_PROBLEM is applicable here.
"""
problem_events = []
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
for event_model in event_models:
# Row key consists of timestamp and message_id from
# models.Event or purposes of storage event sorted by
# timestamp in the database.
ts = event_model.generated
row = "%d_%s" % (hbase_utils.timestamp(
ts, reverse=False), event_model.message_id)
event_type = event_model.event_type
traits = {}
if event_model.traits:
for trait in event_model.traits:
key = "%s+%d" % (trait.name, trait.dtype)
traits[key] = trait.value
record = hbase_utils.serialize_entry(traits,
event_type=event_type,
timestamp=ts)
try:
events_table.put(row, record)
except Exception as ex:
LOG.debug(_("Failed to record event: %s") % ex)
problem_events.append(
(ev_models.Event.UNKNOWN_PROBLEM, event_model))
return problem_events
def record_events(self, event_models):
"""Write the events to Hbase.
:param event_models: a list of models.Event objects.
"""
error = None
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
for event_model in event_models:
# Row key consists of timestamp and message_id from
# models.Event or purposes of storage event sorted by
# timestamp in the database.
ts = event_model.generated
row = hbase_utils.prepare_key(
hbase_utils.timestamp(ts, reverse=False),
event_model.message_id)
event_type = event_model.event_type
traits = {}
if event_model.traits:
for trait in event_model.traits:
key = hbase_utils.prepare_key(trait.name, trait.dtype)
traits[key] = trait.value
record = hbase_utils.serialize_entry(traits,
event_type=event_type,
timestamp=ts,
raw=event_model.raw)
try:
events_table.put(row, record)
except Exception as ex:
LOG.exception(_LE("Failed to record event: %s") % ex)
error = ex
if error:
raise error
def record_events(self, event_models):
"""Write the events to Hbase.
:param event_models: a list of models.Event objects.
:return problem_events: a list of events that could not be saved in a
(reason, event) tuple. From the reasons that are enumerated in
storage.models.Event only the UNKNOWN_PROBLEM is applicable here.
"""
problem_events = []
with self.conn_pool.connection() as conn:
events_table = conn.table(self.EVENT_TABLE)
for event_model in event_models:
# Row key consists of timestamp and message_id from
# models.Event or purposes of storage event sorted by
# timestamp in the database.
ts = event_model.generated
row = "%d_%s" % (hbase_utils.timestamp(ts, reverse=False),
event_model.message_id)
event_type = event_model.event_type
traits = {}
if event_model.traits:
for trait in event_model.traits:
key = "%s+%d" % (trait.name, trait.dtype)
traits[key] = trait.value
record = hbase_utils.serialize_entry(traits,
event_type=event_type,
timestamp=ts)
try:
events_table.put(row, record)
except Exception as ex:
LOG.debug(_("Failed to record event: %s") % ex)
problem_events.append((models.Event.UNKNOWN_PROBLEM,
event_model))
return problem_events
def record_metering_data(self, data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.meter.meter_message_from_counter
"""
with self.conn_pool.connection() as conn:
resource_table = conn.table(self.RESOURCE_TABLE)
meter_table = conn.table(self.METER_TABLE)
resource_metadata = data.get("resource_metadata", {})
# Determine the name of new meter
rts = hbase_utils.timestamp(data["timestamp"])
new_meter = hbase_utils.format_meter_reference(
data["counter_name"], data["counter_type"], data["counter_unit"], rts, data["source"]
)
# TODO(nprivalova): try not to store resource_id
resource = hbase_utils.serialize_entry(
**{
"source": data["source"],
"meter": {new_meter: data["timestamp"]},
"resource_metadata": resource_metadata,
"resource_id": data["resource_id"],
"project_id": data["project_id"],
"user_id": data["user_id"],
}
)
# Here we put entry in HBase with our own timestamp. This is needed
# when samples arrive out-of-order
# If we use timestamp=data['timestamp'] the newest data will be
# automatically 'on the top'. It is needed to keep metadata
# up-to-date: metadata from newest samples is considered as actual.
ts = int(time.mktime(data["timestamp"].timetuple()) * 1000)
resource_table.put(data["resource_id"], resource, ts)
# TODO(nprivalova): improve uniqueness
# Rowkey consists of reversed timestamp, meter and an md5 of
# user+resource+project for purposes of uniqueness
m = hashlib.md5()
m.update("%s%s%s" % (data["user_id"], data["resource_id"], data["project_id"]))
row = "%s_%d_%s" % (data["counter_name"], rts, m.hexdigest())
record = hbase_utils.serialize_entry(
data, **{"source": data["source"], "rts": rts, "message": data, "recorded_at": timeutils.utcnow()}
)
meter_table.put(row, record)
def record_alarm_change(self, alarm_change):
"""Record alarm change event."""
alarm_change_dict = hbase_utils.serialize_entry(alarm_change)
ts = alarm_change.get('timestamp') or datetime.datetime.now()
rts = hbase_utils.timestamp(ts)
with self.conn_pool.connection() as conn:
alarm_history_table = conn.table(self.ALARM_HISTORY_TABLE)
alarm_history_table.put(alarm_change.get('alarm_id') + "_" +
str(rts), alarm_change_dict)
def record_alarm_change(self, alarm_change):
"""Record alarm change event."""
alarm_change_dict = hbase_utils.serialize_entry(alarm_change)
ts = alarm_change.get('timestamp') or datetime.datetime.now()
rts = hbase_utils.timestamp(ts)
with self.conn_pool.connection() as conn:
alarm_history_table = conn.table(self.ALARM_HISTORY_TABLE)
alarm_history_table.put(alarm_change.get('alarm_id') + "_" +
str(rts), alarm_change_dict)
def record_metering_data(self, data):
"""Write the data to the backend storage system.
:param data: a dictionary such as returned by
ceilometer.publisher.utils.meter_message_from_counter
"""
with self.conn_pool.connection() as conn:
resource_table = conn.table(self.RESOURCE_TABLE)
meter_table = conn.table(self.METER_TABLE)
resource_metadata = data.get('resource_metadata', {})
# Determine the name of new meter
rts = hbase_utils.timestamp(data['timestamp'])
new_meter = hbase_utils.prepare_key(
rts, data['source'], data['counter_name'],
data['counter_type'], data['counter_unit'])
# TODO(nprivalova): try not to store resource_id
resource = hbase_utils.serialize_entry(**{
'source': data['source'],
'meter': {new_meter: data['timestamp']},
'resource_metadata': resource_metadata,
'resource_id': data['resource_id'],
'project_id': data['project_id'], 'user_id': data['user_id']})
# Here we put entry in HBase with our own timestamp. This is needed
# when samples arrive out-of-order
# If we use timestamp=data['timestamp'] the newest data will be
# automatically 'on the top'. It is needed to keep metadata
# up-to-date: metadata from newest samples is considered as actual.
ts = int(time.mktime(data['timestamp'].timetuple()) * 1000)
resource_table.put(hbase_utils.encode_unicode(data['resource_id']),
resource, ts)
# Rowkey consists of reversed timestamp, meter and a
# message uuid for purposes of uniqueness
row = hbase_utils.prepare_key(data['counter_name'], rts,
data['message_id'])
record = hbase_utils.serialize_entry(
data, **{'source': data['source'], 'rts': rts,
'message': data, 'recorded_at': timeutils.utcnow()})
meter_table.put(row, record)
def update_alarm(self, alarm):
"""Create an alarm.
:param alarm: The alarm to create. It is Alarm object, so we need to
call as_dict()
"""
_id = alarm.alarm_id
alarm_to_store = hbase_utils.serialize_entry(alarm.as_dict())
with self.conn_pool.connection() as conn:
alarm_table = conn.table(self.ALARM_TABLE)
alarm_table.put(_id, alarm_to_store)
stored_alarm = hbase_utils.deserialize_entry(alarm_table.row(_id))[0]
return alarm_models.Alarm(**stored_alarm)
def update_alarm(self, alarm):
"""Create an alarm.
:param alarm: The alarm to create. It is Alarm object, so we need to
call as_dict()
"""
_id = alarm.alarm_id
alarm_to_store = hbase_utils.serialize_entry(alarm.as_dict())
with self.conn_pool.connection() as conn:
alarm_table = conn.table(self.ALARM_TABLE)
alarm_table.put(_id, alarm_to_store)
stored_alarm = hbase_utils.deserialize_entry(
alarm_table.row(_id))[0]
return models.Alarm(**stored_alarm)