def make_test_data(conn, name, meter_type, unit, volume, random_min,
random_max, user_id, project_id, resource_id, start,
end, interval, resource_metadata={}, source='artificial',):
# Compute start and end timestamps for the new data.
if isinstance(start, datetime.datetime):
timestamp = start
else:
timestamp = timeutils.parse_strtime(start)
if not isinstance(end, datetime.datetime):
end = timeutils.parse_strtime(end)
increment = datetime.timedelta(minutes=interval)
print('Adding new events for meter %s.' % (name))
# Generate events
n = 0
total_volume = volume
while timestamp <= end:
if (random_min >= 0 and random_max >= 0):
# If there is a random element defined, we will add it to
# user given volume.
if isinstance(random_min, int) and isinstance(random_max, int):
total_volume += random.randint(random_min, random_max)
else:
total_volume += random.uniform(random_min, random_max)
c = sample.Sample(name=name,
type=meter_type,
unit=unit,
volume=total_volume,
user_id=user_id,
project_id=project_id,
resource_id=resource_id,
timestamp=timestamp,
resource_metadata=resource_metadata,
source=source,
)
data = utils.meter_message_from_counter(
c,
cfg.CONF.publisher.metering_secret)
conn.record_metering_data(data)
n += 1
timestamp = timestamp + increment
if (meter_type == 'gauge' or meter_type == 'delta'):
# For delta and gauge, we don't want to increase the value
# in time by random element. So we always set it back to
# volume.
total_volume = volume
print('Added %d new events for meter %s.' % (n, name))
def get_event_interval(self, event_filter):
"""Return the min and max timestamps from samples,
using the event_filter to limit the samples seen.
( datetime.datetime(), datetime.datetime() )
"""
q, start, stop = make_query_from_filter(event_filter)
LOG.debug("q: %s" % q)
gen = self.meter.scan(filter=q, row_start=start, row_stop=stop)
a_min = None
a_max = None
for ignored, meter in gen:
timestamp = timeutils.parse_strtime(meter['f:timestamp'])
if a_min is None:
a_min = timestamp
else:
if timestamp < a_min:
a_min = timestamp
if a_max is None:
a_max = timestamp
else:
if timestamp > a_max:
a_max = timestamp
return a_min, a_max
def get_samples(self, sample_filter):
"""Return an iterable of models.Sample instances
"""
q, start, stop = make_query_from_filter(sample_filter,
require_meter=False)
LOG.debug("q: %s" % q)
gen = self.meter.scan(filter=q, row_start=start, row_stop=stop)
for ignored, meter in gen:
meter = json.loads(meter['f:message'])
meter['timestamp'] = timeutils.parse_strtime(meter['timestamp'])
yield models.Sample(**meter)
def get_samples(self, event_filter):
"""Return an iterable of models.Sample instances
"""
q, start, stop = make_query_from_filter(event_filter,
require_meter=False)
LOG.debug("q: %s" % q)
gen = self.meter.scan(filter=q, row_start=start, row_stop=stop)
for ignored, meter in gen:
meter = json.loads(meter['f:message'])
meter['timestamp'] = timeutils.parse_strtime(meter['timestamp'])
yield models.Sample(**meter)
def get_raw_events(self, event_filter):
"""Return an iterable of raw event data as created by
:func:`ceilometer.meter.meter_message_from_counter`.
"""
q, start, stop = make_query_from_filter(event_filter,
require_meter=False)
LOG.debug("q: %s" % q)
gen = self.meter.scan(filter=q, row_start=start, row_stop=stop)
meters = []
for ignored, meter in gen:
meter = json.loads(meter['f:message'])
meter['timestamp'] = timeutils.parse_strtime(meter['timestamp'])
meters.append(meter)
return meters
def handle_sample(self, context, sample):
if not self.initial_timestamp:
self.initial_timestamp = timeutils.parse_strtime(sample.timestamp)
self.aggregated_samples += 1
key = self._get_unique_key(sample)
if key not in self.samples:
self.samples[key] = self._convert(sample)
if self.merged_attribute_policy['resource_metadata'] == 'drop':
self.samples[key].resource_metadata = {}
else:
self.samples[key].volume += self._scale(sample)
for field in self.merged_attribute_policy:
if self.merged_attribute_policy[field] == 'last':
setattr(self.samples[key], field, getattr(sample, field))
def _get_sample(message, name):
try:
for metric in message['payload']['metrics']:
if name == metric['name']:
info = {'payload': metric,
'event_type': message['event_type'],
'publisher_id': message['publisher_id'],
'resource_id': '%s_%s' % (
message['payload']['host'],
message['payload']['nodename']),
'timestamp': str(timeutils.parse_strtime(
metric['timestamp']))}
return info
except Exception as err:
LOG.warning(_('An error occurred while building %(m)s '
'sample: %(e)s') % {'m': name, 'e': err})
def _get_sample(message, name):
try:
for metric in message['payload']['metrics']:
if name == metric['name']:
info = {}
info['payload'] = metric
info['event_type'] = message['event_type']
info['publisher_id'] = message['publisher_id']
info['resource_id'] = '%s_%s' % (
message['payload']['host'],
message['payload']['nodename'])
info['timestamp'] = str(timeutils.parse_strtime(
metric['timestamp']))
return info
except Exception as err:
LOG.warning(_('An error occurred while building %(m)s '
'sample: %(e)s') % {'m': name, 'e': err})
def handle_sample(self, context, sample):
if not self.initial_timestamp:
self.initial_timestamp = timeutils.parse_strtime(
sample.timestamp)
self.aggregated_samples += 1
key = self._get_unique_key(sample)
if key not in self.samples:
self.samples[key] = self._convert(sample)
if self.merged_attribute_policy[
'resource_metadata'] == 'drop':
self.samples[key].resource_metadata = {}
else:
self.samples[key].volume += self._scale(sample)
for field in self.merged_attribute_policy:
if self.merged_attribute_policy[field] == 'last':
setattr(self.samples[key], field,
getattr(sample, field))
def _update_meter_stats(stat, meter):
"""Do the stats calculation on a requested time bucket in stats dict
:param stats: dict where aggregated stats are kept
:param index: time bucket index in stats
:param meter: meter record as returned from HBase
:param start_time: query start time
:param period: length of the time bucket
"""
vol = int(meter["f:counter_volume"])
ts = timeutils.parse_strtime(meter["f:timestamp"])
stat.unit = meter["f:counter_unit"]
stat.min = min(vol, stat.min or vol)
stat.max = max(vol, stat.max)
stat.sum = vol + (stat.sum or 0)
stat.count += 1
stat.avg = stat.sum / float(stat.count)
stat.duration_start = min(ts, stat.duration_start or ts)
stat.duration_end = max(ts, stat.duration_end or ts)
stat.duration = timeutils.delta_seconds(stat.duration_start, stat.duration_end)
def _update_meter_stats(self, stat, meter):
"""Do the stats calculation on a requested time bucket in stats dict
:param stats: dict where aggregated stats are kept
:param index: time bucket index in stats
:param meter: meter record as returned from HBase
:param start_time: query start time
:param period: length of the time bucket
"""
vol = int(meter['f:counter_volume'])
ts = timeutils.parse_strtime(meter['f:timestamp'])
stat.min = min(vol, stat.min or vol)
stat.max = max(vol, stat.max)
stat.sum = vol + (stat.sum or 0)
stat.count += 1
stat.avg = (stat.sum / float(stat.count))
stat.duration_start = min(ts, stat.duration_start or ts)
stat.duration_end = max(ts, stat.duration_end or ts)
stat.duration = \
timeutils.delta_seconds(stat.duration_start,
stat.duration_end)
def _update_meter_stats(self, stat, meter):
"""Do the stats calculation on a requested time bucket in stats dict
:param stats: dict where aggregated stats are kept
:param index: time bucket index in stats
:param meter: meter record as returned from HBase
:param start_time: query start time
:param period: length of the time bucket
"""
vol = int(meter['f:counter_volume'])
ts = timeutils.parse_strtime(meter['f:timestamp'])
stat['min'] = min(vol, stat['min'] or vol)
stat['max'] = max(vol, stat['max'])
stat['sum'] = vol + (stat['sum'] or 0)
stat['count'] += 1
stat['avg'] = (stat['sum'] / float(stat['count']))
stat['duration_start'] = min(ts, stat['duration_start'] or ts)
stat['duration_end'] = max(ts, stat['duration_end'] or ts)
stat['duration'] = \
timeutils.delta_seconds(stat['duration_start'],
stat['duration_end'])
def get_resources(self,
user=None,
project=None,
source=None,
start_timestamp=None,
start_timestamp_op=None,
end_timestamp=None,
end_timestamp_op=None,
metaquery={},
resource=None,
pagination=None):
"""Return an iterable of models.Resource instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param start_timestamp_op: Optional start time operator, like ge, gt.
:param end_timestamp: Optional modified timestamp end range.
:param end_timestamp_op: Optional end time operator, like lt, le.
:param metaquery: Optional dict with metadata to match on.
:param resource: Optional resource filter.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
def make_resource(data, first_ts, last_ts):
"""Transform HBase fields to Resource model."""
# convert HBase metadata e.g. f:r_display_name to display_name
data['f:metadata'] = dict((k[4:], v) for k, v in data.iteritems()
if k.startswith('f:r_'))
return models.Resource(
resource_id=data['f:resource_id'],
first_sample_timestamp=first_ts,
last_sample_timestamp=last_ts,
project_id=data['f:project_id'],
source=data['f:source'],
user_id=data['f:user_id'],
metadata=data['f:metadata'],
meter=[
models.ResourceMeter(*(m[4:].split("!"))) for m in data
if m.startswith('f:m_')
],
)
resource_table = self.conn.table(self.RESOURCE_TABLE)
meter_table = self.conn.table(self.METER_TABLE)
q, start_row, stop_row = make_query(user=user,
project=project,
source=source,
resource=resource,
start=start_timestamp,
start_op=start_timestamp_op,
end=end_timestamp,
end_op=end_timestamp_op,
require_meter=False,
query_only=False)
LOG.debug("Query Meter table: %s" % q)
meters = meter_table.scan(filter=q,
row_start=start_row,
row_stop=stop_row)
resources = {}
for resource_id, r_meters in itertools.groupby(
meters, key=lambda x: x[1]['f:resource_id']):
timestamps = tuple(
timeutils.parse_strtime(m[1]['f:timestamp']) for m in r_meters)
resources[resource_id] = (min(timestamps), max(timestamps))
# handle metaquery
if len(metaquery) > 0:
for ignored, data in resource_table.rows(resources.iterkeys()):
for k, v in metaquery.iteritems():
# if metaquery matches, yield the resource model
# e.g. metaquery: metadata.display_name
# equals
# HBase: f:r_display_name
if data['f:r_' + k.split('.', 1)[1]] == v:
yield make_resource(
data, resources[data['f:resource_id']][0],
resources[data['f:resource_id']][1])
else:
for ignored, data in resource_table.rows(resources.iterkeys()):
yield make_resource(data, resources[data['f:resource_id']][0],
resources[data['f:resource_id']][1])
def get_meter_statistics(self, sample_filter, period=None):
"""Return an iterable of models.Statistics instances containing meter
statistics described by the query parameters.
The filter must have a meter value set.
.. note::
Due to HBase limitations the aggregations are implemented
in the driver itself, therefore this method will be quite slow
because of all the Thrift traffic it is going to create.
"""
meter_table = self.conn.table(self.METER_TABLE)
q, start, stop = make_query_from_filter(sample_filter)
meters = list(meter for (ignored, meter) in
meter_table.scan(filter=q, row_start=start,
row_stop=stop)
)
if sample_filter.start:
start_time = sample_filter.start
elif meters:
start_time = timeutils.parse_strtime(meters[-1]['f:timestamp'])
else:
start_time = None
if sample_filter.end:
end_time = sample_filter.end
elif meters:
end_time = timeutils.parse_strtime(meters[0]['f:timestamp'])
else:
end_time = None
results = []
if not period:
period = 0
period_start = start_time
period_end = end_time
# As our HBase meters are stored as newest-first, we need to iterate
# in the reverse order
for meter in meters[::-1]:
ts = timeutils.parse_strtime(meter['f:timestamp'])
if period:
offset = int(timeutils.delta_seconds(
start_time, ts) / period) * period
period_start = start_time + datetime.timedelta(0, offset)
if not len(results) or not results[-1].period_start == \
period_start:
if period:
period_end = period_start + datetime.timedelta(
0, period)
results.append(
models.Statistics(count=0,
min=0,
max=0,
avg=0,
sum=0,
period=period,
period_start=period_start,
period_end=period_end,
duration=None,
duration_start=None,
duration_end=None)
)
self._update_meter_stats(results[-1], meter)
return results
def make_sample(data):
"""Transform HBase fields to Sample model."""
data = json.loads(data['f:message'])
data['timestamp'] = timeutils.parse_strtime(data['timestamp'])
return models.Sample(**data)
def _timestamp_from_record_tuple(record):
"""Extract timestamp from HBase tuple record
"""
return timeutils.parse_strtime(record[1]['f:timestamp'])
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, start_timestamp_op=None,
end_timestamp=None, end_timestamp_op=None,
metaquery={}, resource=None):
"""Return an iterable of models.Resource instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param start_timestamp_op: Optional start time operator, like ge, gt.
:param end_timestamp: Optional modified timestamp end range.
:param end_timestamp_op: Optional end time operator, like lt, le.
:param metaquery: Optional dict with metadata to match on.
:param resource: Optional resource filter.
"""
def make_resource(data, first_ts, last_ts):
"""Transform HBase fields to Resource model."""
# convert HBase metadata e.g. f:r_display_name to display_name
data['f:metadata'] = dict((k[4:], v)
for k, v in data.iteritems()
if k.startswith('f:r_'))
return models.Resource(
resource_id=data['f:resource_id'],
first_sample_timestamp=first_ts,
last_sample_timestamp=last_ts,
project_id=data['f:project_id'],
source=data['f:source'],
user_id=data['f:user_id'],
metadata=data['f:metadata'],
meter=[
models.ResourceMeter(*(m[4:].split("!")))
for m in data
if m.startswith('f:m_')
],
)
resource_table = self.conn.table(self.RESOURCE_TABLE)
meter_table = self.conn.table(self.METER_TABLE)
q, start_row, stop_row = make_query(user=user,
project=project,
source=source,
resource=resource,
start=start_timestamp,
start_op=start_timestamp_op,
end=end_timestamp,
end_op=end_timestamp_op,
require_meter=False,
query_only=False)
LOG.debug("Query Meter table: %s" % q)
meters = meter_table.scan(filter=q, row_start=start_row,
row_stop=stop_row)
resources = {}
for resource_id, r_meters in itertools.groupby(
meters, key=lambda x: x[1]['f:resource_id']):
timestamps = tuple(timeutils.parse_strtime(m[1]['f:timestamp'])
for m in r_meters)
resources[resource_id] = (min(timestamps), max(timestamps))
# handle metaquery
if len(metaquery) > 0:
for ignored, data in resource_table.rows(resources.iterkeys()):
for k, v in metaquery.iteritems():
# if metaquery matches, yield the resource model
# e.g. metaquery: metadata.display_name
# equals
# HBase: f:r_display_name
if data['f:r_' + k.split('.', 1)[1]] == v:
yield make_resource(
data,
resources[data['f:resource_id']][0],
resources[data['f:resource_id']][1])
else:
for ignored, data in resource_table.rows(resources.iterkeys()):
yield make_resource(
data,
resources[data['f:resource_id']][0],
resources[data['f:resource_id']][1])
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, end_timestamp=None,
metaquery={}):
"""Return an iterable of dictionaries containing resource information.
:type end_timestamp: object
{ 'resource_id': UUID of the resource,
'project_id': UUID of project owning the resource,
'user_id': UUID of user owning the resource,
'timestamp': UTC datetime of last update to the resource,
'metadata': most current metadata for the resource,
'meter': list of the meters reporting data for the resource,
}
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param end_timestamp: Optional modified timestamp end range.
"""
q, start_row, end_row = make_query(user=user,
project=project,
source=source,
start=start_timestamp,
end=end_timestamp,
require_meter=False)
LOG.debug("q: %s" % q)
# TODO implement metaquery support
if len(metaquery) > 0:
raise NotImplementedError('metaquery not implemented')
resource_ids = {}
if start_timestamp or end_timestamp:
# Look for resources matching the above criteria and with
# events in the time range we care about, then change the
# resource query to return just those resources by id.
g = self.meter.scan(filter=q, row_start=start_row,
row_stop=end_row)
for ignored, data in g:
resource_ids[data['f:resource_id']] = data['f:resource_id']
q = make_query(user=user, project=project, source=source,
query_only=True, require_meter=False)
LOG.debug("q: %s" % q)
for resource_id, data in self.resource.scan(filter=q):
if not resource_ids or resource_id in resource_ids:
r = {'resource_id': resource_id,
'metadata': json.loads(data['f:metadata']),
'project_id': data['f:project_id'],
'received_timestamp': data['f:received_timestamp'],
'source': data['f:source'],
'timestamp':
timeutils.parse_strtime(data['f:timestamp']),
'user_id': data['f:user_id'],
'meter': []}
for m in data:
if m.startswith('f:m_'):
name, type, unit = m[4:].split("!")
r['meter'].append({"counter_name": name,
"counter_type": type,
"counter_unit": unit})
yield r
def get_meter_statistics(self, event_filter, period=None):
"""Return a dictionary containing meter statistics.
described by the query parameters.
The filter must have a meter value set.
{ 'min':
'max':
'avg':
'sum':
'count':
'period':
'period_start':
'period_end':
'duration':
'duration_start':
'duration_end':
}
.. note::
Due to HBase limitations the aggregations are implemented
in the driver itself, therefore this method will be quite slow
because of all the Thrift traffic it is going to create.
"""
q, start, stop = make_query_from_filter(event_filter)
meters = list(meter for (ignored, meter) in
self.meter.scan(filter=q,
row_start=start,
row_stop=stop)
)
start_time = event_filter.start \
or timeutils.parse_strtime(meters[-1]['f:timestamp'])
end_time = event_filter.end \
or timeutils.parse_strtime(meters[0]['f:timestamp'])
results = []
if not period:
period = 0
period_start = start_time
period_end = end_time
# As our HBase meters are stored as newest-first, we need to iterate
# in the reverse order
for meter in meters[::-1]:
ts = timeutils.parse_strtime(meter['f:timestamp'])
if period:
offset = int(timeutils.delta_seconds(
start_time, ts) / period) * period
period_start = start_time + datetime.timedelta(0, offset)
if not len(results) or not results[-1]['period_start'] == \
period_start:
if period:
period_end = period_start + datetime.timedelta(
0, period)
results.append({'count': 0,
'min': 0,
'max': 0,
'avg': 0,
'sum': 0,
'period': period,
'period_start': period_start,
'period_end': period_end,
'duration': None,
'duration_start': None,
'duration_end': None,
})
self._update_meter_stats(results[-1], meter)
return list(results)
def _timestamp_from_record_tuple(record):
"""Extract timestamp from HBase tuple record
"""
return timeutils.parse_strtime(record[1]['f:timestamp'])
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, start_timestamp_op=None,
end_timestamp=None, end_timestamp_op=None,
metaquery={}, resource=None, pagination=None):
"""Return an iterable of models.Resource instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param start_timestamp_op: Optional start time operator, like ge, gt.
:param end_timestamp: Optional modified timestamp end range.
:param end_timestamp_op: Optional end time operator, like lt, le.
:param metaquery: Optional dict with metadata to match on.
:param resource: Optional resource filter.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
def make_resource(data, first_ts, last_ts):
"""Transform HBase fields to Resource model."""
# convert HBase metadata e.g. f:r_display_name to display_name
data['f:metadata'] = _metadata_from_document(data)
return models.Resource(
resource_id=data['f:resource_id'],
first_sample_timestamp=first_ts,
last_sample_timestamp=last_ts,
project_id=data['f:project_id'],
source=data['f:source'],
user_id=data['f:user_id'],
metadata=data['f:metadata'],
)
meter_table = self.conn.table(self.METER_TABLE)
q, start_row, stop_row = make_query(user=user,
project=project,
source=source,
resource=resource,
start=start_timestamp,
start_op=start_timestamp_op,
end=end_timestamp,
end_op=end_timestamp_op,
require_meter=False,
query_only=False)
LOG.debug(_("Query Meter table: %s") % q)
meters = meter_table.scan(filter=q, row_start=start_row,
row_stop=stop_row)
# We have to sort on resource_id before we can group by it. According
# to the itertools documentation a new group is generated when the
# value of the key function changes (it breaks there).
meters = sorted(meters, key=_resource_id_from_record_tuple)
for resource_id, r_meters in itertools.groupby(
meters, key=_resource_id_from_record_tuple):
meter_rows = [data[1] for data in sorted(
r_meters, key=_timestamp_from_record_tuple)]
latest_data = meter_rows[-1]
min_ts = timeutils.parse_strtime(meter_rows[0]['f:timestamp'])
max_ts = timeutils.parse_strtime(latest_data['f:timestamp'])
if metaquery:
for k, v in metaquery.iteritems():
if latest_data['f:r_' + k.split('.', 1)[1]] == v:
yield make_resource(
latest_data,
min_ts,
max_ts
)
else:
yield make_resource(
latest_data,
min_ts,
max_ts
)
def make_sample(data):
"""Transform HBase fields to Sample model."""
data = json.loads(data['f:message'])
data['timestamp'] = timeutils.parse_strtime(data['timestamp'])
return models.Sample(**data)
def get_meter_statistics(self, sample_filter, period=None, groupby=None):
"""Return an iterable of models.Statistics instances containing meter
statistics described by the query parameters.
The filter must have a meter value set.
.. note::
Due to HBase limitations the aggregations are implemented
in the driver itself, therefore this method will be quite slow
because of all the Thrift traffic it is going to create.
"""
if groupby:
raise NotImplementedError("Group by not implemented.")
meter_table = self.conn.table(self.METER_TABLE)
q, start, stop = make_query_from_filter(sample_filter)
meters = list(meter for (ignored, meter) in meter_table.scan(
filter=q, row_start=start, row_stop=stop))
if sample_filter.start:
start_time = sample_filter.start
elif meters:
start_time = timeutils.parse_strtime(meters[-1]['f:timestamp'])
else:
start_time = None
if sample_filter.end:
end_time = sample_filter.end
elif meters:
end_time = timeutils.parse_strtime(meters[0]['f:timestamp'])
else:
end_time = None
results = []
if not period:
period = 0
period_start = start_time
period_end = end_time
# As our HBase meters are stored as newest-first, we need to iterate
# in the reverse order
for meter in meters[::-1]:
ts = timeutils.parse_strtime(meter['f:timestamp'])
if period:
offset = int(
timeutils.delta_seconds(start_time, ts) / period) * period
period_start = start_time + datetime.timedelta(0, offset)
if not len(results) or not results[-1].period_start == \
period_start:
if period:
period_end = period_start + datetime.timedelta(0, period)
results.append(
models.Statistics(unit='',
count=0,
min=0,
max=0,
avg=0,
sum=0,
period=period,
period_start=period_start,
period_end=period_end,
duration=None,
duration_start=None,
duration_end=None,
groupby=None))
self._update_meter_stats(results[-1], meter)
return results
def get_resources(self, user=None, project=None, source=None,
start_timestamp=None, start_timestamp_op=None,
end_timestamp=None, end_timestamp_op=None,
metaquery={}, resource=None, pagination=None):
"""Return an iterable of models.Resource instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param source: Optional source filter.
:param start_timestamp: Optional modified timestamp start range.
:param start_timestamp_op: Optional start time operator, like ge, gt.
:param end_timestamp: Optional modified timestamp end range.
:param end_timestamp_op: Optional end time operator, like lt, le.
:param metaquery: Optional dict with metadata to match on.
:param resource: Optional resource filter.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
def make_resource(data, first_ts, last_ts, meter_refs):
"""Transform HBase fields to Resource model."""
# convert HBase metadata e.g. f:r_display_name to display_name
data['f:metadata'] = _metadata_from_document(data)
return models.Resource(
resource_id=data['f:resource_id'],
first_sample_timestamp=first_ts,
last_sample_timestamp=last_ts,
project_id=data['f:project_id'],
source=data['f:source'],
user_id=data['f:user_id'],
metadata=data['f:metadata'],
meter=[
models.ResourceMeter(*(m.split("!")))
for m in meter_refs
],
)
meter_table = self.conn.table(self.METER_TABLE)
q, start_row, stop_row = make_query(user=user,
project=project,
source=source,
resource=resource,
start=start_timestamp,
start_op=start_timestamp_op,
end=end_timestamp,
end_op=end_timestamp_op,
require_meter=False,
query_only=False)
LOG.debug("Query Meter table: %s" % q)
meters = meter_table.scan(filter=q, row_start=start_row,
row_stop=stop_row)
# We have to sort on resource_id before we can group by it. According
# to the itertools documentation a new group is generated when the
# value of the key function changes (it breaks there).
meters = sorted(meters, key=_resource_id_from_record_tuple)
for resource_id, r_meters in itertools.groupby(
meters, key=_resource_id_from_record_tuple):
meter_rows = [data[1] for data in sorted(
r_meters, key=_timestamp_from_record_tuple)]
meter_references = [
_format_meter_reference(m['f:counter_name'],
m['f:counter_type'],
m['f:counter_unit'])
for m in meter_rows]
latest_data = meter_rows[-1]
min_ts = timeutils.parse_strtime(meter_rows[0]['f:timestamp'])
max_ts = timeutils.parse_strtime(latest_data['f:timestamp'])
if metaquery:
for k, v in metaquery.iteritems():
if latest_data['f:r_' + k.split('.', 1)[1]] == v:
yield make_resource(
latest_data,
min_ts,
max_ts,
meter_references
)
else:
yield make_resource(
latest_data,
min_ts,
max_ts,
meter_references
)
