def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery=None,
limit=None):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param limit: Maximum number of results to return.
"""
if limit == 0:
return
s_filter = storage.SampleFilter(user=user,
project=project,
source=source,
metaquery=metaquery,
resource=resource)
# NOTE(gordc): get latest sample of each meter/resource. we do not
# filter here as we want to filter only on latest record.
session = self._engine_facade.get_session()
subq = session.query(func.max(models.Sample.id).label('id')).join(
models.Resource,
models.Resource.internal_id == models.Sample.resource_id).group_by(
models.Sample.meter_id, models.Resource.resource_id)
if resource:
subq = subq.filter(models.Resource.resource_id == resource)
subq = subq.subquery()
# get meter details for samples.
query_sample = (session.query(
models.Sample.meter_id, models.Meter.name, models.Meter.type,
models.Meter.unit, models.Resource.resource_id,
models.Resource.project_id,
models.Resource.source_id, models.Resource.user_id).join(
subq, subq.c.id == models.Sample.id).join(
models.Meter,
models.Meter.id == models.Sample.meter_id).join(
models.Resource, models.Resource.internal_id ==
models.Sample.resource_id))
query_sample = make_query_from_filter(session,
query_sample,
s_filter,
require_meter=False)
query_sample = query_sample.limit(limit) if limit else query_sample
for row in query_sample.all():
yield api_models.Meter(name=row.name,
type=row.type,
unit=row.unit,
resource_id=row.resource_id,
project_id=row.project_id,
source=row.source_id,
user_id=row.user_id)
def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery={}):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
q = {}
if user is not None:
q['user_id'] = user
if project is not None:
q['project_id'] = project
if resource is not None:
q['_id'] = resource
if source is not None:
q['source'] = source
q.update(metaquery)
for r in self.db.resource.find(q):
for r_meter in r['meter']:
yield models.Meter(
name=r_meter['counter_name'],
type=r_meter['counter_type'],
# Return empty string if 'counter_unit' is not valid for
# backward compaitiblity.
unit=r_meter.get('counter_unit', ''),
resource_id=r['_id'],
project_id=r['project_id'],
user_id=r['user_id'],
)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery={}):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
q = make_query(user=user,
project=project,
resource=resource,
source=source,
require_meter=False,
query_only=True)
LOG.debug("Query Resource table: %s" % q)
# handle metaquery
if len(metaquery) > 0:
meta_q = []
for k, v in metaquery.iteritems():
meta_q.append(
"SingleColumnValueFilter ('f', '%s', =, 'binary:%s')" %
('r_' + k.split('.', 1)[1], v))
meta_q = " AND ".join(meta_q)
# join query and metaquery
if q is not None:
q += " AND " + meta_q
else:
q = meta_q # metaquery only
gen = self.resource.scan(filter=q)
for ignored, data in gen:
# Meter columns are stored like this:
# "m_{counter_name}|{counter_type}|{counter_unit}" => "1"
# where 'm' is a prefix (m for meter), value is always set to 1
meter = None
for m in data:
if m.startswith('f:m_'):
meter = m
break
if meter is None:
continue
name, type, unit = meter[4:].split("!")
yield models.Meter(
name=name,
type=type,
unit=unit,
resource_id=data['f:resource_id'],
project_id=data['f:project_id'],
source=data['f:source'],
user_id=data['f:user_id'],
)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery=None,
pagination=None):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param pagination: Optional pagination query.
"""
metaquery = metaquery or {}
if pagination:
raise ceilometer.NotImplementedError(
_('Pagination not implemented'))
with self.conn_pool.connection() as conn:
resource_table = conn.table(self.RESOURCE_TABLE)
q = hbase_utils.make_query(metaquery=metaquery,
user_id=user,
project_id=project,
resource_id=resource,
source=source)
LOG.debug(_("Query Resource table: %s") % q)
gen = resource_table.scan(filter=q)
# We need result set to be sure that user doesn't receive several
# same meters. Please see bug
# https://bugs.launchpad.net/ceilometer/+bug/1301371
result = set()
for ignored, data in gen:
flatten_result, s, meters, md = hbase_utils.deserialize_entry(
data)
for m in meters:
_m_rts, m_source, m_raw = m[0].split("+")
name, type, unit = m_raw.split('!')
meter_dict = {
'name': name,
'type': type,
'unit': unit,
'resource_id': flatten_result['resource_id'],
'project_id': flatten_result['project_id'],
'user_id': flatten_result['user_id']
}
frozen_meter = frozenset(meter_dict.items())
if frozen_meter in result:
continue
result.add(frozen_meter)
meter_dict.update(
{'source': m_source if m_source else None})
yield models.Meter(**meter_dict)
def get_meters(user=None,
project=None,
resource=None,
source=None,
metaquery={}):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
session = sqlalchemy_session.get_session()
# Meter table will store large records and join with resource
# will be very slow.
# subquery_meter is used to reduce meter records
# by selecting a record for each (resource_id, counter_name).
# max() is used to choice a meter record, so the latest record
# is selected for each (resource_id, counter_name).
#
subquery_meter = session.query(func.max(Meter.id).label('id')).\
group_by(Meter.resource_id, Meter.counter_name).subquery()
# The SQL of query_meter is essentially:
#
# SELECT meter.* FROM meter INNER JOIN
# (SELECT max(meter.id) AS id FROM meter
# GROUP BY meter.resource_id, meter.counter_name) AS anon_2
# ON meter.id = anon_2.id
#
query_meter = session.query(Meter).\
join(subquery_meter, Meter.id == subquery_meter.c.id)
alias_meter = aliased(Meter, query_meter.subquery())
query = session.query(Resource, alias_meter).join(
alias_meter, Resource.id == alias_meter.resource_id)
if user is not None:
query = query.filter(Resource.user_id == user)
if source is not None:
query = query.filter(Resource.sources.any(id=source))
if resource:
query = query.filter(Resource.id == resource)
if project is not None:
query = query.filter(Resource.project_id == project)
if metaquery:
raise NotImplementedError('metaquery not implemented')
for resource, meter in query.all():
yield api_models.Meter(name=meter.counter_name,
type=meter.counter_type,
unit=meter.counter_unit,
resource_id=resource.id,
project_id=resource.project_id,
source=resource.sources[0].id,
user_id=resource.user_id)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery=None,
pagination=None):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError('Pagination not implemented')
s_filter = storage.SampleFilter(user=user,
project=project,
source=source,
metaquery=metaquery,
resource=resource)
session = self._engine_facade.get_session()
# sample_subq is used to reduce sample records
# by selecting a record for each (resource_id, meter_id).
# max() is used to choice a sample record, so the latest record
# is selected for each (resource_id, meter_id).
sample_subq = (session.query(func.max(
models.Sample.id).label('id')).group_by(models.Sample.meter_id,
models.Sample.resource_id))
sample_subq = sample_subq.subquery()
# SELECT sample.* FROM sample INNER JOIN
# (SELECT max(sample.id) AS id FROM sample
# GROUP BY sample.resource_id, sample.meter_id) AS anon_2
# ON sample.id = anon_2.id
query_sample = (session.query(models.MeterSample).join(
sample_subq, models.MeterSample.id == sample_subq.c.id))
query_sample = make_query_from_filter(session,
query_sample,
s_filter,
require_meter=False)
for sample in query_sample.all():
yield api_models.Meter(name=sample.counter_name,
type=sample.counter_type,
unit=sample.counter_unit,
resource_id=sample.resource_id,
project_id=sample.project_id,
source=sample.source_id,
user_id=sample.user_id)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery=None,
limit=None):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param limit: Maximum number of results to return.
"""
if limit == 0:
return
metaquery = pymongo_utils.improve_keys(metaquery, metaquery=True) or {}
q = {}
if user == 'None':
q['user_id'] = None
elif user is not None:
q['user_id'] = user
if project == 'None':
q['project_id'] = None
elif project is not None:
q['project_id'] = project
if resource == 'None':
q['_id'] = None
elif resource is not None:
q['_id'] = resource
if source is not None:
q['source'] = source
q.update(metaquery)
count = 0
for r in self.db.resource.find(q):
for r_meter in r['meter']:
if limit and count >= limit:
return
else:
count += 1
yield models.Meter(
name=r_meter['counter_name'],
type=r_meter['counter_type'],
# Return empty string if 'counter_unit' is not valid for
# backward compatibility.
unit=r_meter.get('counter_unit', ''),
resource_id=r['_id'],
project_id=r['project_id'],
source=r['source'],
user_id=r['user_id'],
)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery={},
limit=None,
marker_pairs=None,
sort_key=None,
sort_dir=None):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param limit: Number of documents should be returned.
:param marker_pairs: Attribute-value pairs to identify the last item of
the previous page.
:param sort_key: Attribute by which results be sorted.
:param sort_dir: Direction with which results be sorted(asc, desc).
"""
q = {}
if user is not None:
q['user_id'] = user
if project is not None:
q['project_id'] = project
if resource is not None:
q['_id'] = resource
if source is not None:
q['source'] = source
q.update(metaquery)
marker = self._get_marker(self.db.resource, marker_pairs=marker_pairs)
sort_keys = base._handle_sort_key('meter', sort_key)
for r in self.paginate_query(q,
self.db.resource,
limit=limit,
marker=marker,
sort_keys=sort_keys,
sort_dir=sort_dir):
for r_meter in r['meter']:
yield models.Meter(
name=r_meter['counter_name'],
type=r_meter['counter_type'],
# Return empty string if 'counter_unit' is not valid for
# backward compatibility.
unit=r_meter.get('counter_unit', ''),
resource_id=r['_id'],
project_id=r['project_id'],
source=r['source'],
user_id=r['user_id'],
)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
limit=None,
metaquery=None,
pagination=None):
"""Return an iterable of dictionaries containing meter information.
{ 'name': name of the meter,
'type': type of the meter (gauge, delta, cumulative),
'resource_id': UUID of the resource,
'project_id': UUID of project owning the resource,
'user_id': UUID of user owning the resource,
}
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param limit: Maximum number of results to return.
:param metaquery: Optional dict with metadata to match on.
:param pagination: Optional pagination query.
"""
if pagination:
raise ceilometer.NotImplementedError('Pagination not implemented')
if metaquery:
raise ceilometer.NotImplementedError('Metaquery not implemented')
_dimensions = dict(user_id=user,
project_id=project,
resource_id=resource,
source=source)
_dimensions = {k: v for k, v in _dimensions.items() if v is not None}
_search_kwargs = {'dimensions': _dimensions}
if limit:
_search_kwargs['limit'] = limit
for metric in self.mc.metrics_list(**_search_kwargs):
yield api_models.Meter(
name=metric['name'],
type=metric['dimensions'].get('type') or 'cumulative',
unit=metric['dimensions'].get('unit'),
resource_id=metric['dimensions'].get('resource_id'),
project_id=metric['dimensions'].get('project_id'),
source=metric['dimensions'].get('source'),
user_id=metric['dimensions'].get('user_id'))
def get_meters(self, user=None, project=None, resource=None, source=None,
metaquery=None, limit=None):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param limit: Maximum number of results to return.
"""
if limit == 0:
return
metaquery = metaquery or {}
with self.conn_pool.connection() as conn:
resource_table = conn.table(self.RESOURCE_TABLE)
q = hbase_utils.make_query(metaquery=metaquery, user_id=user,
project_id=project,
resource_id=resource,
source=source)
LOG.debug("Query Resource table: %s", q)
gen = resource_table.scan(filter=q)
# We need result set to be sure that user doesn't receive several
# same meters. Please see bug
# https://bugs.launchpad.net/ceilometer/+bug/1301371
result = set()
for ignored, data in gen:
flatten_result, s, meters, md = hbase_utils.deserialize_entry(
data)
for m in meters:
if limit and len(result) >= limit:
return
_m_rts, m_source, name, m_type, unit = m[0]
meter_dict = {'name': name,
'type': m_type,
'unit': unit,
'resource_id': flatten_result['resource_id'],
'project_id': flatten_result['project_id'],
'user_id': flatten_result['user_id']}
frozen_meter = frozenset(meter_dict.items())
if frozen_meter in result:
continue
result.add(frozen_meter)
meter_dict.update({'source': m_source
if m_source else None})
yield models.Meter(**meter_dict)
def search_results_to_meters(results, limit=None, unique=False):
"""Transforms results of the search to the Meter instances."""
if unique:
meter_names = set()
count = 0
for record in results['hits']['hits']:
for meter_name, specials in six.iteritems(record['_source']['meters']):
meter_name = urllib.unquote(meter_name)
if limit and count >= limit:
return
else:
count += 1
if unique:
if meter_name in meter_names:
continue
else:
meter_names.add(meter_name)
yield models.Meter(name=unquote(meter_name),
type=specials['type'],
unit=specials.get('unit', ''),
resource_id=None,
project_id=None,
source=None,
user_id=None)
else:
yield models.Meter(
name=unquote(meter_name),
type=specials['type'],
unit=specials.get('unit', ''),
resource_id=record['_id'],
source=record['_source']["source"],
project_id=record['_source'].get('project_id'),
user_id=record['_source'].get('user_id'),
)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery={},
pagination=None):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
with self.conn_pool.connection() as conn:
resource_table = conn.table(self.RESOURCE_TABLE)
q = make_query(metaquery=metaquery,
user_id=user,
project_id=project,
resource_id=resource,
source=source)
LOG.debug(_("Query Resource table: %s") % q)
gen = resource_table.scan(filter=q)
for ignored, data in gen:
flatten_result, s, m, md = deserialize_entry(data)
if not m:
continue
# Meter table may have only one "meter" and "source". That's
# why only first lists element is get in this method
name, type, unit = m[0].split("!")
yield models.Meter(
name=name,
type=type,
unit=unit,
resource_id=flatten_result['resource_id'],
project_id=flatten_result['project_id'],
source=s[0] if s else None,
user_id=flatten_result['user_id'],
)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery={}):
"""Return an iterable of models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional resource filter.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
q, ignored, ignored = make_query(user=user,
project=project,
resource=resource,
source=source,
require_meter=False)
LOG.debug("q: %s" % q)
# TODO implement metaquery support
if len(metaquery) > 0:
raise NotImplementedError('metaquery not implemented')
gen = self.resource.scan(filter=q)
for ignored, data in gen:
# Meter columns are stored like this:
# "m_{counter_name}|{counter_type}|{counter_unit}" => "1"
# where 'm' is a prefix (m for meter), value is always set to 1
meter = None
for m in data:
if m.startswith('f:m_'):
meter = m
break
if meter is None:
continue
name, type, unit = meter[4:].split("!")
yield models.Meter(
name=name,
type=type,
unit=unit,
resource_id=data['f:resource_id'],
project_id=data['f:project_id'],
user_id=data['f:user_id'],
)
def get_meters(user=None,
project=None,
resource=None,
source=None,
metaquery={}):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
"""
session = sqlalchemy_session.get_session()
query = session.query(Resource)
if user is not None:
query = query.filter(Resource.user_id == user)
if source is not None:
query = query.filter(Resource.sources.any(id=source))
if resource:
query = query.filter(Resource.id == resource)
if project is not None:
query = query.filter(Resource.project_id == project)
query = query.options(
sqlalchemy_session.sqlalchemy.orm.joinedload('meters'))
if metaquery:
raise NotImplementedError('metaquery not implemented')
for resource in query.all():
meter_names = set()
for meter in resource.meters:
if meter.counter_name in meter_names:
continue
meter_names.add(meter.counter_name)
yield api_models.Meter(
name=meter.counter_name,
type=meter.counter_type,
unit=meter.counter_unit,
resource_id=resource.id,
project_id=resource.project_id,
source=resource.sources[0].id,
user_id=resource.user_id,
)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery=None,
pagination=None):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param pagination: Optional pagination query.
"""
if pagination:
raise ceilometer.NotImplementedError('Pagination not implemented')
s_filter = storage.SampleFilter(user=user,
project=project,
source=source,
metaquery=metaquery,
resource=resource)
# NOTE(gordc): get latest sample of each meter/resource. we do not
# filter here as we want to filter only on latest record.
session = self._engine_facade.get_session()
subq = session.query(func.max(models.Sample.id).label('id')).join(
models.Resource,
models.Resource.internal_id == models.Sample.resource_id).group_by(
models.Sample.meter_id, models.Resource.resource_id)
if resource:
subq = subq.filter(models.Resource.resource_id == resource)
subq = subq.subquery()
# get meter details for samples.
query_sample = (session.query(
models.Sample.meter_id, models.Meter.name, models.Meter.type,
models.Meter.unit, models.Resource.resource_id,
models.Resource.project_id,
models.Resource.source_id, models.Resource.user_id).join(
subq, subq.c.id == models.Sample.id).join(
models.Meter,
models.Meter.id == models.Sample.meter_id).join(
models.Resource, models.Resource.internal_id ==
models.Sample.resource_id))
query_sample = make_query_from_filter(session,
query_sample,
s_filter,
require_meter=False)
if project is None:
query_sample = (query_sample.limit(
cfg.CONF.sqlalchemy.max_sample_limit))
for row in query_sample.all():
yield api_models.Meter(name=row.name,
type=row.type,
unit=row.unit,
resource_id=row.resource_id,
project_id=row.project_id,
source=row.source_id,
user_id=row.user_id)
def get_meters(self,
user=None,
project=None,
resource=None,
source=None,
metaquery=None,
pagination=None):
"""Return an iterable of api_models.Meter instances
:param user: Optional ID for user that owns the resource.
:param project: Optional ID for project that owns the resource.
:param resource: Optional ID of the resource.
:param source: Optional source filter.
:param metaquery: Optional dict with metadata to match on.
:param pagination: Optional pagination query.
"""
if pagination:
raise NotImplementedError('Pagination not implemented')
metaquery = metaquery or {}
def _apply_filters(query):
# TODO(gordc) this should be merged with make_query_from_filter
for column, value in [(models.Sample.resource_id, resource),
(models.Sample.user_id, user),
(models.Sample.project_id, project),
(models.Sample.source_id, source)]:
if value:
query = query.filter(column == value)
if metaquery:
query = apply_metaquery_filter(session, query, metaquery)
return query
session = self._engine_facade.get_session()
# sample_subq is used to reduce sample records
# by selecting a record for each (resource_id, meter_id).
# max() is used to choice a sample record, so the latest record
# is selected for each (resource_id, meter_id).
sample_subq = session.query(
func.max(models.Sample.id).label('id'))\
.group_by(models.Sample.meter_id, models.Sample.resource_id)
sample_subq = sample_subq.subquery()
# SELECT sample.* FROM sample INNER JOIN
# (SELECT max(sample.id) AS id FROM sample
# GROUP BY sample.resource_id, sample.meter_id) AS anon_2
# ON sample.id = anon_2.id
query_sample = session.query(models.MeterSample).\
join(sample_subq, models.MeterSample.id == sample_subq.c.id)
query_sample = _apply_filters(query_sample)
for sample in query_sample.all():
yield api_models.Meter(name=sample.counter_name,
type=sample.counter_type,
unit=sample.counter_unit,
resource_id=sample.resource_id,
project_id=sample.project_id,
source=sample.source_id,
user_id=sample.user_id)
