def summary(self,
begin=None,
end=None,
tenant_id=None,
service=None,
groupby=None,
all_tenants=False):
"""Return the summary to pay for a given period.
"""
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if all_tenants:
tenant_id = None
else:
tenant_context = pecan.request.context.tenant
tenant_id = tenant_context if not tenant_id else tenant_id
policy.authorize(pecan.request.context, 'report:get_summary',
{"tenant_id": tenant_id})
storage = pecan.request.storage_backend
summarymodels = []
results = storage.get_total(begin,
end,
tenant_id,
service,
groupby=groupby)
for result in results:
summarymodel = report_models.SummaryModel(**result)
summarymodels.append(summarymodel)
return report_models.SummaryCollectionModel(summary=summarymodels)
def get(self,
offset=0,
limit=100,
scope_id=None,
scope_key=None,
fetcher=None,
collector=None):
policy.authorize(
flask.request.context, 'scope:get_state',
{'tenant_id': scope_id or flask.request.context.project_id})
results = storage_state.StateManager().get_all(
identifier=scope_id,
scope_key=scope_key,
fetcher=fetcher,
collector=collector,
offset=offset,
limit=limit,
)
if len(results) < 1:
raise http_exceptions.NotFound(
"No resource found for provided filters.")
return {
'results': [{
'scope_id': r.identifier,
'scope_key': r.scope_key,
'fetcher': r.fetcher,
'collector': r.collector,
'state': str(r.state),
} for r in results]
}
def get(self,
groupby=None,
filters={},
begin=None,
end=None,
offset=0,
limit=100):
policy.authorize(flask.request.context, 'summary:get_summary',
{'tenant_id': flask.request.context.project_id})
begin = begin or utils.get_month_start()
end = end or utils.get_next_month()
if not flask.request.context.is_admin:
filters['project_id'] = flask.request.context.project_id
total = self._storage.total(
begin=begin,
end=end,
groupby=groupby,
filters=filters,
offset=offset,
limit=limit,
paginate=True,
)
columns = []
if len(total['results']) > 0:
columns = list(total['results'][0].keys())
return {
'total': total['total'],
'columns': columns,
'results': [list(res.values()) for res in total['results']]
}
def get_all(self,
begin=None,
end=None,
tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.authorize(pecan.request.context, 'storage:list_data_frames', {})
scope_key = CONF.collect.scope_key
backend = pecan.request.storage_backend
dataframes = []
group_filters = {scope_key: tenant_id} if tenant_id else None
if begin:
begin = ck_utils.dt2ts(begin)
if end:
end = ck_utils.dt2ts(end)
try:
resp = backend.retrieve(begin,
end,
group_filters=group_filters,
metric_types=resource_type,
paginate=False)
except storage.NoTimeFrame:
return storage_models.DataFrameCollection(dataframes=[])
for frame in resp['dataframes']:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
# This means we use a v1 storage backend
if 'desc' in data.keys():
desc = data['desc']
else:
desc = data['metadata'].copy()
desc.update(data.get('groupby', {}))
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
if frame_tenant is None:
frame_tenant = desc[scope_key]
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
return storage_models.DataFrameCollection(dataframes=dataframes)
def _route(self, args, request):
try:
policy.authorize(request.context, 'rating:module_config', {})
except policy.PolicyNotAuthorized as e:
pecan.abort(403, six.text_type(e))
return super(RatingRestControllerBase, self)._route(args, request)
def total(self,
begin=None,
end=None,
tenant_id=None,
service=None,
all_tenants=False):
"""Return the amount to pay for a given period.
"""
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if all_tenants:
tenant_id = None
else:
tenant_context = pecan.request.context.tenant
tenant_id = tenant_context if not tenant_id else tenant_id
policy.authorize(pecan.request.context, 'report:get_total',
{"tenant_id": tenant_id})
storage = pecan.request.storage_backend
# FIXME(sheeprine): We should filter on user id.
# Use keystone token information by default but make it overridable and
# enforce it by policy engine
total = storage.get_total(begin, end, tenant_id, service)
# TODO(Aaron): `get_total` return a list of dict,
# Get value of rate from index[0]
total = total[0].get('rate', decimal.Decimal('0'))
return total if total else decimal.Decimal('0')
def test_ignore_case_role_check(self):
lowercase_action = "test:lowercase_admin"
uppercase_action = "test:uppercase_admin"
admin_context = context.RequestContext('admin',
'fake',
roles=['AdMiN'])
policy.authorize(admin_context, lowercase_action, self.target)
policy.authorize(admin_context, uppercase_action, self.target)
def reload_modules(self):
"""Trigger a rating module list reload.
"""
policy.authorize(pecan.request.context, 'rating:module_config', {})
self.modules.reload_extensions()
self.module_config.reload_extensions()
self.module_config.expose_modules()
def post(self,
scope_id,
scope_key=None,
fetcher=None,
collector=None,
active=None):
policy.authorize(
flask.request.context, 'scope:post_state',
{'tenant_id': scope_id or flask.request.context.project_id})
results = self._storage_state.get_all(identifier=scope_id)
if len(results) >= 1:
LOG.debug(
"There is already a scope with ID [%s], "
"scopes found: [%s].", scope_id, results)
raise http_exceptions.NotFound("Cannot create a scope with an "
"already existing scope_id: %s." %
scope_id)
LOG.debug(
"Creating storage scope with data: [scope_id=%s, "
"scope_key=%s, fetcher=%s, collector=%s, active=%s].", scope_id,
scope_key, fetcher, collector, active)
self._storage_state.create_scope(scope_id,
None,
fetcher=fetcher,
collector=collector,
scope_key=scope_key,
active=active)
storage_scopes = self._storage_state.get_all(identifier=scope_id)
update_storage_scope = storage_scopes[0]
last_processed_timestamp = None
if update_storage_scope.last_processed_timestamp.isoformat():
last_processed_timestamp =\
update_storage_scope.last_processed_timestamp.isoformat()
return {
'scope_id':
update_storage_scope.identifier,
'scope_key':
update_storage_scope.scope_key,
'fetcher':
update_storage_scope.fetcher,
'collector':
update_storage_scope.collector,
'state':
last_processed_timestamp,
'last_processed_timestamp':
last_processed_timestamp,
'active':
update_storage_scope.active,
'scope_activation_toggle_date':
update_storage_scope.scope_activation_toggle_date.isoformat()
}
def put(self,
all_scopes=False,
scope_id=None,
scope_key=None,
fetcher=None,
collector=None,
last_processed_timestamp=None,
state=None):
policy.authorize(
flask.request.context, 'scope:reset_state',
{'project_id': scope_id or flask.request.context.project_id})
if not all_scopes and scope_id is None:
raise http_exceptions.BadRequest(
"Either all_scopes or a scope_id should be specified.")
if not state and not last_processed_timestamp:
raise http_exceptions.BadRequest(
"Variables 'state' and 'last_processed_timestamp' cannot be "
"empty/None. We expect at least one of them.")
if state:
LOG.warning("The use of 'state' variable is deprecated, and will "
"be removed in the next upcomming release. You should "
"consider using 'last_processed_timestamp' variable.")
results = self._storage_state.get_all(
identifier=scope_id,
scope_key=scope_key,
fetcher=fetcher,
collector=collector,
)
if len(results) < 1:
raise http_exceptions.NotFound(
"No resource found for provided filters.")
serialized_results = [{
'scope_id': r.identifier,
'scope_key': r.scope_key,
'fetcher': r.fetcher,
'collector': r.collector,
} for r in results]
if not last_processed_timestamp:
last_processed_timestamp = state
self._client.cast({},
'reset_state',
res_data={
'scopes':
serialized_results,
'last_processed_timestamp':
last_processed_timestamp.isoformat()
})
return {}, 202
def get(self, name):
"""Query the enable state of a collector.
:param name: Name of the collector.
:return: State of the collector.
"""
policy.authorize(pecan.request.context, 'collector:get_state', {})
enabled = self._db.get_state('collector_{}'.format(name))
collector = collector_models.CollectorInfos(name=name, enabled=enabled)
return collector
def delete(self, service):
"""Delete a service to collector mapping.
:param service: Name of the service to filter on.
"""
policy.authorize(pecan.request.context, 'collector:manage_mapping', {})
try:
self._db.delete_mapping(service)
except db_api.NoSuchMapping as e:
pecan.abort(404, six.text_type(e))
def post(self, collector, service):
"""Create a service to collector mapping.
:param collector: Name of the collector to apply mapping on.
:param service: Name of the service to apply mapping on.
"""
policy.authorize(pecan.request.context, 'collector:manage_mapping', {})
new_mapping = self._db.set_mapping(service, collector)
return collector_models.ServiceToCollectorMapping(
service=new_mapping.service, collector=new_mapping.collector)
def post(self, fruit=None):
policy.authorize(flask.request.context, 'example:submit_fruit', {})
if not fruit:
raise http_exceptions.BadRequest('You must submit a fruit', )
if fruit not in ['banana', 'strawberry']:
raise http_exceptions.Forbidden(
'You submitted a forbidden fruit', )
return {
'message': 'Your fruit is a ' + fruit,
}
def get_all(self, begin=None, end=None, tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
project_id = tenant_id or pecan.request.context.project_id
policy.authorize(pecan.request.context, 'storage:list_data_frames', {
'tenant_id': project_id,
})
scope_key = CONF.collect.scope_key
backend = pecan.request.storage_backend
dataframes = []
filters = {scope_key: tenant_id} if tenant_id else None
try:
resp = backend.retrieve(
begin, end,
filters=filters,
metric_types=resource_type,
paginate=False)
except storage.NoTimeFrame:
return storage_models.DataFrameCollection(dataframes=[])
for frame in resp['dataframes']:
frame_tenant = None
for type_, points in frame.itertypes():
resources = []
for point in points:
resource = storage_models.RatedResource(
service=type_,
desc=point.desc,
volume=point.qty,
rating=point.price)
if frame_tenant is None:
# NOTE(jferrieu): Since DataFrame/DataPoint
# implementation patch we cannot guarantee
# anymore that a DataFrame does contain a scope_id
# therefore the __UNDEF__ default value has been
# retained to maintain backward compatibility
# if it would occur being absent
frame_tenant = point.desc.get(scope_key, '__UNDEF__')
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=tzutils.local_to_utc(frame.start, naive=True),
end=tzutils.local_to_utc(frame.end, naive=True),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
return storage_models.DataFrameCollection(dataframes=dataframes)
def delete(self, service):
"""Delete a service to collector mapping.
:param service: Name of the service to filter on.
"""
LOG.warning("Collector mappings are deprecated and shouldn't be used.")
policy.authorize(pecan.request.context, 'collector:manage_mapping', {})
try:
self._db.delete_mapping(service)
except db_api.NoSuchMapping as e:
pecan.abort(404, e.args[0])
def put(self, name, infos):
"""Set the enable state of a collector.
:param name: Name of the collector.
:param infos: New state informations of the collector.
:return: State of the collector.
"""
policy.authorize(pecan.request.context, 'collector:update_state', {})
enabled = self._db.set_state('collector_{}'.format(name),
infos.enabled)
collector = collector_models.CollectorInfos(name=name, enabled=enabled)
return collector
def get_one(self, service):
"""Return a service to collector mapping.
:param service: Name of the service to filter on.
"""
policy.authorize(pecan.request.context, 'collector:get_mapping', {})
try:
mapping = self._db.get_mapping(service)
return collector_models.ServiceToCollectorMapping(
**mapping.as_dict())
except db_api.NoSuchMapping as e:
pecan.abort(404, six.text_type(e))
def patch(self,
scope_id,
scope_key=None,
fetcher=None,
collector=None,
active=None):
policy.authorize(
flask.request.context, 'scope:patch_state',
{'tenant_id': scope_id or flask.request.context.project_id})
results = self._storage_state.get_all(identifier=scope_id, active=None)
if len(results) < 1:
raise http_exceptions.NotFound(
"No resource found for provided filters.")
if len(results) > 1:
LOG.debug(
"Too many resources found with the same scope_id [%s], "
"scopes found: [%s].", scope_id, results)
raise http_exceptions.NotFound("Too many resources found with "
"the same scope_id: %s." % scope_id)
scope_to_update = results[0]
LOG.debug("Executing update of storage scope: [%s].", scope_to_update)
self._storage_state.update_storage_scope(scope_to_update,
scope_key=scope_key,
fetcher=fetcher,
collector=collector,
active=active)
storage_scopes = self._storage_state.get_all(identifier=scope_id,
active=active)
update_storage_scope = storage_scopes[0]
return {
'scope_id':
update_storage_scope.identifier,
'scope_key':
update_storage_scope.scope_key,
'fetcher':
update_storage_scope.fetcher,
'collector':
update_storage_scope.collector,
'state':
update_storage_scope.state.isoformat(),
'last_processed_timestamp':
update_storage_scope.last_processed_timestamp.isoformat(),
'active':
update_storage_scope.active,
'scope_activation_toggle_date':
update_storage_scope.scope_activation_toggle_date.isoformat()
}
def get_all(self, collector=None):
"""Return the list of every services mapped to a collector.
:param collector: Filter on the collector name.
:return: Service to collector mappings collection.
"""
policy.authorize(pecan.request.context, 'collector:list_mappings', {})
mappings = [
collector_models.ServiceToCollectorMapping(**mapping.as_dict())
for mapping in self._db.list_mappings(collector)
]
return collector_models.ServiceToCollectorMappingCollection(
mappings=mappings)
def get_one(self, service):
"""Return a service to collector mapping.
:param service: Name of the service to filter on.
"""
LOG.warning("Collector mappings are deprecated and shouldn't be used.")
policy.authorize(pecan.request.context, 'collector:get_mapping', {})
try:
mapping = self._db.get_mapping(service)
return collector_models.ServiceToCollectorMapping(
**mapping.as_dict())
except db_api.NoSuchMapping as e:
pecan.abort(404, e.args[0])
def tenants(self, begin=None, end=None):
"""Return the list of rated tenants.
"""
policy.authorize(pecan.request.context, 'report:list_tenants', {})
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
storage = pecan.request.storage_backend
tenants = storage.get_tenants(begin, end)
return tenants
def post(self, dataframes=[]):
policy.authorize(
flask.request.context,
'dataframes:add',
{},
)
if not dataframes:
raise http_exceptions.BadRequest(
"Parameter dataframes must not be empty.")
self._storage.push(dataframes)
return {}, 204
def get_all(self, begin=None, end=None, tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.authorize(pecan.request.context, 'storage:list_data_frames', {})
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
begin_ts = ck_utils.dt2ts(begin)
end_ts = ck_utils.dt2ts(end)
backend = pecan.request.storage_backend
dataframes = []
try:
frames = backend.get_time_frame(begin_ts,
end_ts,
tenant_id=tenant_id,
res_type=resource_type)
for frame in frames:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
desc = data['desc'] if data['desc'] else {}
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
frame_tenant = data['tenant_id']
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
except ck_storage.NoTimeFrame:
pass
return storage_models.DataFrameCollection(dataframes=dataframes)
def summary(self,
begin=None,
end=None,
tenant_id=None,
service=None,
groupby=None,
all_tenants=False):
"""Return the summary to pay for a given period.
"""
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if all_tenants:
tenant_id = None
else:
tenant_context = pecan.request.context.project_id
tenant_id = tenant_context if not tenant_id else tenant_id
policy.authorize(pecan.request.context, 'report:get_summary',
{"tenant_id": tenant_id})
storage = pecan.request.storage_backend
scope_key = CONF.collect.scope_key
storage_groupby = []
if groupby is not None and 'tenant_id' in groupby:
storage_groupby.append(scope_key)
if groupby is not None and 'res_type' in groupby:
storage_groupby.append('type')
filters = {scope_key: tenant_id} if tenant_id else None
result = storage.total(groupby=storage_groupby,
begin=begin,
end=end,
metric_types=service,
filters=filters)
summarymodels = []
for res in result['results']:
kwargs = {
'res_type': res.get('type') or res.get('res_type'),
'tenant_id': res.get(scope_key) or res.get('tenant_id'),
'begin': res['begin'],
'end': res['end'],
'rate': res['rate'],
}
summarymodel = report_models.SummaryModel(**kwargs)
summarymodels.append(summarymodel)
return report_models.SummaryCollectionModel(summary=summarymodels)
def get(self,
response_format=TABLE_RESPONSE_FORMAT,
custom_fields=None,
groupby=None,
filters={},
begin=None,
end=None,
offset=0,
limit=100):
if response_format not in ALL_RESPONSE_FORMATS:
raise voluptuous.Invalid("Invalid response format [%s]. Valid "
"format are [%s]." %
(response_format, ALL_RESPONSE_FORMATS))
policy.authorize(flask.request.context, 'summary:get_summary',
{'project_id': flask.request.context.project_id})
begin = begin or tzutils.get_month_start()
end = end or tzutils.get_next_month()
if not flask.request.context.is_admin:
if flask.request.context.project_id is None:
# Unscoped non-admin user
return {
'total': 0,
'columns': [],
'results': [],
}
filters['project_id'] = flask.request.context.project_id
metric_types = filters.pop('type', [])
if not isinstance(metric_types, list):
metric_types = [metric_types]
arguments = {
'begin': begin,
'end': end,
'groupby': groupby,
'filters': filters,
'metric_types': metric_types,
'offset': offset,
'limit': limit,
'paginate': True
}
if custom_fields:
arguments['custom_fields'] = custom_fields
total = self._storage.total(**arguments)
return self.generate_response(response_format, total)
def get(self, module_id):
policy.authorize(flask.request.context, 'v2_rating:get_module', {})
try:
module = self.rating_modules[module_id]
except KeyError:
raise http_exceptions.NotFound(
"Module '{}' not found".format(module_id))
infos = module.obj.module_info.copy()
return {
'module_id': module_id,
'description': infos['description'],
'enabled': infos['enabled'],
'hot_config': infos['hot_config'],
'priority': infos['priority'],
}
def get_one_metric(metric_name):
try:
metrics_conf = collector.validate_conf(
ck_utils.load_conf(CONF.collect.metrics_conf))
except (voluptuous.Invalid, voluptuous.MultipleInvalid):
msg = 'Invalid endpoint: no metrics in current configuration.'
pecan.abort(405, msg)
policy.authorize(pecan.request.context, 'info:get_metric_info', {})
metric = _find_metric(metric_name, metrics_conf)
if not metric:
pecan.abort(404, str(metric_name))
info = metric.copy()
info['metric_id'] = info['alt_name']
return info_models.CloudkittyMetricInfo(**info)
def get_one(self, module_id):
"""return a module
:return: CloudKittyModule
"""
policy.authorize(pecan.request.context, 'rating:get_module', {})
try:
lock = lockutils.lock('rating-modules')
with lock:
module = self.extensions[module_id]
except KeyError:
pecan.abort(404, 'Module not found.')
infos = module.obj.module_info.copy()
infos['module_id'] = infos.pop('name')
return rating_models.CloudkittyModule(**infos)
def get_all_metrics():
try:
metrics_conf = collector.validate_conf(
ck_utils.load_conf(CONF.collect.metrics_conf))
except (voluptuous.Invalid, voluptuous.MultipleInvalid):
msg = 'Invalid endpoint: no metrics in current configuration.'
pecan.abort(405, msg)
policy.authorize(pecan.request.context, 'info:list_metrics_info', {})
metrics_info_list = []
for metric_name, metric in metrics_conf.items():
info = metric.copy()
info['metric_id'] = info['alt_name']
metrics_info_list.append(info_models.CloudkittyMetricInfo(**info))
return info_models.CloudkittyMetricInfoCollection(
metrics=metrics_info_list)