def show_aggregated_resource_rate(self, begin=None, end=None, tenant_ids=None, resource_id=None):
model = models.AggregatedDataFrame
# Boundary calculation
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
session = db.get_session()
q = session.query(model.begin.label('begin'), model.end.label('end'),
model.rate.label('rate'), model.qty.label('qty'),
model.unit.label('unit'), model.unit_price.label('unit_price'),
model.begining_event_type.label('begin_event'), model.ending_event_type.label('end_event'))
if tenant_ids:
q = q.filter(
model.tenant_id.in_(tenant_ids)
)
q = q.filter(
model.begin >= begin,
model.end <= end,
model.res_type != '_NO_DATA_',
model.resource_id == resource_id
)
resources_usage_rate = q.all()
return [model.to_resource_usage_rate(entry) for entry in resources_usage_rate]
def total(self,
begin=None,
end=None,
tenant_id=None,
service=None,
all_tenants=False):
"""Return the amount to pay for a given period.
"""
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if all_tenants:
tenant_id = None
else:
tenant_context = pecan.request.context.tenant
tenant_id = tenant_context if not tenant_id else tenant_id
policy.authorize(pecan.request.context, 'report:get_total',
{"tenant_id": tenant_id})
storage = pecan.request.storage_backend
# FIXME(sheeprine): We should filter on user id.
# Use keystone token information by default but make it overridable and
# enforce it by policy engine
total = storage.get_total(begin, end, tenant_id, service)
# TODO(Aaron): `get_total` return a list of dict,
# Get value of rate from index[0]
total = total[0].get('rate', decimal.Decimal('0'))
return total if total else decimal.Decimal('0')
def summary(self,
begin=None,
end=None,
tenant_id=None,
service=None,
groupby=None,
all_tenants=False):
"""Return the summary to pay for a given period.
"""
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if all_tenants:
tenant_id = None
else:
tenant_context = pecan.request.context.tenant
tenant_id = tenant_context if not tenant_id else tenant_id
policy.authorize(pecan.request.context, 'report:get_summary',
{"tenant_id": tenant_id})
storage = pecan.request.storage_backend
summarymodels = []
results = storage.get_total(begin,
end,
tenant_id,
service,
groupby=groupby)
for result in results:
summarymodel = report_models.SummaryModel(**result)
summarymodels.append(summarymodel)
return report_models.SummaryCollectionModel(summary=summarymodels)
def get_total(self, begin=None, end=None, tenant_id=None, service=None, instance_id=None):
model = models.RatedDataFrame
# Boundary calculation
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
session = db.get_session()
q = session.query(
sqlalchemy.func.sum(model.rate).label('rate'))
if tenant_id:
q = q.filter(
models.RatedDataFrame.tenant_id == tenant_id)
if service:
q = q.filter(
models.RatedDataFrame.res_type == service)
if instance_id:
q = q.filter(
models.RatedDataFrame.desc.like('%'+instance_id+'%'))
q = q.filter(
model.begin >= begin,
model.end <= end)
rate = q.scalar()
return rate
def get_total(self, begin=None, end=None, tenant_id=None, service=None, instance_id=None):
model = models.RatedDataFrame
# Boundary calculation
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
session = db.get_session()
q = session.query(
sqlalchemy.func.sum(model.rate).label('rate'))
if tenant_id:
q = q.filter(
models.RatedDataFrame.tenant_id == tenant_id)
if service:
q = q.filter(
models.RatedDataFrame.res_type == service)
if instance_id:
q = q.filter(
models.RatedDataFrame.desc.like('%'+instance_id+'%'))
q = q.filter(
model.begin >= begin,
model.end <= end)
rate = q.scalar()
return rate
def get(self,
groupby=None,
filters={},
begin=None,
end=None,
offset=0,
limit=100):
policy.authorize(flask.request.context, 'summary:get_summary',
{'tenant_id': flask.request.context.project_id})
begin = begin or utils.get_month_start()
end = end or utils.get_next_month()
if not flask.request.context.is_admin:
filters['project_id'] = flask.request.context.project_id
total = self._storage.total(
begin=begin,
end=end,
groupby=groupby,
filters=filters,
offset=offset,
limit=limit,
paginate=True,
)
columns = []
if len(total['results']) > 0:
columns = list(total['results'][0].keys())
return {
'total': total['total'],
'columns': columns,
'results': [list(res.values()) for res in total['results']]
}
def tenants_list(self):
if not CONF.command.begin:
CONF.command.begin = ck_utils.get_month_start()
if not CONF.command.end:
CONF.command.end = ck_utils.get_next_month()
tenants = self._storage.get_tenants(CONF.command.begin,
CONF.command.end)
print('Tenant list:')
for tenant in tenants:
print(tenant)
def check_state(self):
timestamp = self._storage.get_state(self._tenant_id)
if not timestamp:
month_start = ck_utils.local2utc(ck_utils.get_month_start())
return ck_utils.dt2ts(ck_utils.iso2dt(month_start))
now = ck_utils.utcnow_ts()
if timestamp + self._period + self._wait_time < now:
return timestamp
return 0
def tenants_list(self):
if not CONF.command.begin:
CONF.command.begin = ck_utils.get_month_start()
if not CONF.command.end:
CONF.command.end = ck_utils.get_next_month()
tenants = self._storage.get_tenants(CONF.command.begin,
CONF.command.end)
print('Tenant list:')
for tenant in tenants:
print(tenant)
def check_state(self):
timestamp = self._storage.get_state(self._tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + self._period
if next_timestamp + self._wait_time < now:
return next_timestamp
return 0
def check_state(self):
timestamp = self._storage.get_state(self._tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + self._period
if next_timestamp + self._wait_time < now:
return next_timestamp
return 0
def _check_state(self, tenant_id):
timestamp = self.storage.get_state(tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + CONF.collect.period
wait_time = CONF.collect.wait_periods * CONF.collect.period
if next_timestamp + wait_time < now:
return next_timestamp
return 0
def _check_state(self, tenant_id):
timestamp = self.storage.get_state(tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + CONF.collect.period
wait_time = CONF.collect.wait_periods * CONF.collect.period
if next_timestamp + wait_time < now:
return next_timestamp
return 0
def tenants(self, begin=None, end=None):
"""Return the list of rated tenants.
"""
policy.authorize(pecan.request.context, 'report:list_tenants', {})
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
storage = pecan.request.storage_backend
tenants = storage.get_tenants(begin, end)
return tenants
def get_all(self, begin=None, end=None, tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.authorize(pecan.request.context, 'storage:list_data_frames', {})
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
begin_ts = ck_utils.dt2ts(begin)
end_ts = ck_utils.dt2ts(end)
backend = pecan.request.storage_backend
dataframes = []
try:
frames = backend.get_time_frame(begin_ts,
end_ts,
tenant_id=tenant_id,
res_type=resource_type)
for frame in frames:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
desc = data['desc'] if data['desc'] else {}
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
frame_tenant = data['tenant_id']
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
except ck_storage.NoTimeFrame:
pass
return storage_models.DataFrameCollection(dataframes=dataframes)
def get_tenants(self, begin=None, end=None):
model = models.RatedDataFrame
# Boundary calculation
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
session = db.get_session()
q = utils.model_query(model, session).filter(model.begin >= begin,
model.end <= end)
tenants = q.distinct().values(model.tenant_id)
return [tenant.tenant_id for tenant in tenants]
def summary(self,
begin=None,
end=None,
tenant_id=None,
service=None,
groupby=None,
all_tenants=False):
"""Return the summary to pay for a given period.
"""
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if all_tenants:
tenant_id = None
else:
tenant_context = pecan.request.context.project_id
tenant_id = tenant_context if not tenant_id else tenant_id
policy.authorize(pecan.request.context, 'report:get_summary',
{"tenant_id": tenant_id})
storage = pecan.request.storage_backend
scope_key = CONF.collect.scope_key
storage_groupby = []
if groupby is not None and 'tenant_id' in groupby:
storage_groupby.append(scope_key)
if groupby is not None and 'res_type' in groupby:
storage_groupby.append('type')
filters = {scope_key: tenant_id} if tenant_id else None
result = storage.total(groupby=storage_groupby,
begin=begin,
end=end,
metric_types=service,
filters=filters)
summarymodels = []
for res in result['results']:
kwargs = {
'res_type': res.get('type') or res.get('res_type'),
'tenant_id': res.get(scope_key) or res.get('tenant_id'),
'begin': res['begin'],
'end': res['end'],
'rate': res['rate'],
}
summarymodel = report_models.SummaryModel(**kwargs)
summarymodels.append(summarymodel)
return report_models.SummaryCollectionModel(summary=summarymodels)
def _check_begin_end(begin, end):
if not begin:
begin = utils.get_month_start()
if not end:
end = utils.get_next_month()
if isinstance(begin, six.text_type):
begin = utils.iso2dt(begin)
if isinstance(begin, int):
begin = utils.ts2dt(begin)
if isinstance(end, six.text_type):
end = utils.iso2dt(end)
if isinstance(end, int):
end = utils.ts2dt(end)
return begin, end
def _check_begin_end(begin, end):
if not begin:
begin = utils.get_month_start()
if not end:
end = utils.get_next_month()
if isinstance(begin, six.text_type):
begin = utils.iso2dt(begin)
if isinstance(begin, int):
begin = utils.ts2dt(begin)
if isinstance(end, six.text_type):
end = utils.iso2dt(end)
if isinstance(end, int):
end = utils.ts2dt(end)
return begin, end
def get_total(self,
begin=None,
end=None,
tenant_id=None,
service=None,
groupby=None):
session = db.get_session()
querymodels = [
sqlalchemy.func.sum(self.frame_model.rate).label('rate')
]
# Boundary calculation
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if tenant_id:
querymodels.append(self.frame_model.tenant_id)
if service:
querymodels.append(self.frame_model.res_type)
if groupby:
groupbyfields = groupby.split(",")
for field in groupbyfields:
field_obj = self.frame_model.__dict__.get(field, None)
if field_obj and field_obj not in querymodels:
querymodels.append(field_obj)
q = session.query(*querymodels)
if tenant_id:
q = q.filter(self.frame_model.tenant_id == tenant_id)
if service:
q = q.filter(self.frame_model.res_type == service)
q = q.filter(self.frame_model.begin >= begin,
self.frame_model.end <= end,
self.frame_model.res_type != '_NO_DATA_')
if groupby:
q = q.group_by(groupby)
# Order by sum(rate)
q = q.order_by(sqlalchemy.func.sum(self.frame_model.rate))
results = q.all()
totallist = []
for r in results:
total = {model.name: value for model, value in zip(querymodels, r)}
total["begin"] = begin
total["end"] = end
totallist.append(total)
return totallist
def get_tenants(self, begin=None, end=None):
# Boundary calculation
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
session = db.get_session()
q = utils.model_query(
self.frame_model,
session)
q = q.filter(
self.frame_model.begin >= begin,
self.frame_model.end <= end)
tenants = q.distinct().values(
self.frame_model.tenant_id)
return [tenant.tenant_id for tenant in tenants]
def generate(self):
if not CONF.command.tenant:
if not CONF.command.begin:
CONF.command.begin = ck_utils.get_month_start()
if not CONF.command.end:
CONF.command.end = ck_utils.get_next_month()
tenants = self._storage.get_tenants(CONF.command.begin,
CONF.command.end)
else:
tenants = [CONF.command.tenant]
for tenant in tenants:
wo = write_orchestrator.WriteOrchestrator(self._output, tenant,
self._storage,
CONF.output.basepath)
wo.init_writing_pipeline()
if not CONF.command.begin:
wo.restart_month()
wo.process()
def generate(self):
if not CONF.command.tenant:
if not CONF.command.begin:
CONF.command.begin = ck_utils.get_month_start()
if not CONF.command.end:
CONF.command.end = ck_utils.get_next_month()
tenants = self._storage.get_tenants(CONF.command.begin,
CONF.command.end)
else:
tenants = [CONF.command.tenant]
for tenant in tenants:
wo = write_orchestrator.WriteOrchestrator(self._output,
tenant,
self._storage,
CONF.output.basepath)
wo.init_writing_pipeline()
if not CONF.command.begin:
wo.restart_month()
wo.process()
def get_time_frame(self, begin, end, **filters):
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
session = db.get_session()
q = utils.model_query(self.frame_model, session)
q = q.filter(self.frame_model.begin >= begin,
self.frame_model.end <= end)
for filter_name, filter_value in filters.items():
if filter_value:
q = q.filter(
getattr(self.frame_model, filter_name) == filter_value)
if not filters.get('res_type'):
q = q.filter(self.frame_model.res_type != '_NO_DATA_')
count = q.count()
if not count:
raise NoTimeFrame()
r = q.all()
return [entry.to_cloudkitty(self._collector) for entry in r]
def total(self,
begin=None,
end=None,
tenant_id=None,
service=None,
all_tenants=False):
"""Return the amount to pay for a given period.
"""
LOG.warning('/v1/report/total is deprecated, please use '
'/v1/report/summary instead.')
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
if all_tenants:
tenant_id = None
else:
tenant_context = pecan.request.context.project_id
tenant_id = tenant_context if not tenant_id else tenant_id
policy.authorize(pecan.request.context, 'report:get_total',
{"tenant_id": tenant_id})
storage = pecan.request.storage_backend
# FIXME(sheeprine): We should filter on user id.
# Use keystone token information by default but make it overridable and
# enforce it by policy engine
scope_key = CONF.collect.scope_key
groupby = [scope_key]
group_filters = {scope_key: tenant_id} if tenant_id else None
total_resources = storage.total(groupby=groupby,
begin=begin,
end=end,
metric_types=service,
group_filters=group_filters)
# TODO(Aaron): `get_total` return a list of dict,
# Get value of rate from index[0]
total = sum(total['rate'] for total in total_resources)
return total if total else decimal.Decimal('0')
def get_total(self, begin=None, end=None, tenant_id=None, service=None):
# Boundary calculation
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
session = db.get_session()
q = session.query(
sqlalchemy.func.sum(self.frame_model.rate).label('rate'))
if tenant_id:
q = q.filter(
self.frame_model.tenant_id == tenant_id)
if service:
q = q.filter(
self.frame_model.res_type == service)
q = q.filter(
self.frame_model.begin >= begin,
self.frame_model.end <= end)
rate = q.scalar()
return rate
def _get_metadata(self, resource_type, transformers):
info = {}
try:
met = list(METRICS_CONF['metrics_units'][resource_type].values())
info['unit'] = met[0]['unit']
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
info['unit'] = self.units_mappings[resource_type][1]
except (KeyError, IndexError):
info['unit'] = self.default_unit[1]
start = ck_utils.dt2ts(ck_utils.get_month_start())
end = ck_utils.dt2ts(ck_utils.get_month_end())
try:
resource = self.active_resources(resource_type, start, end,
None)[0]
except IndexError:
resource = {}
info['metadata'] = resource.get('dimensions', {}).keys()
try:
service_metrics = METRICS_CONF['services_metrics'][resource_type]
for service_metric in service_metrics:
metric, statistics = list(service_metric.items())[0]
info['metadata'].append(metric)
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
for metric, statistics in self.metrics_mappings[resource_type]:
info['metadata'].append(metric)
except (KeyError, IndexError):
pass
return info
def _get_metadata(self, resource_type, transformers, conf):
info = {}
info['unit'] = conf['metrics'][resource_type]['unit']
start = ck_utils.dt2ts(ck_utils.get_month_start())
end = ck_utils.dt2ts(ck_utils.get_month_end())
try:
resource = self.active_resources(
resource_type,
start,
end,
None,
)[0]
except IndexError:
resource = {}
info['metadata'] = resource.get('dimensions', {}).keys()
service_metrics = METRICS_CONF['services_metrics'][resource_type]
for service_metric in service_metrics:
metric, statistics = list(service_metric.items())[0]
info['metadata'].append(metric)
return info
def current_month():
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
def last_month():
month_start = ck_utils.get_month_start()
month_end = ck_utils.get_month_end()
start_ts = ck_utils.dt2ts(month_start)
end_ts = ck_utils.dt2ts(month_end)
return start_ts, end_ts
def test_month_start_without_dt(self, patch_utcnow_mock):
date = datetime.datetime(2014, 1, 1)
trans_dt = ck_utils.get_month_start()
self.assertEqual(date, trans_dt)
patch_utcnow_mock.assert_called_once_with()
def restart_month(self):
self._load_state_manager_data()
month_start = ck_utils.get_month_start()
self.usage_end = ck_utils.dt2ts(month_start)
self._update_state_manager_data()
def restart_month(self):
self._load_state_manager_data()
month_start = ck_utils.get_month_start()
self.usage_end = ck_utils.dt2ts(month_start)
self._update_state_manager_data()
def current_month():
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
def last_month():
month_start = ck_utils.get_month_start()
month_end = ck_utils.get_month_end()
start_ts = ck_utils.dt2ts(month_start)
end_ts = ck_utils.dt2ts(month_end)
return start_ts, end_ts
def _check_begin_end(begin, end):
if not begin:
begin = utils.get_month_start()
if not end:
end = utils.get_next_month()
return begin, end
def test_month_start_without_dt(self, patch_utcnow_mock):
date = datetime.datetime(2014, 1, 1)
trans_dt = ck_utils.get_month_start()
self.assertEqual(date, trans_dt)
patch_utcnow_mock.assert_called_once_with()
def initialize_data(self):
data = test_utils.generate_v2_storage_data(
start=ck_utils.get_month_start(),
end=ck_utils.utcnow().replace(hour=0),
)
self.storage.push([data])