def get_all(self, begin=None, end=None, tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
project_id = tenant_id or pecan.request.context.project_id
policy.authorize(pecan.request.context, 'storage:list_data_frames', {
'tenant_id': project_id,
})
scope_key = CONF.collect.scope_key
backend = pecan.request.storage_backend
dataframes = []
filters = {scope_key: tenant_id} if tenant_id else None
if begin:
begin = ck_utils.dt2ts(begin)
if end:
end = ck_utils.dt2ts(end)
try:
resp = backend.retrieve(
begin, end,
filters=filters,
metric_types=resource_type,
paginate=False)
except storage.NoTimeFrame:
return storage_models.DataFrameCollection(dataframes=[])
for frame in resp['dataframes']:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
# This means we use a v1 storage backend
if 'desc' in data.keys():
desc = data['desc']
else:
desc = data['metadata'].copy()
desc.update(data.get('groupby', {}))
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
if frame_tenant is None:
frame_tenant = desc[scope_key]
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
return storage_models.DataFrameCollection(dataframes=dataframes)
def _crumple(self):
# Reset states
self._usage_data = []
self.total = 0
# Recover state from file
if self._report is not None:
self._report.seek(0)
reader = csv.reader(self._report)
# Skip header
for dummy in range(2):
line = reader.next()
self.usage_start_dt = datetime.datetime.strptime(
line[0], '%Y/%m/%d %H:%M:%S')
self.usage_start = ck_utils.dt2ts(self.usage_start_dt)
self.usage_end_dt = datetime.datetime.strptime(
line[1], '%Y/%m/%d %H:%M:%S')
self.usage_end = ck_utils.dt2ts(self.usage_end_dt)
# Reset file
self._report.seek(0)
self._report.truncate()
self._write_header()
timeframe = self._write_orchestrator.get_timeframe(self.usage_start)
start = self.usage_start
self.usage_start = None
for data in timeframe:
self.append(data['usage'], start, None)
self.usage_start = self.usage_end
def get_all(self,
begin=None,
end=None,
tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.authorize(pecan.request.context, 'storage:list_data_frames', {})
scope_key = CONF.collect.scope_key
backend = pecan.request.storage_backend
dataframes = []
group_filters = {scope_key: tenant_id} if tenant_id else None
if begin:
begin = ck_utils.dt2ts(begin)
if end:
end = ck_utils.dt2ts(end)
try:
resp = backend.retrieve(begin,
end,
group_filters=group_filters,
metric_types=resource_type,
paginate=False)
except storage.NoTimeFrame:
return storage_models.DataFrameCollection(dataframes=[])
for frame in resp['dataframes']:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
# This means we use a v1 storage backend
if 'desc' in data.keys():
desc = data['desc']
else:
desc = data['metadata'].copy()
desc.update(data.get('groupby', {}))
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
if frame_tenant is None:
frame_tenant = desc[scope_key]
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
return storage_models.DataFrameCollection(dataframes=dataframes)
def get_state(self, tenant_id=None):
# Return the last written frame's timestamp.
query = {"=": {"project_id": tenant_id}} if tenant_id else {}
state_resource_id = self._get_or_create_resource(
CLOUDKITTY_STATE_RESOURCE, tenant_id)
try:
# (aolwas) add "refresh=True" to be sure to get all posted
# measures for this particular metric
r = self._conn.metric.get_measures(metric=CLOUDKITTY_STATE_METRIC,
resource_id=state_resource_id,
query=query,
aggregation="sum",
limit=1,
granularity=self._period,
needed_overlap=0,
refresh=True)
except gexceptions.MetricNotFound:
return
if len(r) > 0:
# NOTE(lukapeschke) Since version 5.0.0, gnocchiclient returns a
# datetime object instead of a timestamp. This fixture is made
# to ensure compatibility with all versions
try:
# (aolwas) According http://gnocchi.xyz/rest.html#metrics,
# gnocchi always returns measures ordered by timestamp
return ck_utils.dt2ts(dateutil.parser.parse(r[-1][0]))
except TypeError:
return ck_utils.dt2ts(r[-1][0])
def generate_v2_storage_data(min_length=10,
nb_projects=2,
project_ids=None,
start=datetime(2018, 1, 1),
end=datetime(2018, 1, 1, 1)):
if isinstance(start, datetime):
start = ck_utils.dt2ts(start)
if isinstance(end, datetime):
end = ck_utils.dt2ts(end)
if not project_ids:
project_ids = [uuidutils.generate_uuid() for i in range(nb_projects)]
elif not isinstance(project_ids, list):
project_ids = [project_ids]
usage = {}
for metric_name, sample in samples.V2_STORAGE_SAMPLE.items():
dataframes = []
for project_id in project_ids:
data = [
copy.deepcopy(sample)
for i in range(min_length + random.randint(1, 10))
]
for elem in data:
elem['groupby']['id'] = uuidutils.generate_uuid()
elem['groupby']['project_id'] = project_id
dataframes += data
usage[metric_name] = dataframes
return {'usage': usage, 'period': {'begin': start, 'end': end}}
def _crumple(self):
# Reset states
self._usage_data = []
self.total = 0
# Recover state from file
if self._report is not None:
self._report.seek(0)
reader = csv.reader(self._report)
# Skip header
for dummy in range(2):
line = reader.next()
self.usage_start_dt = datetime.datetime.strptime(
line[0],
'%Y/%m/%d %H:%M:%S')
self.usage_start = ck_utils.dt2ts(self.usage_start_dt)
self.usage_end_dt = datetime.datetime.strptime(
line[1],
'%Y/%m/%d %H:%M:%S')
self.usage_end = ck_utils.dt2ts(self.usage_end_dt)
# Reset file
self._report.seek(0)
self._report.truncate()
self._write_header()
timeframe = self._write_orchestrator.get_timeframe(
self.usage_start)
start = self.usage_start
self.usage_start = None
for data in timeframe:
self.append(data['usage'],
start,
None)
self.usage_start = self.usage_end
def last_month():
now = datetime.datetime.now()
month_end = (datetime.datetime(now.year, now.month, 1) -
datetime.timedelta(days=1))
month_start = month_end.replace(day=1)
start_ts = utils.dt2ts(month_start)
end_ts = utils.dt2ts(month_end)
return start_ts, end_ts
def last_month():
now = datetime.datetime.now()
month_end = (datetime.datetime(now.year, now.month, 1) -
datetime.timedelta(days=1))
month_start = month_end.replace(day=1)
start_ts = utils.dt2ts(month_start)
end_ts = utils.dt2ts(month_end)
return start_ts, end_ts
def get_all(self, begin=None, end=None, tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.authorize(pecan.request.context, 'storage:list_data_frames', {})
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
begin_ts = ck_utils.dt2ts(begin)
end_ts = ck_utils.dt2ts(end)
backend = pecan.request.storage_backend
dataframes = []
try:
frames = backend.get_time_frame(begin_ts,
end_ts,
tenant_id=tenant_id,
res_type=resource_type)
for frame in frames:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
desc = data['desc'] if data['desc'] else {}
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
frame_tenant = data['tenant_id']
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
except ck_storage.NoTimeFrame:
pass
return storage_models.DataFrameCollection(dataframes=dataframes)
def get_state(self, tenant_id=None):
session = db.get_session()
q = utils.model_query(self.state_model, session)
if tenant_id:
q = q.filter(self.state_model.tenant_id == tenant_id)
q = q.order_by(self.state_model.state.desc())
r = q.first()
return ck_utils.dt2ts(r.state) if r else None
def get_state(self, tenant_id=None):
session = db.get_session()
q = utils.model_query(self.state_model, session)
if tenant_id:
q = q.filter(self.state_model.tenant_id == tenant_id)
q = q.order_by(self.state_model.state.desc())
r = q.first()
return ck_utils.dt2ts(r.state) if r else None
def get_state(self, tenant_id=None):
session = db.get_session()
q = utils.model_query(models.RatedDataFrame, session)
if tenant_id:
q = q.filter(models.RatedDataFrame.tenant_id == tenant_id)
r = q.order_by(models.RatedDataFrame.begin.desc()).first()
if r:
return ck_utils.dt2ts(r.begin)
def get_state(self, identifier,
fetcher=None, collector=None, scope_key=None):
session = db.get_session()
session.begin()
r = self._get_db_item(
session, identifier, fetcher, collector, scope_key)
session.close()
return ck_utils.dt2ts(r.state) if r else None
def delete(self, begin, end, filters):
beg = utils.dt2ts(begin) if begin else None
end = utils.dt2ts(end) if end else None
def __filter_func(elem):
def __time(elem):
return ((beg and beg > elem['time'])
or (end and end <= elem['time']))
def __filt(elem):
return all((elem['tags'].get(k, None) == v
or elem['fields'].get(k, None) == v)
for k, v in filters.items())
return __time(elem) and __filt(elem)
self._points = list(filter(__filter_func, self._points))
def get_state(self, tenant_id=None):
session = db.get_session()
q = utils.model_query(self.frame_model, session)
if tenant_id:
q = q.filter(self.frame_model.tenant_id == tenant_id)
q = q.order_by(self.frame_model.begin.desc())
r = q.first()
if r:
return ck_utils.dt2ts(r.begin)
def get_all(self, begin, end, tenant_id=None, resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.enforce(pecan.request.context, 'storage:list_data_frames', {})
begin_ts = ck_utils.dt2ts(begin)
end_ts = ck_utils.dt2ts(end)
backend = pecan.request.storage_backend
dataframes = []
try:
frames = backend.get_time_frame(begin_ts,
end_ts,
tenant_id=tenant_id,
res_type=resource_type)
for frame in frames:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
desc = data['desc'] if data['desc'] else {}
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
frame_tenant = data['tenant_id']
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
except ck_storage.NoTimeFrame:
pass
return storage_models.DataFrameCollection(dataframes=dataframes)
def check_state(self):
timestamp = self._storage.get_state(self._tenant_id)
if not timestamp:
month_start = ck_utils.local2utc(ck_utils.get_month_start())
return ck_utils.dt2ts(ck_utils.iso2dt(month_start))
now = ck_utils.utcnow_ts()
if timestamp + self._period + self._wait_time < now:
return timestamp
return 0
def test_fetch_measures_kwargs_no_forced_project(self):
with mock.patch.object(self.collector._conn.metrics,
'list_statistics') as m:
start = datetime.datetime(2019, 1, 1)
end = datetime.datetime(2019, 1, 1, 1)
self.collector._fetch_measures(
'metric_one',
utils.dt2ts(start),
utils.dt2ts(end),
)
m.assert_called_once_with(
name='metric_one',
merge_metrics=True,
dimensions={},
start_time=start,
end_time=end,
period=3600,
statistics='max',
group_by=['project_id', 'resource_id'],
)
def check_state(self):
timestamp = self._storage.get_state(self._tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + self._period
if next_timestamp + self._wait_time < now:
return next_timestamp
return 0
def check_state(self):
timestamp = self._storage.get_state(self._tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + self._period
if next_timestamp + self._wait_time < now:
return next_timestamp
return 0
def get_state(self,
identifier,
fetcher=None,
collector=None,
scope_key=None):
session = db.get_session()
session.begin()
r = self._get_db_item(session, identifier, fetcher, collector,
scope_key)
session.close()
return ck_utils.dt2ts(r.state) if r else None
def _check_state(self, tenant_id):
timestamp = self.storage.get_state(tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + CONF.collect.period
wait_time = CONF.collect.wait_periods * CONF.collect.period
if next_timestamp + wait_time < now:
return next_timestamp
return 0
def _check_state(self, tenant_id):
timestamp = self.storage.get_state(tenant_id)
if not timestamp:
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
now = ck_utils.utcnow_ts()
next_timestamp = timestamp + CONF.collect.period
wait_time = CONF.collect.wait_periods * CONF.collect.period
if next_timestamp + wait_time < now:
return next_timestamp
return 0
def get_state(self, tenant_id=None):
session = db.get_session()
q = utils.model_query(
self.frame_model,
session)
if tenant_id:
q = q.filter(
self.frame_model.tenant_id == tenant_id)
q = q.order_by(
self.frame_model.begin.desc())
r = q.first()
if r:
return ck_utils.dt2ts(r.begin)
def _get_metadata(self, resource_type, transformers):
info = {}
try:
met = list(METRICS_CONF['metrics_units'][resource_type].values())
info['unit'] = met[0]['unit']
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
info['unit'] = self.units_mappings[resource_type][1]
except (KeyError, IndexError):
info['unit'] = self.default_unit[1]
start = ck_utils.dt2ts(ck_utils.get_month_start())
end = ck_utils.dt2ts(ck_utils.get_month_end())
try:
resource = self.active_resources(resource_type, start, end,
None)[0]
except IndexError:
resource = {}
info['metadata'] = resource.get('dimensions', {}).keys()
try:
service_metrics = METRICS_CONF['services_metrics'][resource_type]
for service_metric in service_metrics:
metric, statistics = list(service_metric.items())[0]
info['metadata'].append(metric)
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
for metric, statistics in self.metrics_mappings[resource_type]:
info['metadata'].append(metric)
except (KeyError, IndexError):
pass
return info
def _get_metadata(self, resource_type, transformers, conf):
info = {}
info['unit'] = conf['metrics'][resource_type]['unit']
start = ck_utils.dt2ts(ck_utils.get_month_start())
end = ck_utils.dt2ts(ck_utils.get_month_end())
try:
resource = self.active_resources(
resource_type,
start,
end,
None,
)[0]
except IndexError:
resource = {}
info['metadata'] = resource.get('dimensions', {}).keys()
service_metrics = METRICS_CONF['services_metrics'][resource_type]
for service_metric in service_metrics:
metric, statistics = list(service_metric.items())[0]
info['metadata'].append(metric)
return info
def to_cloudkitty(self, collector=None):
if not collector:
raise Exception('Gnocchi storage needs a reference '
'to the collector.')
# Rating informations
rating_dict = {}
rating_dict['price'] = self.rate
# Resource information from gnocchi
resource_data = collector.resource_info(resource_name=self.res_type,
start=ck_utils.dt2ts(
self.begin),
end=ck_utils.dt2ts(self.end),
project_id=self.tenant_id)
# Encapsulate informations in a resource dict
res_dict = {}
resource_data = resource_data[0]
res_dict['desc'] = resource_data['desc']
res_dict['vol'] = resource_data['vol']
res_dict['rating'] = rating_dict
res_dict['tenant_id'] = self.tenant_id
# Add resource to the usage dict
usage_dict = {}
usage_dict[self.res_type] = [res_dict]
# Time informations
period_dict = {}
period_dict['begin'] = ck_utils.dt2iso(self.begin)
period_dict['end'] = ck_utils.dt2iso(self.end)
# Add period to the resource informations
ck_dict = {}
ck_dict['period'] = period_dict
ck_dict['usage'] = usage_dict
return ck_dict
def build_period_usage(self, begining_event, ending_event=None, service=None):
if ending_event:
self.period_end = iso2dt(str(ending_event.generated))
ending_event_type = str(ending_event.event_type)
else:
self.period_end = self.end
ending_event_type = 'period_cutting_event'
period_usage = {}
if type(begining_event) is dict:
usage_dict = begining_event['usage']
for service in usage_dict:
period_usage = usage_dict[service][0]
period_dict = period_usage['period']
period_dict['end'] = self.period_end
period_dict['period_duration'] = ck_utils.dt2ts(period_dict['end']) - ck_utils.dt2ts(period_dict['begin'])
period_dict['ending_event_type'] = ending_event_type
chargeable = self.is_chargeable(period_usage['desc'])
period_dict['chargeable'] = chargeable
else:
self.period_start = iso2dt(str(begining_event.generated))
period_resource_info = self.t_ceilometer.strip_event_data(event_data=begining_event, service=service)
self.duration = ck_utils.dt2ts(self.period_end) - ck_utils.dt2ts(self.period_start)
chargeable = self.is_chargeable(period_resource_info)
period_usage = self.t_cloudkitty.format_item(period_resource_info, service)
period_usage['period'] = {'begin': self.period_start,
'end': self.period_end,
'period_duration': self.duration,
'begining_event_type': str(begining_event.event_type),
'ending_event_type': ending_event_type,
'chargeable': chargeable}
return period_usage
def get_state(self, tenant_id=None):
session = db.get_session()
q = utils.model_query(
models.RatedDataFrame,
session
)
if tenant_id:
q = q.filter(
models.RatedDataFrame.tenant_id == tenant_id
)
r = q.order_by(
models.RatedDataFrame.begin.desc()
).first()
if r:
return ck_utils.dt2ts(r.begin)
def current_month():
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
def current_month():
month_start = ck_utils.get_month_start()
return ck_utils.dt2ts(month_start)
def last_month():
month_start = ck_utils.get_month_start()
month_end = ck_utils.get_month_end()
start_ts = ck_utils.dt2ts(month_start)
end_ts = ck_utils.dt2ts(month_end)
return start_ts, end_ts
def current_month():
now = datetime.now()
month_start = datetime(now.year, now.month, 1)
return utils.dt2ts(month_start)
def test_dt2ts(self):
date = datetime.datetime(**self.date_params)
trans_ts = ck_utils.dt2ts(date)
self.assertEqual(self.date_ts, trans_ts)
def restart_month(self):
self._load_state_manager_data()
month_start = ck_utils.get_month_start()
self.usage_end = ck_utils.dt2ts(month_start)
self._update_state_manager_data()
def test_dt2ts(self):
date = datetime.datetime(**self.date_params)
trans_ts = ck_utils.dt2ts(date)
self.assertEqual(self.date_ts, trans_ts)
def _format_data(self, timeframe, data):
beg = utils.dt2ts(timeframe)
end = beg + self._period
final_data = {'period': {'begin': beg, 'end': end}}
final_data['usage'] = data
return [final_data]
def last_month():
month_start = ck_utils.get_month_start()
month_end = ck_utils.get_month_end()
start_ts = ck_utils.dt2ts(month_start)
end_ts = ck_utils.dt2ts(month_end)
return start_ts, end_ts
def _format_data(self, timeframe, data):
beg = utils.dt2ts(timeframe)
end = beg + self._period
final_data = {'period': {'begin': beg, 'end': end}}
final_data['usage'] = data
return [final_data]
def get_state(self, identifier):
session = db.get_session()
session.begin()
r = self._get_db_item(session, identifier)
session.close()
return ck_utils.dt2ts(r.state) if r else None
def restart_month(self):
self._load_state_manager_data()
month_start = ck_utils.get_month_start()
self.usage_end = ck_utils.dt2ts(month_start)
self._update_state_manager_data()
def current_month():
now = datetime.now()
month_start = datetime(now.year, now.month, 1)
return utils.dt2ts(month_start)
