def get_all(self, begin=None, end=None, tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
project_id = tenant_id or pecan.request.context.project_id
policy.authorize(pecan.request.context, 'storage:list_data_frames', {
'tenant_id': project_id,
})
scope_key = CONF.collect.scope_key
backend = pecan.request.storage_backend
dataframes = []
filters = {scope_key: tenant_id} if tenant_id else None
if begin:
begin = ck_utils.dt2ts(begin)
if end:
end = ck_utils.dt2ts(end)
try:
resp = backend.retrieve(
begin, end,
filters=filters,
metric_types=resource_type,
paginate=False)
except storage.NoTimeFrame:
return storage_models.DataFrameCollection(dataframes=[])
for frame in resp['dataframes']:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
# This means we use a v1 storage backend
if 'desc' in data.keys():
desc = data['desc']
else:
desc = data['metadata'].copy()
desc.update(data.get('groupby', {}))
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
if frame_tenant is None:
frame_tenant = desc[scope_key]
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
return storage_models.DataFrameCollection(dataframes=dataframes)
def get_all(self,
begin=None,
end=None,
tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.authorize(pecan.request.context, 'storage:list_data_frames', {})
scope_key = CONF.collect.scope_key
backend = pecan.request.storage_backend
dataframes = []
group_filters = {scope_key: tenant_id} if tenant_id else None
if begin:
begin = ck_utils.dt2ts(begin)
if end:
end = ck_utils.dt2ts(end)
try:
resp = backend.retrieve(begin,
end,
group_filters=group_filters,
metric_types=resource_type,
paginate=False)
except storage.NoTimeFrame:
return storage_models.DataFrameCollection(dataframes=[])
for frame in resp['dataframes']:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
# This means we use a v1 storage backend
if 'desc' in data.keys():
desc = data['desc']
else:
desc = data['metadata'].copy()
desc.update(data.get('groupby', {}))
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
if frame_tenant is None:
frame_tenant = desc[scope_key]
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
return storage_models.DataFrameCollection(dataframes=dataframes)
def get_all(self, begin=None, end=None, tenant_id=None,
resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.authorize(pecan.request.context, 'storage:list_data_frames', {})
if not begin:
begin = ck_utils.get_month_start()
if not end:
end = ck_utils.get_next_month()
begin_ts = ck_utils.dt2ts(begin)
end_ts = ck_utils.dt2ts(end)
backend = pecan.request.storage_backend
dataframes = []
try:
frames = backend.get_time_frame(begin_ts,
end_ts,
tenant_id=tenant_id,
res_type=resource_type)
for frame in frames:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
desc = data['desc'] if data['desc'] else {}
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
frame_tenant = data['tenant_id']
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
except ck_storage.NoTimeFrame:
pass
return storage_models.DataFrameCollection(dataframes=dataframes)
def _check_begin_end(begin, end):
if not begin:
begin = utils.get_month_start()
if not end:
end = utils.get_next_month()
if isinstance(begin, six.text_type):
begin = utils.iso2dt(begin)
if isinstance(begin, int):
begin = utils.ts2dt(begin)
if isinstance(end, six.text_type):
end = utils.iso2dt(end)
if isinstance(end, int):
end = utils.ts2dt(end)
return begin, end
def _check_begin_end(begin, end):
if not begin:
begin = utils.get_month_start()
if not end:
end = utils.get_next_month()
if isinstance(begin, six.text_type):
begin = utils.iso2dt(begin)
if isinstance(begin, int):
begin = utils.ts2dt(begin)
if isinstance(end, six.text_type):
end = utils.iso2dt(end)
if isinstance(end, int):
end = utils.ts2dt(end)
return begin, end
def reset_state(self, ctxt, res_data):
LOG.info('Received state reset command. {}'.format(res_data))
random.shuffle(res_data['scopes'])
for scope in res_data['scopes']:
lock_name, lock = get_lock(self._coord, scope['scope_id'])
LOG.debug('[ScopeEndpoint] Trying to acquire lock "{}" ...'.format(
lock_name, ))
if lock.acquire(blocking=True):
LOG.debug('[ScopeEndpoint] Acquired lock "{}".'.format(
lock_name, ))
state_dt = ck_utils.iso2dt(res_data['state'])
try:
self._storage.delete(begin=state_dt,
end=None,
filters={
scope['scope_key']:
scope['scope_id'],
'collector': scope['collector'],
'fetcher': scope['fetcher'],
})
self._state.set_state(
scope['scope_id'],
state_dt,
fetcher=scope['fetcher'],
collector=scope['collector'],
scope_key=scope['scope_key'],
)
finally:
lock.release()
LOG.debug('[ScopeEndpoint] Released lock "{}" .'.format(
lock_name, ))
def get_all(self, begin, end, tenant_id=None, resource_type=None):
"""Return a list of rated resources for a time period and a tenant.
:param begin: Start of the period
:param end: End of the period
:param tenant_id: UUID of the tenant to filter on.
:param resource_type: Type of the resource to filter on.
:return: Collection of DataFrame objects.
"""
policy.enforce(pecan.request.context, 'storage:list_data_frames', {})
begin_ts = ck_utils.dt2ts(begin)
end_ts = ck_utils.dt2ts(end)
backend = pecan.request.storage_backend
dataframes = []
try:
frames = backend.get_time_frame(begin_ts,
end_ts,
tenant_id=tenant_id,
res_type=resource_type)
for frame in frames:
for service, data_list in frame['usage'].items():
frame_tenant = None
resources = []
for data in data_list:
desc = data['desc'] if data['desc'] else {}
price = decimal.Decimal(str(data['rating']['price']))
resource = storage_models.RatedResource(
service=service,
desc=desc,
volume=data['vol']['qty'],
rating=price)
frame_tenant = data['tenant_id']
resources.append(resource)
dataframe = storage_models.DataFrame(
begin=ck_utils.iso2dt(frame['period']['begin']),
end=ck_utils.iso2dt(frame['period']['end']),
tenant_id=frame_tenant,
resources=resources)
dataframes.append(dataframe)
except ck_storage.NoTimeFrame:
pass
return storage_models.DataFrameCollection(dataframes=dataframes)
def check_state(self):
timestamp = self._storage.get_state(self._tenant_id)
if not timestamp:
month_start = ck_utils.local2utc(ck_utils.get_month_start())
return ck_utils.dt2ts(ck_utils.iso2dt(month_start))
now = ck_utils.utcnow_ts()
if timestamp + self._period + self._wait_time < now:
return timestamp
return 0
def build_period_usage(self, begining_event, ending_event=None, service=None):
if ending_event:
self.period_end = iso2dt(str(ending_event.generated))
ending_event_type = str(ending_event.event_type)
else:
self.period_end = self.end
ending_event_type = 'period_cutting_event'
period_usage = {}
if type(begining_event) is dict:
usage_dict = begining_event['usage']
for service in usage_dict:
period_usage = usage_dict[service][0]
period_dict = period_usage['period']
period_dict['end'] = self.period_end
period_dict['period_duration'] = ck_utils.dt2ts(period_dict['end']) - ck_utils.dt2ts(period_dict['begin'])
period_dict['ending_event_type'] = ending_event_type
chargeable = self.is_chargeable(period_usage['desc'])
period_dict['chargeable'] = chargeable
else:
self.period_start = iso2dt(str(begining_event.generated))
period_resource_info = self.t_ceilometer.strip_event_data(event_data=begining_event, service=service)
self.duration = ck_utils.dt2ts(self.period_end) - ck_utils.dt2ts(self.period_start)
chargeable = self.is_chargeable(period_resource_info)
period_usage = self.t_cloudkitty.format_item(period_resource_info, service)
period_usage['period'] = {'begin': self.period_start,
'end': self.period_end,
'period_duration': self.duration,
'begining_event_type': str(begining_event.event_type),
'ending_event_type': ending_event_type,
'chargeable': chargeable}
return period_usage
def _build_dataframes(self, points):
dataframes = {}
for point in points:
point_type = point['type']
if point['time'] not in dataframes.keys():
dataframes[point['time']] = {
'period': {
'begin': point['time'],
'end': utils.isotime(
utils.iso2dt(point['time'])
+ datetime.timedelta(seconds=self._period)),
},
'usage': {},
}
usage = dataframes[point['time']]['usage']
if point_type not in usage.keys():
usage[point_type] = []
usage[point_type].append(self._point_to_dataframe_entry(point))
output = list(dataframes.values())
output.sort(key=lambda x: x['period']['begin'])
return output
def _build_dataframes(self, points):
dataframes = {}
for point in points:
point_type = point['type']
if point['time'] not in dataframes.keys():
dataframes[point['time']] = {
'period': {
'begin': point['time'],
'end': utils.isotime(
utils.iso2dt(point['time'])
+ datetime.timedelta(seconds=self._period)),
},
'usage': {},
}
usage = dataframes[point['time']]['usage']
if point_type not in usage.keys():
usage[point_type] = []
usage[point_type].append(self._point_to_dataframe_entry(point))
output = list(dataframes.values())
output.sort(key=lambda x: x['period']['begin'])
return output
def parse_events(self, start, end=None, instance_events=None, resource_id=None, service=None):
if service == 'compute':
deletion_event = u'compute.instance.delete.end'
elif service == 'volume':
deletion_event = u'volume.delete.end'
self.start = ts2dt(start)
self.end = ts2dt(end)
items = []
if len(instance_events)>1:
i=0
while len(instance_events)-1 > i:
items.append(self.build_period_usage(instance_events[i], instance_events[i+1], service=service))
i = i+1
if type(instance_events[-1]) is dict:
items.append(self.build_period_usage(instance_events[-1], ending_event=None, service=service))
else:
if instance_events[-1].event_type != deletion_event and ck_utils.iso2dt(instance_events[-1].generated) != self.end:
items.append(self.build_period_usage(instance_events[-1], ending_event=None, service=service))
return items
def test_iso2dt(self):
date = datetime.datetime(**self.date_params)
trans_dt = ck_utils.iso2dt(self.date_iso)
self.assertEqual(date, trans_dt)
def test_load_timestamp(self):
calc_dt = ck_utils.iso2dt(self.date_iso)
check_dt = ck_utils.ts2dt(self.date_ts)
self.assertEqual(calc_dt, check_dt)
def test_iso2dt(self):
date = datetime.datetime(**self.date_params)
trans_dt = ck_utils.iso2dt(self.date_iso)
self.assertEqual(date, trans_dt)
def test_load_timestamp(self):
calc_dt = ck_utils.iso2dt(self.date_iso)
check_dt = ck_utils.ts2dt(self.date_ts)
self.assertEqual(calc_dt, check_dt)
