def init_data(self):
project_ids = [self._project_id, self._other_project_id]
start_base = tzutils.utc_to_local(datetime.datetime(2018, 1, 1))
for i in range(3):
start_delta = datetime.timedelta(seconds=3600 * i)
end_delta = start_delta + datetime.timedelta(seconds=3600)
start = tzutils.add_delta(start_base, start_delta)
end = tzutils.add_delta(start_base, end_delta)
data = test_utils.generate_v2_storage_data(
project_ids=project_ids,
start=start,
end=end)
self.data.append(data)
self.storage.push([data])
def _test_add_substract_delta(self, obj, tzone):
delta = datetime.timedelta(seconds=3600)
naive = obj.astimezone(tz.UTC).replace(tzinfo=None)
self.assertEqual(
tzutils.add_delta(obj, delta).astimezone(tzone),
(naive + delta).replace(tzinfo=tz.UTC).astimezone(tzone),
)
self.assertEqual(
tzutils.substract_delta(obj, delta).astimezone(tzone),
(naive - delta).replace(tzinfo=tz.UTC).astimezone(tzone),
)
def _collect(self, metric, start_timestamp):
next_timestamp = tzutils.add_delta(start_timestamp,
timedelta(seconds=self._period))
name, data = self._collector.retrieve(
metric,
start_timestamp,
next_timestamp,
self._tenant_id,
)
if not data:
raise collector.NoDataCollected
return name, data
def check_time_state(timestamp=None, period=0, wait_periods=0):
"""Checks the state of a timestamp compared to the current time.
Returns the next timestamp based on the current timestamp and the period if
the next timestamp is inferior to the current time and the waiting period
or None if not.
:param timestamp: Current timestamp
:type timestamp: datetime.datetime
:param period: Period, in seconds
:type period: int
:param wait_periods: periods to wait before the current timestamp.
:type wait_periods: int
:rtype: datetime.datetime
"""
if not timestamp:
return tzutils.get_month_start()
period_delta = datetime.timedelta(seconds=period)
next_timestamp = tzutils.add_delta(timestamp, period_delta)
wait_time = wait_periods * period_delta
if tzutils.add_delta(next_timestamp, wait_time) < tzutils.localized_now():
return next_timestamp
return None
def _fetch_resources(self, metric_name, start, end,
project_id=None, q_filter=None):
"""Get resources during the timeframe.
:type metric_name: str
:param start: Start of the timeframe.
:param end: End of the timeframe if needed.
:param project_id: Filter on a specific tenant/project.
:type project_id: str
:param q_filter: Append a custom filter.
:type q_filter: list
"""
# Get gnocchi specific conf
extra_args = self.conf[metric_name]['extra_args']
resource_type = extra_args['resource_type']
scope_key = CONF.collect.scope_key
# Build query
# FIXME(peschk_l): In order not to miss any resource whose metrics may
# contain measures after its destruction, we scan resources over three
# collect periods.
delta = timedelta(seconds=CONF.collect.period)
start = tzutils.substract_delta(start, delta)
end = tzutils.add_delta(end, delta)
query_parameters = self._generate_time_filter(start, end)
if project_id:
kwargs = {scope_key: project_id}
query_parameters.append(self.gen_filter(**kwargs))
if q_filter:
query_parameters.append(q_filter)
sorts = [extra_args['resource_key'] + ':asc']
resources = []
marker = None
while True:
resources_chunk = self._conn.resource.search(
resource_type=resource_type,
query=self.extend_filter(*query_parameters),
sorts=sorts,
marker=marker)
if len(resources_chunk) < 1:
break
resources += resources_chunk
marker = resources_chunk[-1][extra_args['resource_key']]
return {res[extra_args['resource_key']]: res for res in resources}
def _build_dataframes(self, points):
dataframes = {}
for point in points:
point_type = point['type']
time = tzutils.dt_from_iso(point['time'])
period = point.get(PERIOD_FIELD_NAME) or self._default_period
timekey = (time,
tzutils.add_delta(time,
datetime.timedelta(seconds=period)))
if timekey not in dataframes.keys():
dataframes[timekey] = dataframe.DataFrame(start=timekey[0],
end=timekey[1])
dataframes[timekey].add_point(
self._point_to_dataframe_entry(point), point_type)
output = list(dataframes.values())
output.sort(key=lambda frame: (frame.start, frame.end))
return output
def run(self):
while True:
timestamp = self._check_state()
if not timestamp:
break
metrics = list(self._conf['metrics'].keys())
# Collection
usage_data = self._do_collection(metrics, timestamp)
frame = dataframe.DataFrame(
start=timestamp,
end=tzutils.add_delta(timestamp,
timedelta(seconds=self._period)),
usage=usage_data,
)
# Rating
for processor in self._processors:
frame = processor.obj.process(frame)
# Writing
self._storage.push([frame], self._tenant_id)
self._state.set_state(self._tenant_id, timestamp)