def generate_v2_storage_data(min_length=10,
nb_projects=2,
project_ids=None,
start=None,
end=None):
if not project_ids:
project_ids = [uuidutils.generate_uuid() for i in range(nb_projects)]
elif not isinstance(project_ids, list):
project_ids = [project_ids]
df = dataframe.DataFrame(start=start, end=end)
for metric_name, sample in samples.V2_STORAGE_SAMPLE.items():
datapoints = []
for project_id in project_ids:
data = [copy.deepcopy(sample)
for i in range(min_length + random.randint(1, 10))]
for elem in data:
elem['groupby']['id'] = uuidutils.generate_uuid()
elem['groupby']['project_id'] = project_id
datapoints += [dataframe.DataPoint(
elem['vol']['unit'],
elem['vol']['qty'],
elem['rating']['price'],
elem['groupby'],
elem['metadata'],
) for elem in data]
df.add_points(datapoints, metric_name)
return df
def process(self, data):
output = dataframe.DataFrame(start=data.start, end=data.end)
for service_name, point in data.iterpoints():
self._res = {}
self.process_services(service_name, point)
self.process_fields(service_name, point)
output.add_point(self.add_rating_informations(point), service_name)
return output
def test_repr(self):
start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC)
end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC)
df = dataframe.DataFrame(start=start, end=end)
points = [
dataframe.DataPoint(**TestDataPoint.default_params)
for _ in range(4)
]
df.add_points(points, 'metric_x')
self.assertEqual(str(df), "DataFrame(metrics=[metric_x])")
df.add_points(points, 'metric_y')
self.assertEqual(str(df), "DataFrame(metrics=[metric_x,metric_y])")
def test_iterpoints(self):
start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC)
end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC)
df = dataframe.DataFrame(start=start, end=end)
points = [
dataframe.DataPoint(**TestDataPoint.default_params)
for _ in range(4)
]
df.add_points(points, 'metric_x')
expected = [('metric_x',
dataframe.DataPoint(**TestDataPoint.default_params))
for _ in range(4)]
self.assertEqual(list(df.iterpoints()), expected)
def execute_measurements_rating(self, end_time, start_time, usage_data):
frame = dataframe.DataFrame(
start=start_time,
end=end_time,
usage=usage_data,
)
for processor in self._processors:
original_data = copy.deepcopy(frame)
frame = processor.obj.process(frame)
LOG.debug("Results [%s] for processing [%s] of data points [%s].",
frame, processor.obj.process, original_data)
return frame
def create_fake_data(self, begin, end, project_id):
cpu_point = dataframe.DataPoint(
unit="nothing",
qty=1,
groupby={
"fake_meta": 1.0,
"project_id": project_id
},
metadata={"dummy": True},
price=decimal.Decimal('1.337'),
)
image_point = dataframe.DataPoint(
unit="nothing",
qty=1,
groupby={
"fake_meta": 1.0,
"project_id": project_id
},
metadata={"dummy": True},
price=decimal.Decimal('0.121'),
)
data = [
dataframe.DataFrame(
start=begin,
end=end,
usage=collections.OrderedDict({"cpu": [cpu_point, cpu_point]}),
),
dataframe.DataFrame(
start=begin,
end=end,
usage=collections.OrderedDict(
{"image.size": [image_point, image_point]}),
),
]
return data
def test_from_dict_valid_dict_date_as_str(self):
start = datetime.datetime(2019, 1, 2, 12, tzinfo=tz.UTC)
end = datetime.datetime(2019, 1, 2, 13, tzinfo=tz.UTC)
point = dataframe.DataPoint('unit', 0, 0, {'g_one': 'one'},
{'m_two': 'two'})
usage = {'metric_x': [point]}
dict_usage = {'metric_x': [point.as_dict(mutable=True)]}
self.assertEqual(
dataframe.DataFrame(start, end, usage).as_dict(),
dataframe.DataFrame.from_dict({
'period': {
'begin': start.isoformat(),
'end': end.isoformat()
},
'usage': dict_usage,
}).as_dict(),
)
def _build_dataframes(self, docs):
dataframes = {}
nb_points = 0
for doc in docs:
source = doc['_source']
start = tzutils.dt_from_iso(source['start'])
end = tzutils.dt_from_iso(source['end'])
key = (start, end)
if key not in dataframes.keys():
dataframes[key] = dataframe.DataFrame(start=start, end=end)
dataframes[key].add_point(self._doc_to_datapoint(source),
source['type'])
nb_points += 1
output = list(dataframes.values())
output.sort(key=lambda frame: (frame.start, frame.end))
return output
def _build_dataframes(self, points):
dataframes = {}
for point in points:
point_type = point['type']
time = tzutils.dt_from_iso(point['time'])
period = point.get(PERIOD_FIELD_NAME) or self._default_period
timekey = (time,
tzutils.add_delta(time,
datetime.timedelta(seconds=period)))
if timekey not in dataframes.keys():
dataframes[timekey] = dataframe.DataFrame(start=timekey[0],
end=timekey[1])
dataframes[timekey].add_point(
self._point_to_dataframe_entry(point), point_type)
output = list(dataframes.values())
output.sort(key=lambda frame: (frame.start, frame.end))
return output
def test_dataframe_add_points(self):
start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC)
end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC)
df = dataframe.DataFrame(start=start, end=end)
a_points = [
dataframe.DataPoint(**TestDataPoint.default_params)
for _ in range(2)
]
b_points = [
dataframe.DataPoint(**TestDataPoint.default_params)
for _ in range(4)
]
df.add_point(a_points[0], 'service_a')
df.add_points(a_points[1:], 'service_a')
df.add_points(b_points[:2], 'service_b')
df.add_points(b_points[2:3], 'service_b')
df.add_point(b_points[3], 'service_b')
self.assertEqual(
dict(df.as_dict()), {
'period': {
'begin': start,
'end': end
},
'usage': {
'service_a': [
dataframe.DataPoint(
**TestDataPoint.default_params).as_dict()
for _ in range(2)
],
'service_b': [
dataframe.DataPoint(
**TestDataPoint.default_params).as_dict()
for _ in range(4)
],
}
})
def test_json(self):
start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC)
end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC)
df = dataframe.DataFrame(start=start, end=end)
a_points = [
dataframe.DataPoint(**TestDataPoint.default_params)
for _ in range(2)
]
b_points = [
dataframe.DataPoint(**TestDataPoint.default_params)
for _ in range(4)
]
df.add_points(a_points, 'service_a')
df.add_points(b_points, 'service_b')
self.maxDiff = None
self.assertEqual(
json.loads(df.json()),
json.loads(
json.dumps({
'period': {
'begin': start.isoformat(),
'end': end.isoformat()
},
'usage': {
'service_a': [
dataframe.DataPoint(
**TestDataPoint.default_params).as_dict()
for _ in range(2)
],
'service_b': [
dataframe.DataPoint(
**TestDataPoint.default_params).as_dict()
for _ in range(4)
],
}
})))
def run(self):
while True:
timestamp = self._check_state()
if not timestamp:
break
metrics = list(self._conf['metrics'].keys())
# Collection
usage_data = self._do_collection(metrics, timestamp)
frame = dataframe.DataFrame(
start=timestamp,
end=tzutils.add_delta(timestamp,
timedelta(seconds=self._period)),
usage=usage_data,
)
# Rating
for processor in self._processors:
frame = processor.obj.process(frame)
# Writing
self._storage.push([frame], self._tenant_id)
self._state.set_state(self._tenant_id, timestamp)
def quote(self, ctxt, res_data):
LOG.debug('Received quote request [%s] from RPC.', res_data)
worker = APIWorker()
start = tzutils.localized_now()
end = tzutils.add_delta(start, timedelta(seconds=CONF.collect.period))
# Need to prepare data to support the V2 processing format
usage = {}
for k in res_data['usage']:
all_data_points_for_metric = []
all_quote_data_entries = res_data['usage'][k]
for p in all_quote_data_entries:
vol = p['vol']
desc = p.get('desc', {})
data_point = dataframe.DataPoint(
vol['unit'],
vol['qty'],
0,
desc.get('groupby', []),
desc.get('metadata', []),
)
all_data_points_for_metric.append(data_point)
usage[k] = all_data_points_for_metric
frame = dataframe.DataFrame(
start=start,
end=end,
usage=usage,
)
quote_result = worker.quote(frame)
LOG.debug("Quote result [%s] for input data [%s].",
quote_result, res_data)
return str(quote_result)
def test_properties(self):
start = datetime.datetime(2019, 6, 1, tzinfo=tz.UTC)
end = datetime.datetime(2019, 6, 1, 1, tzinfo=tz.UTC)
df = dataframe.DataFrame(start=start, end=end)
self.assertEqual(df.start, start)
self.assertEqual(df.end, end)
IMAGE_GROUPBY = {
'id': '7b5b73f2-9181-4307-a710-b1aa6472526d',
}
FIRST_PERIOD = {
'begin': FIRST_PERIOD_BEGIN,
'end': FIRST_PERIOD_END,
}
SECOND_PERIOD = {
'begin': SECOND_PERIOD_BEGIN,
'end': SECOND_PERIOD_END,
}
COLLECTED_DATA = [
dataframe.DataFrame(start=FIRST_PERIOD["begin"], end=FIRST_PERIOD["end"]),
dataframe.DataFrame(start=SECOND_PERIOD["begin"],
end=SECOND_PERIOD["end"]),
]
_INSTANCE_POINT = dataframe.DataPoint('instance', '1.0', '0.42',
COMPUTE_GROUPBY, COMPUTE_METADATA)
_IMAGE_SIZE_POINT = dataframe.DataPoint('image', '1.0', '0.1337',
IMAGE_GROUPBY, IMAGE_METADATA)
COLLECTED_DATA[0].add_point(_INSTANCE_POINT, 'instance')
COLLECTED_DATA[0].add_point(_IMAGE_SIZE_POINT, 'image.size')
COLLECTED_DATA[1].add_point(_INSTANCE_POINT, 'instance')
RATED_DATA = copy.deepcopy(COLLECTED_DATA)
