def generate_v2_storage_data(min_length=10, nb_projects=2, project_ids=None, start=None, end=None): if not project_ids: project_ids = [uuidutils.generate_uuid() for i in range(nb_projects)] elif not isinstance(project_ids, list): project_ids = [project_ids] df = dataframe.DataFrame(start=start, end=end) for metric_name, sample in samples.V2_STORAGE_SAMPLE.items(): datapoints = [] for project_id in project_ids: data = [copy.deepcopy(sample) for i in range(min_length + random.randint(1, 10))] for elem in data: elem['groupby']['id'] = uuidutils.generate_uuid() elem['groupby']['project_id'] = project_id datapoints += [dataframe.DataPoint( elem['vol']['unit'], elem['vol']['qty'], elem['rating']['price'], elem['groupby'], elem['metadata'], ) for elem in data] df.add_points(datapoints, metric_name) return df
def process(self, data): output = dataframe.DataFrame(start=data.start, end=data.end) for service_name, point in data.iterpoints(): self._res = {} self.process_services(service_name, point) self.process_fields(service_name, point) output.add_point(self.add_rating_informations(point), service_name) return output
def test_repr(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_points(points, 'metric_x') self.assertEqual(str(df), "DataFrame(metrics=[metric_x])") df.add_points(points, 'metric_y') self.assertEqual(str(df), "DataFrame(metrics=[metric_x,metric_y])")
def test_iterpoints(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_points(points, 'metric_x') expected = [('metric_x', dataframe.DataPoint(**TestDataPoint.default_params)) for _ in range(4)] self.assertEqual(list(df.iterpoints()), expected)
def execute_measurements_rating(self, end_time, start_time, usage_data): frame = dataframe.DataFrame( start=start_time, end=end_time, usage=usage_data, ) for processor in self._processors: original_data = copy.deepcopy(frame) frame = processor.obj.process(frame) LOG.debug("Results [%s] for processing [%s] of data points [%s].", frame, processor.obj.process, original_data) return frame
def create_fake_data(self, begin, end, project_id): cpu_point = dataframe.DataPoint( unit="nothing", qty=1, groupby={ "fake_meta": 1.0, "project_id": project_id }, metadata={"dummy": True}, price=decimal.Decimal('1.337'), ) image_point = dataframe.DataPoint( unit="nothing", qty=1, groupby={ "fake_meta": 1.0, "project_id": project_id }, metadata={"dummy": True}, price=decimal.Decimal('0.121'), ) data = [ dataframe.DataFrame( start=begin, end=end, usage=collections.OrderedDict({"cpu": [cpu_point, cpu_point]}), ), dataframe.DataFrame( start=begin, end=end, usage=collections.OrderedDict( {"image.size": [image_point, image_point]}), ), ] return data
def test_from_dict_valid_dict_date_as_str(self): start = datetime.datetime(2019, 1, 2, 12, tzinfo=tz.UTC) end = datetime.datetime(2019, 1, 2, 13, tzinfo=tz.UTC) point = dataframe.DataPoint('unit', 0, 0, {'g_one': 'one'}, {'m_two': 'two'}) usage = {'metric_x': [point]} dict_usage = {'metric_x': [point.as_dict(mutable=True)]} self.assertEqual( dataframe.DataFrame(start, end, usage).as_dict(), dataframe.DataFrame.from_dict({ 'period': { 'begin': start.isoformat(), 'end': end.isoformat() }, 'usage': dict_usage, }).as_dict(), )
def _build_dataframes(self, docs): dataframes = {} nb_points = 0 for doc in docs: source = doc['_source'] start = tzutils.dt_from_iso(source['start']) end = tzutils.dt_from_iso(source['end']) key = (start, end) if key not in dataframes.keys(): dataframes[key] = dataframe.DataFrame(start=start, end=end) dataframes[key].add_point(self._doc_to_datapoint(source), source['type']) nb_points += 1 output = list(dataframes.values()) output.sort(key=lambda frame: (frame.start, frame.end)) return output
def _build_dataframes(self, points): dataframes = {} for point in points: point_type = point['type'] time = tzutils.dt_from_iso(point['time']) period = point.get(PERIOD_FIELD_NAME) or self._default_period timekey = (time, tzutils.add_delta(time, datetime.timedelta(seconds=period))) if timekey not in dataframes.keys(): dataframes[timekey] = dataframe.DataFrame(start=timekey[0], end=timekey[1]) dataframes[timekey].add_point( self._point_to_dataframe_entry(point), point_type) output = list(dataframes.values()) output.sort(key=lambda frame: (frame.start, frame.end)) return output
def test_dataframe_add_points(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) a_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(2) ] b_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_point(a_points[0], 'service_a') df.add_points(a_points[1:], 'service_a') df.add_points(b_points[:2], 'service_b') df.add_points(b_points[2:3], 'service_b') df.add_point(b_points[3], 'service_b') self.assertEqual( dict(df.as_dict()), { 'period': { 'begin': start, 'end': end }, 'usage': { 'service_a': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(2) ], 'service_b': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(4) ], } })
def test_json(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) a_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(2) ] b_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_points(a_points, 'service_a') df.add_points(b_points, 'service_b') self.maxDiff = None self.assertEqual( json.loads(df.json()), json.loads( json.dumps({ 'period': { 'begin': start.isoformat(), 'end': end.isoformat() }, 'usage': { 'service_a': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(2) ], 'service_b': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(4) ], } })))
def run(self): while True: timestamp = self._check_state() if not timestamp: break metrics = list(self._conf['metrics'].keys()) # Collection usage_data = self._do_collection(metrics, timestamp) frame = dataframe.DataFrame( start=timestamp, end=tzutils.add_delta(timestamp, timedelta(seconds=self._period)), usage=usage_data, ) # Rating for processor in self._processors: frame = processor.obj.process(frame) # Writing self._storage.push([frame], self._tenant_id) self._state.set_state(self._tenant_id, timestamp)
def quote(self, ctxt, res_data): LOG.debug('Received quote request [%s] from RPC.', res_data) worker = APIWorker() start = tzutils.localized_now() end = tzutils.add_delta(start, timedelta(seconds=CONF.collect.period)) # Need to prepare data to support the V2 processing format usage = {} for k in res_data['usage']: all_data_points_for_metric = [] all_quote_data_entries = res_data['usage'][k] for p in all_quote_data_entries: vol = p['vol'] desc = p.get('desc', {}) data_point = dataframe.DataPoint( vol['unit'], vol['qty'], 0, desc.get('groupby', []), desc.get('metadata', []), ) all_data_points_for_metric.append(data_point) usage[k] = all_data_points_for_metric frame = dataframe.DataFrame( start=start, end=end, usage=usage, ) quote_result = worker.quote(frame) LOG.debug("Quote result [%s] for input data [%s].", quote_result, res_data) return str(quote_result)
def test_properties(self): start = datetime.datetime(2019, 6, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 6, 1, 1, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) self.assertEqual(df.start, start) self.assertEqual(df.end, end)
IMAGE_GROUPBY = { 'id': '7b5b73f2-9181-4307-a710-b1aa6472526d', } FIRST_PERIOD = { 'begin': FIRST_PERIOD_BEGIN, 'end': FIRST_PERIOD_END, } SECOND_PERIOD = { 'begin': SECOND_PERIOD_BEGIN, 'end': SECOND_PERIOD_END, } COLLECTED_DATA = [ dataframe.DataFrame(start=FIRST_PERIOD["begin"], end=FIRST_PERIOD["end"]), dataframe.DataFrame(start=SECOND_PERIOD["begin"], end=SECOND_PERIOD["end"]), ] _INSTANCE_POINT = dataframe.DataPoint('instance', '1.0', '0.42', COMPUTE_GROUPBY, COMPUTE_METADATA) _IMAGE_SIZE_POINT = dataframe.DataPoint('image', '1.0', '0.1337', IMAGE_GROUPBY, IMAGE_METADATA) COLLECTED_DATA[0].add_point(_INSTANCE_POINT, 'instance') COLLECTED_DATA[0].add_point(_IMAGE_SIZE_POINT, 'image.size') COLLECTED_DATA[1].add_point(_INSTANCE_POINT, 'instance') RATED_DATA = copy.deepcopy(COLLECTED_DATA)