def test_iterpoints(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_points(points, 'metric_x') expected = [('metric_x', dataframe.DataPoint(**TestDataPoint.default_params)) for _ in range(4)] self.assertEqual(list(df.iterpoints()), expected)
def generate_v2_storage_data(min_length=10, nb_projects=2, project_ids=None, start=None, end=None): if not project_ids: project_ids = [uuidutils.generate_uuid() for i in range(nb_projects)] elif not isinstance(project_ids, list): project_ids = [project_ids] df = dataframe.DataFrame(start=start, end=end) for metric_name, sample in samples.V2_STORAGE_SAMPLE.items(): datapoints = [] for project_id in project_ids: data = [copy.deepcopy(sample) for i in range(min_length + random.randint(1, 10))] for elem in data: elem['groupby']['id'] = uuidutils.generate_uuid() elem['groupby']['project_id'] = project_id datapoints += [dataframe.DataPoint( elem['vol']['unit'], elem['vol']['qty'], elem['rating']['price'], elem['groupby'], elem['metadata'], ) for elem in data] df.add_points(datapoints, metric_name) return df
def fetch_all(self, metric_name, start, end, project_id=None, q_filter=None): met = self.conf[metric_name] data = self._fetch_measures( metric_name, start, end, project_id=project_id, q_filter=q_filter, ) resources_info = None if met['metadata']: resources_info = self._fetch_metrics( metric_name, start, end, project_id=project_id, q_filter=q_filter, ) formated_resources = list() for d in data: if len(d['statistics']): metadata, groupby, qty = self._format_data( met, d, resources_info) formated_resources.append(dataframe.DataPoint( met['unit'], qty, 0, groupby, metadata, )) return formated_resources
def test_add_point_no_autocommit(self): point = dataframe.DataPoint('unit', '0.42', '0.1337', {}, {}) start = datetime.datetime(2019, 1, 1) end = datetime.datetime(2019, 1, 1, 1) with mock.patch.object(self.client, 'commit') as func_mock: with mock.patch.object(self.client, '_autocommit', new=False): with mock.patch.object(self.client, '_chunk_size', new=3): self.client._docs = [] for _ in range(5): self.client.add_point(point, 'awesome_type', start, end) func_mock.assert_not_called() self.assertEqual(self.client._docs, [{ 'start': start, 'end': end, 'type': 'awesome_type', 'unit': point.unit, 'qty': point.qty, 'price': point.price, 'groupby': point.groupby, 'metadata': point.metadata, } for _ in range(5)]) self.client._docs = []
def test_add_point_with_autocommit(self): point = dataframe.DataPoint('unit', '0.42', '0.1337', {}, {}) start = datetime.datetime(2019, 1, 1) end = datetime.datetime(2019, 1, 1, 1) commit_calls = {'count': 0} def commit(): # We can't re-assign nonlocal variables in python2 commit_calls['count'] += 1 self.client._docs = [] with mock.patch.object(self.client, 'commit', new=commit): with mock.patch.object(self.client, '_autocommit', new=True): with mock.patch.object(self.client, '_chunk_size', new=3): self.client._docs = [] for i in range(5): self.client.add_point(point, 'awesome_type', start, end) self.assertEqual(commit_calls['count'], 1) self.assertEqual(self.client._docs, [{ 'start': start, 'end': end, 'type': 'awesome_type', 'unit': point.unit, 'qty': point.qty, 'price': point.price, 'groupby': point.groupby, 'metadata': point.metadata, } for _ in range(2)]) # cleanup self.client._docs = []
def test_from_dict_valid_dict(self): self.assertEqual( dataframe.DataPoint( unit="amazing_unit", qty=3, price=0, groupby={ "g_one": "one", "g_two": "two" }, metadata={ "m_one": "one", "m_two": "two" }, ).as_dict(), dataframe.DataPoint.from_dict({ "vol": { "unit": "amazing_unit", "qty": 3 }, "groupby": { "g_one": "one", "g_two": "two" }, "metadata": { "m_one": "one", "m_two": "two" }, }).as_dict(), )
def test_properties(self): params = copy.deepcopy(self.default_params) groupby = {"group_one": "one", "group_two": "two"} metadata = {"meta_one": "one", "meta_two": "two"} params.update({'groupby': groupby, 'metadata': metadata}) point = dataframe.DataPoint(**params) self.assertEqual(point.groupby, groupby) self.assertEqual(point.metadata, metadata)
def _doc_to_datapoint(doc): return dataframe.DataPoint( doc['unit'], doc['qty'], doc['price'], doc['groupby'], doc['metadata'], )
def _point_to_dataframe_entry(point): groupby = filter(bool, (point.pop('groupby', None) or '').split('|')) metadata = filter(bool, (point.pop('metadata', None) or '').split('|')) return dataframe.DataPoint( point['unit'], point['qty'], point['price'], {key: point.get(key, '') for key in groupby}, {key: point.get(key, '') for key in metadata}, )
def fetch_all(self, metric_name, start, end, project_id=None, q_filter=None): met = self.conf[metric_name] data = self._fetch_metric( metric_name, start, end, project_id=project_id, q_filter=q_filter, ) data = GnocchiCollector.filter_unecessary_measurements( data, met, metric_name) resources_info = None if met['metadata']: resources_info = self._fetch_resources(metric_name, start, end, project_id=project_id, q_filter=q_filter) formated_resources = list() for d in data: # Only if aggregates have been found LOG.debug( "Processing entry [%s] for [%s] in timestamp [" "start=%s, end=%s] and project id [%s]", d, metric_name, start, end, project_id) if d['measures']['measures']['aggregated']: try: metadata, groupby, qty = self._format_data( met, d, resources_info) except AssociatedResourceNotFound as e: LOG.warning( '[{}] An error occured during data collection ' 'between {} and {}: {}'.format(project_id, start, end, e), ) continue formated_resources.append( dataframe.DataPoint( met['unit'], qty, 0, groupby, metadata, )) return formated_resources
def test_point_to_dataframe_entry_valid_point(self): self.assertEqual( influx.InfluxStorage._point_to_dataframe_entry(self.point), dataframe.DataPoint( 'banana', 42, 1, {'one': '1', 'two': '2'}, {'1': 'one', '2': 'two'}, ), )
def test_repr(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_points(points, 'metric_x') self.assertEqual(str(df), "DataFrame(metrics=[metric_x])") df.add_points(points, 'metric_y') self.assertEqual(str(df), "DataFrame(metrics=[metric_x,metric_y])")
def test_desc(self): params = copy.deepcopy(self.default_params) params['groupby'] = {'group_one': 'one', 'group_two': 'two'} params['metadata'] = {'meta_one': 'one', 'meta_two': 'two'} point = dataframe.DataPoint(**params) self.assertEqual( point.desc, { 'group_one': 'one', 'group_two': 'two', 'meta_one': 'one', 'meta_two': 'two', })
def fetch_all(self, metric_name, start, end, scope_id, q_filter=None): """Returns metrics to be valorized.""" scope_key = CONF.collect.scope_key method = self.conf[metric_name]['extra_args']['aggregation_method'] groupby = self.conf[metric_name].get('groupby', []) metadata = self.conf[metric_name].get('metadata', []) period = tzutils.diff_seconds(end, start) time = end query = '{0}({0}_over_time({1}{{{2}="{3}"}}[{4}s])) by ({5})'.format( method, metric_name, scope_key, scope_id, period, ', '.join(groupby + metadata), ) try: res = self._conn.get_instant( query, time.isoformat(), ) except PrometheusResponseError as e: raise CollectError(*e.args) # If the query returns an empty dataset, # return an empty list if not res['data']['result']: return [] formatted_resources = [] for item in res['data']['result']: metadata, groupby, qty = self._format_data( metric_name, scope_key, scope_id, start, end, item, ) formatted_resources.append( dataframe.DataPoint( self.conf[metric_name]['unit'], qty, 0, groupby, metadata, )) return formatted_resources
def test_as_dict_mutable_standard(self): self.assertEqual( dataframe.DataPoint(**self.default_params).as_dict(mutable=True), { "vol": { "unit": "undefined", "qty": decimal.Decimal(0) }, "rating": { "price": decimal.Decimal(0) }, "groupby": {}, "metadata": {}, })
def test_json_standard(self): self.assertEqual( json.loads(dataframe.DataPoint(**self.default_params).json()), { "vol": { "unit": "undefined", "qty": decimal.Decimal(0) }, "rating": { "price": decimal.Decimal(0) }, "groupby": {}, "metadata": {}, })
def test_dataframe_add_points(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) a_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(2) ] b_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_point(a_points[0], 'service_a') df.add_points(a_points[1:], 'service_a') df.add_points(b_points[:2], 'service_b') df.add_points(b_points[2:3], 'service_b') df.add_point(b_points[3], 'service_b') self.assertEqual( dict(df.as_dict()), { 'period': { 'begin': start, 'end': end }, 'usage': { 'service_a': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(2) ], 'service_b': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(4) ], } })
def test_as_dict_mutable_legacy(self): self.assertEqual( dataframe.DataPoint(**self.default_params).as_dict(legacy=True, mutable=True), { "vol": { "unit": "undefined", "qty": decimal.Decimal(0) }, "rating": { "price": decimal.Decimal(0) }, "desc": {}, })
def test_json_legacy(self): self.assertEqual( json.loads( dataframe.DataPoint(**self.default_params).json(legacy=True)), { "vol": { "unit": "undefined", "qty": decimal.Decimal(0) }, "rating": { "price": decimal.Decimal(0) }, "desc": {}, })
def test_point_to_dataframe_entry_invalid_groupby_metadata(self): point = copy.deepcopy(self.point) point['groupby'] = 'a' point['metadata'] = None self.assertEqual( influx.InfluxStorage._point_to_dataframe_entry(point), dataframe.DataPoint( 'banana', 42, 1, {'a': ''}, {}, ), )
def test_json(self): start = datetime.datetime(2019, 3, 4, 1, tzinfo=tz.UTC) end = datetime.datetime(2019, 3, 4, 2, tzinfo=tz.UTC) df = dataframe.DataFrame(start=start, end=end) a_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(2) ] b_points = [ dataframe.DataPoint(**TestDataPoint.default_params) for _ in range(4) ] df.add_points(a_points, 'service_a') df.add_points(b_points, 'service_b') self.maxDiff = None self.assertEqual( json.loads(df.json()), json.loads( json.dumps({ 'period': { 'begin': start.isoformat(), 'end': end.isoformat() }, 'usage': { 'service_a': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(2) ], 'service_b': [ dataframe.DataPoint( **TestDataPoint.default_params).as_dict() for _ in range(4) ], } })))
def test_format_retrieve(self): expected_name = 'http_requests_total' expected_data = [ dataframe.DataPoint('instance', '7', '0', { 'bar': '', 'foo': '', 'project_id': '' }, { 'code': '200', 'instance': 'localhost:9090' }), dataframe.DataPoint('instance', '42', '0', { 'bar': '', 'foo': '', 'project_id': '' }, { 'code': '200', 'instance': 'localhost:9090' }), ] no_response = mock.patch( 'cloudkitty.common.prometheus_client.PrometheusClient.get_instant', return_value=samples.PROMETHEUS_RESP_INSTANT_QUERY, ) with no_response: actual_name, actual_data = self.collector.retrieve( metric_name='http_requests_total', start=samples.FIRST_PERIOD_BEGIN, end=samples.FIRST_PERIOD_END, project_id=samples.TENANT, q_filter=None, ) self.assertEqual(expected_name, actual_name) self.assertEqual(expected_data, actual_data)
def create_fake_data(self, begin, end, project_id): cpu_point = dataframe.DataPoint( unit="nothing", qty=1, groupby={ "fake_meta": 1.0, "project_id": project_id }, metadata={"dummy": True}, price=decimal.Decimal('1.337'), ) image_point = dataframe.DataPoint( unit="nothing", qty=1, groupby={ "fake_meta": 1.0, "project_id": project_id }, metadata={"dummy": True}, price=decimal.Decimal('0.121'), ) data = [ dataframe.DataFrame( start=begin, end=end, usage=collections.OrderedDict({"cpu": [cpu_point, cpu_point]}), ), dataframe.DataFrame( start=begin, end=end, usage=collections.OrderedDict( {"image.size": [image_point, image_point]}), ), ] return data
def test_as_dict_immutable(self): point_dict = dataframe.DataPoint(**self.default_params).as_dict() self.assertIsInstance(point_dict, datastructures.ImmutableDict) self.assertEqual( dict(point_dict), { "vol": { "unit": "undefined", "qty": decimal.Decimal(0) }, "rating": { "price": decimal.Decimal(0) }, "groupby": {}, "metadata": {}, })
def test_from_dict_valid_dict_date_as_str(self): start = datetime.datetime(2019, 1, 2, 12, tzinfo=tz.UTC) end = datetime.datetime(2019, 1, 2, 13, tzinfo=tz.UTC) point = dataframe.DataPoint('unit', 0, 0, {'g_one': 'one'}, {'m_two': 'two'}) usage = {'metric_x': [point]} dict_usage = {'metric_x': [point.as_dict(mutable=True)]} self.assertEqual( dataframe.DataFrame(start, end, usage).as_dict(), dataframe.DataFrame.from_dict({ 'period': { 'begin': start.isoformat(), 'end': end.isoformat() }, 'usage': dict_usage, }).as_dict(), )
def fetch_all(self, metric_name, start, end, project_id=None, q_filter=None): met = self.conf[metric_name] data = self._fetch_metric( metric_name, start, end, project_id=project_id, q_filter=q_filter, ) resources_info = None if met['metadata']: resources_info = self._fetch_resources( metric_name, start, end, project_id=project_id, q_filter=q_filter ) formated_resources = list() for d in data: # Only if aggregates have been found if d['measures']['measures']['aggregated']: try: metadata, groupby, qty = self._format_data( met, d, resources_info) except AssociatedResourceNotFound as e: LOG.warning( '[{}] An error occured during data collection ' 'between {} and {}: {}'.format( project_id, start, end, e), ) continue formated_resources.append(dataframe.DataPoint( met['unit'], qty, 0, groupby, metadata, )) return formated_resources
def quote(self, ctxt, res_data): LOG.debug('Received quote request [%s] from RPC.', res_data) worker = APIWorker() start = tzutils.localized_now() end = tzutils.add_delta(start, timedelta(seconds=CONF.collect.period)) # Need to prepare data to support the V2 processing format usage = {} for k in res_data['usage']: all_data_points_for_metric = [] all_quote_data_entries = res_data['usage'][k] for p in all_quote_data_entries: vol = p['vol'] desc = p.get('desc', {}) data_point = dataframe.DataPoint( vol['unit'], vol['qty'], 0, desc.get('groupby', []), desc.get('metadata', []), ) all_data_points_for_metric.append(data_point) usage[k] = all_data_points_for_metric frame = dataframe.DataFrame( start=start, end=end, usage=usage, ) quote_result = worker.quote(frame) LOG.debug("Quote result [%s] for input data [%s].", quote_result, res_data) return str(quote_result)
def test_create_empty_datapoint(self): point = dataframe.DataPoint(**self.default_params) self.assertEqual(point.qty, decimal.Decimal(0)) self.assertEqual(point.price, decimal.Decimal(0)) self.assertEqual(point.unit, "undefined") self.assertEqual(point.groupby, {})
def test_set_price(self): point = dataframe.DataPoint(**self.default_params) self.assertEqual(point.price, decimal.Decimal(0)) self.assertEqual(point.set_price(42).price, decimal.Decimal(42)) self.assertEqual(point.set_price(1337).price, decimal.Decimal(1337))
def test_readonly_attrs(self): point = dataframe.DataPoint(**self.default_params) for attr in ("qty", "price", "unit"): self.assertRaises(AttributeError, setattr, point, attr, 'x')