def test_get_resolution_empty_default(self): """Test get_resolution returns default when query params are empty.""" query_params = {} handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') self.assertEqual(handler.get_resolution(), 'daily') self.assertEqual(handler.get_resolution(), 'daily')
def test_get_time_scope_value_empty_default(self): """Test get_time_scope_value returns default when query params are empty.""" query_params = {} handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') self.assertEqual(handler.get_time_scope_value(), -10) self.assertEqual(handler.get_time_scope_value(), -10)
def test_get_group_by_with_service_list(self): """Test the get_group_by_data method with no data in the query params.""" expected = ['a', 'b'] query_string = '?group_by[service]=a&group_by[service]=b' handler = ReportQueryHandler({'group_by': { 'service': expected }}, query_string, self.tenant, 'unblended_cost', 'currency_code') service = handler.get_group_by_data('service') self.assertEqual(expected, service)
def test_has_filter_with_filter(self): """Test the has_filter method with filter in the query params.""" query_params = { 'filter': { 'resolution': 'monthly', 'time_scope_value': -1 } } handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') self.assertIsNotNone( handler.check_query_params('filter', 'time_scope_value'))
def _set_access_ocp_all(self, provider, filter_key, access_key, raise_exception=True): """Alter query parameters based on user access.""" access_list = self.access.get(access_key, {}).get("read", []) access_filter_applied = False if ReportQueryHandler.has_wildcard(access_list): with tenant_context(self.tenant): access_list = list( OCPAllCostLineItemDailySummary.objects.filter( source_type=provider).values_list( "usage_account_id", flat=True).distinct()) # check group by group_by = self.parameters.get("group_by", {}) if group_by.get(filter_key): items = set(group_by.get(filter_key)) items.update(access_list) if set(group_by.get(filter_key)) != items: self.parameters["group_by"][filter_key] = list(items) access_filter_applied = True if not access_filter_applied: if self.parameters.get("filter", {}).get(filter_key): items = set(self.get_filter(filter_key)) items.update(access_list) self.parameters["filter"][filter_key] = list(items) elif access_list: self.parameters["filter"][filter_key] = access_list
def _set_access(self, provider, filter_key, access_key, raise_exception=True): """Alter query parameters based on user access.""" access_list = self.access.get(access_key, {}).get("read", []) access_filter_applied = False if ReportQueryHandler.has_wildcard(access_list): return # check group by group_by = self.parameters.get("group_by", {}) if access_key == "aws.organizational_unit": if "org_unit_id" in group_by or "or:org_unit_id" in group_by: # Only check the tree hierarchy if we are grouping by org units. # we will want to overwrite the access_list here to include the sub orgs in # the hierarchy for later checks regarding filtering. access_list = self._check_org_unit_tree_hierarchy(group_by, access_list) if group_by.get(filter_key): items = set(group_by.get(filter_key)) result = get_replacement_result(items, access_list, raise_exception) if result: self.parameters["access"][filter_key] = result access_filter_applied = True if not access_filter_applied: if self.parameters.get("filter", {}).get(filter_key): items = set(self.get_filter(filter_key)) result = get_replacement_result(items, access_list, raise_exception) if result: self.parameters["access"][filter_key] = result elif access_list: self.parameters["access"][filter_key] = access_list
def _set_access(self, filter_key, access_key, raise_exception=True): """Alter query parameters based on user access.""" access_list = self.access.get(access_key, {}).get('read', []) access_filter_applied = False if ReportQueryHandler.has_wildcard(access_list): return # check group by group_by = self.parameters.get('group_by', {}) if group_by.get(filter_key): items = set(group_by.get(filter_key)) result = get_replacement_result(items, access_list, raise_exception) if result: self.parameters['group_by'][filter_key] = result access_filter_applied = True if not access_filter_applied: if self.parameters.get('filter', {}).get(filter_key): items = set(self.get_filter(filter_key)) result = get_replacement_result(items, access_list, raise_exception) if result: self.parameters['filter'][filter_key] = result elif access_list: self.parameters['filter'][filter_key] = access_list
def test_execute_query_current_month_monthly(self): """Test execute_query for current month on monthly breakdown.""" query_params = { 'filter': { 'resolution': 'monthly', 'time_scope_value': -1, 'time_scope_units': 'month' } } handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') query_output = handler.execute_query() self.assertIsNotNone(query_output.get('data')) self.assertIsNotNone(query_output.get('total')) total = query_output.get('total') self.assertIsNotNone(total.get('value')) self.assertEqual(total.get('value'), Decimal('4.776000000'))
def _get_replacement_result(param_res_list, access_list, raise_exception=True): if ReportQueryHandler.has_wildcard(param_res_list): return access_list if not access_list and not raise_exception: return list(param_res_list) intersection = param_res_list & set(access_list) if not intersection: raise PermissionDenied() return list(intersection)
def _check_restrictions(self, set_access_list): """Check if all non-ocp providers have wildcard access.""" all_wildcard = [] for set_access in set_access_list: provider, __, access_key, *__ = set_access if provider != Provider.PROVIDER_OCP: access_list = self.access.get(access_key, {}).get("read", []) all_wildcard.append(ReportQueryHandler.has_wildcard(access_list)) return False in all_wildcard
def test_n_days_ago(self): """Test the n_days_ago method.""" delta_day = datetime.timedelta(days=1) today = timezone.now().replace(microsecond=0, second=0, minute=0, hour=0) two_days_ago = (today - delta_day) - delta_day self.assertEqual(two_days_ago, ReportQueryHandler.n_days_ago(today, 2))
def _generic_report(request, aggregate_key, units_key, **kwargs): """Generically query for reports. Args: request (Request): The HTTP request object aggregate_key (str): The report metric to be aggregated e.g. 'usage_amount' or 'unblended_cost' units_key (str): The field used to establish the reporting unit Returns: (Response): The report in a Response object """ LOG.info(f'API: {request.path} USER: {request.user.username}') url_data = request.GET.urlencode() validation, params = process_query_parameters(url_data) if not validation: return Response(data=params, status=status.HTTP_400_BAD_REQUEST) tenant = get_tenant(request.user) if kwargs: kwargs['accept_type'] = request.META.get('HTTP_ACCEPT') else: kwargs = {'accept_type': request.META.get('HTTP_ACCEPT')} handler = ReportQueryHandler(params, url_data, tenant, aggregate_key, units_key, **kwargs) output = handler.execute_query() if 'units' in params: from_unit = _find_unit()(output['data']) if from_unit: try: to_unit = params['units'] unit_converter = UnitConverter() output = _fill_in_missing_units(from_unit)(output) output = _convert_units(unit_converter, output, to_unit) except (DimensionalityError, UndefinedUnitError): error = {'details': _('Unit conversion failed.')} raise ValidationError(error) LOG.debug(f'DATA: {output}') return Response(output)
def test_previous_month(self): """Test the previous_month method.""" current_month = timezone.now().replace(microsecond=0, second=0, minute=0, hour=0, day=1) last_month = current_month.replace(month=(current_month.month - 1)) self.assertEqual(last_month, ReportQueryHandler.previous_month(current_month))
def test_get_time_frame_filter_last_thirty(self): """Test _get_time_frame_filter for last thirty days.""" query_params = { 'filter': { 'resolution': 'daily', 'time_scope_value': -30, 'time_scope_units': 'day' } } handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') current_day = timezone.now().replace(microsecond=0, second=0, minute=0) ten_days_ago = ReportQueryHandler.n_days_ago(current_day, 30) start = handler.start_datetime end = handler.end_datetime interval = handler.time_interval self.assertEqual(start, ten_days_ago) self.assertEqual(end, current_day) self.assertIsInstance(interval, list) self.assertTrue(len(interval) == 31)
def test_list_days(self): """Test the list_days method.""" first = timezone.now().replace(microsecond=0, second=0, minute=0, hour=0, day=1) second = first.replace(day=2) third = first.replace(day=3) expected = [first, second, third] self.assertEqual(expected, ReportQueryHandler.list_days(first, third))
def test_execute_query_by_account_by_service(self): """Test execute_query for current month breakdown by account by service.""" query_params = { 'filter': { 'resolution': 'monthly', 'time_scope_value': -1, 'time_scope_units': 'month' }, 'group_by': { 'account': ['*'], 'service': ['*'] } } query_string = '?group_by[account]=*&group_by[service]=Compute Instance' handler = ReportQueryHandler(query_params, query_string, self.tenant, 'unblended_cost', 'currency_code') query_output = handler.execute_query() data = query_output.get('data') self.assertIsNotNone(data) self.assertIsNotNone(query_output.get('total')) total = query_output.get('total') self.assertIsNotNone(total.get('value')) self.assertEqual(total.get('value'), Decimal('4.776000000')) current_month = timezone.now().replace(microsecond=0, second=0, minute=0, hour=0, day=1) cmonth_str = current_month.strftime('%Y-%m') for data_item in data: month_val = data_item.get('date') month_data = data_item.get('accounts') self.assertEqual(month_val, cmonth_str) self.assertIsInstance(month_data, list) for month_item in month_data: account = month_item.get('account') self.assertEqual(account, self.payer_account_id) self.assertIsInstance(month_item.get('services'), list)
def test_get_time_frame_filter_previous_month(self): """Test _get_time_frame_filter for previous month.""" query_params = { 'filter': { 'resolution': 'daily', 'time_scope_value': -2, 'time_scope_units': 'month' } } handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') current_month = timezone.now().replace(microsecond=0, second=0, minute=0, hour=0, day=1) prev_month = ReportQueryHandler.previous_month(current_month) start = handler.start_datetime end = handler.end_datetime interval = handler.time_interval self.assertEqual(start, prev_month) self.assertEqual(end, current_month) self.assertIsInstance(interval, list) self.assertTrue(len(interval) >= 28)
def get_replacement_result(param_res_list, access_list, raise_exception=True): """Adjust param list based on access list.""" if ReportQueryHandler.has_wildcard(param_res_list): return access_list if not (access_list or raise_exception): return list(param_res_list) access_difference = param_res_list.difference(set(access_list)) if access_difference: LOG.warning( "User does not have permissions for the requested params: %s. Current access: %s.", param_res_list, access_list, ) raise PermissionDenied() return param_res_list
def get_replacement_result(param_res_list, access_list, raise_exception=True): """Adjust param list based on access list.""" if ReportQueryHandler.has_wildcard(param_res_list): return access_list if not access_list and not raise_exception: return list(param_res_list) intersection = param_res_list & set(access_list) if not intersection: LOG.warning( "User does not have permissions for the " "requested params: %s. Current access: %s.", param_res_list, access_list, ) raise PermissionDenied() return list(intersection)
def add_data_to_tenant(self): """Populate tenant with data.""" payer_account_id = self.fake.ean(length=13) # pylint: disable=no-member self.payer_account_id = payer_account_id one_day = datetime.timedelta(days=1) one_hour = datetime.timedelta(minutes=60) this_hour = timezone.now().replace(microsecond=0, second=0, minute=0) yesterday = this_hour - one_day bill_start = this_hour.replace(microsecond=0, second=0, minute=0, hour=0, day=1) bill_end = ReportQueryHandler.next_month(bill_start) with tenant_context(self.tenant): bill = AWSCostEntryBill(bill_type='Anniversary', payer_account_id=payer_account_id, billing_period_start=bill_start, billing_period_end=bill_end) bill.save() rate = 0.199 amount = 1 cost = rate * amount # pylint: disable=no-member sku = self.fake.pystr(min_chars=12, max_chars=12).upper() prod_ec2 = 'Amazon Elastic Compute Cloud' ce_product = AWSCostEntryProduct(sku=sku, product_name=prod_ec2, product_family='Compute Instance', service_code='AmazonEC2', region='US East (N. Virginia)', instance_type='c4.xlarge', memory=7.5, vcpu=4) ce_product.save() ce_pricing = AWSCostEntryPricing(public_on_demand_cost=rate * 1, public_on_demand_rate=rate, term='OnDemand', unit='Hrs') ce_pricing.save() current = yesterday while current < this_hour: end_hour = current + one_hour self.create_hourly_instance_usage(payer_account_id, bill, ce_pricing, ce_product, rate, cost, current, end_hour) current = end_hour
def _update_query_parameters(query_parameters, filter_key, access, access_key, raise_exception=True): """Alter query parameters based on user access.""" access_list = access.get(access_key, {}).get('read', []) access_filter_applied = False if ReportQueryHandler.has_wildcard(access_list): return query_parameters # check group by group_by = query_parameters.get('group_by', {}) if group_by.get(filter_key): items = set(group_by.get(filter_key)) result = _get_replacement_result(items, access_list, raise_exception=True) if result: query_parameters['group_by'][filter_key] = result access_filter_applied = True if not access_filter_applied: if query_parameters.get('filter', {}).get(filter_key): items = set(query_parameters.get('filter', {}).get(filter_key)) result = _get_replacement_result(items, access_list, raise_exception) if result: if query_parameters.get('filter') is None: query_parameters['filter'] = {} query_parameters['filter'][filter_key] = result elif access_list: if query_parameters.get('filter') is None: query_parameters['filter'] = {} query_parameters['filter'][filter_key] = access_list return query_parameters
def test_has_wildcard_none(self): """Test an empty list doesn't have a wildcard.""" result = ReportQueryHandler.has_wildcard([]) self.assertFalse(result)
def test_has_wildcard_no(self): """Test a list doesn't have a wildcard.""" result = ReportQueryHandler.has_wildcard(['abc', 'def']) self.assertFalse(result)
def test_has_wildcard_yes(self): """Test a list has a wildcard.""" result = ReportQueryHandler.has_wildcard(['abc', '*']) self.assertTrue(result)
def test_get_group_by_no_data(self): """Test the get_group_by_data method with no data in the query params.""" handler = ReportQueryHandler({}, '', self.tenant, 'unblended_cost', 'currency_code') self.assertFalse(handler.get_group_by_data('service'))
def test_group_data_by_list(self): """Test the _group_data_by_list method.""" group_by = ['account', 'service'] data = [{ 'account': 'a1', 'service': 's1', 'units': 'USD', 'total': 4 }, { 'account': 'a1', 'service': 's2', 'units': 'USD', 'total': 5 }, { 'account': 'a2', 'service': 's1', 'units': 'USD', 'total': 6 }, { 'account': 'a2', 'service': 's2', 'units': 'USD', 'total': 5 }, { 'account': 'a1', 'service': 's3', 'units': 'USD', 'total': 5 }] out_data = ReportQueryHandler._group_data_by_list(group_by, 0, data) expected = { 'a1': { 's1': [{ 'account': 'a1', 'service': 's1', 'units': 'USD', 'total': 4 }], 's2': [{ 'account': 'a1', 'service': 's2', 'units': 'USD', 'total': 5 }], 's3': [{ 'account': 'a1', 'service': 's3', 'units': 'USD', 'total': 5 }] }, 'a2': { 's1': [{ 'account': 'a2', 'service': 's1', 'units': 'USD', 'total': 6 }], 's2': [{ 'account': 'a2', 'service': 's2', 'units': 'USD', 'total': 5 }] } } self.assertEqual(expected, out_data)
def test_get_time_scope_value_empty_day_time_scope(self): """Test get_time_scope_value returns default when time_scope is month.""" query_params = {'filter': {'time_scope_units': 'day'}} handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') self.assertEqual(handler.get_time_scope_value(), -10)
def costs(request): """Get cost data. @api {get} /api/v1/reports/costs/ Get cost data @apiName getCostData @apiGroup Report @apiVersion 1.0.0 @apiDescription Get cost data. @apiHeader {String} token User authorization token. @apiHeaderExample {json} Header-Example: { "Authorization": "Token 45138a913da44ab89532bab0352ef84b" } @apiParam (Query Param) {Object} filter The filter to apply to the report. @apiParam (Query Param) {Object} group_by The grouping to apply to the report. @apiParam (Query Param) {Object} order_by The ordering to apply to the report. @apiParamExample {json} Query Param: ?filter[resolution]=daily&filter[time_scope_value]=-10&order_by[cost]=asc @apiSuccess {Object} group_by The grouping to applied to the report. @apiSuccess {Object} order_by The ordering to applied to the report @apiSuccess {Object} filter The filter to applied to the report. @apiSuccess {Object} data The report data. @apiSuccessExample {json} Success-Response: HTTP/1.1 200 OK { "group_by": { "account": [ "*" ] }, "order_by": { "cost": "asc" }, "filter": { "resolution": "daily", "time_scope_value": -10, "time_scope_units": "day", "resource_scope": [] }, "data": [ [ { "date": "2018-05-28", "accounts": [ { "account": "8577742690384", "values": [ { "date": "2018-05-28", "units": "USD", "account": "8577742690384", "total": 1498.92962634 } ] }, { "account": "9420673783214", "values": [ { "date": "2018-05-28", "units": "USD", "account": "9420673783214", "total": 1065.845524241 } ] } ] } ] ] } """ url_data = request.GET.urlencode() validation, value = process_query_parameters(url_data) if not validation: return Response(data=value, status=status.HTTP_400_BAD_REQUEST) tenant = get_tenant(request.user) handler = ReportQueryHandler(value, url_data, tenant, 'unblended_cost', 'currency_code') output = handler.execute_query() return Response(output)
def test_get_resolution_empty_day_time_scope(self): """Test get_resolution returns default when time_scope is month.""" query_params = {'filter': {'time_scope_value': -10}} handler = ReportQueryHandler(query_params, '', self.tenant, 'unblended_cost', 'currency_code') self.assertEqual(handler.get_resolution(), 'daily')
def test_has_filter_no_filter(self): """Test the has_filter method with no filter in the query params.""" handler = ReportQueryHandler({}, '', self.tenant, 'unblended_cost', 'currency_code') self.assertFalse( handler.check_query_params('filter', 'time_scope_value'))