def test_get_resolution_empty_default(self):
"""Test get_resolution returns default when query params are empty."""
query_params = {}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
self.assertEqual(handler.get_resolution(), 'daily')
self.assertEqual(handler.get_resolution(), 'daily')
def test_get_time_scope_value_empty_default(self):
"""Test get_time_scope_value returns default when query params are empty."""
query_params = {}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
self.assertEqual(handler.get_time_scope_value(), -10)
self.assertEqual(handler.get_time_scope_value(), -10)
def test_get_group_by_with_service_list(self):
"""Test the get_group_by_data method with no data in the query params."""
expected = ['a', 'b']
query_string = '?group_by[service]=a&group_by[service]=b'
handler = ReportQueryHandler({'group_by': {
'service': expected
}}, query_string, self.tenant, 'unblended_cost', 'currency_code')
service = handler.get_group_by_data('service')
self.assertEqual(expected, service)
def test_has_filter_with_filter(self):
"""Test the has_filter method with filter in the query params."""
query_params = {
'filter': {
'resolution': 'monthly',
'time_scope_value': -1
}
}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
self.assertIsNotNone(
handler.check_query_params('filter', 'time_scope_value'))
def _set_access_ocp_all(self,
provider,
filter_key,
access_key,
raise_exception=True):
"""Alter query parameters based on user access."""
access_list = self.access.get(access_key, {}).get("read", [])
access_filter_applied = False
if ReportQueryHandler.has_wildcard(access_list):
with tenant_context(self.tenant):
access_list = list(
OCPAllCostLineItemDailySummary.objects.filter(
source_type=provider).values_list(
"usage_account_id", flat=True).distinct())
# check group by
group_by = self.parameters.get("group_by", {})
if group_by.get(filter_key):
items = set(group_by.get(filter_key))
items.update(access_list)
if set(group_by.get(filter_key)) != items:
self.parameters["group_by"][filter_key] = list(items)
access_filter_applied = True
if not access_filter_applied:
if self.parameters.get("filter", {}).get(filter_key):
items = set(self.get_filter(filter_key))
items.update(access_list)
self.parameters["filter"][filter_key] = list(items)
elif access_list:
self.parameters["filter"][filter_key] = access_list
def _set_access(self, provider, filter_key, access_key, raise_exception=True):
"""Alter query parameters based on user access."""
access_list = self.access.get(access_key, {}).get("read", [])
access_filter_applied = False
if ReportQueryHandler.has_wildcard(access_list):
return
# check group by
group_by = self.parameters.get("group_by", {})
if access_key == "aws.organizational_unit":
if "org_unit_id" in group_by or "or:org_unit_id" in group_by:
# Only check the tree hierarchy if we are grouping by org units.
# we will want to overwrite the access_list here to include the sub orgs in
# the hierarchy for later checks regarding filtering.
access_list = self._check_org_unit_tree_hierarchy(group_by, access_list)
if group_by.get(filter_key):
items = set(group_by.get(filter_key))
result = get_replacement_result(items, access_list, raise_exception)
if result:
self.parameters["access"][filter_key] = result
access_filter_applied = True
if not access_filter_applied:
if self.parameters.get("filter", {}).get(filter_key):
items = set(self.get_filter(filter_key))
result = get_replacement_result(items, access_list, raise_exception)
if result:
self.parameters["access"][filter_key] = result
elif access_list:
self.parameters["access"][filter_key] = access_list
def _set_access(self, filter_key, access_key, raise_exception=True):
"""Alter query parameters based on user access."""
access_list = self.access.get(access_key, {}).get('read', [])
access_filter_applied = False
if ReportQueryHandler.has_wildcard(access_list):
return
# check group by
group_by = self.parameters.get('group_by', {})
if group_by.get(filter_key):
items = set(group_by.get(filter_key))
result = get_replacement_result(items, access_list,
raise_exception)
if result:
self.parameters['group_by'][filter_key] = result
access_filter_applied = True
if not access_filter_applied:
if self.parameters.get('filter', {}).get(filter_key):
items = set(self.get_filter(filter_key))
result = get_replacement_result(items, access_list,
raise_exception)
if result:
self.parameters['filter'][filter_key] = result
elif access_list:
self.parameters['filter'][filter_key] = access_list
def test_execute_query_current_month_monthly(self):
"""Test execute_query for current month on monthly breakdown."""
query_params = {
'filter': {
'resolution': 'monthly',
'time_scope_value': -1,
'time_scope_units': 'month'
}
}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get('data'))
self.assertIsNotNone(query_output.get('total'))
total = query_output.get('total')
self.assertIsNotNone(total.get('value'))
self.assertEqual(total.get('value'), Decimal('4.776000000'))
def _get_replacement_result(param_res_list, access_list, raise_exception=True):
if ReportQueryHandler.has_wildcard(param_res_list):
return access_list
if not access_list and not raise_exception:
return list(param_res_list)
intersection = param_res_list & set(access_list)
if not intersection:
raise PermissionDenied()
return list(intersection)
def _check_restrictions(self, set_access_list):
"""Check if all non-ocp providers have wildcard access."""
all_wildcard = []
for set_access in set_access_list:
provider, __, access_key, *__ = set_access
if provider != Provider.PROVIDER_OCP:
access_list = self.access.get(access_key, {}).get("read", [])
all_wildcard.append(ReportQueryHandler.has_wildcard(access_list))
return False in all_wildcard
def test_n_days_ago(self):
"""Test the n_days_ago method."""
delta_day = datetime.timedelta(days=1)
today = timezone.now().replace(microsecond=0,
second=0,
minute=0,
hour=0)
two_days_ago = (today - delta_day) - delta_day
self.assertEqual(two_days_ago, ReportQueryHandler.n_days_ago(today, 2))
def _generic_report(request, aggregate_key, units_key, **kwargs):
"""Generically query for reports.
Args:
request (Request): The HTTP request object
aggregate_key (str): The report metric to be aggregated
e.g. 'usage_amount' or 'unblended_cost'
units_key (str): The field used to establish the reporting unit
Returns:
(Response): The report in a Response object
"""
LOG.info(f'API: {request.path} USER: {request.user.username}')
url_data = request.GET.urlencode()
validation, params = process_query_parameters(url_data)
if not validation:
return Response(data=params, status=status.HTTP_400_BAD_REQUEST)
tenant = get_tenant(request.user)
if kwargs:
kwargs['accept_type'] = request.META.get('HTTP_ACCEPT')
else:
kwargs = {'accept_type': request.META.get('HTTP_ACCEPT')}
handler = ReportQueryHandler(params, url_data, tenant, aggregate_key,
units_key, **kwargs)
output = handler.execute_query()
if 'units' in params:
from_unit = _find_unit()(output['data'])
if from_unit:
try:
to_unit = params['units']
unit_converter = UnitConverter()
output = _fill_in_missing_units(from_unit)(output)
output = _convert_units(unit_converter, output, to_unit)
except (DimensionalityError, UndefinedUnitError):
error = {'details': _('Unit conversion failed.')}
raise ValidationError(error)
LOG.debug(f'DATA: {output}')
return Response(output)
def test_previous_month(self):
"""Test the previous_month method."""
current_month = timezone.now().replace(microsecond=0,
second=0,
minute=0,
hour=0,
day=1)
last_month = current_month.replace(month=(current_month.month - 1))
self.assertEqual(last_month,
ReportQueryHandler.previous_month(current_month))
def test_get_time_frame_filter_last_thirty(self):
"""Test _get_time_frame_filter for last thirty days."""
query_params = {
'filter': {
'resolution': 'daily',
'time_scope_value': -30,
'time_scope_units': 'day'
}
}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
current_day = timezone.now().replace(microsecond=0, second=0, minute=0)
ten_days_ago = ReportQueryHandler.n_days_ago(current_day, 30)
start = handler.start_datetime
end = handler.end_datetime
interval = handler.time_interval
self.assertEqual(start, ten_days_ago)
self.assertEqual(end, current_day)
self.assertIsInstance(interval, list)
self.assertTrue(len(interval) == 31)
def test_list_days(self):
"""Test the list_days method."""
first = timezone.now().replace(microsecond=0,
second=0,
minute=0,
hour=0,
day=1)
second = first.replace(day=2)
third = first.replace(day=3)
expected = [first, second, third]
self.assertEqual(expected, ReportQueryHandler.list_days(first, third))
def test_execute_query_by_account_by_service(self):
"""Test execute_query for current month breakdown by account by service."""
query_params = {
'filter': {
'resolution': 'monthly',
'time_scope_value': -1,
'time_scope_units': 'month'
},
'group_by': {
'account': ['*'],
'service': ['*']
}
}
query_string = '?group_by[account]=*&group_by[service]=Compute Instance'
handler = ReportQueryHandler(query_params, query_string, self.tenant,
'unblended_cost', 'currency_code')
query_output = handler.execute_query()
data = query_output.get('data')
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get('total'))
total = query_output.get('total')
self.assertIsNotNone(total.get('value'))
self.assertEqual(total.get('value'), Decimal('4.776000000'))
current_month = timezone.now().replace(microsecond=0,
second=0,
minute=0,
hour=0,
day=1)
cmonth_str = current_month.strftime('%Y-%m')
for data_item in data:
month_val = data_item.get('date')
month_data = data_item.get('accounts')
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
account = month_item.get('account')
self.assertEqual(account, self.payer_account_id)
self.assertIsInstance(month_item.get('services'), list)
def test_get_time_frame_filter_previous_month(self):
"""Test _get_time_frame_filter for previous month."""
query_params = {
'filter': {
'resolution': 'daily',
'time_scope_value': -2,
'time_scope_units': 'month'
}
}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
current_month = timezone.now().replace(microsecond=0,
second=0,
minute=0,
hour=0,
day=1)
prev_month = ReportQueryHandler.previous_month(current_month)
start = handler.start_datetime
end = handler.end_datetime
interval = handler.time_interval
self.assertEqual(start, prev_month)
self.assertEqual(end, current_month)
self.assertIsInstance(interval, list)
self.assertTrue(len(interval) >= 28)
def get_replacement_result(param_res_list, access_list, raise_exception=True):
"""Adjust param list based on access list."""
if ReportQueryHandler.has_wildcard(param_res_list):
return access_list
if not (access_list or raise_exception):
return list(param_res_list)
access_difference = param_res_list.difference(set(access_list))
if access_difference:
LOG.warning(
"User does not have permissions for the requested params: %s. Current access: %s.",
param_res_list,
access_list,
)
raise PermissionDenied()
return param_res_list
def get_replacement_result(param_res_list, access_list, raise_exception=True):
"""Adjust param list based on access list."""
if ReportQueryHandler.has_wildcard(param_res_list):
return access_list
if not access_list and not raise_exception:
return list(param_res_list)
intersection = param_res_list & set(access_list)
if not intersection:
LOG.warning(
"User does not have permissions for the "
"requested params: %s. Current access: %s.",
param_res_list,
access_list,
)
raise PermissionDenied()
return list(intersection)
def add_data_to_tenant(self):
"""Populate tenant with data."""
payer_account_id = self.fake.ean(length=13) # pylint: disable=no-member
self.payer_account_id = payer_account_id
one_day = datetime.timedelta(days=1)
one_hour = datetime.timedelta(minutes=60)
this_hour = timezone.now().replace(microsecond=0, second=0, minute=0)
yesterday = this_hour - one_day
bill_start = this_hour.replace(microsecond=0,
second=0,
minute=0,
hour=0,
day=1)
bill_end = ReportQueryHandler.next_month(bill_start)
with tenant_context(self.tenant):
bill = AWSCostEntryBill(bill_type='Anniversary',
payer_account_id=payer_account_id,
billing_period_start=bill_start,
billing_period_end=bill_end)
bill.save()
rate = 0.199
amount = 1
cost = rate * amount
# pylint: disable=no-member
sku = self.fake.pystr(min_chars=12, max_chars=12).upper()
prod_ec2 = 'Amazon Elastic Compute Cloud'
ce_product = AWSCostEntryProduct(sku=sku,
product_name=prod_ec2,
product_family='Compute Instance',
service_code='AmazonEC2',
region='US East (N. Virginia)',
instance_type='c4.xlarge',
memory=7.5,
vcpu=4)
ce_product.save()
ce_pricing = AWSCostEntryPricing(public_on_demand_cost=rate * 1,
public_on_demand_rate=rate,
term='OnDemand',
unit='Hrs')
ce_pricing.save()
current = yesterday
while current < this_hour:
end_hour = current + one_hour
self.create_hourly_instance_usage(payer_account_id, bill,
ce_pricing, ce_product, rate,
cost, current, end_hour)
current = end_hour
def _update_query_parameters(query_parameters,
filter_key,
access,
access_key,
raise_exception=True):
"""Alter query parameters based on user access."""
access_list = access.get(access_key, {}).get('read', [])
access_filter_applied = False
if ReportQueryHandler.has_wildcard(access_list):
return query_parameters
# check group by
group_by = query_parameters.get('group_by', {})
if group_by.get(filter_key):
items = set(group_by.get(filter_key))
result = _get_replacement_result(items,
access_list,
raise_exception=True)
if result:
query_parameters['group_by'][filter_key] = result
access_filter_applied = True
if not access_filter_applied:
if query_parameters.get('filter', {}).get(filter_key):
items = set(query_parameters.get('filter', {}).get(filter_key))
result = _get_replacement_result(items, access_list,
raise_exception)
if result:
if query_parameters.get('filter') is None:
query_parameters['filter'] = {}
query_parameters['filter'][filter_key] = result
elif access_list:
if query_parameters.get('filter') is None:
query_parameters['filter'] = {}
query_parameters['filter'][filter_key] = access_list
return query_parameters
def test_has_wildcard_none(self):
"""Test an empty list doesn't have a wildcard."""
result = ReportQueryHandler.has_wildcard([])
self.assertFalse(result)
def test_has_wildcard_no(self):
"""Test a list doesn't have a wildcard."""
result = ReportQueryHandler.has_wildcard(['abc', 'def'])
self.assertFalse(result)
def test_has_wildcard_yes(self):
"""Test a list has a wildcard."""
result = ReportQueryHandler.has_wildcard(['abc', '*'])
self.assertTrue(result)
def test_get_group_by_no_data(self):
"""Test the get_group_by_data method with no data in the query params."""
handler = ReportQueryHandler({}, '', self.tenant, 'unblended_cost',
'currency_code')
self.assertFalse(handler.get_group_by_data('service'))
def test_group_data_by_list(self):
"""Test the _group_data_by_list method."""
group_by = ['account', 'service']
data = [{
'account': 'a1',
'service': 's1',
'units': 'USD',
'total': 4
}, {
'account': 'a1',
'service': 's2',
'units': 'USD',
'total': 5
}, {
'account': 'a2',
'service': 's1',
'units': 'USD',
'total': 6
}, {
'account': 'a2',
'service': 's2',
'units': 'USD',
'total': 5
}, {
'account': 'a1',
'service': 's3',
'units': 'USD',
'total': 5
}]
out_data = ReportQueryHandler._group_data_by_list(group_by, 0, data)
expected = {
'a1': {
's1': [{
'account': 'a1',
'service': 's1',
'units': 'USD',
'total': 4
}],
's2': [{
'account': 'a1',
'service': 's2',
'units': 'USD',
'total': 5
}],
's3': [{
'account': 'a1',
'service': 's3',
'units': 'USD',
'total': 5
}]
},
'a2': {
's1': [{
'account': 'a2',
'service': 's1',
'units': 'USD',
'total': 6
}],
's2': [{
'account': 'a2',
'service': 's2',
'units': 'USD',
'total': 5
}]
}
}
self.assertEqual(expected, out_data)
def test_get_time_scope_value_empty_day_time_scope(self):
"""Test get_time_scope_value returns default when time_scope is month."""
query_params = {'filter': {'time_scope_units': 'day'}}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
self.assertEqual(handler.get_time_scope_value(), -10)
def costs(request):
"""Get cost data.
@api {get} /api/v1/reports/costs/ Get cost data
@apiName getCostData
@apiGroup Report
@apiVersion 1.0.0
@apiDescription Get cost data.
@apiHeader {String} token User authorization token.
@apiHeaderExample {json} Header-Example:
{
"Authorization": "Token 45138a913da44ab89532bab0352ef84b"
}
@apiParam (Query Param) {Object} filter The filter to apply to the report.
@apiParam (Query Param) {Object} group_by The grouping to apply to the report.
@apiParam (Query Param) {Object} order_by The ordering to apply to the report.
@apiParamExample {json} Query Param:
?filter[resolution]=daily&filter[time_scope_value]=-10&order_by[cost]=asc
@apiSuccess {Object} group_by The grouping to applied to the report.
@apiSuccess {Object} order_by The ordering to applied to the report
@apiSuccess {Object} filter The filter to applied to the report.
@apiSuccess {Object} data The report data.
@apiSuccessExample {json} Success-Response:
HTTP/1.1 200 OK
{
"group_by": {
"account": [
"*"
]
},
"order_by": {
"cost": "asc"
},
"filter": {
"resolution": "daily",
"time_scope_value": -10,
"time_scope_units": "day",
"resource_scope": []
},
"data": [
[
{
"date": "2018-05-28",
"accounts": [
{
"account": "8577742690384",
"values": [
{
"date": "2018-05-28",
"units": "USD",
"account": "8577742690384",
"total": 1498.92962634
}
]
},
{
"account": "9420673783214",
"values": [
{
"date": "2018-05-28",
"units": "USD",
"account": "9420673783214",
"total": 1065.845524241
}
]
}
]
}
]
]
}
"""
url_data = request.GET.urlencode()
validation, value = process_query_parameters(url_data)
if not validation:
return Response(data=value, status=status.HTTP_400_BAD_REQUEST)
tenant = get_tenant(request.user)
handler = ReportQueryHandler(value, url_data, tenant, 'unblended_cost',
'currency_code')
output = handler.execute_query()
return Response(output)
def test_get_resolution_empty_day_time_scope(self):
"""Test get_resolution returns default when time_scope is month."""
query_params = {'filter': {'time_scope_value': -10}}
handler = ReportQueryHandler(query_params, '', self.tenant,
'unblended_cost', 'currency_code')
self.assertEqual(handler.get_resolution(), 'daily')
def test_has_filter_no_filter(self):
"""Test the has_filter method with no filter in the query params."""
handler = ReportQueryHandler({}, '', self.tenant, 'unblended_cost',
'currency_code')
self.assertFalse(
handler.check_query_params('filter', 'time_scope_value'))