def test_set_access_filter_with_list(self):
"""
Tests that when an access restriction, filters, and a filter list are passed in,
the correct query filters are added
"""
# create the elements needed to mock the query handler
term = self.mock_tag_key
operation = "icontains"
params = self.mocked_query_params("", self.mock_view)
mapper = {"filter": [{}], "filters": {term: {"field": term, "operation": operation}}}
rqh = create_test_handler(params, mapper=mapper)
# set filters and access to be used in function
filters = QueryFilterCollection()
access = ["589173575009"]
filt = [
{"field": "account_alias__account_alias", "operation": "icontains", "composition_key": "account_filter"},
{"field": "usage_account_id", "operation": "icontains", "composition_key": "account_filter"},
]
expected = QueryFilterCollection(
filters=[
QueryFilter(field="account_alias__account_alias", operation="in", composition_key="account_filter"),
QueryFilter(field="usage_account_id", operation="in", composition_key="account_filter"),
]
)
rqh._set_access_filters(access, filt, filters)
self.assertIsInstance(filters, QueryFilterCollection)
assertSameQ(filters.compose(), expected.compose())
def _set_operator_specified_filters(self, operator):
"""Set any filters using AND instead of OR."""
filters = QueryFilterCollection()
composed_filter = Q()
for filter_key in self.SUPPORTED_FILTERS:
operator_key = operator + ":" + filter_key
filter_value = self.parameters.get_filter(operator_key)
logical_operator = operator
if filter_value and len(filter_value) < 2:
logical_operator = "or"
if filter_value and not OrgQueryHandler.has_wildcard(filter_value):
filter_obj = self.FILTER_MAP.get(filter_key)
if isinstance(filter_obj, list):
for _filt in filter_obj:
filt_filters = QueryFilterCollection()
for item in filter_value:
q_filter = QueryFilter(parameter=item, logical_operator=logical_operator, **_filt)
filt_filters.add(q_filter)
composed_filter = composed_filter | filt_filters.compose()
else:
for item in filter_value:
q_filter = QueryFilter(parameter=item, logical_operator=logical_operator, **filter_obj)
filters.add(q_filter)
if filters:
composed_filter = composed_filter & filters.compose()
return composed_filter
def test_set_access_filter_with_list(self):
"""
Tests that when an access restriction, filters, and a filter list are passed in,
the correct query filters are added
"""
# create the elements needed to mock the query handler
params = self.mocked_query_params("", AWSCostForecastView)
instance = AWSForecast(params)
# set filters and access to be used in function
filters = QueryFilterCollection()
access = ["589173575009"]
filt = [
{
"field": "account_alias__account_alias",
"operation": "icontains",
"composition_key": "account_filter"
},
{
"field": "usage_account_id",
"operation": "icontains",
"composition_key": "account_filter"
},
]
expected = QueryFilterCollection(filters=[
QueryFilter(field="account_alias__account_alias",
operation="in",
composition_key="account_filter"),
QueryFilter(field="usage_account_id",
operation="in",
composition_key="account_filter"),
])
instance.set_access_filters(access, filt, filters)
self.assertIsInstance(filters, QueryFilterCollection)
assertSameQ(filters.compose(), expected.compose())
def _set_operator_specified_filters(self, operator):
"""Set any filters using AND instead of OR."""
fields = self._mapper._provider_map.get("filters")
filters = QueryFilterCollection()
composed_filter = Q()
for q_param, filt in fields.items():
q_param = operator + ":" + q_param
group_by = self.parameters.get_group_by(q_param, list())
filter_ = self.parameters.get_filter(q_param, list())
list_ = list(set(group_by + filter_)) # uniquify the list
logical_operator = operator
# This is a flexibilty feature allowing a user to set
# a single and: value and still get a result instead
# of erroring on validation
if len(list_) < 2:
logical_operator = "or"
if list_ and not ReportQueryHandler.has_wildcard(list_):
if isinstance(filt, list):
for _filt in filt:
filt_filters = QueryFilterCollection()
for item in list_:
q_filter = QueryFilter(
parameter=item,
logical_operator=logical_operator,
**_filt)
filt_filters.add(q_filter)
# List filter are a complex mix of and/or logic
# Each filter in the list must be ORed together
# regardless of the operator on the item in the filter
# Ex:
# (OR:
# (AND:
# ('cluster_alias__icontains', 'ni'),
# ('cluster_alias__icontains', 'se')
# ),
# (AND:
# ('cluster_id__icontains', 'ni'),
# ('cluster_id__icontains', 'se')
# )
# )
composed_filter = composed_filter | filt_filters.compose(
)
else:
list_ = self._build_custom_filter_list(
q_param, filt.get("custom"), list_)
for item in list_:
q_filter = QueryFilter(
parameter=item,
logical_operator=logical_operator,
**filt)
filters.add(q_filter)
if filters:
composed_filter = composed_filter & filters.compose()
return composed_filter
def _get_exclusions(self, delta=False):
"""Create dictionary for filter parameters for exclude clause.
Returns:
(Dict): query filter dictionary
"""
exclusions = QueryFilterCollection()
tag_column = self._mapper.tag_column
tag_group_by = self.get_tag_group_by_keys()
if tag_group_by:
for tag in tag_group_by:
tag_db_name = tag_column + '__' + strip_tag_prefix(tag)
filt = {
'field': tag_db_name,
'operation': 'isnull',
'parameter': True
}
q_filter = QueryFilter(**filt)
exclusions.add(q_filter)
composed_exclusions = exclusions.compose()
LOG.debug(f'_get_exclusions: {composed_exclusions}')
return composed_exclusions
def _create_accounts_mapping(self):
"""Returns a mapping of org ids to accounts."""
account_mapping = {}
with tenant_context(self.tenant):
for source in self.data_sources:
# Grab columns for this query
account_info = source.get("account_alias_column")
# Create filters & Query
filters = QueryFilterCollection()
no_org_units = QueryFilter(field=f"{account_info}", operation="isnull", parameter=False)
filters.add(no_org_units)
composed_filters = filters.compose()
account_query = source.get("db_table").objects
account_query = account_query.filter(composed_filters)
account_query = account_query.exclude(deleted_timestamp__lte=self.start_datetime)
account_query = account_query.exclude(created_timestamp__gt=self.end_datetime)
if self.access:
accounts_to_filter = self.access.get("aws.account", {}).get("read", [])
if accounts_to_filter and "*" not in accounts_to_filter:
account_query = account_query.filter(account_alias__account_id__in=accounts_to_filter)
account_query = account_query.order_by(f"{account_info}", "-created_timestamp")
account_query = account_query.distinct(f"{account_info}")
account_query = account_query.annotate(
alias=Coalesce(F(f"{account_info}__account_alias"), F(f"{account_info}__account_id"))
)
for account in account_query:
org_id = account.org_unit_id
alias = account.alias
if account_mapping.get(org_id):
account_list = account_mapping[org_id]
account_list.append(alias)
account_mapping[org_id] = account_list
else:
account_mapping[org_id] = [alias]
return account_mapping
def test_set_operator_specified_tag_filters_or(self):
"""Test that AND/OR terms are correctly applied to tag filters."""
operator = "or"
term = self.mock_tag_key
first = FAKE.word()
second = FAKE.word()
operation = "icontains"
url = (
f"?filter[time_scope_value]=-1&"
f"filter[or:tag:{term}]={first}&"
f"filter[or:tag:{term}]={second}&"
f"group_by[or:tag:{term}]={first}&"
f"group_by[or:tag:{term}]={second}"
)
params = self.mocked_query_params(url, self.mock_view)
mapper = {"filter": [{}], "filters": {term: {"field": term, "operation": operation}}}
rqh = create_test_handler(params, mapper=mapper)
output = rqh._set_operator_specified_tag_filters(QueryFilterCollection(), operator)
self.assertIsNotNone(output)
expected = QueryFilterCollection(
filters=[
QueryFilter(field=term, operation=operation, parameter=second, logical_operator=operator),
QueryFilter(field=term, operation=operation, parameter=first, logical_operator=operator),
]
)
self.assertIsInstance(output, QueryFilterCollection)
assertSameQ(output.compose(), expected.compose())
def _get_filter(self, delta=False): # noqa: C901
"""Create dictionary for filter parameters.
Args:
delta (Boolean): Construct timeframe for delta
Returns:
(Dict): query filter dictionary
"""
filters = QueryFilterCollection()
for filter_key in self.SUPPORTED_FILTERS:
filter_value = self.parameters.get_filter(filter_key)
if filter_value and not OrgQueryHandler.has_wildcard(filter_value):
filter_obj = self.FILTER_MAP.get(filter_key)
for item in filter_value:
q_filter = QueryFilter(parameter=item, **filter_obj)
filters.add(q_filter)
# Update filters that specifiy and or or in the query parameter
and_composed_filters = self._set_operator_specified_filters("and")
or_composed_filters = self._set_operator_specified_filters("or")
composed_filters = filters.compose()
filter_list = [composed_filters, and_composed_filters, or_composed_filters]
final_filters = None
for filter_option in filter_list:
if filter_option:
if final_filters is not None:
final_filters & filter_option
else:
final_filters = filter_option
LOG.debug(f"_get_filter: {final_filters}")
return final_filters
def _get_sub_ou_list(self, data, org_ids):
"""Get a list of the sub org units for a org unit."""
level = data.get("level")
level = level + 1
unit_path = data.get("org_unit_path")
final_sub_ou_list = []
with tenant_context(self.tenant):
for source in self.data_sources:
# Grab columns for this query
account_info = source.get("account_alias_column")
level_column = source.get("level_column")
org_path = source.get("org_path_column")
# Build filters
filters = QueryFilterCollection()
no_accounts = QueryFilter(field=f"{account_info}", operation="isnull", parameter=True)
filters.add(no_accounts)
exact_parent_id = QueryFilter(field=f"{level_column}", operation="exact", parameter=level)
filters.add(exact_parent_id)
path_on_like = QueryFilter(field=f"{org_path}", operation="icontains", parameter=unit_path)
filters.add(path_on_like)
composed_filters = filters.compose()
# Start quering
sub_org_query = source.get("db_table").objects
sub_org_query = sub_org_query.filter(composed_filters)
sub_org_query = sub_org_query.filter(id__in=org_ids)
sub_ou_list = sub_org_query.values_list("org_unit_id", flat=True)
final_sub_ou_list.extend(sub_ou_list)
return final_sub_ou_list
def _get_key_filter(self):
"""
Add new `exact` QueryFilter that filters on the key name.
If filtering on value, uses the tags summary table to find the key
"""
filters = QueryFilterCollection()
filters.add(QueryFilter(field="key", operation="exact", parameter=self.key))
return self.query_filter & filters.compose()
def get_org_units(self):
"""Get a list of org keys to build upon."""
org_units = list()
org_id_list = list()
with tenant_context(self.tenant):
for source in self.data_sources:
# Grab columns for this query
org_id = source.get("org_id_column")
org_path = source.get("org_path_column")
org_name = source.get("org_name_column")
level = source.get("level_column")
account_info = source.get("account_alias_column")
created_field = source.get("created_time_column")
# Create filters & Query
account_filter = QueryFilterCollection()
no_accounts = QueryFilter(field=f"{account_info}",
operation="isnull",
parameter=True)
account_filter.add(no_accounts)
remove_accounts = account_filter.compose()
org_unit_query = source.get("db_table").objects
org_unit_query = org_unit_query.filter(remove_accounts)
org_unit_query = org_unit_query.exclude(
deleted_timestamp__lte=self.start_datetime)
org_unit_query = org_unit_query.exclude(
created_timestamp__gt=self.end_datetime)
val_list = [org_id, org_name, org_path, level]
org_unit_query = org_unit_query.order_by(
f"{org_id}", f"-{created_field}").distinct(f"{org_id}")
org_ids = org_unit_query.values_list("id", flat=True)
if self.access:
acceptable_ous = self.access.get("aws.organizational_unit",
{}).get("read", [])
if acceptable_ous and "*" not in acceptable_ous:
allowed_ids_query = source.get("db_table").objects
allowed_ids_query = allowed_ids_query.filter(
reduce(operator.or_,
(Q(org_unit_path__icontains=rbac)
for rbac in acceptable_ous
))).filter(remove_accounts)
allowed_ids = allowed_ids_query.values_list("id",
flat=True)
org_ids = list(set(org_ids) & set(allowed_ids))
org_unit_query = org_unit_query.filter(id__in=org_ids)
org_id_list.extend(org_ids)
# Note: you want to collect the org_id_list before you implement the self.query_filter
# so that way the get_sub_ou list will still work when you do filter[org_unit_id]=OU_002
if self.query_filter:
org_unit_query = org_unit_query.filter(self.query_filter)
org_unit_query = org_unit_query.values(*val_list)
org_units.extend(org_unit_query)
return org_units, org_id_list
def _get_search_filter(self, filters):
"""Populate the query filter collection for search filters.
Args:
filters (QueryFilterCollection): collection of query filters
Returns:
(QueryFilterCollection): populated collection of query filters
"""
# define filter parameters using API query params.
fields = self._mapper._provider_map.get("filters")
access_filters = QueryFilterCollection()
for q_param, filt in fields.items():
access = self.parameters.get_access(q_param, list())
group_by = self.parameters.get_group_by(q_param, list())
filter_ = self.parameters.get_filter(q_param, list())
list_ = list(set(group_by + filter_)) # uniquify the list
if list_ and not ReportQueryHandler.has_wildcard(list_):
if isinstance(filt, list):
for _filt in filt:
for item in list_:
q_filter = QueryFilter(parameter=item, **_filt)
filters.add(q_filter)
else:
list_ = self._build_custom_filter_list(
q_param, filt.get("custom"), list_)
for item in list_:
q_filter = QueryFilter(parameter=item, **filt)
filters.add(q_filter)
if access:
access_filt = copy.deepcopy(filt)
self.set_access_filters(access, access_filt, access_filters)
# Update filters with tag filters
filters = self._set_tag_filters(filters)
filters = self._set_operator_specified_tag_filters(filters, "and")
filters = self._set_operator_specified_tag_filters(filters, "or")
# Update filters that specifiy and or or in the query parameter
and_composed_filters = self._set_operator_specified_filters("and")
or_composed_filters = self._set_operator_specified_filters("or")
multi_field_or_composed_filters = self._set_or_filters()
composed_filters = filters.compose()
composed_filters = composed_filters & and_composed_filters & or_composed_filters
if access_filters:
composed_access_filters = access_filters.compose()
composed_filters = composed_filters & composed_access_filters
if multi_field_or_composed_filters:
composed_filters = composed_filters & multi_field_or_composed_filters
LOG.debug(f"_get_search_filter: {composed_filters}")
return composed_filters
def _set_or_filters(self):
"""Create a composed filter collection of ORed filters.
This is designed to handle specific cases in the provider_map
not to accomodate user input via the API.
"""
filters = QueryFilterCollection()
or_filter = self._mapper._report_type_map.get("or_filter", [])
for filt in or_filter:
q_filter = QueryFilter(**filt)
filters.add(q_filter)
return filters.compose(logical_operator="or")
def _get_filter(self, delta=False): # noqa: C901
"""Create dictionary for filter parameters.
Args:
delta (Boolean): Construct timeframe for delta
Returns:
(Dict): query filter dictionary
"""
filters = QueryFilterCollection()
if not self.parameters.get_filter("value"):
for source in self.data_sources:
start_filter, end_filter = self._get_time_based_filters(
source, delta)
filters.add(query_filter=start_filter)
filters.add(query_filter=end_filter)
for filter_key in self.SUPPORTED_FILTERS:
if self.parameters.get_filter("value") and filter_key == "enabled":
continue
filter_value = self.parameters.get_filter(filter_key)
if filter_value and not TagQueryHandler.has_wildcard(filter_value):
filter_obj = self.FILTER_MAP.get(filter_key)
if isinstance(filter_value, bool):
filters.add(QueryFilter(**filter_obj))
elif isinstance(filter_obj, list):
for _filt in filter_obj:
for item in filter_value:
q_filter = QueryFilter(parameter=item, **_filt)
filters.add(q_filter)
else:
for item in filter_value:
q_filter = QueryFilter(parameter=item, **filter_obj)
filters.add(q_filter)
access = self.parameters.get_access(filter_key)
filt = self.FILTER_MAP.get(filter_key, [])
if access and filt:
filt = self.FILTER_MAP.get(filter_key)
q_filter = QueryFilter(parameter=access, **filt)
filters.add(q_filter)
# Update filters that specifiy and or or in the query parameter
and_composed_filters = self._set_operator_specified_filters("and")
or_composed_filters = self._set_operator_specified_filters("or")
composed_filters = filters.compose()
composed_filters = composed_filters & and_composed_filters & or_composed_filters
LOG.debug(f"_get_filter: {composed_filters}")
return composed_filters
def test_compose_with_or_operator(self):
"""Test the compose() method with or operator on the compose method."""
qf_coll = QueryFilterCollection()
operation = self.fake.word()
filts = [
QueryFilter(table=self.fake.word(),
field=self.fake.word(),
operation=operation,
parameter=self.fake.word()) for _ in range(2)
]
expected = filts[0].composed_Q() | filts[1].composed_Q()
qf_coll.add(filts[0])
qf_coll.add(filts[1])
self.assertEqual(qf_coll.compose(logical_operator="or"), expected)
def test_compose(self):
"""Test the compose() method."""
qf_coll = QueryFilterCollection()
table = self.fake.word()
field = self.fake.word()
operation = self.fake.word()
parameter = self.fake.word()
filt = QueryFilter(table=table,
field=field,
operation=operation,
parameter=parameter)
expected = filt.composed_Q()
qf_coll.add(table=table,
field=field,
operation=operation,
parameter=parameter)
self.assertEqual(qf_coll.compose(), expected)
def test_compose_with_filter_with_and_operator(self):
"""Test the compose() method with and operator on the filter."""
qf_coll = QueryFilterCollection()
table = self.fake.word()
field = self.fake.word()
operation = self.fake.word()
filts = [
QueryFilter(table=table,
field=field,
operation=operation,
parameter=self.fake.word(),
logical_operator='and') for _ in range(2)
]
expected = filts[0].composed_Q() & filts[1].composed_Q()
qf_coll.add(filts[0])
qf_coll.add(filts[1])
self.assertEqual(qf_coll.compose(), expected)
def _get_exclusions(self, column):
"""Create dictionary for filter parameters for exclude clause.
For tags this is to filter items that have null values for the
specified tag field.
Args:
column (str): The tag column being queried
Returns:
(Dict): query filter dictionary
"""
exclusions = QueryFilterCollection()
filt = {"field": column, "operation": "isnull", "parameter": True}
q_filter = QueryFilter(**filt)
exclusions.add(q_filter)
composed_exclusions = exclusions.compose()
LOG.debug(f"_get_exclusions: {composed_exclusions}")
return composed_exclusions
def get_tag_values(self):
"""
Gets the values associated with a tag when filtering on a value.
"""
final_data = []
with tenant_context(self.tenant):
tag_keys = {}
for source in self.TAGS_VALUES_SOURCE:
vals_filter = QueryFilterCollection()
for key_field in source.get("fields"):
vals_filter.add(
QueryFilter(field=key_field,
operation="exact",
parameter=self.key,
composition_key="filter_key"))
tag_values_query = source.get("db_table").objects
filt = self.query_filter & vals_filter.compose()
tag_keys = list(tag_values_query.filter(filt))
tag_tup = self._value_filter_dict(tag_keys)
converted = self._convert_to_dict(tag_tup)
self.append_to_final_data_without_type(final_data, converted)
self.deduplicate_and_sort(final_data)
return final_data
def test_set_operator_specified_filters_and(self):
"""Test that AND/OR terms are correctly applied to param filters."""
operator = "and"
term = FAKE.word()
first = FAKE.word()
second = FAKE.word()
operation = FAKE.word()
url = f"?filter[time_scope_value]=-1&group_by[and:{term}]={first}&group_by[and:{term}]={second}"
params = self.mocked_query_params(url, self.mock_view)
mapper = {"filter": [{}], "filters": {term: {"field": term, "operation": operation}}}
rqh = create_test_handler(params, mapper=mapper)
output = rqh._set_operator_specified_filters(operator)
self.assertIsNotNone(output)
expected = QueryFilterCollection(
filters=[
QueryFilter(field=term, operation=operation, parameter=second, logical_operator=operator),
QueryFilter(field=term, operation=operation, parameter=first, logical_operator=operator),
]
)
assertSameQ(output, expected.compose())
class Forecast:
"""Base forecasting class."""
# the minimum number of data points needed to use the current month's data.
# if we have fewer than this many data points, fall back to using the previous month's data.
#
# this number is chosen in part because statsmodels.stats.stattools.omni_normtest() needs at least eight data
# points to test for normal distribution.
MINIMUM = 8
# the precision of the floats returned in the forecast response.
PRECISION = 8
REPORT_TYPE = "costs"
def __init__(self, query_params): # noqa: C901
"""Class Constructor.
Instance Attributes:
- cost_summary_table (Model)
- aggregates (dict)
- filters (QueryFilterCollection)
- query_range (tuple)
"""
self.dh = DateHelper()
self.params = query_params
# select appropriate model based on access
access = query_params.get("access", {})
access_key = "default"
self.cost_summary_table = self.provider_map.views.get("costs").get(access_key)
if access:
access_key = tuple(access.keys())
filter_fields = self.provider_map.provider_map.get("filters")
materialized_view = self.provider_map.views.get("costs").get(access_key)
if materialized_view:
# We found a matching materialized view, use that
self.cost_summary_table = materialized_view
else:
# We have access constraints, but no view to accomodate, default to daily summary table
self.cost_summary_table = self.provider_map.query_table
self.forecast_days_required = (self.dh.this_month_end - self.dh.yesterday).days
# forecasts use a rolling window
self.query_range = (self.dh.n_days_ago(self.dh.yesterday, 30), self.dh.yesterday)
self.filters = QueryFilterCollection()
self.filters.add(field="usage_start", operation="gte", parameter=self.query_range[0])
self.filters.add(field="usage_end", operation="lte", parameter=self.query_range[1])
# filter queries based on access
if access_key != "default":
for q_param, filt in filter_fields.items():
access = query_params.get_access(q_param, list())
if access:
self.set_access_filters(access, filt, self.filters)
@property
def provider_map(self):
"""Return the provider map instance."""
return self.provider_map_class(self.provider, self.REPORT_TYPE)
@property
def total_cost_term(self):
"""Return the provider map value for total cost."""
return self.provider_map.report_type_map.get("aggregates", {}).get("cost_total")
@property
def supplementary_cost_term(self):
"""Return the provider map value for total supplemenatry cost."""
return self.provider_map.report_type_map.get("aggregates", {}).get("sup_total")
@property
def infrastructure_cost_term(self):
"""Return the provider map value for total inftrastructure cost."""
return self.provider_map.report_type_map.get("aggregates", {}).get("infra_total")
def predict(self):
"""Define ORM query to run forecast and return prediction."""
cost_predictions = {}
with tenant_context(self.params.tenant):
data = (
self.cost_summary_table.objects.filter(self.filters.compose())
.order_by("usage_start")
.values("usage_start")
.annotate(
total_cost=self.total_cost_term,
supplementary_cost=self.supplementary_cost_term,
infrastructure_cost=self.infrastructure_cost_term,
)
)
for fieldname in ["total_cost", "infrastructure_cost", "supplementary_cost"]:
uniq_data = self._uniquify_qset(data.values("usage_start", fieldname), field=fieldname)
cost_predictions[fieldname] = self._predict(uniq_data)
cost_predictions = self._key_results_by_date(cost_predictions)
return self.format_result(cost_predictions)
def _predict(self, data):
"""Handle pre and post prediction work.
This function handles arranging incoming data to conform with statsmodels requirements.
Then after receiving the forecast output, this function handles formatting to conform to
API reponse requirements.
Args:
data (list) a list of (datetime, float) tuples
Returns:
(LinearForecastResult) linear forecast results object
"""
LOG.debug("Forecast input data: %s", data)
if len(data) < self.MINIMUM:
LOG.warning(
"Number of data elements (%s) is fewer than the minimum (%s). Unable to generate forecast.",
len(data),
self.MINIMUM,
)
return []
dates, costs = zip(*data)
X = self._enumerate_dates(dates)
Y = [float(c) for c in costs]
# calculate x-values for the prediction range
pred_x = [i for i in range(X[-1] + 1, X[-1] + 1 + self.forecast_days_required)]
# run the forecast
results = self._run_forecast(X, Y, to_predict=pred_x)
result_dict = {}
for i, value in enumerate(results.prediction):
# extrapolate confidence intervals to align with prediction.
# this reduces the confidence interval below 95th percentile, but is a better UX.
if i < len(results.confidence_lower):
lower = results.confidence_lower[i]
else:
lower = results.confidence_lower[-1] + results.slope * (i - len(results.confidence_lower))
if i < len(results.confidence_upper):
upper = results.confidence_upper[i]
else:
upper = results.confidence_upper[-1] + results.slope * (i - len(results.confidence_upper))
# ensure that there are no negative numbers.
result_dict[self.dh.today.date() + timedelta(days=i)] = {
"total_cost": max((value, 0)),
"confidence_min": max((lower, 0)),
"confidence_max": max((upper, 0)),
}
return (result_dict, results.rsquared, results.pvalues)
def _enumerate_dates(self, date_list):
"""Given a list of dates, return a list of integers.
This method works in conjunction with _remove_outliers(). This method works to preserve any gaps
in the data created by _remove_outliers() so that the integers used for the X-axis are aligned
appropriately.
Example:
If _remove_outliers() returns {"2000-01-01": 1.0, "2000-01-03": 1.5}
then _enumerate_dates() returns [0, 2]
"""
days = self.dh.list_days(
datetime.combine(date_list[0], self.dh.midnight), datetime.combine(date_list[-1], self.dh.midnight)
)
out = [i for i, day in enumerate(days) if day.date() in date_list]
return out
def _remove_outliers(self, data):
"""Remove outliers from our dateset before predicting.
We use a box plot method without plotting the box.
"""
values = list(data.values())
if values:
third_quartile, first_quartile = np.percentile(values, [Decimal(75), Decimal(25)])
interquartile_range = third_quartile - first_quartile
upper_boundary = third_quartile + (Decimal(1.5) * interquartile_range)
lower_boundary = first_quartile - (Decimal(1.5) * interquartile_range)
return {key: value for key, value in data.items() if (value >= lower_boundary and value <= upper_boundary)}
return data
def _key_results_by_date(self, results, check_term="total_cost"):
"""Take results formatted by cost type, and return results keyed by date."""
results_by_date = defaultdict(dict)
date_based_dict = results[check_term][0] if results[check_term] else []
for date in date_based_dict:
for cost_term in results:
if results[cost_term][0].get(date):
results_by_date[date][cost_term] = (
results[cost_term][0][date],
{"rsquared": results[cost_term][1]},
{"pvalues": results[cost_term][2]},
)
return results_by_date
def format_result(self, results):
"""Format results for API consumption."""
f_format = f"%.{self.PRECISION}f" # avoid converting floats to e-notation
units = "USD"
response = []
for key in results:
if key > self.dh.this_month_end.date():
continue
dikt = {
"date": key,
"values": [
{
"date": key,
"infrastructure": {
"total": {
"value": round(results[key]["infrastructure_cost"][0]["total_cost"], 3),
"units": units,
},
"confidence_max": {
"value": round(results[key]["infrastructure_cost"][0]["confidence_max"], 3),
"units": units,
},
"confidence_min": {
"value": round(max(results[key]["infrastructure_cost"][0]["confidence_min"], 0), 3),
"units": units,
},
"rsquared": {
"value": f_format % results[key]["infrastructure_cost"][1]["rsquared"],
"units": None,
},
"pvalues": {"value": results[key]["infrastructure_cost"][2]["pvalues"], "units": None},
},
"supplementary": {
"total": {
"value": round(results[key]["supplementary_cost"][0]["total_cost"], 3),
"units": units,
},
"confidence_max": {
"value": round(results[key]["supplementary_cost"][0]["confidence_max"], 3),
"units": units,
},
"confidence_min": {
"value": round(max(results[key]["supplementary_cost"][0]["confidence_min"], 0), 3),
"units": units,
},
"rsquared": {
"value": f_format % results[key]["supplementary_cost"][1]["rsquared"],
"units": None,
},
"pvalues": {"value": results[key]["supplementary_cost"][2]["pvalues"], "units": None},
},
"cost": {
"total": {"value": round(results[key]["total_cost"][0]["total_cost"], 3), "units": units},
"confidence_max": {
"value": round(results[key]["total_cost"][0]["confidence_max"], 3),
"units": units,
},
"confidence_min": {
"value": round(max(results[key]["total_cost"][0]["confidence_min"], 0), 3),
"units": units,
},
"rsquared": {"value": f_format % results[key]["total_cost"][1]["rsquared"], "units": None},
"pvalues": {"value": results[key]["total_cost"][2]["pvalues"], "units": None},
},
}
],
}
response.append(dikt)
return response
def _run_forecast(self, x, y, to_predict=None):
"""Apply the forecast model.
Args:
x (list) a list of exogenous variables
y (list) a list of endogenous variables
to_predict (list) a list of exogenous variables used in the forecast results
Note:
both x and y MUST be the same number of elements
Returns:
(tuple)
(numpy.ndarray) prediction values
(numpy.ndarray) confidence interval lower bound
(numpy.ndarray) confidence interval upper bound
(float) R-squared value
(list) P-values
"""
x = sm.add_constant(x)
to_predict = sm.add_constant(to_predict)
model = sm.OLS(y, x)
results = model.fit()
return LinearForecastResult(results, exog=to_predict)
def _uniquify_qset(self, qset, field="total_cost"):
"""Take a QuerySet list, sum costs within the same day, and arrange it into a list of tuples.
Args:
qset (QuerySet)
field (str) - field name in the QuerySet to be summed
Returns:
[(date, cost), ...]
"""
# FIXME: this QuerySet->dict->list conversion probably isn't ideal.
# FIXME: there's probably a way to aggregate multiple sources by date using just the ORM.
result = defaultdict(Decimal)
for item in qset:
result[item.get("usage_start")] += Decimal(item.get(field, 0.0))
result = self._remove_outliers(result)
out = [(k, v) for k, v in result.items()]
return out
def set_access_filters(self, access, filt, filters):
"""Set access filters to ensure RBAC restrictions adhere to user's access and filters.
Args:
access (list) the list containing the users relevant access
filt (list or dict) contains the filters to be updated
filters (QueryFilterCollection) the filter collection to add the new filters to
returns:
None
"""
if isinstance(filt, list):
for _filt in filt:
_filt["operation"] = "in"
q_filter = QueryFilter(parameter=access, **_filt)
filters.add(q_filter)
else:
filt["operation"] = "in"
q_filter = QueryFilter(parameter=access, **filt)
filters.add(q_filter)
class Forecast:
"""Base forecasting class."""
# the minimum number of data points needed to use the current month's data.
# if we have fewer than this many data points, fall back to using the previous month's data.
#
# this number is chosen in part because statsmodels.stats.stattools.omni_normtest() needs at least eight data
# points to test for normal distribution.
MINIMUM = 8
# the precision of the floats returned in the forecast response.
PRECISION = 8
REPORT_TYPE = "costs"
dh = DateHelper()
def __init__(self, query_params): # noqa: C901
"""Class Constructor.
Instance Attributes:
- cost_summary_table (Model)
- aggregates (dict)
- filters (QueryFilterCollection)
- query_range (tuple)
"""
self.params = query_params
# select appropriate model based on access
access = query_params.get("access", {})
access_key = "default"
self.cost_summary_table = self.provider_map.views.get("costs").get(
access_key)
if access:
access_key = tuple(access.keys())
filter_fields = self.provider_map.provider_map.get("filters")
materialized_view = self.provider_map.views.get("costs").get(
access_key)
if materialized_view:
# We found a matching materialized view, use that
self.cost_summary_table = materialized_view
else:
# We have access constraints, but no view to accomodate, default to daily summary table
self.cost_summary_table = self.provider_map.report_type_map.get(
"tables", {}).get("query")
current_day_of_month = self.dh.today.day
yesterday = (self.dh.today - timedelta(days=1)).day
last_day_of_month = self.dh.this_month_end.day
if current_day_of_month == 1:
self.forecast_days_required = last_day_of_month
else:
self.forecast_days_required = last_day_of_month - yesterday
if current_day_of_month <= self.MINIMUM:
self.query_range = (self.dh.last_month_start,
self.dh.last_month_end)
else:
self.query_range = (self.dh.this_month_start,
self.dh.today - timedelta(days=1))
self.filters = QueryFilterCollection()
self.filters.add(field="usage_start",
operation="gte",
parameter=self.query_range[0])
self.filters.add(field="usage_end",
operation="lte",
parameter=self.query_range[1])
# filter queries based on access
if access_key != "default":
for q_param, filt in filter_fields.items():
access = query_params.get_access(q_param, list())
if access:
self.set_access_filters(access, filt, self.filters)
@property
def provider_map(self):
"""Return the provider map instance."""
return self.provider_map_class(self.provider, self.REPORT_TYPE)
@property
def total_cost_term(self):
"""Return the provider map value for total cost."""
return self.provider_map.report_type_map.get("aggregates",
{}).get("cost_total")
@property
def supplementary_cost_term(self):
"""Return the provider map value for total supplemenatry cost."""
return self.provider_map.report_type_map.get("aggregates",
{}).get("sup_total")
@property
def infrastructure_cost_term(self):
"""Return the provider map value for total inftrastructure cost."""
return self.provider_map.report_type_map.get("aggregates",
{}).get("infra_total")
def predict(self):
"""Define ORM query to run forecast and return prediction."""
cost_predictions = {}
with tenant_context(self.params.tenant):
data = (self.cost_summary_table.objects.filter(
self.filters.compose()).order_by("usage_start").values(
"usage_start").annotate(
total_cost=self.total_cost_term,
supplementary_cost=self.supplementary_cost_term,
infrastructure_cost=self.infrastructure_cost_term,
))
total_cost_data = data.values("usage_start", "total_cost")
infra_cost_data = data.values("usage_start", "infrastructure_cost")
supp_cost_data = data.values("usage_start", "supplementary_cost")
cost_predictions["total_cost"] = self._predict(
self._uniquify_qset(total_cost_data, field="total_cost"))
cost_predictions["infrastructure_cost"] = self._predict(
self._uniquify_qset(infra_cost_data,
field="infrastructure_cost"))
cost_predictions["supplementary_cost"] = self._predict(
self._uniquify_qset(supp_cost_data,
field="supplementary_cost"))
cost_predictions = self._key_results_by_date(cost_predictions)
return self.format_result(cost_predictions)
def _predict(self, data):
"""Handle pre and post prediction work.
Args:
data (list) a list of (datetime, float) tuples
Returns:
(LinearForecastResult) linear forecast results object
"""
LOG.debug("Forecast input data: %s", data)
if len(data) < 2:
LOG.warning(
"Unable to calculate forecast. Insufficient data for %s.",
self.params.tenant)
return []
if len(data) < self.MINIMUM:
LOG.warning("Number of data elements is fewer than the minimum.")
# arrange the data into a form that statsmodels will accept.
dates, costs = zip(*data)
X = [int(d.strftime("%Y%m%d")) for d in dates]
Y = [float(c) for c in costs]
# run the forecast
results = self._run_forecast(X, Y)
result_dict = {}
for i, value in enumerate(results.prediction):
result_dict[self.dh.today.date() + timedelta(days=i)] = {
"total_cost": value,
"confidence_min": results.confidence_lower[i],
"confidence_max": results.confidence_upper[i],
}
result_dict = self._add_additional_data_points(result_dict,
results.slope)
return (result_dict, results.rsquared, results.pvalues)
def _remove_outliers(self, data):
"""Remove outliers from our dateset before predicting.
We use a box plot method without plotting the box.
"""
values = list(data.values())
if values:
third_quartile, first_quartile = np.percentile(
values, [Decimal(75), Decimal(25)])
interquartile_range = third_quartile - first_quartile
upper_boundary = third_quartile + (Decimal(1.5) *
interquartile_range)
lower_boundary = first_quartile - (Decimal(1.5) *
interquartile_range)
return {
key: value
for key, value in data.items()
if (value >= lower_boundary and value <= upper_boundary)
}
return data
def _key_results_by_date(self, results, check_term="total_cost"):
"""Take results formatted by cost type, and return results keyed by date."""
results_by_date = defaultdict(dict)
date_based_dict = results[check_term][0] if results[check_term] else []
for date in date_based_dict:
for cost_term in results:
results_by_date[date][cost_term] = (
results[cost_term][0][date],
{
"rsquared": results[cost_term][1]
},
{
"pvalues": results[cost_term][2]
},
)
return results_by_date
def format_result(self, results):
"""Format results for API consumption."""
f_format = f"%.{self.PRECISION}f" # avoid converting floats to e-notation
units = "USD"
response = []
for key in results:
if key > self.dh.this_month_end.date():
continue
dikt = {
"date":
key,
"values": [{
"date": key,
"infrastructure": {
"total": {
"value":
round(
results[key]["infrastructure_cost"][0]
["total_cost"], 3),
"units":
units,
},
"confidence_max": {
"value":
round(
results[key]["infrastructure_cost"][0]
["confidence_max"], 3),
"units":
units,
},
"confidence_min": {
"value":
round(
max(
results[key]["infrastructure_cost"][0]
["confidence_min"], 0), 3),
"units":
units,
},
"rsquared": {
"value":
f_format %
results[key]["infrastructure_cost"][1]["rsquared"],
"units":
None,
},
"pvalues": {
"value":
f_format %
results[key]["infrastructure_cost"][2]["pvalues"],
"units":
None,
},
},
"supplementary": {
"total": {
"value":
round(
results[key]["supplementary_cost"][0]
["total_cost"], 3),
"units":
units,
},
"confidence_max": {
"value":
round(
results[key]["supplementary_cost"][0]
["confidence_max"], 3),
"units":
units,
},
"confidence_min": {
"value":
round(
max(
results[key]["supplementary_cost"][0]
["confidence_min"], 0), 3),
"units":
units,
},
"rsquared": {
"value":
f_format %
results[key]["supplementary_cost"][1]["rsquared"],
"units":
None,
},
"pvalues": {
"value":
f_format %
results[key]["supplementary_cost"][2]["pvalues"],
"units":
None,
},
},
"cost": {
"total": {
"value":
round(results[key]["total_cost"][0]["total_cost"],
3),
"units":
units
},
"confidence_max": {
"value":
round(
results[key]["total_cost"][0]
["confidence_max"], 3),
"units":
units,
},
"confidence_min": {
"value":
round(
max(
results[key]["total_cost"][0]
["confidence_min"], 0), 3),
"units":
units,
},
"rsquared": {
"value":
f_format %
results[key]["total_cost"][1]["rsquared"],
"units":
None
},
"pvalues": {
"value":
f_format %
results[key]["total_cost"][2]["pvalues"],
"units":
None
},
},
}],
}
response.append(dikt)
return response
def _add_additional_data_points(self, results, slope):
"""Add extra entries to make sure we predict the full month."""
additional_days_needed = 0
dates = results.keys()
last_predicted_date = max(dates)
days_already_predicted = len(dates)
last_predicted_cost = results[last_predicted_date]["total_cost"]
last_predicted_max = results[last_predicted_date]["confidence_max"]
last_predicted_min = results[last_predicted_date]["confidence_min"]
if days_already_predicted < self.forecast_days_required:
additional_days_needed = self.forecast_days_required - days_already_predicted
for i in range(1, additional_days_needed + 1):
results[last_predicted_date + timedelta(days=i)] = {
"total_cost": last_predicted_cost + (slope * i),
"confidence_min": last_predicted_min + (slope * i),
"confidence_max": last_predicted_max + (slope * i),
}
return results
def _run_forecast(self, x, y):
"""Apply the forecast model.
Args:
x (list) a list of exogenous variables
y (list) a list of endogenous variables
Note:
both x and y MUST be the same number of elements
Returns:
(tuple)
(numpy.ndarray) prediction values
(numpy.ndarray) confidence interval lower bound
(numpy.ndarray) confidence interval upper bound
(float) R-squared value
(list) P-values
"""
model = sm.OLS(y, x)
results = model.fit()
return LinearForecastResult(results)
def _uniquify_qset(self, qset, field="total_cost"):
"""Take a QuerySet list, sum costs within the same day, and arrange it into a list of tuples.
Args:
qset (QuerySet)
Returns:
[(date, cost), ...]
"""
# FIXME: this QuerySet->dict->list conversion probably isn't ideal.
# FIXME: there's probably a way to aggregate multiple sources by date using just the ORM.
result = defaultdict(Decimal)
for item in qset:
result[item.get("usage_start")] += Decimal(item.get(field, 0.0))
result = self._remove_outliers(result)
out = [(k, v) for k, v in result.items()]
return out
def set_access_filters(self, access, filt, filters):
"""Set access filters to ensure RBAC restrictions adhere to user's access and filters.
Args:
access (list) the list containing the users relevant access
filt (list or dict) contains the filters to be updated
filters (QueryFilterCollection) the filter collection to add the new filters to
returns:
None
"""
if isinstance(filt, list):
for _filt in filt:
_filt["operation"] = "in"
q_filter = QueryFilter(parameter=access, **_filt)
filters.add(q_filter)
else:
filt["operation"] = "in"
q_filter = QueryFilter(parameter=access, **filt)
filters.add(q_filter)
def _get_key_filter(self):
"""Add new `exact` QueryFilter that filters on the key name."""
filters = QueryFilterCollection()
filters.add(
QueryFilter(field="key", operation="exact", parameter=self.key))
return self.query_filter & filters.compose()