def test_predict_response_date(self):
"""Test that predict() returns expected date range."""
dh = DateHelper()
expected = []
for n in range(0, 10):
expected.append({
"usage_start":
dh.n_days_ago(dh.today, 10 - n).date(),
"total_cost":
5,
"infrastructure_cost":
3,
"supplementary_cost":
2,
})
mock_qset = MockQuerySet(expected)
mocked_table = Mock()
mocked_table.objects.filter.return_value.order_by.return_value.values.return_value.annotate.return_value = ( # noqa: E501
mock_qset)
mocked_table.len = mock_qset.len
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
instance.cost_summary_table = mocked_table
results = instance.predict()
for item in results:
self.assertIsInstance(item.get("date"), date)
self.assertLessEqual(item.get("date"), dh.this_month_end.date())
def test_forecast_days_required(self):
"""Test that we accurately select the number of days."""
dh = DateHelper()
params = self.mocked_query_params("?", AWSCostForecastView)
with patch("forecast.forecast.Forecast.dh") as mock_dh:
mock_dh.today = dh.this_month_start
mock_dh.this_month_start = dh.this_month_start
mock_dh.this_month_end = dh.this_month_end
mock_dh.last_month_start = dh.last_month_start
mock_dh.last_month_end = dh.last_month_end
forecast = AWSForecast(params)
self.assertEqual(forecast.forecast_days_required,
dh.this_month_end.day)
with patch("forecast.forecast.Forecast.dh") as mock_dh:
fake_yesterday = dh.this_month_start
fake_today = dh.this_month_start + timedelta(days=1)
mock_dh.today = fake_today
mock_dh.this_month_start = dh.this_month_start
mock_dh.this_month_end = dh.this_month_end
mock_dh.last_month_start = dh.last_month_start
mock_dh.last_month_end = dh.last_month_end
forecast = AWSForecast(params)
self.assertEqual(forecast.forecast_days_required,
dh.this_month_end.day - fake_yesterday.day)
def test_set_access_filter_with_list(self):
"""
Tests that when an access restriction, filters, and a filter list are passed in,
the correct query filters are added
"""
# create the elements needed to mock the query handler
params = self.mocked_query_params("", AWSCostForecastView)
instance = AWSForecast(params)
# set filters and access to be used in function
filters = QueryFilterCollection()
access = ["589173575009"]
filt = [
{
"field": "account_alias__account_alias",
"operation": "icontains",
"composition_key": "account_filter"
},
{
"field": "usage_account_id",
"operation": "icontains",
"composition_key": "account_filter"
},
]
expected = QueryFilterCollection(filters=[
QueryFilter(field="account_alias__account_alias",
operation="in",
composition_key="account_filter"),
QueryFilter(field="usage_account_id",
operation="in",
composition_key="account_filter"),
])
instance.set_access_filters(access, filt, filters)
self.assertIsInstance(filters, QueryFilterCollection)
assertSameQ(filters.compose(), expected.compose())
def test_query_range(self):
"""Test that we select the correct range based on day of month."""
dh = DateHelper()
params = self.mocked_query_params("?", AWSCostForecastView)
with patch("forecast.forecast.Forecast.dh") as mock_dh:
mock_dh.today = dh.this_month_start + timedelta(
days=AWSForecast.MINIMUM - 1)
mock_dh.this_month_start = dh.this_month_start
mock_dh.this_month_end = dh.this_month_end
mock_dh.last_month_start = dh.last_month_start
mock_dh.last_month_end = dh.last_month_end
expected = (dh.last_month_start, dh.last_month_end)
forecast = AWSForecast(params)
self.assertEqual(forecast.query_range, expected)
with patch("forecast.forecast.Forecast.dh") as mock_dh:
mock_dh.today = dh.this_month_start + timedelta(
days=(AWSForecast.MINIMUM))
mock_dh.this_month_start = dh.this_month_start
mock_dh.this_month_end = dh.this_month_end
mock_dh.last_month_start = dh.last_month_start
mock_dh.last_month_end = dh.last_month_end
expected = (dh.this_month_start, dh.this_month_start +
timedelta(days=AWSForecast.MINIMUM - 1))
forecast = AWSForecast(params)
self.assertEqual(forecast.query_range, expected)
def test_predict_end_of_month(self):
"""COST-1091: Test that predict() returns empty list on the last day of a month."""
scenario = [(date(2000, 1, 31), 1.5)]
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
out = instance._predict(scenario)
self.assertEqual(out, [])
def test_predict_flat(self):
"""Test that predict() returns expected values for flat costs."""
dh = DateHelper()
expected = []
for n in range(0, 10):
expected.append({
"usage_start":
(dh.this_month_start + timedelta(days=n)).date(),
"total_cost":
5 + (0.01 * n),
"infrastructure_cost":
3 + (0.01 * n),
"supplementary_cost":
2 + (0.01 * n),
})
mock_qset = MockQuerySet(expected)
mocked_table = Mock()
mocked_table.objects.filter.return_value.order_by.return_value.values.return_value.annotate.return_value = ( # noqa: E501
mock_qset)
mocked_table.len = mock_qset.len
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
instance.cost_summary_table = mocked_table
results = instance.predict()
for result in results:
for val in result.get("values", []):
with self.subTest(values=val):
self.assertIsInstance(val.get("date"), date)
for item, cost, delta in [
(val.get("cost"), 5, 1),
(val.get("infrastructure"), 3, 1),
(val.get("supplementary"), 2, 1),
]:
with self.subTest(cost=cost, delta=delta, item=item):
self.assertAlmostEqual(float(
item.get("total").get("value")),
cost,
delta=delta)
self.assertAlmostEqual(float(
item.get("confidence_max").get("value")),
cost,
delta=delta)
self.assertAlmostEqual(float(
item.get("confidence_min").get("value")),
cost,
delta=delta)
self.assertGreater(
float(item.get("rsquared").get("value")), 0)
for pval in item.get("pvalues").get("value"):
self.assertGreaterEqual(float(pval), 0)
def test_predict_flat(self):
"""Test that predict() returns expected values for flat costs."""
dh = DateHelper()
expected = []
for n in range(0, 10):
expected.append({
"usage_start":
dh.n_days_ago(dh.today, 10 - n).date(),
"total_cost":
5,
"infrastructure_cost":
3,
"supplementary_cost":
2,
})
mock_qset = MockQuerySet(expected)
mocked_table = Mock()
mocked_table.objects.filter.return_value.order_by.return_value.values.return_value.annotate.return_value = ( # noqa: E501
mock_qset)
mocked_table.len = mock_qset.len
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
instance.cost_summary_table = mocked_table
results = instance.predict()
for result in results:
for val in result.get("values", []):
self.assertIsInstance(val.get("date"), date)
for item, cost in [
(val.get("cost"), 5),
(val.get("infrastructure"), 3),
(val.get("supplementary"), 2),
]:
self.assertAlmostEqual(float(
item.get("total").get("value")),
cost,
delta=0.0001)
self.assertAlmostEqual(float(
item.get("confidence_max").get("value")),
cost,
delta=0.0001)
self.assertAlmostEqual(float(
item.get("confidence_min").get("value")),
cost,
delta=0.0001)
self.assertAlmostEqual(float(
item.get("rsquared").get("value")),
1,
delta=0.0001)
self.assertGreaterEqual(
float(item.get("pvalues").get("value")), 0)
def test_results_never_outside_curren_month(self):
"""Test that our results stop at the end of the current month."""
dh = DateHelper()
params = self.mocked_query_params("?", AWSCostForecastView)
forecast = AWSForecast(params)
forecast.forecast_days_required = 100
results = forecast.predict()
dates = [result.get("date") for result in results]
self.assertNotIn(dh.next_month_start, dates)
self.assertEqual(dh.this_month_end.date(), max(dates))
def test_predict_few_values(self):
"""Test that predict() behaves well with a limited data set."""
dh = DateHelper()
num_elements = [AWSForecast.MINIMUM - 1, AWSForecast.MINIMUM, AWSForecast.MINIMUM + 1]
for number in num_elements:
with self.subTest(num_elements=number):
expected = []
for n in range(0, number):
# the test data needs to include some jitter to avoid
# division-by-zero in the underlying dot-product maths.
expected.append(
{
"usage_start": dh.n_days_ago(dh.today, 10 - n).date(),
"total_cost": 5 + (0.01 * n),
"infrastructure_cost": 3 + (0.01 * n),
"supplementary_cost": 2 + (0.01 * n),
}
)
mock_qset = MockQuerySet(expected)
mocked_table = Mock()
mocked_table.objects.filter.return_value.order_by.return_value.values.return_value.annotate.return_value = ( # noqa: E501
mock_qset
)
mocked_table.len = mock_qset.len
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
instance.cost_summary_table = mocked_table
if number < AWSForecast.MINIMUM:
# forecasting isn't useful with less than the minimum number of data points.
with self.assertLogs(logger="forecast.forecast", level=logging.WARNING):
results = instance.predict()
self.assertEqual(results, [])
else:
results = instance.predict()
self.assertNotEqual(results, [])
for result in results:
for val in result.get("values", []):
self.assertIsInstance(val.get("date"), date)
item = val.get("cost")
self.assertGreaterEqual(float(item.get("total").get("value")), 0)
self.assertGreaterEqual(float(item.get("confidence_max").get("value")), 0)
self.assertGreaterEqual(float(item.get("confidence_min").get("value")), 0)
self.assertGreaterEqual(float(item.get("rsquared").get("value")), 0)
for pval in item.get("pvalues").get("value"):
self.assertGreaterEqual(float(pval), 0)
# test that the results always stop at the end of the month.
self.assertEqual(results[-1].get("date"), dh.this_month_end.date())
def test_remove_outliers(self):
"""Test that we remove outliers before predicting."""
params = self.mocked_query_params("?", AWSCostForecastView)
dh = DateHelper()
days_in_month = dh.this_month_end.day
data = {}
for i in range(days_in_month):
data[dh.this_month_start + timedelta(days=i)] = Decimal(20)
outlier = Decimal(100)
data[dh.this_month_start] = outlier
forecast = AWSForecast(params)
result = forecast._remove_outliers(data)
self.assertNotIn(dh.this_month_start, result.keys())
self.assertNotIn(outlier, result.values())
def test_negative_values(self, mock_enumerate_dates, mock_run_forecast, mock_format_result):
"""COST-1110: ensure that the forecast response does not include negative numbers."""
mock_run_forecast.return_value = Mock(
prediction=[1, 0, -1, -2, -3], confidence_lower=[2, 1, 0, -1, -2], confidence_upper=[3, 2, 1, 0, -1]
)
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
instance.predict()
self.assertIsInstance(mock_format_result.call_args[0][0], dict)
for key, val_dict in mock_format_result.call_args[0][0].items():
for inner_key, inner_val in val_dict.items():
if "cost" in inner_key:
self.assertGreaterEqual(inner_val[0]["total_cost"], 0)
self.assertGreaterEqual(inner_val[0]["confidence_min"], 0)
self.assertGreaterEqual(inner_val[0]["confidence_max"], 0)
def test_enumerate_dates(self):
"""Test that the _enumerate_dates() method gives expected results."""
test_scenarios = [
{"dates": [date(2000, 1, 1), date(2000, 1, 2), date(2000, 1, 3)], "expected": [0, 1, 2]},
{"dates": [date(2000, 1, 1), date(2000, 1, 3), date(2000, 1, 5)], "expected": [0, 2, 4]},
{"dates": [date(2000, 1, 1), date(2000, 1, 2), date(2000, 1, 5)], "expected": [0, 1, 4]},
{"dates": [date(2000, 1, 1), date(2000, 1, 4), date(2000, 1, 5)], "expected": [0, 3, 4]},
]
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
for scenario in test_scenarios:
with self.subTest(dates=scenario["dates"], expected=scenario["expected"]):
out = instance._enumerate_dates(scenario["dates"])
self.assertEqual(out, scenario["expected"])
def test_predict_increasing(self):
"""Test that predict() returns expected values for increasing costs."""
dh = DateHelper()
expected = []
for n in range(0, 10):
# the test data needs to include some jitter to avoid
# division-by-zero in the underlying dot-product maths.
expected.append({
"usage_start":
dh.n_days_ago(dh.today, 10 - n).date(),
"total_cost":
5 + random.random(),
"infrastructure_cost":
3 + random.random(),
"supplementary_cost":
2 + random.random(),
})
mock_qset = MockQuerySet(expected)
mocked_table = Mock()
mocked_table.objects.filter.return_value.order_by.return_value.values.return_value.annotate.return_value = ( # noqa: E501
mock_qset)
mocked_table.len = mock_qset.len
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
instance.cost_summary_table = mocked_table
results = instance.predict()
for result in results:
for val in result.get("values", []):
self.assertIsInstance(val.get("date"), date)
item = val.get("cost")
self.assertGreaterEqual(float(item.get("total").get("value")),
0)
self.assertGreaterEqual(
float(item.get("confidence_max").get("value")), 0)
self.assertGreaterEqual(
float(item.get("confidence_min").get("value")), 0)
self.assertGreaterEqual(
float(item.get("rsquared").get("value")), 0)
for pval in item.get("pvalues").get("value"):
self.assertGreaterEqual(float(pval), 0)
def test_summary_table(self):
"""COST-908: Test that the expected summary table is used."""
mock_access = {"aws.organizational_unit": {"read": ["1234", "5678"]}}
params = self.mocked_query_params("?",
AWSCostForecastView,
access=mock_access)
instance = AWSForecast(params)
self.assertEqual(instance.cost_summary_table,
AWSCostEntryLineItemDailySummary)
def test_add_additional_data_points(self):
"""Test that we fill in data to the end of the month."""
dh = DateHelper()
params = self.mocked_query_params("?", AWSCostForecastView)
last_day_of_data = dh.last_month_start + timedelta(days=10)
with patch("forecast.forecast.Forecast.dh") as mock_dh:
mock_dh.today = dh.this_month_start
mock_dh.this_month_end = dh.this_month_end
mock_dh.last_month_start = dh.last_month_start
mock_dh.last_month_end = last_day_of_data
forecast = AWSForecast(params)
results = forecast.predict()
self.assertEqual(len(results), dh.this_month_end.day)
for i, result in enumerate(results):
self.assertEqual(
result.get("date"),
dh.this_month_start.date() + timedelta(days=i))
for val in result.get("values", []):
cost = val.get("cost", {}).get("total", {}).get("value")
self.assertNotEqual(cost, 0)
def test_query_range(self):
"""Test that we select the correct range based on day of month."""
params = self.mocked_query_params("?", AWSCostForecastView)
dh = DateHelper()
mock_dh = Mock(spec=DateHelper)
scenarios = [
{
"today": dh.today,
"yesterday": dh.yesterday,
"this_month_end": dh.this_month_end,
"expected": (dh.yesterday + timedelta(days=-30), dh.yesterday),
},
{
"today":
datetime(2000, 1, 1, 0, 0, 0, 0),
"yesterday":
datetime(1999, 12, 31, 0, 0, 0, 0),
"this_month_end":
datetime(2000, 1, 31, 0, 0, 0, 0),
"expected": (
datetime(1999, 12, 31, 0, 0, 0, 0) + timedelta(days=-30),
datetime(1999, 12, 31, 0, 0, 0, 0),
),
},
{
"today":
datetime(2000, 1, 31, 0, 0, 0, 0),
"yesterday":
datetime(2000, 1, 30, 0, 0, 0, 0),
"this_month_end":
datetime(2000, 1, 31, 0, 0, 0, 0),
"expected": (
datetime(2000, 1, 30, 0, 0, 0, 0) + timedelta(days=-30),
datetime(2000, 1, 30, 0, 0, 0, 0),
),
},
]
mock_dh.return_value.n_days_ago = dh.n_days_ago # pass-thru to real function
for test in scenarios:
with self.subTest(scenario=test):
mock_dh.return_value.today = test["today"]
mock_dh.return_value.yesterday = test["yesterday"]
mock_dh.return_value.this_month_end = test["this_month_end"]
with patch("forecast.forecast.DateHelper",
new_callable=lambda: mock_dh) as mock_dh:
forecast = AWSForecast(params)
self.assertEqual(forecast.query_range, test["expected"])
def test_forecast_days_required(self):
"""Test that we accurately select the number of days."""
params = self.mocked_query_params("?", AWSCostForecastView)
dh = DateHelper()
mock_dh = Mock(spec=DateHelper)
scenarios = [
{
"today": dh.today,
"yesterday": dh.yesterday,
"this_month_end": dh.this_month_end,
"expected": max((dh.this_month_end - dh.yesterday).days, 2),
},
{
"today": datetime(2000, 1, 1, 0, 0, 0, 0),
"yesterday": datetime(1999, 12, 31, 0, 0, 0, 0),
"this_month_end": datetime(2000, 1, 31, 0, 0, 0, 0),
"expected": 31,
},
{
"today": datetime(2000, 1, 31, 0, 0, 0, 0),
"yesterday": datetime(2000, 1, 30, 0, 0, 0, 0),
"this_month_end": datetime(2000, 1, 31, 0, 0, 0, 0),
"expected": 2,
},
]
mock_dh.return_value.n_days_ago = dh.n_days_ago # pass-thru to real function
for test in scenarios:
with self.subTest(scenario=test):
mock_dh.return_value.today = test["today"]
mock_dh.return_value.yesterday = test["yesterday"]
mock_dh.return_value.this_month_end = test["this_month_end"]
with patch("forecast.forecast.DateHelper",
new_callable=lambda: mock_dh) as mock_dh:
forecast = AWSForecast(params)
self.assertEqual(forecast.forecast_days_required,
test["expected"])
def test_constructor(self):
"""Test the constructor."""
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
self.assertIsInstance(instance, AWSForecast)
def test_predict_few_values(self):
"""Test that predict() behaves well with a limited data set."""
dh = DateHelper()
num_elements = [1, 2, 3, 4, 5]
for number in num_elements:
with self.subTest(num_elements=number):
expected = []
for n in range(0, number):
expected.append({
"usage_start":
dh.n_days_ago(dh.today, 10 - n).date(),
"total_cost":
5,
"infrastructure_cost":
3,
"supplementary_cost":
2,
})
mock_qset = MockQuerySet(expected)
mocked_table = Mock()
mocked_table.objects.filter.return_value.order_by.return_value.values.return_value.annotate.return_value = ( # noqa: E501
mock_qset)
mocked_table.len = mock_qset.len
params = self.mocked_query_params("?", AWSCostForecastView)
instance = AWSForecast(params)
instance.cost_summary_table = mocked_table
if number == 1:
# forecasting isn't possible with only 1 data point.
with self.assertLogs(logger="forecast.forecast",
level=logging.WARNING):
results = instance.predict()
self.assertEqual(results, [])
else:
with self.assertLogs(logger="forecast.forecast",
level=logging.WARNING):
results = instance.predict()
for result in results:
for val in result.get("values", []):
self.assertIsInstance(val.get("date"), date)
item = val.get("cost")
self.assertGreaterEqual(
float(item.get("total").get("value")), 0)
self.assertGreaterEqual(
float(
item.get("confidence_max").get(
"value")), 0)
self.assertGreaterEqual(
float(
item.get("confidence_min").get(
"value")), 0)
self.assertGreaterEqual(
float(item.get("rsquared").get("value")),
0)
self.assertGreaterEqual(
float(item.get("pvalues").get("value")), 0)
# test that the results always stop at the end of the month.
self.assertEqual(results[-1].get("date"),
dh.this_month_end.date())