class TestUpdateSummaryTablesTask(MasuTestCase):
"""Test cases for Processor summary table Celery tasks."""
@classmethod
def setUpClass(cls):
"""Set up for the class."""
super().setUpClass()
cls.aws_tables = list(AWS_CUR_TABLE_MAP.values())
cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + list(
OCP_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.schema)
def setUp(self):
"""Set up each test."""
super().setUp()
self.aws_accessor = AWSReportDBAccessor(schema=self.schema)
self.ocp_accessor = OCPReportDBAccessor(schema=self.schema)
# Populate some line item data so that the summary tables
# have something to pull from
self.start_date = DateHelper().today.replace(day=1)
@patch("masu.processor.tasks.chain")
@patch("masu.processor.tasks.refresh_materialized_views")
@patch("masu.processor.tasks.update_cost_model_costs")
def test_update_summary_tables_aws(self, mock_charge_info, mock_views,
mock_chain):
"""Test that the summary table task runs."""
provider = Provider.PROVIDER_AWS
provider_aws_uuid = self.aws_provider_uuid
daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"]
summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"]
start_date = self.start_date.replace(
day=1) + relativedelta.relativedelta(months=-1)
with schema_context(self.schema):
daily_query = self.aws_accessor._get_db_obj_query(daily_table_name)
summary_query = self.aws_accessor._get_db_obj_query(
summary_table_name)
daily_query.delete()
summary_query.delete()
initial_daily_count = daily_query.count()
initial_summary_count = summary_query.count()
self.assertEqual(initial_daily_count, 0)
self.assertEqual(initial_summary_count, 0)
update_summary_tables(self.schema, provider, provider_aws_uuid,
start_date)
with schema_context(self.schema):
self.assertNotEqual(daily_query.count(), initial_daily_count)
self.assertNotEqual(summary_query.count(), initial_summary_count)
mock_chain.return_value.apply_async.assert_called()
@patch("masu.processor.tasks.update_cost_model_costs")
def test_update_summary_tables_aws_end_date(self, mock_charge_info):
"""Test that the summary table task respects a date range."""
provider = Provider.PROVIDER_AWS_LOCAL
provider_aws_uuid = self.aws_provider_uuid
ce_table_name = AWS_CUR_TABLE_MAP["cost_entry"]
daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"]
summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"]
start_date = DateHelper().last_month_start
end_date = DateHelper().last_month_end
daily_table = getattr(self.aws_accessor.report_schema,
daily_table_name)
summary_table = getattr(self.aws_accessor.report_schema,
summary_table_name)
ce_table = getattr(self.aws_accessor.report_schema, ce_table_name)
with schema_context(self.schema):
daily_table.objects.all().delete()
summary_table.objects.all().delete()
ce_start_date = ce_table.objects.filter(
interval_start__gte=start_date.date()).aggregate(
Min("interval_start"))["interval_start__min"]
ce_end_date = ce_table.objects.filter(
interval_start__lte=end_date.date()).aggregate(
Max("interval_start"))["interval_start__max"]
# The summary tables will only include dates where there is data
expected_start_date = max(start_date, ce_start_date)
expected_start_date = expected_start_date.replace(hour=0,
minute=0,
second=0,
microsecond=0)
expected_end_date = min(end_date, ce_end_date)
expected_end_date = expected_end_date.replace(hour=0,
minute=0,
second=0,
microsecond=0)
update_summary_tables(self.schema, provider, provider_aws_uuid,
start_date, end_date)
with schema_context(self.schema):
daily_entry = daily_table.objects.all().aggregate(
Min("usage_start"), Max("usage_end"))
result_start_date = daily_entry["usage_start__min"]
result_end_date = daily_entry["usage_end__max"]
self.assertEqual(result_start_date, expected_start_date.date())
self.assertEqual(result_end_date, expected_end_date.date())
with schema_context(self.schema):
summary_entry = summary_table.objects.all().aggregate(
Min("usage_start"), Max("usage_end"))
result_start_date = summary_entry["usage_start__min"]
result_end_date = summary_entry["usage_end__max"]
self.assertEqual(result_start_date, expected_start_date.date())
self.assertEqual(result_end_date, expected_end_date.date())
@patch("masu.processor.tasks.chain")
@patch("masu.processor.tasks.refresh_materialized_views")
@patch("masu.processor.tasks.update_cost_model_costs")
@patch("masu.processor.ocp.ocp_cost_model_cost_updater.CostModelDBAccessor"
)
def test_update_summary_tables_ocp(self, mock_cost_model, mock_charge_info,
mock_view, mock_chain):
"""Test that the summary table task runs."""
infrastructure_rates = {
"cpu_core_usage_per_hour": 1.5,
"memory_gb_usage_per_hour": 2.5,
"storage_gb_usage_per_month": 0.5,
}
markup = {}
mock_cost_model.return_value.__enter__.return_value.infrastructure_rates = infrastructure_rates
mock_cost_model.return_value.__enter__.return_value.supplementary_rates = {}
mock_cost_model.return_value.__enter__.return_value.markup = markup
provider = Provider.PROVIDER_OCP
provider_ocp_uuid = self.ocp_test_provider_uuid
daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"]
start_date = DateHelper().last_month_start
end_date = DateHelper().last_month_end
with schema_context(self.schema):
daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name)
daily_query.delete()
initial_daily_count = daily_query.count()
self.assertEqual(initial_daily_count, 0)
update_summary_tables(self.schema, provider, provider_ocp_uuid,
start_date, end_date)
with schema_context(self.schema):
self.assertNotEqual(daily_query.count(), initial_daily_count)
update_cost_model_costs(schema_name=self.schema,
provider_uuid=provider_ocp_uuid,
start_date=start_date,
end_date=end_date)
table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"]
with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor:
provider_obj = provider_accessor.get_provider()
usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(
provider_obj.uuid)
with schema_context(self.schema):
cluster_id = usage_period_qry.first().cluster_id
items = self.ocp_accessor._get_db_obj_query(table_name).filter(
usage_start__gte=start_date,
usage_start__lte=end_date,
cluster_id=cluster_id,
data_source="Pod")
for item in items:
self.assertNotEqual(item.infrastructure_usage_cost.get("cpu"),
0)
self.assertNotEqual(
item.infrastructure_usage_cost.get("memory"), 0)
storage_daily_name = OCP_REPORT_TABLE_MAP[
"storage_line_item_daily"]
items = self.ocp_accessor._get_db_obj_query(
storage_daily_name).filter(cluster_id=cluster_id)
for item in items:
self.assertIsNotNone(item.volume_request_storage_byte_seconds)
self.assertIsNotNone(
item.persistentvolumeclaim_usage_byte_seconds)
storage_summary_name = OCP_REPORT_TABLE_MAP[
"line_item_daily_summary"]
items = self.ocp_accessor._get_db_obj_query(
storage_summary_name).filter(cluster_id=cluster_id,
data_source="Storage")
for item in items:
self.assertIsNotNone(
item.volume_request_storage_gigabyte_months)
self.assertIsNotNone(
item.persistentvolumeclaim_usage_gigabyte_months)
mock_chain.return_value.apply_async.assert_called()
@patch("masu.processor.tasks.update_cost_model_costs")
@patch(
"masu.database.cost_model_db_accessor.CostModelDBAccessor.get_memory_gb_usage_per_hour_rates"
)
@patch(
"masu.database.cost_model_db_accessor.CostModelDBAccessor.get_cpu_core_usage_per_hour_rates"
)
def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate,
mock_mem_rate,
mock_charge_info):
"""Test that the summary table task respects a date range."""
mock_cpu_rate.return_value = 1.5
mock_mem_rate.return_value = 2.5
provider = Provider.PROVIDER_OCP
provider_ocp_uuid = self.ocp_test_provider_uuid
ce_table_name = OCP_REPORT_TABLE_MAP["report"]
daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"]
start_date = DateHelper().last_month_start
end_date = DateHelper().last_month_end
daily_table = getattr(self.ocp_accessor.report_schema,
daily_table_name)
ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name)
with schema_context(self.schema):
daily_table.objects.all().delete()
ce_start_date = ce_table.objects.filter(
interval_start__gte=start_date.date()).aggregate(
Min("interval_start"))["interval_start__min"]
ce_end_date = ce_table.objects.filter(
interval_start__lte=end_date.date()).aggregate(
Max("interval_start"))["interval_start__max"]
# The summary tables will only include dates where there is data
expected_start_date = max(start_date, ce_start_date)
expected_end_date = min(end_date, ce_end_date)
update_summary_tables(self.schema, provider, provider_ocp_uuid,
start_date, end_date)
with schema_context(self.schema):
daily_entry = daily_table.objects.all().aggregate(
Min("usage_start"), Max("usage_end"))
result_start_date = daily_entry["usage_start__min"]
result_end_date = daily_entry["usage_end__max"]
self.assertEqual(result_start_date, expected_start_date.date())
self.assertEqual(result_end_date, expected_end_date.date())
@patch("masu.processor.tasks.update_summary_tables")
def test_get_report_data_for_all_providers(self, mock_update):
"""Test GET report_data endpoint with provider_uuid=*."""
start_date = date.today()
update_all_summary_tables(start_date)
mock_update.delay.assert_called_with(ANY, ANY, ANY, str(start_date),
ANY)
def test_refresh_materialized_views(self):
"""Test that materialized views are refreshed."""
manifest_dict = {
"assembly_id": "12345",
"billing_period_start_datetime": DateHelper().today,
"num_total_files": 2,
"provider_uuid": self.aws_provider_uuid,
"task": "170653c0-3e66-4b7e-a764-336496d7ca5a",
}
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.add(**manifest_dict)
manifest.save()
refresh_materialized_views(self.schema,
Provider.PROVIDER_AWS,
manifest_id=manifest.id)
views_to_check = [
view for view in AWS_MATERIALIZED_VIEWS
if "Cost" in view._meta.db_table
]
with schema_context(self.schema):
for view in views_to_check:
self.assertNotEqual(view.objects.count(), 0)
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest_by_id(manifest.id)
self.assertIsNotNone(manifest.manifest_completed_datetime)
@patch("masu.processor.tasks.connection")
def test_vacuum_schema(self, mock_conn):
"""Test that the vacuum schema task runs."""
logging.disable(logging.NOTSET)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("table", )
]
expected = "INFO:masu.processor.tasks:VACUUM ANALYZE acct10001.table"
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
vacuum_schema(self.schema)
self.assertIn(expected, logger.output)
@patch("masu.processor.tasks.connection")
def test_autovacuum_tune_schema_default_table(self, mock_conn):
"""Test that the autovacuum tuning runs."""
logging.disable(logging.NOTSET)
# Make sure that the AUTOVACUUM_TUNING environment variable is unset!
if "AUTOVACUUM_TUNING" in os.environ:
del os.environ["AUTOVACUUM_TUNING"]
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 20000000, {})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.01);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 2000000, {})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.02);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 200000, {})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.05);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 200000, {
"autovacuum_vacuum_scale_factor": Decimal("0.05")
})
]
expected = "INFO:masu.processor.tasks:Altered autovacuum_vacuum_scale_factor on 0 tables"
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 20000, {
"autovacuum_vacuum_scale_factor": Decimal("0.02")
})
]
expected = "INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model reset (autovacuum_vacuum_scale_factor);"
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
@patch("masu.processor.tasks.connection")
def test_autovacuum_tune_schema_custom_table(self, mock_conn):
"""Test that the autovacuum tuning runs."""
logging.disable(logging.NOTSET)
scale_table = [(10000000, "0.0001"), (1000000, "0.004"),
(100000, "0.011")]
os.environ["AUTOVACUUM_TUNING"] = json.dumps(scale_table)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 20000000, {})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.0001);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 2000000, {})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.004);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 200000, {})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.011);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 200000, {
"autovacuum_vacuum_scale_factor": Decimal("0.011")
})
]
expected = "INFO:masu.processor.tasks:Altered autovacuum_vacuum_scale_factor on 0 tables"
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 20000, {
"autovacuum_vacuum_scale_factor": Decimal("0.004")
})
]
expected = "INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model reset (autovacuum_vacuum_scale_factor);"
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
del os.environ["AUTOVACUUM_TUNING"]
@patch("masu.processor.tasks.connection")
def test_autovacuum_tune_schema_manual_setting(self, mock_conn):
"""Test that the autovacuum tuning runs."""
logging.disable(logging.NOTSET)
# Make sure that the AUTOVACUUM_TUNING environment variable is unset!
if "AUTOVACUUM_TUNING" in os.environ:
del os.environ["AUTOVACUUM_TUNING"]
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 200000, {
"autovacuum_vacuum_scale_factor": Decimal("0.04")
})
]
expected = "INFO:masu.processor.tasks:Altered autovacuum_vacuum_scale_factor on 0 tables"
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 200000, {
"autovacuum_vacuum_scale_factor": Decimal("0.06")
})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.05);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
@patch("masu.processor.tasks.connection")
def test_autovacuum_tune_schema_invalid_setting(self, mock_conn):
"""Test that the autovacuum tuning runs."""
logging.disable(logging.NOTSET)
# Make sure that the AUTOVACUUM_TUNING environment variable is unset!
if "AUTOVACUUM_TUNING" in os.environ:
del os.environ["AUTOVACUUM_TUNING"]
# This invalid setting should be treated as though there was no setting
mock_conn.cursor.return_value.__enter__.return_value.fetchall.return_value = [
("cost_model", 20000000, {
"autovacuum_vacuum_scale_factor": ""
})
]
expected = (
"INFO:masu.processor.tasks:ALTER TABLE acct10001.cost_model set (autovacuum_vacuum_scale_factor = 0.01);"
)
with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
autovacuum_tune_schema(self.schema)
self.assertIn(expected, logger.output)
def test_autovacuum_tune_schedule(self):
vh = next(
iter(koku_celery.app.conf.beat_schedule["vacuum-schemas"]
["schedule"].hour))
avh = next(
iter(koku_celery.app.conf.beat_schedule["autovacuum-tune-schemas"]
["schedule"].hour))
self.assertTrue(avh == (23 if vh == 0 else (vh - 1)))
def update_summary_tables(schema_name,
provider,
provider_uuid,
start_date,
end_date=None,
manifest_id=None):
"""Populate the summary tables for reporting.
Args:
schema_name (str) The DB schema name.
provider (str) The provider type.
provider_uuid (str) The provider uuid.
report_dict (dict) The report data dict from previous task.
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
worker_stats.REPORT_SUMMARY_ATTEMPTS_COUNTER.labels(
provider_type=provider).inc()
stmt = (f"update_summary_tables called with args:\n"
f" schema_name: {schema_name},\n"
f" provider: {provider},\n"
f" start_date: {start_date},\n"
f" end_date: {end_date},\n"
f" manifest_id: {manifest_id}")
LOG.info(stmt)
updater = ReportSummaryUpdater(schema_name, provider_uuid, manifest_id)
start_date, end_date = updater.update_daily_tables(start_date, end_date)
updater.update_summary_tables(start_date, end_date)
if provider_uuid:
dh = DateHelper(utc=True)
prev_month_last_day = dh.last_month_end
start_date_obj = datetime.datetime.strptime(start_date, "%Y-%m-%d")
prev_month_last_day = prev_month_last_day.replace(tzinfo=None)
prev_month_last_day = prev_month_last_day.replace(microsecond=0,
second=0,
minute=0,
hour=0,
day=1)
if manifest_id and (start_date_obj <= prev_month_last_day):
# We want make sure that the manifest_id is not none, because
# we only want to call the delete line items after the summarize_reports
# task above
simulate = False
line_items_only = True
chain(
update_cost_model_costs.s(schema_name, provider_uuid,
start_date, end_date),
refresh_materialized_views.si(schema_name, provider,
manifest_id),
remove_expired_data.si(schema_name, provider, simulate,
provider_uuid, line_items_only),
).apply_async()
else:
chain(
update_cost_model_costs.s(schema_name, provider_uuid,
start_date, end_date),
refresh_materialized_views.si(schema_name, provider,
manifest_id),
).apply_async()
else:
refresh_materialized_views.delay(schema_name, provider, manifest_id)
def setUpClass(cls):
"""Set up the test class."""
super().setUpClass()
cls.dh = DateHelper()
cls.ten_days_ago = cls.dh.n_days_ago(cls.dh._now, 9)
class Forecast:
"""Base forecasting class."""
# the minimum number of data points needed to use the current month's data.
# if we have fewer than this many data points, fall back to using the previous month's data.
#
# this number is chosen in part because statsmodels.stats.stattools.omni_normtest() needs at least eight data
# points to test for normal distribution.
MINIMUM = 8
# the precision of the floats returned in the forecast response.
PRECISION = 8
REPORT_TYPE = "costs"
def __init__(self, query_params): # noqa: C901
"""Class Constructor.
Instance Attributes:
- cost_summary_table (Model)
- aggregates (dict)
- filters (QueryFilterCollection)
- query_range (tuple)
"""
self.dh = DateHelper()
self.params = query_params
# select appropriate model based on access
access = query_params.get("access", {})
access_key = "default"
self.cost_summary_table = self.provider_map.views.get("costs").get(access_key)
if access:
access_key = tuple(access.keys())
filter_fields = self.provider_map.provider_map.get("filters")
materialized_view = self.provider_map.views.get("costs").get(access_key)
if materialized_view:
# We found a matching materialized view, use that
self.cost_summary_table = materialized_view
else:
# We have access constraints, but no view to accomodate, default to daily summary table
self.cost_summary_table = self.provider_map.query_table
self.forecast_days_required = (self.dh.this_month_end - self.dh.yesterday).days
# forecasts use a rolling window
self.query_range = (self.dh.n_days_ago(self.dh.yesterday, 30), self.dh.yesterday)
self.filters = QueryFilterCollection()
self.filters.add(field="usage_start", operation="gte", parameter=self.query_range[0])
self.filters.add(field="usage_end", operation="lte", parameter=self.query_range[1])
# filter queries based on access
if access_key != "default":
for q_param, filt in filter_fields.items():
access = query_params.get_access(q_param, list())
if access:
self.set_access_filters(access, filt, self.filters)
@property
def provider_map(self):
"""Return the provider map instance."""
return self.provider_map_class(self.provider, self.REPORT_TYPE)
@property
def total_cost_term(self):
"""Return the provider map value for total cost."""
return self.provider_map.report_type_map.get("aggregates", {}).get("cost_total")
@property
def supplementary_cost_term(self):
"""Return the provider map value for total supplemenatry cost."""
return self.provider_map.report_type_map.get("aggregates", {}).get("sup_total")
@property
def infrastructure_cost_term(self):
"""Return the provider map value for total inftrastructure cost."""
return self.provider_map.report_type_map.get("aggregates", {}).get("infra_total")
def predict(self):
"""Define ORM query to run forecast and return prediction."""
cost_predictions = {}
with tenant_context(self.params.tenant):
data = (
self.cost_summary_table.objects.filter(self.filters.compose())
.order_by("usage_start")
.values("usage_start")
.annotate(
total_cost=self.total_cost_term,
supplementary_cost=self.supplementary_cost_term,
infrastructure_cost=self.infrastructure_cost_term,
)
)
for fieldname in ["total_cost", "infrastructure_cost", "supplementary_cost"]:
uniq_data = self._uniquify_qset(data.values("usage_start", fieldname), field=fieldname)
cost_predictions[fieldname] = self._predict(uniq_data)
cost_predictions = self._key_results_by_date(cost_predictions)
return self.format_result(cost_predictions)
def _predict(self, data):
"""Handle pre and post prediction work.
This function handles arranging incoming data to conform with statsmodels requirements.
Then after receiving the forecast output, this function handles formatting to conform to
API reponse requirements.
Args:
data (list) a list of (datetime, float) tuples
Returns:
(LinearForecastResult) linear forecast results object
"""
LOG.debug("Forecast input data: %s", data)
if len(data) < self.MINIMUM:
LOG.warning(
"Number of data elements (%s) is fewer than the minimum (%s). Unable to generate forecast.",
len(data),
self.MINIMUM,
)
return []
dates, costs = zip(*data)
X = self._enumerate_dates(dates)
Y = [float(c) for c in costs]
# calculate x-values for the prediction range
pred_x = [i for i in range(X[-1] + 1, X[-1] + 1 + self.forecast_days_required)]
# run the forecast
results = self._run_forecast(X, Y, to_predict=pred_x)
result_dict = {}
for i, value in enumerate(results.prediction):
# extrapolate confidence intervals to align with prediction.
# this reduces the confidence interval below 95th percentile, but is a better UX.
if i < len(results.confidence_lower):
lower = results.confidence_lower[i]
else:
lower = results.confidence_lower[-1] + results.slope * (i - len(results.confidence_lower))
if i < len(results.confidence_upper):
upper = results.confidence_upper[i]
else:
upper = results.confidence_upper[-1] + results.slope * (i - len(results.confidence_upper))
# ensure that there are no negative numbers.
result_dict[self.dh.today.date() + timedelta(days=i)] = {
"total_cost": max((value, 0)),
"confidence_min": max((lower, 0)),
"confidence_max": max((upper, 0)),
}
return (result_dict, results.rsquared, results.pvalues)
def _enumerate_dates(self, date_list):
"""Given a list of dates, return a list of integers.
This method works in conjunction with _remove_outliers(). This method works to preserve any gaps
in the data created by _remove_outliers() so that the integers used for the X-axis are aligned
appropriately.
Example:
If _remove_outliers() returns {"2000-01-01": 1.0, "2000-01-03": 1.5}
then _enumerate_dates() returns [0, 2]
"""
days = self.dh.list_days(
datetime.combine(date_list[0], self.dh.midnight), datetime.combine(date_list[-1], self.dh.midnight)
)
out = [i for i, day in enumerate(days) if day.date() in date_list]
return out
def _remove_outliers(self, data):
"""Remove outliers from our dateset before predicting.
We use a box plot method without plotting the box.
"""
values = list(data.values())
if values:
third_quartile, first_quartile = np.percentile(values, [Decimal(75), Decimal(25)])
interquartile_range = third_quartile - first_quartile
upper_boundary = third_quartile + (Decimal(1.5) * interquartile_range)
lower_boundary = first_quartile - (Decimal(1.5) * interquartile_range)
return {key: value for key, value in data.items() if (value >= lower_boundary and value <= upper_boundary)}
return data
def _key_results_by_date(self, results, check_term="total_cost"):
"""Take results formatted by cost type, and return results keyed by date."""
results_by_date = defaultdict(dict)
date_based_dict = results[check_term][0] if results[check_term] else []
for date in date_based_dict:
for cost_term in results:
if results[cost_term][0].get(date):
results_by_date[date][cost_term] = (
results[cost_term][0][date],
{"rsquared": results[cost_term][1]},
{"pvalues": results[cost_term][2]},
)
return results_by_date
def format_result(self, results):
"""Format results for API consumption."""
f_format = f"%.{self.PRECISION}f" # avoid converting floats to e-notation
units = "USD"
response = []
for key in results:
if key > self.dh.this_month_end.date():
continue
dikt = {
"date": key,
"values": [
{
"date": key,
"infrastructure": {
"total": {
"value": round(results[key]["infrastructure_cost"][0]["total_cost"], 3),
"units": units,
},
"confidence_max": {
"value": round(results[key]["infrastructure_cost"][0]["confidence_max"], 3),
"units": units,
},
"confidence_min": {
"value": round(max(results[key]["infrastructure_cost"][0]["confidence_min"], 0), 3),
"units": units,
},
"rsquared": {
"value": f_format % results[key]["infrastructure_cost"][1]["rsquared"],
"units": None,
},
"pvalues": {"value": results[key]["infrastructure_cost"][2]["pvalues"], "units": None},
},
"supplementary": {
"total": {
"value": round(results[key]["supplementary_cost"][0]["total_cost"], 3),
"units": units,
},
"confidence_max": {
"value": round(results[key]["supplementary_cost"][0]["confidence_max"], 3),
"units": units,
},
"confidence_min": {
"value": round(max(results[key]["supplementary_cost"][0]["confidence_min"], 0), 3),
"units": units,
},
"rsquared": {
"value": f_format % results[key]["supplementary_cost"][1]["rsquared"],
"units": None,
},
"pvalues": {"value": results[key]["supplementary_cost"][2]["pvalues"], "units": None},
},
"cost": {
"total": {"value": round(results[key]["total_cost"][0]["total_cost"], 3), "units": units},
"confidence_max": {
"value": round(results[key]["total_cost"][0]["confidence_max"], 3),
"units": units,
},
"confidence_min": {
"value": round(max(results[key]["total_cost"][0]["confidence_min"], 0), 3),
"units": units,
},
"rsquared": {"value": f_format % results[key]["total_cost"][1]["rsquared"], "units": None},
"pvalues": {"value": results[key]["total_cost"][2]["pvalues"], "units": None},
},
}
],
}
response.append(dikt)
return response
def _run_forecast(self, x, y, to_predict=None):
"""Apply the forecast model.
Args:
x (list) a list of exogenous variables
y (list) a list of endogenous variables
to_predict (list) a list of exogenous variables used in the forecast results
Note:
both x and y MUST be the same number of elements
Returns:
(tuple)
(numpy.ndarray) prediction values
(numpy.ndarray) confidence interval lower bound
(numpy.ndarray) confidence interval upper bound
(float) R-squared value
(list) P-values
"""
x = sm.add_constant(x)
to_predict = sm.add_constant(to_predict)
model = sm.OLS(y, x)
results = model.fit()
return LinearForecastResult(results, exog=to_predict)
def _uniquify_qset(self, qset, field="total_cost"):
"""Take a QuerySet list, sum costs within the same day, and arrange it into a list of tuples.
Args:
qset (QuerySet)
field (str) - field name in the QuerySet to be summed
Returns:
[(date, cost), ...]
"""
# FIXME: this QuerySet->dict->list conversion probably isn't ideal.
# FIXME: there's probably a way to aggregate multiple sources by date using just the ORM.
result = defaultdict(Decimal)
for item in qset:
result[item.get("usage_start")] += Decimal(item.get(field, 0.0))
result = self._remove_outliers(result)
out = [(k, v) for k, v in result.items()]
return out
def set_access_filters(self, access, filt, filters):
"""Set access filters to ensure RBAC restrictions adhere to user's access and filters.
Args:
access (list) the list containing the users relevant access
filt (list or dict) contains the filters to be updated
filters (QueryFilterCollection) the filter collection to add the new filters to
returns:
None
"""
if isinstance(filt, list):
for _filt in filt:
_filt["operation"] = "in"
q_filter = QueryFilter(parameter=access, **_filt)
filters.add(q_filter)
else:
filt["operation"] = "in"
q_filter = QueryFilter(parameter=access, **filt)
filters.add(q_filter)
def create_daily_archives(tracing_id,
account,
provider_uuid,
filename,
filepath,
manifest_id,
start_date,
last_export_time,
context={}):
"""
Create daily CSVs from incoming report and archive to S3.
Args:
tracing_id (str): The tracing id
account (str): The account number
provider_uuid (str): The uuid of a provider
filename (str): The OCP file name
filepath (str): The full path name of the file
manifest_id (int): The manifest identifier
start_date (Datetime): The start datetime of incoming report
context (Dict): Logging context dictionary
"""
download_hash = None
daily_file_names = []
if last_export_time:
download_hash = hashlib.md5(str(last_export_time).encode())
download_hash = download_hash.hexdigest()
if settings.ENABLE_S3_ARCHIVING or enable_trino_processing(
provider_uuid, Provider.PROVIDER_GCP, account):
dh = DateHelper()
directory = os.path.dirname(filepath)
try:
data_frame = pd.read_csv(filepath)
except Exception as error:
LOG.error(
f"File {filepath} could not be parsed. Reason: {str(error)}")
raise error
for invoice_month in data_frame["invoice.month"].unique():
# daily_files = []
invoice_filter = data_frame["invoice.month"] == invoice_month
invoice_data = data_frame[invoice_filter]
unique_times = invoice_data.partition_date.unique()
days = list({cur_dt[:10] for cur_dt in unique_times})
daily_data_frames = [{
"data_frame":
invoice_data[invoice_data.partition_date.str.contains(
cur_day)],
"date":
cur_day
} for cur_day in days]
start_of_invoice = dh.gcp_invoice_month_start(invoice_month)
s3_csv_path = get_path_prefix(account, Provider.PROVIDER_GCP,
provider_uuid, start_of_invoice,
Config.CSV_DATA_TYPE)
for daily_data in daily_data_frames:
day = daily_data.get("date")
df = daily_data.get("data_frame")
if download_hash:
day_file = f"{invoice_month}_{day}_{download_hash}.csv"
else:
day_file = f"{invoice_month}_{day}.csv"
day_filepath = f"{directory}/{day_file}"
df.to_csv(day_filepath, index=False, header=True)
copy_local_report_file_to_s3_bucket(tracing_id, s3_csv_path,
day_filepath, day_file,
manifest_id, start_date,
context)
daily_file_names.append(day_filepath)
return daily_file_names
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.accessor = OCPReportDBAccessor(schema=cls.schema)
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.dh = DateHelper()
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.dh = DateHelper()
def report_data(request):
"""Update report summary tables in the database."""
if request.method == "GET":
async_results = []
params = request.query_params
async_result = None
all_providers = False
provider_uuid = params.get("provider_uuid")
provider_type = params.get("provider_type")
schema_name = params.get("schema")
start_date = params.get("start_date")
end_date = params.get("end_date")
queue_name = params.get("queue") or PRIORITY_QUEUE
if provider_uuid is None and provider_type is None:
errmsg = "provider_uuid or provider_type must be supplied as a parameter."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if queue_name not in QUEUE_LIST:
errmsg = f"'queue' must be one of {QUEUE_LIST}."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if provider_uuid == "*":
all_providers = True
elif provider_uuid:
with ProviderDBAccessor(provider_uuid) as provider_accessor:
provider = provider_accessor.get_type()
else:
provider = provider_type
if start_date is None:
errmsg = "start_date is a required parameter."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
start_date = ciso8601.parse_datetime(start_date).replace(
tzinfo=pytz.UTC)
end_date = ciso8601.parse_datetime(end_date).replace(
tzinfo=pytz.UTC) if end_date else DateHelper().today
months = DateHelper().list_month_tuples(start_date, end_date)
num_months = len(months)
first_month = months[0]
months[0] = (start_date, first_month[1])
last_month = months[num_months - 1]
months[num_months - 1] = (last_month[0], end_date)
# need to format all the datetimes into strings with the format "%Y-%m-%d" for the celery task
for i, month in enumerate(months):
start, end = month
start_date = start.date().strftime("%Y-%m-%d")
end_date = end.date().strftime("%Y-%m-%d")
months[i] = (start_date, end_date)
if not all_providers:
if schema_name is None:
errmsg = "schema is a required parameter."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if provider is None:
errmsg = "Unable to determine provider type."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if provider_type and provider_type != provider:
errmsg = "provider_uuid and provider_type have mismatched provider types."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
for month in months:
async_result = update_summary_tables.s(
schema_name,
provider,
provider_uuid,
month[0],
month[1],
queue_name=queue_name).apply_async(
queue=queue_name or PRIORITY_QUEUE)
async_results.append({str(month): str(async_result)})
else:
# TODO: when DEVELOPMENT=False, disable resummarization for all providers to prevent burning the db.
# this query could be re-enabled if we need it, but we should consider limiting its use to a schema.
if not settings.DEVELOPMENT:
errmsg = "?provider_uuid=* is invalid query."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
for month in months:
async_result = update_all_summary_tables.delay(
month[0], month[1])
async_results.append({str(month): str(async_result)})
return Response({REPORT_DATA_KEY: async_results})
if request.method == "DELETE":
params = request.query_params
schema_name = params.get("schema")
provider = params.get("provider")
provider_uuid = params.get("provider_uuid")
simulate = params.get("simulate")
if schema_name is None:
errmsg = "schema is a required parameter."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if provider is None:
errmsg = "provider is a required parameter."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if provider_uuid is None:
errmsg = "provider_uuid is a required parameter."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if simulate is not None and simulate.lower() not in ("true", "false"):
errmsg = "simulate must be a boolean."
return Response({"Error": errmsg},
status=status.HTTP_400_BAD_REQUEST)
if simulate is not None and simulate.lower() == "true":
simulate = True
else:
simulate = False
LOG.info("Calling remove_expired_data async task.")
async_result = remove_expired_data.delay(schema_name, provider,
simulate, provider_uuid)
return Response({"Report Data Task ID": str(async_result)})
def setUp(self):
"""Set up the tests."""
super().setUp()
self.dh = DateHelper()
class AzureReportDataGenerator:
"""Populate the database with Azure fake report data."""
def __init__(self,
tenant,
provider,
current_month_only=False,
config=None):
"""Set up the class."""
self.tenant = tenant
self.config = config if config else FakeAzureConfig()
self.fake = Faker()
self.dh = DateHelper()
self.provider_uuid = provider.uuid
# generate a list of dicts with unique keys.
self.period_ranges, self.report_ranges = self.report_period_and_range(
current_month_only=current_month_only)
def _randomize_line_item(self, retained_fields=None):
"""Update our FakeAzureConfig to generate a new line item."""
DEFAULT_FIELDS = ["subscription_guid", "resource_location", "tags"]
if not retained_fields:
retained_fields = DEFAULT_FIELDS
config_dict = {}
for field in retained_fields:
if field in self.config:
config_dict[field] = getattr(self.config, field)
self.config = FakeAzureConfig(**config_dict)
def report_period_and_range(self, current_month_only=False):
"""Return the report period and range."""
period = []
ranges = []
if current_month_only:
report_days = 10
diff_from_first = self.dh.today - self.dh.this_month_start
if diff_from_first.days < 10:
report_days = 1 + diff_from_first.days
period = [(self.dh.this_month_start, self.dh.this_month_end)]
ranges = [
list(self.dh.this_month_start + relativedelta(days=i)
for i in range(report_days))
]
else:
period = [(self.dh.this_month_start, self.dh.this_month_end)]
ranges = [
list(self.dh.today - relativedelta(days=i)
for i in range(report_days))
]
else:
period = [
(self.dh.last_month_start, self.dh.last_month_end),
(self.dh.this_month_start, self.dh.this_month_end),
]
one_month_ago = self.dh.today - relativedelta(months=1)
diff_from_first = self.dh.today - self.dh.this_month_start
if diff_from_first.days < 10:
report_days = 1 + diff_from_first.days
ranges = [
list(self.dh.last_month_start + relativedelta(days=i)
for i in range(report_days)),
list(self.dh.this_month_start + relativedelta(days=i)
for i in range(report_days)),
]
else:
ranges = [
list(one_month_ago - relativedelta(days=i)
for i in range(10)),
list(self.dh.today - relativedelta(days=i)
for i in range(10)),
]
return (period, ranges)
def add_data_to_tenant(self, fixed_fields=None):
"""Populate tenant with data.
Args:
provider_uuid (uuid) - provider uuid
fixed_fields (list) - a list of field names the data generator should NOT randomize.
"""
with tenant_context(self.tenant):
for i, period in enumerate(self.period_ranges):
manifest = self._manifest(period[0], self.provider_uuid)
self._report_status(manifest.billing_period_start_datetime,
manifest.id)
for report_date in self.report_ranges[i]:
self._randomize_line_item(retained_fields=fixed_fields)
self._cost_entry_line_item_daily_summary(report_date)
self._tag_summary()
def remove_data_from_tenant(self):
"""Remove the added data."""
with tenant_context(self.tenant):
for table in (
AzureCostEntryLineItemDaily,
AzureCostEntryLineItemDailySummary,
AzureCostEntryBill,
AzureCostEntryProductService,
AzureMeter,
AzureTagsSummary,
):
table.objects.all().delete()
# pylint: disable=no-self-use
def remove_data_from_reporting_common(self):
"""Remove the public report statistics."""
for table in (CostUsageReportManifest, CostUsageReportStatus):
table.objects.all().delete()
# pylint: disable=no-self-use
def _manifest(self, start, provider_uuid):
"""Populate a report manifest entry."""
manifest_creation_datetime = start + relativedelta(
days=random.randint(1, 27))
manifest_updated_datetime = manifest_creation_datetime + relativedelta(
days=random.randint(1, 2))
data = {
"assembly_id": uuid4(),
"manifest_creation_datetime": manifest_creation_datetime,
"manifest_updated_datetime": manifest_updated_datetime,
"billing_period_start_datetime": start,
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": provider_uuid,
}
manifest_entry = CostUsageReportManifest(**data)
manifest_entry.save()
return manifest_entry
# pylint: disable=no-self-use
def _report_status(self, billing_period_start, manifest_id):
"""Populate a report status entry."""
etag_hash = hashlib.new("ripemd160")
etag_hash.update(bytes(str(billing_period_start), "utf-8"))
last_started_datetime = billing_period_start + relativedelta(
days=random.randint(1, 3))
last_completed_datetime = last_started_datetime + relativedelta(days=1)
data = {
"report_name": uuid4(),
"last_completed_datetime": last_completed_datetime,
"last_started_datetime": last_started_datetime,
"etag": etag_hash.hexdigest(),
"manifest_id": manifest_id,
}
status_entry = CostUsageReportStatus(**data)
status_entry.save()
return status_entry
def _cost_entry_bill(self, report_date=None):
"""Populate AzureCostEntryBill."""
billing_period_start = self.config.billing_period_start
billing_period_end = self.config.billing_period_end
if report_date:
billing_period_start = report_date.replace(day=1)
last_day = self.dh.days_in_month(report_date)
billing_period_end = report_date.replace(day=last_day)
obj, _ = AzureCostEntryBill.objects.get_or_create(
billing_period_start=billing_period_start,
billing_period_end=billing_period_end,
summary_data_creation_datetime=self.dh.today,
summary_data_updated_datetime=self.dh.today,
finalized_datetime=None,
derived_cost_datetime=self.dh.today,
provider_id=self.provider_uuid,
)
# obj.save()
return obj
def _cost_entry_product(self):
"""Populate AzureCostEntryProduct."""
fake_dict = {self.fake.word(): self.fake.word()}
obj = AzureCostEntryProductService(
instance_id=self.config.instance_id,
resource_location=self.config.resource_location,
consumed_service=self.config.resource_service,
resource_type=self.config.resource_type,
resource_group=self.fake.word(),
additional_info=fake_dict,
service_name=self.config.service_name,
service_tier=self.config.service_tier,
service_info1=random.choice([None, self.fake.word()]),
service_info2=random.choice([None, self.fake.word()]),
)
obj.save()
return obj
def _meter(self):
"""Populate AzureMeter."""
obj = AzureMeter(
meter_id=uuid4(),
meter_name=self.config.meter_name,
meter_category=self.config.meter_category,
meter_subcategory=self.config.meter_subcategory,
meter_region=self.config.resource_location,
resource_rate=random.random(),
currency=random.choice(_currency_symbols()),
)
obj.save()
return obj
def _cost_entry_line_item_daily(self, report_date=None):
"""Populate AzureCostEntryLineItemDaily."""
if report_date:
usage_dt = report_date
else:
usage_dt = self.fake.date_time_between_dates(
self.dh.this_month_start, self.dh.today)
usage_qty = random.random() * random.randrange(0, 100)
obj = AzureCostEntryLineItemDaily(
cost_entry_bill=self._cost_entry_bill(report_date),
cost_entry_product=self._cost_entry_product(),
meter=self._meter(),
subscription_guid=self.config.subscription_guid,
tags=self.select_tags(),
usage_date=usage_dt.date()
if isinstance(usage_dt, datetime) else usage_dt,
usage_quantity=usage_qty,
pretax_cost=usage_qty * self.config.meter_rate,
offer_id=random.choice([None, self.fake.pyint()]),
)
obj.save()
return obj
def _cost_entry_line_item_daily_summary(self, report_date=None):
"""Populate AzureCostEntryLineItemDailySummary."""
line_item = self._cost_entry_line_item_daily(report_date)
obj = AzureCostEntryLineItemDailySummary(
cost_entry_bill=line_item.cost_entry_bill,
meter=line_item.meter,
subscription_guid=line_item.subscription_guid,
instance_type=self.config.instance_type,
service_name=line_item.cost_entry_product.service_name,
resource_location=line_item.cost_entry_product.resource_location,
tags=line_item.tags,
usage_start=line_item.usage_date,
usage_end=line_item.usage_date,
usage_quantity=line_item.usage_quantity,
pretax_cost=line_item.pretax_cost,
markup_cost=line_item.pretax_cost * 0.1,
offer_id=line_item.offer_id,
)
obj.save()
return obj
def _tag_summary(self):
"""Populate AzureTagsSummary."""
agg_sql = pkgutil.get_data("masu.database",
"sql/reporting_azuretags_summary.sql")
agg_sql = agg_sql.decode("utf-8")
agg_sql_params = {"schema": connection.schema_name}
agg_sql, agg_sql_params = JinjaSql().prepare_query(
agg_sql, agg_sql_params)
with connection.cursor() as cursor:
cursor.execute(agg_sql)
def select_tags(self):
"""Return a random selection of the defined tags."""
return {
key: self.config.tags[key]
for key in random.choices(list(self.config.tags.keys()),
k=random.randrange(
2, len(self.config.tags.keys())))
}
def test_populate_ocp_on_azure_cost_daily_summary(self):
"""Test the method to run OpenShift on Azure SQL."""
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
project_summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_project_daily_summary"]
markup_value = decimal.Decimal(0.1)
summary_table = getattr(self.accessor.report_schema,
summary_table_name)
project_table = getattr(self.accessor.report_schema,
project_summary_table_name)
today = DateHelper().today
last_month = DateHelper().last_month_start
azure_bills = get_bills_from_provider(self.azure_provider_uuid,
self.schema, last_month, today)
with schema_context(self.schema):
bill_ids = [str(bill.id) for bill in azure_bills]
cluster_id = self.ocp_on_azure_ocp_provider.authentication.credentials.get(
"cluster_id")
self.accessor.populate_ocp_on_azure_cost_daily_summary(
last_month, today, cluster_id, bill_ids, markup_value)
li_table_name = AZURE_REPORT_TABLE_MAP["line_item"]
with schema_context(self.schema):
li_table = getattr(self.accessor.report_schema, li_table_name)
sum_azure_cost = li_table.objects.aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
with schema_context(self.schema):
# These names are defined in the `azure_static_data.yml` used by Nise to populate the Azure data
namespaces = [
"kube-system", "openshift", "banking", "mobile", "news-site",
"weather"
]
for namespace in namespaces:
with self.subTest(namespace=namespace):
with connection.cursor() as cursor:
cursor.execute(f"""
SELECT sum(pretax_cost / cardinality(namespace)) AS pretax_cost
FROM {summary_table._meta.db_table}
WHERE namespace @> array['{namespace}'::varchar]
""")
sum_cost = cursor.fetchone()[0]
sum_project_cost = project_table.objects.filter(
namespace=namespace).aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
self.assertNotEqual(sum_cost, 0)
self.assertAlmostEqual(sum_cost, sum_project_cost, 4)
self.assertLessEqual(sum_cost, sum_azure_cost)
with schema_context(self.schema):
sum_cost = summary_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
sum_markup_cost = summary_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("markup_cost"))["markup_cost__sum"]
sum_project_cost = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
sum_pod_cost = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("pod_cost"))["pod_cost__sum"]
sum_markup_cost_project = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("markup_cost"))["markup_cost__sum"]
sum_project_markup_cost_project = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("project_markup_cost"))["project_markup_cost__sum"]
self.assertLessEqual(sum_cost, sum_azure_cost)
self.assertAlmostEqual(sum_markup_cost, sum_cost * markup_value, 4)
self.assertAlmostEqual(sum_markup_cost_project,
sum_project_cost * markup_value, 4)
self.assertAlmostEqual(sum_project_markup_cost_project,
sum_pod_cost * markup_value, 4)
class FakeAzureConfig(UserDict):
"""Azure Configuration Data Mocker.
This object holds any values that may need to be preserved across multiple
generated line items. The AzureReportDataGenerator does its own
randomization of the fields where state never needs to be preserved.
"""
fake = Faker()
dh = DateHelper()
def _get_prop(self, name, default):
"""Get property value or use provided default."""
if name not in self.data:
self.data[name] = default
return self.data.get(name)
@property
def billing_period_start(self):
"""Return the datetime."""
return self._get_prop("billing_period_start", self.dh.this_month_start)
@property
def billing_period_end(self):
"""Return the datetime."""
return self._get_prop("billing_period_end", self.dh.this_month_end)
@property
def instance_id(self):
"""Return the instance_id."""
template = "/subscriptions/{uuid}/resourceGroups/{group}/providers/" + "{service}/{resource_type}/{name}"
return self._get_prop(
"instance_id",
template.format(
uuid=self.subscription_guid,
group=self.fake.word(),
name=self.fake.word(),
service=self.resource_service,
resource_type=self.resource_type,
),
)
@property
def instance_type(self):
"""Return the instance type."""
return self._get_prop("instance_type",
random.choice(AZURE_INSTANCE_TYPES))
@property
def meter_category(self):
"""Return the meter category."""
return self._get_prop(
"meter_category",
random.choice(list(AZURE_METERS[self.meter_name].keys())))
@property
def meter_name(self):
"""Return the meter name."""
return self._get_prop("meter_name",
random.choice(list(AZURE_METERS.keys())))
@property
def meter_rate(self):
"""Return the meter rate."""
return self._get_prop("meter_rate", random.random())
@property
def meter_subcategory(self):
"""Return the meter subcategory."""
return self._get_prop(
"meter_subcategory",
random.choice(
list(AZURE_METERS[self.meter_name][self.meter_category])))
@property
def resource(self):
"""Return the resource name."""
return self._get_prop(
"resource", random.choice(AZURE_RESOURCES[self.resource_service]))
@property
def resource_location(self):
"""Return the location."""
return self._get_prop("resource_location",
random.choice(AZURE_LOCATIONS))
@property
def resource_service(self):
"""Return the service name."""
return self._get_prop("resource_service",
random.choice(list(AZURE_RESOURCES.keys())))
@property
def resource_type(self):
"""Return the resource type."""
return self._get_prop("resource_type",
str(self.resource).split("/")[-1])
@property
def tags(self):
"""Return the list of tag dicts."""
new_tags = {
key: self.fake.word()
for key in
{self.fake.word()
for _ in range(0, random.randrange(4, 10))}
}
return self._get_prop("tags", new_tags)
@property
def service_name(self):
"""Return the service name."""
return self._get_prop("service_name",
random.choice(list(AZURE_SERVICES.keys())))
@service_name.setter
def service_name(self, name):
"""Set the service name."""
self.data["service_name"] = name
@property
def service_tier(self):
"""Return the service tier."""
return self._get_prop("service_tier",
random.choice(AZURE_SERVICES[self.service_name]))
@property
def subscription_guid(self):
"""Return the uuid."""
return self._get_prop("subscription_guid", uuid4())
def update_provider_uuids(self, provider_uuids):
"""Update rate with new provider uuids."""
current_providers_for_instance = []
for rate_map_instance in CostModelMap.objects.filter(
cost_model=self._model):
current_providers_for_instance.append(
str(rate_map_instance.provider_uuid))
providers_to_delete = set(current_providers_for_instance).difference(
provider_uuids)
providers_to_create = set(provider_uuids).difference(
current_providers_for_instance)
all_providers = set(current_providers_for_instance).union(
provider_uuids)
for provider_uuid in providers_to_delete:
CostModelMap.objects.filter(provider_uuid=provider_uuid,
cost_model=self._model).delete()
for provider_uuid in providers_to_create:
# Raise exception if source is already associated with another cost model.
existing_cost_model = CostModelMap.objects.filter(
provider_uuid=provider_uuid)
if existing_cost_model.exists():
cost_model_uuid = existing_cost_model.first().cost_model.uuid
log_msg = f"Source {provider_uuid} is already associated with cost model: {cost_model_uuid}."
LOG.warning(log_msg)
raise CostModelException(log_msg)
CostModelMap.objects.create(cost_model=self._model,
provider_uuid=provider_uuid)
start_date = DateHelper().this_month_start.strftime("%Y-%m-%d")
end_date = DateHelper().today.strftime("%Y-%m-%d")
for provider_uuid in all_providers:
tracing_id = uuid.uuid4()
# Update cost-model costs for each provider, on every PUT/DELETE
try:
provider = Provider.objects.get(uuid=provider_uuid)
except Provider.DoesNotExist:
LOG.info(
f"Provider {provider_uuid} does not exist. Skipping cost-model update."
)
else:
if provider.active:
schema_name = provider.customer.schema_name
# Because this is triggered from the UI, we use the priority queue
LOG.info(
f"provider {provider_uuid} update for cost model {self._cost_model_uuid} "
+ f"with tracing_id {tracing_id}")
chain(
update_cost_model_costs.s(
schema_name,
provider.uuid,
start_date,
end_date,
tracing_id=tracing_id,
queue_name=PRIORITY_QUEUE,
).set(queue=PRIORITY_QUEUE),
refresh_materialized_views.si(
schema_name,
provider.type,
provider_uuid=provider.uuid,
tracing_id=tracing_id,
queue_name=PRIORITY_QUEUE,
).set(queue=PRIORITY_QUEUE),
).apply_async()
def test_update_summary_tables_ocp(self, mock_cost_model, mock_charge_info,
mock_view, mock_chain):
"""Test that the summary table task runs."""
infrastructure_rates = {
"cpu_core_usage_per_hour": 1.5,
"memory_gb_usage_per_hour": 2.5,
"storage_gb_usage_per_month": 0.5,
}
markup = {}
mock_cost_model.return_value.__enter__.return_value.infrastructure_rates = infrastructure_rates
mock_cost_model.return_value.__enter__.return_value.supplementary_rates = {}
mock_cost_model.return_value.__enter__.return_value.markup = markup
provider = Provider.PROVIDER_OCP
provider_ocp_uuid = self.ocp_test_provider_uuid
daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"]
start_date = DateHelper().last_month_start
end_date = DateHelper().last_month_end
with schema_context(self.schema):
daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name)
daily_query.delete()
initial_daily_count = daily_query.count()
self.assertEqual(initial_daily_count, 0)
update_summary_tables(self.schema, provider, provider_ocp_uuid,
start_date, end_date)
with schema_context(self.schema):
self.assertNotEqual(daily_query.count(), initial_daily_count)
update_cost_model_costs(schema_name=self.schema,
provider_uuid=provider_ocp_uuid,
start_date=start_date,
end_date=end_date)
table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"]
with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor:
provider_obj = provider_accessor.get_provider()
usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(
provider_obj.uuid)
with schema_context(self.schema):
cluster_id = usage_period_qry.first().cluster_id
items = self.ocp_accessor._get_db_obj_query(table_name).filter(
usage_start__gte=start_date,
usage_start__lte=end_date,
cluster_id=cluster_id,
data_source="Pod")
for item in items:
self.assertNotEqual(item.infrastructure_usage_cost.get("cpu"),
0)
self.assertNotEqual(
item.infrastructure_usage_cost.get("memory"), 0)
storage_daily_name = OCP_REPORT_TABLE_MAP[
"storage_line_item_daily"]
items = self.ocp_accessor._get_db_obj_query(
storage_daily_name).filter(cluster_id=cluster_id)
for item in items:
self.assertIsNotNone(item.volume_request_storage_byte_seconds)
self.assertIsNotNone(
item.persistentvolumeclaim_usage_byte_seconds)
storage_summary_name = OCP_REPORT_TABLE_MAP[
"line_item_daily_summary"]
items = self.ocp_accessor._get_db_obj_query(
storage_summary_name).filter(cluster_id=cluster_id,
data_source="Storage")
for item in items:
self.assertIsNotNone(
item.volume_request_storage_gigabyte_months)
self.assertIsNotNone(
item.persistentvolumeclaim_usage_gigabyte_months)
mock_chain.return_value.apply_async.assert_called()
def update_summary_tables(schema_name,
provider,
provider_uuid,
start_date,
end_date=None,
manifest_id=None):
"""Populate the summary tables for reporting.
Args:
schema_name (str) The DB schema name.
provider (str) The provider type.
provider_uuid (str) The provider uuid.
report_dict (dict) The report data dict from previous task.
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
worker_stats.REPORT_SUMMARY_ATTEMPTS_COUNTER.labels(
provider_type=provider).inc()
stmt = (f"update_summary_tables called with args:\n"
f" schema_name: {schema_name},\n"
f" provider: {provider},\n"
f" start_date: {start_date},\n"
f" end_date: {end_date},\n"
f" manifest_id: {manifest_id}")
LOG.info(stmt)
updater = ReportSummaryUpdater(schema_name, provider_uuid, manifest_id)
start_date, end_date = updater.update_daily_tables(start_date, end_date)
updater.update_summary_tables(start_date, end_date)
if not provider_uuid:
refresh_materialized_views.delay(schema_name, provider, manifest_id)
return
with CostModelDBAccessor(schema_name,
provider_uuid) as cost_model_accessor:
cost_model = cost_model_accessor.cost_model
if cost_model is not None:
linked_tasks = update_cost_model_costs.s(
schema_name, provider_uuid,
start_date, end_date) | refresh_materialized_views.si(
schema_name, provider, manifest_id)
else:
stmt = (
f"\n update_cost_model_costs skipped. No cost model available for \n"
f" schema_name: {schema_name},\n"
f" provider_uuid: {provider_uuid}")
LOG.info(stmt)
linked_tasks = refresh_materialized_views.s(schema_name, provider,
manifest_id)
dh = DateHelper(utc=True)
prev_month_start_day = dh.last_month_start.replace(tzinfo=None)
start_date_obj = datetime.datetime.strptime(start_date, "%Y-%m-%d")
if manifest_id and (start_date_obj <= prev_month_start_day):
# We want make sure that the manifest_id is not none, because
# we only want to call the delete line items after the summarize_reports
# task above
simulate = False
line_items_only = True
linked_tasks |= remove_expired_data.si(schema_name, provider, simulate,
provider_uuid, line_items_only)
chain(linked_tasks).apply_async()
def end_date(self):
"""Return an end date."""
dh = DateHelper()
return dh.month_end(self.start_date)
class FakeAWSCostData:
"""Object to generate and store fake AWS cost data."""
fake = Faker()
dh = DateHelper()
SOME_INSTANCE_TYPES = [
"t3.small",
"t3.medium",
"t3.large",
"m5.large",
"m5.xlarge",
"m5.2xlarge",
"c5.large",
"c5.xlarge",
"c5.2xlarge",
"r5.large",
"r5.xlarge",
"r5.2xlarge",
]
SOME_REGIONS = [
"us-east-2",
"us-east-1",
"us-west-1",
"us-west-2",
"ap-south-1",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-southeast-1",
"ap-southeast-2",
"ca-central-1",
"eu-central-1",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"sa-east-1",
]
def __init__(
self,
provider,
account_alias=None,
account_id=None,
availability_zone=None,
bill=None,
billing_period_end=None,
billing_period_start=None,
cost_entry=None,
instance_type=None,
line_item=None,
pricing=None,
region=None,
usage_end=None,
usage_start=None,
resource_id=None,
):
"""Constructor."""
# properties
self.provider = provider
self._account_alias = account_alias
self._account_id = account_id
self._availability_zone = availability_zone
self._bill = bill
self._billing_period_end = billing_period_end
self._billing_period_start = billing_period_start
self._cost_entry = cost_entry
self._instance_type = instance_type
self._line_item = line_item
self._pricing = pricing
self._region = region
self._usage_end = usage_end
self._usage_start = usage_start
self._resource_id = resource_id
self._products = {
"fake": {
"sku": self.fake.pystr(min_chars=12, max_chars=12).upper(),
"product_name": self.fake.words(nb=5),
"product_family": self.fake.words(nb=3),
"service_code": self.fake.word(),
"region": self.region,
"instance_type": self.instance_type,
"memory": random.randint(1, 100),
"vcpu": random.randint(1, 100),
},
"ec2": {
"sku": self.fake.pystr(min_chars=12, max_chars=12).upper(),
"product_name": "Amazon Elastic Compute Cloud",
"product_family": "Compute Instance",
"service_code": "AmazonEC2",
"region": self.region,
"instance_type": self.instance_type,
"memory": random.choice([8, 16, 32, 64]),
"vcpu": random.choice([2, 4, 8, 16]),
},
"ebs": {
"sku": self.fake.pystr(min_chars=12, max_chars=12).upper(),
"product_name": "Amazon Elastic Compute Cloud",
"product_family": "Storage",
"service_code": "AmazonEC2",
"region": self.region,
},
}
short_region = self._usage_transform(self.region)
self.SOME_USAGE_OPERATIONS = {
"ec2": [
(f"BoxUsage:{self.region}", "RunInstances"),
("DataTransfer-In-Bytes", "RunInstances"),
("DataTransfer-Out-Bytes", "RunInstances"),
(f"{short_region}-DataTransfer-In-Bytes", "RunInstances"),
(f"{short_region}-DataTransfer-Out-Bytes", "RunInstances"),
],
"ebs": [
("EBS:VolumeUsage.gp2", "CreateVolume-Gp2"),
("EBS:VolumeUsage", "CreateVolume"),
(f"{short_region}-EBS:VolumeUsage", "CreateVolume"),
(f"{short_region}-EBS:VolumeUsage.gp2", "CreateVolume-Gp2"),
("EBS:SnapshotUsage", "CreateSnapshot"),
],
}
def __str__(self):
"""Represent data as string."""
return str(self.to_dict())
def _usage_transform(self, region):
"""Translate region into shortened string used in usage.
Example: 'us-east-1' becomes 'USE1'
Note: Real-world line items can be formatted using 'EUC1' or 'EU', depending
on the context. Additional work will be required to support the
second format.
"""
regex = r"(\w+)-(\w+)-(\d+)"
groups = re.search(regex, region).groups()
output = "{}{}{}".format(groups[0].upper(), groups[1][0].upper(),
groups[2])
return output
@property
def account_alias(self):
"""Randomly generated account alias."""
if not self._account_alias:
self._account_alias = self.fake.company()
return self._account_alias
@account_alias.setter
def account_alias(self, alias):
"""Account alias setter."""
self._account_alias = alias
@property
def account_id(self):
"""Randomly generated account id."""
if not self._account_id:
self._account_id = self.fake.ean(length=13)
return self._account_id
@account_id.setter
def account_id(self, account_id):
"""Account id setter."""
self._account_id = account_id
if self.bill:
self.bill["payer_account_id"] = account_id
if self._line_item:
self._line_item["usage_account_id"] = account_id
@property
def availability_zone(self):
"""Availability zone."""
if not self._availability_zone:
self._availability_zone = self.region + random.choice(
["a", "b", "c"])
return self._availability_zone
@availability_zone.setter
def availability_zone(self, zone):
"""Availability zone."""
self._availability_zone = zone
if self._line_item:
self._line_item["availability_zone"] = zone
@property
def bill(self):
"""Bill."""
if not self._bill:
self._bill = {
"bill_type": "Anniversary",
"payer_account_id": self.account_id,
"billing_period_start": self.billing_period_start,
"billing_period_end": self.billing_period_end,
"provider_id": self.provider.uuid,
}
return self._bill
@bill.setter
def bill(self, obj):
"""Bill setter."""
self._bill = obj
@property
def billing_period_end(self):
"""Billing period end date."""
if not self._billing_period_end:
self._billing_period_end = self.dh.this_month_end
return self._billing_period_end
@billing_period_end.setter
def billing_period_end(self, date):
"""Billing period end date setter."""
self._billing_period_end = date
if self.bill:
self.bill["billing_period_end"] = date
if self.cost_entry:
self.cost_entry["interval_end"] = date
@property
def billing_period_start(self):
"""Billing period start date."""
if not self._billing_period_start:
self._billing_period_start = self.dh.this_month_start
return self._billing_period_start
@billing_period_start.setter
def billing_period_start(self, date):
"""Billing period start date setter."""
self._billing_period_start = date
if self.bill:
self.bill["billing_period_start"] = date
if self.cost_entry:
self.cost_entry["interval_start"] = date
@property
def cost_entry(self):
"""Cost entry."""
if not self._cost_entry:
self._cost_entry = {
"interval_start": self.billing_period_start,
"interval_end": self.billing_period_end,
"bill": self.bill,
}
return self._cost_entry
@cost_entry.setter
def cost_entry(self, obj):
"""Cost entry setter."""
self._cost_entry = obj
@property
def instance_type(self):
"""Randomly selected instance type."""
if not self._instance_type:
self._instance_type = random.choice(self.SOME_INSTANCE_TYPES)
return self._instance_type
@instance_type.setter
def instance_type(self, instance_type):
"""Instance type setter."""
self._instance_type = instance_type
for prod in self._products:
self._products[prod]["instance_type"] = instance_type
if self._line_item:
self._line_item["cost_entry_product"][
"instance_type"] = instance_type
def line_item(self, product="ec2"):
"""Fake line item.
Args:
product (string) Either 'ec2' or 'ebs'
"""
if not self._line_item:
usage = random.randint(1, 100)
ub_rate = random.random()
b_rate = random.random()
usage_type, operation = random.choice(
self.SOME_USAGE_OPERATIONS[product])
self._line_item = {
"invoice_id": self.fake.sha1(raw_output=False),
"availability_zone": self.availability_zone,
"blended_cost": b_rate * usage,
"blended_rate": b_rate,
"cost_entry": self.cost_entry,
"cost_entry_bill": self.bill,
"cost_entry_pricing": self.pricing,
"cost_entry_product": self.product(product),
"currency_code": "USD",
"line_item_type": "Usage",
"operation": operation,
"product_code": "AmazonEC2",
"resource_id": f"i-{self.resource_id}",
"usage_amount": usage,
"unblended_cost": ub_rate * usage,
"unblended_rate": ub_rate,
"usage_account_id": self.account_id,
"usage_end": self.usage_end,
"usage_start": self.usage_start,
"usage_type": usage_type,
"tags": self._get_tags(),
}
return self._line_item
@property
def pricing(self):
"""Product pricing."""
if not self._pricing:
self._pricing = {"term": "OnDemand", "unit": "Hrs"}
return self._pricing
@pricing.setter
def pricing(self, obj):
"""Pricing setter."""
self._pricing = obj
if self._line_item:
self._line_item["cost_entry_pricing"] = obj
def product(self, product="ec2"):
"""Product."""
return self._products.get(product, self._products["fake"])
@property
def region(self):
"""Randomly selected region."""
if not self._region:
self._region = random.choice(self.SOME_REGIONS)
return self._region
@region.setter
def region(self, region):
"""Region setter."""
self._region = region
for prod in self._products:
self._products[prod]["region"] = region
if self._line_item:
self._line_item["cost_entry_product"]["region"] = region
def to_dict(self):
"""Return a copy of object data as a dict."""
return {
"account_alias": self.account_alias,
"account_id": self.account_id,
"availability_zone": self.availability_zone,
"bill": self.bill,
"billing_period_end": self.billing_period_end,
"billing_period_start": self.billing_period_start,
"cost_entry": self.cost_entry,
"instance_type": self.instance_type,
"line_item": self.line_item(),
"pricing": self.pricing,
"region": self.region,
"usage_end": self.usage_end,
"usage_start": self.usage_start,
}
@property
def usage_end(self):
"""Usage end date."""
if not self._usage_end:
self._usage_end = self.dh.this_month_start + self.dh.one_day
return self._usage_end
@usage_end.setter
def usage_end(self, date):
"""Usage end date setter."""
self._usage_end = date
if self._line_item:
self._line_item["usage_end"] = date
@property
def resource_id(self):
"""resource_id."""
if not self._resource_id:
self._resource_id = self.fake.ean8()
return self._resource_id
@property
def usage_start(self):
"""Usage start date."""
if not self._usage_start:
self._usage_start = self.dh.this_month_start
return self._usage_start
@usage_start.setter
def usage_start(self, date):
"""Usage start date setter."""
self._usage_start = date
if self._line_item:
self._line_item["usage_start"] = date
def _get_tags(self):
"""Create tags for output data."""
apps = [
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
]
organizations = [
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word()
]
markets = [
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
]
versions = [
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
]
seeded_labels = {
"environment": ["dev", "ci", "qa", "stage", "prod"],
"app": apps,
"organization": organizations,
"market": markets,
"version": versions,
}
gen_label_keys = [
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
self.fake.word(),
]
all_label_keys = list(seeded_labels.keys()) + gen_label_keys
num_labels = random.randint(2, len(all_label_keys))
chosen_label_keys = random.choices(all_label_keys, k=num_labels)
labels = {}
for label_key in chosen_label_keys:
label_value = self.fake.word()
if label_key in seeded_labels:
label_value = random.choice(seeded_labels[label_key])
labels[f"{label_key}_label"] = label_value
return labels
class AWSReportViewTest(IamTestCase):
"""Tests the report view."""
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.client = APIClient()
self.dh = DateHelper()
self.ten_days_ago = self.dh.n_days_ago(self.dh.today, 9)
self.report = {
"group_by": {
"account": ["*"]
},
"filter": {
"resolution": "monthly",
"time_scope_value": -1,
"time_scope_units": "month",
"resource_scope": [],
},
"data": [{
"date":
"2018-07",
"accounts": [
{
"account":
"4418636104713",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "4418636104713",
"total": 1826.74238146924,
}],
},
{
"account":
"8577742690384",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "8577742690384",
"total": 1137.74036198065,
}],
},
{
"account":
"3474227945050",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "3474227945050",
"total": 1045.80659412797,
}],
},
{
"account":
"7249815104968",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "7249815104968",
"total": 807.326470618818,
}],
},
{
"account":
"9420673783214",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "9420673783214",
"total": 658.306642830709,
}],
},
],
}],
"total": {
"value": 5475.922451027388,
"units": "GB-Mo"
},
}
def test_execute_query_w_delta_total(self):
"""Test that delta=total returns deltas."""
qs = "delta=cost"
url = reverse("reports-aws-costs") + "?" + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_execute_query_w_delta_bad_choice(self):
"""Test invalid delta value."""
bad_delta = "Invalid"
expected = f'"{bad_delta}" is not a valid choice.'
qs = f"group_by[account]=*&filter[limit]=2&delta={bad_delta}"
url = reverse("reports-aws-costs") + "?" + qs
response = self.client.get(url, **self.headers)
result = str(response.data.get("delta")[0])
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(result, expected)
def test_convert_units_success(self):
"""Test unit conversion succeeds."""
converter = UnitConverter()
to_unit = "byte"
expected_unit = f"{to_unit}-Mo"
report_total = self.report.get("total", {}).get("value")
result = _convert_units(converter, self.report, to_unit)
result_unit = result.get("total", {}).get("units")
result_total = result.get("total", {}).get("value")
self.assertEqual(expected_unit, result_unit)
self.assertEqual(report_total * 1e9, result_total)
def test_convert_units_list(self):
"""Test that the list check is hit."""
converter = UnitConverter()
to_unit = "byte"
expected_unit = f"{to_unit}-Mo"
report_total = self.report.get("total", {}).get("value")
report = [self.report]
result = _convert_units(converter, report, to_unit)
result_unit = result[0].get("total", {}).get("units")
result_total = result[0].get("total", {}).get("value")
self.assertEqual(expected_unit, result_unit)
self.assertEqual(report_total * 1e9, result_total)
def test_convert_units_total_not_dict(self):
"""Test that the total not dict block is hit."""
converter = UnitConverter()
to_unit = "byte"
expected_unit = f"{to_unit}-Mo"
report = self.report["data"][0]["accounts"][0]["values"][0]
report_total = report.get("total")
result = _convert_units(converter, report, to_unit)
result_unit = result.get("units")
result_total = result.get("total")
self.assertEqual(expected_unit, result_unit)
self.assertEqual(report_total * 1e9, result_total)
@RbacPermissions({
"aws.account": {
"read": ["*"]
},
"aws.organizational_unit": {
"read":
["R_001", "OU_001", "OU_002", "OU_003", "OU_004", "OU_005"]
},
})
def test_execute_query_w_group_by_rbac_explicit_access(self):
"""Test that explicit access results in all accounts/orgs listed."""
ou_to_account_subou_map = {
"R_001": {
"accounts": ["9999999999990"],
"org_units": ["OU_001"]
},
"OU_001": {
"accounts": ["9999999999991", "9999999999992"],
"org_units": []
},
"OU_002": {
"accounts": [],
"org_units": ["OU_003"]
},
"OU_003": {
"accounts": ["9999999999993"],
"org_units": []
},
"OU_004": {
"accounts": [],
"org_units": []
},
"OU_005": {
"accounts": [],
"org_units": []
},
}
for org_unit in list(ou_to_account_subou_map):
qs = f"?group_by[org_unit_id]={org_unit}"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
accounts_and_subous = _calculate_accounts_and_subous(
response.data.get("data"))
# These accounts are tied to this org unit inside of the
# aws_org_tree.yml that populates the data for tests
for account in ou_to_account_subou_map.get(org_unit).get(
"accounts"):
self.assertIn(account, accounts_and_subous)
for ou in ou_to_account_subou_map.get(org_unit).get("org_units"):
self.assertIn(ou, accounts_and_subous)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@RbacPermissions({
"aws.account": {
"read": ["*"]
},
"aws.organizational_unit": {
"read": ["R_001", "OU_002", "OU_003", "OU_004", "OU_005"]
},
})
def test_execute_query_w_group_by_rbac_restricted_org_access(self):
"""Test that total account access/restricted org results in all accounts/ accessible orgs."""
ou_to_account_subou_map = {
"R_001": {
"accounts": ["9999999999990"],
"org_units": []
},
"OU_002": {
"accounts": [],
"org_units": ["OU_003"]
},
"OU_003": {
"accounts": ["9999999999993"],
"org_units": []
},
"OU_004": {
"accounts": [],
"org_units": []
},
"OU_005": {
"accounts": [],
"org_units": []
},
}
for org_unit in list(ou_to_account_subou_map):
qs = f"?group_by[org_unit_id]={org_unit}"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
accounts_and_subous = _calculate_accounts_and_subous(
response.data.get("data"))
# These accounts are tied to this org unit inside of the
# aws_org_tree.yml that populates the data for tests
for account in ou_to_account_subou_map.get(org_unit).get(
"accounts"):
self.assertIn(account, accounts_and_subous)
for ou in ou_to_account_subou_map.get(org_unit).get("org_units"):
self.assertIn(ou, accounts_and_subous)
self.assertEqual(response.status_code, status.HTTP_200_OK)
# test that OU_001 raises a 403
qs = "?group_by[org_unit_id]=OU_001"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
@RbacPermissions({
"aws.account": {
"read": ["*"]
},
"aws.organizational_unit": {
"read": ["*"]
}
})
def test_execute_query_w_group_by_rbac_no_restrictions(self):
"""Test that total access results in all accounts and orgs."""
ou_to_account_subou_map = {
"R_001": {
"accounts": ["9999999999990"],
"org_units": ["OU_001"]
},
"OU_001": {
"accounts": ["9999999999991", "9999999999992"],
"org_units": []
},
"OU_002": {
"accounts": [],
"org_units": ["OU_003"]
},
"OU_003": {
"accounts": ["9999999999993"],
"org_units": []
},
"OU_004": {
"accounts": [],
"org_units": []
},
"OU_005": {
"accounts": [],
"org_units": []
},
}
for org_unit in list(ou_to_account_subou_map):
qs = f"?group_by[org_unit_id]={org_unit}"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
accounts_and_subous = _calculate_accounts_and_subous(
response.data.get("data"))
# These accounts are tied to this org unit inside of the
# aws_org_tree.yml that populates the data for tests
for account in ou_to_account_subou_map.get(org_unit).get(
"accounts"):
self.assertIn(account, accounts_and_subous)
for ou in ou_to_account_subou_map.get(org_unit).get("org_units"):
self.assertIn(ou, accounts_and_subous)
self.assertEqual(response.status_code, status.HTTP_200_OK)
# test filter
qs = f"?filter[org_unit_id]={org_unit}"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@RbacPermissions({
"aws.account": {
"read": ["9999999999990"]
},
"aws.organizational_unit": {
"read": ["*"]
}
})
def test_execute_query_w_group_by_rbac_account_restrictions(self):
"""Test that restricted access results in the accessible orgs/accounts."""
ou_to_account_subou_map = {
"R_001": {
"accounts": ["9999999999990"],
"org_units": []
}
}
# since we only have access to the account directly under root - no org units will show up
# because they only show up when they have costs associated with the accounts under them
for org_unit in list(ou_to_account_subou_map):
qs = f"?group_by[org_unit_id]={org_unit}"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
accounts_and_subous = _calculate_accounts_and_subous(
response.data.get("data"))
# These accounts are tied to this org unit inside of the
# aws_org_tree.yml that populates the data for tests
for account in ou_to_account_subou_map.get(org_unit).get(
"accounts"):
self.assertIn(account, accounts_and_subous)
for ou in ou_to_account_subou_map.get(org_unit).get("org_units"):
self.assertIn(ou, accounts_and_subous)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@RbacPermissions({
"aws.account": {
"read": ["9999999999991"]
},
"aws.organizational_unit": {
"read": ["*"]
}
})
def test_execute_query_w_group_by_rbac_restriction(self):
"""Test limited access results in only the account that the user can see."""
qs = "group_by[org_unit_id]=OU_001"
url = reverse("reports-aws-costs") + "?" + qs
response = self.client.get(url, **self.headers)
accounts_and_subous = _calculate_accounts_and_subous(
response.data.get("data"))
self.assertEqual(accounts_and_subous, ["9999999999991"])
self.assertEqual(response.status_code, status.HTTP_200_OK)
@RbacPermissions({
"aws.account": {
"read": ["fakeaccount"]
},
"aws.organizational_unit": {
"read": ["fake_org"]
}
})
def test_execute_query_w_group_by_rbac_no_accounts_or_orgs(self):
"""Test that no access to relevant results in a 403."""
for org in ["R_001", "OU_001", "OU_002", "OU_003", "OU_004", "OU_005"]:
qs = f"?group_by[org_unit_id]={org}"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
# test filters
qs = f"?filter[org_unit_id]={org}"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_group_by_org_unit_non_costs_reports(self):
"""Test that grouping by org unit on non costs reports raises a validation error."""
qs = "?group_by[org_unit_id]=*"
url = reverse("reports-aws-storage") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_group_by_org_unit_wildcard_costs_reports(self):
"""Test that grouping by org unit with a wildcard raises a validation error."""
qs = "?group_by[org_unit_id]=*"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_ou_group_by_default_pagination(self):
"""Test that the default pagination works."""
qs = "?group_by[org_unit_id]=R_001&filter[resolution]=monthly&filter[time_scope_value]=-1&filter[time_scope_units]=month" # noqa: E501
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_data = response.json()
data = response_data.get("data", [])
meta = response_data.get("meta", {})
count = meta.get("count", 0)
self.assertIn("total", meta)
self.assertIn("filter", meta)
self.assertIn("count", meta)
for entry in data:
org_entities = entry.get("org_entities", [])
self.assertEqual(len(org_entities), count)
def test_ou_group_by_filter_limit_offset_pagination(self):
"""Test that the ranked group pagination works."""
limit = 1
offset = 0
qs = f"?group_by[org_unit_id]=R_001&filter[resolution]=monthly&filter[time_scope_value]=-1&filter[time_scope_units]=month&filter[limit]={limit}&filter[offset]={offset}" # noqa: E501
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_data = response.json()
data = response_data.get("data", [])
meta = response_data.get("meta", {})
count = meta.get("count", 0)
self.assertIn("total", meta)
self.assertIn("filter", meta)
self.assertIn("count", meta)
for entry in data:
org_entities = entry.get("org_entities", [])
if limit + offset > count:
self.assertEqual(len(org_entities), max((count - offset), 0))
else:
self.assertEqual(len(org_entities), limit)
def test_ou_group_by_filter_limit_high_offset_pagination(self):
"""Test that high offset pagination works."""
limit = 1
offset = 10
qs = f"?group_by[org_unit_id]=R_001&filter[resolution]=monthly&filter[time_scope_value]=-1&filter[time_scope_units]=month&filter[limit]={limit}&filter[offset]={offset}" # noqa: E501
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_data = response.json()
data = response_data.get("data", [])
meta = response_data.get("meta", {})
count = meta.get("count", 0)
self.assertIn("total", meta)
self.assertIn("filter", meta)
self.assertIn("count", meta)
for entry in data:
org_entities = entry.get("org_entities", [])
if limit + offset > count:
self.assertEqual(len(org_entities), max((count - offset), 0))
else:
self.assertEqual(len(org_entities), limit)
def test_group_by_org_unit_order_by_cost_asc(self):
"""Test that ordering by cost=asc works as expected"""
qs = "?group_by[org_unit_id]=R_001&order_by[cost]=asc"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
data = response.data.get("data", [])
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Now we need to loop through the results and make sure that
# the org units are in asc order according to cost
for entry in data:
org_entities = entry.get("org_entities", [])
sorted_org_entities = copy.deepcopy(org_entities)
sorted_org_entities.sort(
key=lambda e: e["values"][0]["cost"]["total"]["value"],
reverse=False)
self.assertEqual(org_entities, sorted_org_entities)
def test_group_by_org_unit_order_by_cost_desc(self):
"""Test that ordering by cost=descworks as expected"""
qs = "?group_by[org_unit_id]=R_001&order_by[cost]=desc"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
data = response.data.get("data")
self.assertEqual(response.status_code, status.HTTP_200_OK)
# Now we need to loop through the results and make sure that
# the org units are in desc order according to cost
for entry in data:
org_entities = entry.get("org_entities", [])
sorted_org_entities = copy.deepcopy(org_entities)
sorted_org_entities.sort(
key=lambda e: e["values"][0]["cost"]["total"]["value"],
reverse=True)
self.assertEqual(org_entities, sorted_org_entities)
def test_multiple_and_group_by_org_unit_bad_request(self):
"""Test that grouping by org unit on non costs reports raises a validation error."""
qs = "?group_by[org_unit_id]=R_001&group_by[org_unit_id]=OU_001"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_multiple_mixed_group_by_org_unit_bad_request(self):
"""Test that grouping by org unit on non costs reports raises a validation error."""
qs = "?group_by[org_unit_id]=R_001&group_by[or:org_unit_id]=OU_001"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_group_by_org_unit_or_wildcard_bad_request(self):
"""Test that grouping by org unit on non costs reports raises a validation error."""
qs = "?group_by[or:org_unit_id]=*"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_group_by_org_unit_id_and_wildcard_region(self):
"""Test multiple group by with org unit id and region."""
# The ui team uses these to populate graphs
qs = "?group_by[or:org_unit_id]=R_001&group_by[region]=*"
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_group_by_org_unit_id_and_wildcard_account(self):
"""Test multiple group by with org unit id and account."""
qs = "?group_by[or:org_unit_id]=R_001&group_by[account]=*"
# The ui team uses these to populate graphs
url = reverse("reports-aws-costs") + qs
response = self.client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.client = APIClient()
self.factory = RequestFactory()
self.dh = DateHelper()
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.client = APIClient()
self.dh = DateHelper()
self.ten_days_ago = self.dh.n_days_ago(self.dh.today, 9)
self.report = {
"group_by": {
"account": ["*"]
},
"filter": {
"resolution": "monthly",
"time_scope_value": -1,
"time_scope_units": "month",
"resource_scope": [],
},
"data": [{
"date":
"2018-07",
"accounts": [
{
"account":
"4418636104713",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "4418636104713",
"total": 1826.74238146924,
}],
},
{
"account":
"8577742690384",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "8577742690384",
"total": 1137.74036198065,
}],
},
{
"account":
"3474227945050",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "3474227945050",
"total": 1045.80659412797,
}],
},
{
"account":
"7249815104968",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "7249815104968",
"total": 807.326470618818,
}],
},
{
"account":
"9420673783214",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"account": "9420673783214",
"total": 658.306642830709,
}],
},
],
}],
"total": {
"value": 5475.922451027388,
"units": "GB-Mo"
},
}
class OCPAzureQueryHandlerTest(IamTestCase):
"""Tests for the OCP report query handler."""
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.dh = DateHelper()
_, self.provider = create_generic_provider(Provider.PROVIDER_OCP, self.headers)
self.this_month_filter = {"usage_start__gte": self.dh.this_month_start}
self.ten_day_filter = {"usage_start__gte": self.dh.n_days_ago(self.dh.today, 9)}
self.thirty_day_filter = {"usage_start__gte": self.dh.n_days_ago(self.dh.today, 29)}
self.last_month_filter = {
"usage_start__gte": self.dh.last_month_start,
"usage_end__lte": self.dh.last_month_end,
}
self.generator = OCPAzureReportDataGenerator(self.tenant, self.provider)
def get_totals_by_time_scope(self, aggregates, filters=None):
"""Return the total aggregates for a time period."""
if filters is None:
filters = self.ten_day_filter
with tenant_context(self.tenant):
return OCPAzureCostLineItemDailySummary.objects.filter(**filters).aggregate(**aggregates)
def test_execute_sum_query_storage(self):
"""Test that the sum query runs properly."""
self.generator.add_data_to_tenant(service_name="Storage")
url = "?"
query_params = self.mocked_query_params(url, OCPAzureStorageView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
aggregates = handler._mapper.report_type_map.get("aggregates")
filt = {"service_name__contains": "Storage"}
filt.update(self.ten_day_filter)
current_totals = self.get_totals_by_time_scope(aggregates, filt)
total = query_output.get("total")
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
def test_execute_sum_query_instance_types(self):
"""Test that the sum query runs properly."""
self.generator.add_data_to_tenant()
url = "?"
query_params = self.mocked_query_params(url, OCPAzureInstanceTypeView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.ten_day_filter)
total = query_output.get("total")
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
def test_execute_query_current_month_daily(self):
"""Test execute_query for current month on daily breakdown."""
self.generator.add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=daily"
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
def test_execute_query_current_month_by_account(self):
"""Test execute_query for current month on monthly breakdown by account."""
self.generator.add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("subscription_guids")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
self.assertIsInstance(month_item.get("values"), list)
def test_execute_query_current_month_by_service(self):
"""Test execute_query for current month on monthly breakdown by service."""
self.generator.add_data_to_tenant()
valid_services = list(AZURE_SERVICES.keys())
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[service_name]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("service_names")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
name = month_item.get("service_name")
self.assertIn(name, valid_services)
self.assertIsInstance(month_item.get("values"), list)
def test_execute_query_by_filtered_service(self):
"""Test execute_query monthly breakdown by filtered service."""
self.generator.add_data_to_tenant(
fixed_fields=["subscription_guid", "resource_location", "tags", "service_name"]
)
valid_services = list(AZURE_SERVICES.keys())
service = self.generator.config.service_name
url = f"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[service_name]={service}" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
filters = {**self.this_month_filter, "service_name__icontains": service}
for filt in handler._mapper.report_type_map.get("filter"):
if filt:
qf = QueryFilter(**filt)
filters.update({qf.composed_query_string(): qf.parameter})
current_totals = self.get_totals_by_time_scope(aggregates, filters)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("service_names")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
name = month_item.get("service_name")
self.assertIn(name, valid_services)
self.assertIsInstance(month_item.get("values"), list)
def test_execute_query_curr_month_by_subscription_guid_w_limit(self):
"""Test execute_query for current month on monthly breakdown by subscription_guid with limit."""
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=2&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("subscription_guids")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
self.assertEqual(2, len(month_data))
for month_item in month_data:
self.assertIsInstance(month_item.get("subscription_guid"), str)
self.assertIsInstance(month_item.get("values"), list)
def test_execute_query_curr_month_by_subscription_guid_w_order(self):
"""Test execute_query for current month on monthly breakdown by subscription_guid with asc order."""
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&order_by[cost]=asc&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("subscription_guids")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
self.assertEqual(len(month_data), 2)
current_total = 0
for month_item in month_data:
self.assertIsInstance(month_item.get("subscription_guid"), str)
self.assertIsInstance(month_item.get("values"), list)
self.assertIsNotNone(month_item.get("values")[0].get("cost", {}).get("value"))
data_point_total = month_item.get("values")[0].get("cost", {}).get("value")
self.assertLess(current_total, data_point_total)
current_total = data_point_total
def test_execute_query_curr_month_by_subscription_guid_w_order_by_subscription_guid(self):
"""Test execute_query for current month on monthly breakdown by subscription_guid with asc order."""
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&order_by[subscription_guid]=asc&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("subscription_guids")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
self.assertEqual(len(month_data), 2)
current = "0"
for month_item in month_data:
self.assertIsInstance(month_item.get("subscription_guid"), str)
self.assertIsInstance(month_item.get("values"), list)
self.assertIsNotNone(month_item.get("values")[0].get("subscription_guid"))
data_point = month_item.get("values")[0].get("subscription_guid")
if data_point == "1 Other":
continue
self.assertLess(current, data_point)
current = data_point
def test_execute_query_curr_month_by_cluster(self):
"""Test execute_query for current month on monthly breakdown by group_by cluster."""
self.generator.add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[cluster]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("clusters")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
self.assertIsInstance(month_item.get("cluster"), str)
self.assertIsInstance(month_item.get("values"), list)
self.assertIsNotNone(month_item.get("values")[0].get("cost"))
def test_execute_query_by_filtered_cluster(self):
"""Test execute_query monthly breakdown by filtered cluster."""
self.generator.add_data_to_tenant()
cluster = self.generator.cluster_id
url = f"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[cluster]={cluster}" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
filters = {**self.this_month_filter, "cluster_id__icontains": cluster}
for filt in handler._mapper.report_type_map.get("filter"):
if filt:
qf = QueryFilter(**filt)
filters.update({qf.composed_query_string(): qf.parameter})
current_totals = self.get_totals_by_time_scope(aggregates, filters)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("clusters")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
self.assertIsInstance(month_item.get("cluster"), str)
self.assertIsInstance(month_item.get("values"), list)
self.assertIsNotNone(month_item.get("values")[0].get("cost"))
def test_execute_query_curr_month_by_filtered_resource_location(self):
"""Test execute_query for current month on monthly breakdown by filtered resource_location."""
self.generator.add_data_to_tenant()
location = self.generator.config.resource_location
url = f"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[resource_location]={location}" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("resource_locations")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
self.assertIsInstance(month_item.get("resource_location"), str)
self.assertIsInstance(month_item.get("values"), list)
self.assertIsNotNone(month_item.get("values")[0].get("cost"))
def test_execute_query_current_month_filter_subscription_guid(self):
"""Test execute_query for current month on monthly filtered by subscription_guid."""
self.generator.add_data_to_tenant()
guid = self.generator.config.subscription_guid
url = f"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[subscription_guid]={guid}" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("values")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
def test_execute_query_current_month_filter_service(self):
"""Test execute_query for current month on monthly filtered by service."""
self.generator = OCPAzureReportDataGenerator(self.tenant, self.provider, current_month_only=True)
self.generator.add_data_to_tenant(
fixed_fields=["subscription_guid", "resource_location", "tags", "service_name"]
)
service = self.generator.config.service_name
url = f"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[service_name]={service}" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
filters = {**self.this_month_filter, "service_name__icontains": service}
for filt in handler._mapper.report_type_map.get("filter"):
if filt:
qf = QueryFilter(**filt)
filters.update({qf.composed_query_string(): qf.parameter})
current_totals = self.get_totals_by_time_scope(aggregates, filters)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("values")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
def test_execute_query_current_month_filter_resource_location(self):
"""Test execute_query for current month on monthly filtered by resource_location."""
self.generator.add_data_to_tenant()
location = self.generator.config.resource_location
url = f"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[resource_location]={location}" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("values")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
@patch("api.query_params.QueryParameters.accept_type", new_callable=PropertyMock)
def test_execute_query_current_month_filter_resource_location_csv(self, mock_accept):
"""Test execute_query on monthly filtered by resource_location for csv."""
self.generator.add_data_to_tenant()
mock_accept.return_value = "text/csv"
url = f"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[resource_location]={self.generator.config.resource_location}" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
self.assertEqual(len(data), 1)
for data_item in data:
month_val = data_item.get("date", "not-a-date")
self.assertEqual(month_val, cmonth_str)
@patch("api.query_params.QueryParameters.accept_type", new_callable=PropertyMock)
def test_execute_query_curr_month_by_subscription_guid_w_limit_csv(self, mock_accept):
"""Test execute_query for current month on monthly by subscription_guid with limt as csv."""
mock_accept.return_value = "text/csv"
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=2&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertIsNotNone(total.get("cost"))
self.assertEqual(total.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
self.assertEqual(len(data), 2)
for data_item in data:
month = data_item.get("date", "not-a-date")
self.assertEqual(month, cmonth_str)
def test_execute_query_w_delta(self):
"""Test grouped by deltas."""
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
path = reverse("reports-openshift-azure-costs")
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[subscription_guid]=*&delta=cost" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView, path)
handler = OCPAzureReportQueryHandler(query_params)
# test the calculations
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
subs = data[0].get("subscription_guids", [{}])
for sub in subs:
current_total = Decimal(0)
prev_total = Decimal(0)
# fetch the expected sums from the DB.
with tenant_context(self.tenant):
curr = OCPAzureCostLineItemDailySummary.objects.filter(
usage_start__date__gte=self.dh.this_month_start,
usage_start__date__lte=self.dh.today,
subscription_guid=sub.get("subscription_guid"),
).aggregate(value=Sum(F("pretax_cost") + F("markup_cost")))
current_total = Decimal(curr.get("value"))
prev = OCPAzureCostLineItemDailySummary.objects.filter(
usage_start__date__gte=self.dh.last_month_start,
usage_start__date__lte=self.dh.today - relativedelta(months=1),
subscription_guid=sub.get("subscription_guid"),
).aggregate(value=Sum(F("pretax_cost") + F("markup_cost")))
prev_total = Decimal(prev.get("value", Decimal(0)))
expected_delta_value = Decimal(current_total - prev_total)
expected_delta_percent = Decimal((current_total - prev_total) / prev_total * 100)
values = sub.get("values", [{}])[0]
self.assertIn("delta_value", values)
self.assertIn("delta_percent", values)
self.assertEqual(values.get("delta_value", "str"), expected_delta_value)
self.assertEqual(values.get("delta_percent", "str"), expected_delta_percent)
current_total = Decimal(0)
prev_total = Decimal(0)
# fetch the expected sums from the DB.
with tenant_context(self.tenant):
curr = OCPAzureCostLineItemDailySummary.objects.filter(
usage_start__gte=self.dh.this_month_start, usage_start__lte=self.dh.today
).aggregate(value=Sum(F("pretax_cost") + F("markup_cost")))
current_total = Decimal(curr.get("value"))
prev = OCPAzureCostLineItemDailySummary.objects.filter(
usage_start__gte=self.dh.last_month_start, usage_start__lte=self.dh.today - relativedelta(months=1)
).aggregate(value=Sum(F("pretax_cost") + F("markup_cost")))
prev_total = Decimal(prev.get("value"))
expected_delta_value = Decimal(current_total - prev_total)
expected_delta_percent = Decimal((current_total - prev_total) / prev_total * 100)
delta = query_output.get("delta")
self.assertIsNotNone(delta.get("value"))
self.assertIsNotNone(delta.get("percent"))
self.assertEqual(delta.get("value", "str"), expected_delta_value)
self.assertEqual(delta.get("percent", "str"), expected_delta_percent)
def test_execute_query_w_delta_no_previous_data(self):
"""Test deltas with no previous data."""
self.generator = OCPAzureReportDataGenerator(self.tenant, self.provider, current_month_only=True)
self.generator.add_data_to_tenant()
url = "?filter[time_scope_value]=-1&delta=cost"
path = reverse("reports-openshift-azure-costs")
query_params = self.mocked_query_params(url, OCPAzureCostView, path)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
total_cost = query_output.get("total", {}).get("cost", {}).get("value", 1)
delta = query_output.get("delta")
self.assertIsNotNone(delta.get("value"))
self.assertIsNone(delta.get("percent", 0))
self.assertEqual(delta.get("value", 0), total_cost)
def test_execute_query_orderby_delta(self):
"""Test execute_query with ordering by delta ascending."""
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&order_by[delta]=asc&group_by[subscription_guid]=*&delta=cost" # noqa: E501
path = reverse("reports-openshift-azure-costs")
query_params = self.mocked_query_params(url, OCPAzureCostView, path)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
cmonth_str = self.dh.this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("subscription_guids")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
self.assertIsInstance(month_item.get("subscription_guid"), str)
self.assertIsInstance(month_item.get("values"), list)
self.assertIsInstance(month_item.get("values")[0].get("delta_value"), Decimal)
def test_calculate_total(self):
"""Test that calculated totals return correctly."""
self.generator.add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly"
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
expected_units = "USD"
with tenant_context(self.tenant):
result = handler.calculate_total(**{"cost_units": expected_units})
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, self.this_month_filter)
self.assertEqual(result.get("cost", {}).get("value", 0), current_totals.get("cost", 1))
self.assertEqual(result.get("cost", {}).get("units", "not-USD"), expected_units)
def test_percent_delta(self):
"""Test _percent_delta() utility method."""
url = "?"
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
self.assertEqual(handler._percent_delta(10, 5), 100)
def test_rank_list_by_subscription_guid(self):
"""Test rank list limit with subscription_guid alias."""
# No need to fill db
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=2&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
data_list = [
{"subscription_guid": "1", "total": 5, "rank": 1},
{"subscription_guid": "2", "total": 4, "rank": 2},
{"subscription_guid": "3", "total": 3, "rank": 3},
{"subscription_guid": "4", "total": 2, "rank": 4},
]
expected = [
{"subscription_guid": "1", "total": 5, "rank": 1},
{"subscription_guid": "2", "total": 4, "rank": 2},
{
"subscription_guid": "2 Others",
"cost": 0,
"markup_cost": 0,
"derived_cost": 0,
"infrastructure_cost": 0,
"total": 5,
"rank": 3,
},
]
ranked_list = handler._ranked_list(data_list)
self.assertEqual(ranked_list, expected)
def test_rank_list_by_service_name(self):
"""Test rank list limit with service_name grouping."""
# No need to fill db
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=2&group_by[service_name]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
data_list = [
{"service_name": "1", "total": 5, "rank": 1},
{"service_name": "2", "total": 4, "rank": 2},
{"service_name": "3", "total": 3, "rank": 3},
{"service_name": "4", "total": 2, "rank": 4},
]
expected = [
{"service_name": "1", "total": 5, "rank": 1},
{"service_name": "2", "total": 4, "rank": 2},
{
"cost": 0,
"derived_cost": 0,
"infrastructure_cost": 0,
"markup_cost": 0,
"service_name": "2 Others",
"total": 5,
"rank": 3,
},
]
ranked_list = handler._ranked_list(data_list)
self.assertEqual(ranked_list, expected)
def test_rank_list_with_offset(self):
"""Test rank list limit and offset with subscription_guid alias."""
# No need to fill db
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=1&filter[offset]=1&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
data_list = [
{"subscription_guid": "1", "total": 5, "rank": 1},
{"subscription_guid": "2", "total": 4, "rank": 2},
{"subscription_guid": "3", "total": 3, "rank": 3},
{"subscription_guid": "4", "total": 2, "rank": 4},
]
expected = [{"subscription_guid": "2", "total": 4, "rank": 2}]
ranked_list = handler._ranked_list(data_list)
self.assertEqual(ranked_list, expected)
def test_query_costs_with_totals(self):
"""Test execute_query() - costs with totals.
Query for instance_types, validating that cost totals are present.
"""
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[subscription_guid]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
for data_item in data:
subscription_guids = data_item.get("subscription_guids")
for subscription_guid in subscription_guids:
self.assertIsNotNone(subscription_guid.get("values"))
self.assertGreater(len(subscription_guid.get("values")), 0)
for value in subscription_guid.get("values"):
self.assertIsInstance(value.get("cost", {}).get("value"), Decimal)
self.assertGreater(value.get("cost", {}).get("value"), Decimal(0))
def test_query_instance_types_with_totals(self):
"""Test execute_query() - instance types with totals.
Query for instance_types, validating that cost totals are present.
"""
self.generator.add_data_to_tenant()
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant()
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[instance_type]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureInstanceTypeView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
for data_item in data:
instance_types = data_item.get("instance_types")
for it in instance_types:
self.assertIsNotNone(it.get("values"))
self.assertGreater(len(it.get("values")), 0)
for value in it.get("values"):
self.assertIsInstance(value.get("cost", {}).get("value"), Decimal)
self.assertGreaterEqual(
value.get("cost", {}).get("value").quantize(Decimal(".0001"), ROUND_HALF_UP), Decimal(0)
)
# FIXME: usage doesn't have units yet. waiting on MSFT
# self.assertIsInstance(value.get('usage', {}).get('value'), Decimal)
# self.assertGreater(value.get('usage', {}).get('value'), Decimal(0))
self.assertIsInstance(value.get("usage", {}), dict)
self.assertGreaterEqual(
value.get("usage", {}).get("value", {}).quantize(Decimal(".0001"), ROUND_HALF_UP), Decimal(0)
)
def test_query_storage_with_totals(self):
"""Test execute_query() - storage with totals.
Query for storage, validating that cost totals are present.
"""
self.generator.add_data_to_tenant(service_name="Storage")
OCPAzureReportDataGenerator(self.tenant, self.provider).add_data_to_tenant(service_name="Storage")
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[service_name]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAzureStorageView)
handler = OCPAzureReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
for data_item in data:
services = data_item.get("service_names")
self.assertIsNotNone(services)
for srv in services:
self.assertIsNotNone(srv.get("values"))
self.assertGreater(len(srv.get("values")), 0)
for value in srv.get("values"):
self.assertIsInstance(value.get("cost", {}).get("value"), Decimal)
self.assertGreater(value.get("cost", {}).get("value"), Decimal(0))
# FIXME: usage doesn't have units yet. waiting on MSFT
# self.assertIsInstance(value.get('usage', {}).get('value'), Decimal)
# self.assertGreater(value.get('usage', {}).get('value'), Decimal(0))
self.assertIsInstance(value.get("usage", {}), dict)
self.assertGreater(value.get("usage", {}).get("value", {}), Decimal(0))
def test_order_by(self):
"""Test that order_by returns properly sorted data."""
# Do not need to fill db
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly"
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
unordered_data = [
{"date": self.dh.today, "delta_percent": 8, "total": 6.2, "rank": 2},
{"date": self.dh.yesterday, "delta_percent": 4, "total": 2.2, "rank": 1},
{"date": self.dh.today, "delta_percent": 7, "total": 8.2, "rank": 1},
{"date": self.dh.yesterday, "delta_percent": 4, "total": 2.2, "rank": 2},
]
order_fields = ["date", "rank"]
expected = [
{"date": self.dh.yesterday, "delta_percent": 4, "total": 2.2, "rank": 1},
{"date": self.dh.yesterday, "delta_percent": 4, "total": 2.2, "rank": 2},
{"date": self.dh.today, "delta_percent": 7, "total": 8.2, "rank": 1},
{"date": self.dh.today, "delta_percent": 8, "total": 6.2, "rank": 2},
]
ordered_data = handler.order_by(unordered_data, order_fields)
self.assertEqual(ordered_data, expected)
order_fields = ["date", "-delta"]
expected = [
{"date": self.dh.yesterday, "delta_percent": 4, "total": 2.2, "rank": 1},
{"date": self.dh.yesterday, "delta_percent": 4, "total": 2.2, "rank": 2},
{"date": self.dh.today, "delta_percent": 8, "total": 6.2, "rank": 2},
{"date": self.dh.today, "delta_percent": 7, "total": 8.2, "rank": 1},
]
ordered_data = handler.order_by(unordered_data, order_fields)
self.assertEqual(ordered_data, expected)
def test_order_by_null_values(self):
"""Test that order_by returns properly sorted data with null data."""
# Do not need to fill db
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly"
query_params = self.mocked_query_params(url, OCPAzureCostView)
handler = OCPAzureReportQueryHandler(query_params)
unordered_data = [
{"node": None, "cluster": "cluster-1"},
{"node": "alpha", "cluster": "cluster-2"},
{"node": "bravo", "cluster": "cluster-3"},
{"node": "oscar", "cluster": "cluster-4"},
]
order_fields = ["node"]
expected = [
{"node": "alpha", "cluster": "cluster-2"},
{"node": "bravo", "cluster": "cluster-3"},
{"node": "no-node", "cluster": "cluster-1"},
{"node": "oscar", "cluster": "cluster-4"},
]
ordered_data = handler.order_by(unordered_data, order_fields)
self.assertEqual(ordered_data, expected)
class AzureTagsViewTest(IamTestCase):
"""Tests the report view."""
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.dh = DateHelper()
self.ten_days_ago = self.dh.n_days_ago(self.dh.today, 9)
_, self.provider = create_generic_provider(Provider.PROVIDER_AZURE,
self.headers)
self.data_generator = AzureReportDataGenerator(self.tenant,
self.provider)
self.data_generator.add_data_to_tenant()
def test_execute_tags_queries_keys_only(self):
"""Test that tag key data is for the correct time queries."""
test_cases = [{
'value': '-1',
'unit': 'month',
'resolution': 'monthly'
}, {
'value': '-2',
'unit': 'month',
'resolution': 'monthly'
}, {
'value': '-10',
'unit': 'day',
'resolution': 'daily'
}, {
'value': '-30',
'unit': 'day',
'resolution': 'daily'
}]
for case in test_cases:
url = reverse('azure-tags')
client = APIClient()
params = {
'filter[resolution]': case.get('resolution'),
'filter[time_scope_value]': case.get('value'),
'filter[time_scope_units]': case.get('unit'),
'key_only': True
}
url = url + '?' + urlencode(params, quote_via=quote_plus)
response = client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.json().get('data')
self.assertTrue(data)
self.assertTrue(isinstance(data, list))
for tag in data:
self.assertTrue(isinstance(tag, str))
def test_execute_tags_queries(self):
"""Test that tag data is for the correct time queries."""
test_cases = [{
'value': '-1',
'unit': 'month',
'resolution': 'monthly'
}, {
'value': '-2',
'unit': 'month',
'resolution': 'monthly'
}, {
'value': '-10',
'unit': 'day',
'resolution': 'daily'
}, {
'value': '-30',
'unit': 'day',
'resolution': 'daily'
}]
for case in test_cases:
url = reverse('azure-tags')
client = APIClient()
params = {
'filter[resolution]': case.get('resolution'),
'filter[time_scope_value]': case.get('value'),
'filter[time_scope_units]': case.get('unit'),
'key_only': False
}
url = url + '?' + urlencode(params, quote_via=quote_plus)
response = client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.json().get('data')
self.assertTrue(data)
self.assertTrue(isinstance(data, list))
for tag in data:
self.assertTrue(isinstance(tag, dict))
self.assertIn('key', tag)
self.assertIn('values', tag)
self.assertIsNotNone(tag.get('key'))
self.assertIn(tag.get('values').__class__, [list, str])
self.assertTrue(tag.get('values'))
def test_execute_tags_type_queries(self):
"""Test that tag data is for the correct type queries."""
test_cases = [{
'value':
'-1',
'unit':
'month',
'resolution':
'monthly',
'subscription_guid':
self.data_generator.config.subscription_guid
}, {
'value':
'-2',
'unit':
'month',
'resolution':
'monthly',
'subscription_guid':
self.data_generator.config.subscription_guid
}, {
'value':
'-10',
'unit':
'day',
'resolution':
'daily',
'subscription_guid':
self.data_generator.config.subscription_guid
}, {
'value':
'-30',
'unit':
'day',
'resolution':
'daily',
'subscription_guid':
self.data_generator.config.subscription_guid
}]
for case in test_cases:
url = reverse('azure-tags')
client = APIClient()
params = {
'filter[resolution]': case.get('resolution'),
'filter[time_scope_value]': case.get('value'),
'filter[time_scope_units]': case.get('unit'),
'key_only': False,
'filter[subscription_guid]': case.get('subscription_guid')
}
url = url + '?' + urlencode(params, quote_via=quote_plus)
response = client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.json().get('data')
self.assertTrue(data)
self.assertTrue(isinstance(data, list))
for tag in data:
self.assertTrue(isinstance(tag, dict))
self.assertIn('key', tag)
self.assertIn('values', tag)
self.assertIsNotNone(tag.get('key'))
self.assertIn(tag.get('values').__class__, [list, str])
self.assertTrue(tag.get('values'))
def test_execute_query_with_and_filter(self):
"""Test the filter[and:] param in the view."""
AzureReportDataGenerator(self.tenant,
self.provider).add_data_to_tenant()
url = reverse('azure-tags')
client = APIClient()
with tenant_context(self.tenant):
subs = AzureCostEntryLineItemDailySummary.objects\
.filter(usage_start__gte=self.ten_days_ago)\
.values('subscription_guid').distinct()
subscription_guids = [sub.get('subscription_guid') for sub in subs]
params = {
'filter[resolution]': 'daily',
'filter[time_scope_value]': '-10',
'filter[time_scope_units]': 'day',
'filter[and:subscription_guid]': subscription_guids
}
url = url + '?' + urlencode(params, quote_via=quote_plus)
response = client.get(url, **self.headers)
self.assertEqual(response.status_code, status.HTTP_200_OK)
data = response.json().get('data')
self.assertEqual(data, [])
def download_file(self,
key,
stored_etag=None,
manifest_id=None,
start_date=None):
"""
Download a file from GCP storage bucket.
If we have a stored etag and it matches the current GCP blob, we can
safely skip download since the blob/file content must not have changed.
Args:
key (str): name of the blob in the GCP storage bucket
stored_etag (str): optional etag stored in our DB for comparison
Returns:
tuple(str, str) with the local filesystem path to file and GCP's etag.
"""
try:
filename = os.path.splitext(key)[0]
date_range = filename.split("_")[-1]
scan_start, scan_end = date_range.split(":")
last_export_time = self._get_export_time_for_big_query(
scan_start, scan_end, key)
if not last_export_time:
if str(DateAccessor().today().date()) == scan_end:
scan_end = DateAccessor().today().date() + relativedelta(
days=1)
query = f"""
SELECT {self.build_query_select_statement()}
FROM {self.table_name}
WHERE DATE(_PARTITIONTIME) >= '{scan_start}'
AND DATE(_PARTITIONTIME) < '{scan_end}'
"""
else:
query = f"""
SELECT {self.build_query_select_statement()}
FROM {self.table_name}
WHERE DATE(_PARTITIONTIME) >= '{scan_start}'
AND DATE(_PARTITIONTIME) < '{scan_end}'
AND export_time > '{last_export_time}'
"""
client = bigquery.Client()
LOG.info(f"{query}")
query_job = client.query(query)
except GoogleCloudError as err:
err_msg = ("Could not query table for billing information."
f"\n Provider: {self._provider_uuid}"
f"\n Customer: {self.customer_name}"
f"\n Response: {err.message}")
LOG.warning(err_msg)
raise GCPReportDownloaderError(err_msg)
except UnboundLocalError:
err_msg = f"Error recovering start and end date from csv key ({key})."
raise GCPReportDownloaderError(err_msg)
directory_path = self._get_local_directory_path()
full_local_path = self._get_local_file_path(directory_path, key)
os.makedirs(directory_path, exist_ok=True)
msg = f"Downloading {key} to {full_local_path}"
LOG.info(log_json(self.tracing_id, msg, self.context))
try:
with open(full_local_path, "w") as f:
writer = csv.writer(f)
column_list = self.gcp_big_query_columns.copy()
column_list.append("partition_date")
LOG.info(f"writing columns: {column_list}")
writer.writerow(column_list)
for row in query_job:
writer.writerow(row)
except OSError as exc:
err_msg = ("Could not create GCP billing data csv file."
f"\n Provider: {self._provider_uuid}"
f"\n Customer: {self.customer_name}"
f"\n Response: {exc}")
raise GCPReportDownloaderError(err_msg)
msg = f"Returning full_file_path: {full_local_path}"
LOG.info(log_json(self.tracing_id, msg, self.context))
dh = DateHelper()
file_names = create_daily_archives(
self.tracing_id,
self.account,
self._provider_uuid,
key,
full_local_path,
manifest_id,
start_date,
last_export_time,
self.context,
)
return full_local_path, self.etag, dh.today, file_names
class GCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the GCPReportDownloader object."""
def setUp(self):
"""Setup vars for test."""
super().setUp()
self.etag = "1234"
self.today = DateHelper().today
def tearDown(self):
"""Remove files and directories created during the test run."""
super().tearDown()
shutil.rmtree(DATA_DIR, ignore_errors=True)
def create_gcp_downloader_with_mocked_values(
self,
customer_name=FAKE.name(),
dataset=FAKE.slug(),
provider_uuid=uuid4(),
project_id=FAKE.slug(),
table_id=FAKE.slug(),
):
"""
Create a GCPReportDownloader that skips the initial GCP bigquery check creates etag.
This also results in Mock objects being set to instance variables that can be patched
inside other test functions.
Args:
customer_name (str): optional customer name; will be randomly generated if None
bucket_name (str): optional bucket name; will be randomly generated if None
provider_uuid (uuid): optional provider UUID; will be randomly generated if None
Returns:
GCPReportDownloader instance with faked argument data and Mocks in
self.etag.
"""
billing_source = {"table_id": table_id, "dataset": dataset}
credentials = {"project_id": project_id}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
), patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag",
return_value=self.etag,
):
downloader = GCPReportDownloader(
customer_name=customer_name,
data_source=billing_source,
provider_uuid=provider_uuid,
credentials=credentials,
)
return downloader
@patch("masu.external.downloader.gcp.gcp_report_downloader.GCPProvider")
def test_generate_etag_big_query_client_error(self, gcp_provider):
"""Test BigQuery client is handled correctly in generate etag method."""
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
err_msg = "GCP Error"
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.bigquery"
) as bigquery:
bigquery.Client.side_effect = GoogleCloudError(err_msg)
with self.assertRaisesRegex(ReportDownloaderWarning, err_msg):
GCPReportDownloader(
customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials,
)
@patch("masu.external.downloader.gcp.gcp_report_downloader.GCPProvider")
def test_generate_etag(self, gcp_provider):
"""Test BigQuery client is handled correctly in generate etag method."""
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.bigquery"
) as bigquery:
bigquery.Client.return_value.get_table.return_value.modified.return_value = self.today
downloader = GCPReportDownloader(customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials)
self.assertIsNotNone(downloader.etag)
@patch("masu.external.downloader.gcp.gcp_report_downloader.os.makedirs")
@patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery")
def test_download_file_failure_on_file_open(self, mock_bigquery,
mock_makedirs):
"""Assert download_file successful scenario"""
mock_bigquery.client.return_value.query.return_value = ["This", "test"]
key = "202011_1234_2020-12-05:2020-12-08.csv"
downloader = self.create_gcp_downloader_with_mocked_values()
with patch("masu.external.downloader.gcp.gcp_report_downloader.open"
) as mock_open:
err_msg = "bad open"
mock_open.side_effect = IOError(err_msg)
with self.assertRaisesRegex(GCPReportDownloaderError, err_msg):
downloader.download_file(key)
def test_generate_monthly_pseudo_manifest(self):
"""Assert _generate_monthly_pseudo_manifest returns a manifest-like dict."""
provider_uuid = uuid4()
dh = DateHelper()
start_date = dh.this_month_start
invoice_month = start_date.strftime("%Y%m")
expected_assembly_id = ":".join(
[str(provider_uuid), self.etag, invoice_month])
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=provider_uuid)
downloader.scan_end = dh.this_month_end.date()
if self.today.date() < downloader.scan_end:
expected_end_date = self.today.date()
else:
expected_end_date = downloader.scan_end
result_manifest = downloader._generate_monthly_pseudo_manifest(
start_date.date())
expected_files = create_expected_csv_files(dh.this_month_start.date(),
downloader.scan_end,
invoice_month, self.etag)
expected_manifest_data = {
"assembly_id": expected_assembly_id,
"compression": UNCOMPRESSED,
"start_date": start_date.date(),
"end_date": expected_end_date, # inclusive end date
"file_names": expected_files,
}
self.assertEqual(result_manifest, expected_manifest_data)
def test_generate_assembly_id(self):
"""Assert appropriate generation of assembly ID."""
provider_uuid = uuid4()
expected_assembly_id = ":".join([str(provider_uuid), self.etag, "1"])
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=provider_uuid)
assembly_id = downloader._generate_assembly_id("1")
self.assertEqual(assembly_id, expected_assembly_id)
def test_relevant_file_names(self):
"""Assert relevant file name is generated correctly."""
downloader = self.create_gcp_downloader_with_mocked_values()
mock_invoice_month = self.today.strftime("%Y%m")
end_date = downloader.scan_end + relativedelta(days=1)
expected_file_name = [
f"{mock_invoice_month}_{self.etag}_{downloader.scan_start}:{end_date}.csv"
]
result_file_names = downloader._get_relevant_file_names(
mock_invoice_month)
self.assertEqual(expected_file_name, result_file_names)
def test_get_local_file_for_report(self):
"""Assert that get_local_file_for_report is a simple pass-through."""
downloader = self.create_gcp_downloader_with_mocked_values()
report_name = FAKE.file_path()
local_name = downloader.get_local_file_for_report(report_name)
self.assertEqual(local_name, report_name)
@patch("masu.external.downloader.gcp.gcp_report_downloader.os.makedirs")
@patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery")
def test_download_file_success(self, mock_bigquery, mock_makedirs):
"""Assert download_file successful scenario"""
mock_bigquery.client.return_value.query.return_value = ["This", "test"]
key = "202011_1234_2020-12-05:2020-12-08.csv"
mock_name = "mock-test-customer-success"
expected_full_path = f"{DATA_DIR}/{mock_name}/gcp/{key}"
downloader = self.create_gcp_downloader_with_mocked_values(
customer_name=mock_name)
with patch("masu.external.downloader.gcp.gcp_report_downloader.open"):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.create_daily_archives"
):
full_path, etag, date, _ = downloader.download_file(key)
mock_makedirs.assert_called()
self.assertEqual(etag, self.etag)
self.assertEqual(date, self.today)
self.assertEqual(full_path, expected_full_path)
@patch("masu.external.downloader.gcp.gcp_report_downloader.os.makedirs")
@patch("masu.external.downloader.gcp.gcp_report_downloader.bigquery")
def test_download_file_success_end_date_today(self, mock_bigquery,
mock_makedirs):
"""Assert download_file successful scenario"""
mock_bigquery.client.return_value.query.return_value = ["This", "test"]
end_date = DateAccessor().today().date()
key = f"202011_1234_2020-12-05:{end_date}.csv"
mock_name = "mock-test-customer-end-date"
expected_full_path = f"{DATA_DIR}/{mock_name}/gcp/{key}"
downloader = self.create_gcp_downloader_with_mocked_values(
customer_name=mock_name)
with patch("masu.external.downloader.gcp.gcp_report_downloader.open"):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.create_daily_archives"
):
full_path, etag, date, _ = downloader.download_file(key)
mock_makedirs.assert_called()
self.assertEqual(etag, self.etag)
self.assertEqual(date, self.today)
self.assertEqual(full_path, expected_full_path)
@patch("masu.external.downloader.gcp.gcp_report_downloader.open")
def test_download_file_query_client_error(self, mock_open):
"""Test BigQuery client is handled correctly in download file method."""
key = "202011_1234_2020-12-05:2020-12-08.csv"
downloader = self.create_gcp_downloader_with_mocked_values()
err_msg = "GCP Error"
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.bigquery"
) as bigquery:
bigquery.Client.side_effect = GoogleCloudError(err_msg)
with self.assertRaisesRegex(GCPReportDownloaderError, err_msg):
downloader.download_file(key)
@patch("masu.external.downloader.gcp.gcp_report_downloader.GCPProvider")
def test_download_with_unreachable_source(self, gcp_provider):
"""Assert errors correctly when source is unreachable."""
gcp_provider.return_value.cost_usage_source_is_reachable.side_effect = ValidationError
billing_source = {"table_id": FAKE.slug(), "dataset": FAKE.slug()}
credentials = {"project_id": FAKE.slug()}
with self.assertRaises(GCPReportDownloaderError):
GCPReportDownloader(FAKE.name(),
billing_source,
credentials=credentials)
def test_get_manifest_context_for_date(self):
"""Test successful return of get manifest context for date."""
self.maxDiff = None
dh = DateHelper()
start_date = dh.this_month_start
invoice_month = start_date.strftime("%Y%m")
p_uuid = uuid4()
expected_assembly_id = f"{p_uuid}:{self.etag}:{invoice_month}"
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=p_uuid)
if self.today.date() < downloader.scan_end:
expected_end = self.today.date()
else:
expected_end = downloader.scan_end
expected_files = create_expected_csv_files(dh.this_month_start.date(),
expected_end, invoice_month,
self.etag, True)
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._process_manifest_db_record",
return_value=2,
):
report_dict = downloader.get_manifest_context_for_date(
start_date.date())
self.assertEqual(report_dict.get("manifest_id"), 2)
self.assertEqual(report_dict.get("files"), expected_files)
self.assertEqual(report_dict.get("compression"), UNCOMPRESSED)
self.assertEqual(report_dict.get("assembly_id"), expected_assembly_id)
def test_generate_monthly_pseudo_no_manifest(self):
"""Test get monly psuedo manifest with no manifest."""
dh = DateHelper()
downloader = self.create_gcp_downloader_with_mocked_values(
provider_uuid=uuid4())
start_date = dh.last_month_start
manifest_dict = downloader._generate_monthly_pseudo_manifest(
start_date)
self.assertIsNotNone(manifest_dict)
@override_settings(ENABLE_PARQUET_PROCESSING=True)
@patch(
"masu.external.downloader.gcp.gcp_report_downloader.copy_local_report_file_to_s3_bucket"
)
def test_create_daily_archives(self, mock_s3):
"""Test that we load daily files to S3."""
# Use the processor example for data:
file_path = "./koku/masu/test/data/gcp/202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv"
file_name = "202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv"
temp_dir = tempfile.gettempdir()
temp_path = os.path.join(temp_dir, file_name)
shutil.copy2(file_path, temp_path)
expected_daily_files = [
f"{temp_dir}/202011_2020-11-08.csv",
f"{temp_dir}/202011_2020-11-09.csv",
f"{temp_dir}/202011_2020-11-10.csv",
f"{temp_dir}/202011_2020-11-11.csv",
]
start_date = DateHelper().this_month_start
daily_file_names = create_daily_archives("request_id", "account",
self.gcp_provider_uuid,
file_name, temp_path, None,
start_date, None)
mock_s3.assert_called()
self.assertEqual(sorted(daily_file_names),
sorted(expected_daily_files))
for daily_file in expected_daily_files:
self.assertTrue(os.path.exists(daily_file))
os.remove(daily_file)
os.remove(temp_path)
def test_get_dataset_name(self):
"""Test _get_dataset_name helper."""
project_id = FAKE.slug()
dataset_name = FAKE.slug()
datasets = [f"{project_id}:{dataset_name}", dataset_name]
for dataset in datasets:
billing_source = {"table_id": FAKE.slug(), "dataset": dataset}
credentials = {"project_id": project_id}
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
), patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPReportDownloader._generate_etag",
return_value=self.etag,
):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.GCPProvider"
):
downloader = GCPReportDownloader(
customer_name=FAKE.name(),
data_source=billing_source,
provider_uuid=uuid4(),
credentials=credentials,
)
self.assertEqual(downloader._get_dataset_name(), dataset_name)
def setUpClass(cls):
"""Set up the test class."""
super().setUpClass()
cls.dh = DateHelper()
def setUp(self):
"""Setup vars for test."""
super().setUp()
self.etag = "1234"
self.today = DateHelper().today
class OCPReportQueryHandlerTest(IamTestCase):
"""Tests for the OCP report query handler."""
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.dh = DateHelper()
self.this_month_filter = {"usage_start__gte": self.dh.this_month_start}
self.ten_day_filter = {"usage_start__gte": self.dh.n_days_ago(self.dh.today, 9)}
self.thirty_day_filter = {"usage_start__gte": self.dh.n_days_ago(self.dh.today, 29)}
self.last_month_filter = {
"usage_start__gte": self.dh.last_month_start,
"usage_end__lte": self.dh.last_month_end,
}
def get_totals_by_time_scope(self, aggregates, filters=None):
"""Return the total aggregates for a time period."""
if filters is None:
filters = self.ten_day_filter
with tenant_context(self.tenant):
return OCPUsageLineItemDailySummary.objects.filter(**filters).aggregate(**aggregates)
def get_totals_costs_by_time_scope(self, aggregates, filters=None):
"""Return the total costs aggregates for a time period."""
if filters is None:
filters = self.this_month_filter
with tenant_context(self.tenant):
return OCPUsageLineItemDailySummary.objects.filter(**filters).aggregate(**aggregates)
def test_execute_sum_query(self):
"""Test that the sum query runs properly."""
url = "?"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertEqual(total.get("usage", {}).get("value"), current_totals.get("usage"))
self.assertEqual(total.get("request", {}).get("value"), current_totals.get("request"))
self.assertEqual(total.get("cost", {}).get("value"), current_totals.get("cost"))
self.assertEqual(total.get("limit", {}).get("value"), current_totals.get("limit"))
def test_execute_sum_query_costs(self):
"""Test that the sum query runs properly for the costs endpoint."""
url = "?"
query_params = self.mocked_query_params(url, OCPCostView)
handler = OCPReportQueryHandler(query_params)
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_costs_by_time_scope(aggregates, self.ten_day_filter)
expected_cost_total = current_totals.get("cost_total")
self.assertIsNotNone(expected_cost_total)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
result_cost_total = total.get("cost", {}).get("total", {}).get("value")
self.assertIsNotNone(result_cost_total)
self.assertEqual(result_cost_total, expected_cost_total)
def test_get_cluster_capacity_monthly_resolution(self):
"""Test that cluster capacity returns a full month's capacity."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = [{"row": 1}]
query_data, total_capacity = handler.get_cluster_capacity(query_data)
self.assertTrue("capacity" in total_capacity)
self.assertTrue(isinstance(total_capacity["capacity"], Decimal))
self.assertTrue("capacity" in query_data[0])
self.assertIsNotNone(query_data[0].get("capacity"))
self.assertIsNotNone(total_capacity.get("capacity"))
self.assertEqual(query_data[0].get("capacity"), total_capacity.get("capacity"))
def test_get_cluster_capacity_monthly_resolution_group_by_cluster(self):
"""Test that cluster capacity returns capacity by cluster."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[cluster]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
capacity_by_cluster = defaultdict(Decimal)
total_capacity = Decimal(0)
query_filter = handler.query_filter
query_group_by = ["usage_start", "cluster_id"]
annotations = {"capacity": Max("cluster_capacity_cpu_core_hours")}
cap_key = list(annotations.keys())[0]
q_table = handler._mapper.provider_map.get("tables").get("query")
query = q_table.objects.filter(query_filter)
with tenant_context(self.tenant):
cap_data = query.values(*query_group_by).annotate(**annotations)
for entry in cap_data:
cluster_id = entry.get("cluster_id", "")
capacity_by_cluster[cluster_id] += entry.get(cap_key, 0)
total_capacity += entry.get(cap_key, 0)
for entry in query_data.get("data", []):
for cluster in entry.get("clusters", []):
cluster_name = cluster.get("cluster", "")
capacity = cluster.get("values")[0].get("capacity", {}).get("value")
self.assertEqual(capacity, capacity_by_cluster[cluster_name])
self.assertEqual(query_data.get("total", {}).get("capacity", {}).get("value"), total_capacity)
def test_get_cluster_capacity_daily_resolution(self):
"""Test that total capacity is returned daily resolution."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=daily"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
daily_capacity = defaultdict(Decimal)
total_capacity = Decimal(0)
query_filter = handler.query_filter
query_group_by = ["usage_start", "cluster_id"]
annotations = {"capacity": Max("cluster_capacity_cpu_core_hours")}
cap_key = list(annotations.keys())[0]
q_table = handler._mapper.provider_map.get("tables").get("query")
query = q_table.objects.filter(query_filter)
with tenant_context(self.tenant):
cap_data = query.values(*query_group_by).annotate(**annotations)
for entry in cap_data:
date = handler.date_to_string(entry.get("usage_start"))
daily_capacity[date] += entry.get(cap_key, 0)
cap_data = query.values(*query_group_by).annotate(**annotations)
for entry in cap_data:
total_capacity += entry.get(cap_key, 0)
self.assertEqual(query_data.get("total", {}).get("capacity", {}).get("value"), total_capacity)
for entry in query_data.get("data", []):
date = entry.get("date")
values = entry.get("values")
if values:
capacity = values[0].get("capacity", {}).get("value")
self.assertEqual(capacity, daily_capacity[date])
def test_get_cluster_capacity_daily_resolution_group_by_clusters(self):
"""Test that cluster capacity returns daily capacity by cluster."""
url = (
"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=daily&group_by[cluster]=*"
)
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
daily_capacity_by_cluster = defaultdict(dict)
total_capacity = Decimal(0)
query_filter = handler.query_filter
query_group_by = ["usage_start", "cluster_id"]
annotations = {"capacity": Max("cluster_capacity_cpu_core_hours")}
cap_key = list(annotations.keys())[0]
q_table = handler._mapper.query_table
query = q_table.objects.filter(query_filter)
with tenant_context(self.tenant):
cap_data = query.values(*query_group_by).annotate(**annotations)
for entry in cap_data:
date = handler.date_to_string(entry.get("usage_start"))
cluster_id = entry.get("cluster_id", "")
if cluster_id in daily_capacity_by_cluster[date]:
daily_capacity_by_cluster[date][cluster_id] += entry.get(cap_key, 0)
else:
daily_capacity_by_cluster[date][cluster_id] = entry.get(cap_key, 0)
total_capacity += entry.get(cap_key, 0)
for entry in query_data.get("data", []):
date = entry.get("date")
for cluster in entry.get("clusters", []):
cluster_name = cluster.get("cluster", "")
capacity = cluster.get("values")[0].get("capacity", {}).get("value")
self.assertEqual(capacity, daily_capacity_by_cluster[date][cluster_name])
self.assertEqual(query_data.get("total", {}).get("capacity", {}).get("value"), total_capacity)
@patch("api.report.ocp.query_handler.ReportQueryHandler.add_deltas")
@patch("api.report.ocp.query_handler.OCPReportQueryHandler.add_current_month_deltas")
def test_add_deltas_current_month(self, mock_current_deltas, mock_deltas):
"""Test that the current month method is called for deltas."""
url = "?"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
handler._delta = "usage__request"
handler.add_deltas([], [])
mock_current_deltas.assert_called()
mock_deltas.assert_not_called()
@patch("api.report.ocp.query_handler.ReportQueryHandler.add_deltas")
@patch("api.report.ocp.query_handler.OCPReportQueryHandler.add_current_month_deltas")
def test_add_deltas_super_delta(self, mock_current_deltas, mock_deltas):
"""Test that the super delta method is called for deltas."""
url = "?"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
handler._delta = "usage"
handler.add_deltas([], [])
mock_current_deltas.assert_not_called()
mock_deltas.assert_called()
def test_add_current_month_deltas(self):
"""Test that current month deltas are calculated."""
url = "?"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
handler._delta = "usage__request"
q_table = handler._mapper.provider_map.get("tables").get("query")
with tenant_context(self.tenant):
query = q_table.objects.filter(handler.query_filter)
query_data = query.annotate(**handler.annotations)
group_by_value = handler._get_group_by()
query_group_by = ["date"] + group_by_value
query_order_by = ("-date",)
query_order_by += (handler.order,)
annotations = handler.report_annotations
query_data = query_data.values(*query_group_by).annotate(**annotations)
aggregates = handler._mapper.report_type_map.get("aggregates")
metric_sum = query.aggregate(**aggregates)
query_sum = {key: metric_sum.get(key) for key in aggregates}
result = handler.add_current_month_deltas(query_data, query_sum)
delta_field_one, delta_field_two = handler._delta.split("__")
field_one_total = Decimal(0)
field_two_total = Decimal(0)
for entry in result:
field_one_total += entry.get(delta_field_one, 0)
field_two_total += entry.get(delta_field_two, 0)
delta_percent = entry.get("delta_percent")
expected = (
(entry.get(delta_field_one, 0) / entry.get(delta_field_two, 0) * 100)
if entry.get(delta_field_two)
else 0
)
self.assertEqual(delta_percent, expected)
expected_total = field_one_total / field_two_total * 100 if field_two_total != 0 else 0
self.assertEqual(handler.query_delta.get("percent"), expected_total)
def test_add_current_month_deltas_no_previous_data_w_query_data(self):
"""Test that current month deltas are calculated with no previous data for field two."""
url = "?filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=1"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
handler._delta = "usage__foo"
q_table = handler._mapper.provider_map.get("tables").get("query")
with tenant_context(self.tenant):
query = q_table.objects.filter(handler.query_filter)
query_data = query.annotate(**handler.annotations)
group_by_value = handler._get_group_by()
query_group_by = ["date"] + group_by_value
query_order_by = ("-date",)
query_order_by += (handler.order,)
annotations = annotations = handler.report_annotations
query_data = query_data.values(*query_group_by).annotate(**annotations)
aggregates = handler._mapper.report_type_map.get("aggregates")
metric_sum = query.aggregate(**aggregates)
query_sum = {key: metric_sum.get(key) if metric_sum.get(key) else Decimal(0) for key in aggregates}
result = handler.add_current_month_deltas(query_data, query_sum)
self.assertEqual(result, query_data)
self.assertIsNotNone(handler.query_delta["value"])
self.assertIsNone(handler.query_delta["percent"])
def test_get_tag_filter_keys(self):
"""Test that filter params with tag keys are returned."""
url = "?"
query_params = self.mocked_query_params(url, OCPTagView)
handler = OCPTagQueryHandler(query_params)
tag_keys = handler.get_tag_keys(filters=False)
url = f"?filter[tag:{tag_keys[0]}]=*"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
results = handler.get_tag_filter_keys()
self.assertEqual(results, ["tag:" + tag_keys[0]])
def test_get_tag_group_by_keys(self):
"""Test that group_by params with tag keys are returned."""
url = "?"
query_params = self.mocked_query_params(url, OCPTagView)
handler = OCPTagQueryHandler(query_params)
tag_keys = handler.get_tag_keys(filters=False)
group_by_key = tag_keys[0]
url = f"?group_by[tag:{group_by_key}]=*"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
results = handler.get_tag_group_by_keys()
self.assertEqual(results, ["tag:" + group_by_key])
def test_set_tag_filters(self):
"""Test that tag filters are created properly."""
filters = QueryFilterCollection()
url = "?"
query_params = self.mocked_query_params(url, OCPTagView)
handler = OCPTagQueryHandler(query_params)
tag_keys = handler.get_tag_keys(filters=False)
filter_key = tag_keys[0]
filter_value = "filter"
group_by_key = tag_keys[1]
group_by_value = "group_By"
url = f"?filter[tag:{filter_key}]={filter_value}&group_by[tag:{group_by_key}]={group_by_value}"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
filters = handler._set_tag_filters(filters)
expected = f"""<class 'api.query_filter.QueryFilterCollection'>: (AND: ('pod_labels__{filter_key}__icontains', '{filter_value}')), (AND: ('pod_labels__{group_by_key}__icontains', '{group_by_value}')), """ # noqa: E501
self.assertEqual(repr(filters), expected)
def test_get_tag_group_by(self):
"""Test that tag based group bys work."""
url = "?"
query_params = self.mocked_query_params(url, OCPTagView)
handler = OCPTagQueryHandler(query_params)
tag_keys = handler.get_tag_keys(filters=False)
group_by_key = tag_keys[0]
group_by_value = "group_by"
url = f"?group_by[tag:{group_by_key}]={group_by_value}"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
group_by = handler._get_tag_group_by()
group = group_by[0]
expected = "pod_labels__" + group_by_key
self.assertEqual(len(group_by), 1)
self.assertEqual(group[0], expected)
def test_get_tag_order_by(self):
"""Verify that a propery order by is returned."""
tag = "pod_labels__key"
expected_param = (tag.split("__")[1],)
url = "?"
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
result = handler.get_tag_order_by(tag)
expression = result.expression
self.assertIsInstance(result, OrderBy)
self.assertEqual(expression.sql, "pod_labels -> %s")
self.assertEqual(expression.params, expected_param)
def test_filter_by_infrastructure_ocp_on_aws(self):
"""Test that filter by infrastructure for ocp on aws."""
url = "?filter[resolution]=monthly&filter[time_scope_value]=-1&filter[time_scope_units]=month&filter[infrastructures]=aws" # noqa: E501
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
self.assertTrue(query_data.get("data")) # check that returned list is not empty
for entry in query_data.get("data"):
self.assertTrue(entry.get("values"))
for value in entry.get("values"):
self.assertIsNotNone(value.get("usage").get("value"))
self.assertIsNotNone(value.get("request").get("value"))
def test_filter_by_infrastructure_ocp_on_azure(self):
"""Test that filter by infrastructure for ocp on azure."""
url = "?filter[resolution]=monthly&filter[time_scope_value]=-1&filter[time_scope_units]=month&filter[infrastructures]=azure" # noqa: E501
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
self.assertTrue(query_data.get("data")) # check that returned list is not empty
for entry in query_data.get("data"):
self.assertTrue(entry.get("values"))
for value in entry.get("values"):
self.assertIsNotNone(value.get("usage").get("value"))
self.assertIsNotNone(value.get("request").get("value"))
def test_filter_by_infrastructure_ocp(self):
"""Test that filter by infrastructure for ocp not on aws."""
url = "?filter[resolution]=monthly&filter[time_scope_value]=-1&filter[time_scope_units]=month&filter[cluster]=OCP-On-Azure&filter[infrastructures]=aws" # noqa: E501
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
self.assertTrue(query_data.get("data")) # check that returned list is not empty
for entry in query_data.get("data"):
for value in entry.get("values"):
self.assertEqual(value.get("usage").get("value"), 0)
self.assertEqual(value.get("request").get("value"), 0)
def test_order_by_null_values(self):
"""Test that order_by returns properly sorted data with null data."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly" # noqa: E501
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
unordered_data = [
{"node": None, "cluster": "cluster-1"},
{"node": "alpha", "cluster": "cluster-2"},
{"node": "bravo", "cluster": "cluster-3"},
{"node": "oscar", "cluster": "cluster-4"},
]
order_fields = ["node"]
expected = [
{"node": "alpha", "cluster": "cluster-2"},
{"node": "bravo", "cluster": "cluster-3"},
{"node": "no-node", "cluster": "cluster-1"},
{"node": "oscar", "cluster": "cluster-4"},
]
ordered_data = handler.order_by(unordered_data, order_fields)
self.assertEqual(ordered_data, expected)
def test_ocp_cpu_query_group_by_cluster(self):
"""Test that group by cluster includes cluster and cluster_alias."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=3&group_by[cluster]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
for data in query_data.get("data"):
self.assertIn("clusters", data)
for cluster_data in data.get("clusters"):
self.assertIn("cluster", cluster_data)
self.assertIn("values", cluster_data)
for cluster_value in cluster_data.get("values"):
# cluster_value is a dictionary
self.assertIn("cluster", cluster_value.keys())
self.assertIn("clusters", cluster_value.keys())
self.assertIsNotNone(cluster_value["cluster"])
self.assertIsNotNone(cluster_value["clusters"])
def test_other_clusters(self):
"""Test that group by cluster includes cluster and cluster_alias."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=1&group_by[cluster]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
for data in query_data.get("data"):
for cluster_data in data.get("clusters"):
cluster_name = cluster_data.get("cluster", "")
if cluster_name == "Other":
for cluster_value in cluster_data.get("values"):
self.assertTrue(len(cluster_value.get("clusters", [])) == 1)
self.assertTrue(len(cluster_value.get("source_uuid", [])) == 1)
elif cluster_name == "Others":
for cluster_value in cluster_data.get("values"):
self.assertTrue(len(cluster_value.get("clusters", [])) > 1)
self.assertTrue(len(cluster_value.get("source_uuid", [])) > 1)
def test_subtotals_add_up_to_total(self):
"""Test the apply_group_by handles different grouping scenerios."""
group_by_list = [
("project", "cluster", "node"),
("project", "node", "cluster"),
("cluster", "project", "node"),
("cluster", "node", "project"),
("node", "cluster", "project"),
("node", "project", "cluster"),
]
base_url = (
"?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&filter[limit]=3"
) # noqa: E501
tolerance = 1
for group_by in group_by_list:
sub_url = "&group_by[%s]=*&group_by[%s]=*&group_by[%s]=*" % group_by
url = base_url + sub_url
query_params = self.mocked_query_params(url, OCPCpuView)
handler = OCPReportQueryHandler(query_params)
query_data = handler.execute_query()
the_sum = handler.query_sum
data = query_data["data"][0]
result_cost, result_infra, result_sup = _calculate_subtotals(data, [], [], [])
test_dict = {
"cost": {
"expected": the_sum.get("cost", {}).get("total", {}).get("value"),
"result": sum(result_cost),
},
"infra": {
"expected": the_sum.get("infrastructure", {}).get("total", {}).get("value"),
"result": sum(result_infra),
},
"sup": {
"expected": the_sum.get("supplementary", {}).get("total", {}).get("value"),
"result": sum(result_sup),
},
}
for _, data in test_dict.items():
expected = data["expected"]
result = data["result"]
self.assertIsNotNone(expected)
self.assertIsNotNone(result)
self.assertLessEqual(abs(expected - result), tolerance)
def test_source_uuid_mapping(self): # noqa: C901
"""Test source_uuid is mapped to the correct source."""
endpoints = [OCPCostView, OCPCpuView, OCPVolumeView, OCPMemoryView]
with tenant_context(self.tenant):
expected_source_uuids = list(OCPUsageReportPeriod.objects.all().values_list("provider_id", flat=True))
source_uuid_list = []
for endpoint in endpoints:
urls = ["?", "?group_by[project]=*"]
if endpoint == OCPCostView:
urls.append("?group_by[node]=*")
for url in urls:
query_params = self.mocked_query_params(url, endpoint)
handler = OCPReportQueryHandler(query_params)
query_output = handler.execute_query()
for dictionary in query_output.get("data"):
for _, value in dictionary.items():
if isinstance(value, list):
for item in value:
if isinstance(item, dict):
if "values" in item.keys():
self.assertEqual(len(item["values"]), 1)
value = item["values"][0]
source_uuid_list.extend(value.get("source_uuid"))
self.assertNotEquals(source_uuid_list, [])
for source_uuid in source_uuid_list:
self.assertIn(source_uuid, expected_source_uuids)
def test_group_by_project_w_limit(self):
"""COST-1252: Test that grouping by project with limit works as expected."""
url = "?group_by[project]=*&order_by[project]=asc&filter[limit]=2" # noqa: E501
query_params = self.mocked_query_params(url, OCPCostView)
handler = OCPReportQueryHandler(query_params)
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_costs_by_time_scope(aggregates, self.ten_day_filter)
expected_cost_total = current_totals.get("cost_total")
self.assertIsNotNone(expected_cost_total)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
result_cost_total = total.get("cost", {}).get("total", {}).get("value")
self.assertIsNotNone(result_cost_total)
self.assertEqual(result_cost_total, expected_cost_total)
class OCPAWSQueryHandlerTest(IamTestCase):
"""Tests for the OCP report query handler."""
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.dh = DateHelper()
self.this_month_filter = {"usage_start__gte": self.dh.this_month_start}
self.ten_day_filter = {
"usage_start__gte": self.dh.n_days_ago(self.dh.today, 9)
}
self.thirty_day_filter = {
"usage_start__gte": self.dh.n_days_ago(self.dh.today, 29)
}
self.last_month_filter = {
"usage_start__gte": self.dh.last_month_start,
"usage_end__lte": self.dh.last_month_end,
}
with tenant_context(self.tenant):
self.services = OCPAWSCostLineItemDailySummary.objects.values(
"product_code").distinct()
self.services = [
entry.get("product_code") for entry in self.services
]
def get_totals_by_time_scope(self, aggregates, filters=None):
"""Return the total aggregates for a time period."""
if filters is None:
filters = self.ten_day_filter
with tenant_context(self.tenant):
return OCPAWSCostLineItemDailySummary.objects.filter(
**filters).aggregate(**aggregates)
def test_execute_sum_query_storage(self):
"""Test that the sum query runs properly."""
url = "?"
query_params = self.mocked_query_params(url, OCPAWSStorageView)
handler = OCPAWSReportQueryHandler(query_params)
filt = {"product_family__contains": "Storage"}
filt.update(self.ten_day_filter)
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, filt)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
def test_execute_query_current_month_daily(self):
"""Test execute_query for current month on daily breakdown."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=daily"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates,
self.this_month_filter)
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
def test_execute_query_current_month_monthly(self):
"""Test execute_query for current month on monthly breakdown."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=daily"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
self.assertIsNotNone(query_output.get("data"))
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates,
self.this_month_filter)
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
def test_execute_query_current_month_by_service(self):
"""Test execute_query for current month on monthly breakdown by service."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[service]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates,
self.this_month_filter)
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date")
month_data = data_item.get("services")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
service = month_item.get("service")
self.assertIn(service, self.services)
self.assertIsInstance(month_item.get("values"), list)
def test_execute_query_by_filtered_service(self):
"""Test execute_query monthly breakdown by filtered service."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[service]=AmazonEC2" # noqa: E501
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
filt = copy.deepcopy(self.this_month_filter)
filt["product_code"] = "AmazonEC2"
current_totals = self.get_totals_by_time_scope(aggregates, filt)
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date")
month_data = data_item.get("services")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
compute = month_item.get("service")
self.assertEqual(compute, "AmazonEC2")
self.assertIsInstance(month_item.get("values"), list)
def test_query_by_partial_filtered_service(self):
"""Test execute_query monthly breakdown by filtered service."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[service]=eC2" # noqa: E501
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
filt = copy.deepcopy(self.this_month_filter)
filt["product_code__icontains"] = "ec2"
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates, filt)
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date")
month_data = data_item.get("services")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
compute = month_item.get("service")
self.assertEqual(compute, "AmazonEC2")
self.assertIsInstance(month_item.get("values"), list)
def test_execute_query_current_month_by_account(self):
"""Test execute_query for current month on monthly breakdown by account."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[account]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates,
self.this_month_filter)
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("accounts", "not-a-list")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
self.assertIsInstance(month_item.get("values"), list)
def test_execute_query_by_account_by_service(self):
"""Test execute_query for current month breakdown by account by service."""
url = "?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=monthly&group_by[account]=*&group_by[service]=*" # noqa: E501
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
data = query_output.get("data")
self.assertIsNotNone(data)
self.assertIsNotNone(query_output.get("total"))
total = query_output.get("total")
self.assertIsNotNone(total.get("cost"))
aggregates = handler._mapper.report_type_map.get("aggregates")
current_totals = self.get_totals_by_time_scope(aggregates,
self.this_month_filter)
self.assertEqual(
total.get("cost", {}).get("total", {}).get("value", 0),
current_totals.get("cost_total", 1))
cmonth_str = DateHelper().this_month_start.strftime("%Y-%m")
for data_item in data:
month_val = data_item.get("date", "not-a-date")
month_data = data_item.get("accounts", "not-a-string")
self.assertEqual(month_val, cmonth_str)
self.assertIsInstance(month_data, list)
for month_item in month_data:
self.assertIsInstance(month_item.get("services"), list)
def test_check_view_filter_and_group_by_criteria(self):
"""Test that all filter and group by checks return the correct result."""
good_group_by_options = [
"account", "service", "region", "cluster", "product_family"
]
bad_group_by_options = ["project", "node"]
for option in good_group_by_options:
filter_keys = {option}
group_by_keys = set()
self.assertTrue(
check_view_filter_and_group_by_criteria(
filter_keys, group_by_keys))
filter_keys = set()
group_by_keys = {option}
self.assertTrue(
check_view_filter_and_group_by_criteria(
filter_keys, group_by_keys))
# Different group by and filter
filter_keys = {"account"}
group_by_keys = {"cluster"}
self.assertTrue(
check_view_filter_and_group_by_criteria(filter_keys,
group_by_keys))
# Multiple group bys
filter_keys = set()
group_by_keys = {"cluster", "account"}
self.assertTrue(
check_view_filter_and_group_by_criteria(filter_keys,
group_by_keys))
# Multiple filters
filter_keys = {"cluster", "account"}
group_by_keys = set()
self.assertTrue(
check_view_filter_and_group_by_criteria(filter_keys,
group_by_keys))
# Project and node unsupported
for option in bad_group_by_options:
filter_keys = {option}
group_by_keys = set()
self.assertFalse(
check_view_filter_and_group_by_criteria(
filter_keys, group_by_keys))
filter_keys = set()
group_by_keys = {option}
self.assertFalse(
check_view_filter_and_group_by_criteria(
filter_keys, group_by_keys))
def test_query_table(self):
"""Test that the correct view is assigned by query table property."""
url = "?"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSCostSummary)
url = "?group_by[account]=*"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSCostSummaryByAccount)
url = "?group_by[region]=*"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSCostSummaryByRegion)
url = "?group_by[region]=*&group_by[account]=*"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSCostSummaryByRegion)
url = "?group_by[service]=*"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSCostSummaryByService)
url = "?group_by[service]=*&group_by[account]=*"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSCostSummaryByService)
url = "?"
query_params = self.mocked_query_params(url, OCPAWSInstanceTypeView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSComputeSummary)
url = "?group_by[account]=*"
query_params = self.mocked_query_params(url, OCPAWSInstanceTypeView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSComputeSummary)
url = "?"
query_params = self.mocked_query_params(url, OCPAWSStorageView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSStorageSummary)
url = "?group_by[account]=*"
query_params = self.mocked_query_params(url, OCPAWSStorageView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSStorageSummary)
url = "?filter[service]=AmazonVPC,AmazonCloudFront,AmazonRoute53,AmazonAPIGateway"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSNetworkSummary)
url = "?filter[service]=AmazonVPC,AmazonCloudFront,AmazonRoute53,AmazonAPIGateway&group_by[account]=*"
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSNetworkSummary)
url = (
"?filter[service]=AmazonRDS,AmazonDynamoDB,AmazonElastiCache,AmazonNeptune,AmazonRedshift,AmazonDocumentDB"
)
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSDatabaseSummary)
url = (
"?filter[service]=AmazonRDS,AmazonDynamoDB,AmazonElastiCache,AmazonNeptune,AmazonRedshift,AmazonDocumentDB"
"&group_by[account]=*")
query_params = self.mocked_query_params(url, OCPAWSCostView)
handler = OCPAWSReportQueryHandler(query_params)
self.assertEqual(handler.query_table, OCPAWSDatabaseSummary)
def test_source_uuid_mapping(self): # noqa: C901
"""Test source_uuid is mapped to the correct source."""
endpoints = [OCPAWSCostView, OCPAWSInstanceTypeView, OCPAWSStorageView]
with tenant_context(self.tenant):
expected_source_uuids = list(
AWSCostEntryBill.objects.distinct().values_list("provider_id",
flat=True))
source_uuid_list = []
for endpoint in endpoints:
urls = ["?"]
if endpoint == OCPAWSCostView:
urls.extend([
"?group_by[account]=*", "?group_by[service]=*",
"?group_by[region]=*"
])
for url in urls:
query_params = self.mocked_query_params(url, endpoint)
handler = OCPAWSReportQueryHandler(query_params)
query_output = handler.execute_query()
for dictionary in query_output.get("data"):
for _, value in dictionary.items():
if isinstance(value, list):
for item in value:
if isinstance(item, dict):
if "values" in item.keys():
value = item["values"][0]
source_uuid_list.extend(
value.get("source_uuid"))
self.assertNotEquals(source_uuid_list, [])
for source_uuid in source_uuid_list:
self.assertIn(source_uuid, expected_source_uuids)
class NiseDataLoader:
"""Loads nise generated test data for different source types."""
def __init__(self, schema, num_days=10):
"""Initialize the data loader."""
self.dh = DateHelper()
self.schema = schema
self.nise_data_path = Config.TMP_DIR
if not os.path.exists(self.nise_data_path):
os.makedirs(self.nise_data_path)
self.dates = self.get_test_data_dates(num_days)
def get_test_data_dates(self, num_days):
"""Return a list of tuples with dates for nise data."""
end_date = self.dh.today
if end_date.day == 1:
end_date += relativedelta(days=1)
n_days_ago = self.dh.n_days_ago(end_date, num_days)
start_date = n_days_ago
if self.dh.this_month_start > n_days_ago:
start_date = self.dh.this_month_start
prev_month_start = start_date - relativedelta(months=1)
prev_month_end = end_date - relativedelta(months=1)
days_of_data = prev_month_end.day - prev_month_start.day
if days_of_data < num_days:
extra_days = num_days - days_of_data
prev_month_end = prev_month_end + relativedelta(days=extra_days)
return [
(prev_month_start, prev_month_end, self.dh.last_month_start),
(start_date, end_date, self.dh.this_month_start),
]
def prepare_template(self, provider_type, static_data_file):
"""Prepare the Jinja template for static data."""
static_data = pkgutil.get_data("api.report.test", static_data_file)
template = Template(static_data.decode("utf8"))
static_data_path = f"/tmp/{provider_type}_static_data.yml"
return template, static_data_path
def build_report_path(self, provider_type, bill_date, base_path):
"""Build a path to report files."""
this_month_str = bill_date.strftime("%Y%m%d")
next_month = bill_date + relativedelta(months=1)
if provider_type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
next_month = next_month - relativedelta(days=1)
next_month_str = next_month.strftime("%Y%m%d")
return f"{base_path}/{this_month_str}-{next_month_str}"
def process_report(self, report, compression, provider_type, provider,
manifest):
"""Run the report processor on a report."""
status = baker.make("CostUsageReportStatus",
manifest=manifest,
report_name=report)
status.last_started_datetime = self.dh.now
ReportProcessor(self.schema, report, compression, provider_type,
provider.uuid, manifest.id).process()
status.last_completed_datetime = self.dh.now
status.save()
def load_openshift_data(self, customer, static_data_file, cluster_id):
"""Load OpenShift data into the database."""
provider_type = Provider.PROVIDER_OCP
credentials = {"cluster_id": cluster_id}
with override_settings(AUTO_DATA_INGEST=False):
ocp_billing_source, _ = ProviderBillingSource.objects.get_or_create(
data_source={})
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source=ocp_billing_source,
customer=customer,
)
template, static_data_path = self.prepare_template(
provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"insights_upload": self.nise_data_path,
"ocp_cluster_id": cluster_id,
}
base_path = f"{self.nise_data_path}/{cluster_id}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
num_total_files=3,
)
with open(static_data_path, "w") as f:
f.write(
template.render(start_date=start_date, end_date=end_date))
run(provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date,
base_path)
for report in os.scandir(report_path):
shutil.move(report.path, f"{base_path}/{report.name}")
for report in [f.path for f in os.scandir(base_path)]:
if os.path.isdir(report):
continue
elif "manifest" in report.lower():
continue
self.process_report(report, "PLAIN", provider_type, provider,
manifest)
with patch("masu.processor.tasks.chain"):
update_summary_tables(
self.schema,
provider_type,
provider.uuid,
start_date,
end_date,
manifest_id=manifest.id,
synchronous=True,
)
update_cost_model_costs.s(self.schema,
provider.uuid,
self.dh.last_month_start,
self.dh.today,
synchronous=True).apply()
refresh_materialized_views.s(self.schema,
provider_type,
provider_uuid=provider.uuid,
synchronous=True).apply()
shutil.rmtree(report_path, ignore_errors=True)
def load_aws_data(self,
customer,
static_data_file,
account_id=None,
role_arn=None,
day_list=None):
"""Load AWS data into the database."""
provider_type = Provider.PROVIDER_AWS_LOCAL
if account_id is None:
account_id = "9999999999999"
if role_arn is None:
role_arn = "arn:aws:iam::999999999999:role/CostManagement"
nise_provider_type = provider_type.replace("-local", "")
report_name = "Test"
credentials = {"role_arn": role_arn}
data_source = {"bucket": "test-bucket"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
# chicken/egg probrem. I need the provider_uuid to upload aws org unit tree
# but the tree needs to be created first in order to populate the org unit
# foreign key on the daily summary table.
if day_list:
org_tree_obj = InsertAwsOrgTree(schema=self.schema,
provider_uuid=provider.uuid,
start_date=self.dates[0][0])
org_tree_obj.insert_tree(day_list=day_list)
template, static_data_path = self.prepare_template(
provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"aws_report_name": report_name,
"aws_bucket_name": self.nise_data_path,
}
base_path = f"{self.nise_data_path}/{report_name}"
with schema_context(self.schema):
baker.make("AWSAccountAlias",
account_id=account_id,
account_alias="Test Account")
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(
template.render(start_date=start_date,
end_date=end_date,
account_id=account_id))
run(nise_provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date,
base_path)
for report in os.scandir(report_path):
if os.path.isdir(report):
for report in [
f.path
for f in os.scandir(f"{report_path}/{report.name}")
]:
if os.path.isdir(report):
continue
elif "manifest" in report.lower():
continue
self.process_report(report, "GZIP", provider_type,
provider, manifest)
with patch("masu.processor.tasks.chain"), patch.object(
settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema,
provider_type,
provider.uuid,
start_date,
end_date,
manifest_id=manifest.id,
synchronous=True,
)
update_cost_model_costs.s(self.schema,
provider.uuid,
self.dh.last_month_start,
self.dh.today,
synchronous=True).apply()
refresh_materialized_views.s(self.schema,
provider_type,
provider_uuid=provider.uuid,
synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_azure_data(self,
customer,
static_data_file,
credentials=None,
data_source=None):
"""Load Azure data into the database."""
provider_type = Provider.PROVIDER_AZURE_LOCAL
nise_provider_type = provider_type.replace("-local", "")
report_name = "Test"
if credentials is None:
credentials = {
"subscription_id": "11111111-1111-1111-1111-11111111",
"tenant_id": "22222222-2222-2222-2222-22222222",
"client_id": "33333333-3333-3333-3333-33333333",
"client_secret": "MyPassW0rd!",
}
if data_source is None:
data_source = {
"resource_group": "resourcegroup1",
"storage_account": "storageaccount1"
}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
template, static_data_path = self.prepare_template(
provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"azure_report_name": report_name,
"azure_container_name": self.nise_data_path,
}
base_path = f"{self.nise_data_path}/{report_name}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(
template.render(start_date=start_date, end_date=end_date))
run(nise_provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date,
base_path)
for report in os.scandir(report_path):
if os.path.isdir(report):
continue
elif "manifest" in report.name.lower():
continue
self.process_report(report, "PLAIN", provider_type, provider,
manifest)
with patch("masu.processor.tasks.chain"), patch.object(
settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema,
provider_type,
provider.uuid,
start_date,
end_date,
manifest_id=manifest.id,
synchronous=True,
)
update_cost_model_costs.s(self.schema,
provider.uuid,
self.dh.last_month_start,
self.dh.today,
synchronous=True).apply()
refresh_materialized_views.s(self.schema,
provider_type,
provider_uuid=provider.uuid,
synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_gcp_data(self, customer, static_data_file):
"""Load GCP data into the database."""
provider_type = Provider.PROVIDER_GCP_LOCAL
nise_provider_type = provider_type.replace("-local", "")
credentials = {"project_id": "test_project_id"}
data_source = {"table_id": "test_table_id", "dataset": "test_dataset"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
etag = uuid4()
template, static_data_path = self.prepare_template(
provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"gcp_bucket_name": self.nise_data_path,
"gcp_etag": etag
}
base_path = f"{self.nise_data_path}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(
template.render(start_date=start_date, end_date=end_date))
run(nise_provider_type.lower(), options)
report_path = f"{base_path}/{etag}"
for report in os.scandir(report_path):
if os.path.isdir(report):
continue
self.process_report(report, "PLAIN", provider_type, provider,
manifest)
with patch("masu.processor.tasks.chain"), patch.object(
settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema,
provider_type,
provider.uuid,
start_date,
end_date,
manifest_id=manifest.id,
synchronous=True,
)
update_cost_model_costs.s(self.schema,
provider.uuid,
self.dh.last_month_start,
self.dh.today,
synchronous=True).apply()
refresh_materialized_views.s(self.schema,
provider_type,
provider_uuid=provider.uuid,
synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def test_yesterday(self):
"""Test yesterday property."""
date_helper = DateHelper()
date_helper._now = datetime.datetime(1970, 1, 1, 12, 59, 59)
expected = datetime.datetime(1969, 12, 31, 0, 0, 0, 0)
self.assertEqual(date_helper.yesterday, expected)
