def test_populate_ocp_on_azure_cost_daily_summary(self):
"""Test the method to run OpenShift on Azure SQL."""
summary_table_name = AZURE_REPORT_TABLE_MAP["ocp_on_azure_daily_summary"]
project_summary_table_name = AZURE_REPORT_TABLE_MAP["ocp_on_azure_project_daily_summary"]
markup_value = decimal.Decimal(0.1)
summary_table = getattr(self.accessor.report_schema, summary_table_name)
project_table = getattr(self.accessor.report_schema, project_summary_table_name)
today = DateHelper().today
last_month = DateHelper().last_month_start
azure_bills = get_bills_from_provider(self.azure_provider_uuid, self.schema, last_month, today)
with schema_context(self.schema):
bill_ids = [str(bill.id) for bill in azure_bills]
cluster_id = self.ocp_on_azure_ocp_provider.authentication.provider_resource_name
self.accessor.populate_ocp_on_azure_cost_daily_summary(last_month, today, cluster_id, bill_ids, markup_value)
li_table_name = AZURE_REPORT_TABLE_MAP["line_item"]
with schema_context(self.schema):
li_table = getattr(self.accessor.report_schema, li_table_name)
sum_azure_cost = li_table.objects.aggregate(Sum("pretax_cost"))["pretax_cost__sum"]
with schema_context(self.schema):
sum_cost = summary_table.objects.all().aggregate(Sum("pretax_cost"))["pretax_cost__sum"]
sum_project_cost = project_table.objects.all().aggregate(Sum("pretax_cost"))["pretax_cost__sum"]
self.assertNotEqual(sum_cost, 0)
self.assertAlmostEqual(sum_cost, sum_project_cost, 4)
self.assertLessEqual(sum_cost, sum_azure_cost)
with schema_context(self.schema):
sum_cost = summary_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("pretax_cost"))[
"pretax_cost__sum"
]
sum_project_cost = project_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("pretax_cost"))[
"pretax_cost__sum"
]
sum_pod_cost = project_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("pod_cost"))[
"pod_cost__sum"
]
sum_markup_cost = summary_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("markup_cost"))[
"markup_cost__sum"
]
sum_markup_cost_project = project_table.objects.filter(cluster_id=cluster_id).aggregate(
Sum("markup_cost")
)["markup_cost__sum"]
sum_project_markup_cost_project = project_table.objects.filter(cluster_id=cluster_id).aggregate(
Sum("project_markup_cost")
)["project_markup_cost__sum"]
self.assertLessEqual(sum_cost, sum_azure_cost)
self.assertAlmostEqual(sum_cost, sum_project_cost, 4)
self.assertAlmostEqual(sum_markup_cost, sum_cost * markup_value, 4)
self.assertAlmostEqual(sum_markup_cost_project, sum_cost * markup_value, 4)
self.assertAlmostEqual(sum_project_markup_cost_project, sum_pod_cost * markup_value, 4)
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
LOG.info("update_summary_tables for: %s-%s", str(start_date),
str(end_date))
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid,
self._schema,
datetime.datetime.strptime(start_date, "%Y-%m-%d"),
datetime.datetime.strptime(end_date, "%Y-%m-%d"),
)
bill_ids = []
with schema_context(self._schema):
bill_ids = [str(bill.id) for bill in bills]
with AzureReportDBAccessor(self._schema) as accessor:
# Need these bills on the session to update dates after processing
bills = accessor.bills_for_provider_uuid(self._provider.uuid,
start_date)
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating Azure report summary tables: \n\tSchema: %s"
"\n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
start,
end,
)
accessor.populate_line_item_daily_summary_table(
start, end, bill_ids)
accessor.populate_tags_summary_table(bill_ids, start_date,
end_date)
for bill in bills:
if bill.summary_data_creation_datetime is None:
bill.summary_data_creation_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.summary_data_updated_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.save()
return start_date, end_date
def _update_markup_cost(self, start_date, end_date):
"""Store markup costs."""
try:
bills = get_bills_from_provider(self._provider.uuid, self._schema, start_date, end_date)
with CostModelDBAccessor(self._schema, self._provider.uuid, self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = float(markup.get("value", 0)) / 100
with AzureReportDBAccessor(self._schema, self._column_map) as report_accessor:
with schema_context(self._schema):
bill_ids = [str(bill.id) for bill in bills]
report_accessor.populate_markup_cost(markup_value, bill_ids)
except AzureReportChargeUpdaterError as error:
LOG.error("Unable to update markup costs. Error: %s", str(error))
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
LOG.info('update_summary_tables for: %s-%s', str(start_date), str(end_date))
start_date, end_date = self._get_sql_inputs(start_date, end_date)
bills = get_bills_from_provider(
self._provider.uuid,
self._schema_name,
datetime.datetime.strptime(start_date, '%Y-%m-%d'),
datetime.datetime.strptime(end_date, '%Y-%m-%d')
)
bill_ids = []
with schema_context(self._schema_name):
bill_ids = [str(bill.id) for bill in bills]
with AzureReportDBAccessor(self._schema_name, self._column_map) as accessor:
# Need these bills on the session to update dates after processing
bills = accessor.bills_for_provider_id(self._provider.id, start_date)
LOG.info('Updating Azure report summary tables: \n\tSchema: %s'
'\n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_line_item_daily_summary_table(start_date, end_date, bill_ids)
accessor.populate_tags_summary_table()
for bill in bills:
if bill.summary_data_creation_datetime is None:
bill.summary_data_creation_datetime = \
self._date_accessor.today_with_timezone('UTC')
bill.summary_data_updated_datetime = \
self._date_accessor.today_with_timezone('UTC')
bill.save()
accessor.commit()
return start_date, end_date
def test_populate_ocp_on_azure_cost_daily_summary(self):
"""Test the method to run OpenShift on Azure SQL."""
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
project_summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_project_daily_summary"]
markup_value = decimal.Decimal(0.1)
summary_table = getattr(self.accessor.report_schema,
summary_table_name)
project_table = getattr(self.accessor.report_schema,
project_summary_table_name)
today = DateHelper().today
last_month = DateHelper().last_month_start
azure_bills = get_bills_from_provider(self.azure_provider_uuid,
self.schema, last_month, today)
with schema_context(self.schema):
bill_ids = [str(bill.id) for bill in azure_bills]
cluster_id = self.ocp_on_azure_ocp_provider.authentication.credentials.get(
"cluster_id")
self.accessor.populate_ocp_on_azure_cost_daily_summary(
last_month, today, cluster_id, bill_ids, markup_value)
li_table_name = AZURE_REPORT_TABLE_MAP["line_item"]
with schema_context(self.schema):
li_table = getattr(self.accessor.report_schema, li_table_name)
sum_azure_cost = li_table.objects.aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
with schema_context(self.schema):
# These names are defined in the `azure_static_data.yml` used by Nise to populate the Azure data
namespaces = [
"kube-system", "openshift", "banking", "mobile", "news-site",
"weather"
]
for namespace in namespaces:
with self.subTest(namespace=namespace):
with connection.cursor() as cursor:
cursor.execute(f"""
SELECT sum(pretax_cost / cardinality(namespace)) AS pretax_cost
FROM {summary_table._meta.db_table}
WHERE namespace @> array['{namespace}'::varchar]
""")
sum_cost = cursor.fetchone()[0]
sum_project_cost = project_table.objects.filter(
namespace=namespace).aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
self.assertNotEqual(sum_cost, 0)
self.assertAlmostEqual(sum_cost, sum_project_cost, 4)
self.assertLessEqual(sum_cost, sum_azure_cost)
with schema_context(self.schema):
sum_cost = summary_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
sum_markup_cost = summary_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("markup_cost"))["markup_cost__sum"]
sum_project_cost = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
sum_pod_cost = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("pod_cost"))["pod_cost__sum"]
sum_markup_cost_project = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("markup_cost"))["markup_cost__sum"]
sum_project_markup_cost_project = project_table.objects.filter(
cluster_id=cluster_id).aggregate(
Sum("project_markup_cost"))["project_markup_cost__sum"]
self.assertLessEqual(sum_cost, sum_azure_cost)
self.assertAlmostEqual(sum_markup_cost, sum_cost * markup_value, 4)
self.assertAlmostEqual(sum_markup_cost_project,
sum_project_cost * markup_value, 4)
self.assertAlmostEqual(sum_project_markup_cost_project,
sum_pod_cost * markup_value, 4)
