def test_populate_ocp_on_azure_cost_daily_summary(self): """Test the method to run OpenShift on Azure SQL.""" summary_table_name = AZURE_REPORT_TABLE_MAP["ocp_on_azure_daily_summary"] project_summary_table_name = AZURE_REPORT_TABLE_MAP["ocp_on_azure_project_daily_summary"] markup_value = decimal.Decimal(0.1) summary_table = getattr(self.accessor.report_schema, summary_table_name) project_table = getattr(self.accessor.report_schema, project_summary_table_name) today = DateHelper().today last_month = DateHelper().last_month_start azure_bills = get_bills_from_provider(self.azure_provider_uuid, self.schema, last_month, today) with schema_context(self.schema): bill_ids = [str(bill.id) for bill in azure_bills] cluster_id = self.ocp_on_azure_ocp_provider.authentication.provider_resource_name self.accessor.populate_ocp_on_azure_cost_daily_summary(last_month, today, cluster_id, bill_ids, markup_value) li_table_name = AZURE_REPORT_TABLE_MAP["line_item"] with schema_context(self.schema): li_table = getattr(self.accessor.report_schema, li_table_name) sum_azure_cost = li_table.objects.aggregate(Sum("pretax_cost"))["pretax_cost__sum"] with schema_context(self.schema): sum_cost = summary_table.objects.all().aggregate(Sum("pretax_cost"))["pretax_cost__sum"] sum_project_cost = project_table.objects.all().aggregate(Sum("pretax_cost"))["pretax_cost__sum"] self.assertNotEqual(sum_cost, 0) self.assertAlmostEqual(sum_cost, sum_project_cost, 4) self.assertLessEqual(sum_cost, sum_azure_cost) with schema_context(self.schema): sum_cost = summary_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("pretax_cost"))[ "pretax_cost__sum" ] sum_project_cost = project_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("pretax_cost"))[ "pretax_cost__sum" ] sum_pod_cost = project_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("pod_cost"))[ "pod_cost__sum" ] sum_markup_cost = summary_table.objects.filter(cluster_id=cluster_id).aggregate(Sum("markup_cost"))[ "markup_cost__sum" ] sum_markup_cost_project = project_table.objects.filter(cluster_id=cluster_id).aggregate( Sum("markup_cost") )["markup_cost__sum"] sum_project_markup_cost_project = project_table.objects.filter(cluster_id=cluster_id).aggregate( Sum("project_markup_cost") )["project_markup_cost__sum"] self.assertLessEqual(sum_cost, sum_azure_cost) self.assertAlmostEqual(sum_cost, sum_project_cost, 4) self.assertAlmostEqual(sum_markup_cost, sum_cost * markup_value, 4) self.assertAlmostEqual(sum_markup_cost_project, sum_cost * markup_value, 4) self.assertAlmostEqual(sum_project_markup_cost_project, sum_pod_cost * markup_value, 4)
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str) A start date and end date. """ LOG.info("update_summary_tables for: %s-%s", str(start_date), str(end_date)) start_date, end_date = self._get_sql_inputs(start_date, end_date) bills = get_bills_from_provider( self._provider.uuid, self._schema, datetime.datetime.strptime(start_date, "%Y-%m-%d"), datetime.datetime.strptime(end_date, "%Y-%m-%d"), ) bill_ids = [] with schema_context(self._schema): bill_ids = [str(bill.id) for bill in bills] with AzureReportDBAccessor(self._schema) as accessor: # Need these bills on the session to update dates after processing bills = accessor.bills_for_provider_uuid(self._provider.uuid, start_date) for start, end in date_range_pair(start_date, end_date): LOG.info( "Updating Azure report summary tables: \n\tSchema: %s" "\n\tProvider: %s \n\tDates: %s - %s", self._schema, self._provider.uuid, start, end, ) accessor.populate_line_item_daily_summary_table( start, end, bill_ids) accessor.populate_tags_summary_table(bill_ids, start_date, end_date) for bill in bills: if bill.summary_data_creation_datetime is None: bill.summary_data_creation_datetime = self._date_accessor.today_with_timezone( "UTC") bill.summary_data_updated_datetime = self._date_accessor.today_with_timezone( "UTC") bill.save() return start_date, end_date
def _update_markup_cost(self, start_date, end_date): """Store markup costs.""" try: bills = get_bills_from_provider(self._provider.uuid, self._schema, start_date, end_date) with CostModelDBAccessor(self._schema, self._provider.uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() markup_value = float(markup.get("value", 0)) / 100 with AzureReportDBAccessor(self._schema, self._column_map) as report_accessor: with schema_context(self._schema): bill_ids = [str(bill.id) for bill in bills] report_accessor.populate_markup_cost(markup_value, bill_ids) except AzureReportChargeUpdaterError as error: LOG.error("Unable to update markup costs. Error: %s", str(error))
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns (str, str) A start date and end date. """ LOG.info('update_summary_tables for: %s-%s', str(start_date), str(end_date)) start_date, end_date = self._get_sql_inputs(start_date, end_date) bills = get_bills_from_provider( self._provider.uuid, self._schema_name, datetime.datetime.strptime(start_date, '%Y-%m-%d'), datetime.datetime.strptime(end_date, '%Y-%m-%d') ) bill_ids = [] with schema_context(self._schema_name): bill_ids = [str(bill.id) for bill in bills] with AzureReportDBAccessor(self._schema_name, self._column_map) as accessor: # Need these bills on the session to update dates after processing bills = accessor.bills_for_provider_id(self._provider.id, start_date) LOG.info('Updating Azure report summary tables: \n\tSchema: %s' '\n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_line_item_daily_summary_table(start_date, end_date, bill_ids) accessor.populate_tags_summary_table() for bill in bills: if bill.summary_data_creation_datetime is None: bill.summary_data_creation_datetime = \ self._date_accessor.today_with_timezone('UTC') bill.summary_data_updated_datetime = \ self._date_accessor.today_with_timezone('UTC') bill.save() accessor.commit() return start_date, end_date
def test_populate_ocp_on_azure_cost_daily_summary(self): """Test the method to run OpenShift on Azure SQL.""" summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] project_summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_project_daily_summary"] markup_value = decimal.Decimal(0.1) summary_table = getattr(self.accessor.report_schema, summary_table_name) project_table = getattr(self.accessor.report_schema, project_summary_table_name) today = DateHelper().today last_month = DateHelper().last_month_start azure_bills = get_bills_from_provider(self.azure_provider_uuid, self.schema, last_month, today) with schema_context(self.schema): bill_ids = [str(bill.id) for bill in azure_bills] cluster_id = self.ocp_on_azure_ocp_provider.authentication.credentials.get( "cluster_id") self.accessor.populate_ocp_on_azure_cost_daily_summary( last_month, today, cluster_id, bill_ids, markup_value) li_table_name = AZURE_REPORT_TABLE_MAP["line_item"] with schema_context(self.schema): li_table = getattr(self.accessor.report_schema, li_table_name) sum_azure_cost = li_table.objects.aggregate( Sum("pretax_cost"))["pretax_cost__sum"] with schema_context(self.schema): # These names are defined in the `azure_static_data.yml` used by Nise to populate the Azure data namespaces = [ "kube-system", "openshift", "banking", "mobile", "news-site", "weather" ] for namespace in namespaces: with self.subTest(namespace=namespace): with connection.cursor() as cursor: cursor.execute(f""" SELECT sum(pretax_cost / cardinality(namespace)) AS pretax_cost FROM {summary_table._meta.db_table} WHERE namespace @> array['{namespace}'::varchar] """) sum_cost = cursor.fetchone()[0] sum_project_cost = project_table.objects.filter( namespace=namespace).aggregate( Sum("pretax_cost"))["pretax_cost__sum"] self.assertNotEqual(sum_cost, 0) self.assertAlmostEqual(sum_cost, sum_project_cost, 4) self.assertLessEqual(sum_cost, sum_azure_cost) with schema_context(self.schema): sum_cost = summary_table.objects.filter( cluster_id=cluster_id).aggregate( Sum("pretax_cost"))["pretax_cost__sum"] sum_markup_cost = summary_table.objects.filter( cluster_id=cluster_id).aggregate( Sum("markup_cost"))["markup_cost__sum"] sum_project_cost = project_table.objects.filter( cluster_id=cluster_id).aggregate( Sum("pretax_cost"))["pretax_cost__sum"] sum_pod_cost = project_table.objects.filter( cluster_id=cluster_id).aggregate( Sum("pod_cost"))["pod_cost__sum"] sum_markup_cost_project = project_table.objects.filter( cluster_id=cluster_id).aggregate( Sum("markup_cost"))["markup_cost__sum"] sum_project_markup_cost_project = project_table.objects.filter( cluster_id=cluster_id).aggregate( Sum("project_markup_cost"))["project_markup_cost__sum"] self.assertLessEqual(sum_cost, sum_azure_cost) self.assertAlmostEqual(sum_markup_cost, sum_cost * markup_value, 4) self.assertAlmostEqual(sum_markup_cost_project, sum_project_cost * markup_value, 4) self.assertAlmostEqual(sum_project_markup_cost_project, sum_pod_cost * markup_value, 4)