def test_update_summary_tables_azure(self, mock_markup, mock_refresh): """Test that summary tables are updated correctly.""" markup = {"value": 10, "unit": "percent"} mock_markup.return_value = markup with ProviderDBAccessor(self.ocp_provider_uuid) as accessor: accessor.set_infrastructure(self.azure_provider_uuid, Provider.PROVIDER_AZURE) self._generate_ocp_on_azure_data() start_date = self.date_accessor.today_with_timezone("UTC") end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") with ProviderDBAccessor( self.azure_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) initial_count = query.count() updater.update_summary_tables(start_date_str, end_date_str) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: query = azure_accessor._get_db_obj_query(summary_table_name) self.assertNotEqual(query.count(), initial_count) markup_cost = query.aggregate( Sum("markup_cost"))["markup_cost__sum"] pretax_cost = query.aggregate( Sum("pretax_cost"))["pretax_cost__sum"] self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get("value") / 100), places=9) daily_summary_table_name = OCP_REPORT_TABLE_MAP[ "line_item_daily_summary"] with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: query = ocp_accessor._get_db_obj_query(daily_summary_table_name) infra_cost = query.aggregate(Sum("infra_cost"))["infra_cost__sum"] project_infra_cost = query.aggregate( Sum("project_infra_cost"))["project_infra_cost__sum"] self.assertIsNotNone(infra_cost) self.assertIsNotNone(project_infra_cost) self.assertNotEqual(infra_cost, decimal.Decimal(0)) self.assertNotEqual(project_infra_cost, decimal.Decimal(0)) mock_refresh.assert_called()
def test_update_project_markup_cost(self, mock_cost_model): """Test that summary tables are updated correctly.""" markup = {"value": 10, "unit": "percent"} mock_cost_model.return_value.__enter__.return_value.markup = markup markup_dec = decimal.Decimal(markup.get("value") / 100) start_date = self.dh.this_month_start end_date = self.dh.this_month_end updater = OCPCloudReportSummaryUpdater( schema=self.schema, provider=self.ocp_on_aws_ocp_provider, manifest=None) updater.update_summary_tables(start_date, end_date, self.ocp_on_aws_ocp_provider.uuid, self.aws_provider.uuid, Provider.PROVIDER_AWS) summary_table_name = AWS_CUR_TABLE_MAP[ "ocp_on_aws_project_daily_summary"] with AWSReportDBAccessor(self.schema) as aws_accessor: query = (aws_accessor._get_db_obj_query(summary_table_name).filter( cost_entry_bill__billing_period_start=start_date, data_source="Pod").all()) for item in query: self.assertAlmostEqual(item.project_markup_cost, item.pod_cost * markup_dec)
def _set_updater(self): """ Create the report summary updater object. Object is specific to the report provider. Args: None Returns: (Object) : Provider-specific report summary updater """ if self._provider.type in (AMAZON_WEB_SERVICES, AWS_LOCAL_SERVICE_PROVIDER): return (AWSReportSummaryUpdater(self._schema, self._provider, self._manifest), OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest)) if self._provider.type in (AZURE, AZURE_LOCAL_SERVICE_PROVIDER): return (AzureReportSummaryUpdater(self._schema, self._provider, self._manifest), OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest)) if self._provider.type in (OPENSHIFT_CONTAINER_PLATFORM, ): return (OCPReportSummaryUpdater(self._schema, self._provider, self._manifest), OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest)) return None
def test_update_summary_tables_with_aws_provider(self, mock_cluster_id_utility, mock_utility, mock_ocp, mock_ocp_on_aws): """Test that summary tables are properly run for an OCP provider.""" fake_cluster_id = 'my-ocp-cluster' mock_cluster_id_utility.return_value = fake_cluster_id fake_bills = [Mock(), Mock()] fake_bills[0].id = 1 fake_bills[1].id = 2 bill_ids = [str(bill.id) for bill in fake_bills] mock_utility.return_value = fake_bills start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) mock_ocp_on_aws.assert_called_with(start_date_str, end_date_str, fake_cluster_id, bill_ids)
def test_update_markup_cost(self, mock_cost_model): """Test that summary tables are updated correctly.""" markup = {"value": 10, "unit": "percent"} markup_dec = decimal.Decimal(markup.get("value") / 100) mock_cost_model.markup = markup start_date = self.dh.this_month_start end_date = self.dh.this_month_end manifest = CostUsageReportManifest.objects.filter( provider=self.ocp_on_aws_ocp_provider, billing_period_start_datetime=start_date).first() updater = OCPCloudReportSummaryUpdater( schema=self.schema, provider=self.ocp_on_aws_ocp_provider, manifest=manifest) updater.update_summary_tables(start_date, end_date) summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"] with AWSReportDBAccessor(self.schema) as aws_accessor: query = (aws_accessor._get_db_obj_query(summary_table_name).filter( cost_entry_bill__billing_period_start=start_date).all()) for item in query: self.assertAlmostEqual(item.markup_cost, item.unblended_cost * markup_dec)
def test_update_summary_tables_with_aws_provider( self, mock_utility, mock_ocpall_proj_summ, mock_ocpall_summ, mock_ocpall_persp, mock_ocp_on_aws, mock_map): """Test that summary tables are properly run for an OCP provider.""" fake_bills = [Mock(), Mock()] fake_bills[0].id = 1 fake_bills[1].id = 2 bill_ids = [str(bill.id) for bill in fake_bills] mock_utility.return_value = fake_bills start_date = self.dh.today end_date = start_date + datetime.timedelta(days=1) with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() with ProviderDBAccessor( self.ocp_on_aws_ocp_provider.uuid) as provider_accessor: credentials = provider_accessor.get_credentials() cluster_id = credentials.get("cluster_id") mock_map.return_value = { self.ocp_test_provider_uuid: (self.aws_provider_uuid, Provider.PROVIDER_AWS) } updater = OCPCloudReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) updater.update_summary_tables(start_date, end_date, self.ocp_on_aws_ocp_provider.uuid, self.aws_provider.uuid, Provider.PROVIDER_AWS) mock_ocp_on_aws.assert_called_with(start_date.date(), end_date.date(), cluster_id, bill_ids, decimal.Decimal(0))
def test_update_summary_tables_no_ocp_on_aws(self, mock_ocp_on_aws, mock_ocpall_proj_summ, mock_ocpall_summ, mock_ocpall_persp): """Test that summary tables do not run when OCP-on-AWS does not exist.""" new_aws_provider = baker.make("Provider", type="AWS") new_ocp_provider = baker.make("Provider", type="OCP") test_provider_list = [ str(new_aws_provider.uuid), str(new_ocp_provider.uuid) ] for provider_uuid in test_provider_list: start_date = self.dh.today end_date = start_date + datetime.timedelta(days=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") with ProviderDBAccessor(provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) infra_map = updater.get_infra_map(start_date_str, end_date_str) for openshift_provider_uuid, infrastructure_tuple in infra_map.items( ): updater.update_summary_tables( start_date_str, end_date_str, openshift_provider_uuid, infrastructure_tuple[0], infrastructure_tuple[1], ) mock_ocp_on_aws.assert_not_called()
def test_update_summary_tables(self): """Test that summary tables are updated correctly.""" self._generate_ocp_on_aws_data() start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta.relativedelta(months=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: summary_table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_daily_summary'] query = aws_accessor._get_db_obj_query(summary_table_name) initial_count = query.count() updater.update_summary_tables(start_date_str, end_date_str) with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: query = aws_accessor._get_db_obj_query(summary_table_name) self.assertNotEqual(query.count(), initial_count)
def test_update_summary_tables_with_ocp_provider(self, mock_ocp, mock_ocp_on_aws, mock_map, mock_refresh): """Test that summary tables are properly run for an OCP provider.""" start_date = self.dh.today end_date = start_date + datetime.timedelta(days=1) cluster_id = self.ocp_on_aws_ocp_provider.authentication.provider_resource_name manifest = CostUsageReportManifest.objects.filter( provider=self.ocp_on_aws_ocp_provider, billing_period_start_datetime=self.dh.this_month_start) mock_map.return_value = { str(self.ocp_on_aws_ocp_provider.uuid): (self.aws_provider_uuid, Provider.PROVIDER_AWS) } updater = OCPCloudReportSummaryUpdater( schema=self.schema, provider=self.ocp_on_aws_ocp_provider, manifest=manifest) updater.update_summary_tables(start_date, end_date) with schema_context(self.schema): bill = AWSCostEntryBill.objects.filter( provider=self.aws_provider, billing_period_start=self.dh.this_month_start).first() mock_ocp_on_aws.assert_called_with(start_date.date(), end_date.date(), cluster_id, [str(bill.id)]) mock_refresh.assert_called()
def _set_updater(self): """ Create the report summary updater object. Object is specific to the report provider. Args: None Returns: (Object) : Provider-specific report summary updater """ if self._provider.type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL): return (AWSReportSummaryUpdater(self._schema, self._provider, self._manifest), OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest)) if self._provider.type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL): return (AzureReportSummaryUpdater(self._schema, self._provider, self._manifest), OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest)) if self._provider.type in (Provider.PROVIDER_OCP, ): return (OCPReportSummaryUpdater(self._schema, self._provider, self._manifest), OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest)) return (None, None)
def test_update_summary_tables_with_aws_provider(self, mock_utility, mock_ocp, mock_ocp_on_aws, mock_map, mock_refresh): """Test that summary tables are properly run for an OCP provider.""" fake_bills = [Mock(), Mock()] fake_bills[0].id = 1 fake_bills[1].id = 2 bill_ids = [str(bill.id) for bill in fake_bills] mock_utility.return_value = fake_bills start_date = self.dh.today end_date = start_date + datetime.timedelta(days=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() with ProviderDBAccessor( self.ocp_test_provider_uuid) as provider_accessor: cluster_id = provider_accessor.get_authentication() mock_map.return_value = { self.ocp_test_provider_uuid: (self.aws_provider_uuid, Provider.PROVIDER_AWS) } updater = OCPCloudReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) mock_ocp_on_aws.assert_called_with(start_date.date(), end_date.date(), cluster_id, bill_ids) mock_refresh.assert_called()
def test_update_summary_tables_no_ocp_on_aws(self, mock_ocp, mock_ocp_on_aws, mock_refresh): """Test that summary tables do not run when OCP-on-AWS does not exist.""" new_aws_provider = baker.make("Provider", type="AWS") new_ocp_provider = baker.make("Provider", type="OCP") test_provider_list = [ str(new_aws_provider.uuid), str(new_ocp_provider.uuid) ] for provider_uuid in test_provider_list: start_date = self.dh.today end_date = start_date + datetime.timedelta(days=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") with ProviderDBAccessor(provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) mock_ocp_on_aws.assert_not_called() mock_refresh.assert_not_called()
def test_refresh_openshift_on_infrastructure_views(self): """Test that the combined OpenShift views are refreshed.""" self._generate_ocp_on_aws_data(cluster_id="my-ocp-cluster-1") self._generate_ocp_on_azure_data(cluster_id="my-ocp-cluster-2") start_date = self.date_accessor.today_with_timezone("UTC") end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") with ProviderDBAccessor( self.azure_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_count = query.count() summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_project_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_project_count = query.count() with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_count = query.count() summary_table_name = AWS_CUR_TABLE_MAP[ "ocp_on_aws_project_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_project_count = query.count() updater.refresh_openshift_on_infrastructure_views() with schema_context(self.schema): all_count = OCPAllCostLineItemDailySummary.objects.count() all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count( ) self.assertEqual(all_count, azure_count + aws_count) self.assertEqual(all_project_count, azure_project_count + aws_project_count)
def test_refresh_openshift_on_infrastructure_views(self, mock_view_refresh): """Test that the combined OpenShift views are refreshed.""" start_date = self.dh.today end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime("%Y-%m-%d") end_date_str = end_date.strftime("%Y-%m-%d") updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.azure_provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) mock_view_refresh.assert_called_with( OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) mock_view_refresh.reset_mock() updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.aws_provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) expected_calls = [ call(OCP_ON_AWS_MATERIALIZED_VIEWS), call(OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) ] mock_view_refresh.assert_has_calls(expected_calls) with AzureReportDBAccessor(self.schema) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_count = query.count() summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_project_daily_summary"] query = azure_accessor._get_db_obj_query(summary_table_name) azure_project_count = query.count() with AWSReportDBAccessor(self.schema) as aws_accessor: summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_count = query.count() summary_table_name = AWS_CUR_TABLE_MAP[ "ocp_on_aws_project_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) aws_project_count = query.count() updater.refresh_openshift_on_infrastructure_views( OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) with schema_context(self.schema): all_count = OCPAllCostLineItemDailySummary.objects.count() all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count( ) self.assertEqual(all_count, azure_count + aws_count) self.assertEqual(all_project_count, azure_project_count + aws_project_count)
def test_update_summary_tables_azure( self, mock_cost_model # , mock_ocpall_proj_summ, mock_ocpall_summ, mock_ocpall_persp ): """Test that summary tables are updated correctly.""" markup = {"value": 10, "unit": "percent"} mock_cost_model.markup = markup start_date = self.dh.this_month_start end_date = self.dh.this_month_end updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.azure_provider, manifest=None) updater.update_summary_tables( start_date, end_date, self.ocp_on_azure_ocp_provider.uuid, self.azure_provider.uuid, Provider.PROVIDER_AZURE, ) summary_table_name = AZURE_REPORT_TABLE_MAP[ "ocp_on_azure_daily_summary"] with AzureReportDBAccessor(self.schema) as azure_accessor: query = azure_accessor._get_db_obj_query( summary_table_name).filter( cost_entry_bill__billing_period_start=start_date) markup_cost = query.aggregate( Sum("markup_cost"))["markup_cost__sum"] pretax_cost = query.aggregate( Sum("pretax_cost"))["pretax_cost__sum"] self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get("value") / 100), places=5) daily_summary_table_name = OCP_REPORT_TABLE_MAP[ "line_item_daily_summary"] with OCPReportDBAccessor(self.schema) as ocp_accessor: query = ocp_accessor._get_db_obj_query( daily_summary_table_name).filter( report_period__provider=self.ocp_on_azure_ocp_provider, report_period__report_period_start=self.dh. this_month_start, ) infra_cost = query.aggregate( Sum("infrastructure_raw_cost"))["infrastructure_raw_cost__sum"] project_infra_cost = query.aggregate( Sum("infrastructure_project_raw_cost") )["infrastructure_project_raw_cost__sum"] self.assertIsNotNone(infra_cost) self.assertIsNotNone(project_infra_cost) self.assertNotEqual(infra_cost, decimal.Decimal(0)) self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def test_update_summary_tables_azure(self, mock_markup): """Test that summary tables are updated correctly.""" markup = {'value': 10, 'unit': 'percent'} mock_markup.return_value = markup with ProviderDBAccessor(self.ocp_provider_uuid) as accessor: accessor.set_infrastructure(self.azure_provider_uuid, 'AZURE') self._generate_ocp_on_azure_data() start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta(months=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.azure_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: summary_table_name = AZURE_REPORT_TABLE_MAP[ 'ocp_on_azure_daily_summary'] query = azure_accessor._get_db_obj_query(summary_table_name) initial_count = query.count() updater.update_summary_tables(start_date_str, end_date_str) with AzureReportDBAccessor(self.schema, self.column_map) as azure_accessor: query = azure_accessor._get_db_obj_query(summary_table_name) self.assertNotEqual(query.count(), initial_count) markup_cost = query.aggregate( Sum('markup_cost'))['markup_cost__sum'] pretax_cost = query.aggregate( Sum('pretax_cost'))['pretax_cost__sum'] self.assertAlmostEqual(markup_cost, pretax_cost * decimal.Decimal(markup.get('value') / 100), places=9) daily_summary_table_name = OCP_REPORT_TABLE_MAP[ 'line_item_daily_summary'] with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: query = ocp_accessor._get_db_obj_query(daily_summary_table_name) infra_cost = query.aggregate(Sum('infra_cost'))['infra_cost__sum'] project_infra_cost = query.aggregate( Sum('project_infra_cost'))['project_infra_cost__sum'] self.assertIsNotNone(infra_cost) self.assertIsNotNone(project_infra_cost) self.assertNotEqual(infra_cost, decimal.Decimal(0)) self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def test_update_summary_tables_with_ocp_provider(self, mock_ocp, mock_ocp_on_aws, mock_map): """Test that summary tables are properly run for an OCP provider.""" start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor(self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() cluster_id = provider_accessor.get_authentication() mock_map.return_value = {self.ocp_test_provider_uuid: (self.aws_provider_uuid, 'AWS')} updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) mock_ocp_on_aws.assert_called_with(start_date_str, end_date_str, cluster_id, [])
def _set_updater(self): """ Create the report summary updater object. Object is specific to the report provider. Args: None Returns: (Object) : Provider-specific report summary updater """ if self._provider.type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL): report_summary_updater = ( AWSReportParquetSummaryUpdater if settings.ENABLE_PARQUET_PROCESSING else AWSReportSummaryUpdater ) elif self._provider.type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL): report_summary_updater = ( AzureReportParquetSummaryUpdater if settings.ENABLE_PARQUET_PROCESSING else AzureReportSummaryUpdater ) elif self._provider.type in (Provider.PROVIDER_OCP,): report_summary_updater = ( OCPReportParquetSummaryUpdater if settings.ENABLE_PARQUET_PROCESSING else OCPReportSummaryUpdater ) elif self._provider.type in (Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL): report_summary_updater = GCPReportSummaryUpdater else: return (None, None) return ( report_summary_updater(self._schema, self._provider, self._manifest), OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest), )
def test_update_cost_summary_tables(self, mock_ocp): """Test that cost_summary tables are updated correctly""" self._generate_ocp_on_aws_data() start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta.relativedelta(months=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) updater.update_cost_summary_table(start_date_str, end_date_str) mock_ocp.assert_called()
def test_update_project_markup_cost(self, mock_markup): """Test that summary tables are updated correctly.""" markup = {'value': 10, 'unit': 'percent'} mock_markup.return_value = markup self._generate_ocp_on_aws_data() start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta.relativedelta(months=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: table_name = AWS_CUR_TABLE_MAP['line_item'] tag_query = aws_accessor._get_db_obj_query(table_name) summary_table_name = AWS_CUR_TABLE_MAP[ 'ocp_on_aws_project_daily_summary'] query = aws_accessor._get_db_obj_query(summary_table_name) possible_values = {} with schema_context(self.schema): for item in tag_query: possible_values.update({ item.cost_entry_bill_id: (item.unblended_cost * decimal.Decimal(0.1)) }) updater.update_summary_tables(start_date_str, end_date_str) with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: query = aws_accessor._get_db_obj_query(summary_table_name) found_values = {} for item in query: found_values.update( {item.cost_entry_bill_id: item.project_markup_cost}) for k, v in found_values.items(): self.assertAlmostEqual(v, possible_values[k], places=6)
def test_update_summary_tables(self): """Test that summary tables are updated correctly.""" start_date = self.dh.this_month_start end_date = self.dh.this_month_end updater = OCPCloudReportSummaryUpdater( schema=self.schema, provider=self.ocp_on_aws_ocp_provider, manifest=None ) with AWSReportDBAccessor(self.schema) as aws_accessor: summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"] query = aws_accessor._get_db_obj_query(summary_table_name) query.delete() initial_count = query.count() updater.update_summary_tables(start_date, end_date) with AWSReportDBAccessor(self.schema) as aws_accessor: query = aws_accessor._get_db_obj_query(summary_table_name) self.assertNotEqual(query.count(), initial_count)
def test_update_summary_tables_no_ocp_on_aws(self, mock_ocp, mock_ocp_on_aws): """Test that summary tables do not run when OCP-on-AWS does not exist.""" test_provider_list = [self.aws_provider_uuid, self.ocp_test_provider_uuid] for provider_uuid in test_provider_list: start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor(provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater( schema='acct10001', provider=provider, manifest=None ) updater.update_summary_tables(start_date_str, end_date_str) mock_ocp_on_aws.assert_not_called()
def test_partition_handler_str_table(self): new_table_sql = f""" create table {self.schema}._eek_pt0 (usage_start date not null, id int) partition by range (usage_start); """ with schema_context(self.schema): with connection.cursor() as cur: cur.execute(new_table_sql) partable = get_model("PartitionedTable") default_part = partable( schema_name=self.schema, table_name="_eek_pt0_default", partition_of_table_name="_eek_pt0", partition_type=partable.RANGE, partition_col="usage_start", partition_parameters={"default": True}, active=True, ) default_part.save() ocrsu = OCPCloudReportSummaryUpdater(self.schema, self.ocp_on_aws_ocp_provider, None) num_eek = partable.objects.filter( schema_name=self.schema, partition_of_table_name="_eek_pt0").count() self.assertEqual(num_eek, 1) ocrsu._handle_partitions(self.schema, "_eek_pt0", datetime.date(1970, 10, 1), datetime.date(1970, 12, 1)) eek_p = partable.objects.filter( schema_name=self.schema, partition_of_table_name="_eek_pt0", partition_parameters__default=False).all() self.assertEqual(len(eek_p), 3) eek_p.delete() default_part.delete() with connection.cursor() as cur: cur.execute(f"drop table {self.schema}._eek_pt0 ;")
def test_update_summary_tables_with_ocp_provider(self, mock_utility, mock_ocp, mock_ocp_on_aws): """Test that summary tables are properly run for an OCP provider.""" fake_cluster = 'my-ocp-cluster' mock_utility.return_value = fake_cluster start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) mock_ocp_on_aws.assert_called_with(start_date_str, end_date_str, fake_cluster, []) mock_ocp.assert_called_with(fake_cluster, start_date_str, end_date_str)
def test_get_infra_map(self): """Test that an infrastructure map is returned.""" infrastructure_type = 'AWS' with ProviderDBAccessor(self.ocp_provider_uuid) as accessor: accessor.set_infrastructure(self.aws_provider_uuid, infrastructure_type) ocp_provider = accessor.get_provider() updater = OCPCloudReportSummaryUpdater( schema=self.schema, provider=ocp_provider, manifest=None ) expected_mapping = (self.aws_provider_uuid, 'AWS') infra_map = updater.get_infra_map() self.assertEqual(len(infra_map.keys()), 1) self.assertIn(self.ocp_provider_uuid, infra_map) self.assertEqual(infra_map.get(self.ocp_provider_uuid), expected_mapping) with ProviderDBAccessor(self.aws_provider_uuid) as accessor: aws_provider = accessor.get_provider() updater = OCPCloudReportSummaryUpdater( schema=self.schema, provider=aws_provider, manifest=None ) infra_map = updater.get_infra_map() self.assertEqual(len(infra_map.keys()), 1) self.assertIn(self.ocp_provider_uuid, infra_map) self.assertEqual(infra_map.get(self.ocp_provider_uuid), expected_mapping)
def test_get_infra_map(self): """Test that an infrastructure map is returned.""" updater = OCPCloudReportSummaryUpdater( schema=self.schema, provider=self.ocp_on_aws_ocp_provider, manifest=None) expected_mapping = (self.aws_provider_uuid, Provider.PROVIDER_AWS_LOCAL) infra_map = updater.get_infra_map() self.assertEqual(len(infra_map.keys()), 1) self.assertIn(str(self.ocp_on_aws_ocp_provider.uuid), infra_map) self.assertEqual(infra_map.get(str(self.ocp_on_aws_ocp_provider.uuid)), expected_mapping) updater = OCPCloudReportSummaryUpdater(schema=self.schema, provider=self.aws_provider, manifest=None) infra_map = updater.get_infra_map() self.assertEqual(len(infra_map.keys()), 1) self.assertIn(str(self.ocp_on_aws_ocp_provider.uuid), infra_map) self.assertEqual(infra_map.get(str(self.ocp_on_aws_ocp_provider.uuid)), expected_mapping)
def test_get_infra_db_key_for_provider_type(self): """Test db_key private method for OCP-on-AWS infrastructure map.""" with ProviderDBAccessor(self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater( schema='acct10001', provider=provider, manifest=None ) self.assertEqual(updater._get_infra_db_key_for_provider_type('AWS'), 'aws_uuid') self.assertEqual(updater._get_infra_db_key_for_provider_type('AWS-local'), 'aws_uuid') self.assertEqual(updater._get_infra_db_key_for_provider_type('OCP'), 'ocp_uuid') self.assertEqual(updater._get_infra_db_key_for_provider_type('WRONG'), None)
def test_update_cost_summary_table_for_markup(self, mock_markup): """Test that summary tables are updated correctly.""" markup = {'value': 10, 'unit': 'percent'} mock_markup.return_value = markup self._generate_ocp_on_aws_data() start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta.relativedelta(months=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid) with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: ocp_accessor.populate_cost_summary_table(cluster_id, start_date, end_date) cost_summary_table = OCP_REPORT_TABLE_MAP['cost_summary'] cost_summary_query = ocp_accessor._get_db_obj_query( cost_summary_table) possible_values = {} with schema_context(self.schema): for item in cost_summary_query: possible_values.update({ item.cluster_id: ((item.pod_charge_cpu_core_hours + item.pod_charge_memory_gigabyte_hours + item.persistentvolumeclaim_charge_gb_month + item.infra_cost) * decimal.Decimal(0.1)) }) updater.update_cost_summary_table(start_date_str, end_date_str) with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: query = ocp_accessor._get_db_obj_query(cost_summary_table) found_values = {} for item in query: found_values.update({item.cluster_id: item.markup_cost}) for k, v in found_values.items(): self.assertAlmostEqual(v, possible_values[k], places=6)