def load_gcp_data(self, customer): """Load GCP data into the database.""" provider_type = Provider.PROVIDER_GCP_LOCAL credentials = {"project_id": "test_project_id"} data_source = {"table_id": "test_table_id", "dataset": "test_dataset"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply()
def load_azure_data(self, customer, static_data_file, credentials=None, data_source=None): """Load Azure data into the database.""" provider_type = Provider.PROVIDER_AZURE_LOCAL nise_provider_type = provider_type.replace("-local", "") report_name = "Test" if credentials is None: credentials = { "subscription_id": "11111111-1111-1111-1111-11111111", "tenant_id": "22222222-2222-2222-2222-22222222", "client_id": "33333333-3333-3333-3333-33333333", "client_secret": "MyPassW0rd!", } if data_source is None: data_source = {"resource_group": "resourcegroup1", "storage_account": "storageaccount1"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "azure_report_name": report_name, "azure_container_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): continue elif "manifest" in report.name.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def test_update_summary_tables_aws(self, mock_charge_info, mock_views, mock_chain): """Test that the summary table task runs.""" provider = Provider.PROVIDER_AWS provider_aws_uuid = self.aws_provider_uuid daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"] summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) with schema_context(self.schema): daily_query = self.aws_accessor._get_db_obj_query(daily_table_name) summary_query = self.aws_accessor._get_db_obj_query(summary_table_name) initial_daily_count = daily_query.count() initial_summary_count = summary_query.count() self.assertEqual(initial_daily_count, 0) self.assertEqual(initial_summary_count, 0) update_summary_tables(self.schema, provider, provider_aws_uuid, start_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) self.assertNotEqual(summary_query.count(), initial_summary_count) mock_chain.return_value.apply_async.assert_called()
def test_update_summary_tables(self): """Test that the summary table task runs.""" daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] agg_table_name = AWS_CUR_TABLE_MAP['line_item_aggregates'] start_date = datetime.utcnow() start_date = start_date.replace(day=1, month=(start_date.month - 1)) daily_query = self.accessor._get_db_obj_query(daily_table_name) summary_query = self.accessor._get_db_obj_query(summary_table_name) agg_query = self.accessor._get_db_obj_query(agg_table_name) initial_daily_count = daily_query.count() initial_summary_count = summary_query.count() initial_agg_count = agg_query.count() self.assertEqual(initial_daily_count, 0) self.assertEqual(initial_summary_count, 0) self.assertEqual(initial_agg_count, 0) update_summary_tables('testcustomer', start_date) self.assertNotEqual(daily_query.count(), initial_daily_count) self.assertNotEqual(summary_query.count(), initial_summary_count) self.assertNotEqual(agg_query.count(), initial_agg_count)
def load_aws_data(self, customer, static_data_file, account_id=None, role_arn=None): """Load AWS data into the database.""" provider_type = Provider.PROVIDER_AWS_LOCAL if account_id is None: account_id = "9999999999999" if role_arn is None: role_arn = "arn:aws:iam::999999999999:role/CostManagement" nise_provider_type = provider_type.replace("-local", "") report_name = "Test" credentials = {"role_arn": role_arn} data_source = {"bucket": "test-bucket"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "aws_report_name": report_name, "aws_bucket_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" with schema_context(self.schema): baker.make("AWSAccountAlias", account_id=account_id, account_alias="Test Account") for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date, account_id=account_id)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): for report in [f.path for f in os.scandir(f"{report_path}/{report.name}")]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "GZIP", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def test_update_summary_tables_aws_end_date(self, mock_charge_info): """Test that the summary table task respects a date range.""" provider = 'AWS' provider_aws_uuid = self.aws_provider_uuid ce_table_name = AWS_CUR_TABLE_MAP['cost_entry'] daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.aws_accessor.report_schema, daily_table_name) summary_table = getattr(self.aws_accessor.report_schema, summary_table_name) ce_table = getattr(self.aws_accessor.report_schema, ce_table_name) with schema_context(self.schema): ce_start_date = ce_table.objects\ .filter(interval_start__gte=start_date)\ .aggregate(Min('interval_start'))['interval_start__min'] ce_end_date = ce_table.objects\ .filter(interval_start__lte=end_date)\ .aggregate(Max('interval_start'))['interval_start__max'] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace( hour=0, minute=0, second=0, microsecond=0 ) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace( hour=0, minute=0, second=0, microsecond=0 ) update_summary_tables( self.schema, provider, provider_aws_uuid, start_date, end_date ) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate( Min('usage_start'), Max('usage_end') ) result_start_date = daily_entry['usage_start__min'] result_end_date = daily_entry['usage_end__max'] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) with schema_context(self.schema): summary_entry = summary_table.objects.all().aggregate( Min('usage_start'), Max('usage_end') ) result_start_date = summary_entry['usage_start__min'] result_end_date = summary_entry['usage_end__max'] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date)
def load_gcp_data(self, customer, static_data_file): """Load GCP data into the database.""" provider_type = Provider.PROVIDER_GCP_LOCAL nise_provider_type = provider_type.replace("-local", "") credentials = {"project_id": "test_project_id"} data_source = {"table_id": "test_table_id", "dataset": "test_dataset"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) etag = uuid4() template, static_data_path = self.prepare_template( provider_type, static_data_file) options = { "static_report_file": static_data_path, "gcp_bucket_name": self.nise_data_path, "gcp_etag": etag } base_path = f"{self.nise_data_path}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write( template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = f"{base_path}/{etag}" for report in os.scandir(report_path): if os.path.isdir(report): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object( settings, "AUTO_DATA_INGEST", False): update_summary_tables(self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id) update_cost_model_costs.s(self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def test_update_summary_tables_aws_end_date(self, mock_charge_info): """Test that the summary table task respects a date range.""" provider = Provider.PROVIDER_AWS_LOCAL provider_aws_uuid = self.aws_provider_uuid ce_table_name = AWS_CUR_TABLE_MAP["cost_entry"] daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"] summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"] start_date = DateHelper().last_month_start end_date = DateHelper().last_month_end daily_table = getattr(self.aws_accessor.report_schema, daily_table_name) summary_table = getattr(self.aws_accessor.report_schema, summary_table_name) ce_table = getattr(self.aws_accessor.report_schema, ce_table_name) with schema_context(self.schema): daily_table.objects.all().delete() summary_table.objects.all().delete() ce_start_date = ce_table.objects.filter( interval_start__gte=start_date.date()).aggregate( Min("interval_start"))["interval_start__min"] ce_end_date = ce_table.objects.filter( interval_start__lte=end_date.date()).aggregate( Max("interval_start"))["interval_start__max"] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema, provider, provider_aws_uuid, start_date, end_date) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate( Min("usage_start"), Max("usage_end")) result_start_date = daily_entry["usage_start__min"] result_end_date = daily_entry["usage_end__max"] self.assertEqual(result_start_date, expected_start_date.date()) self.assertEqual(result_end_date, expected_end_date.date()) with schema_context(self.schema): summary_entry = summary_table.objects.all().aggregate( Min("usage_start"), Max("usage_end")) result_start_date = summary_entry["usage_start__min"] result_end_date = summary_entry["usage_end__max"] self.assertEqual(result_start_date, expected_start_date.date()) self.assertEqual(result_end_date, expected_end_date.date())
def load_openshift_data(self, customer, static_data_file, cluster_id): """Load OpenShift data into the database.""" provider_type = Provider.PROVIDER_OCP with override_settings(AUTO_DATA_INGEST=False): provider = baker.make( "Provider", type=provider_type, authentication__provider_resource_name=cluster_id, billing_source__bucket="", customer=customer, ) template, static_data_path = self.prepare_template( provider_type, static_data_file) options = { "static_report_file": static_data_path, "insights_upload": self.nise_data_path, "ocp_cluster_id": cluster_id, } base_path = f"{self.nise_data_path}/{cluster_id}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, num_processed_files=0, num_total_files=3, ) with open(static_data_path, "w") as f: f.write( template.render(start_date=start_date, end_date=end_date)) run(provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): shutil.move(report.path, f"{base_path}/{report.name}") for report in [f.path for f in os.scandir(base_path)]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"): update_summary_tables(self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id) update_cost_model_costs(self.schema, provider.uuid, self.dh.last_month_start, self.dh.today) refresh_materialized_views(self.schema, provider_type) shutil.rmtree(report_path, ignore_errors=True)
def test_update_summary_tables_aws_end_date(self, mock_charge_info): """Test that the summary table task respects a date range.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid ce_table_name = AWS_CUR_TABLE_MAP['cost_entry'] daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.aws_accessor.report_schema, daily_table_name) summary_table = getattr(self.aws_accessor.report_schema, summary_table_name) ce_table = getattr(self.aws_accessor.report_schema, ce_table_name) ce_start_date = self.aws_accessor._session\ .query(func.min(ce_table.interval_start))\ .filter(ce_table.interval_start >= start_date).first()[0] ce_end_date = self.aws_accessor._session\ .query(func.max(ce_table.interval_start))\ .filter(ce_table.interval_start <= end_date).first()[0] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date, end_date) result_start_date, result_end_date = self.aws_accessor._session.query( func.min(daily_table.usage_start), func.max(daily_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) result_start_date, result_end_date = self.aws_accessor._session.query( func.min(summary_table.usage_start), func.max(summary_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date)
def test_update_summary_tables_ocp(self, mock_cpu_rate, mock_mem_rate, mock_charge_info): """Test that the summary table task runs.""" mem_rate = {'tiered_rate': [{'value': '1.5', 'unit': 'USD'}]} cpu_rate = {'tiered_rate': [{'value': '2.5', 'unit': 'USD'}]} mock_cpu_rate.return_value = cpu_rate mock_mem_rate.return_value = mem_rate provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema_name, provider, provider_ocp_uuid, start_date) self.assertNotEqual(daily_query.count(), initial_daily_count) update_charge_info(schema_name=self.test_schema, provider_uuid=provider_ocp_uuid) table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary'] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(provider_obj.id) cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours) self.assertIsNotNone(item.pod_charge_cpu_core_hours) storage_daily_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily'] items = self.ocp_accessor._get_db_obj_query(storage_daily_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone(item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily_summary'] items = self.ocp_accessor._get_db_obj_query(storage_summary_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_gigabyte_months) self.assertIsNotNone(item.persistentvolumeclaim_usage_gigabyte_months)
def test_update_summary_tables_end_date(self): """Test that the summary table task respects a date range.""" ce_table_name = AWS_CUR_TABLE_MAP['cost_entry'] daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = datetime.utcnow() start_date = start_date.replace(day=1, month=(start_date.month - 1)).date() end_date = start_date + timedelta(days=10) daily_table = getattr(self.accessor.report_schema, daily_table_name) summary_table = getattr(self.accessor.report_schema, summary_table_name) ce_table = getattr(self.accessor.report_schema, ce_table_name) ce_start_date = self.accessor._session\ .query(func.min(ce_table.interval_start))\ .filter(func.date(ce_table.interval_start) >= start_date).first()[0] ce_end_date = self.accessor._session\ .query(func.max(ce_table.interval_start))\ .filter(func.date(ce_table.interval_start) <= end_date).first()[0] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date.date()) expected_end_date = min(end_date, ce_end_date.date()) update_summary_tables('testcustomer', start_date, end_date) result_start_date, result_end_date = self.accessor._session.query( func.min(daily_table.usage_start), func.max(daily_table.usage_start) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) result_start_date, result_end_date = self.accessor._session.query( func.min(summary_table.usage_start), func.max(summary_table.usage_start) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date)
def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info): """Test that the summary table task respects a date range.""" mock_cpu_rate.return_value = 1.5 mock_mem_rate.return_value = 2.5 provider = Provider.PROVIDER_OCP provider_ocp_uuid = self.ocp_test_provider_uuid ce_table_name = OCP_REPORT_TABLE_MAP["report"] daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name) ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name) with schema_context(self.schema): ce_start_date = ce_table.objects.filter(interval_start__gte=start_date).aggregate(Min("interval_start"))[ "interval_start__min" ] ce_end_date = ce_table.objects.filter(interval_start__lte=end_date).aggregate(Max("interval_start"))[ "interval_start__max" ] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate(Min("usage_start"), Max("usage_end")) result_start_date = daily_entry["usage_start__min"] result_end_date = daily_entry["usage_end__max"] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date)
def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info): """Test that the summary table task respects a date range.""" mock_cpu_rate.return_value = 1.5 mock_mem_rate.return_value = 2.5 provider = Provider.PROVIDER_OCP provider_ocp_uuid = self.ocp_test_provider_uuid ce_table_name = OCP_REPORT_TABLE_MAP["report"] daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"] start_date = DateHelper().last_month_start end_date = DateHelper().last_month_end daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name) ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name) with schema_context(self.schema): daily_table.objects.all().delete() ce_start_date = ce_table.objects.filter( interval_start__gte=start_date.date()).aggregate( Min("interval_start"))["interval_start__min"] ce_end_date = ce_table.objects.filter( interval_start__lte=end_date.date()).aggregate( Max("interval_start"))["interval_start__max"] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_end_date = min(end_date, ce_end_date) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate( Min("usage_start"), Max("usage_end")) result_start_date = daily_entry["usage_start__min"] result_end_date = daily_entry["usage_end__max"] self.assertEqual(result_start_date, expected_start_date.date()) self.assertEqual(result_end_date, expected_end_date.date())
def test_update_summary_tables_aws(self, mock_charge_info): """Test that the summary table task runs.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) daily_query = self.aws_accessor._get_db_obj_query(daily_table_name) summary_query = self.aws_accessor._get_db_obj_query(summary_table_name) initial_daily_count = daily_query.count() initial_summary_count = summary_query.count() self.assertEqual(initial_daily_count, 0) self.assertEqual(initial_summary_count, 0) update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date) self.assertNotEqual(daily_query.count(), initial_daily_count) self.assertNotEqual(summary_query.count(), initial_summary_count)
def test_update_summary_tables_ocp(self, mock_markup, mock_rate_map, mock_charge_info, mock_view, mock_chain): """Test that the summary table task runs.""" markup = {} mem_rate = {"tiered_rates": [{"value": "1.5", "unit": "USD"}]} cpu_rate = {"tiered_rates": [{"value": "2.5", "unit": "USD"}]} rate_metric_map = {"cpu_core_usage_per_hour": cpu_rate, "memory_gb_usage_per_hour": mem_rate} mock_markup.return_value = markup mock_rate_map.return_value = rate_metric_map provider = Provider.PROVIDER_OCP provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) with schema_context(self.schema): daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) update_charge_info( schema_name=self.schema, provider_uuid=provider_ocp_uuid, start_date=start_date, end_date=end_date ) table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(provider_obj.uuid) with schema_context(self.schema): cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours) self.assertIsNotNone(item.pod_charge_cpu_core_hours) storage_daily_name = OCP_REPORT_TABLE_MAP["storage_line_item_daily"] items = self.ocp_accessor._get_db_obj_query(storage_daily_name).filter(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone(item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"] items = self.ocp_accessor._get_db_obj_query(storage_summary_name).filter( cluster_id=cluster_id, data_source="Storage" ) for item in items: self.assertIsNotNone(item.volume_request_storage_gigabyte_months) self.assertIsNotNone(item.persistentvolumeclaim_usage_gigabyte_months) mock_chain.return_value.apply_async.assert_called()
def test_update_summary_tables_ocp(self, mock_markup, mock_rate_map, mock_charge_info, mock_cost_summary): """Test that the summary table task runs.""" markup = {} mem_rate = {'tiered_rates': [{'value': '1.5', 'unit': 'USD'}]} cpu_rate = {'tiered_rates': [{'value': '2.5', 'unit': 'USD'}]} rate_metric_map = { 'cpu_core_usage_per_hour': cpu_rate, 'memory_gb_usage_per_hour': mem_rate } mock_markup.return_value = markup mock_rate_map.return_value = rate_metric_map provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace( day=1) + relativedelta.relativedelta(months=-1) with schema_context(self.schema): daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) update_charge_info(schema_name=self.schema, provider_uuid=provider_ocp_uuid) table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary'] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider( provider_obj.id) with schema_context(self.schema): cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter( cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours) self.assertIsNotNone(item.pod_charge_cpu_core_hours) storage_daily_name = OCP_REPORT_TABLE_MAP[ 'storage_line_item_daily'] items = self.ocp_accessor._get_db_obj_query( storage_daily_name).filter(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone( item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP[ 'line_item_daily_summary'] items = self.ocp_accessor._get_db_obj_query( storage_summary_name).filter(cluster_id=cluster_id, data_source='Storage') for item in items: self.assertIsNotNone( item.volume_request_storage_gigabyte_months) self.assertIsNotNone( item.persistentvolumeclaim_usage_gigabyte_months) mock_charge_info.apply_async.assert_called() mock_cost_summary.si.assert_called()
def test_update_summary_tables_ocp(self, mock_cost_model, mock_charge_info, mock_view, mock_chain): """Test that the summary table task runs.""" infrastructure_rates = { "cpu_core_usage_per_hour": 1.5, "memory_gb_usage_per_hour": 2.5, "storage_gb_usage_per_month": 0.5, } markup = {} mock_cost_model.return_value.__enter__.return_value.infrastructure_rates = infrastructure_rates mock_cost_model.return_value.__enter__.return_value.supplementary_rates = {} mock_cost_model.return_value.__enter__.return_value.markup = markup provider = Provider.PROVIDER_OCP provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"] start_date = DateHelper().last_month_start end_date = DateHelper().last_month_end with schema_context(self.schema): daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) daily_query.delete() initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) update_cost_model_costs(schema_name=self.schema, provider_uuid=provider_ocp_uuid, start_date=start_date, end_date=end_date) table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider( provider_obj.uuid) with schema_context(self.schema): cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter( usage_start__gte=start_date, usage_start__lte=end_date, cluster_id=cluster_id, data_source="Pod") for item in items: self.assertNotEqual(item.infrastructure_usage_cost.get("cpu"), 0) self.assertNotEqual( item.infrastructure_usage_cost.get("memory"), 0) storage_daily_name = OCP_REPORT_TABLE_MAP[ "storage_line_item_daily"] items = self.ocp_accessor._get_db_obj_query( storage_daily_name).filter(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone( item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP[ "line_item_daily_summary"] items = self.ocp_accessor._get_db_obj_query( storage_summary_name).filter(cluster_id=cluster_id, data_source="Storage") for item in items: self.assertIsNotNone( item.volume_request_storage_gigabyte_months) self.assertIsNotNone( item.persistentvolumeclaim_usage_gigabyte_months) mock_chain.return_value.apply_async.assert_called()