def setUp(self): """Set up a test with database objects.""" super().setUp() today = DateAccessor().today_with_timezone('UTC') billing_start = today.replace(day=1) self.cluster_id = 'testcluster' self.manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_id': self.aws_provider.id, } bill = self.creator.create_cost_entry_bill( provider_id=self.aws_provider.id, bill_date=today, ) cost_entry = self.creator.create_cost_entry(bill, entry_datetime=today) product = self.creator.create_cost_entry_product() pricing = self.creator.create_cost_entry_pricing() reservation = self.creator.create_cost_entry_reservation() self.creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation) self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def _calculate_expiration_date(self): """ Calculate the expiration date based on the retention policy. Args: None Returns: (datetime.datetime) Expiration date """ today = DateAccessor().today() LOG.info('Current date time is %s', today) middle_of_current_month = today.replace(day=15) num_of_days_to_expire_date = self._months_to_keep * timedelta(days=30) middle_of_expire_date_month = middle_of_current_month - num_of_days_to_expire_date expiration_date = datetime(year=middle_of_expire_date_month.year, month=middle_of_expire_date_month.month, day=1, tzinfo=pytz.UTC) expiration_msg = 'Report data expiration is {} for a {} month retention policy' msg = expiration_msg.format(expiration_date, self._months_to_keep) LOG.info(msg) return expiration_date
def _calculate_expiration_date(self, line_items_only=False): """ Calculate the expiration date based on the retention policy. Args: None Returns: (datetime.datetime) Expiration date """ if line_items_only: months = self._line_items_months expiration_msg = "Line items expiration is {} for a {} month retention policy" else: months = self._months_to_keep expiration_msg = "Report data expiration is {} for a {} month retention policy" today = DateAccessor().today() LOG.info("Current date time is %s", today) middle_of_current_month = today.replace(day=15) num_of_days_to_expire_date = months * timedelta(days=30) middle_of_expire_date_month = middle_of_current_month - num_of_days_to_expire_date expiration_date = datetime(year=middle_of_expire_date_month.year, month=middle_of_expire_date_month.month, day=1, tzinfo=pytz.UTC) msg = expiration_msg.format(expiration_date, months) LOG.info(msg) return expiration_date
def test_update_summary_tables_without_manifest(self, mock_daily, mock_summary): """Test that summary tables are properly run without a manifest.""" self.updater = AWSReportSummaryUpdater('acct10001', self.provider, None) start_date = DateAccessor().today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) bill_date = start_date.replace(day=1).date() with schema_context(self.schema): bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0] bill.summary_data_updated_datetime = start_date self.accessor.commit() start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') expected_start_date = start_date.strftime('%Y-%m-%d') expected_end_date = end_date.strftime('%Y-%m-%d') self.updater.update_daily_tables(start_date_str, end_date_str) mock_daily.assert_called_with(expected_start_date, expected_end_date, [str(bill.id)]) mock_summary.assert_not_called() self.updater.update_summary_tables(start_date_str, end_date_str) mock_summary.assert_called_with(expected_start_date, expected_end_date, [str(bill.id)]) with AWSReportDBAccessor('acct10001', self.column_map) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] self.assertIsNotNone(bill.summary_data_creation_datetime) self.assertGreater(bill.summary_data_updated_datetime, start_date)
def _populate_storage_summary(self): """Helper to generate storage summary data.""" report_table_name = OCP_REPORT_TABLE_MAP['report'] summary_table_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily_summary'] report_table = getattr(self.accessor.report_schema, report_table_name) summary_table = getattr(self.accessor.report_schema, summary_table_name) start_date = DateAccessor().today_with_timezone('UTC') cluster_id = 'testcluster' period = self.creator.create_ocp_report_period(start_date, provider_id=self.ocp_provider_id, cluster_id=cluster_id) report = self.creator.create_ocp_report(period, start_date) for _ in range(25): self.creator.create_ocp_usage_line_item(period, report) start_date, end_date = self.accessor._session.query( func.min(report_table.interval_start), func.max(report_table.interval_start) ).first() start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace(hour=0, minute=0, second=0, microsecond=0) query = self.accessor._get_db_obj_query(summary_table_name) initial_count = query.count() self.accessor.populate_storage_line_item_daily_table(start_date, end_date, cluster_id) self.accessor.populate_storage_line_item_daily_summary_table(start_date, end_date, cluster_id)
def test_populate_line_item_daily_table(self): """Test that the line item daily table populates.""" report_table_name = OCP_REPORT_TABLE_MAP['report'] daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] report_table = getattr(self.accessor.report_schema, report_table_name) daily_table = getattr(self.accessor.report_schema, daily_table_name) start_date = DateAccessor().today_with_timezone('UTC') period = self.creator.create_ocp_report_period( start_date, provider_id=self.ocp_provider_id, cluster_id=self.cluster_id) report = self.creator.create_ocp_report(period, start_date) for _ in range(25): self.creator.create_ocp_usage_line_item(period, report) start_date, end_date = self.accessor._session.query( func.min(report_table.interval_start), func.max(report_table.interval_start)).first() start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace(hour=0, minute=0, second=0, microsecond=0) query = self.accessor._get_db_obj_query(daily_table_name) initial_count = query.count() self.accessor.populate_line_item_daily_table(start_date, end_date, self.cluster_id) self.assertNotEqual(query.count(), initial_count) result_start_date, result_end_date = self.accessor._session.query( func.min(daily_table.usage_start), func.max(daily_table.usage_start)).first() self.assertEqual(result_start_date, start_date) self.assertEqual(result_end_date, end_date) entry = query.first() summary_columns = [ 'cluster_id', 'namespace', 'node', 'node_capacity_cpu_core_seconds', 'node_capacity_cpu_cores', 'node_capacity_memory_byte_seconds', 'node_capacity_memory_bytes', 'pod', 'pod_labels', 'pod_limit_cpu_core_seconds', 'pod_limit_memory_byte_seconds', 'pod_request_cpu_core_seconds', 'pod_request_memory_byte_seconds', 'pod_usage_cpu_core_seconds', 'pod_usage_memory_byte_seconds', 'total_seconds', 'usage_end', 'usage_start' ] for column in summary_columns: self.assertIsNotNone(getattr(entry, column))
def test_azure_update_summary_cost_model_costs(self): """Test to verify Azure derived cost summary is calculated.""" updater = AzureCostModelCostUpdater(schema=self.schema, provider=self.azure_provider) start_date = DateAccessor().today_with_timezone("UTC") bill_date = start_date.replace(day=1).date() updater.update_summary_cost_model_costs() with AzureReportDBAccessor(self.schema) as accessor: bill = accessor.get_cost_entry_bills_by_date(bill_date)[0] self.assertIsNotNone(bill.derived_cost_datetime)
def setUp(self): """Set up a test with database objects.""" super().setUp() today = DateAccessor().today_with_timezone("UTC") billing_start = today.replace(day=1) self.manifest_dict = { "assembly_id": "1234", "billing_period_start_datetime": billing_start, "num_total_files": 2, "provider_id": self.gcp_provider.uuid, } self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def setUp(self): """Set up a test with database objects.""" super().setUp() today = DateAccessor().today_with_timezone("UTC") billing_start = today.replace(day=1) self.cluster_id = "testcluster" self.manifest_dict = { "assembly_id": "1234", "billing_period_start_datetime": billing_start, "num_total_files": 2, "provider_id": self.aws_provider.uuid, } self.creator.create_cost_entry_reservation()
def test_update_summary_tables_without_manifest( self, mock_daily, mock_sum, mock_storage_daily, mock_storage_summary ): """Test that summary tables are properly run without a manifest.""" # Create an updater that doesn't have a manifest updater = OCPReportSummaryUpdater(self.schema, self.provider, None) start_date = DateAccessor().today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) bill_date = start_date.replace(day=1).date() with schema_context(self.schema): period = self.accessor.get_usage_periods_by_date(bill_date)[0] period.summary_data_updated_datetime = start_date period.save() start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') expected_start_date = start_date.strftime('%Y-%m-%d') expected_end_date = end_date.strftime('%Y-%m-%d') updater.update_daily_tables(start_date_str, end_date_str) mock_daily.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) mock_storage_daily.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) mock_sum.assert_not_called() mock_storage_summary.assert_not_called() updater.update_summary_tables(start_date_str, end_date_str) mock_sum.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) mock_storage_summary.assert_called_with( expected_start_date, expected_end_date, self.report_period.cluster_id ) with OCPReportDBAccessor(self.schema, self.column_map) as accessor: period = accessor.get_usage_periods_by_date(bill_date)[0] self.assertIsNotNone(period.summary_data_creation_datetime) self.assertGreater(period.summary_data_updated_datetime, start_date)
def setUp(self): """Set up a test with database objects.""" super().setUp() today = DateAccessor().today_with_timezone("UTC") billing_start = today.replace(day=1) self.manifest_dict = { "assembly_id": "1234", "billing_period_start_datetime": billing_start, "num_total_files": 2, "provider_uuid": self.azure_provider_uuid, } bill = self.creator.create_azure_cost_entry_bill( provider_uuid=self.azure_provider_uuid, bill_date=today) product = self.creator.create_azure_cost_entry_product( provider_uuid=self.azure_provider_uuid) meter = self.creator.create_azure_meter( provider_uuid=self.azure_provider_uuid) self.creator.create_azure_cost_entry_line_item(bill, product, meter) self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def setUp(self): """Set up a test with database objects.""" super().setUp() today = DateAccessor().today_with_timezone('UTC') billing_start = today.replace(day=1) self.manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_id': self.azure_provider_id, } bill = self.creator.create_azure_cost_entry_bill( provider_id=self.azure_provider.id, bill_date=today, ) product = self.creator.create_azure_cost_entry_product() meter = self.creator.create_azure_meter() self.creator.create_azure_cost_entry_line_item(bill, product, meter) self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def setUp(self): """Set up a test with database objects.""" super().setUp() today = DateAccessor().today_with_timezone("UTC") billing_start = today.replace(day=1) self.cluster_id = "testcluster" self.manifest_dict = { "assembly_id": "1234", "billing_period_start_datetime": billing_start, "num_total_files": 2, "provider_id": self.aws_provider.uuid, } bill = self.creator.create_cost_entry_bill(provider_uuid=self.aws_provider.uuid, bill_date=today) cost_entry = self.creator.create_cost_entry(bill, entry_datetime=today) product = self.creator.create_cost_entry_product() pricing = self.creator.create_cost_entry_pricing() reservation = self.creator.create_cost_entry_reservation() self.creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation) self.manifest = self.manifest_accessor.add(**self.manifest_dict)
class TestUpdateSummaryTablesTask(MasuTestCase): """Test cases for Processor summary table Celery tasks.""" @classmethod def setUpClass(cls): """Set up for the class.""" super().setUpClass() cls.aws_tables = list(AWS_CUR_TABLE_MAP.values()) cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values()) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + list(OCP_REPORT_TABLE_MAP.values()) with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.creator = ReportObjectCreator(cls.schema, cls.column_map) def setUp(self): """Set up each test.""" super().setUp() self.aws_accessor = AWSReportDBAccessor(schema=self.schema, column_map=self.column_map) self.ocp_accessor = OCPReportDBAccessor(schema=self.schema, column_map=self.column_map) # Populate some line item data so that the summary tables # have something to pull from self.start_date = DateAccessor().today_with_timezone("UTC").replace(day=1) last_month = self.start_date - relativedelta.relativedelta(months=1) for cost_entry_date in (self.start_date, last_month): bill = self.creator.create_cost_entry_bill(provider_uuid=self.aws_provider_uuid, bill_date=cost_entry_date) cost_entry = self.creator.create_cost_entry(bill, cost_entry_date) for family in ["Storage", "Compute Instance", "Database Storage", "Database Instance"]: product = self.creator.create_cost_entry_product(family) pricing = self.creator.create_cost_entry_pricing() reservation = self.creator.create_cost_entry_reservation() self.creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation) provider_ocp_uuid = self.ocp_test_provider_uuid with ProviderDBAccessor(provider_uuid=provider_ocp_uuid) as provider_accessor: provider_uuid = provider_accessor.get_provider().uuid cluster_id = self.ocp_provider_resource_name for period_date in (self.start_date, last_month): period = self.creator.create_ocp_report_period( provider_uuid=provider_uuid, period_date=period_date, cluster_id=cluster_id ) report = self.creator.create_ocp_report(period, period_date) for _ in range(25): self.creator.create_ocp_usage_line_item(period, report) @patch("masu.processor.tasks.chain") @patch("masu.processor.tasks.refresh_materialized_views") @patch("masu.processor.tasks.update_charge_info") def test_update_summary_tables_aws(self, mock_charge_info, mock_views, mock_chain): """Test that the summary table task runs.""" provider = Provider.PROVIDER_AWS provider_aws_uuid = self.aws_provider_uuid daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"] summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) with schema_context(self.schema): daily_query = self.aws_accessor._get_db_obj_query(daily_table_name) summary_query = self.aws_accessor._get_db_obj_query(summary_table_name) initial_daily_count = daily_query.count() initial_summary_count = summary_query.count() self.assertEqual(initial_daily_count, 0) self.assertEqual(initial_summary_count, 0) update_summary_tables(self.schema, provider, provider_aws_uuid, start_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) self.assertNotEqual(summary_query.count(), initial_summary_count) mock_chain.return_value.apply_async.assert_called() @patch("masu.processor.tasks.update_charge_info") def test_update_summary_tables_aws_end_date(self, mock_charge_info): """Test that the summary table task respects a date range.""" provider = Provider.PROVIDER_AWS provider_aws_uuid = self.aws_provider_uuid ce_table_name = AWS_CUR_TABLE_MAP["cost_entry"] daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"] summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.aws_accessor.report_schema, daily_table_name) summary_table = getattr(self.aws_accessor.report_schema, summary_table_name) ce_table = getattr(self.aws_accessor.report_schema, ce_table_name) with schema_context(self.schema): ce_start_date = ce_table.objects.filter(interval_start__gte=start_date).aggregate(Min("interval_start"))[ "interval_start__min" ] ce_end_date = ce_table.objects.filter(interval_start__lte=end_date).aggregate(Max("interval_start"))[ "interval_start__max" ] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema, provider, provider_aws_uuid, start_date, end_date) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate(Min("usage_start"), Max("usage_end")) result_start_date = daily_entry["usage_start__min"] result_end_date = daily_entry["usage_end__max"] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) with schema_context(self.schema): summary_entry = summary_table.objects.all().aggregate(Min("usage_start"), Max("usage_end")) result_start_date = summary_entry["usage_start__min"] result_end_date = summary_entry["usage_end__max"] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) @patch("masu.processor.tasks.chain") @patch("masu.processor.tasks.refresh_materialized_views") @patch("masu.processor.tasks.update_charge_info") @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor._make_rate_by_metric_map") @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor.get_markup") def test_update_summary_tables_ocp(self, mock_markup, mock_rate_map, mock_charge_info, mock_view, mock_chain): """Test that the summary table task runs.""" markup = {} mem_rate = {"tiered_rates": [{"value": "1.5", "unit": "USD"}]} cpu_rate = {"tiered_rates": [{"value": "2.5", "unit": "USD"}]} rate_metric_map = {"cpu_core_usage_per_hour": cpu_rate, "memory_gb_usage_per_hour": mem_rate} mock_markup.return_value = markup mock_rate_map.return_value = rate_metric_map provider = Provider.PROVIDER_OCP provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) with schema_context(self.schema): daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) update_charge_info( schema_name=self.schema, provider_uuid=provider_ocp_uuid, start_date=start_date, end_date=end_date ) table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(provider_obj.uuid) with schema_context(self.schema): cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours) self.assertIsNotNone(item.pod_charge_cpu_core_hours) storage_daily_name = OCP_REPORT_TABLE_MAP["storage_line_item_daily"] items = self.ocp_accessor._get_db_obj_query(storage_daily_name).filter(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone(item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"] items = self.ocp_accessor._get_db_obj_query(storage_summary_name).filter( cluster_id=cluster_id, data_source="Storage" ) for item in items: self.assertIsNotNone(item.volume_request_storage_gigabyte_months) self.assertIsNotNone(item.persistentvolumeclaim_usage_gigabyte_months) mock_chain.return_value.apply_async.assert_called() @patch("masu.processor.tasks.update_charge_info") @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor.get_memory_gb_usage_per_hour_rates") @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor.get_cpu_core_usage_per_hour_rates") def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info): """Test that the summary table task respects a date range.""" mock_cpu_rate.return_value = 1.5 mock_mem_rate.return_value = 2.5 provider = Provider.PROVIDER_OCP provider_ocp_uuid = self.ocp_test_provider_uuid ce_table_name = OCP_REPORT_TABLE_MAP["report"] daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name) ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name) with schema_context(self.schema): ce_start_date = ce_table.objects.filter(interval_start__gte=start_date).aggregate(Min("interval_start"))[ "interval_start__min" ] ce_end_date = ce_table.objects.filter(interval_start__lte=end_date).aggregate(Max("interval_start"))[ "interval_start__max" ] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate(Min("usage_start"), Max("usage_end")) result_start_date = daily_entry["usage_start__min"] result_end_date = daily_entry["usage_end__max"] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) @patch("masu.processor.tasks.update_summary_tables") def test_get_report_data_for_all_providers(self, mock_update): """Test GET report_data endpoint with provider_uuid=*.""" start_date = date.today() update_all_summary_tables(start_date) mock_update.delay.assert_called_with(ANY, ANY, ANY, str(start_date), ANY) def test_refresh_materialized_views(self): """Test that materialized views are refreshed.""" manifest_dict = { "assembly_id": "12345", "billing_period_start_datetime": DateAccessor().today_with_timezone("UTC"), "num_total_files": 2, "provider_uuid": self.aws_provider_uuid, "task": "170653c0-3e66-4b7e-a764-336496d7ca5a", } fake_aws = FakeAWSCostData(self.aws_provider) generator = AWSReportDataGenerator(self.tenant) generator.add_data_to_tenant(fake_aws) with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.add(**manifest_dict) manifest.save() refresh_materialized_views(self.schema, Provider.PROVIDER_AWS, manifest_id=manifest.id) views_to_check = [view for view in AWS_MATERIALIZED_VIEWS if "Cost" in view._meta.db_table] with schema_context(self.schema): for view in views_to_check: self.assertNotEqual(view.objects.count(), 0) with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(manifest.id) self.assertIsNotNone(manifest.manifest_completed_datetime) def test_vacuum_schema(self): """Test that the vacuum schema task runs.""" logging.disable(logging.NOTSET) expected = "INFO:masu.processor.tasks:VACUUM" with self.assertLogs("masu.processor.tasks", level="INFO") as logger: vacuum_schema(self.schema) self.assertIn(expected, logger.output)
class TestUpdateSummaryTablesTask(MasuTestCase): """Test cases for Processor summary table Celery tasks.""" @classmethod def setUpClass(cls): """Setup for the class.""" super().setUpClass() cls.aws_tables = list(AWS_CUR_TABLE_MAP.values()) cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values()) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + \ list(OCP_REPORT_TABLE_MAP.values()) report_common_db = ReportingCommonDBAccessor() cls.column_map = report_common_db.column_map report_common_db.close_session() @classmethod def tearDownClass(cls): """Tear down the test class.""" super().tearDownClass() def setUp(self): """Set up each test.""" super().setUp() self.schema_name = self.test_schema self.aws_accessor = AWSReportDBAccessor(schema=self.schema_name, column_map=self.column_map) self.ocp_accessor = OCPReportDBAccessor(schema=self.schema_name, column_map=self.column_map) self.creator = ReportObjectCreator( self.aws_accessor, self.column_map, self.aws_accessor.report_schema.column_types ) # Populate some line item data so that the summary tables # have something to pull from self.start_date = DateAccessor().today_with_timezone('UTC').replace(day=1) last_month = self.start_date - relativedelta.relativedelta(months=1) for cost_entry_date in (self.start_date, last_month): bill = self.creator.create_cost_entry_bill(cost_entry_date) cost_entry = self.creator.create_cost_entry(bill, cost_entry_date) for family in ['Storage', 'Compute Instance', 'Database Storage', 'Database Instance']: product = self.creator.create_cost_entry_product(family) pricing = self.creator.create_cost_entry_pricing() reservation = self.creator.create_cost_entry_reservation() self.creator.create_cost_entry_line_item( bill, cost_entry, product, pricing, reservation ) provider_ocp_uuid = self.ocp_test_provider_uuid with ProviderDBAccessor(provider_uuid=provider_ocp_uuid) as provider_accessor: provider_id = provider_accessor.get_provider().id cluster_id = self.ocp_provider_resource_name for period_date in (self.start_date, last_month): period = self.creator.create_ocp_report_period(period_date, provider_id=provider_id, cluster_id=cluster_id) report = self.creator.create_ocp_report(period, period_date) for _ in range(25): self.creator.create_ocp_usage_line_item(period, report) def tearDown(self): """Return the database to a pre-test state.""" for table_name in self.aws_tables: tables = self.aws_accessor._get_db_obj_query(table_name).all() for table in tables: self.aws_accessor._session.delete(table) self.aws_accessor.commit() for table_name in self.ocp_tables: tables = self.ocp_accessor._get_db_obj_query(table_name).all() for table in tables: self.ocp_accessor._session.delete(table) self.ocp_accessor.commit() self.aws_accessor._session.rollback() self.aws_accessor.close_connections() self.aws_accessor.close_session() self.ocp_accessor.close_connections() self.ocp_accessor.close_session() super().tearDown() @patch('masu.processor.tasks.update_charge_info') def test_update_summary_tables_aws(self, mock_charge_info): """Test that the summary table task runs.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) daily_query = self.aws_accessor._get_db_obj_query(daily_table_name) summary_query = self.aws_accessor._get_db_obj_query(summary_table_name) initial_daily_count = daily_query.count() initial_summary_count = summary_query.count() self.assertEqual(initial_daily_count, 0) self.assertEqual(initial_summary_count, 0) update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date) self.assertNotEqual(daily_query.count(), initial_daily_count) self.assertNotEqual(summary_query.count(), initial_summary_count) @patch('masu.processor.tasks.update_charge_info') def test_update_summary_tables_aws_end_date(self, mock_charge_info): """Test that the summary table task respects a date range.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid ce_table_name = AWS_CUR_TABLE_MAP['cost_entry'] daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.aws_accessor.report_schema, daily_table_name) summary_table = getattr(self.aws_accessor.report_schema, summary_table_name) ce_table = getattr(self.aws_accessor.report_schema, ce_table_name) ce_start_date = self.aws_accessor._session\ .query(func.min(ce_table.interval_start))\ .filter(ce_table.interval_start >= start_date).first()[0] ce_end_date = self.aws_accessor._session\ .query(func.max(ce_table.interval_start))\ .filter(ce_table.interval_start <= end_date).first()[0] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date, end_date) result_start_date, result_end_date = self.aws_accessor._session.query( func.min(daily_table.usage_start), func.max(daily_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) result_start_date, result_end_date = self.aws_accessor._session.query( func.min(summary_table.usage_start), func.max(summary_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) @patch('masu.processor.tasks.update_charge_info') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_memory_gb_usage_per_hour_rates') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_cpu_core_usage_per_hour_rates') def test_update_summary_tables_ocp(self, mock_cpu_rate, mock_mem_rate, mock_charge_info): """Test that the summary table task runs.""" mem_rate = {'tiered_rate': [{'value': '1.5', 'unit': 'USD'}]} cpu_rate = {'tiered_rate': [{'value': '2.5', 'unit': 'USD'}]} mock_cpu_rate.return_value = cpu_rate mock_mem_rate.return_value = mem_rate provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema_name, provider, provider_ocp_uuid, start_date) self.assertNotEqual(daily_query.count(), initial_daily_count) update_charge_info(schema_name=self.test_schema, provider_uuid=provider_ocp_uuid) table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary'] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(provider_obj.id) cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours) self.assertIsNotNone(item.pod_charge_cpu_core_hours) storage_daily_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily'] items = self.ocp_accessor._get_db_obj_query(storage_daily_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone(item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily_summary'] items = self.ocp_accessor._get_db_obj_query(storage_summary_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_gigabyte_months) self.assertIsNotNone(item.persistentvolumeclaim_usage_gigabyte_months) @patch('masu.processor.tasks.update_charge_info') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_memory_gb_usage_per_hour_rates') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_cpu_core_usage_per_hour_rates') def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info, ): """Test that the summary table task respects a date range.""" mock_cpu_rate.return_value = 1.5 mock_mem_rate.return_value = 2.5 provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid ce_table_name = OCP_REPORT_TABLE_MAP['report'] daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name) ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name) ce_start_date = self.ocp_accessor._session\ .query(func.min(ce_table.interval_start))\ .filter(ce_table.interval_start >= start_date).first()[0] ce_end_date = self.ocp_accessor._session\ .query(func.max(ce_table.interval_start))\ .filter(ce_table.interval_start <= end_date).first()[0] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema_name, provider, provider_ocp_uuid, start_date, end_date) result_start_date, result_end_date = self.ocp_accessor._session.query( func.min(daily_table.usage_start), func.max(daily_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) def test_update_charge_info_aws(self): """Test that update_charge_info is not called for AWS.""" update_charge_info(schema_name=self.test_schema, provider_uuid=self.aws_test_provider_uuid) # FIXME: no asserts on test @patch('masu.processor.tasks.update_summary_tables') def test_get_report_data_for_all_providers(self, mock_update): """Test GET report_data endpoint with provider_uuid=*.""" start_date = date.today() update_all_summary_tables(start_date) mock_update.delay.assert_called_with( ANY, ANY, ANY, str(start_date), ANY)
class TestUpdateSummaryTablesTask(MasuTestCase): """Test cases for Processor summary table Celery tasks.""" @classmethod def setUpClass(cls): """Setup for the class.""" super().setUpClass() cls.aws_tables = list(AWS_CUR_TABLE_MAP.values()) cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values()) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + list( OCP_REPORT_TABLE_MAP.values()) with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.creator = ReportObjectCreator(cls.schema, cls.column_map) def setUp(self): """Set up each test.""" super().setUp() self.aws_accessor = AWSReportDBAccessor(schema=self.schema, column_map=self.column_map) self.ocp_accessor = OCPReportDBAccessor(schema=self.schema, column_map=self.column_map) # Populate some line item data so that the summary tables # have something to pull from self.start_date = DateAccessor().today_with_timezone('UTC').replace( day=1) last_month = self.start_date - relativedelta.relativedelta(months=1) for cost_entry_date in (self.start_date, last_month): bill = self.creator.create_cost_entry_bill( provider_id=self.aws_provider.id, bill_date=cost_entry_date) cost_entry = self.creator.create_cost_entry(bill, cost_entry_date) for family in [ 'Storage', 'Compute Instance', 'Database Storage', 'Database Instance', ]: product = self.creator.create_cost_entry_product(family) pricing = self.creator.create_cost_entry_pricing() reservation = self.creator.create_cost_entry_reservation() self.creator.create_cost_entry_line_item( bill, cost_entry, product, pricing, reservation) provider_ocp_uuid = self.ocp_test_provider_uuid with ProviderDBAccessor( provider_uuid=provider_ocp_uuid) as provider_accessor: provider_id = provider_accessor.get_provider().id cluster_id = self.ocp_provider_resource_name for period_date in (self.start_date, last_month): period = self.creator.create_ocp_report_period( period_date, provider_id=provider_id, cluster_id=cluster_id) report = self.creator.create_ocp_report(period, period_date) for _ in range(25): self.creator.create_ocp_usage_line_item(period, report) @patch('masu.processor.tasks.update_cost_summary_table') @patch('masu.processor.tasks.update_charge_info') def test_update_summary_tables_aws(self, mock_charge_info, mock_cost_summary): """Test that the summary table task runs.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace( day=1) + relativedelta.relativedelta(months=-1) with schema_context(self.schema): daily_query = self.aws_accessor._get_db_obj_query(daily_table_name) summary_query = self.aws_accessor._get_db_obj_query( summary_table_name) initial_daily_count = daily_query.count() initial_summary_count = summary_query.count() self.assertEqual(initial_daily_count, 0) self.assertEqual(initial_summary_count, 0) update_summary_tables(self.schema, provider, provider_aws_uuid, start_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) self.assertNotEqual(summary_query.count(), initial_summary_count) mock_charge_info.apply_async.assert_called() mock_cost_summary.si.assert_called() @patch('masu.processor.tasks.update_charge_info') def test_update_summary_tables_aws_end_date(self, mock_charge_info): """Test that the summary table task respects a date range.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid ce_table_name = AWS_CUR_TABLE_MAP['cost_entry'] daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.aws_accessor.report_schema, daily_table_name) summary_table = getattr(self.aws_accessor.report_schema, summary_table_name) ce_table = getattr(self.aws_accessor.report_schema, ce_table_name) with schema_context(self.schema): ce_start_date = ce_table.objects\ .filter(interval_start__gte=start_date)\ .aggregate(Min('interval_start'))['interval_start__min'] ce_end_date = ce_table.objects\ .filter(interval_start__lte=end_date)\ .aggregate(Max('interval_start'))['interval_start__max'] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema, provider, provider_aws_uuid, start_date, end_date) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate( Min('usage_start'), Max('usage_end')) result_start_date = daily_entry['usage_start__min'] result_end_date = daily_entry['usage_end__max'] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) with schema_context(self.schema): summary_entry = summary_table.objects.all().aggregate( Min('usage_start'), Max('usage_end')) result_start_date = summary_entry['usage_start__min'] result_end_date = summary_entry['usage_end__max'] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) @patch('masu.processor.tasks.update_cost_summary_table') @patch('masu.processor.tasks.update_charge_info') @patch( 'masu.database.cost_model_db_accessor.CostModelDBAccessor._make_rate_by_metric_map' ) @patch( 'masu.database.cost_model_db_accessor.CostModelDBAccessor.get_markup') def test_update_summary_tables_ocp(self, mock_markup, mock_rate_map, mock_charge_info, mock_cost_summary): """Test that the summary table task runs.""" markup = {} mem_rate = {'tiered_rates': [{'value': '1.5', 'unit': 'USD'}]} cpu_rate = {'tiered_rates': [{'value': '2.5', 'unit': 'USD'}]} rate_metric_map = { 'cpu_core_usage_per_hour': cpu_rate, 'memory_gb_usage_per_hour': mem_rate } mock_markup.return_value = markup mock_rate_map.return_value = rate_metric_map provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace( day=1) + relativedelta.relativedelta(months=-1) with schema_context(self.schema): daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date) with schema_context(self.schema): self.assertNotEqual(daily_query.count(), initial_daily_count) update_charge_info(schema_name=self.schema, provider_uuid=provider_ocp_uuid) table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary'] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider( provider_obj.id) with schema_context(self.schema): cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter( cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours) self.assertIsNotNone(item.pod_charge_cpu_core_hours) storage_daily_name = OCP_REPORT_TABLE_MAP[ 'storage_line_item_daily'] items = self.ocp_accessor._get_db_obj_query( storage_daily_name).filter(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone( item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP[ 'line_item_daily_summary'] items = self.ocp_accessor._get_db_obj_query( storage_summary_name).filter(cluster_id=cluster_id, data_source='Storage') for item in items: self.assertIsNotNone( item.volume_request_storage_gigabyte_months) self.assertIsNotNone( item.persistentvolumeclaim_usage_gigabyte_months) mock_charge_info.apply_async.assert_called() mock_cost_summary.si.assert_called() @patch('masu.processor.tasks.update_charge_info') @patch( 'masu.database.cost_model_db_accessor.CostModelDBAccessor.get_memory_gb_usage_per_hour_rates' ) @patch( 'masu.database.cost_model_db_accessor.CostModelDBAccessor.get_cpu_core_usage_per_hour_rates' ) def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info): """Test that the summary table task respects a date range.""" mock_cpu_rate.return_value = 1.5 mock_mem_rate.return_value = 2.5 provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid ce_table_name = OCP_REPORT_TABLE_MAP['report'] daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name) ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name) with schema_context(self.schema): ce_start_date = ce_table.objects\ .filter(interval_start__gte=start_date)\ .aggregate(Min('interval_start'))['interval_start__min'] ce_end_date = ce_table.objects\ .filter(interval_start__lte=end_date)\ .aggregate(Max('interval_start'))['interval_start__max'] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date) with schema_context(self.schema): daily_entry = daily_table.objects.all().aggregate( Min('usage_start'), Max('usage_end')) result_start_date = daily_entry['usage_start__min'] result_end_date = daily_entry['usage_end__max'] self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) @patch('masu.processor.tasks.update_summary_tables') def test_get_report_data_for_all_providers(self, mock_update): """Test GET report_data endpoint with provider_uuid=*.""" start_date = date.today() update_all_summary_tables(start_date) mock_update.delay.assert_called_with(ANY, ANY, ANY, str(start_date), ANY) @patch( 'masu.database.ocp_report_db_accessor.OCPReportDBAccessor.populate_cost_summary_table' ) def test_update_cost_summary_table(self, mock_update): """Tests that the updater updates the cost summary table.""" provider = 'OCP' provider_aws_uuid = self.ocp_test_provider_uuid manifest_id = None start_date = self.start_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) update_cost_summary_table(self.schema, provider_aws_uuid, start_date=start_date) mock_update.assert_called()