def _generate_ocp_on_aws_data(self): """Test that the OCP on AWS cost summary table is populated.""" creator = ReportObjectCreator(self.accessor, self.column_map, self.accessor.report_schema.column_types) bill_ids = [] today = DateAccessor().today_with_timezone('UTC') last_month = today - relativedelta.relativedelta(months=1) resource_id = 'i-12345' for cost_entry_date in (today, last_month): bill = creator.create_cost_entry_bill(cost_entry_date) bill_ids.append(str(bill.id)) cost_entry = creator.create_cost_entry(bill, cost_entry_date) product = creator.create_cost_entry_product('Compute Instance') pricing = creator.create_cost_entry_pricing() reservation = creator.create_cost_entry_reservation() creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation, resource_id=resource_id) self.accessor.populate_line_item_daily_table(last_month, today, bill_ids) with OCPReportDBAccessor(self.test_schema, self.column_map) as ocp_accessor: cluster_id = self.ocp_provider_resource_name with ProviderDBAccessor(provider_uuid=self.ocp_test_provider_uuid ) as provider_access: provider_id = provider_access.get_provider().id for cost_entry_date in (today, last_month): period = creator.create_ocp_report_period( cost_entry_date, provider_id=provider_id, cluster_id=cluster_id) report = creator.create_ocp_report(period, cost_entry_date) creator.create_ocp_usage_line_item(period, report, resource_id=resource_id) cluster_id = get_cluster_id_from_provider( self.ocp_test_provider_uuid) ocp_accessor.populate_line_item_daily_table( last_month, today, cluster_id)
class OCPUtilTests(MasuTestCase): """Test the OCP utility functions.""" def setUp(self): super().setUp() self.common_accessor = ReportingCommonDBAccessor() self.column_map = self.common_accessor.column_map self.accessor = OCPReportDBAccessor(schema=self.test_schema, column_map=self.column_map) self.provider_accessor = ProviderDBAccessor( provider_uuid=self.ocp_test_provider_uuid) self.report_schema = self.accessor.report_schema self.creator = ReportObjectCreator(self.accessor, self.column_map, self.report_schema.column_types) self.all_tables = list(OCP_REPORT_TABLE_MAP.values()) self.provider_id = self.provider_accessor.get_provider().id reporting_period = self.creator.create_ocp_report_period( provider_id=self.provider_id) report = self.creator.create_ocp_report( reporting_period, reporting_period.report_period_start) self.creator.create_ocp_usage_line_item(reporting_period, report) self.creator.create_ocp_storage_line_item(reporting_period, report) def tearDown(self): """Return the database to a pre-test state.""" self.accessor._session.rollback() for table_name in self.all_tables: tables = self.accessor._get_db_obj_query(table_name).all() for table in tables: self.accessor._session.delete(table) self.accessor.commit() self.accessor.close_session() self.provider_accessor.close_session() self.common_accessor.close_session() def test_get_cluster_id_from_provider(self): """Test that the cluster ID is returned from OCP provider.""" cluster_id = utils.get_cluster_id_from_provider( self.ocp_test_provider_uuid) self.assertIsNotNone(cluster_id) def test_get_cluster_id_from_non_ocp_provider(self): """Test that None is returned when getting cluster ID on non-OCP provider.""" cluster_id = utils.get_cluster_id_from_provider( self.aws_test_provider_uuid) self.assertIsNone(cluster_id) def test_get_provider_uuid_from_cluster_id(self): """Test that the provider uuid is returned for a cluster ID.""" cluster_id = self.ocp_provider_resource_name provider_uuid = utils.get_provider_uuid_from_cluster_id(cluster_id) try: UUID(provider_uuid) except ValueError: self.fail('{} is not a valid uuid.'.format(str(provider_uuid))) def test_get_provider_uuid_from_invalid_cluster_id(self): """Test that the provider uuid is not returned for an invalid cluster ID.""" cluster_id = 'bad_cluster_id' provider_uuid = utils.get_provider_uuid_from_cluster_id(cluster_id) self.assertIsNone(provider_uuid) def test_poll_ingest_override_for_provider(self): """Test that OCP polling override returns True if insights local path exists.""" fake_dir = tempfile.mkdtemp() with patch.object(Config, 'INSIGHTS_LOCAL_REPORT_DIR', fake_dir): cluster_id = utils.get_cluster_id_from_provider( self.ocp_test_provider_uuid) expected_path = '{}/{}/'.format(Config.INSIGHTS_LOCAL_REPORT_DIR, cluster_id) os.makedirs(expected_path, exist_ok=True) self.assertTrue( utils.poll_ingest_override_for_provider( self.ocp_test_provider_uuid)) shutil.rmtree(fake_dir)
class TestUpdateSummaryTablesTask(MasuTestCase): """Test cases for Processor summary table Celery tasks.""" @classmethod def setUpClass(cls): """Setup for the class.""" super().setUpClass() cls.aws_tables = list(AWS_CUR_TABLE_MAP.values()) cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values()) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + \ list(OCP_REPORT_TABLE_MAP.values()) report_common_db = ReportingCommonDBAccessor() cls.column_map = report_common_db.column_map report_common_db.close_session() @classmethod def tearDownClass(cls): """Tear down the test class.""" super().tearDownClass() def setUp(self): """Set up each test.""" super().setUp() self.schema_name = self.test_schema self.aws_accessor = AWSReportDBAccessor(schema=self.schema_name, column_map=self.column_map) self.ocp_accessor = OCPReportDBAccessor(schema=self.schema_name, column_map=self.column_map) self.creator = ReportObjectCreator( self.aws_accessor, self.column_map, self.aws_accessor.report_schema.column_types ) # Populate some line item data so that the summary tables # have something to pull from self.start_date = DateAccessor().today_with_timezone('UTC').replace(day=1) last_month = self.start_date - relativedelta.relativedelta(months=1) for cost_entry_date in (self.start_date, last_month): bill = self.creator.create_cost_entry_bill(cost_entry_date) cost_entry = self.creator.create_cost_entry(bill, cost_entry_date) for family in ['Storage', 'Compute Instance', 'Database Storage', 'Database Instance']: product = self.creator.create_cost_entry_product(family) pricing = self.creator.create_cost_entry_pricing() reservation = self.creator.create_cost_entry_reservation() self.creator.create_cost_entry_line_item( bill, cost_entry, product, pricing, reservation ) provider_ocp_uuid = self.ocp_test_provider_uuid with ProviderDBAccessor(provider_uuid=provider_ocp_uuid) as provider_accessor: provider_id = provider_accessor.get_provider().id cluster_id = self.ocp_provider_resource_name for period_date in (self.start_date, last_month): period = self.creator.create_ocp_report_period(period_date, provider_id=provider_id, cluster_id=cluster_id) report = self.creator.create_ocp_report(period, period_date) for _ in range(25): self.creator.create_ocp_usage_line_item(period, report) def tearDown(self): """Return the database to a pre-test state.""" for table_name in self.aws_tables: tables = self.aws_accessor._get_db_obj_query(table_name).all() for table in tables: self.aws_accessor._session.delete(table) self.aws_accessor.commit() for table_name in self.ocp_tables: tables = self.ocp_accessor._get_db_obj_query(table_name).all() for table in tables: self.ocp_accessor._session.delete(table) self.ocp_accessor.commit() self.aws_accessor._session.rollback() self.aws_accessor.close_connections() self.aws_accessor.close_session() self.ocp_accessor.close_connections() self.ocp_accessor.close_session() super().tearDown() @patch('masu.processor.tasks.update_charge_info') def test_update_summary_tables_aws(self, mock_charge_info): """Test that the summary table task runs.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) daily_query = self.aws_accessor._get_db_obj_query(daily_table_name) summary_query = self.aws_accessor._get_db_obj_query(summary_table_name) initial_daily_count = daily_query.count() initial_summary_count = summary_query.count() self.assertEqual(initial_daily_count, 0) self.assertEqual(initial_summary_count, 0) update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date) self.assertNotEqual(daily_query.count(), initial_daily_count) self.assertNotEqual(summary_query.count(), initial_summary_count) @patch('masu.processor.tasks.update_charge_info') def test_update_summary_tables_aws_end_date(self, mock_charge_info): """Test that the summary table task respects a date range.""" provider = 'AWS' provider_aws_uuid = self.aws_test_provider_uuid ce_table_name = AWS_CUR_TABLE_MAP['cost_entry'] daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily'] summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary'] start_date = self.start_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.aws_accessor.report_schema, daily_table_name) summary_table = getattr(self.aws_accessor.report_schema, summary_table_name) ce_table = getattr(self.aws_accessor.report_schema, ce_table_name) ce_start_date = self.aws_accessor._session\ .query(func.min(ce_table.interval_start))\ .filter(ce_table.interval_start >= start_date).first()[0] ce_end_date = self.aws_accessor._session\ .query(func.max(ce_table.interval_start))\ .filter(ce_table.interval_start <= end_date).first()[0] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date, end_date) result_start_date, result_end_date = self.aws_accessor._session.query( func.min(daily_table.usage_start), func.max(daily_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) result_start_date, result_end_date = self.aws_accessor._session.query( func.min(summary_table.usage_start), func.max(summary_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) @patch('masu.processor.tasks.update_charge_info') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_memory_gb_usage_per_hour_rates') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_cpu_core_usage_per_hour_rates') def test_update_summary_tables_ocp(self, mock_cpu_rate, mock_mem_rate, mock_charge_info): """Test that the summary table task runs.""" mem_rate = {'tiered_rate': [{'value': '1.5', 'unit': 'USD'}]} cpu_rate = {'tiered_rate': [{'value': '2.5', 'unit': 'USD'}]} mock_cpu_rate.return_value = cpu_rate mock_mem_rate.return_value = mem_rate provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1) daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name) initial_daily_count = daily_query.count() self.assertEqual(initial_daily_count, 0) update_summary_tables(self.schema_name, provider, provider_ocp_uuid, start_date) self.assertNotEqual(daily_query.count(), initial_daily_count) update_charge_info(schema_name=self.test_schema, provider_uuid=provider_ocp_uuid) table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary'] with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor: provider_obj = provider_accessor.get_provider() usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(provider_obj.id) cluster_id = usage_period_qry.first().cluster_id items = self.ocp_accessor._get_db_obj_query(table_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours) self.assertIsNotNone(item.pod_charge_cpu_core_hours) storage_daily_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily'] items = self.ocp_accessor._get_db_obj_query(storage_daily_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_byte_seconds) self.assertIsNotNone(item.persistentvolumeclaim_usage_byte_seconds) storage_summary_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily_summary'] items = self.ocp_accessor._get_db_obj_query(storage_summary_name).filter_by(cluster_id=cluster_id) for item in items: self.assertIsNotNone(item.volume_request_storage_gigabyte_months) self.assertIsNotNone(item.persistentvolumeclaim_usage_gigabyte_months) @patch('masu.processor.tasks.update_charge_info') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_memory_gb_usage_per_hour_rates') @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_cpu_core_usage_per_hour_rates') def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info, ): """Test that the summary table task respects a date range.""" mock_cpu_rate.return_value = 1.5 mock_mem_rate.return_value = 2.5 provider = 'OCP' provider_ocp_uuid = self.ocp_test_provider_uuid ce_table_name = OCP_REPORT_TABLE_MAP['report'] daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily'] start_date = self.start_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + relativedelta.relativedelta(months=-1) end_date = start_date + timedelta(days=10) end_date = end_date.replace(hour=23, minute=59, second=59) daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name) ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name) ce_start_date = self.ocp_accessor._session\ .query(func.min(ce_table.interval_start))\ .filter(ce_table.interval_start >= start_date).first()[0] ce_end_date = self.ocp_accessor._session\ .query(func.max(ce_table.interval_start))\ .filter(ce_table.interval_start <= end_date).first()[0] # The summary tables will only include dates where there is data expected_start_date = max(start_date, ce_start_date) expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0) expected_end_date = min(end_date, ce_end_date) expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0) update_summary_tables(self.schema_name, provider, provider_ocp_uuid, start_date, end_date) result_start_date, result_end_date = self.ocp_accessor._session.query( func.min(daily_table.usage_start), func.max(daily_table.usage_end) ).first() self.assertEqual(result_start_date, expected_start_date) self.assertEqual(result_end_date, expected_end_date) def test_update_charge_info_aws(self): """Test that update_charge_info is not called for AWS.""" update_charge_info(schema_name=self.test_schema, provider_uuid=self.aws_test_provider_uuid) # FIXME: no asserts on test @patch('masu.processor.tasks.update_summary_tables') def test_get_report_data_for_all_providers(self, mock_update): """Test GET report_data endpoint with provider_uuid=*.""" start_date = date.today() update_all_summary_tables(start_date) mock_update.delay.assert_called_with( ANY, ANY, ANY, str(start_date), ANY)