Exemplo n.º 1
0
    def setUp(self):
        """Set up a test with database objects."""
        super().setUp()
        today = DateAccessor().today_with_timezone('UTC')
        billing_start = today.replace(day=1)

        self.cluster_id = 'testcluster'

        self.manifest_dict = {
            'assembly_id': '1234',
            'billing_period_start_datetime': billing_start,
            'num_total_files': 2,
            'provider_id': self.aws_provider.id,
        }

        bill = self.creator.create_cost_entry_bill(
            provider_id=self.aws_provider.id,
            bill_date=today,
        )
        cost_entry = self.creator.create_cost_entry(bill, entry_datetime=today)
        product = self.creator.create_cost_entry_product()
        pricing = self.creator.create_cost_entry_pricing()
        reservation = self.creator.create_cost_entry_reservation()
        self.creator.create_cost_entry_line_item(bill, cost_entry, product,
                                                 pricing, reservation)
        self.manifest = self.manifest_accessor.add(**self.manifest_dict)
Exemplo n.º 2
0
    def _calculate_expiration_date(self):
        """
        Calculate the expiration date based on the retention policy.

        Args:
            None

        Returns:
            (datetime.datetime) Expiration date

        """
        today = DateAccessor().today()
        LOG.info('Current date time is %s', today)

        middle_of_current_month = today.replace(day=15)
        num_of_days_to_expire_date = self._months_to_keep * timedelta(days=30)
        middle_of_expire_date_month = middle_of_current_month - num_of_days_to_expire_date
        expiration_date = datetime(year=middle_of_expire_date_month.year,
                                   month=middle_of_expire_date_month.month,
                                   day=1,
                                   tzinfo=pytz.UTC)
        expiration_msg = 'Report data expiration is {} for a {} month retention policy'
        msg = expiration_msg.format(expiration_date, self._months_to_keep)
        LOG.info(msg)
        return expiration_date
Exemplo n.º 3
0
    def _calculate_expiration_date(self, line_items_only=False):
        """
        Calculate the expiration date based on the retention policy.

        Args:
            None

        Returns:
            (datetime.datetime) Expiration date

        """
        if line_items_only:
            months = self._line_items_months
            expiration_msg = "Line items expiration is {} for a {} month retention policy"
        else:
            months = self._months_to_keep
            expiration_msg = "Report data expiration is {} for a {} month retention policy"
        today = DateAccessor().today()
        LOG.info("Current date time is %s", today)

        middle_of_current_month = today.replace(day=15)
        num_of_days_to_expire_date = months * timedelta(days=30)
        middle_of_expire_date_month = middle_of_current_month - num_of_days_to_expire_date
        expiration_date = datetime(year=middle_of_expire_date_month.year,
                                   month=middle_of_expire_date_month.month,
                                   day=1,
                                   tzinfo=pytz.UTC)
        msg = expiration_msg.format(expiration_date, months)
        LOG.info(msg)
        return expiration_date
Exemplo n.º 4
0
    def test_update_summary_tables_without_manifest(self, mock_daily,
                                                    mock_summary):
        """Test that summary tables are properly run without a manifest."""
        self.updater = AWSReportSummaryUpdater('acct10001', self.provider,
                                               None)

        start_date = DateAccessor().today_with_timezone('UTC')
        end_date = start_date + datetime.timedelta(days=1)
        bill_date = start_date.replace(day=1).date()

        with schema_context(self.schema):
            bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
            bill.summary_data_updated_datetime = start_date
            self.accessor.commit()

        start_date_str = start_date.strftime('%Y-%m-%d')
        end_date_str = end_date.strftime('%Y-%m-%d')

        expected_start_date = start_date.strftime('%Y-%m-%d')
        expected_end_date = end_date.strftime('%Y-%m-%d')
        self.updater.update_daily_tables(start_date_str, end_date_str)
        mock_daily.assert_called_with(expected_start_date, expected_end_date,
                                      [str(bill.id)])
        mock_summary.assert_not_called()

        self.updater.update_summary_tables(start_date_str, end_date_str)
        mock_summary.assert_called_with(expected_start_date, expected_end_date,
                                        [str(bill.id)])

        with AWSReportDBAccessor('acct10001', self.column_map) as accessor:
            bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
            self.assertIsNotNone(bill.summary_data_creation_datetime)
            self.assertGreater(bill.summary_data_updated_datetime, start_date)
Exemplo n.º 5
0
    def _populate_storage_summary(self):
        """Helper to generate storage summary data."""
        report_table_name = OCP_REPORT_TABLE_MAP['report']
        summary_table_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily_summary']

        report_table = getattr(self.accessor.report_schema, report_table_name)
        summary_table = getattr(self.accessor.report_schema, summary_table_name)

        start_date = DateAccessor().today_with_timezone('UTC')

        cluster_id = 'testcluster'
        period = self.creator.create_ocp_report_period(start_date, provider_id=self.ocp_provider_id, cluster_id=cluster_id)
        report = self.creator.create_ocp_report(period, start_date)
        for _ in range(25):
            self.creator.create_ocp_usage_line_item(period, report)

        start_date, end_date = self.accessor._session.query(
            func.min(report_table.interval_start),
            func.max(report_table.interval_start)
        ).first()

        start_date = start_date.replace(hour=0, minute=0, second=0,
                                        microsecond=0)
        end_date = end_date.replace(hour=0, minute=0, second=0, microsecond=0)

        query = self.accessor._get_db_obj_query(summary_table_name)
        initial_count = query.count()

        self.accessor.populate_storage_line_item_daily_table(start_date, end_date, cluster_id)
        self.accessor.populate_storage_line_item_daily_summary_table(start_date, end_date, cluster_id)
Exemplo n.º 6
0
    def test_populate_line_item_daily_table(self):
        """Test that the line item daily table populates."""
        report_table_name = OCP_REPORT_TABLE_MAP['report']
        daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily']

        report_table = getattr(self.accessor.report_schema, report_table_name)
        daily_table = getattr(self.accessor.report_schema, daily_table_name)

        start_date = DateAccessor().today_with_timezone('UTC')

        period = self.creator.create_ocp_report_period(
            start_date,
            provider_id=self.ocp_provider_id,
            cluster_id=self.cluster_id)
        report = self.creator.create_ocp_report(period, start_date)
        for _ in range(25):
            self.creator.create_ocp_usage_line_item(period, report)

        start_date, end_date = self.accessor._session.query(
            func.min(report_table.interval_start),
            func.max(report_table.interval_start)).first()

        start_date = start_date.replace(hour=0,
                                        minute=0,
                                        second=0,
                                        microsecond=0)
        end_date = end_date.replace(hour=0, minute=0, second=0, microsecond=0)

        query = self.accessor._get_db_obj_query(daily_table_name)
        initial_count = query.count()

        self.accessor.populate_line_item_daily_table(start_date, end_date,
                                                     self.cluster_id)

        self.assertNotEqual(query.count(), initial_count)

        result_start_date, result_end_date = self.accessor._session.query(
            func.min(daily_table.usage_start),
            func.max(daily_table.usage_start)).first()

        self.assertEqual(result_start_date, start_date)
        self.assertEqual(result_end_date, end_date)

        entry = query.first()

        summary_columns = [
            'cluster_id', 'namespace', 'node',
            'node_capacity_cpu_core_seconds', 'node_capacity_cpu_cores',
            'node_capacity_memory_byte_seconds', 'node_capacity_memory_bytes',
            'pod', 'pod_labels', 'pod_limit_cpu_core_seconds',
            'pod_limit_memory_byte_seconds', 'pod_request_cpu_core_seconds',
            'pod_request_memory_byte_seconds', 'pod_usage_cpu_core_seconds',
            'pod_usage_memory_byte_seconds', 'total_seconds', 'usage_end',
            'usage_start'
        ]

        for column in summary_columns:
            self.assertIsNotNone(getattr(entry, column))
    def test_azure_update_summary_cost_model_costs(self):
        """Test to verify Azure derived cost summary is calculated."""
        updater = AzureCostModelCostUpdater(schema=self.schema, provider=self.azure_provider)
        start_date = DateAccessor().today_with_timezone("UTC")
        bill_date = start_date.replace(day=1).date()

        updater.update_summary_cost_model_costs()

        with AzureReportDBAccessor(self.schema) as accessor:
            bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
            self.assertIsNotNone(bill.derived_cost_datetime)
    def setUp(self):
        """Set up a test with database objects."""
        super().setUp()
        today = DateAccessor().today_with_timezone("UTC")
        billing_start = today.replace(day=1)

        self.manifest_dict = {
            "assembly_id": "1234",
            "billing_period_start_datetime": billing_start,
            "num_total_files": 2,
            "provider_id": self.gcp_provider.uuid,
        }
        self.manifest = self.manifest_accessor.add(**self.manifest_dict)
Exemplo n.º 9
0
    def setUp(self):
        """Set up a test with database objects."""
        super().setUp()
        today = DateAccessor().today_with_timezone("UTC")
        billing_start = today.replace(day=1)

        self.cluster_id = "testcluster"

        self.manifest_dict = {
            "assembly_id": "1234",
            "billing_period_start_datetime": billing_start,
            "num_total_files": 2,
            "provider_id": self.aws_provider.uuid,
        }
        self.creator.create_cost_entry_reservation()
Exemplo n.º 10
0
    def test_update_summary_tables_without_manifest(
        self, mock_daily, mock_sum, mock_storage_daily, mock_storage_summary
    ):
        """Test that summary tables are properly run without a manifest."""
        # Create an updater that doesn't have a manifest
        updater = OCPReportSummaryUpdater(self.schema, self.provider, None)
        start_date = DateAccessor().today_with_timezone('UTC')
        end_date = start_date + datetime.timedelta(days=1)
        bill_date = start_date.replace(day=1).date()

        with schema_context(self.schema):
            period = self.accessor.get_usage_periods_by_date(bill_date)[0]
            period.summary_data_updated_datetime = start_date
            period.save()

        start_date_str = start_date.strftime('%Y-%m-%d')
        end_date_str = end_date.strftime('%Y-%m-%d')

        expected_start_date = start_date.strftime('%Y-%m-%d')
        expected_end_date = end_date.strftime('%Y-%m-%d')

        updater.update_daily_tables(start_date_str, end_date_str)
        mock_daily.assert_called_with(
            expected_start_date, expected_end_date, self.report_period.cluster_id
        )
        mock_storage_daily.assert_called_with(
            expected_start_date, expected_end_date, self.report_period.cluster_id
        )
        mock_sum.assert_not_called()
        mock_storage_summary.assert_not_called()

        updater.update_summary_tables(start_date_str, end_date_str)
        mock_sum.assert_called_with(
            expected_start_date, expected_end_date, self.report_period.cluster_id
        )
        mock_storage_summary.assert_called_with(
            expected_start_date, expected_end_date, self.report_period.cluster_id
        )

        with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
            period = accessor.get_usage_periods_by_date(bill_date)[0]
            self.assertIsNotNone(period.summary_data_creation_datetime)
            self.assertGreater(period.summary_data_updated_datetime, start_date)
Exemplo n.º 11
0
    def setUp(self):
        """Set up a test with database objects."""
        super().setUp()
        today = DateAccessor().today_with_timezone("UTC")
        billing_start = today.replace(day=1)

        self.manifest_dict = {
            "assembly_id": "1234",
            "billing_period_start_datetime": billing_start,
            "num_total_files": 2,
            "provider_uuid": self.azure_provider_uuid,
        }

        bill = self.creator.create_azure_cost_entry_bill(
            provider_uuid=self.azure_provider_uuid, bill_date=today)
        product = self.creator.create_azure_cost_entry_product(
            provider_uuid=self.azure_provider_uuid)
        meter = self.creator.create_azure_meter(
            provider_uuid=self.azure_provider_uuid)
        self.creator.create_azure_cost_entry_line_item(bill, product, meter)
        self.manifest = self.manifest_accessor.add(**self.manifest_dict)
Exemplo n.º 12
0
    def setUp(self):
        """Set up a test with database objects."""
        super().setUp()
        today = DateAccessor().today_with_timezone('UTC')
        billing_start = today.replace(day=1)

        self.manifest_dict = {
            'assembly_id': '1234',
            'billing_period_start_datetime': billing_start,
            'num_total_files': 2,
            'provider_id': self.azure_provider_id,
        }

        bill = self.creator.create_azure_cost_entry_bill(
            provider_id=self.azure_provider.id,
            bill_date=today,
        )
        product = self.creator.create_azure_cost_entry_product()
        meter = self.creator.create_azure_meter()
        self.creator.create_azure_cost_entry_line_item(bill, product, meter)
        self.manifest = self.manifest_accessor.add(**self.manifest_dict)
Exemplo n.º 13
0
    def setUp(self):
        """Set up a test with database objects."""
        super().setUp()
        today = DateAccessor().today_with_timezone("UTC")
        billing_start = today.replace(day=1)

        self.cluster_id = "testcluster"

        self.manifest_dict = {
            "assembly_id": "1234",
            "billing_period_start_datetime": billing_start,
            "num_total_files": 2,
            "provider_id": self.aws_provider.uuid,
        }

        bill = self.creator.create_cost_entry_bill(provider_uuid=self.aws_provider.uuid, bill_date=today)
        cost_entry = self.creator.create_cost_entry(bill, entry_datetime=today)
        product = self.creator.create_cost_entry_product()
        pricing = self.creator.create_cost_entry_pricing()
        reservation = self.creator.create_cost_entry_reservation()
        self.creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation)
        self.manifest = self.manifest_accessor.add(**self.manifest_dict)
Exemplo n.º 14
0
class TestUpdateSummaryTablesTask(MasuTestCase):
    """Test cases for Processor summary table Celery tasks."""

    @classmethod
    def setUpClass(cls):
        """Set up for the class."""
        super().setUpClass()
        cls.aws_tables = list(AWS_CUR_TABLE_MAP.values())
        cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values())
        cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + list(OCP_REPORT_TABLE_MAP.values())
        with ReportingCommonDBAccessor() as report_common_db:
            cls.column_map = report_common_db.column_map

        cls.creator = ReportObjectCreator(cls.schema, cls.column_map)

    def setUp(self):
        """Set up each test."""
        super().setUp()
        self.aws_accessor = AWSReportDBAccessor(schema=self.schema, column_map=self.column_map)
        self.ocp_accessor = OCPReportDBAccessor(schema=self.schema, column_map=self.column_map)

        # Populate some line item data so that the summary tables
        # have something to pull from
        self.start_date = DateAccessor().today_with_timezone("UTC").replace(day=1)
        last_month = self.start_date - relativedelta.relativedelta(months=1)

        for cost_entry_date in (self.start_date, last_month):
            bill = self.creator.create_cost_entry_bill(provider_uuid=self.aws_provider_uuid, bill_date=cost_entry_date)
            cost_entry = self.creator.create_cost_entry(bill, cost_entry_date)
            for family in ["Storage", "Compute Instance", "Database Storage", "Database Instance"]:
                product = self.creator.create_cost_entry_product(family)
                pricing = self.creator.create_cost_entry_pricing()
                reservation = self.creator.create_cost_entry_reservation()
                self.creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation)
        provider_ocp_uuid = self.ocp_test_provider_uuid

        with ProviderDBAccessor(provider_uuid=provider_ocp_uuid) as provider_accessor:
            provider_uuid = provider_accessor.get_provider().uuid

        cluster_id = self.ocp_provider_resource_name
        for period_date in (self.start_date, last_month):
            period = self.creator.create_ocp_report_period(
                provider_uuid=provider_uuid, period_date=period_date, cluster_id=cluster_id
            )
            report = self.creator.create_ocp_report(period, period_date)
            for _ in range(25):
                self.creator.create_ocp_usage_line_item(period, report)

    @patch("masu.processor.tasks.chain")
    @patch("masu.processor.tasks.refresh_materialized_views")
    @patch("masu.processor.tasks.update_charge_info")
    def test_update_summary_tables_aws(self, mock_charge_info, mock_views, mock_chain):
        """Test that the summary table task runs."""
        provider = Provider.PROVIDER_AWS
        provider_aws_uuid = self.aws_provider_uuid

        daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"]
        summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"]
        start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1)

        with schema_context(self.schema):
            daily_query = self.aws_accessor._get_db_obj_query(daily_table_name)
            summary_query = self.aws_accessor._get_db_obj_query(summary_table_name)

            initial_daily_count = daily_query.count()
            initial_summary_count = summary_query.count()

        self.assertEqual(initial_daily_count, 0)
        self.assertEqual(initial_summary_count, 0)

        update_summary_tables(self.schema, provider, provider_aws_uuid, start_date)

        with schema_context(self.schema):
            self.assertNotEqual(daily_query.count(), initial_daily_count)
            self.assertNotEqual(summary_query.count(), initial_summary_count)

        mock_chain.return_value.apply_async.assert_called()

    @patch("masu.processor.tasks.update_charge_info")
    def test_update_summary_tables_aws_end_date(self, mock_charge_info):
        """Test that the summary table task respects a date range."""
        provider = Provider.PROVIDER_AWS
        provider_aws_uuid = self.aws_provider_uuid
        ce_table_name = AWS_CUR_TABLE_MAP["cost_entry"]
        daily_table_name = AWS_CUR_TABLE_MAP["line_item_daily"]
        summary_table_name = AWS_CUR_TABLE_MAP["line_item_daily_summary"]

        start_date = self.start_date.replace(
            day=1, hour=0, minute=0, second=0, microsecond=0
        ) + relativedelta.relativedelta(months=-1)

        end_date = start_date + timedelta(days=10)
        end_date = end_date.replace(hour=23, minute=59, second=59)

        daily_table = getattr(self.aws_accessor.report_schema, daily_table_name)
        summary_table = getattr(self.aws_accessor.report_schema, summary_table_name)
        ce_table = getattr(self.aws_accessor.report_schema, ce_table_name)

        with schema_context(self.schema):
            ce_start_date = ce_table.objects.filter(interval_start__gte=start_date).aggregate(Min("interval_start"))[
                "interval_start__min"
            ]
            ce_end_date = ce_table.objects.filter(interval_start__lte=end_date).aggregate(Max("interval_start"))[
                "interval_start__max"
            ]

        # The summary tables will only include dates where there is data
        expected_start_date = max(start_date, ce_start_date)
        expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0)
        expected_end_date = min(end_date, ce_end_date)
        expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0)

        update_summary_tables(self.schema, provider, provider_aws_uuid, start_date, end_date)

        with schema_context(self.schema):
            daily_entry = daily_table.objects.all().aggregate(Min("usage_start"), Max("usage_end"))
            result_start_date = daily_entry["usage_start__min"]
            result_end_date = daily_entry["usage_end__max"]

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

        with schema_context(self.schema):
            summary_entry = summary_table.objects.all().aggregate(Min("usage_start"), Max("usage_end"))
            result_start_date = summary_entry["usage_start__min"]
            result_end_date = summary_entry["usage_end__max"]

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

    @patch("masu.processor.tasks.chain")
    @patch("masu.processor.tasks.refresh_materialized_views")
    @patch("masu.processor.tasks.update_charge_info")
    @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor._make_rate_by_metric_map")
    @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor.get_markup")
    def test_update_summary_tables_ocp(self, mock_markup, mock_rate_map, mock_charge_info, mock_view, mock_chain):
        """Test that the summary table task runs."""
        markup = {}
        mem_rate = {"tiered_rates": [{"value": "1.5", "unit": "USD"}]}
        cpu_rate = {"tiered_rates": [{"value": "2.5", "unit": "USD"}]}
        rate_metric_map = {"cpu_core_usage_per_hour": cpu_rate, "memory_gb_usage_per_hour": mem_rate}

        mock_markup.return_value = markup
        mock_rate_map.return_value = rate_metric_map

        provider = Provider.PROVIDER_OCP
        provider_ocp_uuid = self.ocp_test_provider_uuid

        daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"]
        start_date = self.start_date.replace(
            day=1, hour=0, minute=0, second=0, microsecond=0
        ) + relativedelta.relativedelta(months=-1)
        end_date = start_date + timedelta(days=10)

        with schema_context(self.schema):
            daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name)

            initial_daily_count = daily_query.count()

        self.assertEqual(initial_daily_count, 0)
        update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date)

        with schema_context(self.schema):
            self.assertNotEqual(daily_query.count(), initial_daily_count)

        update_charge_info(
            schema_name=self.schema, provider_uuid=provider_ocp_uuid, start_date=start_date, end_date=end_date
        )

        table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"]
        with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor:
            provider_obj = provider_accessor.get_provider()

        usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(provider_obj.uuid)
        with schema_context(self.schema):
            cluster_id = usage_period_qry.first().cluster_id

            items = self.ocp_accessor._get_db_obj_query(table_name).filter(cluster_id=cluster_id)
            for item in items:
                self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours)
                self.assertIsNotNone(item.pod_charge_cpu_core_hours)

            storage_daily_name = OCP_REPORT_TABLE_MAP["storage_line_item_daily"]

            items = self.ocp_accessor._get_db_obj_query(storage_daily_name).filter(cluster_id=cluster_id)
            for item in items:
                self.assertIsNotNone(item.volume_request_storage_byte_seconds)
                self.assertIsNotNone(item.persistentvolumeclaim_usage_byte_seconds)

            storage_summary_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"]
            items = self.ocp_accessor._get_db_obj_query(storage_summary_name).filter(
                cluster_id=cluster_id, data_source="Storage"
            )
            for item in items:
                self.assertIsNotNone(item.volume_request_storage_gigabyte_months)
                self.assertIsNotNone(item.persistentvolumeclaim_usage_gigabyte_months)

        mock_chain.return_value.apply_async.assert_called()

    @patch("masu.processor.tasks.update_charge_info")
    @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor.get_memory_gb_usage_per_hour_rates")
    @patch("masu.database.cost_model_db_accessor.CostModelDBAccessor.get_cpu_core_usage_per_hour_rates")
    def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info):
        """Test that the summary table task respects a date range."""
        mock_cpu_rate.return_value = 1.5
        mock_mem_rate.return_value = 2.5
        provider = Provider.PROVIDER_OCP
        provider_ocp_uuid = self.ocp_test_provider_uuid
        ce_table_name = OCP_REPORT_TABLE_MAP["report"]
        daily_table_name = OCP_REPORT_TABLE_MAP["line_item_daily"]

        start_date = self.start_date.replace(
            day=1, hour=0, minute=0, second=0, microsecond=0
        ) + relativedelta.relativedelta(months=-1)

        end_date = start_date + timedelta(days=10)
        end_date = end_date.replace(hour=23, minute=59, second=59)

        daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name)
        ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name)

        with schema_context(self.schema):
            ce_start_date = ce_table.objects.filter(interval_start__gte=start_date).aggregate(Min("interval_start"))[
                "interval_start__min"
            ]

            ce_end_date = ce_table.objects.filter(interval_start__lte=end_date).aggregate(Max("interval_start"))[
                "interval_start__max"
            ]

        # The summary tables will only include dates where there is data
        expected_start_date = max(start_date, ce_start_date)
        expected_start_date = expected_start_date.replace(hour=0, minute=0, second=0, microsecond=0)
        expected_end_date = min(end_date, ce_end_date)
        expected_end_date = expected_end_date.replace(hour=0, minute=0, second=0, microsecond=0)

        update_summary_tables(self.schema, provider, provider_ocp_uuid, start_date, end_date)
        with schema_context(self.schema):
            daily_entry = daily_table.objects.all().aggregate(Min("usage_start"), Max("usage_end"))
            result_start_date = daily_entry["usage_start__min"]
            result_end_date = daily_entry["usage_end__max"]

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

    @patch("masu.processor.tasks.update_summary_tables")
    def test_get_report_data_for_all_providers(self, mock_update):
        """Test GET report_data endpoint with provider_uuid=*."""
        start_date = date.today()
        update_all_summary_tables(start_date)

        mock_update.delay.assert_called_with(ANY, ANY, ANY, str(start_date), ANY)

    def test_refresh_materialized_views(self):
        """Test that materialized views are refreshed."""
        manifest_dict = {
            "assembly_id": "12345",
            "billing_period_start_datetime": DateAccessor().today_with_timezone("UTC"),
            "num_total_files": 2,
            "provider_uuid": self.aws_provider_uuid,
            "task": "170653c0-3e66-4b7e-a764-336496d7ca5a",
        }
        fake_aws = FakeAWSCostData(self.aws_provider)
        generator = AWSReportDataGenerator(self.tenant)
        generator.add_data_to_tenant(fake_aws)

        with ReportManifestDBAccessor() as manifest_accessor:
            manifest = manifest_accessor.add(**manifest_dict)
            manifest.save()

        refresh_materialized_views(self.schema, Provider.PROVIDER_AWS, manifest_id=manifest.id)

        views_to_check = [view for view in AWS_MATERIALIZED_VIEWS if "Cost" in view._meta.db_table]

        with schema_context(self.schema):
            for view in views_to_check:
                self.assertNotEqual(view.objects.count(), 0)

        with ReportManifestDBAccessor() as manifest_accessor:
            manifest = manifest_accessor.get_manifest_by_id(manifest.id)
            self.assertIsNotNone(manifest.manifest_completed_datetime)

    def test_vacuum_schema(self):
        """Test that the vacuum schema task runs."""
        logging.disable(logging.NOTSET)
        expected = "INFO:masu.processor.tasks:VACUUM"
        with self.assertLogs("masu.processor.tasks", level="INFO") as logger:
            vacuum_schema(self.schema)
            self.assertIn(expected, logger.output)
Exemplo n.º 15
0
class TestUpdateSummaryTablesTask(MasuTestCase):
    """Test cases for Processor summary table Celery tasks."""

    @classmethod
    def setUpClass(cls):
        """Setup for the class."""
        super().setUpClass()
        cls.aws_tables = list(AWS_CUR_TABLE_MAP.values())
        cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values())
        cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + \
            list(OCP_REPORT_TABLE_MAP.values())
        report_common_db = ReportingCommonDBAccessor()
        cls.column_map = report_common_db.column_map
        report_common_db.close_session()

    @classmethod
    def tearDownClass(cls):
        """Tear down the test class."""
        super().tearDownClass()

    def setUp(self):
        """Set up each test."""
        super().setUp()
        self.schema_name = self.test_schema
        self.aws_accessor = AWSReportDBAccessor(schema=self.schema_name,
                                                column_map=self.column_map)
        self.ocp_accessor = OCPReportDBAccessor(schema=self.schema_name,
                                                column_map=self.column_map)

        self.creator = ReportObjectCreator(
            self.aws_accessor,
            self.column_map,
            self.aws_accessor.report_schema.column_types
        )

        # Populate some line item data so that the summary tables
        # have something to pull from
        self.start_date = DateAccessor().today_with_timezone('UTC').replace(day=1)
        last_month = self.start_date - relativedelta.relativedelta(months=1)

        for cost_entry_date in (self.start_date, last_month):
            bill = self.creator.create_cost_entry_bill(cost_entry_date)
            cost_entry = self.creator.create_cost_entry(bill, cost_entry_date)
            for family in ['Storage', 'Compute Instance', 'Database Storage',
                           'Database Instance']:
                product = self.creator.create_cost_entry_product(family)
                pricing = self.creator.create_cost_entry_pricing()
                reservation = self.creator.create_cost_entry_reservation()
                self.creator.create_cost_entry_line_item(
                    bill,
                    cost_entry,
                    product,
                    pricing,
                    reservation
                )
        provider_ocp_uuid = self.ocp_test_provider_uuid

        with ProviderDBAccessor(provider_uuid=provider_ocp_uuid) as provider_accessor:
            provider_id = provider_accessor.get_provider().id

        cluster_id = self.ocp_provider_resource_name
        for period_date in (self.start_date, last_month):
            period = self.creator.create_ocp_report_period(period_date, provider_id=provider_id,
                                                           cluster_id=cluster_id)
            report = self.creator.create_ocp_report(period, period_date)
            for _ in range(25):
                self.creator.create_ocp_usage_line_item(period, report)

    def tearDown(self):
        """Return the database to a pre-test state."""
        for table_name in self.aws_tables:
            tables = self.aws_accessor._get_db_obj_query(table_name).all()
            for table in tables:
                self.aws_accessor._session.delete(table)
        self.aws_accessor.commit()
        for table_name in self.ocp_tables:
            tables = self.ocp_accessor._get_db_obj_query(table_name).all()
            for table in tables:
                self.ocp_accessor._session.delete(table)
        self.ocp_accessor.commit()

        self.aws_accessor._session.rollback()
        self.aws_accessor.close_connections()
        self.aws_accessor.close_session()
        self.ocp_accessor.close_connections()
        self.ocp_accessor.close_session()
        super().tearDown()

    @patch('masu.processor.tasks.update_charge_info')
    def test_update_summary_tables_aws(self, mock_charge_info):
        """Test that the summary table task runs."""
        provider = 'AWS'
        provider_aws_uuid = self.aws_test_provider_uuid

        daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily']
        summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary']
        start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1)

        daily_query = self.aws_accessor._get_db_obj_query(daily_table_name)
        summary_query = self.aws_accessor._get_db_obj_query(summary_table_name)

        initial_daily_count = daily_query.count()
        initial_summary_count = summary_query.count()

        self.assertEqual(initial_daily_count, 0)
        self.assertEqual(initial_summary_count, 0)

        update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date)

        self.assertNotEqual(daily_query.count(), initial_daily_count)
        self.assertNotEqual(summary_query.count(), initial_summary_count)

    @patch('masu.processor.tasks.update_charge_info')
    def test_update_summary_tables_aws_end_date(self, mock_charge_info):
        """Test that the summary table task respects a date range."""
        provider = 'AWS'
        provider_aws_uuid = self.aws_test_provider_uuid
        ce_table_name = AWS_CUR_TABLE_MAP['cost_entry']
        daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily']
        summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary']

        start_date = self.start_date.replace(day=1,
                                             hour=0,
                                             minute=0,
                                             second=0,
                                             microsecond=0) + relativedelta.relativedelta(months=-1)

        end_date = start_date + timedelta(days=10)
        end_date = end_date.replace(hour=23, minute=59, second=59)

        daily_table = getattr(self.aws_accessor.report_schema, daily_table_name)
        summary_table = getattr(self.aws_accessor.report_schema, summary_table_name)
        ce_table = getattr(self.aws_accessor.report_schema, ce_table_name)

        ce_start_date = self.aws_accessor._session\
            .query(func.min(ce_table.interval_start))\
            .filter(ce_table.interval_start >= start_date).first()[0]

        ce_end_date = self.aws_accessor._session\
            .query(func.max(ce_table.interval_start))\
            .filter(ce_table.interval_start <= end_date).first()[0]

        # The summary tables will only include dates where there is data
        expected_start_date = max(start_date, ce_start_date)
        expected_start_date = expected_start_date.replace(hour=0, minute=0,
                                                          second=0,
                                                          microsecond=0)
        expected_end_date = min(end_date, ce_end_date)
        expected_end_date = expected_end_date.replace(hour=0, minute=0,
                                                      second=0, microsecond=0)

        update_summary_tables(self.schema_name, provider, provider_aws_uuid, start_date, end_date)

        result_start_date, result_end_date = self.aws_accessor._session.query(
            func.min(daily_table.usage_start),
            func.max(daily_table.usage_end)
        ).first()

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

        result_start_date, result_end_date = self.aws_accessor._session.query(
            func.min(summary_table.usage_start),
            func.max(summary_table.usage_end)
        ).first()

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

    @patch('masu.processor.tasks.update_charge_info')
    @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_memory_gb_usage_per_hour_rates')
    @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_cpu_core_usage_per_hour_rates')
    def test_update_summary_tables_ocp(self, mock_cpu_rate, mock_mem_rate, mock_charge_info):
        """Test that the summary table task runs."""
        mem_rate = {'tiered_rate': [{'value': '1.5', 'unit': 'USD'}]}
        cpu_rate = {'tiered_rate': [{'value': '2.5', 'unit': 'USD'}]}

        mock_cpu_rate.return_value = cpu_rate
        mock_mem_rate.return_value = mem_rate

        provider = 'OCP'
        provider_ocp_uuid = self.ocp_test_provider_uuid

        daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily']
        start_date = self.start_date.replace(day=1) + relativedelta.relativedelta(months=-1)

        daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name)

        initial_daily_count = daily_query.count()

        self.assertEqual(initial_daily_count, 0)
        update_summary_tables(self.schema_name, provider, provider_ocp_uuid, start_date)

        self.assertNotEqual(daily_query.count(), initial_daily_count)

        update_charge_info(schema_name=self.test_schema, provider_uuid=provider_ocp_uuid)

        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']
        with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor:
            provider_obj = provider_accessor.get_provider()

        usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(provider_obj.id)
        cluster_id = usage_period_qry.first().cluster_id

        items = self.ocp_accessor._get_db_obj_query(table_name).filter_by(cluster_id=cluster_id)
        for item in items:
            self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours)
            self.assertIsNotNone(item.pod_charge_cpu_core_hours)

        storage_daily_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily']
        items = self.ocp_accessor._get_db_obj_query(storage_daily_name).filter_by(cluster_id=cluster_id)
        for item in items:
            self.assertIsNotNone(item.volume_request_storage_byte_seconds)
            self.assertIsNotNone(item.persistentvolumeclaim_usage_byte_seconds)

        storage_summary_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily_summary']
        items = self.ocp_accessor._get_db_obj_query(storage_summary_name).filter_by(cluster_id=cluster_id)
        for item in items:
            self.assertIsNotNone(item.volume_request_storage_gigabyte_months)
            self.assertIsNotNone(item.persistentvolumeclaim_usage_gigabyte_months)

    @patch('masu.processor.tasks.update_charge_info')
    @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_memory_gb_usage_per_hour_rates')
    @patch('masu.database.ocp_rate_db_accessor.OCPRateDBAccessor.get_cpu_core_usage_per_hour_rates')
    def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate, mock_mem_rate, mock_charge_info, ):
        """Test that the summary table task respects a date range."""
        mock_cpu_rate.return_value = 1.5
        mock_mem_rate.return_value = 2.5
        provider = 'OCP'
        provider_ocp_uuid = self.ocp_test_provider_uuid
        ce_table_name = OCP_REPORT_TABLE_MAP['report']
        daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily']

        start_date = self.start_date.replace(day=1,
                                             hour=0, minute=0, second=0,
                                             microsecond=0) + relativedelta.relativedelta(months=-1)

        end_date = start_date + timedelta(days=10)
        end_date = end_date.replace(hour=23, minute=59, second=59)

        daily_table = getattr(self.ocp_accessor.report_schema, daily_table_name)
        ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name)

        ce_start_date = self.ocp_accessor._session\
            .query(func.min(ce_table.interval_start))\
            .filter(ce_table.interval_start >= start_date).first()[0]

        ce_end_date = self.ocp_accessor._session\
            .query(func.max(ce_table.interval_start))\
            .filter(ce_table.interval_start <= end_date).first()[0]

        # The summary tables will only include dates where there is data
        expected_start_date = max(start_date, ce_start_date)
        expected_start_date = expected_start_date.replace(hour=0, minute=0,
                                                          second=0,
                                                          microsecond=0)
        expected_end_date = min(end_date, ce_end_date)
        expected_end_date = expected_end_date.replace(hour=0, minute=0,
                                                      second=0, microsecond=0)

        update_summary_tables(self.schema_name, provider, provider_ocp_uuid, start_date, end_date)
        result_start_date, result_end_date = self.ocp_accessor._session.query(
            func.min(daily_table.usage_start),
            func.max(daily_table.usage_end)
        ).first()

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

    def test_update_charge_info_aws(self):
        """Test that update_charge_info is not called for AWS."""
        update_charge_info(schema_name=self.test_schema,
                           provider_uuid=self.aws_test_provider_uuid)
        # FIXME: no asserts on test

    @patch('masu.processor.tasks.update_summary_tables')
    def test_get_report_data_for_all_providers(self, mock_update):
        """Test GET report_data endpoint with provider_uuid=*."""
        start_date = date.today()
        update_all_summary_tables(start_date)

        mock_update.delay.assert_called_with(
            ANY, ANY, ANY, str(start_date), ANY)
Exemplo n.º 16
0
class TestUpdateSummaryTablesTask(MasuTestCase):
    """Test cases for Processor summary table Celery tasks."""
    @classmethod
    def setUpClass(cls):
        """Setup for the class."""
        super().setUpClass()
        cls.aws_tables = list(AWS_CUR_TABLE_MAP.values())
        cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values())
        cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + list(
            OCP_REPORT_TABLE_MAP.values())
        with ReportingCommonDBAccessor() as report_common_db:
            cls.column_map = report_common_db.column_map

        cls.creator = ReportObjectCreator(cls.schema, cls.column_map)

    def setUp(self):
        """Set up each test."""
        super().setUp()
        self.aws_accessor = AWSReportDBAccessor(schema=self.schema,
                                                column_map=self.column_map)
        self.ocp_accessor = OCPReportDBAccessor(schema=self.schema,
                                                column_map=self.column_map)

        # Populate some line item data so that the summary tables
        # have something to pull from
        self.start_date = DateAccessor().today_with_timezone('UTC').replace(
            day=1)
        last_month = self.start_date - relativedelta.relativedelta(months=1)

        for cost_entry_date in (self.start_date, last_month):
            bill = self.creator.create_cost_entry_bill(
                provider_id=self.aws_provider.id, bill_date=cost_entry_date)
            cost_entry = self.creator.create_cost_entry(bill, cost_entry_date)
            for family in [
                    'Storage',
                    'Compute Instance',
                    'Database Storage',
                    'Database Instance',
            ]:
                product = self.creator.create_cost_entry_product(family)
                pricing = self.creator.create_cost_entry_pricing()
                reservation = self.creator.create_cost_entry_reservation()
                self.creator.create_cost_entry_line_item(
                    bill, cost_entry, product, pricing, reservation)
        provider_ocp_uuid = self.ocp_test_provider_uuid

        with ProviderDBAccessor(
                provider_uuid=provider_ocp_uuid) as provider_accessor:
            provider_id = provider_accessor.get_provider().id

        cluster_id = self.ocp_provider_resource_name
        for period_date in (self.start_date, last_month):
            period = self.creator.create_ocp_report_period(
                period_date, provider_id=provider_id, cluster_id=cluster_id)
            report = self.creator.create_ocp_report(period, period_date)
            for _ in range(25):
                self.creator.create_ocp_usage_line_item(period, report)

    @patch('masu.processor.tasks.update_cost_summary_table')
    @patch('masu.processor.tasks.update_charge_info')
    def test_update_summary_tables_aws(self, mock_charge_info,
                                       mock_cost_summary):
        """Test that the summary table task runs."""
        provider = 'AWS'
        provider_aws_uuid = self.aws_test_provider_uuid

        daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily']
        summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary']
        start_date = self.start_date.replace(
            day=1) + relativedelta.relativedelta(months=-1)

        with schema_context(self.schema):
            daily_query = self.aws_accessor._get_db_obj_query(daily_table_name)
            summary_query = self.aws_accessor._get_db_obj_query(
                summary_table_name)

            initial_daily_count = daily_query.count()
            initial_summary_count = summary_query.count()

        self.assertEqual(initial_daily_count, 0)
        self.assertEqual(initial_summary_count, 0)

        update_summary_tables(self.schema, provider, provider_aws_uuid,
                              start_date)

        with schema_context(self.schema):
            self.assertNotEqual(daily_query.count(), initial_daily_count)
            self.assertNotEqual(summary_query.count(), initial_summary_count)

        mock_charge_info.apply_async.assert_called()
        mock_cost_summary.si.assert_called()

    @patch('masu.processor.tasks.update_charge_info')
    def test_update_summary_tables_aws_end_date(self, mock_charge_info):
        """Test that the summary table task respects a date range."""
        provider = 'AWS'
        provider_aws_uuid = self.aws_test_provider_uuid
        ce_table_name = AWS_CUR_TABLE_MAP['cost_entry']
        daily_table_name = AWS_CUR_TABLE_MAP['line_item_daily']
        summary_table_name = AWS_CUR_TABLE_MAP['line_item_daily_summary']

        start_date = self.start_date.replace(
            day=1, hour=0, minute=0, second=0,
            microsecond=0) + relativedelta.relativedelta(months=-1)

        end_date = start_date + timedelta(days=10)
        end_date = end_date.replace(hour=23, minute=59, second=59)

        daily_table = getattr(self.aws_accessor.report_schema,
                              daily_table_name)
        summary_table = getattr(self.aws_accessor.report_schema,
                                summary_table_name)
        ce_table = getattr(self.aws_accessor.report_schema, ce_table_name)

        with schema_context(self.schema):
            ce_start_date = ce_table.objects\
                .filter(interval_start__gte=start_date)\
                .aggregate(Min('interval_start'))['interval_start__min']
            ce_end_date = ce_table.objects\
                .filter(interval_start__lte=end_date)\
                .aggregate(Max('interval_start'))['interval_start__max']

        # The summary tables will only include dates where there is data
        expected_start_date = max(start_date, ce_start_date)
        expected_start_date = expected_start_date.replace(hour=0,
                                                          minute=0,
                                                          second=0,
                                                          microsecond=0)
        expected_end_date = min(end_date, ce_end_date)
        expected_end_date = expected_end_date.replace(hour=0,
                                                      minute=0,
                                                      second=0,
                                                      microsecond=0)

        update_summary_tables(self.schema, provider, provider_aws_uuid,
                              start_date, end_date)

        with schema_context(self.schema):
            daily_entry = daily_table.objects.all().aggregate(
                Min('usage_start'), Max('usage_end'))
            result_start_date = daily_entry['usage_start__min']
            result_end_date = daily_entry['usage_end__max']

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

        with schema_context(self.schema):
            summary_entry = summary_table.objects.all().aggregate(
                Min('usage_start'), Max('usage_end'))
            result_start_date = summary_entry['usage_start__min']
            result_end_date = summary_entry['usage_end__max']

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

    @patch('masu.processor.tasks.update_cost_summary_table')
    @patch('masu.processor.tasks.update_charge_info')
    @patch(
        'masu.database.cost_model_db_accessor.CostModelDBAccessor._make_rate_by_metric_map'
    )
    @patch(
        'masu.database.cost_model_db_accessor.CostModelDBAccessor.get_markup')
    def test_update_summary_tables_ocp(self, mock_markup, mock_rate_map,
                                       mock_charge_info, mock_cost_summary):
        """Test that the summary table task runs."""
        markup = {}
        mem_rate = {'tiered_rates': [{'value': '1.5', 'unit': 'USD'}]}
        cpu_rate = {'tiered_rates': [{'value': '2.5', 'unit': 'USD'}]}
        rate_metric_map = {
            'cpu_core_usage_per_hour': cpu_rate,
            'memory_gb_usage_per_hour': mem_rate
        }

        mock_markup.return_value = markup
        mock_rate_map.return_value = rate_metric_map

        provider = 'OCP'
        provider_ocp_uuid = self.ocp_test_provider_uuid

        daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily']
        start_date = self.start_date.replace(
            day=1) + relativedelta.relativedelta(months=-1)

        with schema_context(self.schema):
            daily_query = self.ocp_accessor._get_db_obj_query(daily_table_name)

            initial_daily_count = daily_query.count()

        self.assertEqual(initial_daily_count, 0)
        update_summary_tables(self.schema, provider, provider_ocp_uuid,
                              start_date)

        with schema_context(self.schema):
            self.assertNotEqual(daily_query.count(), initial_daily_count)

        update_charge_info(schema_name=self.schema,
                           provider_uuid=provider_ocp_uuid)

        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']
        with ProviderDBAccessor(provider_ocp_uuid) as provider_accessor:
            provider_obj = provider_accessor.get_provider()

        usage_period_qry = self.ocp_accessor.get_usage_period_query_by_provider(
            provider_obj.id)
        with schema_context(self.schema):
            cluster_id = usage_period_qry.first().cluster_id

            items = self.ocp_accessor._get_db_obj_query(table_name).filter(
                cluster_id=cluster_id)
            for item in items:
                self.assertIsNotNone(item.pod_charge_memory_gigabyte_hours)
                self.assertIsNotNone(item.pod_charge_cpu_core_hours)

            storage_daily_name = OCP_REPORT_TABLE_MAP[
                'storage_line_item_daily']

            items = self.ocp_accessor._get_db_obj_query(
                storage_daily_name).filter(cluster_id=cluster_id)
            for item in items:
                self.assertIsNotNone(item.volume_request_storage_byte_seconds)
                self.assertIsNotNone(
                    item.persistentvolumeclaim_usage_byte_seconds)

            storage_summary_name = OCP_REPORT_TABLE_MAP[
                'line_item_daily_summary']
            items = self.ocp_accessor._get_db_obj_query(
                storage_summary_name).filter(cluster_id=cluster_id,
                                             data_source='Storage')
            for item in items:
                self.assertIsNotNone(
                    item.volume_request_storage_gigabyte_months)
                self.assertIsNotNone(
                    item.persistentvolumeclaim_usage_gigabyte_months)

        mock_charge_info.apply_async.assert_called()
        mock_cost_summary.si.assert_called()

    @patch('masu.processor.tasks.update_charge_info')
    @patch(
        'masu.database.cost_model_db_accessor.CostModelDBAccessor.get_memory_gb_usage_per_hour_rates'
    )
    @patch(
        'masu.database.cost_model_db_accessor.CostModelDBAccessor.get_cpu_core_usage_per_hour_rates'
    )
    def test_update_summary_tables_ocp_end_date(self, mock_cpu_rate,
                                                mock_mem_rate,
                                                mock_charge_info):
        """Test that the summary table task respects a date range."""
        mock_cpu_rate.return_value = 1.5
        mock_mem_rate.return_value = 2.5
        provider = 'OCP'
        provider_ocp_uuid = self.ocp_test_provider_uuid
        ce_table_name = OCP_REPORT_TABLE_MAP['report']
        daily_table_name = OCP_REPORT_TABLE_MAP['line_item_daily']

        start_date = self.start_date.replace(
            day=1, hour=0, minute=0, second=0,
            microsecond=0) + relativedelta.relativedelta(months=-1)

        end_date = start_date + timedelta(days=10)
        end_date = end_date.replace(hour=23, minute=59, second=59)

        daily_table = getattr(self.ocp_accessor.report_schema,
                              daily_table_name)
        ce_table = getattr(self.ocp_accessor.report_schema, ce_table_name)

        with schema_context(self.schema):
            ce_start_date = ce_table.objects\
                .filter(interval_start__gte=start_date)\
                .aggregate(Min('interval_start'))['interval_start__min']

            ce_end_date = ce_table.objects\
                .filter(interval_start__lte=end_date)\
                .aggregate(Max('interval_start'))['interval_start__max']

        # The summary tables will only include dates where there is data
        expected_start_date = max(start_date, ce_start_date)
        expected_start_date = expected_start_date.replace(hour=0,
                                                          minute=0,
                                                          second=0,
                                                          microsecond=0)
        expected_end_date = min(end_date, ce_end_date)
        expected_end_date = expected_end_date.replace(hour=0,
                                                      minute=0,
                                                      second=0,
                                                      microsecond=0)

        update_summary_tables(self.schema, provider, provider_ocp_uuid,
                              start_date, end_date)
        with schema_context(self.schema):
            daily_entry = daily_table.objects.all().aggregate(
                Min('usage_start'), Max('usage_end'))
            result_start_date = daily_entry['usage_start__min']
            result_end_date = daily_entry['usage_end__max']

        self.assertEqual(result_start_date, expected_start_date)
        self.assertEqual(result_end_date, expected_end_date)

    @patch('masu.processor.tasks.update_summary_tables')
    def test_get_report_data_for_all_providers(self, mock_update):
        """Test GET report_data endpoint with provider_uuid=*."""
        start_date = date.today()
        update_all_summary_tables(start_date)

        mock_update.delay.assert_called_with(ANY, ANY, ANY, str(start_date),
                                             ANY)

    @patch(
        'masu.database.ocp_report_db_accessor.OCPReportDBAccessor.populate_cost_summary_table'
    )
    def test_update_cost_summary_table(self, mock_update):
        """Tests that the updater updates the cost summary table."""
        provider = 'OCP'
        provider_aws_uuid = self.ocp_test_provider_uuid
        manifest_id = None
        start_date = self.start_date.replace(
            day=1, hour=0, minute=0, second=0,
            microsecond=0) + relativedelta.relativedelta(months=-1)

        update_cost_summary_table(self.schema,
                                  provider_aws_uuid,
                                  start_date=start_date)

        mock_update.assert_called()