def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on AWS.""" cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) aws_bills = aws_get_bills_from_provider( aws_provider_uuid, self._schema, datetime.datetime.strptime(start_date, '%Y-%m-%d'), datetime.datetime.strptime(end_date, '%Y-%m-%d')) aws_bill_ids = [] with schema_context(self._schema): aws_bill_ids = [str(bill.id) for bill in aws_bills] with CostModelDBAccessor(self._schema, aws_provider_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() markup_value = Decimal(markup.get('value', 0)) / 100 # OpenShift on AWS with AWSReportDBAccessor(self._schema, self._column_map) as accessor: LOG.info( 'Updating OpenShift on AWS summary table for ' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s' '\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema, self._provider.uuid, start_date, end_date, cluster_id, str(aws_bill_ids)) accessor.populate_ocp_on_aws_cost_daily_summary( start_date, end_date, cluster_id, aws_bill_ids) accessor.populate_ocp_on_aws_markup_cost(markup_value, aws_bill_ids) with OCPReportDBAccessor(self._schema, self._column_map) as accessor: # This call just sends the infrastructure cost to the # OCP usage daily summary table accessor.update_summary_infrastructure_cost( cluster_id, start_date, end_date)
def create_bill(self, bill_date): """Create bill postgres entry.""" if isinstance(bill_date, str): bill_date = ciso8601.parse_datetime(bill_date) report_date_range = month_date_range(bill_date) start_date, end_date = report_date_range.split("-") report_period_start = ciso8601.parse_datetime(start_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) report_period_end = ciso8601.parse_datetime(end_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) # Make end date first of next month report_period_end = report_period_end + datetime.timedelta(days=1) provider = self._get_provider() cluster_id = utils.get_cluster_id_from_provider(provider.uuid) cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id) with schema_context(self._schema_name): OCPUsageReportPeriod.objects.get_or_create( cluster_id=cluster_id, cluster_alias=cluster_alias, report_period_start=report_period_start, report_period_end=report_period_end, provider=provider, )
def _update_markup_cost(self, start_date, end_date): """Populate markup costs for OpenShift. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ infra_markup_value = Decimal(0.0) infra_map = self.get_infra_map() infra_tuple = infra_map.get(self._provider_uuid) cluster_id = get_cluster_id_from_provider(self._provider_uuid) if infra_tuple: infra_uuid = infra_tuple[0] with CostModelDBAccessor(self._schema, infra_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() infra_markup_value = Decimal(markup.get('value', 0)) / 100 with CostModelDBAccessor(self._schema, self._provider_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() ocp_markup_value = Decimal(markup.get('value', 0)) / 100 with OCPReportDBAccessor(self._schema, self._column_map) as accessor: LOG.info('Updating OpenShift markup for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema, self._provider_uuid, start_date, end_date) accessor.populate_markup_cost(infra_markup_value, ocp_markup_value, cluster_id) LOG.info('Finished updating markup.')
def update_summary_charge_info(self, start_date=None, end_date=None): """Update the OCP summary table with the charge information. Args: start_date (str, Optional) - Start date of range to update derived cost. end_date (str, Optional) - End date of range to update derived cost. Returns None """ self._cluster_id = get_cluster_id_from_provider(self._provider_uuid) LOG.info( 'Starting charge calculation updates for provider: %s. Cluster ID: %s.', self._provider_uuid, self._cluster_id) self._update_pod_charge() self._update_storage_charge() with OCPReportDBAccessor(self._schema, self._column_map) as accessor: LOG.info( 'Updating OpenShift on Cloud cost summary for schema: %s and provider: %s', self._schema, self._provider_uuid) accessor.populate_cost_summary_table(self._cluster_id, start_date=start_date, end_date=end_date) report_periods = accessor.report_periods_for_provider_id( self._provider_id, start_date) for period in report_periods: period.derived_cost_datetime = DateAccessor( ).today_with_timezone('UTC') accessor.commit()
def update_summary_charge_info(self, start_date=None, end_date=None): """Update the OCP summary table with the charge information. Args: start_date (str, Optional) - Start date of range to update derived cost. end_date (str, Optional) - End date of range to update derived cost. Returns None """ self._cluster_id = get_cluster_id_from_provider(self._provider_uuid) LOG.info( 'Starting charge calculation updates for provider: %s. Cluster ID: %s.', self._provider_uuid, self._cluster_id) self._update_pod_charge() self._update_storage_charge() self._update_markup_cost() with OCPReportDBAccessor(self._schema, self._column_map) as accessor: report_periods = accessor.report_periods_for_provider_uuid( self._provider_uuid, start_date) with schema_context(self._schema): for period in report_periods: period.derived_cost_datetime = DateAccessor( ).today_with_timezone('UTC') period.save()
def test_update_gcp_summary_tables( self, mock_utility, mock_delete, mock_back_populate, mock_ocp_on_gcp, mock_ui_tables, mock_tag_summary, mock_map, mock_ocpallproj, mock_ocpall, mock_ocpallui, ): """Test that summary tables are properly run for a gcp provider.""" fake_bills = MagicMock() fake_bills.__iter__.return_value = [Mock(), Mock()] first = Mock() bill_id = 1 first.return_value.id = bill_id fake_bills.first = first mock_utility.return_value = fake_bills start_date = self.dh.today.date() end_date = start_date + datetime.timedelta(days=1) with ProviderDBAccessor(self.gcp_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() with OCPReportDBAccessor(self.schema_name) as accessor: report_period = accessor.report_periods_for_provider_uuid(self.ocpgcp_provider_uuid, start_date) with schema_context(self.schema_name): current_ocp_report_period_id = report_period.id mock_map.return_value = {self.ocpgcp_provider_uuid: (self.gcp_provider_uuid, Provider.PROVIDER_GCP)} updater = OCPCloudParquetReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) updater.update_gcp_summary_tables(self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid, start_date, end_date) cluster_id = get_cluster_id_from_provider(self.ocpgcp_provider_uuid) cluster_alias = get_cluster_alias_from_cluster_id(cluster_id) sql_params = { "schema_name": self.schema_name, "start_date": start_date, "end_date": end_date, "source_uuid": self.gcp_test_provider_uuid, "cluster_id": cluster_id, "cluster_alias": cluster_alias, "source_type": "GCP", } distribution = None mock_ocp_on_gcp.assert_called_with( start_date, end_date, self.ocpgcp_provider_uuid, cluster_id, self.gcp_test_provider_uuid, current_ocp_report_period_id, bill_id, decimal.Decimal(0), distribution, ) mock_ui_tables.assert_called_with(sql_params) mock_tag_summary.assert_called_with([str(bill.id) for bill in fake_bills], start_date, end_date)
def test_poll_ingest_override_for_provider(self): """Test that OCP polling override returns True if insights local path exists.""" fake_dir = tempfile.mkdtemp() with patch.object(Config, 'INSIGHTS_LOCAL_REPORT_DIR', fake_dir): cluster_id = utils.get_cluster_id_from_provider(self.ocp_test_provider_uuid) expected_path = '{}/{}/'.format(Config.INSIGHTS_LOCAL_REPORT_DIR, cluster_id) os.makedirs(expected_path, exist_ok=True) self.assertTrue(utils.poll_ingest_override_for_provider(self.ocp_test_provider_uuid)) shutil.rmtree(fake_dir)
def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on AWS.""" if isinstance(start_date, str): start_date = parser.parse(start_date).date() if isinstance(end_date, str): end_date = parser.parse(end_date).date() cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) with OCPReportDBAccessor(self._schema) as accessor: report_period = accessor.report_periods_for_provider_uuid( openshift_provider_uuid, start_date) accessor.delete_infrastructure_raw_cost_from_daily_summary( openshift_provider_uuid, report_period.id, start_date, end_date) aws_bills = aws_get_bills_from_provider(aws_provider_uuid, self._schema, start_date, end_date) with schema_context(self._schema): aws_bill_ids = [str(bill.id) for bill in aws_bills] current_aws_bill_id = aws_bills.first().id if aws_bills else None current_ocp_report_period_id = report_period.id with CostModelDBAccessor(self._schema, aws_provider_uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = Decimal(markup.get("value", 0)) / 100 # OpenShift on AWS with AWSReportDBAccessor(self._schema) as accessor: for start, end in date_range_pair(start_date, end_date, step=settings.TRINO_DATE_STEP): LOG.info( "Updating OpenShift on AWS summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, AWS Bill ID: %s", self._schema, self._provider.uuid, start, end, cluster_id, current_aws_bill_id, ) accessor.populate_ocp_on_aws_cost_daily_summary_presto( start, end, openshift_provider_uuid, aws_provider_uuid, current_ocp_report_period_id, current_aws_bill_id, markup_value, ) accessor.back_populate_ocp_on_aws_daily_summary( start_date, end_date, current_ocp_report_period_id) accessor.populate_ocp_on_aws_tags_summary_table( aws_bill_ids, start_date, end_date)
def test_get_cluster_id_with_no_authentication(self): """Test that a None is correctly returned if authentication is not present.""" # Remove test provider authentication Provider.objects.filter(uuid=self.ocp_test_provider_uuid).update( authentication=None) ocp_provider = Provider.objects.get(uuid=self.ocp_test_provider_uuid) self.assertIsNone(ocp_provider.authentication) # Assert if authentication is empty we return none instead of an error cluster_id = utils.get_cluster_id_from_provider( self.ocp_test_provider_uuid) self.assertIsNone(cluster_id)
def __init__(self, schema, provider, manifest): """Establish the database connection. Args: schema (str): The customer schema to associate with """ self._schema = schema self._provider = provider self._manifest = manifest self._cluster_id = get_cluster_id_from_provider(self._provider.uuid) self._date_accessor = DateAccessor()
def __init__(self, schema, provider, manifest): """Establish the database connection. Args: schema (str): The customer schema to associate with """ self._schema_name = schema self._provider = provider self._manifest = manifest self._cluster_id = get_cluster_id_from_provider(self._provider.uuid) with ReportingCommonDBAccessor() as reporting_common: self._column_map = reporting_common.column_map self._date_accessor = DateAccessor()
def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on Azure.""" if isinstance(start_date, str): start_date = parser.parse(start_date).date() if isinstance(end_date, str): end_date = parser.parse(end_date).date() cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) azure_bills = azure_get_bills_from_provider(azure_provider_uuid, self._schema, start_date, end_date) with schema_context(self._schema): current_azure_bill_id = azure_bills.first( ).id if azure_bills else None with CostModelDBAccessor(self._schema, azure_provider_uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = Decimal(markup.get("value", 0)) / 100 # OpenShift on Azure with AzureReportDBAccessor(self._schema) as accessor: LOG.info( "Updating OpenShift on Azure summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, Azure Bill ID: %s", self._schema, self._provider.uuid, start_date, end_date, cluster_id, current_azure_bill_id, ) accessor.populate_ocp_on_azure_cost_daily_summary_presto( start_date, end_date, openshift_provider_uuid, azure_provider_uuid, cluster_id, current_azure_bill_id, markup_value, ) accessor.populate_ocp_on_azure_tags_summary_table() with OCPReportDBAccessor(self._schema) as accessor: # This call just sends the infrastructure cost to the # OCP usage daily summary table accessor.update_summary_infrastructure_cost( cluster_id, start_date, end_date)
def test_update_cost_summary_table_for_markup(self, mock_markup): """Test that summary tables are updated correctly.""" markup = {'value': 10, 'unit': 'percent'} mock_markup.return_value = markup self._generate_ocp_on_aws_data() start_date = self.date_accessor.today_with_timezone('UTC') end_date = start_date + datetime.timedelta(days=1) start_date = start_date - relativedelta.relativedelta(months=1) start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = end_date.strftime('%Y-%m-%d') with ProviderDBAccessor( self.ocp_test_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None) updater.update_summary_tables(start_date_str, end_date_str) cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid) with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: ocp_accessor.populate_cost_summary_table(cluster_id, start_date, end_date) cost_summary_table = OCP_REPORT_TABLE_MAP['cost_summary'] cost_summary_query = ocp_accessor._get_db_obj_query( cost_summary_table) possible_values = {} with schema_context(self.schema): for item in cost_summary_query: possible_values.update({ item.cluster_id: ((item.pod_charge_cpu_core_hours + item.pod_charge_memory_gigabyte_hours + item.persistentvolumeclaim_charge_gb_month + item.infra_cost) * decimal.Decimal(0.1)) }) updater.update_cost_summary_table(start_date_str, end_date_str) with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: query = ocp_accessor._get_db_obj_query(cost_summary_table) found_values = {} for item in query: found_values.update({item.cluster_id: item.markup_cost}) for k, v in found_values.items(): self.assertAlmostEqual(v, possible_values[k], places=6)
def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on Azure.""" if isinstance(start_date, str): start_date = parser.parse(start_date).date() if isinstance(end_date, str): end_date = parser.parse(end_date).date() cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) azure_bills = azure_get_bills_from_provider(azure_provider_uuid, self._schema, start_date, end_date) with schema_context(self._schema): azure_bill_ids = [str(bill.id) for bill in azure_bills] current_azure_bill_id = azure_bills.first( ).id if azure_bills else None with CostModelDBAccessor(self._schema, azure_provider_uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = Decimal(markup.get("value", 0)) / 100 # OpenShift on Azure with AzureReportDBAccessor(self._schema) as accessor: for start, end in date_range_pair(start_date, end_date, step=settings.TRINO_DATE_STEP): LOG.info( "Updating OpenShift on Azure summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, Azure Bill ID: %s", self._schema, self._provider.uuid, start, end, cluster_id, current_azure_bill_id, ) accessor.populate_ocp_on_azure_cost_daily_summary_presto( start, end, openshift_provider_uuid, azure_provider_uuid, cluster_id, current_azure_bill_id, markup_value, ) accessor.populate_ocp_on_azure_tags_summary_table( azure_bill_ids, start_date, end_date)
def __init__(self, schema, provider): """Establish the database connection. Args: schema (str): The customer schema to associate with """ super().__init__(schema, provider, None) self._cluster_id = get_cluster_id_from_provider(self._provider_uuid) self._cluster_alias = get_cluster_alias_from_cluster_id( self._cluster_id) with CostModelDBAccessor(self._schema, self._provider_uuid) as cost_model_accessor: self._infra_rates = cost_model_accessor.infrastructure_rates self._supplementary_rates = cost_model_accessor.supplementary_rates
def _generate_ocp_on_aws_data(self, cluster_id=None): """Generate OCP and AWS data.""" if not cluster_id: cluster_id = self.ocp_provider_resource_name creator = ReportObjectCreator(self.schema, self.column_map) bill_ids = [] today = DateAccessor().today_with_timezone("UTC") last_month = today - relativedelta(months=1) resource_id = "i-12345" for cost_entry_date in (today, last_month): bill = creator.create_cost_entry_bill( provider_uuid=self.aws_provider_uuid, bill_date=cost_entry_date) bill_ids.append(str(bill.id)) cost_entry = creator.create_cost_entry(bill, cost_entry_date) product = creator.create_cost_entry_product("Compute Instance") pricing = creator.create_cost_entry_pricing() reservation = creator.create_cost_entry_reservation() creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation, resource_id=resource_id) with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor: aws_accessor.populate_line_item_daily_table( last_month.date(), today.date(), bill_ids) provider_uuid = self.ocp_provider_uuid for cost_entry_date in (today, last_month): period = creator.create_ocp_report_period( provider_uuid=provider_uuid, period_date=cost_entry_date, cluster_id=cluster_id) report = creator.create_ocp_report(period, cost_entry_date) creator.create_ocp_usage_line_item(period, report, resource_id=resource_id) cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid) with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor: ocp_accessor.populate_line_item_daily_table( last_month.date(), today.date(), cluster_id)
def _generate_ocp_on_aws_data(self): """Test that the OCP on AWS cost summary table is populated.""" creator = ReportObjectCreator(self.accessor, self.column_map, self.accessor.report_schema.column_types) bill_ids = [] today = DateAccessor().today_with_timezone('UTC') last_month = today - relativedelta.relativedelta(months=1) resource_id = 'i-12345' for cost_entry_date in (today, last_month): bill = creator.create_cost_entry_bill(cost_entry_date) bill_ids.append(str(bill.id)) cost_entry = creator.create_cost_entry(bill, cost_entry_date) product = creator.create_cost_entry_product('Compute Instance') pricing = creator.create_cost_entry_pricing() reservation = creator.create_cost_entry_reservation() creator.create_cost_entry_line_item(bill, cost_entry, product, pricing, reservation, resource_id=resource_id) self.accessor.populate_line_item_daily_table(last_month, today, bill_ids) with OCPReportDBAccessor(self.test_schema, self.column_map) as ocp_accessor: cluster_id = self.ocp_provider_resource_name with ProviderDBAccessor(provider_uuid=self.ocp_test_provider_uuid ) as provider_access: provider_id = provider_access.get_provider().id for cost_entry_date in (today, last_month): period = creator.create_ocp_report_period( cost_entry_date, provider_id=provider_id, cluster_id=cluster_id) report = creator.create_ocp_report(period, cost_entry_date) creator.create_ocp_usage_line_item(period, report, resource_id=resource_id) cluster_id = get_cluster_id_from_provider( self.ocp_test_provider_uuid) ocp_accessor.populate_line_item_daily_table( last_month, today, cluster_id)
def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on AWS.""" if isinstance(start_date, str): start_date = parser.parse(start_date) if isinstance(end_date, str): end_date = parser.parse(end_date) cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) aws_bills = aws_get_bills_from_provider(aws_provider_uuid, self._schema, start_date, end_date) aws_bill_ids = [] with schema_context(self._schema): aws_bill_ids = [str(bill.id) for bill in aws_bills] with CostModelDBAccessor(self._schema, aws_provider_uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = Decimal(markup.get("value", 0)) / 100 # OpenShift on AWS with AWSReportDBAccessor(self._schema) as accessor: for start, end in date_range_pair(start_date, end_date): LOG.info( "Updating OpenShift on AWS summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, AWS Bill IDs: %s", self._schema, self._provider.uuid, start, end, cluster_id, str(aws_bill_ids), ) accessor.populate_ocp_on_aws_cost_daily_summary( start, end, cluster_id, aws_bill_ids, markup_value) accessor.populate_ocp_on_aws_tags_summary_table() self.refresh_openshift_on_infrastructure_views( OCP_ON_AWS_MATERIALIZED_VIEWS) with OCPReportDBAccessor(self._schema) as accessor: # This call just sends the infrastructure cost to the # OCP usage daily summary table accessor.update_summary_infrastructure_cost( cluster_id, start_date, end_date)
def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on Azure.""" cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) azure_bills = azure_get_bills_from_provider( azure_provider_uuid, self._schema, datetime.datetime.strptime(start_date, "%Y-%m-%d"), datetime.datetime.strptime(end_date, "%Y-%m-%d"), ) azure_bill_ids = [] with schema_context(self._schema): azure_bill_ids = [str(bill.id) for bill in azure_bills] with CostModelDBAccessor(self._schema, azure_provider_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() markup_value = Decimal(markup.get("value", 0)) / 100 # OpenShift on Azure with AzureReportDBAccessor(self._schema, self._column_map) as accessor: for start, end in date_range_pair(start_date, end_date): LOG.info( "Updating OpenShift on Azure summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, Azure Bill IDs: %s", self._schema, self._provider.uuid, start, end, cluster_id, str(azure_bill_ids), ) accessor.populate_ocp_on_azure_cost_daily_summary( start, end, cluster_id, azure_bill_ids) accessor.populate_ocp_on_azure_markup_cost(markup_value, azure_bill_ids) with OCPReportDBAccessor(self._schema, self._column_map) as accessor: # This call just sends the infrastructure cost to the # OCP usage daily summary table accessor.update_summary_infrastructure_cost( cluster_id, start_date, end_date)
def __init__(self, schema_name, report_path, compression, provider_uuid): """Initialize base class.""" super().__init__( schema_name=schema_name, report_path=report_path, compression=compression, provider_uuid=provider_uuid, manifest_id=None, processed_report=ProcessedOCPReport(), ) self._report_name = path.basename(report_path) self._cluster_id = utils.get_cluster_id_from_provider(provider_uuid) self._datetime_format = Config.OCP_DATETIME_STR_FORMAT self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE with OCPReportDBAccessor(self._schema) as report_db: self.existing_report_periods_map = report_db.get_report_periods() self.existing_report_map = report_db.get_reports() self.line_item_columns = None
def update_cost_summary_table(self, start_date, end_date): """Populate the cost summary tables. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ cluster_id = get_cluster_id_from_provider(self._provider.uuid) # This needs to always run regardless of whether the OpenShift # cluster is tied to a cloud provider with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date)
def update_summary_tables(self, start_date, end_date): """Populate the summary tables for reporting. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ cluster_id = get_cluster_id_from_provider(self._provider.uuid) aws_bills = get_bills_from_provider(self._provider.uuid, self._schema_name, start_date, end_date) aws_bill_ids = [str(bill.id) for bill in aws_bills] # OpenShift on AWS with AWSReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on AWS summary table for ' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_ocp_on_aws_cost_daily_summary( start_date, end_date, cluster_id, aws_bill_ids) accessor.commit() if cluster_id: with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date) accessor.commit()
def update_cost_summary_table(self, start_date, end_date): """Populate the cost summary tables. Args: start_date (str) The date to start populating the table. end_date (str) The date to end on. Returns None """ cluster_id = get_cluster_id_from_provider(self._provider.uuid) # This needs to always run regardless of whether the OpenShift # cluster is tied to a cloud provider infra_map = self._get_ocp_infra_map(start_date, end_date) aws_uuid = self._get_aws_provider_uuid_from_map( self._provider, infra_map) with CostModelDBAccessor(self._schema_name, aws_uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() aws_markup_value = float(markup.get('value', 0)) / 100 with CostModelDBAccessor(self._schema_name, self._provider.uuid, self._column_map) as cost_model_accessor: markup = cost_model_accessor.get_markup() ocp_markup_value = float(markup.get('value', 0)) / 100 with OCPReportDBAccessor(self._schema_name, self._column_map) as accessor: LOG.info( 'Updating OpenShift on OCP cost summary table for' '\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s', self._schema_name, self._provider.uuid, start_date, end_date) accessor.populate_cost_summary_table(cluster_id, start_date, end_date) accessor.populate_ocp_on_aws_markup_cost(aws_markup_value, ocp_markup_value, cluster_id)
def update_summary_charge_info(self): """Update the OCP summary table with the charge information. Args: None Returns None """ self._cluster_id = get_cluster_id_from_provider(self._provider_uuid) LOG.info( 'Starting charge calculation updates for provider: %s. Cluster ID: %s.', self._provider_uuid, self._cluster_id) self._update_pod_charge() self._update_storage_charge() with OCPReportDBAccessor(self._schema, self._column_map) as accessor: LOG.info( 'Updating OpenShift on Cloud cost summary for schema: %s and provider: %s', self._schema, self._provider_uuid) accessor.populate_cost_summary_table(self._cluster_id) accessor.commit()
def test_get_cluster_id_from_non_ocp_provider(self): """Test that None is returned when getting cluster ID on non-OCP provider.""" cluster_id = utils.get_cluster_id_from_provider( self.aws_test_provider_uuid) self.assertIsNone(cluster_id)
def test_get_cluster_id_from_provider(self): """Test that the cluster ID is returned from OCP provider.""" cluster_id = utils.get_cluster_id_from_provider( self.ocp_test_provider_uuid) self.assertIsNotNone(cluster_id)
def test_populate_ocp_on_aws_cost_daily_summary(self): """Test that the OCP on AWS cost summary table is populated.""" summary_table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_daily_summary'] project_summary_table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_project_daily_summary'] bill_ids = [] summary_table = getattr(self.accessor.report_schema, summary_table_name) project_table = getattr(self.accessor.report_schema, project_summary_table_name) today = DateAccessor().today_with_timezone('UTC') last_month = today - relativedelta.relativedelta(months=1) resource_id = 'i-12345' for cost_entry_date in (today, last_month): bill = self.creator.create_cost_entry_bill(cost_entry_date) bill_ids.append(str(bill.id)) cost_entry = self.creator.create_cost_entry(bill, cost_entry_date) product = self.creator.create_cost_entry_product('Compute Instance') pricing = self.creator.create_cost_entry_pricing() reservation = self.creator.create_cost_entry_reservation() self.creator.create_cost_entry_line_item( bill, cost_entry, product, pricing, reservation, resource_id=resource_id ) self.accessor.populate_line_item_daily_table(last_month, today, bill_ids) li_table_name = AWS_CUR_TABLE_MAP['line_item'] li_table = getattr(self.accessor.report_schema, li_table_name) sum_aws_cost = self.accessor._session.query( func.sum(li_table.unblended_cost) ).first() with OCPReportDBAccessor(self.test_schema, self.column_map) as ocp_accessor: cluster_id = self.ocp_provider_resource_name with ProviderDBAccessor(provider_uuid=self.ocp_test_provider_uuid) as provider_access: provider_id = provider_access.get_provider().id for cost_entry_date in (today, last_month): period = self.creator.create_ocp_report_period(cost_entry_date, provider_id=provider_id, cluster_id=cluster_id) report = self.creator.create_ocp_report(period, cost_entry_date) self.creator.create_ocp_usage_line_item( period, report, resource_id=resource_id ) cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid) ocp_accessor.populate_line_item_daily_table(last_month, today, cluster_id) query = self.accessor._get_db_obj_query(summary_table_name) initial_count = query.count() self.accessor.populate_ocp_on_aws_cost_daily_summary(last_month, today, cluster_id, bill_ids) self.assertNotEqual(query.count(), initial_count) sum_cost = self.accessor._session.query( func.sum(summary_table.unblended_cost) ).first() sum_project_cost = self.accessor._session.query( func.sum(project_table.unblended_cost) ).first() self.assertEqual(sum_cost, sum_project_cost) self.assertLessEqual(sum_cost, sum_aws_cost)
def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on AWS.""" if isinstance(start_date, str): start_date = parser.parse(start_date).date() if isinstance(end_date, str): end_date = parser.parse(end_date).date() cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) cluster_alias = get_cluster_alias_from_cluster_id(cluster_id) with OCPReportDBAccessor(self._schema) as accessor: if not accessor.get_cluster_for_provider(openshift_provider_uuid): LOG.info( f"No cluster information available for OCP Provider: {openshift_provider_uuid}, " f"skipping OCP on Cloud summary table update for AWS source: {aws_provider_uuid}." ) return report_period = accessor.report_periods_for_provider_uuid( openshift_provider_uuid, start_date) if not report_period: LOG.info( f"No report period for AWS provider {openshift_provider_uuid} with start date {start_date}" ) return accessor.delete_infrastructure_raw_cost_from_daily_summary( openshift_provider_uuid, report_period.id, start_date, end_date) aws_bills = aws_get_bills_from_provider(aws_provider_uuid, self._schema, start_date, end_date) with schema_context(self._schema): self._handle_partitions( self._schema, ( "reporting_ocpawscostlineitem_daily_summary_p", "reporting_ocpawscostlineitem_project_daily_summary_p", "reporting_ocpaws_compute_summary_p", "reporting_ocpaws_cost_summary_p", "reporting_ocpaws_cost_summary_by_account_p", "reporting_ocpaws_cost_summary_by_region_p", "reporting_ocpaws_cost_summary_by_service_p", "reporting_ocpaws_storage_summary_p", "reporting_ocpaws_database_summary_p", "reporting_ocpaws_network_summary_p", "reporting_ocpallcostlineitem_daily_summary_p", "reporting_ocpallcostlineitem_project_daily_summary_p", "reporting_ocpall_compute_summary_pt", "reporting_ocpall_cost_summary_pt", ), start_date, end_date, ) aws_bill_ids = [str(bill.id) for bill in aws_bills] current_aws_bill_id = aws_bills.first().id if aws_bills else None current_ocp_report_period_id = report_period.id with CostModelDBAccessor(self._schema, aws_provider_uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = Decimal(markup.get("value", 0)) / 100 with CostModelDBAccessor( self._schema, openshift_provider_uuid) as cost_model_accessor: distribution = cost_model_accessor.distribution # OpenShift on AWS sql_params = { "schema_name": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": aws_provider_uuid, "cluster_id": cluster_id, "cluster_alias": cluster_alias, } with AWSReportDBAccessor(self._schema) as accessor: for start, end in date_range_pair(start_date, end_date, step=settings.TRINO_DATE_STEP): LOG.info( "Updating OpenShift on AWS summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, AWS Bill ID: %s", self._schema, self._provider.uuid, start, end, cluster_id, current_aws_bill_id, ) filters = { "report_period_id": current_ocp_report_period_id } # Use report_period_id to leverage DB index on DELETE accessor.delete_line_item_daily_summary_entries_for_date_range_raw( self._provider.uuid, start, end, filters, table=OCPAWSCostLineItemProjectDailySummaryP) accessor.populate_ocp_on_aws_cost_daily_summary_presto( start, end, openshift_provider_uuid, aws_provider_uuid, current_ocp_report_period_id, current_aws_bill_id, markup_value, distribution, ) accessor.back_populate_ocp_on_aws_daily_summary( start_date, end_date, current_ocp_report_period_id) accessor.populate_ocp_on_aws_tags_summary_table( aws_bill_ids, start_date, end_date) accessor.populate_ocp_on_aws_ui_summary_tables(sql_params) with OCPReportDBAccessor(self._schema) as ocp_accessor: sql_params["source_type"] = "AWS" LOG.info( f"Processing OCP-ALL for AWS (T) (s={start_date} e={end_date})" ) ocp_accessor.populate_ocp_on_all_project_daily_summary( "aws", sql_params) ocp_accessor.populate_ocp_on_all_daily_summary( "aws", sql_params) ocp_accessor.populate_ocp_on_all_ui_summary_tables(sql_params) ocp_accessor.populate_ui_summary_tables( start, end, openshift_provider_uuid, UI_SUMMARY_TABLES_MARKUP_SUBSET) LOG.info( "Updating ocp_on_cloud_updated_datetime OpenShift report periods") with schema_context(self._schema): report_period.ocp_on_cloud_updated_datetime = self._date_accessor.today_with_timezone( "UTC") report_period.save()