def create_bill(self, bill_date): """Create bill postgres entry.""" if isinstance(bill_date, str): bill_date = ciso8601.parse_datetime(bill_date) report_date_range = month_date_range(bill_date) start_date, end_date = report_date_range.split("-") report_period_start = ciso8601.parse_datetime(start_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) report_period_end = ciso8601.parse_datetime(end_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) # Make end date first of next month report_period_end = report_period_end + datetime.timedelta(days=1) provider = self._get_provider() cluster_id = utils.get_cluster_id_from_provider(provider.uuid) cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id) with schema_context(self._schema_name): OCPUsageReportPeriod.objects.get_or_create( cluster_id=cluster_id, cluster_alias=cluster_alias, report_period_start=report_period_start, report_period_end=report_period_end, provider=provider, )
def test_update_gcp_summary_tables( self, mock_utility, mock_delete, mock_back_populate, mock_ocp_on_gcp, mock_ui_tables, mock_tag_summary, mock_map, mock_ocpallproj, mock_ocpall, mock_ocpallui, ): """Test that summary tables are properly run for a gcp provider.""" fake_bills = MagicMock() fake_bills.__iter__.return_value = [Mock(), Mock()] first = Mock() bill_id = 1 first.return_value.id = bill_id fake_bills.first = first mock_utility.return_value = fake_bills start_date = self.dh.today.date() end_date = start_date + datetime.timedelta(days=1) with ProviderDBAccessor(self.gcp_provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() with OCPReportDBAccessor(self.schema_name) as accessor: report_period = accessor.report_periods_for_provider_uuid(self.ocpgcp_provider_uuid, start_date) with schema_context(self.schema_name): current_ocp_report_period_id = report_period.id mock_map.return_value = {self.ocpgcp_provider_uuid: (self.gcp_provider_uuid, Provider.PROVIDER_GCP)} updater = OCPCloudParquetReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None) updater.update_gcp_summary_tables(self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid, start_date, end_date) cluster_id = get_cluster_id_from_provider(self.ocpgcp_provider_uuid) cluster_alias = get_cluster_alias_from_cluster_id(cluster_id) sql_params = { "schema_name": self.schema_name, "start_date": start_date, "end_date": end_date, "source_uuid": self.gcp_test_provider_uuid, "cluster_id": cluster_id, "cluster_alias": cluster_alias, "source_type": "GCP", } distribution = None mock_ocp_on_gcp.assert_called_with( start_date, end_date, self.ocpgcp_provider_uuid, cluster_id, self.gcp_test_provider_uuid, current_ocp_report_period_id, bill_id, decimal.Decimal(0), distribution, ) mock_ui_tables.assert_called_with(sql_params) mock_tag_summary.assert_called_with([str(bill.id) for bill in fake_bills], start_date, end_date)
def __init__(self, schema, provider, manifest): """Establish the database connection. Args: schema (str): The customer schema to associate with """ self._schema = schema self._provider = provider self._manifest = manifest self._cluster_id = get_cluster_id_from_provider(self._provider.uuid) self._cluster_alias = get_cluster_alias_from_cluster_id(self._cluster_id) self._date_accessor = DateAccessor()
def __init__(self, schema, provider): """Establish the database connection. Args: schema (str): The customer schema to associate with """ super().__init__(schema, provider, None) self._cluster_id = get_cluster_id_from_provider(self._provider_uuid) self._cluster_alias = get_cluster_alias_from_cluster_id( self._cluster_id) with CostModelDBAccessor(self._schema, self._provider_uuid) as cost_model_accessor: self._infra_rates = cost_model_accessor.infrastructure_rates self._supplementary_rates = cost_model_accessor.supplementary_rates
def __init__(self, schema_name, report_path, compression, provider_uuid): """Initialize base class.""" super().__init__( schema_name=schema_name, report_path=report_path, compression=compression, provider_uuid=provider_uuid, manifest_id=None, processed_report=ProcessedOCPReport(), ) self._report_name = path.basename(report_path) self._cluster_id = utils.get_cluster_id_from_provider(provider_uuid) self._cluster_alias = utils.get_cluster_alias_from_cluster_id(self._cluster_id) self._datetime_format = Config.OCP_DATETIME_STR_FORMAT self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE with OCPReportDBAccessor(self._schema) as report_db: self.existing_report_periods_map = report_db.get_report_periods() self.existing_report_map = report_db.get_reports() self.line_item_columns = None
def update_summary_charge_info(self, start_date, end_date): """Update the OCP summary table with the charge information. Args: start_date (str, Optional) - Start date of range to update derived cost. end_date (str, Optional) - End date of range to update derived cost. Returns None """ if isinstance(start_date, str): start_date = parse(start_date) if isinstance(end_date, str): end_date = parse(end_date) self._cluster_id = get_cluster_id_from_provider(self._provider_uuid) self._cluster_alias = get_cluster_alias_from_cluster_id( self._cluster_id) LOG.info( "Starting charge calculation updates for provider: %s. Cluster ID: %s.", self._provider_uuid, self._cluster_id, ) self._update_pod_charge(start_date, end_date) self._update_storage_charge(start_date, end_date) self._update_markup_cost(start_date, end_date) self._update_monthly_cost(start_date, end_date) with OCPReportDBAccessor(self._schema, self._column_map) as accessor: report_periods = accessor.report_periods_for_provider_uuid( self._provider_uuid, start_date) with schema_context(self._schema): for period in report_periods: period.derived_cost_datetime = DateAccessor( ).today_with_timezone("UTC") period.save()
def test_get_cluster_alias_from_cluster_id(self): """Test that the cluster alias is returned from cluster_id.""" cluster_id = self.ocp_cluster_id cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id) self.assertIsNotNone(cluster_alias)
def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date): """Update operations specifically for OpenShift on AWS.""" if isinstance(start_date, str): start_date = parser.parse(start_date).date() if isinstance(end_date, str): end_date = parser.parse(end_date).date() cluster_id = get_cluster_id_from_provider(openshift_provider_uuid) cluster_alias = get_cluster_alias_from_cluster_id(cluster_id) with OCPReportDBAccessor(self._schema) as accessor: if not accessor.get_cluster_for_provider(openshift_provider_uuid): LOG.info( f"No cluster information available for OCP Provider: {openshift_provider_uuid}, " f"skipping OCP on Cloud summary table update for AWS source: {aws_provider_uuid}." ) return report_period = accessor.report_periods_for_provider_uuid( openshift_provider_uuid, start_date) if not report_period: LOG.info( f"No report period for AWS provider {openshift_provider_uuid} with start date {start_date}" ) return accessor.delete_infrastructure_raw_cost_from_daily_summary( openshift_provider_uuid, report_period.id, start_date, end_date) aws_bills = aws_get_bills_from_provider(aws_provider_uuid, self._schema, start_date, end_date) with schema_context(self._schema): self._handle_partitions( self._schema, ( "reporting_ocpawscostlineitem_daily_summary_p", "reporting_ocpawscostlineitem_project_daily_summary_p", "reporting_ocpaws_compute_summary_p", "reporting_ocpaws_cost_summary_p", "reporting_ocpaws_cost_summary_by_account_p", "reporting_ocpaws_cost_summary_by_region_p", "reporting_ocpaws_cost_summary_by_service_p", "reporting_ocpaws_storage_summary_p", "reporting_ocpaws_database_summary_p", "reporting_ocpaws_network_summary_p", "reporting_ocpallcostlineitem_daily_summary_p", "reporting_ocpallcostlineitem_project_daily_summary_p", "reporting_ocpall_compute_summary_pt", "reporting_ocpall_cost_summary_pt", ), start_date, end_date, ) aws_bill_ids = [str(bill.id) for bill in aws_bills] current_aws_bill_id = aws_bills.first().id if aws_bills else None current_ocp_report_period_id = report_period.id with CostModelDBAccessor(self._schema, aws_provider_uuid) as cost_model_accessor: markup = cost_model_accessor.markup markup_value = Decimal(markup.get("value", 0)) / 100 with CostModelDBAccessor( self._schema, openshift_provider_uuid) as cost_model_accessor: distribution = cost_model_accessor.distribution # OpenShift on AWS sql_params = { "schema_name": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": aws_provider_uuid, "cluster_id": cluster_id, "cluster_alias": cluster_alias, } with AWSReportDBAccessor(self._schema) as accessor: for start, end in date_range_pair(start_date, end_date, step=settings.TRINO_DATE_STEP): LOG.info( "Updating OpenShift on AWS summary table for " "\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s" "\n\tCluster ID: %s, AWS Bill ID: %s", self._schema, self._provider.uuid, start, end, cluster_id, current_aws_bill_id, ) filters = { "report_period_id": current_ocp_report_period_id } # Use report_period_id to leverage DB index on DELETE accessor.delete_line_item_daily_summary_entries_for_date_range_raw( self._provider.uuid, start, end, filters, table=OCPAWSCostLineItemProjectDailySummaryP) accessor.populate_ocp_on_aws_cost_daily_summary_presto( start, end, openshift_provider_uuid, aws_provider_uuid, current_ocp_report_period_id, current_aws_bill_id, markup_value, distribution, ) accessor.back_populate_ocp_on_aws_daily_summary( start_date, end_date, current_ocp_report_period_id) accessor.populate_ocp_on_aws_tags_summary_table( aws_bill_ids, start_date, end_date) accessor.populate_ocp_on_aws_ui_summary_tables(sql_params) with OCPReportDBAccessor(self._schema) as ocp_accessor: sql_params["source_type"] = "AWS" LOG.info( f"Processing OCP-ALL for AWS (T) (s={start_date} e={end_date})" ) ocp_accessor.populate_ocp_on_all_project_daily_summary( "aws", sql_params) ocp_accessor.populate_ocp_on_all_daily_summary( "aws", sql_params) ocp_accessor.populate_ocp_on_all_ui_summary_tables(sql_params) ocp_accessor.populate_ui_summary_tables( start, end, openshift_provider_uuid, UI_SUMMARY_TABLES_MARKUP_SUBSET) LOG.info( "Updating ocp_on_cloud_updated_datetime OpenShift report periods") with schema_context(self._schema): report_period.ocp_on_cloud_updated_datetime = self._date_accessor.today_with_timezone( "UTC") report_period.save()