def create_bill(self, bill_date):
"""Create bill postgres entry."""
if isinstance(bill_date, str):
bill_date = ciso8601.parse_datetime(bill_date)
report_date_range = month_date_range(bill_date)
start_date, end_date = report_date_range.split("-")
report_period_start = ciso8601.parse_datetime(start_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
report_period_end = ciso8601.parse_datetime(end_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
# Make end date first of next month
report_period_end = report_period_end + datetime.timedelta(days=1)
provider = self._get_provider()
cluster_id = utils.get_cluster_id_from_provider(provider.uuid)
cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id)
with schema_context(self._schema_name):
OCPUsageReportPeriod.objects.get_or_create(
cluster_id=cluster_id,
cluster_alias=cluster_alias,
report_period_start=report_period_start,
report_period_end=report_period_end,
provider=provider,
)
def test_update_gcp_summary_tables(
self,
mock_utility,
mock_delete,
mock_back_populate,
mock_ocp_on_gcp,
mock_ui_tables,
mock_tag_summary,
mock_map,
mock_ocpallproj,
mock_ocpall,
mock_ocpallui,
):
"""Test that summary tables are properly run for a gcp provider."""
fake_bills = MagicMock()
fake_bills.__iter__.return_value = [Mock(), Mock()]
first = Mock()
bill_id = 1
first.return_value.id = bill_id
fake_bills.first = first
mock_utility.return_value = fake_bills
start_date = self.dh.today.date()
end_date = start_date + datetime.timedelta(days=1)
with ProviderDBAccessor(self.gcp_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
with OCPReportDBAccessor(self.schema_name) as accessor:
report_period = accessor.report_periods_for_provider_uuid(self.ocpgcp_provider_uuid, start_date)
with schema_context(self.schema_name):
current_ocp_report_period_id = report_period.id
mock_map.return_value = {self.ocpgcp_provider_uuid: (self.gcp_provider_uuid, Provider.PROVIDER_GCP)}
updater = OCPCloudParquetReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None)
updater.update_gcp_summary_tables(self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid, start_date, end_date)
cluster_id = get_cluster_id_from_provider(self.ocpgcp_provider_uuid)
cluster_alias = get_cluster_alias_from_cluster_id(cluster_id)
sql_params = {
"schema_name": self.schema_name,
"start_date": start_date,
"end_date": end_date,
"source_uuid": self.gcp_test_provider_uuid,
"cluster_id": cluster_id,
"cluster_alias": cluster_alias,
"source_type": "GCP",
}
distribution = None
mock_ocp_on_gcp.assert_called_with(
start_date,
end_date,
self.ocpgcp_provider_uuid,
cluster_id,
self.gcp_test_provider_uuid,
current_ocp_report_period_id,
bill_id,
decimal.Decimal(0),
distribution,
)
mock_ui_tables.assert_called_with(sql_params)
mock_tag_summary.assert_called_with([str(bill.id) for bill in fake_bills], start_date, end_date)
def __init__(self, schema, provider, manifest):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
self._schema = schema
self._provider = provider
self._manifest = manifest
self._cluster_id = get_cluster_id_from_provider(self._provider.uuid)
self._cluster_alias = get_cluster_alias_from_cluster_id(self._cluster_id)
self._date_accessor = DateAccessor()
def __init__(self, schema, provider):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
super().__init__(schema, provider, None)
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
self._cluster_alias = get_cluster_alias_from_cluster_id(
self._cluster_id)
with CostModelDBAccessor(self._schema,
self._provider_uuid) as cost_model_accessor:
self._infra_rates = cost_model_accessor.infrastructure_rates
self._supplementary_rates = cost_model_accessor.supplementary_rates
def __init__(self, schema_name, report_path, compression, provider_uuid):
"""Initialize base class."""
super().__init__(
schema_name=schema_name,
report_path=report_path,
compression=compression,
provider_uuid=provider_uuid,
manifest_id=None,
processed_report=ProcessedOCPReport(),
)
self._report_name = path.basename(report_path)
self._cluster_id = utils.get_cluster_id_from_provider(provider_uuid)
self._cluster_alias = utils.get_cluster_alias_from_cluster_id(self._cluster_id)
self._datetime_format = Config.OCP_DATETIME_STR_FORMAT
self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE
with OCPReportDBAccessor(self._schema) as report_db:
self.existing_report_periods_map = report_db.get_report_periods()
self.existing_report_map = report_db.get_reports()
self.line_item_columns = None
def update_summary_charge_info(self, start_date, end_date):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
if isinstance(start_date, str):
start_date = parse(start_date)
if isinstance(end_date, str):
end_date = parse(end_date)
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
self._cluster_alias = get_cluster_alias_from_cluster_id(
self._cluster_id)
LOG.info(
"Starting charge calculation updates for provider: %s. Cluster ID: %s.",
self._provider_uuid,
self._cluster_id,
)
self._update_pod_charge(start_date, end_date)
self._update_storage_charge(start_date, end_date)
self._update_markup_cost(start_date, end_date)
self._update_monthly_cost(start_date, end_date)
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
report_periods = accessor.report_periods_for_provider_uuid(
self._provider_uuid, start_date)
with schema_context(self._schema):
for period in report_periods:
period.derived_cost_datetime = DateAccessor(
).today_with_timezone("UTC")
period.save()
def test_get_cluster_alias_from_cluster_id(self):
"""Test that the cluster alias is returned from cluster_id."""
cluster_id = self.ocp_cluster_id
cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id)
self.assertIsNotNone(cluster_alias)
def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
if isinstance(start_date, str):
start_date = parser.parse(start_date).date()
if isinstance(end_date, str):
end_date = parser.parse(end_date).date()
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
cluster_alias = get_cluster_alias_from_cluster_id(cluster_id)
with OCPReportDBAccessor(self._schema) as accessor:
if not accessor.get_cluster_for_provider(openshift_provider_uuid):
LOG.info(
f"No cluster information available for OCP Provider: {openshift_provider_uuid}, "
f"skipping OCP on Cloud summary table update for AWS source: {aws_provider_uuid}."
)
return
report_period = accessor.report_periods_for_provider_uuid(
openshift_provider_uuid, start_date)
if not report_period:
LOG.info(
f"No report period for AWS provider {openshift_provider_uuid} with start date {start_date}"
)
return
accessor.delete_infrastructure_raw_cost_from_daily_summary(
openshift_provider_uuid, report_period.id, start_date,
end_date)
aws_bills = aws_get_bills_from_provider(aws_provider_uuid,
self._schema, start_date,
end_date)
with schema_context(self._schema):
self._handle_partitions(
self._schema,
(
"reporting_ocpawscostlineitem_daily_summary_p",
"reporting_ocpawscostlineitem_project_daily_summary_p",
"reporting_ocpaws_compute_summary_p",
"reporting_ocpaws_cost_summary_p",
"reporting_ocpaws_cost_summary_by_account_p",
"reporting_ocpaws_cost_summary_by_region_p",
"reporting_ocpaws_cost_summary_by_service_p",
"reporting_ocpaws_storage_summary_p",
"reporting_ocpaws_database_summary_p",
"reporting_ocpaws_network_summary_p",
"reporting_ocpallcostlineitem_daily_summary_p",
"reporting_ocpallcostlineitem_project_daily_summary_p",
"reporting_ocpall_compute_summary_pt",
"reporting_ocpall_cost_summary_pt",
),
start_date,
end_date,
)
aws_bill_ids = [str(bill.id) for bill in aws_bills]
current_aws_bill_id = aws_bills.first().id if aws_bills else None
current_ocp_report_period_id = report_period.id
with CostModelDBAccessor(self._schema,
aws_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
with CostModelDBAccessor(
self._schema, openshift_provider_uuid) as cost_model_accessor:
distribution = cost_model_accessor.distribution
# OpenShift on AWS
sql_params = {
"schema_name": self._schema,
"start_date": start_date,
"end_date": end_date,
"source_uuid": aws_provider_uuid,
"cluster_id": cluster_id,
"cluster_alias": cluster_alias,
}
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date,
end_date,
step=settings.TRINO_DATE_STEP):
LOG.info(
"Updating OpenShift on AWS summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, AWS Bill ID: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
current_aws_bill_id,
)
filters = {
"report_period_id": current_ocp_report_period_id
} # Use report_period_id to leverage DB index on DELETE
accessor.delete_line_item_daily_summary_entries_for_date_range_raw(
self._provider.uuid,
start,
end,
filters,
table=OCPAWSCostLineItemProjectDailySummaryP)
accessor.populate_ocp_on_aws_cost_daily_summary_presto(
start,
end,
openshift_provider_uuid,
aws_provider_uuid,
current_ocp_report_period_id,
current_aws_bill_id,
markup_value,
distribution,
)
accessor.back_populate_ocp_on_aws_daily_summary(
start_date, end_date, current_ocp_report_period_id)
accessor.populate_ocp_on_aws_tags_summary_table(
aws_bill_ids, start_date, end_date)
accessor.populate_ocp_on_aws_ui_summary_tables(sql_params)
with OCPReportDBAccessor(self._schema) as ocp_accessor:
sql_params["source_type"] = "AWS"
LOG.info(
f"Processing OCP-ALL for AWS (T) (s={start_date} e={end_date})"
)
ocp_accessor.populate_ocp_on_all_project_daily_summary(
"aws", sql_params)
ocp_accessor.populate_ocp_on_all_daily_summary(
"aws", sql_params)
ocp_accessor.populate_ocp_on_all_ui_summary_tables(sql_params)
ocp_accessor.populate_ui_summary_tables(
start, end, openshift_provider_uuid,
UI_SUMMARY_TABLES_MARKUP_SUBSET)
LOG.info(
"Updating ocp_on_cloud_updated_datetime OpenShift report periods")
with schema_context(self._schema):
report_period.ocp_on_cloud_updated_datetime = self._date_accessor.today_with_timezone(
"UTC")
report_period.save()
