def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
aws_bills = aws_get_bills_from_provider(
aws_provider_uuid, self._schema,
datetime.datetime.strptime(start_date, '%Y-%m-%d'),
datetime.datetime.strptime(end_date, '%Y-%m-%d'))
aws_bill_ids = []
with schema_context(self._schema):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
with CostModelDBAccessor(self._schema, aws_provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = Decimal(markup.get('value', 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s'
'\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema,
self._provider.uuid, start_date, end_date, cluster_id,
str(aws_bill_ids))
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.populate_ocp_on_aws_markup_cost(markup_value,
aws_bill_ids)
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
def create_bill(self, bill_date):
"""Create bill postgres entry."""
if isinstance(bill_date, str):
bill_date = ciso8601.parse_datetime(bill_date)
report_date_range = month_date_range(bill_date)
start_date, end_date = report_date_range.split("-")
report_period_start = ciso8601.parse_datetime(start_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
report_period_end = ciso8601.parse_datetime(end_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
# Make end date first of next month
report_period_end = report_period_end + datetime.timedelta(days=1)
provider = self._get_provider()
cluster_id = utils.get_cluster_id_from_provider(provider.uuid)
cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id)
with schema_context(self._schema_name):
OCPUsageReportPeriod.objects.get_or_create(
cluster_id=cluster_id,
cluster_alias=cluster_alias,
report_period_start=report_period_start,
report_period_end=report_period_end,
provider=provider,
)
def _update_markup_cost(self, start_date, end_date):
"""Populate markup costs for OpenShift.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
infra_markup_value = Decimal(0.0)
infra_map = self.get_infra_map()
infra_tuple = infra_map.get(self._provider_uuid)
cluster_id = get_cluster_id_from_provider(self._provider_uuid)
if infra_tuple:
infra_uuid = infra_tuple[0]
with CostModelDBAccessor(self._schema, infra_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
infra_markup_value = Decimal(markup.get('value', 0)) / 100
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
ocp_markup_value = Decimal(markup.get('value', 0)) / 100
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info('Updating OpenShift markup for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema, self._provider_uuid, start_date, end_date)
accessor.populate_markup_cost(infra_markup_value, ocp_markup_value, cluster_id)
LOG.info('Finished updating markup.')
def update_summary_charge_info(self, start_date=None, end_date=None):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
LOG.info(
'Starting charge calculation updates for provider: %s. Cluster ID: %s.',
self._provider_uuid, self._cluster_id)
self._update_pod_charge()
self._update_storage_charge()
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
'Updating OpenShift on Cloud cost summary for schema: %s and provider: %s',
self._schema, self._provider_uuid)
accessor.populate_cost_summary_table(self._cluster_id,
start_date=start_date,
end_date=end_date)
report_periods = accessor.report_periods_for_provider_id(
self._provider_id, start_date)
for period in report_periods:
period.derived_cost_datetime = DateAccessor(
).today_with_timezone('UTC')
accessor.commit()
def update_summary_charge_info(self, start_date=None, end_date=None):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
LOG.info(
'Starting charge calculation updates for provider: %s. Cluster ID: %s.',
self._provider_uuid, self._cluster_id)
self._update_pod_charge()
self._update_storage_charge()
self._update_markup_cost()
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
report_periods = accessor.report_periods_for_provider_uuid(
self._provider_uuid, start_date)
with schema_context(self._schema):
for period in report_periods:
period.derived_cost_datetime = DateAccessor(
).today_with_timezone('UTC')
period.save()
def test_update_gcp_summary_tables(
self,
mock_utility,
mock_delete,
mock_back_populate,
mock_ocp_on_gcp,
mock_ui_tables,
mock_tag_summary,
mock_map,
mock_ocpallproj,
mock_ocpall,
mock_ocpallui,
):
"""Test that summary tables are properly run for a gcp provider."""
fake_bills = MagicMock()
fake_bills.__iter__.return_value = [Mock(), Mock()]
first = Mock()
bill_id = 1
first.return_value.id = bill_id
fake_bills.first = first
mock_utility.return_value = fake_bills
start_date = self.dh.today.date()
end_date = start_date + datetime.timedelta(days=1)
with ProviderDBAccessor(self.gcp_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
with OCPReportDBAccessor(self.schema_name) as accessor:
report_period = accessor.report_periods_for_provider_uuid(self.ocpgcp_provider_uuid, start_date)
with schema_context(self.schema_name):
current_ocp_report_period_id = report_period.id
mock_map.return_value = {self.ocpgcp_provider_uuid: (self.gcp_provider_uuid, Provider.PROVIDER_GCP)}
updater = OCPCloudParquetReportSummaryUpdater(schema="acct10001", provider=provider, manifest=None)
updater.update_gcp_summary_tables(self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid, start_date, end_date)
cluster_id = get_cluster_id_from_provider(self.ocpgcp_provider_uuid)
cluster_alias = get_cluster_alias_from_cluster_id(cluster_id)
sql_params = {
"schema_name": self.schema_name,
"start_date": start_date,
"end_date": end_date,
"source_uuid": self.gcp_test_provider_uuid,
"cluster_id": cluster_id,
"cluster_alias": cluster_alias,
"source_type": "GCP",
}
distribution = None
mock_ocp_on_gcp.assert_called_with(
start_date,
end_date,
self.ocpgcp_provider_uuid,
cluster_id,
self.gcp_test_provider_uuid,
current_ocp_report_period_id,
bill_id,
decimal.Decimal(0),
distribution,
)
mock_ui_tables.assert_called_with(sql_params)
mock_tag_summary.assert_called_with([str(bill.id) for bill in fake_bills], start_date, end_date)
def test_poll_ingest_override_for_provider(self):
"""Test that OCP polling override returns True if insights local path exists."""
fake_dir = tempfile.mkdtemp()
with patch.object(Config, 'INSIGHTS_LOCAL_REPORT_DIR', fake_dir):
cluster_id = utils.get_cluster_id_from_provider(self.ocp_test_provider_uuid)
expected_path = '{}/{}/'.format(Config.INSIGHTS_LOCAL_REPORT_DIR, cluster_id)
os.makedirs(expected_path, exist_ok=True)
self.assertTrue(utils.poll_ingest_override_for_provider(self.ocp_test_provider_uuid))
shutil.rmtree(fake_dir)
def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
if isinstance(start_date, str):
start_date = parser.parse(start_date).date()
if isinstance(end_date, str):
end_date = parser.parse(end_date).date()
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
with OCPReportDBAccessor(self._schema) as accessor:
report_period = accessor.report_periods_for_provider_uuid(
openshift_provider_uuid, start_date)
accessor.delete_infrastructure_raw_cost_from_daily_summary(
openshift_provider_uuid, report_period.id, start_date,
end_date)
aws_bills = aws_get_bills_from_provider(aws_provider_uuid,
self._schema, start_date,
end_date)
with schema_context(self._schema):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
current_aws_bill_id = aws_bills.first().id if aws_bills else None
current_ocp_report_period_id = report_period.id
with CostModelDBAccessor(self._schema,
aws_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date,
end_date,
step=settings.TRINO_DATE_STEP):
LOG.info(
"Updating OpenShift on AWS summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, AWS Bill ID: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
current_aws_bill_id,
)
accessor.populate_ocp_on_aws_cost_daily_summary_presto(
start,
end,
openshift_provider_uuid,
aws_provider_uuid,
current_ocp_report_period_id,
current_aws_bill_id,
markup_value,
)
accessor.back_populate_ocp_on_aws_daily_summary(
start_date, end_date, current_ocp_report_period_id)
accessor.populate_ocp_on_aws_tags_summary_table(
aws_bill_ids, start_date, end_date)
def test_get_cluster_id_with_no_authentication(self):
"""Test that a None is correctly returned if authentication is not present."""
# Remove test provider authentication
Provider.objects.filter(uuid=self.ocp_test_provider_uuid).update(
authentication=None)
ocp_provider = Provider.objects.get(uuid=self.ocp_test_provider_uuid)
self.assertIsNone(ocp_provider.authentication)
# Assert if authentication is empty we return none instead of an error
cluster_id = utils.get_cluster_id_from_provider(
self.ocp_test_provider_uuid)
self.assertIsNone(cluster_id)
def __init__(self, schema, provider, manifest):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
self._schema = schema
self._provider = provider
self._manifest = manifest
self._cluster_id = get_cluster_id_from_provider(self._provider.uuid)
self._date_accessor = DateAccessor()
def __init__(self, schema, provider, manifest):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
self._schema_name = schema
self._provider = provider
self._manifest = manifest
self._cluster_id = get_cluster_id_from_provider(self._provider.uuid)
with ReportingCommonDBAccessor() as reporting_common:
self._column_map = reporting_common.column_map
self._date_accessor = DateAccessor()
def update_azure_summary_tables(self, openshift_provider_uuid,
azure_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on Azure."""
if isinstance(start_date, str):
start_date = parser.parse(start_date).date()
if isinstance(end_date, str):
end_date = parser.parse(end_date).date()
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
azure_bills = azure_get_bills_from_provider(azure_provider_uuid,
self._schema, start_date,
end_date)
with schema_context(self._schema):
current_azure_bill_id = azure_bills.first(
).id if azure_bills else None
with CostModelDBAccessor(self._schema,
azure_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on Azure
with AzureReportDBAccessor(self._schema) as accessor:
LOG.info(
"Updating OpenShift on Azure summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, Azure Bill ID: %s",
self._schema,
self._provider.uuid,
start_date,
end_date,
cluster_id,
current_azure_bill_id,
)
accessor.populate_ocp_on_azure_cost_daily_summary_presto(
start_date,
end_date,
openshift_provider_uuid,
azure_provider_uuid,
cluster_id,
current_azure_bill_id,
markup_value,
)
accessor.populate_ocp_on_azure_tags_summary_table()
with OCPReportDBAccessor(self._schema) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
def test_update_cost_summary_table_for_markup(self, mock_markup):
"""Test that summary tables are updated correctly."""
markup = {'value': 10, 'unit': 'percent'}
mock_markup.return_value = markup
self._generate_ocp_on_aws_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta.relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
ocp_accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
cost_summary_table = OCP_REPORT_TABLE_MAP['cost_summary']
cost_summary_query = ocp_accessor._get_db_obj_query(
cost_summary_table)
possible_values = {}
with schema_context(self.schema):
for item in cost_summary_query:
possible_values.update({
item.cluster_id:
((item.pod_charge_cpu_core_hours +
item.pod_charge_memory_gigabyte_hours +
item.persistentvolumeclaim_charge_gb_month +
item.infra_cost) * decimal.Decimal(0.1))
})
updater.update_cost_summary_table(start_date_str, end_date_str)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(cost_summary_table)
found_values = {}
for item in query:
found_values.update({item.cluster_id: item.markup_cost})
for k, v in found_values.items():
self.assertAlmostEqual(v, possible_values[k], places=6)
def update_azure_summary_tables(self, openshift_provider_uuid,
azure_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on Azure."""
if isinstance(start_date, str):
start_date = parser.parse(start_date).date()
if isinstance(end_date, str):
end_date = parser.parse(end_date).date()
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
azure_bills = azure_get_bills_from_provider(azure_provider_uuid,
self._schema, start_date,
end_date)
with schema_context(self._schema):
azure_bill_ids = [str(bill.id) for bill in azure_bills]
current_azure_bill_id = azure_bills.first(
).id if azure_bills else None
with CostModelDBAccessor(self._schema,
azure_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on Azure
with AzureReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date,
end_date,
step=settings.TRINO_DATE_STEP):
LOG.info(
"Updating OpenShift on Azure summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, Azure Bill ID: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
current_azure_bill_id,
)
accessor.populate_ocp_on_azure_cost_daily_summary_presto(
start,
end,
openshift_provider_uuid,
azure_provider_uuid,
cluster_id,
current_azure_bill_id,
markup_value,
)
accessor.populate_ocp_on_azure_tags_summary_table(
azure_bill_ids, start_date, end_date)
def __init__(self, schema, provider):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
super().__init__(schema, provider, None)
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
self._cluster_alias = get_cluster_alias_from_cluster_id(
self._cluster_id)
with CostModelDBAccessor(self._schema,
self._provider_uuid) as cost_model_accessor:
self._infra_rates = cost_model_accessor.infrastructure_rates
self._supplementary_rates = cost_model_accessor.supplementary_rates
def _generate_ocp_on_aws_data(self, cluster_id=None):
"""Generate OCP and AWS data."""
if not cluster_id:
cluster_id = self.ocp_provider_resource_name
creator = ReportObjectCreator(self.schema, self.column_map)
bill_ids = []
today = DateAccessor().today_with_timezone("UTC")
last_month = today - relativedelta(months=1)
resource_id = "i-12345"
for cost_entry_date in (today, last_month):
bill = creator.create_cost_entry_bill(
provider_uuid=self.aws_provider_uuid,
bill_date=cost_entry_date)
bill_ids.append(str(bill.id))
cost_entry = creator.create_cost_entry(bill, cost_entry_date)
product = creator.create_cost_entry_product("Compute Instance")
pricing = creator.create_cost_entry_pricing()
reservation = creator.create_cost_entry_reservation()
creator.create_cost_entry_line_item(bill,
cost_entry,
product,
pricing,
reservation,
resource_id=resource_id)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
aws_accessor.populate_line_item_daily_table(
last_month.date(), today.date(), bill_ids)
provider_uuid = self.ocp_provider_uuid
for cost_entry_date in (today, last_month):
period = creator.create_ocp_report_period(
provider_uuid=provider_uuid,
period_date=cost_entry_date,
cluster_id=cluster_id)
report = creator.create_ocp_report(period, cost_entry_date)
creator.create_ocp_usage_line_item(period,
report,
resource_id=resource_id)
cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
ocp_accessor.populate_line_item_daily_table(
last_month.date(), today.date(), cluster_id)
def _generate_ocp_on_aws_data(self):
"""Test that the OCP on AWS cost summary table is populated."""
creator = ReportObjectCreator(self.accessor, self.column_map,
self.accessor.report_schema.column_types)
bill_ids = []
today = DateAccessor().today_with_timezone('UTC')
last_month = today - relativedelta.relativedelta(months=1)
resource_id = 'i-12345'
for cost_entry_date in (today, last_month):
bill = creator.create_cost_entry_bill(cost_entry_date)
bill_ids.append(str(bill.id))
cost_entry = creator.create_cost_entry(bill, cost_entry_date)
product = creator.create_cost_entry_product('Compute Instance')
pricing = creator.create_cost_entry_pricing()
reservation = creator.create_cost_entry_reservation()
creator.create_cost_entry_line_item(bill,
cost_entry,
product,
pricing,
reservation,
resource_id=resource_id)
self.accessor.populate_line_item_daily_table(last_month, today,
bill_ids)
with OCPReportDBAccessor(self.test_schema,
self.column_map) as ocp_accessor:
cluster_id = self.ocp_provider_resource_name
with ProviderDBAccessor(provider_uuid=self.ocp_test_provider_uuid
) as provider_access:
provider_id = provider_access.get_provider().id
for cost_entry_date in (today, last_month):
period = creator.create_ocp_report_period(
cost_entry_date,
provider_id=provider_id,
cluster_id=cluster_id)
report = creator.create_ocp_report(period, cost_entry_date)
creator.create_ocp_usage_line_item(period,
report,
resource_id=resource_id)
cluster_id = get_cluster_id_from_provider(
self.ocp_test_provider_uuid)
ocp_accessor.populate_line_item_daily_table(
last_month, today, cluster_id)
def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
if isinstance(start_date, str):
start_date = parser.parse(start_date)
if isinstance(end_date, str):
end_date = parser.parse(end_date)
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
aws_bills = aws_get_bills_from_provider(aws_provider_uuid,
self._schema, start_date,
end_date)
aws_bill_ids = []
with schema_context(self._schema):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
with CostModelDBAccessor(self._schema,
aws_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating OpenShift on AWS summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, AWS Bill IDs: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
str(aws_bill_ids),
)
accessor.populate_ocp_on_aws_cost_daily_summary(
start, end, cluster_id, aws_bill_ids, markup_value)
accessor.populate_ocp_on_aws_tags_summary_table()
self.refresh_openshift_on_infrastructure_views(
OCP_ON_AWS_MATERIALIZED_VIEWS)
with OCPReportDBAccessor(self._schema) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
def update_azure_summary_tables(self, openshift_provider_uuid,
azure_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on Azure."""
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
azure_bills = azure_get_bills_from_provider(
azure_provider_uuid,
self._schema,
datetime.datetime.strptime(start_date, "%Y-%m-%d"),
datetime.datetime.strptime(end_date, "%Y-%m-%d"),
)
azure_bill_ids = []
with schema_context(self._schema):
azure_bill_ids = [str(bill.id) for bill in azure_bills]
with CostModelDBAccessor(self._schema, azure_provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on Azure
with AzureReportDBAccessor(self._schema, self._column_map) as accessor:
for start, end in date_range_pair(start_date, end_date):
LOG.info(
"Updating OpenShift on Azure summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, Azure Bill IDs: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
str(azure_bill_ids),
)
accessor.populate_ocp_on_azure_cost_daily_summary(
start, end, cluster_id, azure_bill_ids)
accessor.populate_ocp_on_azure_markup_cost(markup_value,
azure_bill_ids)
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
def __init__(self, schema_name, report_path, compression, provider_uuid):
"""Initialize base class."""
super().__init__(
schema_name=schema_name,
report_path=report_path,
compression=compression,
provider_uuid=provider_uuid,
manifest_id=None,
processed_report=ProcessedOCPReport(),
)
self._report_name = path.basename(report_path)
self._cluster_id = utils.get_cluster_id_from_provider(provider_uuid)
self._datetime_format = Config.OCP_DATETIME_STR_FORMAT
self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE
with OCPReportDBAccessor(self._schema) as report_db:
self.existing_report_periods_map = report_db.get_report_periods()
self.existing_report_map = report_db.get_reports()
self.line_item_columns = None
def update_cost_summary_table(self, start_date, end_date):
"""Populate the cost summary tables.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
cluster_id = get_cluster_id_from_provider(self._provider.uuid)
# This needs to always run regardless of whether the OpenShift
# cluster is tied to a cloud provider
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
cluster_id = get_cluster_id_from_provider(self._provider.uuid)
aws_bills = get_bills_from_provider(self._provider.uuid,
self._schema_name, start_date,
end_date)
aws_bill_ids = [str(bill.id) for bill in aws_bills]
# OpenShift on AWS
with AWSReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.commit()
if cluster_id:
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date,
end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
accessor.commit()
def update_cost_summary_table(self, start_date, end_date):
"""Populate the cost summary tables.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
cluster_id = get_cluster_id_from_provider(self._provider.uuid)
# This needs to always run regardless of whether the OpenShift
# cluster is tied to a cloud provider
infra_map = self._get_ocp_infra_map(start_date, end_date)
aws_uuid = self._get_aws_provider_uuid_from_map(
self._provider, infra_map)
with CostModelDBAccessor(self._schema_name, aws_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
aws_markup_value = float(markup.get('value', 0)) / 100
with CostModelDBAccessor(self._schema_name, self._provider.uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
ocp_markup_value = float(markup.get('value', 0)) / 100
with OCPReportDBAccessor(self._schema_name,
self._column_map) as accessor:
LOG.info(
'Updating OpenShift on OCP cost summary table for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema_name, self._provider.uuid, start_date, end_date)
accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
accessor.populate_ocp_on_aws_markup_cost(aws_markup_value,
ocp_markup_value,
cluster_id)
def update_summary_charge_info(self):
"""Update the OCP summary table with the charge information.
Args:
None
Returns
None
"""
self._cluster_id = get_cluster_id_from_provider(self._provider_uuid)
LOG.info(
'Starting charge calculation updates for provider: %s. Cluster ID: %s.',
self._provider_uuid, self._cluster_id)
self._update_pod_charge()
self._update_storage_charge()
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
'Updating OpenShift on Cloud cost summary for schema: %s and provider: %s',
self._schema, self._provider_uuid)
accessor.populate_cost_summary_table(self._cluster_id)
accessor.commit()
def test_get_cluster_id_from_non_ocp_provider(self):
"""Test that None is returned when getting cluster ID on non-OCP provider."""
cluster_id = utils.get_cluster_id_from_provider(
self.aws_test_provider_uuid)
self.assertIsNone(cluster_id)
def test_get_cluster_id_from_provider(self):
"""Test that the cluster ID is returned from OCP provider."""
cluster_id = utils.get_cluster_id_from_provider(
self.ocp_test_provider_uuid)
self.assertIsNotNone(cluster_id)
def test_populate_ocp_on_aws_cost_daily_summary(self):
"""Test that the OCP on AWS cost summary table is populated."""
summary_table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_daily_summary']
project_summary_table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_project_daily_summary']
bill_ids = []
summary_table = getattr(self.accessor.report_schema, summary_table_name)
project_table = getattr(self.accessor.report_schema, project_summary_table_name)
today = DateAccessor().today_with_timezone('UTC')
last_month = today - relativedelta.relativedelta(months=1)
resource_id = 'i-12345'
for cost_entry_date in (today, last_month):
bill = self.creator.create_cost_entry_bill(cost_entry_date)
bill_ids.append(str(bill.id))
cost_entry = self.creator.create_cost_entry(bill, cost_entry_date)
product = self.creator.create_cost_entry_product('Compute Instance')
pricing = self.creator.create_cost_entry_pricing()
reservation = self.creator.create_cost_entry_reservation()
self.creator.create_cost_entry_line_item(
bill,
cost_entry,
product,
pricing,
reservation,
resource_id=resource_id
)
self.accessor.populate_line_item_daily_table(last_month, today, bill_ids)
li_table_name = AWS_CUR_TABLE_MAP['line_item']
li_table = getattr(self.accessor.report_schema, li_table_name)
sum_aws_cost = self.accessor._session.query(
func.sum(li_table.unblended_cost)
).first()
with OCPReportDBAccessor(self.test_schema, self.column_map) as ocp_accessor:
cluster_id = self.ocp_provider_resource_name
with ProviderDBAccessor(provider_uuid=self.ocp_test_provider_uuid) as provider_access:
provider_id = provider_access.get_provider().id
for cost_entry_date in (today, last_month):
period = self.creator.create_ocp_report_period(cost_entry_date, provider_id=provider_id, cluster_id=cluster_id)
report = self.creator.create_ocp_report(period, cost_entry_date)
self.creator.create_ocp_usage_line_item(
period,
report,
resource_id=resource_id
)
cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid)
ocp_accessor.populate_line_item_daily_table(last_month, today, cluster_id)
query = self.accessor._get_db_obj_query(summary_table_name)
initial_count = query.count()
self.accessor.populate_ocp_on_aws_cost_daily_summary(last_month,
today,
cluster_id, bill_ids)
self.assertNotEqual(query.count(), initial_count)
sum_cost = self.accessor._session.query(
func.sum(summary_table.unblended_cost)
).first()
sum_project_cost = self.accessor._session.query(
func.sum(project_table.unblended_cost)
).first()
self.assertEqual(sum_cost, sum_project_cost)
self.assertLessEqual(sum_cost, sum_aws_cost)
def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
if isinstance(start_date, str):
start_date = parser.parse(start_date).date()
if isinstance(end_date, str):
end_date = parser.parse(end_date).date()
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
cluster_alias = get_cluster_alias_from_cluster_id(cluster_id)
with OCPReportDBAccessor(self._schema) as accessor:
if not accessor.get_cluster_for_provider(openshift_provider_uuid):
LOG.info(
f"No cluster information available for OCP Provider: {openshift_provider_uuid}, "
f"skipping OCP on Cloud summary table update for AWS source: {aws_provider_uuid}."
)
return
report_period = accessor.report_periods_for_provider_uuid(
openshift_provider_uuid, start_date)
if not report_period:
LOG.info(
f"No report period for AWS provider {openshift_provider_uuid} with start date {start_date}"
)
return
accessor.delete_infrastructure_raw_cost_from_daily_summary(
openshift_provider_uuid, report_period.id, start_date,
end_date)
aws_bills = aws_get_bills_from_provider(aws_provider_uuid,
self._schema, start_date,
end_date)
with schema_context(self._schema):
self._handle_partitions(
self._schema,
(
"reporting_ocpawscostlineitem_daily_summary_p",
"reporting_ocpawscostlineitem_project_daily_summary_p",
"reporting_ocpaws_compute_summary_p",
"reporting_ocpaws_cost_summary_p",
"reporting_ocpaws_cost_summary_by_account_p",
"reporting_ocpaws_cost_summary_by_region_p",
"reporting_ocpaws_cost_summary_by_service_p",
"reporting_ocpaws_storage_summary_p",
"reporting_ocpaws_database_summary_p",
"reporting_ocpaws_network_summary_p",
"reporting_ocpallcostlineitem_daily_summary_p",
"reporting_ocpallcostlineitem_project_daily_summary_p",
"reporting_ocpall_compute_summary_pt",
"reporting_ocpall_cost_summary_pt",
),
start_date,
end_date,
)
aws_bill_ids = [str(bill.id) for bill in aws_bills]
current_aws_bill_id = aws_bills.first().id if aws_bills else None
current_ocp_report_period_id = report_period.id
with CostModelDBAccessor(self._schema,
aws_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
with CostModelDBAccessor(
self._schema, openshift_provider_uuid) as cost_model_accessor:
distribution = cost_model_accessor.distribution
# OpenShift on AWS
sql_params = {
"schema_name": self._schema,
"start_date": start_date,
"end_date": end_date,
"source_uuid": aws_provider_uuid,
"cluster_id": cluster_id,
"cluster_alias": cluster_alias,
}
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date,
end_date,
step=settings.TRINO_DATE_STEP):
LOG.info(
"Updating OpenShift on AWS summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, AWS Bill ID: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
current_aws_bill_id,
)
filters = {
"report_period_id": current_ocp_report_period_id
} # Use report_period_id to leverage DB index on DELETE
accessor.delete_line_item_daily_summary_entries_for_date_range_raw(
self._provider.uuid,
start,
end,
filters,
table=OCPAWSCostLineItemProjectDailySummaryP)
accessor.populate_ocp_on_aws_cost_daily_summary_presto(
start,
end,
openshift_provider_uuid,
aws_provider_uuid,
current_ocp_report_period_id,
current_aws_bill_id,
markup_value,
distribution,
)
accessor.back_populate_ocp_on_aws_daily_summary(
start_date, end_date, current_ocp_report_period_id)
accessor.populate_ocp_on_aws_tags_summary_table(
aws_bill_ids, start_date, end_date)
accessor.populate_ocp_on_aws_ui_summary_tables(sql_params)
with OCPReportDBAccessor(self._schema) as ocp_accessor:
sql_params["source_type"] = "AWS"
LOG.info(
f"Processing OCP-ALL for AWS (T) (s={start_date} e={end_date})"
)
ocp_accessor.populate_ocp_on_all_project_daily_summary(
"aws", sql_params)
ocp_accessor.populate_ocp_on_all_daily_summary(
"aws", sql_params)
ocp_accessor.populate_ocp_on_all_ui_summary_tables(sql_params)
ocp_accessor.populate_ui_summary_tables(
start, end, openshift_provider_uuid,
UI_SUMMARY_TABLES_MARKUP_SUBSET)
LOG.info(
"Updating ocp_on_cloud_updated_datetime OpenShift report periods")
with schema_context(self._schema):
report_period.ocp_on_cloud_updated_datetime = self._date_accessor.today_with_timezone(
"UTC")
report_period.save()
