def _update_markup_cost(self, start_date, end_date):
"""Populate markup costs for OpenShift.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
infra_markup_value = Decimal(0.0)
infra_map = self.get_infra_map()
infra_tuple = infra_map.get(self._provider_uuid)
cluster_id = get_cluster_id_from_provider(self._provider_uuid)
if infra_tuple:
infra_uuid = infra_tuple[0]
with CostModelDBAccessor(self._schema, infra_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
infra_markup_value = Decimal(markup.get('value', 0)) / 100
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
ocp_markup_value = Decimal(markup.get('value', 0)) / 100
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info('Updating OpenShift markup for'
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s',
self._schema, self._provider_uuid, start_date, end_date)
accessor.populate_markup_cost(infra_markup_value, ocp_markup_value, cluster_id)
LOG.info('Finished updating markup.')
def test_get_rates_no_cost_model(self):
"""Test that get_rates returns empty dict when cost model does not exist."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
cpu_usage_rate = cost_model_accessor.get_rates(
"cpu_core_usage_per_hour")
self.assertFalse(cpu_usage_rate)
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
with CostModelDBAccessor(self._schema,
self._provider.uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = float(markup.get("value", 0)) / 100
with GCPReportDBAccessor(self._schema) as accessor:
# Need these bills on the session to update dates after processing
with schema_context(self._schema):
bills = accessor.bills_for_provider_uuid(
self._provider.uuid, start_date)
bill_ids = [str(bill.id) for bill in bills]
current_bill_id = bills.first().id if bills else None
if current_bill_id is None:
msg = f"No bill was found for {start_date}. Skipping summarization"
LOG.info(msg)
return start_date, end_date
for start, end in date_range_pair(start_date,
end_date,
step=settings.TRINO_DATE_STEP):
LOG.info(
"Updating GCP report summary tables from parquet: \n\tSchema: %s"
"\n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
start,
end,
)
accessor.delete_line_item_daily_summary_entries_for_date_range(
self._provider.uuid, start, end)
accessor.populate_line_item_daily_summary_table_presto(
start, end, self._provider.uuid, current_bill_id,
markup_value)
accessor.populate_enabled_tag_keys(start, end, bill_ids)
accessor.populate_tags_summary_table(bill_ids)
accessor.update_line_item_daily_summary_with_enabled_tags(
start_date, end_date, bill_ids)
for bill in bills:
if bill.summary_data_creation_datetime is None:
bill.summary_data_creation_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.summary_data_updated_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.save()
return start_date, end_date
def test_get_rates(self):
"""Test get rates."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
cpu_usage_rate = cost_model_accessor.get_rates(
"cpu_core_usage_per_hour")
self.assertEqual(type(cpu_usage_rate), dict)
mem_usage_rate = cost_model_accessor.get_rates(
"memory_gb_usage_per_hour")
self.assertEqual(type(mem_usage_rate), dict)
cpu_request_rate = cost_model_accessor.get_rates(
"cpu_core_request_per_hour")
self.assertEqual(type(cpu_request_rate), dict)
mem_request_rate = cost_model_accessor.get_rates(
"memory_gb_request_per_hour")
self.assertEqual(type(mem_request_rate), dict)
storage_usage_rate = cost_model_accessor.get_rates(
"storage_gb_usage_per_month")
self.assertEqual(type(storage_usage_rate), dict)
storage_request_rate = cost_model_accessor.get_rates(
"storage_gb_request_per_month")
self.assertEqual(type(storage_request_rate), dict)
storage_request_rate = cost_model_accessor.get_rates(
"node_cost_per_month")
self.assertEqual(type(storage_request_rate), dict)
missing_rate = cost_model_accessor.get_rates("wrong_metric")
self.assertIsNone(missing_rate)
def test_populate_ocp_on_gcp_cost_daily_summary_presto(
self, mock_presto, mock_delete):
"""Test that we construst our SQL and query using Presto."""
dh = DateHelper()
start_date = dh.this_month_start.date()
end_date = dh.this_month_end.date()
bills = self.accessor.get_cost_entry_bills_query_by_provider(
self.gcp_provider.uuid)
with schema_context(self.schema):
current_bill_id = bills.first().id if bills else None
with CostModelDBAccessor(
self.schema, self.gcp_provider.uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = float(markup.get("value", 0)) / 100
distribution = cost_model_accessor.distribution
self.accessor.populate_ocp_on_gcp_cost_daily_summary_presto(
start_date,
end_date,
self.ocp_provider_uuid,
self.ocp_cluster_id,
self.gcp_provider_uuid,
self.ocp_cluster_id,
current_bill_id,
markup_value,
distribution,
)
mock_presto.assert_called()
mock_delete.assert_called()
def test_get_node_cost_per_month(self):
"""Test get memory request rates."""
with CostModelDBAccessor(self.schema, self.provider_uuid,
self.column_map) as cost_model_accessor:
node_cost = cost_model_accessor.get_node_per_month_rates()
self.assertEqual(type(node_cost), dict)
self.assertEqual(node_cost.get('tiered_rates')[0].get('value'), 7.5)
def test_get_markup_no_cost_model(self):
"""Test that get_markup returns empty dict when cost model does not exist."""
with CostModelDBAccessor(
self.schema, self.provider_uuid, self.column_map
) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
self.assertFalse(markup)
def test_update_daily_summary_tables(self, mock_presto, mock_tag_update,
mock_delete):
"""Test that we run Presto summary."""
start_str = self.dh.this_month_start.isoformat()
end_str = self.dh.this_month_end.isoformat()
start, end = self.updater._get_sql_inputs(start_str, end_str)
for s, e in date_range_pair(start, end, step=settings.TRINO_DATE_STEP):
expected_start, expected_end = s, e
with AWSReportDBAccessor(self.schema) as accessor:
with schema_context(self.schema):
bills = accessor.bills_for_provider_uuid(
self.aws_provider.uuid, start)
bill_ids = [str(bill.id) for bill in bills]
current_bill_id = bills.first().id if bills else None
with CostModelDBAccessor(
self.schema, self.aws_provider.uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = float(markup.get("value", 0)) / 100
start_return, end_return = self.updater.update_summary_tables(
start, end)
mock_delete.assert_called_with(self.aws_provider.uuid, expected_start,
expected_end)
mock_presto.assert_called_with(expected_start, expected_end,
self.aws_provider.uuid, current_bill_id,
markup_value)
mock_tag_update.assert_called_with(bill_ids, start, end)
self.assertEqual(start_return, start)
self.assertEqual(end_return, end)
def _update_markup_cost(self, start_date, end_date):
"""Populate markup costs for OpenShift.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
None
"""
with CostModelDBAccessor(self._schema, self._provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup = Decimal(markup.get("value", 0)) / 100
with OCPReportDBAccessor(self._schema) as accessor:
LOG.info(
"Updating markup for" "\n\tSchema: %s \n\t%s Provider: %s (%s) \n\tDates: %s - %s",
self._schema,
self._provider.type,
self._provider.name,
self._provider_uuid,
start_date,
end_date,
)
accessor.populate_markup_cost(markup, start_date, end_date, self._cluster_id)
LOG.info("Finished updating markup.")
def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
aws_bills = aws_get_bills_from_provider(
aws_provider_uuid, self._schema,
datetime.datetime.strptime(start_date, '%Y-%m-%d'),
datetime.datetime.strptime(end_date, '%Y-%m-%d'))
aws_bill_ids = []
with schema_context(self._schema):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
with CostModelDBAccessor(self._schema, aws_provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = Decimal(markup.get('value', 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
'Updating OpenShift on AWS summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s'
'\n\tCluster ID: %s, AWS Bill IDs: %s', self._schema,
self._provider.uuid, start_date, end_date, cluster_id,
str(aws_bill_ids))
accessor.populate_ocp_on_aws_cost_daily_summary(
start_date, end_date, cluster_id, aws_bill_ids)
accessor.populate_ocp_on_aws_markup_cost(markup_value,
aws_bill_ids)
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
def test_get_rates(self):
"""Test get rates."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
cpu_usage_rate = cost_model_accessor.get_rates(
"cpu_core_usage_per_hour")
self.assertIsNone(cpu_usage_rate)
def _update_storage_charge(self):
"""Calculate and store the storage charges."""
try:
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor:
storage_usage_rates = cost_model_accessor.get_storage_gb_usage_per_month_rates(
)
storage_request_rates = cost_model_accessor.get_storage_gb_request_per_month_rates(
)
with OCPReportDBAccessor(self._schema,
self._column_map) as report_accessor:
storage_usage = report_accessor.\
get_persistentvolumeclaim_usage_gigabyte_months(self._cluster_id)
storage_usage_charge = self._calculate_charge(
storage_usage_rates, storage_usage)
storage_request = report_accessor.\
get_volume_request_storage_gigabyte_months(self._cluster_id)
storage_request_charge = self._calculate_charge(
storage_request_rates, storage_request)
total_storage_charge = self._aggregate_charges(
storage_usage_charge, storage_request_charge)
temp_table = self._write_to_temp_table(report_accessor,
total_storage_charge)
report_accessor.populate_storage_charge(temp_table)
except OCPReportChargeUpdaterError as error:
LOG.error('Unable to calculate storage usage charge. Error: %s',
str(error))
def test_get_cpu_core_usage_per_hour_rates(self):
"""Test get cpu usage rates."""
with CostModelDBAccessor(self.schema, self.provider_uuid,
self.column_map) as cost_model_accessor:
cpu_rates = cost_model_accessor.get_cpu_core_usage_per_hour_rates()
self.assertEqual(type(cpu_rates), dict)
self.assertEqual(
cpu_rates.get('tiered_rates')[0].get('value'), 1.5)
def test_get_storage_gb_request_per_month_rates(self):
"""Test get memory request rates."""
with CostModelDBAccessor(
self.schema, self.provider_uuid, self.column_map
) as cost_model_accessor:
storage_rates = cost_model_accessor.get_storage_gb_request_per_month_rates()
self.assertEqual(type(storage_rates), dict)
self.assertEqual(storage_rates.get('tiered_rates')[0].get('value'), 6.5)
def test_get_memory_gb_usage_per_hour_rates(self):
"""Test get memory usage rates."""
with CostModelDBAccessor(
self.schema, self.provider_uuid, self.column_map
) as cost_model_accessor:
mem_rates = cost_model_accessor.get_memory_gb_usage_per_hour_rates()
self.assertEqual(type(mem_rates), dict)
self.assertEqual(mem_rates.get('tiered_rates')[0].get('value'), 2.5)
def test_markup(self):
"""Test to make sure markup dictionary is returned."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
self.assertEqual(markup, self.markup)
markup = cost_model_accessor.markup
self.assertEqual(markup, self.markup)
def test_get_storage_gb_usage_per_month_rates(self):
"""Test get memory request rates."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
storage_rates = cost_model_accessor.get_storage_gb_usage_per_month_rates(
)
self.assertEqual(type(storage_rates), dict)
self.assertEqual(
storage_rates.get("tiered_rates")[0].get("value"), 5.5)
def test_get_cpu_core_request_per_hour_rates(self):
"""Test get cpu request rates."""
with CostModelDBAccessor(self.schema, self.provider_uuid,
self.column_map) as cost_model_accessor:
cpu_rates = cost_model_accessor.get_cpu_core_request_per_hour_rates(
)
self.assertEqual(type(cpu_rates), dict)
self.assertEqual(
cpu_rates.get("tiered_rates")[0].get("value"), 3.5)
def update_aws_summary_tables(self, openshift_provider_uuid,
aws_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on AWS."""
if isinstance(start_date, str):
start_date = parser.parse(start_date).date()
if isinstance(end_date, str):
end_date = parser.parse(end_date).date()
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
with OCPReportDBAccessor(self._schema) as accessor:
report_period = accessor.report_periods_for_provider_uuid(
openshift_provider_uuid, start_date)
accessor.delete_infrastructure_raw_cost_from_daily_summary(
openshift_provider_uuid, report_period.id, start_date,
end_date)
aws_bills = aws_get_bills_from_provider(aws_provider_uuid,
self._schema, start_date,
end_date)
with schema_context(self._schema):
aws_bill_ids = [str(bill.id) for bill in aws_bills]
current_aws_bill_id = aws_bills.first().id if aws_bills else None
current_ocp_report_period_id = report_period.id
with CostModelDBAccessor(self._schema,
aws_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on AWS
with AWSReportDBAccessor(self._schema) as accessor:
for start, end in date_range_pair(start_date,
end_date,
step=settings.TRINO_DATE_STEP):
LOG.info(
"Updating OpenShift on AWS summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, AWS Bill ID: %s",
self._schema,
self._provider.uuid,
start,
end,
cluster_id,
current_aws_bill_id,
)
accessor.populate_ocp_on_aws_cost_daily_summary_presto(
start,
end,
openshift_provider_uuid,
aws_provider_uuid,
current_ocp_report_period_id,
current_aws_bill_id,
markup_value,
)
accessor.back_populate_ocp_on_aws_daily_summary(
start_date, end_date, current_ocp_report_period_id)
accessor.populate_ocp_on_aws_tags_summary_table(
aws_bill_ids, start_date, end_date)
def test_get_memory_gb_request_per_hour_rates(self):
"""Test get memory request rates."""
with CostModelDBAccessor(self.schema, self.provider_uuid,
self.column_map) as cost_model_accessor:
mem_rates = cost_model_accessor.get_memory_gb_request_per_hour_rates(
)
self.assertEqual(type(mem_rates), dict)
self.assertEqual(
mem_rates.get("tiered_rates")[0].get("value"), 4.5)
def test_supplementary_rates(self):
"""Test supplementary rates property."""
cost_type = "Supplementary"
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
result_sup_rates = cost_model_accessor.supplementary_rates
for metric_name in result_sup_rates.keys():
expected_value = self.expected[cost_type][metric_name]
self.assertEqual(result_sup_rates[metric_name], expected_value)
def test_infrastructure_rates(self):
"""Test infrastructure rates property."""
cost_type = "Infrastructure"
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
result_infra_rates = cost_model_accessor.infrastructure_rates
for metric_name in result_infra_rates.keys():
expected_value = self.expected[cost_type][metric_name]
self.assertEqual(result_infra_rates[metric_name],
expected_value)
def test_get_cost_model(self):
"""Test to make sure cost_model is gotten."""
with schema_context(self.schema):
model = CostModel.objects.filter(
costmodelmap__provider_uuid=self.provider_uuid).first()
uuid = model.uuid
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
self.assertEqual(cost_model_accessor.cost_model, model)
self.assertEqual(cost_model_accessor.cost_model.uuid, uuid)
def test_price_list_existing_metric_different_key(self):
"""
Tests that the proper keys and values are added for the rates if
different keys are used for the same metric
"""
expected = {"app": {"smoke": 123}, "web": {"smoker": 456}}
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
result_infra_rates = cost_model_accessor.tag_infrastructure_rates.get(
"node_cost_per_month")
self.assertEqual(result_infra_rates, expected)
def test_make_rate_by_metric_map(self):
"""Test to make sure a dictionary of metric to rates is returned."""
expected_map = {}
for rate in self.rates:
expected_map[rate.get('metric', {}).get('name')] = rate
with CostModelDBAccessor(self.schema, self.provider_uuid,
self.column_map) as cost_model_accessor:
result_rate_map = cost_model_accessor._make_rate_by_metric_map()
for metric, rate in result_rate_map.items():
self.assertIn(rate, self.rates)
self.assertIn(rate, expected_map.values())
self.assertIn(metric, expected_map)
def test_get_storage_gb_request_per_month_rates(self):
"""Test get memory request rates."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
storage_rates = cost_model_accessor.get_storage_gb_request_per_month_rates(
)
self.assertEqual(type(storage_rates), dict)
for cost_type in ["Infrastructure", "Supplementary"]:
value_result = storage_rates.get("tiered_rates", {}).get(
cost_type, {})[0].get("value", 0)
expected_value = self.expected_value_rate_mapping[
"storage_gb_request_per_month"][cost_type]
self.assertEqual(value_result, expected_value)
def test_get_node_cost_per_month(self):
"""Test get memory request rates."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
node_cost = cost_model_accessor.get_node_per_month_rates()
self.assertEqual(type(node_cost), dict)
for cost_type in ["Infrastructure", "Supplementary"]:
value_result = node_cost.get("tiered_rates",
{}).get(cost_type,
{})[0].get("value", 0)
expected_value = self.expected[cost_type][
"node_cost_per_month"]
self.assertEqual(value_result, expected_value)
def test_get_cpu_core_usage_per_hour_rates(self):
"""Test get cpu usage rates."""
with CostModelDBAccessor(self.schema,
self.provider_uuid) as cost_model_accessor:
cpu_rates = cost_model_accessor.get_cpu_core_usage_per_hour_rates()
self.assertEqual(type(cpu_rates), dict)
for cost_type in ["Infrastructure", "Supplementary"]:
value_result = cpu_rates.get("tiered_rates",
{}).get(cost_type,
{})[0].get("value", 0)
expected_value = self.expected[cost_type][
"cpu_core_usage_per_hour"]
self.assertEqual(value_result, expected_value)
def _update_markup_cost(self):
"""Store markup costs."""
try:
with CostModelDBAccessor(self._schema, self._provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = float(markup.get('value', 0)) / 100
with OCPReportDBAccessor(self._schema,
self._column_map) as report_accessor:
report_accessor.populate_markup_cost(markup_value,
self._cluster_id)
except OCPReportChargeUpdaterError as error:
LOG.error('Unable to update markup costs. Error: %s', str(error))
def update_azure_summary_tables(self, openshift_provider_uuid,
azure_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on Azure."""
if isinstance(start_date, str):
start_date = parser.parse(start_date).date()
if isinstance(end_date, str):
end_date = parser.parse(end_date).date()
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
azure_bills = azure_get_bills_from_provider(azure_provider_uuid,
self._schema, start_date,
end_date)
with schema_context(self._schema):
current_azure_bill_id = azure_bills.first(
).id if azure_bills else None
with CostModelDBAccessor(self._schema,
azure_provider_uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = Decimal(markup.get("value", 0)) / 100
# OpenShift on Azure
with AzureReportDBAccessor(self._schema) as accessor:
LOG.info(
"Updating OpenShift on Azure summary table for "
"\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s"
"\n\tCluster ID: %s, Azure Bill ID: %s",
self._schema,
self._provider.uuid,
start_date,
end_date,
cluster_id,
current_azure_bill_id,
)
accessor.populate_ocp_on_azure_cost_daily_summary_presto(
start_date,
end_date,
openshift_provider_uuid,
azure_provider_uuid,
cluster_id,
current_azure_bill_id,
markup_value,
)
accessor.populate_ocp_on_azure_tags_summary_table()
with OCPReportDBAccessor(self._schema) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
