def test_update_summary_tables_azure(self, mock_markup, mock_refresh):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
mock_markup.return_value = markup
with ProviderDBAccessor(self.ocp_provider_uuid) as accessor:
accessor.set_infrastructure(self.azure_provider_uuid,
Provider.PROVIDER_AZURE)
self._generate_ocp_on_azure_data()
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
with ProviderDBAccessor(
self.azure_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema="acct10001",
provider=provider,
manifest=None)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
initial_count = query.count()
updater.update_summary_tables(start_date_str, end_date_str)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
query = azure_accessor._get_db_obj_query(summary_table_name)
self.assertNotEqual(query.count(), initial_count)
markup_cost = query.aggregate(
Sum("markup_cost"))["markup_cost__sum"]
pretax_cost = query.aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
self.assertAlmostEqual(markup_cost,
pretax_cost *
decimal.Decimal(markup.get("value") / 100),
places=9)
daily_summary_table_name = OCP_REPORT_TABLE_MAP[
"line_item_daily_summary"]
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(daily_summary_table_name)
infra_cost = query.aggregate(Sum("infra_cost"))["infra_cost__sum"]
project_infra_cost = query.aggregate(
Sum("project_infra_cost"))["project_infra_cost__sum"]
self.assertIsNotNone(infra_cost)
self.assertIsNotNone(project_infra_cost)
self.assertNotEqual(infra_cost, decimal.Decimal(0))
self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
mock_refresh.assert_called()
def test_update_project_markup_cost(self, mock_cost_model):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
mock_cost_model.return_value.__enter__.return_value.markup = markup
markup_dec = decimal.Decimal(markup.get("value") / 100)
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
updater = OCPCloudReportSummaryUpdater(
schema=self.schema,
provider=self.ocp_on_aws_ocp_provider,
manifest=None)
updater.update_summary_tables(start_date, end_date,
self.ocp_on_aws_ocp_provider.uuid,
self.aws_provider.uuid,
Provider.PROVIDER_AWS)
summary_table_name = AWS_CUR_TABLE_MAP[
"ocp_on_aws_project_daily_summary"]
with AWSReportDBAccessor(self.schema) as aws_accessor:
query = (aws_accessor._get_db_obj_query(summary_table_name).filter(
cost_entry_bill__billing_period_start=start_date,
data_source="Pod").all())
for item in query:
self.assertAlmostEqual(item.project_markup_cost,
item.pod_cost * markup_dec)
def _set_updater(self):
"""
Create the report summary updater object.
Object is specific to the report provider.
Args:
None
Returns:
(Object) : Provider-specific report summary updater
"""
if self._provider.type in (AMAZON_WEB_SERVICES,
AWS_LOCAL_SERVICE_PROVIDER):
return (AWSReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (AZURE, AZURE_LOCAL_SERVICE_PROVIDER):
return (AzureReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (OPENSHIFT_CONTAINER_PLATFORM, ):
return (OCPReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
return None
def test_update_summary_tables_with_aws_provider(self,
mock_cluster_id_utility,
mock_utility, mock_ocp,
mock_ocp_on_aws):
"""Test that summary tables are properly run for an OCP provider."""
fake_cluster_id = 'my-ocp-cluster'
mock_cluster_id_utility.return_value = fake_cluster_id
fake_bills = [Mock(), Mock()]
fake_bills[0].id = 1
fake_bills[1].id = 2
bill_ids = [str(bill.id) for bill in fake_bills]
mock_utility.return_value = fake_bills
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
mock_ocp_on_aws.assert_called_with(start_date_str, end_date_str,
fake_cluster_id, bill_ids)
def test_update_markup_cost(self, mock_cost_model):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
markup_dec = decimal.Decimal(markup.get("value") / 100)
mock_cost_model.markup = markup
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
manifest = CostUsageReportManifest.objects.filter(
provider=self.ocp_on_aws_ocp_provider,
billing_period_start_datetime=start_date).first()
updater = OCPCloudReportSummaryUpdater(
schema=self.schema,
provider=self.ocp_on_aws_ocp_provider,
manifest=manifest)
updater.update_summary_tables(start_date, end_date)
summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]
with AWSReportDBAccessor(self.schema) as aws_accessor:
query = (aws_accessor._get_db_obj_query(summary_table_name).filter(
cost_entry_bill__billing_period_start=start_date).all())
for item in query:
self.assertAlmostEqual(item.markup_cost,
item.unblended_cost * markup_dec)
def test_update_summary_tables_with_aws_provider(
self, mock_utility, mock_ocpall_proj_summ, mock_ocpall_summ,
mock_ocpall_persp, mock_ocp_on_aws, mock_map):
"""Test that summary tables are properly run for an OCP provider."""
fake_bills = [Mock(), Mock()]
fake_bills[0].id = 1
fake_bills[1].id = 2
bill_ids = [str(bill.id) for bill in fake_bills]
mock_utility.return_value = fake_bills
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
with ProviderDBAccessor(
self.ocp_on_aws_ocp_provider.uuid) as provider_accessor:
credentials = provider_accessor.get_credentials()
cluster_id = credentials.get("cluster_id")
mock_map.return_value = {
self.ocp_test_provider_uuid:
(self.aws_provider_uuid, Provider.PROVIDER_AWS)
}
updater = OCPCloudReportSummaryUpdater(schema="acct10001",
provider=provider,
manifest=None)
updater.update_summary_tables(start_date, end_date,
self.ocp_on_aws_ocp_provider.uuid,
self.aws_provider.uuid,
Provider.PROVIDER_AWS)
mock_ocp_on_aws.assert_called_with(start_date.date(), end_date.date(),
cluster_id, bill_ids,
decimal.Decimal(0))
def test_update_summary_tables_no_ocp_on_aws(self, mock_ocp_on_aws,
mock_ocpall_proj_summ,
mock_ocpall_summ,
mock_ocpall_persp):
"""Test that summary tables do not run when OCP-on-AWS does not exist."""
new_aws_provider = baker.make("Provider", type="AWS")
new_ocp_provider = baker.make("Provider", type="OCP")
test_provider_list = [
str(new_aws_provider.uuid),
str(new_ocp_provider.uuid)
]
for provider_uuid in test_provider_list:
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
with ProviderDBAccessor(provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema="acct10001",
provider=provider,
manifest=None)
infra_map = updater.get_infra_map(start_date_str, end_date_str)
for openshift_provider_uuid, infrastructure_tuple in infra_map.items(
):
updater.update_summary_tables(
start_date_str,
end_date_str,
openshift_provider_uuid,
infrastructure_tuple[0],
infrastructure_tuple[1],
)
mock_ocp_on_aws.assert_not_called()
def test_update_summary_tables(self):
"""Test that summary tables are updated correctly."""
self._generate_ocp_on_aws_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta.relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
summary_table_name = AWS_CUR_TABLE_MAP['ocp_on_aws_daily_summary']
query = aws_accessor._get_db_obj_query(summary_table_name)
initial_count = query.count()
updater.update_summary_tables(start_date_str, end_date_str)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
query = aws_accessor._get_db_obj_query(summary_table_name)
self.assertNotEqual(query.count(), initial_count)
def test_update_summary_tables_with_ocp_provider(self, mock_ocp,
mock_ocp_on_aws, mock_map,
mock_refresh):
"""Test that summary tables are properly run for an OCP provider."""
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
cluster_id = self.ocp_on_aws_ocp_provider.authentication.provider_resource_name
manifest = CostUsageReportManifest.objects.filter(
provider=self.ocp_on_aws_ocp_provider,
billing_period_start_datetime=self.dh.this_month_start)
mock_map.return_value = {
str(self.ocp_on_aws_ocp_provider.uuid):
(self.aws_provider_uuid, Provider.PROVIDER_AWS)
}
updater = OCPCloudReportSummaryUpdater(
schema=self.schema,
provider=self.ocp_on_aws_ocp_provider,
manifest=manifest)
updater.update_summary_tables(start_date, end_date)
with schema_context(self.schema):
bill = AWSCostEntryBill.objects.filter(
provider=self.aws_provider,
billing_period_start=self.dh.this_month_start).first()
mock_ocp_on_aws.assert_called_with(start_date.date(), end_date.date(),
cluster_id, [str(bill.id)])
mock_refresh.assert_called()
def _set_updater(self):
"""
Create the report summary updater object.
Object is specific to the report provider.
Args:
None
Returns:
(Object) : Provider-specific report summary updater
"""
if self._provider.type in (Provider.PROVIDER_AWS,
Provider.PROVIDER_AWS_LOCAL):
return (AWSReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
return (AzureReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
if self._provider.type in (Provider.PROVIDER_OCP, ):
return (OCPReportSummaryUpdater(self._schema, self._provider,
self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider,
self._manifest))
return (None, None)
def test_update_summary_tables_with_aws_provider(self, mock_utility,
mock_ocp, mock_ocp_on_aws,
mock_map, mock_refresh):
"""Test that summary tables are properly run for an OCP provider."""
fake_bills = [Mock(), Mock()]
fake_bills[0].id = 1
fake_bills[1].id = 2
bill_ids = [str(bill.id) for bill in fake_bills]
mock_utility.return_value = fake_bills
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
cluster_id = provider_accessor.get_authentication()
mock_map.return_value = {
self.ocp_test_provider_uuid:
(self.aws_provider_uuid, Provider.PROVIDER_AWS)
}
updater = OCPCloudReportSummaryUpdater(schema="acct10001",
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
mock_ocp_on_aws.assert_called_with(start_date.date(), end_date.date(),
cluster_id, bill_ids)
mock_refresh.assert_called()
def test_update_summary_tables_no_ocp_on_aws(self, mock_ocp,
mock_ocp_on_aws,
mock_refresh):
"""Test that summary tables do not run when OCP-on-AWS does not exist."""
new_aws_provider = baker.make("Provider", type="AWS")
new_ocp_provider = baker.make("Provider", type="OCP")
test_provider_list = [
str(new_aws_provider.uuid),
str(new_ocp_provider.uuid)
]
for provider_uuid in test_provider_list:
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
with ProviderDBAccessor(provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema="acct10001",
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
mock_ocp_on_aws.assert_not_called()
mock_refresh.assert_not_called()
def test_refresh_openshift_on_infrastructure_views(self):
"""Test that the combined OpenShift views are refreshed."""
self._generate_ocp_on_aws_data(cluster_id="my-ocp-cluster-1")
self._generate_ocp_on_azure_data(cluster_id="my-ocp-cluster-2")
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
with ProviderDBAccessor(
self.azure_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_count = query.count()
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_project_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_project_count = query.count()
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_count = query.count()
summary_table_name = AWS_CUR_TABLE_MAP[
"ocp_on_aws_project_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_project_count = query.count()
updater.refresh_openshift_on_infrastructure_views()
with schema_context(self.schema):
all_count = OCPAllCostLineItemDailySummary.objects.count()
all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count(
)
self.assertEqual(all_count, azure_count + aws_count)
self.assertEqual(all_project_count,
azure_project_count + aws_project_count)
def test_refresh_openshift_on_infrastructure_views(self,
mock_view_refresh):
"""Test that the combined OpenShift views are refreshed."""
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=self.azure_provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
mock_view_refresh.assert_called_with(
OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
mock_view_refresh.reset_mock()
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=self.aws_provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
expected_calls = [
call(OCP_ON_AWS_MATERIALIZED_VIEWS),
call(OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
]
mock_view_refresh.assert_has_calls(expected_calls)
with AzureReportDBAccessor(self.schema) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_count = query.count()
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_project_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_project_count = query.count()
with AWSReportDBAccessor(self.schema) as aws_accessor:
summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_count = query.count()
summary_table_name = AWS_CUR_TABLE_MAP[
"ocp_on_aws_project_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_project_count = query.count()
updater.refresh_openshift_on_infrastructure_views(
OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
with schema_context(self.schema):
all_count = OCPAllCostLineItemDailySummary.objects.count()
all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count(
)
self.assertEqual(all_count, azure_count + aws_count)
self.assertEqual(all_project_count,
azure_project_count + aws_project_count)
def test_update_summary_tables_azure(
self,
mock_cost_model # , mock_ocpall_proj_summ, mock_ocpall_summ, mock_ocpall_persp
):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
mock_cost_model.markup = markup
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=self.azure_provider,
manifest=None)
updater.update_summary_tables(
start_date,
end_date,
self.ocp_on_azure_ocp_provider.uuid,
self.azure_provider.uuid,
Provider.PROVIDER_AZURE,
)
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
with AzureReportDBAccessor(self.schema) as azure_accessor:
query = azure_accessor._get_db_obj_query(
summary_table_name).filter(
cost_entry_bill__billing_period_start=start_date)
markup_cost = query.aggregate(
Sum("markup_cost"))["markup_cost__sum"]
pretax_cost = query.aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
self.assertAlmostEqual(markup_cost,
pretax_cost *
decimal.Decimal(markup.get("value") / 100),
places=5)
daily_summary_table_name = OCP_REPORT_TABLE_MAP[
"line_item_daily_summary"]
with OCPReportDBAccessor(self.schema) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(
daily_summary_table_name).filter(
report_period__provider=self.ocp_on_azure_ocp_provider,
report_period__report_period_start=self.dh.
this_month_start,
)
infra_cost = query.aggregate(
Sum("infrastructure_raw_cost"))["infrastructure_raw_cost__sum"]
project_infra_cost = query.aggregate(
Sum("infrastructure_project_raw_cost")
)["infrastructure_project_raw_cost__sum"]
self.assertIsNotNone(infra_cost)
self.assertIsNotNone(project_infra_cost)
self.assertNotEqual(infra_cost, decimal.Decimal(0))
self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def test_update_summary_tables_azure(self, mock_markup):
"""Test that summary tables are updated correctly."""
markup = {'value': 10, 'unit': 'percent'}
mock_markup.return_value = markup
with ProviderDBAccessor(self.ocp_provider_uuid) as accessor:
accessor.set_infrastructure(self.azure_provider_uuid, 'AZURE')
self._generate_ocp_on_azure_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.azure_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
'ocp_on_azure_daily_summary']
query = azure_accessor._get_db_obj_query(summary_table_name)
initial_count = query.count()
updater.update_summary_tables(start_date_str, end_date_str)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
query = azure_accessor._get_db_obj_query(summary_table_name)
self.assertNotEqual(query.count(), initial_count)
markup_cost = query.aggregate(
Sum('markup_cost'))['markup_cost__sum']
pretax_cost = query.aggregate(
Sum('pretax_cost'))['pretax_cost__sum']
self.assertAlmostEqual(markup_cost,
pretax_cost *
decimal.Decimal(markup.get('value') / 100),
places=9)
daily_summary_table_name = OCP_REPORT_TABLE_MAP[
'line_item_daily_summary']
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(daily_summary_table_name)
infra_cost = query.aggregate(Sum('infra_cost'))['infra_cost__sum']
project_infra_cost = query.aggregate(
Sum('project_infra_cost'))['project_infra_cost__sum']
self.assertIsNotNone(infra_cost)
self.assertIsNotNone(project_infra_cost)
self.assertNotEqual(infra_cost, decimal.Decimal(0))
self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def test_update_summary_tables_with_ocp_provider(self, mock_ocp, mock_ocp_on_aws, mock_map):
"""Test that summary tables are properly run for an OCP provider."""
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
cluster_id = provider_accessor.get_authentication()
mock_map.return_value = {self.ocp_test_provider_uuid: (self.aws_provider_uuid, 'AWS')}
updater = OCPCloudReportSummaryUpdater(schema='acct10001', provider=provider, manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
mock_ocp_on_aws.assert_called_with(start_date_str, end_date_str, cluster_id, [])
def _set_updater(self):
"""
Create the report summary updater object.
Object is specific to the report provider.
Args:
None
Returns:
(Object) : Provider-specific report summary updater
"""
if self._provider.type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL):
report_summary_updater = (
AWSReportParquetSummaryUpdater if settings.ENABLE_PARQUET_PROCESSING else AWSReportSummaryUpdater
)
elif self._provider.type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL):
report_summary_updater = (
AzureReportParquetSummaryUpdater if settings.ENABLE_PARQUET_PROCESSING else AzureReportSummaryUpdater
)
elif self._provider.type in (Provider.PROVIDER_OCP,):
report_summary_updater = (
OCPReportParquetSummaryUpdater if settings.ENABLE_PARQUET_PROCESSING else OCPReportSummaryUpdater
)
elif self._provider.type in (Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL):
report_summary_updater = GCPReportSummaryUpdater
else:
return (None, None)
return (
report_summary_updater(self._schema, self._provider, self._manifest),
OCPCloudReportSummaryUpdater(self._schema, self._provider, self._manifest),
)
def test_update_cost_summary_tables(self, mock_ocp):
"""Test that cost_summary tables are updated correctly"""
self._generate_ocp_on_aws_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta.relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
updater.update_cost_summary_table(start_date_str, end_date_str)
mock_ocp.assert_called()
def test_update_project_markup_cost(self, mock_markup):
"""Test that summary tables are updated correctly."""
markup = {'value': 10, 'unit': 'percent'}
mock_markup.return_value = markup
self._generate_ocp_on_aws_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta.relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
table_name = AWS_CUR_TABLE_MAP['line_item']
tag_query = aws_accessor._get_db_obj_query(table_name)
summary_table_name = AWS_CUR_TABLE_MAP[
'ocp_on_aws_project_daily_summary']
query = aws_accessor._get_db_obj_query(summary_table_name)
possible_values = {}
with schema_context(self.schema):
for item in tag_query:
possible_values.update({
item.cost_entry_bill_id:
(item.unblended_cost * decimal.Decimal(0.1))
})
updater.update_summary_tables(start_date_str, end_date_str)
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
query = aws_accessor._get_db_obj_query(summary_table_name)
found_values = {}
for item in query:
found_values.update(
{item.cost_entry_bill_id: item.project_markup_cost})
for k, v in found_values.items():
self.assertAlmostEqual(v, possible_values[k], places=6)
def test_update_summary_tables(self):
"""Test that summary tables are updated correctly."""
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
updater = OCPCloudReportSummaryUpdater(
schema=self.schema, provider=self.ocp_on_aws_ocp_provider, manifest=None
)
with AWSReportDBAccessor(self.schema) as aws_accessor:
summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
query.delete()
initial_count = query.count()
updater.update_summary_tables(start_date, end_date)
with AWSReportDBAccessor(self.schema) as aws_accessor:
query = aws_accessor._get_db_obj_query(summary_table_name)
self.assertNotEqual(query.count(), initial_count)
def test_update_summary_tables_no_ocp_on_aws(self, mock_ocp, mock_ocp_on_aws):
"""Test that summary tables do not run when OCP-on-AWS does not exist."""
test_provider_list = [self.aws_provider_uuid, self.ocp_test_provider_uuid]
for provider_uuid in test_provider_list:
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(
schema='acct10001', provider=provider, manifest=None
)
updater.update_summary_tables(start_date_str, end_date_str)
mock_ocp_on_aws.assert_not_called()
def test_partition_handler_str_table(self):
new_table_sql = f"""
create table {self.schema}._eek_pt0 (usage_start date not null, id int) partition by range (usage_start);
"""
with schema_context(self.schema):
with connection.cursor() as cur:
cur.execute(new_table_sql)
partable = get_model("PartitionedTable")
default_part = partable(
schema_name=self.schema,
table_name="_eek_pt0_default",
partition_of_table_name="_eek_pt0",
partition_type=partable.RANGE,
partition_col="usage_start",
partition_parameters={"default": True},
active=True,
)
default_part.save()
ocrsu = OCPCloudReportSummaryUpdater(self.schema,
self.ocp_on_aws_ocp_provider,
None)
num_eek = partable.objects.filter(
schema_name=self.schema,
partition_of_table_name="_eek_pt0").count()
self.assertEqual(num_eek, 1)
ocrsu._handle_partitions(self.schema, "_eek_pt0",
datetime.date(1970, 10, 1),
datetime.date(1970, 12, 1))
eek_p = partable.objects.filter(
schema_name=self.schema,
partition_of_table_name="_eek_pt0",
partition_parameters__default=False).all()
self.assertEqual(len(eek_p), 3)
eek_p.delete()
default_part.delete()
with connection.cursor() as cur:
cur.execute(f"drop table {self.schema}._eek_pt0 ;")
def test_update_summary_tables_with_ocp_provider(self, mock_utility,
mock_ocp,
mock_ocp_on_aws):
"""Test that summary tables are properly run for an OCP provider."""
fake_cluster = 'my-ocp-cluster'
mock_utility.return_value = fake_cluster
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
mock_ocp_on_aws.assert_called_with(start_date_str, end_date_str,
fake_cluster, [])
mock_ocp.assert_called_with(fake_cluster, start_date_str, end_date_str)
def test_get_infra_map(self):
"""Test that an infrastructure map is returned."""
infrastructure_type = 'AWS'
with ProviderDBAccessor(self.ocp_provider_uuid) as accessor:
accessor.set_infrastructure(self.aws_provider_uuid, infrastructure_type)
ocp_provider = accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(
schema=self.schema, provider=ocp_provider, manifest=None
)
expected_mapping = (self.aws_provider_uuid, 'AWS')
infra_map = updater.get_infra_map()
self.assertEqual(len(infra_map.keys()), 1)
self.assertIn(self.ocp_provider_uuid, infra_map)
self.assertEqual(infra_map.get(self.ocp_provider_uuid), expected_mapping)
with ProviderDBAccessor(self.aws_provider_uuid) as accessor:
aws_provider = accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(
schema=self.schema, provider=aws_provider, manifest=None
)
infra_map = updater.get_infra_map()
self.assertEqual(len(infra_map.keys()), 1)
self.assertIn(self.ocp_provider_uuid, infra_map)
self.assertEqual(infra_map.get(self.ocp_provider_uuid), expected_mapping)
def test_get_infra_map(self):
"""Test that an infrastructure map is returned."""
updater = OCPCloudReportSummaryUpdater(
schema=self.schema,
provider=self.ocp_on_aws_ocp_provider,
manifest=None)
expected_mapping = (self.aws_provider_uuid,
Provider.PROVIDER_AWS_LOCAL)
infra_map = updater.get_infra_map()
self.assertEqual(len(infra_map.keys()), 1)
self.assertIn(str(self.ocp_on_aws_ocp_provider.uuid), infra_map)
self.assertEqual(infra_map.get(str(self.ocp_on_aws_ocp_provider.uuid)),
expected_mapping)
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=self.aws_provider,
manifest=None)
infra_map = updater.get_infra_map()
self.assertEqual(len(infra_map.keys()), 1)
self.assertIn(str(self.ocp_on_aws_ocp_provider.uuid), infra_map)
self.assertEqual(infra_map.get(str(self.ocp_on_aws_ocp_provider.uuid)),
expected_mapping)
def test_get_infra_db_key_for_provider_type(self):
"""Test db_key private method for OCP-on-AWS infrastructure map."""
with ProviderDBAccessor(self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(
schema='acct10001',
provider=provider,
manifest=None
)
self.assertEqual(updater._get_infra_db_key_for_provider_type('AWS'), 'aws_uuid')
self.assertEqual(updater._get_infra_db_key_for_provider_type('AWS-local'), 'aws_uuid')
self.assertEqual(updater._get_infra_db_key_for_provider_type('OCP'), 'ocp_uuid')
self.assertEqual(updater._get_infra_db_key_for_provider_type('WRONG'), None)
def test_update_cost_summary_table_for_markup(self, mock_markup):
"""Test that summary tables are updated correctly."""
markup = {'value': 10, 'unit': 'percent'}
mock_markup.return_value = markup
self._generate_ocp_on_aws_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta.relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.ocp_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
cluster_id = get_cluster_id_from_provider(self.ocp_test_provider_uuid)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
ocp_accessor.populate_cost_summary_table(cluster_id, start_date,
end_date)
cost_summary_table = OCP_REPORT_TABLE_MAP['cost_summary']
cost_summary_query = ocp_accessor._get_db_obj_query(
cost_summary_table)
possible_values = {}
with schema_context(self.schema):
for item in cost_summary_query:
possible_values.update({
item.cluster_id:
((item.pod_charge_cpu_core_hours +
item.pod_charge_memory_gigabyte_hours +
item.persistentvolumeclaim_charge_gb_month +
item.infra_cost) * decimal.Decimal(0.1))
})
updater.update_cost_summary_table(start_date_str, end_date_str)
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(cost_summary_table)
found_values = {}
for item in query:
found_values.update({item.cluster_id: item.markup_cost})
for k, v in found_values.items():
self.assertAlmostEqual(v, possible_values[k], places=6)