def test_update_summary_tables_azure(self, mock_markup, mock_refresh):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
mock_markup.return_value = markup
with ProviderDBAccessor(self.ocp_provider_uuid) as accessor:
accessor.set_infrastructure(self.azure_provider_uuid,
Provider.PROVIDER_AZURE)
self._generate_ocp_on_azure_data()
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
with ProviderDBAccessor(
self.azure_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema="acct10001",
provider=provider,
manifest=None)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
initial_count = query.count()
updater.update_summary_tables(start_date_str, end_date_str)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
query = azure_accessor._get_db_obj_query(summary_table_name)
self.assertNotEqual(query.count(), initial_count)
markup_cost = query.aggregate(
Sum("markup_cost"))["markup_cost__sum"]
pretax_cost = query.aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
self.assertAlmostEqual(markup_cost,
pretax_cost *
decimal.Decimal(markup.get("value") / 100),
places=9)
daily_summary_table_name = OCP_REPORT_TABLE_MAP[
"line_item_daily_summary"]
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(daily_summary_table_name)
infra_cost = query.aggregate(Sum("infra_cost"))["infra_cost__sum"]
project_infra_cost = query.aggregate(
Sum("project_infra_cost"))["project_infra_cost__sum"]
self.assertIsNotNone(infra_cost)
self.assertIsNotNone(project_infra_cost)
self.assertNotEqual(infra_cost, decimal.Decimal(0))
self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
mock_refresh.assert_called()
def setUp(self):
"""Set up each test."""
super().setUp()
self.assembly_id = '1234'
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=self.azure_provider_id,
)
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 1,
'provider_id': self.azure_provider.id,
}
self.accessor = AzureReportDBAccessor(self.schema, self.column_map)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.manifest_accessor.commit()
def setUp(self):
"""Set up each test."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f'{self.temp_dir}/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
shutil.copy2(self.test_report_path, self.test_report)
self.assembly_id = '1234'
self.processor = AzureReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.azure_provider_uuid,
)
billing_start = self.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0
)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 1,
'provider_uuid': self.azure_provider_uuid,
}
self.accessor = AzureReportDBAccessor(self.schema, self.column_map)
self.report_schema = self.accessor.report_schema
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def test_azure_update_summary_tables_with_manifest(self, mock_summary):
"""Test that summary tables are properly run."""
self.manifest.num_processed_files = self.manifest.num_total_files
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
bill.summary_data_creation_datetime = start_date
bill.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
expected_start_date = start_date.date()
expected_end_date = end_date.date()
self.assertIsNone(bill.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called_with(expected_start_date, expected_end_date,
[str(bill.id)])
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertIsNotNone(bill.summary_data_updated_datetime)
def test_update_summary_tables_azure(self, mock_markup):
"""Test that summary tables are updated correctly."""
markup = {'value': 10, 'unit': 'percent'}
mock_markup.return_value = markup
with ProviderDBAccessor(self.ocp_provider_uuid) as accessor:
accessor.set_infrastructure(self.azure_provider_uuid, 'AZURE')
self._generate_ocp_on_azure_data()
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
with ProviderDBAccessor(
self.azure_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema='acct10001',
provider=provider,
manifest=None)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
'ocp_on_azure_daily_summary']
query = azure_accessor._get_db_obj_query(summary_table_name)
initial_count = query.count()
updater.update_summary_tables(start_date_str, end_date_str)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
query = azure_accessor._get_db_obj_query(summary_table_name)
self.assertNotEqual(query.count(), initial_count)
markup_cost = query.aggregate(
Sum('markup_cost'))['markup_cost__sum']
pretax_cost = query.aggregate(
Sum('pretax_cost'))['pretax_cost__sum']
self.assertAlmostEqual(markup_cost,
pretax_cost *
decimal.Decimal(markup.get('value') / 100),
places=9)
daily_summary_table_name = OCP_REPORT_TABLE_MAP[
'line_item_daily_summary']
with OCPReportDBAccessor(self.schema, self.column_map) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(daily_summary_table_name)
infra_cost = query.aggregate(Sum('infra_cost'))['infra_cost__sum']
project_infra_cost = query.aggregate(
Sum('project_infra_cost'))['project_infra_cost__sum']
self.assertIsNotNone(infra_cost)
self.assertIsNotNone(project_infra_cost)
self.assertNotEqual(infra_cost, decimal.Decimal(0))
self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def update_azure_summary_tables(self, openshift_provider_uuid,
azure_provider_uuid, start_date, end_date):
"""Update operations specifically for OpenShift on Azure."""
cluster_id = get_cluster_id_from_provider(openshift_provider_uuid)
azure_bills = azure_get_bills_from_provider(
azure_provider_uuid, self._schema,
datetime.datetime.strptime(start_date, '%Y-%m-%d'),
datetime.datetime.strptime(end_date, '%Y-%m-%d'))
azure_bill_ids = []
with schema_context(self._schema):
azure_bill_ids = [str(bill.id) for bill in azure_bills]
with CostModelDBAccessor(self._schema, azure_provider_uuid,
self._column_map) as cost_model_accessor:
markup = cost_model_accessor.get_markup()
markup_value = Decimal(markup.get('value', 0)) / 100
# OpenShift on Azure
with AzureReportDBAccessor(self._schema, self._column_map) as accessor:
LOG.info(
'Updating OpenShift on Azure summary table for '
'\n\tSchema: %s \n\tProvider: %s \n\tDates: %s - %s'
'\n\tCluster ID: %s, Azure Bill IDs: %s', self._schema,
self._provider.uuid, start_date, end_date, cluster_id,
str(azure_bill_ids))
accessor.populate_ocp_on_azure_cost_daily_summary(
start_date, end_date, cluster_id, azure_bill_ids)
accessor.populate_ocp_on_azure_markup_cost(markup_value,
azure_bill_ids)
with OCPReportDBAccessor(self._schema, self._column_map) as accessor:
# This call just sends the infrastructure cost to the
# OCP usage daily summary table
accessor.update_summary_infrastructure_cost(
cluster_id, start_date, end_date)
def update_summary_cost_model_costs(self, start_date=None, end_date=None):
"""Update the Azure summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
LOG.debug(
"Starting charge calculation updates for provider: %s. Dates: %s-%s",
self._provider.uuid,
str(start_date),
str(end_date),
)
self._update_markup_cost(start_date, end_date)
with AzureReportDBAccessor(self._schema) as accessor:
LOG.debug(
"Updating Azure derived cost summary for schema: %s and provider: %s",
self._schema,
self._provider.uuid,
)
bills = accessor.bills_for_provider_uuid(self._provider.uuid,
start_date)
with schema_context(self._schema):
for bill in bills:
bill.derived_cost_datetime = DateAccessor(
).today_with_timezone("UTC")
bill.save()
def create_azure_cost_entry_line_item(self,
bill,
product,
meter,
usage_date_time=None):
"""Create an Azure cost entry line item database object for test."""
table_name = AZURE_REPORT_TABLE_MAP['line_item']
data = self.create_columns_for_table(table_name)
random_usage_date_time = bill.billing_period_start + relativedelta.relativedelta(
days=random.randint(1, 15))
extra_data = {
'cost_entry_bill_id': bill.id,
'cost_entry_product_id': product.id,
'meter_id': meter.id,
'usage_date_time':
usage_date_time if usage_date_time else random_usage_date_time,
'tags': {
'environment': random.choice(['dev', 'qa', 'prod']),
self.fake.pystr()[:8]: self.fake.pystr()[:8],
}
}
data.update(extra_data)
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def test_azure_update_summary_tables_new_bill(self, mock_summary):
"""Test that summary tables are run for a full month."""
self.manifest.num_processed_files = self.manifest.num_total_files
start_date = self.date_accessor.today_with_timezone('UTC')
end_date = start_date
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
bill = self.accessor.get_cost_entry_bills_by_date(bill_date)[0]
last_day_of_month = calendar.monthrange(bill_date.year, bill_date.month)[1]
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
expected_start_date = start_date.replace(day=1).strftime('%Y-%m-%d')
expected_end_date = end_date.replace(day=last_day_of_month).strftime('%Y-%m-%d')
self.assertIsNone(bill.summary_data_creation_datetime)
self.assertIsNone(bill.summary_data_updated_datetime)
self.updater.update_daily_tables(start_date_str, end_date_str)
mock_summary.assert_not_called()
self.updater.update_summary_tables(start_date_str, end_date_str)
mock_summary.assert_called_with(expected_start_date, expected_end_date, [str(bill.id)])
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.summary_data_creation_datetime)
self.assertIsNotNone(bill.summary_data_updated_datetime)
def create_azure_cost_entry_line_item(self,
bill,
product,
meter,
usage_date=None):
"""Create an Azure cost entry line item database object for test."""
table_name = AZURE_REPORT_TABLE_MAP["line_item"]
data = self.create_columns_for_table(table_name)
if usage_date:
usage_date = usage_date.date() if isinstance(
usage_date, datetime.datetime) else usage_date
else:
usage_date = (bill.billing_period_start +
relativedelta.relativedelta(
days=random.randint(1, 15))).date()
extra_data = {
"cost_entry_bill_id": bill.id,
"cost_entry_product_id": product.id,
"meter_id": meter.id,
"usage_date": usage_date,
"tags": {
"environment": random.choice(["dev", "qa", "prod"]),
self.fake.pystr()[:8]: self.fake.pystr()[:8],
},
}
data.update(extra_data)
with AzureReportDBAccessor(self.schema) as accessor:
return accessor.create_db_object(table_name, data)
def _get_sql_inputs(self, start_date, end_date):
"""Get the required inputs for running summary SQL."""
with AzureReportDBAccessor(self._schema) as accessor:
# This is the normal processing route
if self._manifest:
# Override the bill date to correspond with the manifest
bill_date = self._manifest.billing_period_start_datetime.date()
bills = accessor.get_cost_entry_bills_query_by_provider(
self._provider.uuid)
bills = bills.filter(billing_period_start=bill_date).all()
first_bill = bills.filter(
billing_period_start=bill_date).first()
do_month_update = False
with schema_context(self._schema):
if first_bill:
do_month_update = self._determine_if_full_summary_update_needed(
first_bill)
if do_month_update:
last_day_of_month = calendar.monthrange(
bill_date.year, bill_date.month)[1]
start_date = bill_date.strftime("%Y-%m-%d")
end_date = bill_date.replace(day=last_day_of_month)
end_date = end_date.strftime("%Y-%m-%d")
LOG.info(
"Overriding start and end date to process full month.")
return start_date, end_date
def test_update_daily_summary_tables(self, mock_presto, mock_tag_update, mock_delete):
"""Test that we run Presto summary."""
start_str = self.dh.this_month_start.isoformat()
end_str = self.dh.this_month_end.isoformat()
start, end = self.updater._get_sql_inputs(start_str, end_str)
for s, e in date_range_pair(start, end, step=settings.TRINO_DATE_STEP):
expected_start, expected_end = s, e
with AzureReportDBAccessor(self.schema) as accessor:
with schema_context(self.schema):
bills = accessor.bills_for_provider_uuid(self.azure_provider.uuid, start)
bill_ids = [str(bill.id) for bill in bills]
current_bill_id = bills.first().id if bills else None
with CostModelDBAccessor(self.schema, self.azure_provider.uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = float(markup.get("value", 0)) / 100
start_return, end_return = self.updater.update_summary_tables(start, end)
mock_delete.assert_called_with(self.azure_provider.uuid, expected_start, expected_end)
mock_presto.assert_called_with(
expected_start, expected_end, self.azure_provider.uuid, current_bill_id, markup_value
)
mock_tag_update.assert_called_with(bill_ids, start, end)
self.assertEqual(start_return, start)
self.assertEqual(end_return, end)
def create_azure_meter(self):
"""Create an Azure meter database object for test."""
table_name = AZURE_REPORT_TABLE_MAP['meter']
data = self.create_columns_for_table(table_name)
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def db_accessor(self):
"""Return the accessor for the infrastructure provider."""
if self.provider_type == Provider.PROVIDER_AWS:
return AWSReportDBAccessor(self.schema_name)
elif self.provider_type == Provider.PROVIDER_AZURE:
return AzureReportDBAccessor(self.schema_name)
return None
def test_refresh_openshift_on_infrastructure_views(self):
"""Test that the combined OpenShift views are refreshed."""
self._generate_ocp_on_aws_data(cluster_id="my-ocp-cluster-1")
self._generate_ocp_on_azure_data(cluster_id="my-ocp-cluster-2")
start_date = self.date_accessor.today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
with ProviderDBAccessor(
self.azure_test_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
with AzureReportDBAccessor(self.schema,
self.column_map) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_count = query.count()
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_project_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_project_count = query.count()
with AWSReportDBAccessor(self.schema, self.column_map) as aws_accessor:
summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_count = query.count()
summary_table_name = AWS_CUR_TABLE_MAP[
"ocp_on_aws_project_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_project_count = query.count()
updater.refresh_openshift_on_infrastructure_views()
with schema_context(self.schema):
all_count = OCPAllCostLineItemDailySummary.objects.count()
all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count(
)
self.assertEqual(all_count, azure_count + aws_count)
self.assertEqual(all_project_count,
azure_project_count + aws_project_count)
def create_azure_meter(self, provider_uuid):
"""Create an Azure meter database object for test."""
table_name = AZURE_REPORT_TABLE_MAP["meter"]
data = self.create_columns_for_table(table_name)
data["provider_id"] = provider_uuid
with AzureReportDBAccessor(self.schema) as accessor:
return accessor.create_db_object(table_name, data)
def create_azure_cost_entry_product(self, provider_uuid):
"""Create an Azure cost entry product database object for test."""
table_name = AZURE_REPORT_TABLE_MAP['product']
data = self.create_columns_for_table(table_name)
data['provider_id'] = provider_uuid
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def test_refresh_openshift_on_infrastructure_views(self,
mock_view_refresh):
"""Test that the combined OpenShift views are refreshed."""
start_date = self.dh.today
end_date = start_date + datetime.timedelta(days=1)
start_date = start_date - relativedelta(months=1)
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=self.azure_provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
mock_view_refresh.assert_called_with(
OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
mock_view_refresh.reset_mock()
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=self.aws_provider,
manifest=None)
updater.update_summary_tables(start_date_str, end_date_str)
expected_calls = [
call(OCP_ON_AWS_MATERIALIZED_VIEWS),
call(OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
]
mock_view_refresh.assert_has_calls(expected_calls)
with AzureReportDBAccessor(self.schema) as azure_accessor:
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_count = query.count()
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_project_daily_summary"]
query = azure_accessor._get_db_obj_query(summary_table_name)
azure_project_count = query.count()
with AWSReportDBAccessor(self.schema) as aws_accessor:
summary_table_name = AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_count = query.count()
summary_table_name = AWS_CUR_TABLE_MAP[
"ocp_on_aws_project_daily_summary"]
query = aws_accessor._get_db_obj_query(summary_table_name)
aws_project_count = query.count()
updater.refresh_openshift_on_infrastructure_views(
OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
with schema_context(self.schema):
all_count = OCPAllCostLineItemDailySummary.objects.count()
all_project_count = OCPAllCostLineItemProjectDailySummary.objects.count(
)
self.assertEqual(all_count, azure_count + aws_count)
self.assertEqual(all_project_count,
azure_project_count + aws_project_count)
def test_update_summary_tables_azure(
self,
mock_cost_model # , mock_ocpall_proj_summ, mock_ocpall_summ, mock_ocpall_persp
):
"""Test that summary tables are updated correctly."""
markup = {"value": 10, "unit": "percent"}
mock_cost_model.markup = markup
start_date = self.dh.this_month_start
end_date = self.dh.this_month_end
updater = OCPCloudReportSummaryUpdater(schema=self.schema,
provider=self.azure_provider,
manifest=None)
updater.update_summary_tables(
start_date,
end_date,
self.ocp_on_azure_ocp_provider.uuid,
self.azure_provider.uuid,
Provider.PROVIDER_AZURE,
)
summary_table_name = AZURE_REPORT_TABLE_MAP[
"ocp_on_azure_daily_summary"]
with AzureReportDBAccessor(self.schema) as azure_accessor:
query = azure_accessor._get_db_obj_query(
summary_table_name).filter(
cost_entry_bill__billing_period_start=start_date)
markup_cost = query.aggregate(
Sum("markup_cost"))["markup_cost__sum"]
pretax_cost = query.aggregate(
Sum("pretax_cost"))["pretax_cost__sum"]
self.assertAlmostEqual(markup_cost,
pretax_cost *
decimal.Decimal(markup.get("value") / 100),
places=5)
daily_summary_table_name = OCP_REPORT_TABLE_MAP[
"line_item_daily_summary"]
with OCPReportDBAccessor(self.schema) as ocp_accessor:
query = ocp_accessor._get_db_obj_query(
daily_summary_table_name).filter(
report_period__provider=self.ocp_on_azure_ocp_provider,
report_period__report_period_start=self.dh.
this_month_start,
)
infra_cost = query.aggregate(
Sum("infrastructure_raw_cost"))["infrastructure_raw_cost__sum"]
project_infra_cost = query.aggregate(
Sum("infrastructure_project_raw_cost")
)["infrastructure_project_raw_cost__sum"]
self.assertIsNotNone(infra_cost)
self.assertIsNotNone(project_infra_cost)
self.assertNotEqual(infra_cost, decimal.Decimal(0))
self.assertNotEqual(project_infra_cost, decimal.Decimal(0))
def purge_expired_report_data(self, expired_date=None, provider_uuid=None, simulate=False):
"""Remove report data with a billing start period before specified date.
Args:
expired_date (datetime.datetime): The cutoff date for removing data.
provider_uuid (uuid): The DB id of the provider to purge data for.
simulate (bool): Whether to simulate the removal.
Returns:
([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start'
"""
LOG.info("Calling purge_expired_report_data for azure")
with AzureReportDBAccessor(self._schema) as accessor:
if (expired_date is None and provider_uuid is None) or ( # noqa: W504
expired_date is not None and provider_uuid is not None
):
err = "This method must be called with either expired_date or provider_uuid"
raise AzureReportDBCleanerError(err)
removed_items = []
if expired_date is not None:
bill_objects = accessor.get_bill_query_before_date(expired_date)
else:
bill_objects = accessor.get_cost_entry_bills_query_by_provider(provider_uuid)
with schema_context(self._schema):
for bill in bill_objects.all():
bill_id = bill.id
removed_provider_uuid = bill.provider_id
removed_billing_period_start = bill.billing_period_start
if not simulate:
lineitem_query = accessor.get_lineitem_query_for_billid(bill_id)
del_count, remainder = mini_transaction_delete(lineitem_query)
LOG.info("Removing %s cost entry line items for bill id %s", del_count, bill_id)
summary_query = accessor.get_summary_query_for_billid(bill_id)
del_count, remainder = mini_transaction_delete(summary_query)
LOG.info("Removing %s cost entry summary items for bill id %s", del_count, bill_id)
LOG.info(
"Report data removed for Account Payer ID: %s with billing period: %s",
removed_provider_uuid,
removed_billing_period_start,
)
removed_items.append(
{
"provider_uuid": removed_provider_uuid,
"billing_period_start": str(removed_billing_period_start),
}
)
if not simulate:
bill_objects.delete()
return removed_items
def process(self):
"""Process cost/usage file.
Returns:
(None)
"""
row_count = 0
is_full_month = self._should_process_full_month()
self._delete_line_items(AzureReportDBAccessor)
opener, mode = self._get_file_opener(self._compression)
with opener(self._report_path, mode, encoding="utf-8-sig") as f:
header = normalize_header(f.readline())
with AzureReportDBAccessor(self._schema) as report_db:
temp_table = report_db.create_temp_table(
self.table_name._meta.db_table, drop_column="id")
LOG.info("File %s opened for processing", str(f))
reader = csv.DictReader(f, fieldnames=header)
for row in reader:
if not self._should_process_row(row, "UsageDateTime",
is_full_month):
continue
self.create_cost_entry_objects(row, report_db)
if len(self.processed_report.line_items
) >= self._batch_size:
LOG.info(
"Saving report rows %d to %d for %s",
row_count,
row_count + len(self.processed_report.line_items),
self._report_name,
)
self._save_to_db(temp_table, report_db)
row_count += len(self.processed_report.line_items)
self._update_mappings()
if self.processed_report.line_items:
LOG.info(
"Saving report rows %d to %d for %s",
row_count,
row_count + len(self.processed_report.line_items),
self._report_name,
)
self._save_to_db(temp_table, report_db)
row_count += len(self.processed_report.line_items)
if self.processed_report.line_items:
report_db.merge_temp_table(self.table_name._meta.db_table,
temp_table,
self.line_item_columns)
LOG.info(
"Completed report processing for file: %s and schema: %s",
self._report_name, self._schema)
if not settings.DEVELOPMENT:
LOG.info("Removing processed file: %s", self._report_path)
remove(self._report_path)
return True
def update_summary_tables(self, start_date, end_date):
"""Populate the summary tables for reporting.
Args:
start_date (str) The date to start populating the table.
end_date (str) The date to end on.
Returns
(str, str) A start date and end date.
"""
start_date, end_date = self._get_sql_inputs(start_date, end_date)
with CostModelDBAccessor(self._schema,
self._provider.uuid) as cost_model_accessor:
markup = cost_model_accessor.markup
markup_value = float(markup.get("value", 0)) / 100
with AzureReportDBAccessor(self._schema) as accessor:
# Need these bills on the session to update dates after processing
with schema_context(self._schema):
bills = accessor.bills_for_provider_uuid(
self._provider.uuid, start_date)
bill_ids = [str(bill.id) for bill in bills]
current_bill_id = bills.first().id if bills else None
for start, end in date_range_pair(start_date,
end_date,
step=settings.TRINO_DATE_STEP):
LOG.info(
"Updating Azure report summary tables via Presto: \n\tSchema: %s"
"\n\tProvider: %s \n\tDates: %s - %s",
self._schema,
self._provider.uuid,
start,
end,
)
accessor.delete_line_item_daily_summary_entries_for_date_range(
self._provider.uuid, start, end)
accessor.populate_line_item_daily_summary_table_presto(
start, end, self._provider.uuid, current_bill_id,
markup_value)
accessor.populate_enabled_tag_keys(start, end, bill_ids)
accessor.populate_tags_summary_table(bill_ids, start_date,
end_date)
accessor.update_line_item_daily_summary_with_enabled_tags(
start_date, end_date, bill_ids)
for bill in bills:
if bill.summary_data_creation_datetime is None:
bill.summary_data_creation_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.summary_data_updated_datetime = self._date_accessor.today_with_timezone(
"UTC")
bill.save()
return start_date, end_date
def purge_expired_report_data_by_date(self, expired_date, simulate=False):
partition_from = str(date(expired_date.year, expired_date.month, 1))
with AzureReportDBAccessor(self._schema) as accessor:
all_bill_objects = accessor.get_bill_query_before_date(
expired_date).all()
table_names = [
accessor._table_map["ocp_on_azure_daily_summary"],
accessor._table_map["ocp_on_azure_project_daily_summary"],
accessor.line_item_daily_summary_table._meta.db_table,
accessor.ocpall_line_item_daily_summary_table._meta.db_table,
accessor.ocpall_line_item_project_daily_summary_table._meta.
db_table,
]
table_names.extend(UI_SUMMARY_TABLES)
table_models = [get_model(tn) for tn in table_names]
with schema_context(self._schema):
removed_items = []
all_providers = set()
all_period_starts = set()
# Iterate over the remainder as they could involve much larger amounts of data
for bill in all_bill_objects:
removed_items.append({
"provider_uuid":
bill.provider_id,
"billing_period_start":
str(bill.billing_period_start)
})
all_providers.add(bill.provider_id)
all_period_starts.add(str(bill.billing_period_start))
LOG.info(
f"Deleting data for providers {all_providers} and periods {all_period_starts}"
)
if not simulate:
# Will call trigger to detach, truncate, and drop partitions
LOG.info(
"Deleting table partitions total for the following tables: "
+ f"{table_names} with partitions <= {partition_from}")
del_count = execute_delete_sql(
PartitionedTable.objects.filter(
schema_name=self._schema,
partition_of_table_name__in=table_names,
partition_parameters__default=False,
partition_parameters__from__lte=partition_from,
))
LOG.info(f"Deleted {del_count} table partitions")
if not simulate:
cascade_delete(all_bill_objects.query.model,
all_bill_objects,
skip_relations=table_models)
return removed_items
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.accessor = AzureReportDBAccessor("acct10001")
cls.report_schema = cls.accessor.report_schema
cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.schema)
cls.date_accessor = DateHelper()
cls.manifest_accessor = ReportManifestDBAccessor()
def create_azure_cost_entry_product(self, provider_uuid, instance_id=None):
"""Create an Azure cost entry product database object for test."""
table_name = AZURE_REPORT_TABLE_MAP["product"]
data = self.create_columns_for_table(table_name)
data["provider_id"] = provider_uuid
if instance_id:
data["instance_id"] = instance_id
with AzureReportDBAccessor(self.schema) as accessor:
return accessor.create_db_object(table_name, data)
def test_azure_update_summary_charge_info(self):
"""Test to verify Azure derived cost summary is calculated."""
start_date = self.date_accessor.today_with_timezone('UTC')
bill_date = start_date.replace(day=1).date()
self.updater.update_summary_charge_info()
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.derived_cost_datetime)
def purge_expired_report_data(self,
expired_date=None,
provider_uuid=None,
simulate=False):
"""Remove report data with a billing start period before specified date.
Args:
expired_date (datetime.datetime): The cutoff date for removing data.
provider_uuid (uuid): The DB id of the provider to purge data for.
simulate (bool): Whether to simulate the removal.
Returns:
([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start'
"""
LOG.info("Calling purge_expired_report_data for azure")
with AzureReportDBAccessor(self._schema) as accessor:
if (expired_date is None
and provider_uuid is None) or ( # noqa: W504
expired_date is not None
and provider_uuid is not None):
err = "This method must be called with either expired_date or provider_uuid"
raise AzureReportDBCleanerError(err)
removed_items = []
all_providers = set()
all_period_starts = set()
if expired_date is not None:
return self.purge_expired_report_data_by_date(
expired_date, simulate=simulate)
bill_objects = accessor.get_cost_entry_bills_query_by_provider(
provider_uuid)
with schema_context(self._schema):
for bill in bill_objects.all():
removed_items.append({
"provider_uuid":
bill.provider_id,
"billing_period_start":
str(bill.billing_period_start)
})
all_providers.add(bill.provider_id)
all_period_starts.add(str(bill.billing_period_start))
LOG.info(
f"Deleting data for providers {all_providers} and periods {all_period_starts}"
)
if not simulate:
cascade_delete(bill_objects.query.model, bill_objects)
return removed_items
def process(self):
"""Process cost/usage file.
Returns:
(None)
"""
row_count = 0
is_full_month = self._should_process_full_month()
self._delete_line_items(AzureReportDBAccessor, self.column_map)
# pylint: disable=invalid-name
opener, mode = self._get_file_opener(self._compression)
with opener(self._report_path, mode, encoding="utf-8-sig") as f:
with AzureReportDBAccessor(self._schema,
self.column_map) as report_db:
LOG.info("File %s opened for processing", str(f))
reader = csv.DictReader(f)
for row in reader:
if not self._should_process_row(row, "UsageDateTime",
is_full_month):
continue
_ = self.create_cost_entry_objects(row, report_db)
if len(self.processed_report.line_items
) >= self._batch_size:
LOG.info(
"Saving report rows %d to %d for %s",
row_count,
row_count + len(self.processed_report.line_items),
self._report_name,
)
self._save_to_db(AZURE_REPORT_TABLE_MAP["line_item"],
report_db)
row_count += len(self.processed_report.line_items)
self._update_mappings()
if self.processed_report.line_items:
LOG.info(
"Saving report rows %d to %d for %s",
row_count,
row_count + len(self.processed_report.line_items),
self._report_name,
)
self._save_to_db(AZURE_REPORT_TABLE_MAP["line_item"],
report_db)
row_count += len(self.processed_report.line_items)
LOG.info(
"Completed report processing for file: %s and schema: %s",
self._report_name, self._schema)
if not settings.DEVELOPMENT:
LOG.info("Removing processed file: %s", self._report_path)
remove(self._report_path)
return True
def test_azure_update_summary_cost_model_costs(self):
"""Test to verify Azure derived cost summary is calculated."""
updater = AzureCostModelCostUpdater(schema=self.schema, provider=self.azure_provider)
start_date = DateAccessor().today_with_timezone("UTC")
bill_date = start_date.replace(day=1).date()
updater.update_summary_cost_model_costs()
with AzureReportDBAccessor(self.schema) as accessor:
bill = accessor.get_cost_entry_bills_by_date(bill_date)[0]
self.assertIsNotNone(bill.derived_cost_datetime)
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = AzureReportDBAccessor("acct10001", cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
