def purge_expired_report_data(self, expired_date=None, provider_uuid=None, simulate=False):
"""Remove report data with a billing start period before specified date.
Args:
expired_date (datetime.datetime): The cutoff date for removing data.
provider_uuid (uuid): The DB id of the provider to purge data for.
simulate (bool): Whether to simulate the removal.
Returns:
([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start'
"""
LOG.info("Calling purge_expired_report_data for azure")
with AzureReportDBAccessor(self._schema) as accessor:
if (expired_date is None and provider_uuid is None) or ( # noqa: W504
expired_date is not None and provider_uuid is not None
):
err = "This method must be called with either expired_date or provider_uuid"
raise AzureReportDBCleanerError(err)
removed_items = []
if expired_date is not None:
bill_objects = accessor.get_bill_query_before_date(expired_date)
else:
bill_objects = accessor.get_cost_entry_bills_query_by_provider(provider_uuid)
with schema_context(self._schema):
for bill in bill_objects.all():
bill_id = bill.id
removed_provider_uuid = bill.provider_id
removed_billing_period_start = bill.billing_period_start
if not simulate:
lineitem_query = accessor.get_lineitem_query_for_billid(bill_id)
del_count, remainder = mini_transaction_delete(lineitem_query)
LOG.info("Removing %s cost entry line items for bill id %s", del_count, bill_id)
summary_query = accessor.get_summary_query_for_billid(bill_id)
del_count, remainder = mini_transaction_delete(summary_query)
LOG.info("Removing %s cost entry summary items for bill id %s", del_count, bill_id)
LOG.info(
"Report data removed for Account Payer ID: %s with billing period: %s",
removed_provider_uuid,
removed_billing_period_start,
)
removed_items.append(
{
"provider_uuid": removed_provider_uuid,
"billing_period_start": str(removed_billing_period_start),
}
)
if not simulate:
bill_objects.delete()
return removed_items
def test_mini_transaction_delete_remainder(self, remainder_check):
remainder_check.return_value = 1
del_rec_lim_key = "DELETE_CYCLE_RECORD_LIMIT"
del_cycle_try_key = "DELETE_CYCLE_MAX_RETRY"
dcrl = os.getenv(del_rec_lim_key, None)
dcmr = os.getenv(del_cycle_try_key, None)
os.environ[del_rec_lim_key] = "3"
os.environ[del_cycle_try_key] = "1"
for id in range(110, 120):
Sources.objects.create(source_id=-id, offset=1, authentication={})
src_query = Sources.objects.filter(source_id__lt=-109)
start_count = src_query.count()
del_count, remainder = mini_transaction_delete(src_query)
end_count = src_query.count()
if dcrl is None:
del os.environ[del_rec_lim_key]
else:
os.environ[del_rec_lim_key] = dcrl
if dcmr is None:
del os.environ[del_cycle_try_key]
else:
os.environ[del_cycle_try_key] = dcmr
self.assertEqual(start_count, 10)
self.assertEqual(remainder, 1)
self.assertEqual(
end_count, 0
) # Based on how this test is constructed, the records would be deleted anyway.
def purge_expired_line_item(self,
expired_date,
provider_uuid=None,
simulate=False):
"""Remove raw line item report data with a billing start period before specified date.
Args:
expired_date (datetime.datetime): The cutoff date for removing data.
provider_uuid (uuid): The DB id of the provider to purge data for.
simulate (bool): Whether to simluate the removal.
Returns:
([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start'
"""
LOG.info("Calling purge_expired_line_item for gcp")
if not isinstance(expired_date, datetime):
err = "Parameter expired_date must be a datetime.datetime object."
raise GCPReportDBCleanerError(err)
with GCPReportDBAccessor(self._schema) as accessor:
removed_items = []
if provider_uuid is not None:
bill_objects = accessor.get_bill_query_before_date(
expired_date, provider_uuid)
else:
bill_objects = accessor.get_bill_query_before_date(
expired_date)
with schema_context(self._schema):
for bill in bill_objects.all():
bill_id = bill.id
removed_provider_uuid = bill.provider_id
removed_billing_period_start = bill.billing_period_start
if not simulate:
lineitem_query = accessor.get_lineitem_query_for_billid(
bill_id)
del_count, remainder = mini_transaction_delete(
lineitem_query)
LOG.info(
"Removing %s cost entry line items for bill id %s",
del_count, bill_id)
LOG.info(
"Line item data removed for Provider ID: %s with billing period: %s",
removed_provider_uuid,
removed_billing_period_start,
)
removed_items.append({
"removed_provider_uuid":
removed_provider_uuid,
"billing_period_start":
str(removed_billing_period_start),
})
return removed_items
def test_mini_transaction_delete_base(self):
for id in range(100, 110):
Sources.objects.create(source_id=-id, offset=1, authentication={})
src_query = Sources.objects.filter(source_id__lt=-99)
start_count = src_query.count()
self.assertEqual(start_count, 10)
del_count, remainder = mini_transaction_delete(src_query)
end_count = src_query.count()
self.assertEqual(del_count, start_count)
self.assertEqual(remainder, 0)
self.assertEqual(end_count, 0)
def purge_expired_line_item(self,
expired_date,
provider_uuid=None,
simulate=False):
"""Remove raw line item report data with a billing start period before specified date.
Args:
expired_date (datetime.datetime): The cutoff date for removing data.
provider_uuid (uuid): The DB id of the provider to purge data for.
simulate (bool): Whether to simluate the removal.
Returns:
([{}]) List of dictionaries containing 'usage_period_id' and 'interval_start'
"""
LOG.info("Calling purge_expired_line_item for ocp")
if not isinstance(expired_date, datetime):
err = "Parameter expired_date must be a datetime.datetime object."
raise OCPReportDBCleanerError(err)
with OCPReportDBAccessor(self._schema) as accessor:
removed_items = []
if provider_uuid is not None:
usage_period_objs = accessor.get_usage_period_on_or_before_date(
expired_date, provider_uuid)
else:
usage_period_objs = accessor.get_usage_period_on_or_before_date(
expired_date)
with schema_context(self._schema):
for usage_period in usage_period_objs.all():
report_period_id = usage_period.id
removed_usage_start_period = usage_period.report_period_start
if not simulate:
item_query = accessor.get_item_query_report_period_id(
report_period_id)
qty, remainder = mini_transaction_delete(item_query)
LOG.info(
"Removing %s usage period line items for usage period id %s",
qty, report_period_id)
LOG.info(
"Line item data removed for usage period ID: %s with interval start: %s",
report_period_id,
removed_usage_start_period,
)
removed_items.append({
"usage_period_id":
report_period_id,
"interval_start":
str(removed_usage_start_period)
})
return removed_items
def delete_line_item_daily_summary_entries_for_date_range(
self, source_uuid, start_date, end_date):
msg = f"Deleting records from {self.line_item_daily_summary_table} from {start_date} to {end_date}"
LOG.info(msg)
select_query = self.line_item_daily_summary_table.objects.filter(
source_uuid=source_uuid,
usage_start__gte=start_date,
usage_start__lte=end_date)
with schema_context(self.schema):
count, _ = mini_transaction_delete(select_query)
msg = f"Deleted {count} records from {self.line_item_daily_summary_table}"
LOG.info(msg)
def _delete_line_items(self, db_accessor, is_finalized=None):
"""Delete stale data for the report being processed, if necessary."""
if not self._manifest_id:
return False
if is_finalized is None:
is_finalized = False
is_full_month = self._should_process_full_month()
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest_by_id(self._manifest_id)
num_processed_files = manifest_accessor.number_of_files_processed(self._manifest_id)
if num_processed_files != 0:
return False
# Override the bill date to correspond with the manifest
bill_date = manifest.billing_period_start_datetime.date()
provider_uuid = manifest.provider_id
date_filter = self.get_date_column_filter()
with db_accessor(self._schema) as accessor:
bills = accessor.get_cost_entry_bills_query_by_provider(provider_uuid)
bills = bills.filter(billing_period_start=bill_date).all()
with schema_context(self._schema):
for bill in bills:
line_item_query = accessor.get_lineitem_query_for_billid(bill.id)
delete_date = bill_date
if not is_finalized and not is_full_month:
delete_date = self.data_cutoff_date
# This means we are processing a mid-month update
# and only need to delete a small window of data
line_item_query = line_item_query.filter(**date_filter)
log_statement = (
f"Deleting data for:\n"
f" schema_name: {self._schema}\n"
f" provider_uuid: {provider_uuid}\n"
f" bill date: {str(bill_date)}\n"
f" bill ID: {bill.id}\n"
f" on or after {delete_date}."
)
LOG.info(log_statement)
del_count, remainder = mini_transaction_delete(line_item_query)
LOG.info(f"Deleted {del_count} records for bill id {bill.id}")
return True
