def purge_expired_report_data(self,
expired_date=None,
provider_uuid=None,
simulate=False):
"""Remove usage data with a report period before specified date.
Args:
expired_date (datetime.datetime): The cutoff date for removing data.
provider_uuid (uuid): The DB id of the provider to purge data for.
simulate (bool): Whether to simluate the removal.
Returns:
([{}]) List of dictionaries containing 'usage_period_id' and 'interval_start'
"""
LOG.info("Calling purge_expired_report_data for ocp")
with OCPReportDBAccessor(self._schema) as accessor:
if (expired_date is not None
and provider_uuid is not None) or ( # noqa: W504
expired_date is None and provider_uuid is None):
err = "This method must be called with expired_date or provider_uuid"
raise OCPReportDBCleanerError(err)
removed_items = []
all_report_periods = []
all_cluster_ids = set()
all_period_starts = set()
if expired_date is not None:
return self.purge_expired_report_data_by_date(
expired_date, simulate=simulate)
usage_period_objs = accessor.get_usage_period_query_by_provider(
provider_uuid)
with schema_context(self._schema):
for usage_period in usage_period_objs.all():
removed_items.append({
"usage_period_id":
usage_period.id,
"interval_start":
str(usage_period.report_period_start)
})
all_report_periods.append(usage_period.id)
all_cluster_ids.add(usage_period.cluster_id)
all_period_starts.add(str(
usage_period.report_period_start))
all_report_periods.sort()
LOG.info(
f"Removing all data related to report_period_ids: {all_report_periods}; "
f"cluster_ids: {all_cluster_ids}; starting periods {all_period_starts}"
)
if not simulate:
cascade_delete(usage_period_objs.query.model,
usage_period_objs)
return removed_items
def purge_expired_report_data_by_date(self, expired_date, simulate=False):
partition_from = str(date(expired_date.year, expired_date.month, 1))
with AzureReportDBAccessor(self._schema) as accessor:
all_bill_objects = accessor.get_bill_query_before_date(
expired_date).all()
table_names = [
accessor._table_map["ocp_on_azure_daily_summary"],
accessor._table_map["ocp_on_azure_project_daily_summary"],
accessor.line_item_daily_summary_table._meta.db_table,
accessor.ocpall_line_item_daily_summary_table._meta.db_table,
accessor.ocpall_line_item_project_daily_summary_table._meta.
db_table,
]
table_names.extend(UI_SUMMARY_TABLES)
table_models = [get_model(tn) for tn in table_names]
with schema_context(self._schema):
removed_items = []
all_providers = set()
all_period_starts = set()
# Iterate over the remainder as they could involve much larger amounts of data
for bill in all_bill_objects:
removed_items.append({
"provider_uuid":
bill.provider_id,
"billing_period_start":
str(bill.billing_period_start)
})
all_providers.add(bill.provider_id)
all_period_starts.add(str(bill.billing_period_start))
LOG.info(
f"Deleting data for providers {all_providers} and periods {all_period_starts}"
)
if not simulate:
# Will call trigger to detach, truncate, and drop partitions
LOG.info(
"Deleting table partitions total for the following tables: "
+ f"{table_names} with partitions <= {partition_from}")
del_count = execute_delete_sql(
PartitionedTable.objects.filter(
schema_name=self._schema,
partition_of_table_name__in=table_names,
partition_parameters__default=False,
partition_parameters__from__lte=partition_from,
))
LOG.info(f"Deleted {del_count} table partitions")
if not simulate:
cascade_delete(all_bill_objects.query.model,
all_bill_objects,
skip_relations=table_models)
return removed_items
def purge_expired_report_data_by_date(self, expired_date, simulate=False):
partition_from = str(date(expired_date.year, expired_date.month, 1))
removed_items = []
all_account_ids = set()
all_period_start = set()
with AWSReportDBAccessor(self._schema) as accessor:
all_bill_objects = accessor.get_bill_query_before_date(
expired_date).all()
for bill in all_bill_objects:
removed_items.append({
"account_payer_id":
bill.payer_account_id,
"billing_period_start":
str(bill.billing_period_start)
})
all_account_ids.add(bill.payer_account_id)
all_period_start.add(str(bill.billing_period_start))
table_names = [
accessor._table_map["ocp_on_aws_daily_summary"],
accessor._table_map["ocp_on_aws_project_daily_summary"],
accessor.line_item_daily_summary_table._meta.db_table,
accessor.ocpall_line_item_daily_summary_table._meta.db_table,
accessor.ocpall_line_item_project_daily_summary_table._meta.
db_table,
]
table_names.extend(UI_SUMMARY_TABLES)
table_models = [get_model(tn) for tn in table_names]
with schema_context(self._schema):
if not simulate:
# Will call trigger to detach, truncate, and drop partitions
LOG.info(
"Deleting table partitions total for the following tables: "
+ f"{table_names} with partitions <= {partition_from}")
del_count = execute_delete_sql(
PartitionedTable.objects.filter(
schema_name=self._schema,
partition_of_table_name__in=table_names,
partition_parameters__default=False,
partition_parameters__from__lte=partition_from,
))
LOG.info(f"Deleted {del_count} table partitions")
# Using skip_relations here as we have already dropped partitions above
cascade_delete(all_bill_objects.query.model,
all_bill_objects,
skip_relations=table_models)
LOG.info(
f"Deleting data related to billing account ids {all_account_ids} "
f"for billing periods starting {all_period_start}")
return removed_items
def purge_expired_report_data(self,
expired_date=None,
provider_uuid=None,
simulate=False):
"""Remove report data with a billing start period before specified date.
Args:
expired_date (datetime.datetime): The cutoff date for removing data.
provider_uuid (uuid): The DB id of the provider to purge data for.
simulate (bool): Whether to simluate the removal.
Returns:
([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start'
"""
LOG.info("Calling purge_expired_report_data for gcp")
with GCPReportDBAccessor(self._schema) as accessor:
if (expired_date is None
and provider_uuid is None) or ( # noqa: W504
expired_date is not None
and provider_uuid is not None):
err = "This method must be called with either expired_date or provider_uuid"
raise GCPReportDBCleanerError(err)
removed_items = []
all_providers = set()
all_period_starts = set()
if expired_date is not None:
return self.purge_expired_report_data_by_date(
expired_date, simulate=simulate)
bill_objects = accessor.get_cost_entry_bills_query_by_provider(
provider_uuid)
with schema_context(self._schema):
for bill in bill_objects.all():
removed_items.append({
"removed_provider_uuid":
bill.provider_id,
"billing_period_start":
str(bill.billing_period_start)
})
all_providers.add(bill.provider_id)
all_period_starts.add(str(bill.billing_period_start))
LOG.info(
f"Deleting data for providers {all_providers} and periods {all_period_starts}"
)
if not simulate:
cascade_delete(bill_objects.query.model, bill_objects)
return removed_items
def delete(self, *args, **kwargs):
if self.customer:
using = router.db_for_write(self.__class__, isinstance=self)
with schema_context(self.customer.schema_name):
LOG.info(
f"PROVIDER {self.name} ({self.pk}) CASCADE DELETE -- SCHEMA {self.customer.schema_name}"
)
cascade_delete(self.__class__, self.__class__,
self.__class__.objects.filter(pk=self.pk))
post_delete.send(sender=self.__class__,
instance=self,
using=using)
else:
super().delete()
def purge_expired_report_data_by_date(self, expired_date, simulate=False):
LOG.info("Executing purge_expired_report_data_by_date")
partition_from = str(date(expired_date.year, expired_date.month, 1))
removed_items = []
all_report_periods = []
all_cluster_ids = set()
all_period_starts = set()
with OCPReportDBAccessor(self._schema) as accessor:
# all_usage_periods = accessor.get_usage_periods_by_date(expired_date)
all_usage_periods = accessor._get_db_obj_query(
accessor._table_map["report_period"]).filter(
report_period_start__lte=expired_date)
table_names = [
# accessor._aws_table_map["ocp_on_aws_daily_summary"],
# accessor._aws_table_map["ocp_on_aws_project_daily_summary"],
accessor._table_map["line_item_daily_summary"]
]
table_names.extend(UI_SUMMARY_TABLES)
table_models = [get_model(tn) for tn in table_names]
with schema_context(self._schema):
# Iterate over the remainder as they could involve much larger amounts of data
for usage_period in all_usage_periods:
removed_items.append({
"usage_period_id":
usage_period.id,
"interval_start":
str(usage_period.report_period_start)
})
all_report_periods.append(usage_period.id)
all_cluster_ids.add(usage_period.cluster_id)
all_period_starts.add(str(usage_period.report_period_start))
all_report_periods.sort()
LOG.info(
f"Removing all data related to "
f"cluster_ids: {all_cluster_ids}; starting periods {all_period_starts}"
)
if not simulate:
# Will call trigger to detach, truncate, and drop partitions
LOG.info(
"Deleting table partitions total for the following tables: "
+ f"{table_names} with partitions <= {partition_from}")
del_count = execute_delete_sql(
PartitionedTable.objects.filter(
schema_name=self._schema,
partition_of_table_name__in=table_names,
partition_parameters__default=False,
partition_parameters__from__lte=partition_from,
))
LOG.info(f"Deleted {del_count} table partitions")
if not simulate:
cascade_delete(all_usage_periods.query.model,
all_usage_periods,
skip_relations=table_models)
return removed_items