def purge_expired_report_data(self, expired_date=None, provider_uuid=None, simulate=False): """Remove report data with a billing start period before specified date. Args: expired_date (datetime.datetime): The cutoff date for removing data. provider_uuid (uuid): The DB id of the provider to purge data for. simulate (bool): Whether to simulate the removal. Returns: ([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start' """ LOG.info("Calling purge_expired_report_data for azure") with AzureReportDBAccessor(self._schema) as accessor: if (expired_date is None and provider_uuid is None) or ( # noqa: W504 expired_date is not None and provider_uuid is not None ): err = "This method must be called with either expired_date or provider_uuid" raise AzureReportDBCleanerError(err) removed_items = [] if expired_date is not None: bill_objects = accessor.get_bill_query_before_date(expired_date) else: bill_objects = accessor.get_cost_entry_bills_query_by_provider(provider_uuid) with schema_context(self._schema): for bill in bill_objects.all(): bill_id = bill.id removed_provider_uuid = bill.provider_id removed_billing_period_start = bill.billing_period_start if not simulate: lineitem_query = accessor.get_lineitem_query_for_billid(bill_id) del_count, remainder = mini_transaction_delete(lineitem_query) LOG.info("Removing %s cost entry line items for bill id %s", del_count, bill_id) summary_query = accessor.get_summary_query_for_billid(bill_id) del_count, remainder = mini_transaction_delete(summary_query) LOG.info("Removing %s cost entry summary items for bill id %s", del_count, bill_id) LOG.info( "Report data removed for Account Payer ID: %s with billing period: %s", removed_provider_uuid, removed_billing_period_start, ) removed_items.append( { "provider_uuid": removed_provider_uuid, "billing_period_start": str(removed_billing_period_start), } ) if not simulate: bill_objects.delete() return removed_items
def test_mini_transaction_delete_remainder(self, remainder_check): remainder_check.return_value = 1 del_rec_lim_key = "DELETE_CYCLE_RECORD_LIMIT" del_cycle_try_key = "DELETE_CYCLE_MAX_RETRY" dcrl = os.getenv(del_rec_lim_key, None) dcmr = os.getenv(del_cycle_try_key, None) os.environ[del_rec_lim_key] = "3" os.environ[del_cycle_try_key] = "1" for id in range(110, 120): Sources.objects.create(source_id=-id, offset=1, authentication={}) src_query = Sources.objects.filter(source_id__lt=-109) start_count = src_query.count() del_count, remainder = mini_transaction_delete(src_query) end_count = src_query.count() if dcrl is None: del os.environ[del_rec_lim_key] else: os.environ[del_rec_lim_key] = dcrl if dcmr is None: del os.environ[del_cycle_try_key] else: os.environ[del_cycle_try_key] = dcmr self.assertEqual(start_count, 10) self.assertEqual(remainder, 1) self.assertEqual( end_count, 0 ) # Based on how this test is constructed, the records would be deleted anyway.
def purge_expired_line_item(self, expired_date, provider_uuid=None, simulate=False): """Remove raw line item report data with a billing start period before specified date. Args: expired_date (datetime.datetime): The cutoff date for removing data. provider_uuid (uuid): The DB id of the provider to purge data for. simulate (bool): Whether to simluate the removal. Returns: ([{}]) List of dictionaries containing 'account_payer_id' and 'billing_period_start' """ LOG.info("Calling purge_expired_line_item for gcp") if not isinstance(expired_date, datetime): err = "Parameter expired_date must be a datetime.datetime object." raise GCPReportDBCleanerError(err) with GCPReportDBAccessor(self._schema) as accessor: removed_items = [] if provider_uuid is not None: bill_objects = accessor.get_bill_query_before_date( expired_date, provider_uuid) else: bill_objects = accessor.get_bill_query_before_date( expired_date) with schema_context(self._schema): for bill in bill_objects.all(): bill_id = bill.id removed_provider_uuid = bill.provider_id removed_billing_period_start = bill.billing_period_start if not simulate: lineitem_query = accessor.get_lineitem_query_for_billid( bill_id) del_count, remainder = mini_transaction_delete( lineitem_query) LOG.info( "Removing %s cost entry line items for bill id %s", del_count, bill_id) LOG.info( "Line item data removed for Provider ID: %s with billing period: %s", removed_provider_uuid, removed_billing_period_start, ) removed_items.append({ "removed_provider_uuid": removed_provider_uuid, "billing_period_start": str(removed_billing_period_start), }) return removed_items
def test_mini_transaction_delete_base(self): for id in range(100, 110): Sources.objects.create(source_id=-id, offset=1, authentication={}) src_query = Sources.objects.filter(source_id__lt=-99) start_count = src_query.count() self.assertEqual(start_count, 10) del_count, remainder = mini_transaction_delete(src_query) end_count = src_query.count() self.assertEqual(del_count, start_count) self.assertEqual(remainder, 0) self.assertEqual(end_count, 0)
def purge_expired_line_item(self, expired_date, provider_uuid=None, simulate=False): """Remove raw line item report data with a billing start period before specified date. Args: expired_date (datetime.datetime): The cutoff date for removing data. provider_uuid (uuid): The DB id of the provider to purge data for. simulate (bool): Whether to simluate the removal. Returns: ([{}]) List of dictionaries containing 'usage_period_id' and 'interval_start' """ LOG.info("Calling purge_expired_line_item for ocp") if not isinstance(expired_date, datetime): err = "Parameter expired_date must be a datetime.datetime object." raise OCPReportDBCleanerError(err) with OCPReportDBAccessor(self._schema) as accessor: removed_items = [] if provider_uuid is not None: usage_period_objs = accessor.get_usage_period_on_or_before_date( expired_date, provider_uuid) else: usage_period_objs = accessor.get_usage_period_on_or_before_date( expired_date) with schema_context(self._schema): for usage_period in usage_period_objs.all(): report_period_id = usage_period.id removed_usage_start_period = usage_period.report_period_start if not simulate: item_query = accessor.get_item_query_report_period_id( report_period_id) qty, remainder = mini_transaction_delete(item_query) LOG.info( "Removing %s usage period line items for usage period id %s", qty, report_period_id) LOG.info( "Line item data removed for usage period ID: %s with interval start: %s", report_period_id, removed_usage_start_period, ) removed_items.append({ "usage_period_id": report_period_id, "interval_start": str(removed_usage_start_period) }) return removed_items
def delete_line_item_daily_summary_entries_for_date_range( self, source_uuid, start_date, end_date): msg = f"Deleting records from {self.line_item_daily_summary_table} from {start_date} to {end_date}" LOG.info(msg) select_query = self.line_item_daily_summary_table.objects.filter( source_uuid=source_uuid, usage_start__gte=start_date, usage_start__lte=end_date) with schema_context(self.schema): count, _ = mini_transaction_delete(select_query) msg = f"Deleted {count} records from {self.line_item_daily_summary_table}" LOG.info(msg)
def _delete_line_items(self, db_accessor, is_finalized=None): """Delete stale data for the report being processed, if necessary.""" if not self._manifest_id: return False if is_finalized is None: is_finalized = False is_full_month = self._should_process_full_month() with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(self._manifest_id) num_processed_files = manifest_accessor.number_of_files_processed(self._manifest_id) if num_processed_files != 0: return False # Override the bill date to correspond with the manifest bill_date = manifest.billing_period_start_datetime.date() provider_uuid = manifest.provider_id date_filter = self.get_date_column_filter() with db_accessor(self._schema) as accessor: bills = accessor.get_cost_entry_bills_query_by_provider(provider_uuid) bills = bills.filter(billing_period_start=bill_date).all() with schema_context(self._schema): for bill in bills: line_item_query = accessor.get_lineitem_query_for_billid(bill.id) delete_date = bill_date if not is_finalized and not is_full_month: delete_date = self.data_cutoff_date # This means we are processing a mid-month update # and only need to delete a small window of data line_item_query = line_item_query.filter(**date_filter) log_statement = ( f"Deleting data for:\n" f" schema_name: {self._schema}\n" f" provider_uuid: {provider_uuid}\n" f" bill date: {str(bill_date)}\n" f" bill ID: {bill.id}\n" f" on or after {delete_date}." ) LOG.info(log_statement) del_count, remainder = mini_transaction_delete(line_item_query) LOG.info(f"Deleted {del_count} records for bill id {bill.id}") return True