def test_delete_line_items_use_data_cutoff_date(self, mock_should_process): """Test that only three days of data are deleted.""" mock_should_process.return_value = True today = self.date_accessor.today_with_timezone("UTC").replace( hour=0, minute=0, second=0, microsecond=0) first_of_month = today.replace(day=1) self.manifest.billing_period_start_datetime = first_of_month self.manifest.save() processor = AWSReportProcessor( schema_name=self.schema, report_path="", compression=UNCOMPRESSED, provider_uuid=self.aws_provider_uuid, manifest_id=self.manifest.id, ) # Get latest data date. bill_ids = [] with schema_context(self.schema): bills = self.accessor.get_cost_entry_bills() for key, value in bills.items(): if key[1] == DateHelper().this_month_start: bill_ids.append(value) for bill_id in bill_ids: with schema_context(self.schema): line_item_query = self.accessor.get_lineitem_query_for_billid( bill_id) undeleted_max_date = line_item_query.aggregate( max_date=Max("usage_start")) mock_should_process.return_value = False processor._delete_line_items(AWSReportDBAccessor, is_finalized=False) with schema_context(self.schema): # bills = self.accessor.get_cost_entry_bills() # for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid( bill_id) if today.day <= 3: self.assertEqual(line_item_query.count(), 0) else: max_date = line_item_query.aggregate( max_date=Max("usage_start")) self.assertLess( max_date.get("max_date").date(), processor.data_cutoff_date) self.assertLess( max_date.get("max_date").date(), undeleted_max_date.get("max_date").date()) self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_success(self): """Test that data is deleted before processing a manifest.""" manifest = CostUsageReportManifest.objects.filter( provider__uuid=self.aws_provider_uuid, billing_period_start_datetime=DateHelper().this_month_start).first( ) manifest.num_processed_files = 0 manifest.save() bill_date = manifest.billing_period_start_datetime processor = AWSReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.aws_provider_uuid, manifest_id=manifest.id, ) with schema_context(self.schema): bills = self.accessor.get_cost_entry_bills_by_date(bill_date) bill_ids = [bill.id for bill in bills] for bill_id in bill_ids: with schema_context(self.schema): before_count = self.accessor.get_lineitem_query_for_billid( bill_id).count() result = processor._delete_line_items(AWSReportDBAccessor) with schema_context(self.schema): line_item_query = self.accessor.get_lineitem_query_for_billid( bill_id) self.assertTrue(result) self.assertLess(line_item_query.count(), before_count)
def test_delete_line_items_not_first_file_in_manifest(self): """Test that data is not deleted once a file has been processed.""" manifest_helper = ManifestCreationHelper(self.manifest.id, self.manifest.num_total_files, self.manifest.assembly_id) report_file = manifest_helper.generate_one_test_file() manifest_helper.mark_report_file_as_completed(report_file) processor = AWSReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.aws_provider_uuid, manifest_id=self.manifest.id, ) processor.process() result = processor._delete_line_items(AWSReportDBAccessor) with schema_context(self.schema): bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid( bill_id) self.assertFalse(result) self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_no_manifest(self): """Test that no data is deleted without a manifest id.""" processor = AWSReportProcessor( schema_name='acct10001', report_path=self.test_report, compression=UNCOMPRESSED, provider_id=1 ) processor.process() result = processor._delete_line_items() bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id) self.assertFalse(result) self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_success(self): """Test that data is deleted before processing a manifest.""" processor = AWSReportProcessor( schema_name='acct10001', report_path=self.test_report, compression=UNCOMPRESSED, provider_id=1, manifest_id=self.manifest.id ) processor.process() result = processor._delete_line_items() bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id) self.assertTrue(result) self.assertEqual(line_item_query.count(), 0)
def test_delete_line_items_success(self): """Test that data is deleted before processing a manifest.""" processor = AWSReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.aws_provider_uuid, manifest_id=self.manifest.id, ) processor.process() result = processor._delete_line_items(AWSReportDBAccessor, self.column_map) with schema_context(self.schema): bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id) self.assertTrue(result) self.assertEqual(line_item_query.count(), 0)
def test_delete_line_items_not_first_file_in_manifest(self): """Test that data is not deleted once a file has been processed.""" self.manifest.num_processed_files = 1 self.manifest_accessor.commit() processor = AWSReportProcessor( schema_name='acct10001', report_path=self.test_report, compression=UNCOMPRESSED, provider_id=1, manifest_id=self.manifest.id ) processor.process() result = processor._delete_line_items() bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id) self.assertFalse(result) self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_not_first_file_in_manifest(self): """Test that data is not deleted once a file has been processed.""" self.manifest.num_processed_files = 1 self.manifest.save() processor = AWSReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.aws_provider_uuid, manifest_id=self.manifest.id, ) processor.process() result = processor._delete_line_items(AWSReportDBAccessor, self.column_map) with schema_context(self.schema): bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id) self.assertFalse(result) self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_use_data_cutoff_date(self, mock_should_process): """Test that only three days of data are deleted.""" mock_should_process.return_value = True today = self.date_accessor.today_with_timezone('UTC').replace( hour=0, minute=0, second=0, microsecond=0 ) first_of_month = today.replace(day=1) first_of_next_month = first_of_month + relativedelta(months=1) days_in_month = [today - relativedelta(days=i) for i in range(today.day)] self.manifest.billing_period_start_datetime = first_of_month self.manifest.save() data = [] with open(self.test_report, 'r') as f: reader = csv.DictReader(f) for row in reader: data.append(row) for row in data: row['lineItem/UsageStartDate'] = random.choice(days_in_month) row['bill/BillingPeriodStartDate'] = first_of_month row['bill/BillingPeriodEndDate'] = first_of_next_month tmp_file = '/tmp/test_delete_data_cutoff.csv' field_names = data[0].keys() with open(tmp_file, 'w') as f: writer = csv.DictWriter(f, fieldnames=field_names) writer.writeheader() writer.writerows(data) processor = AWSReportProcessor( schema_name=self.schema, report_path=tmp_file, compression=UNCOMPRESSED, provider_uuid=self.aws_provider_uuid, manifest_id=self.manifest.id, ) processor.process() # Get latest data date. with schema_context(self.schema): bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id) undeleted_max_date = line_item_query.aggregate(max_date=Max('usage_start')) mock_should_process.return_value = False processor._delete_line_items(AWSReportDBAccessor, self.column_map, is_finalized=False) with schema_context(self.schema): bills = self.accessor.get_cost_entry_bills() for bill_id in bills.values(): line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id) if today.day <= 3: self.assertEqual(line_item_query.count(), 0) else: max_date = line_item_query.aggregate(max_date=Max('usage_start')) self.assertLess(max_date.get('max_date').date(), processor.data_cutoff_date) self.assertLess( max_date.get('max_date').date(), undeleted_max_date.get('max_date').date() ) self.assertNotEqual(line_item_query.count(), 0)