def test_delete_line_items_use_data_cutoff_date(self, mock_should_process):
"""Test that only three days of data are deleted."""
mock_should_process.return_value = True
today = self.date_accessor.today_with_timezone("UTC").replace(
hour=0, minute=0, second=0, microsecond=0)
first_of_month = today.replace(day=1)
self.manifest.billing_period_start_datetime = first_of_month
self.manifest.save()
processor = AWSReportProcessor(
schema_name=self.schema,
report_path="",
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
manifest_id=self.manifest.id,
)
# Get latest data date.
bill_ids = []
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills()
for key, value in bills.items():
if key[1] == DateHelper().this_month_start:
bill_ids.append(value)
for bill_id in bill_ids:
with schema_context(self.schema):
line_item_query = self.accessor.get_lineitem_query_for_billid(
bill_id)
undeleted_max_date = line_item_query.aggregate(
max_date=Max("usage_start"))
mock_should_process.return_value = False
processor._delete_line_items(AWSReportDBAccessor,
is_finalized=False)
with schema_context(self.schema):
# bills = self.accessor.get_cost_entry_bills()
# for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(
bill_id)
if today.day <= 3:
self.assertEqual(line_item_query.count(), 0)
else:
max_date = line_item_query.aggregate(
max_date=Max("usage_start"))
self.assertLess(
max_date.get("max_date").date(),
processor.data_cutoff_date)
self.assertLess(
max_date.get("max_date").date(),
undeleted_max_date.get("max_date").date())
self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_success(self):
"""Test that data is deleted before processing a manifest."""
manifest = CostUsageReportManifest.objects.filter(
provider__uuid=self.aws_provider_uuid,
billing_period_start_datetime=DateHelper().this_month_start).first(
)
manifest.num_processed_files = 0
manifest.save()
bill_date = manifest.billing_period_start_datetime
processor = AWSReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
manifest_id=manifest.id,
)
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills_by_date(bill_date)
bill_ids = [bill.id for bill in bills]
for bill_id in bill_ids:
with schema_context(self.schema):
before_count = self.accessor.get_lineitem_query_for_billid(
bill_id).count()
result = processor._delete_line_items(AWSReportDBAccessor)
with schema_context(self.schema):
line_item_query = self.accessor.get_lineitem_query_for_billid(
bill_id)
self.assertTrue(result)
self.assertLess(line_item_query.count(), before_count)
def test_delete_line_items_not_first_file_in_manifest(self):
"""Test that data is not deleted once a file has been processed."""
manifest_helper = ManifestCreationHelper(self.manifest.id,
self.manifest.num_total_files,
self.manifest.assembly_id)
report_file = manifest_helper.generate_one_test_file()
manifest_helper.mark_report_file_as_completed(report_file)
processor = AWSReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
manifest_id=self.manifest.id,
)
processor.process()
result = processor._delete_line_items(AWSReportDBAccessor)
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(
bill_id)
self.assertFalse(result)
self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_no_manifest(self):
"""Test that no data is deleted without a manifest id."""
processor = AWSReportProcessor(
schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1
)
processor.process()
result = processor._delete_line_items()
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id)
self.assertFalse(result)
self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_success(self):
"""Test that data is deleted before processing a manifest."""
processor = AWSReportProcessor(
schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1,
manifest_id=self.manifest.id
)
processor.process()
result = processor._delete_line_items()
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id)
self.assertTrue(result)
self.assertEqual(line_item_query.count(), 0)
def test_delete_line_items_success(self):
"""Test that data is deleted before processing a manifest."""
processor = AWSReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
manifest_id=self.manifest.id,
)
processor.process()
result = processor._delete_line_items(AWSReportDBAccessor, self.column_map)
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id)
self.assertTrue(result)
self.assertEqual(line_item_query.count(), 0)
def test_delete_line_items_not_first_file_in_manifest(self):
"""Test that data is not deleted once a file has been processed."""
self.manifest.num_processed_files = 1
self.manifest_accessor.commit()
processor = AWSReportProcessor(
schema_name='acct10001',
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_id=1,
manifest_id=self.manifest.id
)
processor.process()
result = processor._delete_line_items()
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id)
self.assertFalse(result)
self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_not_first_file_in_manifest(self):
"""Test that data is not deleted once a file has been processed."""
self.manifest.num_processed_files = 1
self.manifest.save()
processor = AWSReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
manifest_id=self.manifest.id,
)
processor.process()
result = processor._delete_line_items(AWSReportDBAccessor, self.column_map)
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id)
self.assertFalse(result)
self.assertNotEqual(line_item_query.count(), 0)
def test_delete_line_items_use_data_cutoff_date(self, mock_should_process):
"""Test that only three days of data are deleted."""
mock_should_process.return_value = True
today = self.date_accessor.today_with_timezone('UTC').replace(
hour=0, minute=0, second=0, microsecond=0
)
first_of_month = today.replace(day=1)
first_of_next_month = first_of_month + relativedelta(months=1)
days_in_month = [today - relativedelta(days=i) for i in range(today.day)]
self.manifest.billing_period_start_datetime = first_of_month
self.manifest.save()
data = []
with open(self.test_report, 'r') as f:
reader = csv.DictReader(f)
for row in reader:
data.append(row)
for row in data:
row['lineItem/UsageStartDate'] = random.choice(days_in_month)
row['bill/BillingPeriodStartDate'] = first_of_month
row['bill/BillingPeriodEndDate'] = first_of_next_month
tmp_file = '/tmp/test_delete_data_cutoff.csv'
field_names = data[0].keys()
with open(tmp_file, 'w') as f:
writer = csv.DictWriter(f, fieldnames=field_names)
writer.writeheader()
writer.writerows(data)
processor = AWSReportProcessor(
schema_name=self.schema,
report_path=tmp_file,
compression=UNCOMPRESSED,
provider_uuid=self.aws_provider_uuid,
manifest_id=self.manifest.id,
)
processor.process()
# Get latest data date.
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id)
undeleted_max_date = line_item_query.aggregate(max_date=Max('usage_start'))
mock_should_process.return_value = False
processor._delete_line_items(AWSReportDBAccessor, self.column_map, is_finalized=False)
with schema_context(self.schema):
bills = self.accessor.get_cost_entry_bills()
for bill_id in bills.values():
line_item_query = self.accessor.get_lineitem_query_for_billid(bill_id)
if today.day <= 3:
self.assertEqual(line_item_query.count(), 0)
else:
max_date = line_item_query.aggregate(max_date=Max('usage_start'))
self.assertLess(max_date.get('max_date').date(), processor.data_cutoff_date)
self.assertLess(
max_date.get('max_date').date(), undeleted_max_date.get('max_date').date()
)
self.assertNotEqual(line_item_query.count(), 0)
