def test_delete_expired_cost_usage_report_manifest(self):
"""
Test that expired CostUsageReportManifests are removed.
This test inserts CostUsageReportManifest objects,
And then deletes CostUsageReportManifest objects older than
the calculated expiration_date.
"""
provider_type_dict = {
Provider.PROVIDER_AWS_LOCAL: self.aws_provider_uuid,
Provider.PROVIDER_AZURE_LOCAL: self.azure_provider_uuid,
Provider.PROVIDER_OCP: self.ocp_provider_uuid,
}
for provider_type in provider_type_dict:
remover = ExpiredDataRemover(self.schema, provider_type)
expiration_date = remover._calculate_expiration_date()
current_month = datetime.today().replace(day=1)
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
dates = [current_month, day_before_cutoff, expiration_date]
uuids = []
uuids_to_be_deleted = []
for date in dates:
manifest_creation_datetime = current_month
manifest_updated_datetime = manifest_creation_datetime + relativedelta.relativedelta(
days=2)
uuid = uuid4()
data = {
"assembly_id": uuid,
"manifest_creation_datetime": manifest_creation_datetime,
"manifest_updated_datetime": manifest_updated_datetime,
"billing_period_start_datetime": date,
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": provider_type_dict[provider_type],
}
uuids.append(uuid)
if date == day_before_cutoff:
uuids_to_be_deleted.append(uuid)
manifest_entry = CostUsageReportManifest(**data)
manifest_entry.save()
remover.remove()
for uuid in uuids:
record_count = CostUsageReportManifest.objects.filter(
assembly_id=uuid).count()
if uuid in uuids_to_be_deleted:
self.assertEqual(0, record_count)
else:
self.assertEqual(1, record_count)
def test_simulate_delete_expired_cost_usage_report_manifest_by_provider_uuid(
self):
"""
Test simulating the deletion of expired CostUsageReportManifests.
using remove(provider_uuid)
"""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS)
expiration_date = remover._calculate_expiration_date()
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
manifest_id = 7766
day_before_cutoff_data = {
"id": manifest_id,
"assembly_id": uuid4(),
"manifest_creation_datetime": None,
"manifest_updated_datetime": None,
"billing_period_start_datetime": day_before_cutoff,
"num_total_files": 1,
"provider_id": self.aws_provider_uuid,
}
manifest_entry = CostUsageReportManifest(**day_before_cutoff_data)
manifest_entry.save()
manifest_helper = ManifestCreationHelper(
manifest_id, manifest_entry.num_total_files,
manifest_entry.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
count_records = CostUsageReportManifest.objects.count()
with self.assertLogs(logger="masu.processor.expired_data_remover",
level="INFO") as cm:
logging.disable(logging.NOTSET)
remover.remove(simulate=True, provider_uuid=self.aws_provider_uuid)
expected_log_message = "Removed CostUsageReportManifest"
# Check if the log message exists in the log output:
self.assertTrue(
any(match is not None for match in [
re.search(expected_log_message, line) for line in cm.output
]),
"Expected to see log message: " + expected_log_message +
"in the list of log messages" +
" but the list of log messages was instead : " +
str(cm.output),
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
self.assertEqual(count_records,
CostUsageReportManifest.objects.count())
def create_manifest_entry(self, billing_period_start, provider_uuid):
"""Populate a report manifest entry."""
manifest_creation_datetime = billing_period_start + relativedelta(days=random.randint(1, 27))
manifest_updated_datetime = manifest_creation_datetime + relativedelta(days=random.randint(1, 2))
data = {
'assembly_id': uuid4(),
'manifest_creation_datetime': manifest_creation_datetime,
'manifest_updated_datetime': manifest_updated_datetime,
'billing_period_start_datetime': billing_period_start,
'num_processed_files': 1,
'num_total_files': 1,
'provider_id': provider_uuid
}
manifest_entry = CostUsageReportManifest(**data)
manifest_entry.save()
return manifest_entry
def _manifest(self, start, provider_uuid):
"""Populate a report manifest entry."""
manifest_creation_datetime = start + relativedelta(
days=random.randint(1, 27))
manifest_updated_datetime = manifest_creation_datetime + relativedelta(
days=random.randint(1, 2))
data = {
"assembly_id": uuid4(),
"manifest_creation_datetime": manifest_creation_datetime,
"manifest_updated_datetime": manifest_updated_datetime,
"billing_period_start_datetime": start,
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": provider_uuid,
}
manifest_entry = CostUsageReportManifest(**data)
manifest_entry.save()
return manifest_entry
def test_remove_cost_usage_manifests_by_provider_uuid(self):
"""
Test that calling remove(provider_uuid) deletes CostUsageReportManifests.
CostUsageReportManifests that are associated with the provider_uuid
should be deleted.
"""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS_LOCAL)
expiration_date = remover._calculate_expiration_date()
current_month = datetime.today().replace(day=1)
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
fixture_records = [
(self.aws_provider_uuid,
expiration_date), # not expired, should not delete
(self.aws_provider_uuid,
day_before_cutoff), # expired, should delete
(self.azure_provider_uuid,
day_before_cutoff), # expired, should not delete
]
manifest_uuids = []
manifest_uuids_to_be_deleted = []
manifest_creation_datetime = current_month
manifest_updated_datetime = manifest_creation_datetime + relativedelta.relativedelta(
days=2)
for fixture_record in fixture_records:
manifest_uuid = uuid4()
data = {
"assembly_id": manifest_uuid,
"manifest_creation_datetime": manifest_creation_datetime,
"manifest_updated_datetime": manifest_updated_datetime,
"billing_period_start_datetime": fixture_record[1],
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": fixture_record[0],
}
CostUsageReportManifest(**data).save()
manifest_uuids.append(manifest_uuid)
if fixture_record[1] == day_before_cutoff and fixture_record[
0] == self.aws_provider_uuid:
manifest_uuids_to_be_deleted.append(manifest_uuid)
remover.remove(provider_uuid=self.aws_provider_uuid)
for manifest_uuid in manifest_uuids:
record_count = CostUsageReportManifest.objects.filter(
assembly_id=manifest_uuid).count()
if manifest_uuid in manifest_uuids_to_be_deleted:
self.assertEqual(0, record_count)
else:
self.assertEqual(1, record_count)
def test_simulate_delete_expired_cost_usage_report_manifest(self):
"""
Test that expired CostUsageReportManifest is not removed during simulation.
Test that the number of records that would have been deleted is logged.
"""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS)
expiration_date = remover._calculate_expiration_date()
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
day_before_cutoff_data = {
"assembly_id": uuid4(),
"manifest_creation_datetime": None,
"manifest_updated_datetime": None,
"billing_period_start_datetime": day_before_cutoff,
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": self.aws_provider_uuid,
}
CostUsageReportManifest(**day_before_cutoff_data).save()
with self.assertLogs(logger="masu.processor.expired_data_remover",
level="INFO") as cm:
logging.disable(logging.NOTSET)
remover.remove(simulate=True)
expected_log_message = "Removed CostUsageReportManifest"
# Check if the log message exists in the log output:
self.assertTrue(
any(match is not None for match in [
re.search(expected_log_message, line) for line in cm.output
]),
"Expected to see log message: " + expected_log_message +
"in the list of log messages" +
" but the list of log messages was instead : " +
str(cm.output),
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)