def test_remove_provider(self, mock_purge):
"""Test that remove is called with provider_uuid."""
provider_uuid = self.aws_provider_uuid
remover = ExpiredDataRemover(self.schema, AMAZON_WEB_SERVICES)
remover.remove(provider_uuid=provider_uuid)
mock_purge.assert_called_with(simulate=False,
provider_uuid=provider_uuid)
def test_remove_provider_items_only(self, mock_purge):
"""Test that remove is called with provider_uuid items only."""
provider_uuid = self.aws_provider_uuid
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS)
date = remover._calculate_expiration_date(line_items_only=True)
remover.remove(provider_uuid=provider_uuid, line_items_only=True)
mock_purge.assert_called_with(expired_date=date,
simulate=False,
provider_uuid=provider_uuid)
def test_remove_items_only_azure(self):
"""Test that remove is called with provider_uuid items only."""
azure_types = [Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL]
for az_type in azure_types:
remover = ExpiredDataRemover(self.schema, az_type)
result_no_provider = remover.remove(line_items_only=True)
self.assertIsNone(result_no_provider)
result_with_provider = remover.remove(line_items_only=True,
provider_uuid="1234")
self.assertIsNone(result_with_provider)
def test_delete_expired_cost_usage_report_manifest(self):
"""
Test that expired CostUsageReportManifests are removed.
This test inserts CostUsageReportManifest objects,
And then deletes CostUsageReportManifest objects older than
the calculated expiration_date.
"""
provider_type_dict = {
Provider.PROVIDER_AWS_LOCAL: self.aws_provider_uuid,
Provider.PROVIDER_AZURE_LOCAL: self.azure_provider_uuid,
Provider.PROVIDER_OCP: self.ocp_provider_uuid,
}
for provider_type in provider_type_dict:
remover = ExpiredDataRemover(self.schema, provider_type)
expiration_date = remover._calculate_expiration_date()
current_month = datetime.today().replace(day=1)
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
dates = [current_month, day_before_cutoff, expiration_date]
uuids = []
uuids_to_be_deleted = []
for date in dates:
manifest_creation_datetime = current_month
manifest_updated_datetime = manifest_creation_datetime + relativedelta.relativedelta(
days=2)
uuid = uuid4()
data = {
"assembly_id": uuid,
"manifest_creation_datetime": manifest_creation_datetime,
"manifest_updated_datetime": manifest_updated_datetime,
"billing_period_start_datetime": date,
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": provider_type_dict[provider_type],
}
uuids.append(uuid)
if date == day_before_cutoff:
uuids_to_be_deleted.append(uuid)
manifest_entry = CostUsageReportManifest(**data)
manifest_entry.save()
remover.remove()
for uuid in uuids:
record_count = CostUsageReportManifest.objects.filter(
assembly_id=uuid).count()
if uuid in uuids_to_be_deleted:
self.assertEqual(0, record_count)
else:
self.assertEqual(1, record_count)
def test_simulate_delete_expired_cost_usage_report_manifest_by_provider_uuid(
self):
"""
Test simulating the deletion of expired CostUsageReportManifests.
using remove(provider_uuid)
"""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS)
expiration_date = remover._calculate_expiration_date()
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
manifest_id = 7766
day_before_cutoff_data = {
"id": manifest_id,
"assembly_id": uuid4(),
"manifest_creation_datetime": None,
"manifest_updated_datetime": None,
"billing_period_start_datetime": day_before_cutoff,
"num_total_files": 1,
"provider_id": self.aws_provider_uuid,
}
manifest_entry = CostUsageReportManifest(**day_before_cutoff_data)
manifest_entry.save()
manifest_helper = ManifestCreationHelper(
manifest_id, manifest_entry.num_total_files,
manifest_entry.assembly_id)
manifest_helper.generate_test_report_files()
manifest_helper.process_all_files()
count_records = CostUsageReportManifest.objects.count()
with self.assertLogs(logger="masu.processor.expired_data_remover",
level="INFO") as cm:
logging.disable(logging.NOTSET)
remover.remove(simulate=True, provider_uuid=self.aws_provider_uuid)
expected_log_message = "Removed CostUsageReportManifest"
# Check if the log message exists in the log output:
self.assertTrue(
any(match is not None for match in [
re.search(expected_log_message, line) for line in cm.output
]),
"Expected to see log message: " + expected_log_message +
"in the list of log messages" +
" but the list of log messages was instead : " +
str(cm.output),
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
self.assertEqual(count_records,
CostUsageReportManifest.objects.count())
def test_remove_cost_usage_manifests_by_provider_uuid(self):
"""
Test that calling remove(provider_uuid) deletes CostUsageReportManifests.
CostUsageReportManifests that are associated with the provider_uuid
should be deleted.
"""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS_LOCAL)
expiration_date = remover._calculate_expiration_date()
current_month = datetime.today().replace(day=1)
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
fixture_records = [
(self.aws_provider_uuid,
expiration_date), # not expired, should not delete
(self.aws_provider_uuid,
day_before_cutoff), # expired, should delete
(self.azure_provider_uuid,
day_before_cutoff), # expired, should not delete
]
manifest_uuids = []
manifest_uuids_to_be_deleted = []
manifest_creation_datetime = current_month
manifest_updated_datetime = manifest_creation_datetime + relativedelta.relativedelta(
days=2)
for fixture_record in fixture_records:
manifest_uuid = uuid4()
data = {
"assembly_id": manifest_uuid,
"manifest_creation_datetime": manifest_creation_datetime,
"manifest_updated_datetime": manifest_updated_datetime,
"billing_period_start_datetime": fixture_record[1],
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": fixture_record[0],
}
CostUsageReportManifest(**data).save()
manifest_uuids.append(manifest_uuid)
if fixture_record[1] == day_before_cutoff and fixture_record[
0] == self.aws_provider_uuid:
manifest_uuids_to_be_deleted.append(manifest_uuid)
remover.remove(provider_uuid=self.aws_provider_uuid)
for manifest_uuid in manifest_uuids:
record_count = CostUsageReportManifest.objects.filter(
assembly_id=manifest_uuid).count()
if manifest_uuid in manifest_uuids_to_be_deleted:
self.assertEqual(0, record_count)
else:
self.assertEqual(1, record_count)
def _remove_expired_data(schema_name, provider, simulate, provider_uuid=None):
"""
Task to remove expired data.
Args:
schema_name (String) db schema name
provider (String) provider type
simulate (Boolean) Simulate report data removal
Returns:
None
"""
log_statement = (f'Remove expired data:\n'
f' schema_name: {schema_name}\n'
f' provider: {provider}\n'
f' simulate: {simulate}')
LOG.info(log_statement)
remover = ExpiredDataRemover(schema_name, provider)
removed_data = remover.remove(simulate=simulate, provider_uuid=provider_uuid)
status_msg = 'Expired Data' if simulate else 'Removed Data'
result_msg = f'{status_msg}:\n {str(removed_data)}'
LOG.info(result_msg)
def _remove_expired_data(schema_name,
provider,
simulate,
provider_uuid=None,
line_items_only=False):
"""
Task to remove expired data.
Args:
schema_name (String) db schema name
provider (String) provider type
simulate (Boolean) Simulate report data removal
Returns:
None
"""
log_statement = (f"Remove expired data:\n"
f" schema_name: {schema_name}\n"
f" provider: {provider}\n"
f" simulate: {simulate}\n"
f" line_items_only: {line_items_only}\n")
LOG.info(log_statement)
remover = ExpiredDataRemover(schema_name, provider)
removed_data = remover.remove(simulate=simulate,
provider_uuid=provider_uuid,
line_items_only=line_items_only)
if removed_data:
status_msg = "Expired Data" if simulate else "Removed Data"
result_msg = f"{status_msg}:\n {str(removed_data)}"
LOG.info(result_msg)
def test_simulate_delete_expired_cost_usage_report_manifest(self):
"""
Test that expired CostUsageReportManifest is not removed during simulation.
Test that the number of records that would have been deleted is logged.
"""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS)
expiration_date = remover._calculate_expiration_date()
day_before_cutoff = expiration_date - relativedelta.relativedelta(
days=1)
day_before_cutoff_data = {
"assembly_id": uuid4(),
"manifest_creation_datetime": None,
"manifest_updated_datetime": None,
"billing_period_start_datetime": day_before_cutoff,
"num_processed_files": 1,
"num_total_files": 1,
"provider_id": self.aws_provider_uuid,
}
CostUsageReportManifest(**day_before_cutoff_data).save()
with self.assertLogs(logger="masu.processor.expired_data_remover",
level="INFO") as cm:
logging.disable(logging.NOTSET)
remover.remove(simulate=True)
expected_log_message = "Removed CostUsageReportManifest"
# Check if the log message exists in the log output:
self.assertTrue(
any(match is not None for match in [
re.search(expected_log_message, line) for line in cm.output
]),
"Expected to see log message: " + expected_log_message +
"in the list of log messages" +
" but the list of log messages was instead : " +
str(cm.output),
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def _remove_expired_data(schema_name, simulate):
"""
Task to remove expired data.
Args:
schema_name (String) db schema name
simulate (Boolean) Simulate report data removal
Returns:
None
"""
stmt = ('Remove expired data:' ' schema_name: {},' ' simulate: {}')
log_statement = stmt.format(schema_name, simulate)
LOG.info(log_statement)
remover = ExpiredDataRemover(schema_name)
removed_data = remover.remove(simulate=simulate)
status_msg = 'Expired Data' if simulate else 'Removed Data'
result_msg = '{}: {}'.format(status_msg, str(removed_data))
LOG.info(result_msg)
def test_remove(self):
"""Test that removes the expired data based on the retention policy."""
remover = ExpiredDataRemover(self.schema, AMAZON_WEB_SERVICES)
removed_data = remover.remove()
self.assertEqual(len(removed_data), 0)
def test_remove(self):
"""Test that removes the expired data based on the retention policy."""
remover = ExpiredDataRemover('testcustomer')
removed_data = remover.remove()
self.assertEqual(len(removed_data), 0)
def test_remove(self):
"""Test that removes the expired data based on the retention policy."""
remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS)
removed_data = remover.remove()
self.assertEqual(len(removed_data), 0)
def test_remove_provider(self, mock_purge):
"""Test that remove is called with provider_id."""
provider_id = 1
remover = ExpiredDataRemover('acct10001', 'AWS')
remover.remove(provider_id=provider_id)
mock_purge.assert_called_with(simulate=False, provider_id=provider_id)
