def test_remove_provider(self, mock_purge): """Test that remove is called with provider_uuid.""" provider_uuid = self.aws_provider_uuid remover = ExpiredDataRemover(self.schema, AMAZON_WEB_SERVICES) remover.remove(provider_uuid=provider_uuid) mock_purge.assert_called_with(simulate=False, provider_uuid=provider_uuid)
def test_remove_provider_items_only(self, mock_purge): """Test that remove is called with provider_uuid items only.""" provider_uuid = self.aws_provider_uuid remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS) date = remover._calculate_expiration_date(line_items_only=True) remover.remove(provider_uuid=provider_uuid, line_items_only=True) mock_purge.assert_called_with(expired_date=date, simulate=False, provider_uuid=provider_uuid)
def test_remove_items_only_azure(self): """Test that remove is called with provider_uuid items only.""" azure_types = [Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL] for az_type in azure_types: remover = ExpiredDataRemover(self.schema, az_type) result_no_provider = remover.remove(line_items_only=True) self.assertIsNone(result_no_provider) result_with_provider = remover.remove(line_items_only=True, provider_uuid="1234") self.assertIsNone(result_with_provider)
def test_delete_expired_cost_usage_report_manifest(self): """ Test that expired CostUsageReportManifests are removed. This test inserts CostUsageReportManifest objects, And then deletes CostUsageReportManifest objects older than the calculated expiration_date. """ provider_type_dict = { Provider.PROVIDER_AWS_LOCAL: self.aws_provider_uuid, Provider.PROVIDER_AZURE_LOCAL: self.azure_provider_uuid, Provider.PROVIDER_OCP: self.ocp_provider_uuid, } for provider_type in provider_type_dict: remover = ExpiredDataRemover(self.schema, provider_type) expiration_date = remover._calculate_expiration_date() current_month = datetime.today().replace(day=1) day_before_cutoff = expiration_date - relativedelta.relativedelta( days=1) dates = [current_month, day_before_cutoff, expiration_date] uuids = [] uuids_to_be_deleted = [] for date in dates: manifest_creation_datetime = current_month manifest_updated_datetime = manifest_creation_datetime + relativedelta.relativedelta( days=2) uuid = uuid4() data = { "assembly_id": uuid, "manifest_creation_datetime": manifest_creation_datetime, "manifest_updated_datetime": manifest_updated_datetime, "billing_period_start_datetime": date, "num_processed_files": 1, "num_total_files": 1, "provider_id": provider_type_dict[provider_type], } uuids.append(uuid) if date == day_before_cutoff: uuids_to_be_deleted.append(uuid) manifest_entry = CostUsageReportManifest(**data) manifest_entry.save() remover.remove() for uuid in uuids: record_count = CostUsageReportManifest.objects.filter( assembly_id=uuid).count() if uuid in uuids_to_be_deleted: self.assertEqual(0, record_count) else: self.assertEqual(1, record_count)
def test_simulate_delete_expired_cost_usage_report_manifest_by_provider_uuid( self): """ Test simulating the deletion of expired CostUsageReportManifests. using remove(provider_uuid) """ remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS) expiration_date = remover._calculate_expiration_date() day_before_cutoff = expiration_date - relativedelta.relativedelta( days=1) manifest_id = 7766 day_before_cutoff_data = { "id": manifest_id, "assembly_id": uuid4(), "manifest_creation_datetime": None, "manifest_updated_datetime": None, "billing_period_start_datetime": day_before_cutoff, "num_total_files": 1, "provider_id": self.aws_provider_uuid, } manifest_entry = CostUsageReportManifest(**day_before_cutoff_data) manifest_entry.save() manifest_helper = ManifestCreationHelper( manifest_id, manifest_entry.num_total_files, manifest_entry.assembly_id) manifest_helper.generate_test_report_files() manifest_helper.process_all_files() count_records = CostUsageReportManifest.objects.count() with self.assertLogs(logger="masu.processor.expired_data_remover", level="INFO") as cm: logging.disable(logging.NOTSET) remover.remove(simulate=True, provider_uuid=self.aws_provider_uuid) expected_log_message = "Removed CostUsageReportManifest" # Check if the log message exists in the log output: self.assertTrue( any(match is not None for match in [ re.search(expected_log_message, line) for line in cm.output ]), "Expected to see log message: " + expected_log_message + "in the list of log messages" + " but the list of log messages was instead : " + str(cm.output), ) # Re-enable log suppression logging.disable(logging.CRITICAL) self.assertEqual(count_records, CostUsageReportManifest.objects.count())
def test_remove_cost_usage_manifests_by_provider_uuid(self): """ Test that calling remove(provider_uuid) deletes CostUsageReportManifests. CostUsageReportManifests that are associated with the provider_uuid should be deleted. """ remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS_LOCAL) expiration_date = remover._calculate_expiration_date() current_month = datetime.today().replace(day=1) day_before_cutoff = expiration_date - relativedelta.relativedelta( days=1) fixture_records = [ (self.aws_provider_uuid, expiration_date), # not expired, should not delete (self.aws_provider_uuid, day_before_cutoff), # expired, should delete (self.azure_provider_uuid, day_before_cutoff), # expired, should not delete ] manifest_uuids = [] manifest_uuids_to_be_deleted = [] manifest_creation_datetime = current_month manifest_updated_datetime = manifest_creation_datetime + relativedelta.relativedelta( days=2) for fixture_record in fixture_records: manifest_uuid = uuid4() data = { "assembly_id": manifest_uuid, "manifest_creation_datetime": manifest_creation_datetime, "manifest_updated_datetime": manifest_updated_datetime, "billing_period_start_datetime": fixture_record[1], "num_processed_files": 1, "num_total_files": 1, "provider_id": fixture_record[0], } CostUsageReportManifest(**data).save() manifest_uuids.append(manifest_uuid) if fixture_record[1] == day_before_cutoff and fixture_record[ 0] == self.aws_provider_uuid: manifest_uuids_to_be_deleted.append(manifest_uuid) remover.remove(provider_uuid=self.aws_provider_uuid) for manifest_uuid in manifest_uuids: record_count = CostUsageReportManifest.objects.filter( assembly_id=manifest_uuid).count() if manifest_uuid in manifest_uuids_to_be_deleted: self.assertEqual(0, record_count) else: self.assertEqual(1, record_count)
def _remove_expired_data(schema_name, provider, simulate, provider_uuid=None): """ Task to remove expired data. Args: schema_name (String) db schema name provider (String) provider type simulate (Boolean) Simulate report data removal Returns: None """ log_statement = (f'Remove expired data:\n' f' schema_name: {schema_name}\n' f' provider: {provider}\n' f' simulate: {simulate}') LOG.info(log_statement) remover = ExpiredDataRemover(schema_name, provider) removed_data = remover.remove(simulate=simulate, provider_uuid=provider_uuid) status_msg = 'Expired Data' if simulate else 'Removed Data' result_msg = f'{status_msg}:\n {str(removed_data)}' LOG.info(result_msg)
def _remove_expired_data(schema_name, provider, simulate, provider_uuid=None, line_items_only=False): """ Task to remove expired data. Args: schema_name (String) db schema name provider (String) provider type simulate (Boolean) Simulate report data removal Returns: None """ log_statement = (f"Remove expired data:\n" f" schema_name: {schema_name}\n" f" provider: {provider}\n" f" simulate: {simulate}\n" f" line_items_only: {line_items_only}\n") LOG.info(log_statement) remover = ExpiredDataRemover(schema_name, provider) removed_data = remover.remove(simulate=simulate, provider_uuid=provider_uuid, line_items_only=line_items_only) if removed_data: status_msg = "Expired Data" if simulate else "Removed Data" result_msg = f"{status_msg}:\n {str(removed_data)}" LOG.info(result_msg)
def test_simulate_delete_expired_cost_usage_report_manifest(self): """ Test that expired CostUsageReportManifest is not removed during simulation. Test that the number of records that would have been deleted is logged. """ remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS) expiration_date = remover._calculate_expiration_date() day_before_cutoff = expiration_date - relativedelta.relativedelta( days=1) day_before_cutoff_data = { "assembly_id": uuid4(), "manifest_creation_datetime": None, "manifest_updated_datetime": None, "billing_period_start_datetime": day_before_cutoff, "num_processed_files": 1, "num_total_files": 1, "provider_id": self.aws_provider_uuid, } CostUsageReportManifest(**day_before_cutoff_data).save() with self.assertLogs(logger="masu.processor.expired_data_remover", level="INFO") as cm: logging.disable(logging.NOTSET) remover.remove(simulate=True) expected_log_message = "Removed CostUsageReportManifest" # Check if the log message exists in the log output: self.assertTrue( any(match is not None for match in [ re.search(expected_log_message, line) for line in cm.output ]), "Expected to see log message: " + expected_log_message + "in the list of log messages" + " but the list of log messages was instead : " + str(cm.output), ) # Re-enable log suppression logging.disable(logging.CRITICAL)
def _remove_expired_data(schema_name, simulate): """ Task to remove expired data. Args: schema_name (String) db schema name simulate (Boolean) Simulate report data removal Returns: None """ stmt = ('Remove expired data:' ' schema_name: {},' ' simulate: {}') log_statement = stmt.format(schema_name, simulate) LOG.info(log_statement) remover = ExpiredDataRemover(schema_name) removed_data = remover.remove(simulate=simulate) status_msg = 'Expired Data' if simulate else 'Removed Data' result_msg = '{}: {}'.format(status_msg, str(removed_data)) LOG.info(result_msg)
def test_remove(self): """Test that removes the expired data based on the retention policy.""" remover = ExpiredDataRemover(self.schema, AMAZON_WEB_SERVICES) removed_data = remover.remove() self.assertEqual(len(removed_data), 0)
def test_remove(self): """Test that removes the expired data based on the retention policy.""" remover = ExpiredDataRemover('testcustomer') removed_data = remover.remove() self.assertEqual(len(removed_data), 0)
def test_remove(self): """Test that removes the expired data based on the retention policy.""" remover = ExpiredDataRemover(self.schema, Provider.PROVIDER_AWS) removed_data = remover.remove() self.assertEqual(len(removed_data), 0)
def test_remove_provider(self, mock_purge): """Test that remove is called with provider_id.""" provider_id = 1 remover = ExpiredDataRemover('acct10001', 'AWS') remover.remove(provider_id=provider_id) mock_purge.assert_called_with(simulate=False, provider_id=provider_id)