def test_purge_expired_report_data_for_provider(self): """Test that the provider_uuid deletes all data for the provider.""" bill_table_name = AZURE_REPORT_TABLE_MAP["bill"] line_item_table_name = AZURE_REPORT_TABLE_MAP["line_item"] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): first_bill = ( self.accessor._get_db_obj_query(bill_table_name) .filter(provider_id=self.azure_provider_uuid) .order_by("-billing_period_start") .first() ) expected_count = ( self.accessor._get_db_obj_query(bill_table_name).filter(provider_id=self.azure_provider_uuid).count() ) self.assertIsNotNone(self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNotNone(self.accessor._get_db_obj_query(line_item_table_name).first()) removed_data = cleaner.purge_expired_report_data(provider_uuid=self.azure_provider_uuid) self.assertEqual(len(removed_data), expected_count) self.assertEqual(removed_data[0].get("provider_uuid"), first_bill.provider_id) self.assertIn( str(first_bill.billing_period_start), [entry.get("billing_period_start") for entry in removed_data] ) with schema_context(self.schema): self.assertIsNone(self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNone(self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_on_date(self): """Test to remove report data on a provided date.""" bill_table_name = AZURE_REPORT_TABLE_MAP['bill'] line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item'] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): # Verify that data is cleared for a cutoff date == billing_period_start first_bill = self.accessor._get_db_obj_query( bill_table_name).first() cutoff_date = first_bill.billing_period_start self.assertIsNotNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNotNone( self.accessor._get_db_obj_query(line_item_table_name).first()) removed_data = cleaner.purge_expired_report_data(cutoff_date) self.assertEqual(len(removed_data), 1) self.assertEqual(removed_data[0].get('provider_uuid'), first_bill.provider_id) self.assertEqual( removed_data[0].get('billing_period_start'), str(first_bill.billing_period_start), ) with schema_context(self.schema): self.assertIsNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNone( self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_after_date(self): """Test to remove report data after a provided date.""" bill_table_name = AZURE_REPORT_TABLE_MAP["bill"] line_item_table_name = AZURE_REPORT_TABLE_MAP["line_item"] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): # Verify that data is cleared for a cutoff date > billing_period_start first_bill = self.accessor._get_db_obj_query(bill_table_name).order_by("-billing_period_start").first() cutoff_date = first_bill.billing_period_start later_date = cutoff_date + relativedelta.relativedelta(months=+1) later_cutoff = later_date.replace(month=later_date.month, day=15) expected_count = ( self.accessor._get_db_obj_query(bill_table_name).filter(billing_period_start__lte=later_cutoff).count() ) self.assertIsNotNone(self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNotNone(self.accessor._get_db_obj_query(line_item_table_name).first()) removed_data = cleaner.purge_expired_report_data(later_cutoff) self.assertEqual(len(removed_data), expected_count) self.assertEqual(removed_data[0].get("provider_uuid"), first_bill.provider_id) self.assertIn( str(first_bill.billing_period_start), [entry.get("billing_period_start") for entry in removed_data] ) with schema_context(self.schema): self.assertIsNone(self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNone(self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_for_provider(self): """Test that the provider_uuid deletes all data for the provider.""" bill_table_name = AZURE_REPORT_TABLE_MAP['bill'] line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item'] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): first_bill = self.accessor._get_db_obj_query( bill_table_name).first() self.assertIsNotNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNotNone( self.accessor._get_db_obj_query(line_item_table_name).first()) removed_data = cleaner.purge_expired_report_data( provider_uuid=self.azure_provider_uuid) self.assertEqual(len(removed_data), 1) self.assertEqual(removed_data[0].get('provider_uuid'), first_bill.provider_id) self.assertEqual( removed_data[0].get('billing_period_start'), str(first_bill.billing_period_start), ) with schema_context(self.schema): self.assertIsNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNone( self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_both_args(self): """Test that the provider_uuid deletes all data for the provider.""" now = datetime.datetime.utcnow() cleaner = AzureReportDBCleaner(self.schema) with self.assertRaises(AzureReportDBCleanerError): cleaner.purge_expired_report_data( expired_date=now, provider_uuid=self.azure_provider_uuid)
def test_purge_expired_report_data_after_date(self): """Test to remove report data after a provided date.""" bill_table_name = AZURE_REPORT_TABLE_MAP['bill'] line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item'] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): # Verify that data is cleared for a cutoff date > billing_period_start first_bill = self.accessor._get_db_obj_query( bill_table_name).first() cutoff_date = first_bill.billing_period_start later_date = cutoff_date + relativedelta.relativedelta(months=+1) later_cutoff = later_date.replace(month=later_date.month, day=15) self.assertIsNotNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNotNone( self.accessor._get_db_obj_query(line_item_table_name).first()) removed_data = cleaner.purge_expired_report_data(later_cutoff) self.assertEqual(len(removed_data), 1) self.assertEqual(removed_data[0].get('provider_id'), first_bill.provider_id) self.assertEqual( removed_data[0].get('billing_period_start'), str(first_bill.billing_period_start), ) with schema_context(self.schema): self.assertIsNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNone( self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_before_date(self): """Test to remove report data before a provided date.""" bill_table_name = AZURE_REPORT_TABLE_MAP['bill'] line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item'] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): # Verify that data is not cleared for a cutoff date < billing_period_start first_bill = self.accessor._get_db_obj_query( bill_table_name).first() cutoff_date = first_bill.billing_period_start earlier_cutoff = cutoff_date.replace( day=15) + relativedelta.relativedelta(months=-1) self.assertIsNotNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNotNone( self.accessor._get_db_obj_query(line_item_table_name).first()) removed_data = cleaner.purge_expired_report_data(earlier_cutoff) self.assertEqual(len(removed_data), 0) with schema_context(self.schema): self.assertIsNotNone( self.accessor._get_db_obj_query(bill_table_name).first()) self.assertIsNotNone( self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_correct_expired_report_data_for_provider(self): """Test that the provider_uuid deletes all data for the provider when another provider exists.""" # add another line item with different provider_uuid azure_credentials = { 'subscription_id': '11111111-f248-4ad7-bfb1-9a4cff600e1d', 'tenant_id': '11111111-228a-4aee-a215-3a768cdd0105', 'client_id': '11111111-0f78-42bb-b8e5-90144a025191', 'client_secret': 'notsosecretcode', } azure_data_source = { 'resource_group': 'resourcegroup2', 'storage_account': 'storageaccount2', } test_provider_uuid = '11111111-d05f-488c-a6d9-c2a6f3ee02bb' azure_auth = ProviderAuthentication.objects.create(credentials=azure_credentials) azure_auth.save() azure_billing_source = ProviderBillingSource.objects.create(data_source=azure_data_source) azure_billing_source.save() azure_provider = Provider.objects.create( uuid=test_provider_uuid, name='Test Provider', type='AZURE', authentication=azure_auth, billing_source=azure_billing_source, customer=self.customer, setup_complete=False, ) azure_provider.save() bill_id = self.creator.create_azure_cost_entry_bill(provider_uuid=azure_provider.uuid) product_id = self.creator.create_azure_cost_entry_product(provider_uuid=azure_provider.uuid) meter_id = self.creator.create_azure_meter(provider_uuid=azure_provider.uuid) self.creator.create_azure_cost_entry_line_item(bill_id, product_id, meter_id) bill_table_name = AZURE_REPORT_TABLE_MAP['bill'] line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item'] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): # Verify that data is cleared for a cutoff date == billing_period_start first_bill = self.accessor._get_db_obj_query(bill_table_name).first() self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).all().count(), 2) self.assertEqual(self.accessor._get_db_obj_query(line_item_table_name).all().count(), 2) removed_data = cleaner.purge_expired_report_data(provider_uuid=self.azure_provider_uuid) self.assertEqual(len(removed_data), 1) self.assertEqual(removed_data[0].get('provider_uuid'), first_bill.provider_id) self.assertEqual( removed_data[0].get('billing_period_start'), str(first_bill.billing_period_start), ) with schema_context(self.schema): self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).all().count(), 1) self.assertEqual(self.accessor._get_db_obj_query(line_item_table_name).count(), 1)
class AzureReportDBCleanerTest(MasuTestCase): """Test Cases for the AzureReportChargeUpdater object.""" def setUp(self): """Set up each test.""" super().setUp() self.cleaner = AzureReportDBCleaner( schema=self.schema) def test_purge_expired_report_data(self): """Test to verify that expired Azure report data is cleared.""" self.cleaner.purge_expired_report_data()
def _set_cleaner(self): """ Create the expired report data object. Object is specific to the report provider. Args: None Returns: (Object) : Provider-specific report cleaner """ if self._provider in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL): return AWSReportDBCleaner(self._schema) if self._provider in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL): return AzureReportDBCleaner(self._schema) if self._provider in (Provider.PROVIDER_OCP, ): return OCPReportDBCleaner(self._schema) return None
def test_purge_correct_expired_report_data_for_provider(self): """Test that the provider_id deletes all data for the provider when another provider exists.""" # add another line item with different provider_id bill_id = self.creator.create_azure_cost_entry_bill( provider_id=self.azure_provider_id + 1) product_id = self.creator.create_azure_cost_entry_product() meter_id = self.creator.create_azure_meter() self.creator.create_azure_cost_entry_line_item(bill_id, product_id, meter_id) bill_table_name = AZURE_REPORT_TABLE_MAP['bill'] line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item'] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): # Verify that data is cleared for a cutoff date == billing_period_start first_bill = self.accessor._get_db_obj_query( bill_table_name).first() self.assertEqual( self.accessor._get_db_obj_query(bill_table_name).all().count(), 2) self.assertEqual( self.accessor._get_db_obj_query( line_item_table_name).all().count(), 2) removed_data = cleaner.purge_expired_report_data( provider_id=self.azure_provider_id) self.assertEqual(len(removed_data), 1) self.assertEqual(removed_data[0].get('provider_id'), first_bill.provider_id) self.assertEqual( removed_data[0].get('billing_period_start'), str(first_bill.billing_period_start), ) with schema_context(self.schema): self.assertEqual( self.accessor._get_db_obj_query(bill_table_name).all().count(), 1) self.assertEqual( self.accessor._get_db_obj_query(line_item_table_name).count(), 1)
def _set_cleaner(self): """ Create the expired report data object. Object is specific to the report provider. Args: None Returns: (Object) : Provider-specific report cleaner """ if self._provider in (AMAZON_WEB_SERVICES, AWS_LOCAL_SERVICE_PROVIDER): return AWSReportDBCleaner(self._schema) if self._provider in (AZURE, AZURE_LOCAL_SERVICE_PROVIDER): return AzureReportDBCleaner(self._schema) if self._provider in (OPENSHIFT_CONTAINER_PLATFORM, ): return OCPReportDBCleaner(self._schema) return None
def test_purge_expired_report_data_no_args(self): """Test that the provider_uuid deletes all data for the provider.""" cleaner = AzureReportDBCleaner(self.schema) with self.assertRaises(AzureReportDBCleanerError): cleaner.purge_expired_report_data()
def test_purge_correct_expired_report_data_for_provider(self): """Test that the provider_uuid deletes all data for the provider when another provider exists.""" # add another line item with different provider_uuid azure_credentials = { "subscription_id": "11111111-f248-4ad7-bfb1-9a4cff600e1d", "tenant_id": "11111111-228a-4aee-a215-3a768cdd0105", "client_id": "11111111-0f78-42bb-b8e5-90144a025191", "client_secret": "notsosecretcode", } azure_data_source = {"resource_group": "resourcegroup2", "storage_account": "storageaccount2"} test_provider_uuid = "11111111-d05f-488c-a6d9-c2a6f3ee02bb" azure_auth = ProviderAuthentication.objects.create(credentials=azure_credentials) azure_auth.save() azure_billing_source = ProviderBillingSource.objects.create(data_source=azure_data_source) azure_billing_source.save() with patch("masu.celery.tasks.check_report_updates"): azure_provider = Provider.objects.create( uuid=test_provider_uuid, name="Test Provider", type=Provider.PROVIDER_AZURE_LOCAL, authentication=azure_auth, billing_source=azure_billing_source, customer=self.customer, setup_complete=False, ) azure_provider.save() bill_id = self.creator.create_azure_cost_entry_bill(provider_uuid=azure_provider.uuid) product_id = self.creator.create_azure_cost_entry_product(provider_uuid=azure_provider.uuid) meter_id = self.creator.create_azure_meter(provider_uuid=azure_provider.uuid) self.creator.create_azure_cost_entry_line_item(bill_id, product_id, meter_id) bill_table_name = AZURE_REPORT_TABLE_MAP["bill"] line_item_table_name = AZURE_REPORT_TABLE_MAP["line_item"] cleaner = AzureReportDBCleaner(self.schema) with schema_context(self.schema): # Verify that data is cleared for a cutoff date == billing_period_start first_bill = ( self.accessor._get_db_obj_query(bill_table_name) .filter(provider_id=self.azure_provider_uuid) .order_by("-billing_period_start") .first() ) expected_count = ( self.accessor._get_db_obj_query(bill_table_name).filter(provider_id=self.azure_provider_uuid).count() ) expected_line_item_count = ( self.accessor._get_db_obj_query(line_item_table_name) .filter(cost_entry_bill__provider_id=self.azure_provider_uuid) .count() ) self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).count(), expected_count + 1) self.assertEqual( self.accessor._get_db_obj_query(line_item_table_name).count(), expected_line_item_count + 1 ) removed_data = cleaner.purge_expired_report_data(provider_uuid=self.azure_provider_uuid) self.assertEqual(len(removed_data), expected_count) self.assertEqual(removed_data[0].get("provider_uuid"), first_bill.provider_id) self.assertIn( str(first_bill.billing_period_start), [entry.get("billing_period_start") for entry in removed_data] ) with schema_context(self.schema): self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).all().count(), 1) self.assertEqual(self.accessor._get_db_obj_query(line_item_table_name).count(), 1)
def setUp(self): """Set up each test.""" super().setUp() self.cleaner = AzureReportDBCleaner( schema=self.schema)
def test_drop_report_partitions(self): """Test that cleaner drops report line item daily summary parititons""" provider = Provider.objects.filter( type__in=[Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL ]).first() self.assertIsNotNone(provider) partitioned_table_name = AZURE_REPORT_TABLE_MAP[ "line_item_daily_summary"] report_period_table_name = AZURE_REPORT_TABLE_MAP["bill"] partitioned_table_model = None report_period_model = None for model in django.apps.apps.get_models(): if model._meta.db_table == partitioned_table_name: partitioned_table_model = model elif model._meta.db_table == report_period_table_name: report_period_model = model if partitioned_table_model and report_period_model: break self.assertIsNotNone(partitioned_table_model) self.assertIsNotNone(report_period_model) with schema_context(self.schema): test_part = PartitionedTable( schema_name=self.schema, table_name=f"{partitioned_table_name}_2016_01", partition_of_table_name=partitioned_table_name, partition_type=PartitionedTable.RANGE, partition_col="usage_start", partition_parameters={ "default": False, "from": "2016-01-01", "to": "2016-02-01" }, active=True, ) test_part.save() self.assertTrue(table_exists(self.schema, test_part.table_name)) report_period_start = datetime.datetime(2016, 1, 1, tzinfo=pytz.UTC) report_period_end = datetime.datetime(2016, 1, 31, tzinfo=pytz.UTC) report_period = report_period_model( billing_period_start=report_period_start, billing_period_end=report_period_end, provider_id=provider.uuid, ) report_period.save() usage_date = datetime.date(2016, 1, 12) lids_rec = partitioned_table_model( usage_start=usage_date, usage_end=usage_date, cost_entry_bill_id=report_period.id, uuid=uuid.uuid4()) lids_rec.save() cutoff_date = datetime.datetime(2016, 12, 31, tzinfo=pytz.UTC) cleaner = AzureReportDBCleaner(self.schema) removed_data = cleaner.purge_expired_report_data(cutoff_date, simulate=False) self.assertEqual(len(removed_data), 1) self.assertFalse(table_exists(self.schema, test_part.table_name))