def test_purge_expired_report_data_for_provider(self):
"""Test that the provider_uuid deletes all data for the provider."""
bill_table_name = AZURE_REPORT_TABLE_MAP["bill"]
line_item_table_name = AZURE_REPORT_TABLE_MAP["line_item"]
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
first_bill = (
self.accessor._get_db_obj_query(bill_table_name)
.filter(provider_id=self.azure_provider_uuid)
.order_by("-billing_period_start")
.first()
)
expected_count = (
self.accessor._get_db_obj_query(bill_table_name).filter(provider_id=self.azure_provider_uuid).count()
)
self.assertIsNotNone(self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNotNone(self.accessor._get_db_obj_query(line_item_table_name).first())
removed_data = cleaner.purge_expired_report_data(provider_uuid=self.azure_provider_uuid)
self.assertEqual(len(removed_data), expected_count)
self.assertEqual(removed_data[0].get("provider_uuid"), first_bill.provider_id)
self.assertIn(
str(first_bill.billing_period_start), [entry.get("billing_period_start") for entry in removed_data]
)
with schema_context(self.schema):
self.assertIsNone(self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNone(self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_on_date(self):
"""Test to remove report data on a provided date."""
bill_table_name = AZURE_REPORT_TABLE_MAP['bill']
line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item']
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
# Verify that data is cleared for a cutoff date == billing_period_start
first_bill = self.accessor._get_db_obj_query(
bill_table_name).first()
cutoff_date = first_bill.billing_period_start
self.assertIsNotNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNotNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
removed_data = cleaner.purge_expired_report_data(cutoff_date)
self.assertEqual(len(removed_data), 1)
self.assertEqual(removed_data[0].get('provider_uuid'),
first_bill.provider_id)
self.assertEqual(
removed_data[0].get('billing_period_start'),
str(first_bill.billing_period_start),
)
with schema_context(self.schema):
self.assertIsNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_after_date(self):
"""Test to remove report data after a provided date."""
bill_table_name = AZURE_REPORT_TABLE_MAP["bill"]
line_item_table_name = AZURE_REPORT_TABLE_MAP["line_item"]
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
# Verify that data is cleared for a cutoff date > billing_period_start
first_bill = self.accessor._get_db_obj_query(bill_table_name).order_by("-billing_period_start").first()
cutoff_date = first_bill.billing_period_start
later_date = cutoff_date + relativedelta.relativedelta(months=+1)
later_cutoff = later_date.replace(month=later_date.month, day=15)
expected_count = (
self.accessor._get_db_obj_query(bill_table_name).filter(billing_period_start__lte=later_cutoff).count()
)
self.assertIsNotNone(self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNotNone(self.accessor._get_db_obj_query(line_item_table_name).first())
removed_data = cleaner.purge_expired_report_data(later_cutoff)
self.assertEqual(len(removed_data), expected_count)
self.assertEqual(removed_data[0].get("provider_uuid"), first_bill.provider_id)
self.assertIn(
str(first_bill.billing_period_start), [entry.get("billing_period_start") for entry in removed_data]
)
with schema_context(self.schema):
self.assertIsNone(self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNone(self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_for_provider(self):
"""Test that the provider_uuid deletes all data for the provider."""
bill_table_name = AZURE_REPORT_TABLE_MAP['bill']
line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item']
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
first_bill = self.accessor._get_db_obj_query(
bill_table_name).first()
self.assertIsNotNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNotNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
removed_data = cleaner.purge_expired_report_data(
provider_uuid=self.azure_provider_uuid)
self.assertEqual(len(removed_data), 1)
self.assertEqual(removed_data[0].get('provider_uuid'),
first_bill.provider_id)
self.assertEqual(
removed_data[0].get('billing_period_start'),
str(first_bill.billing_period_start),
)
with schema_context(self.schema):
self.assertIsNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_both_args(self):
"""Test that the provider_uuid deletes all data for the provider."""
now = datetime.datetime.utcnow()
cleaner = AzureReportDBCleaner(self.schema)
with self.assertRaises(AzureReportDBCleanerError):
cleaner.purge_expired_report_data(
expired_date=now, provider_uuid=self.azure_provider_uuid)
def test_purge_expired_report_data_after_date(self):
"""Test to remove report data after a provided date."""
bill_table_name = AZURE_REPORT_TABLE_MAP['bill']
line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item']
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
# Verify that data is cleared for a cutoff date > billing_period_start
first_bill = self.accessor._get_db_obj_query(
bill_table_name).first()
cutoff_date = first_bill.billing_period_start
later_date = cutoff_date + relativedelta.relativedelta(months=+1)
later_cutoff = later_date.replace(month=later_date.month, day=15)
self.assertIsNotNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNotNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
removed_data = cleaner.purge_expired_report_data(later_cutoff)
self.assertEqual(len(removed_data), 1)
self.assertEqual(removed_data[0].get('provider_id'),
first_bill.provider_id)
self.assertEqual(
removed_data[0].get('billing_period_start'),
str(first_bill.billing_period_start),
)
with schema_context(self.schema):
self.assertIsNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_expired_report_data_before_date(self):
"""Test to remove report data before a provided date."""
bill_table_name = AZURE_REPORT_TABLE_MAP['bill']
line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item']
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
# Verify that data is not cleared for a cutoff date < billing_period_start
first_bill = self.accessor._get_db_obj_query(
bill_table_name).first()
cutoff_date = first_bill.billing_period_start
earlier_cutoff = cutoff_date.replace(
day=15) + relativedelta.relativedelta(months=-1)
self.assertIsNotNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNotNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
removed_data = cleaner.purge_expired_report_data(earlier_cutoff)
self.assertEqual(len(removed_data), 0)
with schema_context(self.schema):
self.assertIsNotNone(
self.accessor._get_db_obj_query(bill_table_name).first())
self.assertIsNotNone(
self.accessor._get_db_obj_query(line_item_table_name).first())
def test_purge_correct_expired_report_data_for_provider(self):
"""Test that the provider_uuid deletes all data for the provider when another provider exists."""
# add another line item with different provider_uuid
azure_credentials = {
'subscription_id': '11111111-f248-4ad7-bfb1-9a4cff600e1d',
'tenant_id': '11111111-228a-4aee-a215-3a768cdd0105',
'client_id': '11111111-0f78-42bb-b8e5-90144a025191',
'client_secret': 'notsosecretcode',
}
azure_data_source = {
'resource_group': 'resourcegroup2',
'storage_account': 'storageaccount2',
}
test_provider_uuid = '11111111-d05f-488c-a6d9-c2a6f3ee02bb'
azure_auth = ProviderAuthentication.objects.create(credentials=azure_credentials)
azure_auth.save()
azure_billing_source = ProviderBillingSource.objects.create(data_source=azure_data_source)
azure_billing_source.save()
azure_provider = Provider.objects.create(
uuid=test_provider_uuid,
name='Test Provider',
type='AZURE',
authentication=azure_auth,
billing_source=azure_billing_source,
customer=self.customer,
setup_complete=False,
)
azure_provider.save()
bill_id = self.creator.create_azure_cost_entry_bill(provider_uuid=azure_provider.uuid)
product_id = self.creator.create_azure_cost_entry_product(provider_uuid=azure_provider.uuid)
meter_id = self.creator.create_azure_meter(provider_uuid=azure_provider.uuid)
self.creator.create_azure_cost_entry_line_item(bill_id, product_id, meter_id)
bill_table_name = AZURE_REPORT_TABLE_MAP['bill']
line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item']
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
# Verify that data is cleared for a cutoff date == billing_period_start
first_bill = self.accessor._get_db_obj_query(bill_table_name).first()
self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).all().count(), 2)
self.assertEqual(self.accessor._get_db_obj_query(line_item_table_name).all().count(), 2)
removed_data = cleaner.purge_expired_report_data(provider_uuid=self.azure_provider_uuid)
self.assertEqual(len(removed_data), 1)
self.assertEqual(removed_data[0].get('provider_uuid'), first_bill.provider_id)
self.assertEqual(
removed_data[0].get('billing_period_start'), str(first_bill.billing_period_start),
)
with schema_context(self.schema):
self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).all().count(), 1)
self.assertEqual(self.accessor._get_db_obj_query(line_item_table_name).count(), 1)
class AzureReportDBCleanerTest(MasuTestCase):
"""Test Cases for the AzureReportChargeUpdater object."""
def setUp(self):
"""Set up each test."""
super().setUp()
self.cleaner = AzureReportDBCleaner(
schema=self.schema)
def test_purge_expired_report_data(self):
"""Test to verify that expired Azure report data is cleared."""
self.cleaner.purge_expired_report_data()
def _set_cleaner(self):
"""
Create the expired report data object.
Object is specific to the report provider.
Args:
None
Returns:
(Object) : Provider-specific report cleaner
"""
if self._provider in (Provider.PROVIDER_AWS,
Provider.PROVIDER_AWS_LOCAL):
return AWSReportDBCleaner(self._schema)
if self._provider in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
return AzureReportDBCleaner(self._schema)
if self._provider in (Provider.PROVIDER_OCP, ):
return OCPReportDBCleaner(self._schema)
return None
def test_purge_correct_expired_report_data_for_provider(self):
"""Test that the provider_id deletes all data for the provider when another provider exists."""
# add another line item with different provider_id
bill_id = self.creator.create_azure_cost_entry_bill(
provider_id=self.azure_provider_id + 1)
product_id = self.creator.create_azure_cost_entry_product()
meter_id = self.creator.create_azure_meter()
self.creator.create_azure_cost_entry_line_item(bill_id, product_id,
meter_id)
bill_table_name = AZURE_REPORT_TABLE_MAP['bill']
line_item_table_name = AZURE_REPORT_TABLE_MAP['line_item']
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
# Verify that data is cleared for a cutoff date == billing_period_start
first_bill = self.accessor._get_db_obj_query(
bill_table_name).first()
self.assertEqual(
self.accessor._get_db_obj_query(bill_table_name).all().count(),
2)
self.assertEqual(
self.accessor._get_db_obj_query(
line_item_table_name).all().count(), 2)
removed_data = cleaner.purge_expired_report_data(
provider_id=self.azure_provider_id)
self.assertEqual(len(removed_data), 1)
self.assertEqual(removed_data[0].get('provider_id'),
first_bill.provider_id)
self.assertEqual(
removed_data[0].get('billing_period_start'),
str(first_bill.billing_period_start),
)
with schema_context(self.schema):
self.assertEqual(
self.accessor._get_db_obj_query(bill_table_name).all().count(),
1)
self.assertEqual(
self.accessor._get_db_obj_query(line_item_table_name).count(),
1)
def _set_cleaner(self):
"""
Create the expired report data object.
Object is specific to the report provider.
Args:
None
Returns:
(Object) : Provider-specific report cleaner
"""
if self._provider in (AMAZON_WEB_SERVICES, AWS_LOCAL_SERVICE_PROVIDER):
return AWSReportDBCleaner(self._schema)
if self._provider in (AZURE, AZURE_LOCAL_SERVICE_PROVIDER):
return AzureReportDBCleaner(self._schema)
if self._provider in (OPENSHIFT_CONTAINER_PLATFORM, ):
return OCPReportDBCleaner(self._schema)
return None
def test_purge_expired_report_data_no_args(self):
"""Test that the provider_uuid deletes all data for the provider."""
cleaner = AzureReportDBCleaner(self.schema)
with self.assertRaises(AzureReportDBCleanerError):
cleaner.purge_expired_report_data()
def test_purge_correct_expired_report_data_for_provider(self):
"""Test that the provider_uuid deletes all data for the provider when another provider exists."""
# add another line item with different provider_uuid
azure_credentials = {
"subscription_id": "11111111-f248-4ad7-bfb1-9a4cff600e1d",
"tenant_id": "11111111-228a-4aee-a215-3a768cdd0105",
"client_id": "11111111-0f78-42bb-b8e5-90144a025191",
"client_secret": "notsosecretcode",
}
azure_data_source = {"resource_group": "resourcegroup2", "storage_account": "storageaccount2"}
test_provider_uuid = "11111111-d05f-488c-a6d9-c2a6f3ee02bb"
azure_auth = ProviderAuthentication.objects.create(credentials=azure_credentials)
azure_auth.save()
azure_billing_source = ProviderBillingSource.objects.create(data_source=azure_data_source)
azure_billing_source.save()
with patch("masu.celery.tasks.check_report_updates"):
azure_provider = Provider.objects.create(
uuid=test_provider_uuid,
name="Test Provider",
type=Provider.PROVIDER_AZURE_LOCAL,
authentication=azure_auth,
billing_source=azure_billing_source,
customer=self.customer,
setup_complete=False,
)
azure_provider.save()
bill_id = self.creator.create_azure_cost_entry_bill(provider_uuid=azure_provider.uuid)
product_id = self.creator.create_azure_cost_entry_product(provider_uuid=azure_provider.uuid)
meter_id = self.creator.create_azure_meter(provider_uuid=azure_provider.uuid)
self.creator.create_azure_cost_entry_line_item(bill_id, product_id, meter_id)
bill_table_name = AZURE_REPORT_TABLE_MAP["bill"]
line_item_table_name = AZURE_REPORT_TABLE_MAP["line_item"]
cleaner = AzureReportDBCleaner(self.schema)
with schema_context(self.schema):
# Verify that data is cleared for a cutoff date == billing_period_start
first_bill = (
self.accessor._get_db_obj_query(bill_table_name)
.filter(provider_id=self.azure_provider_uuid)
.order_by("-billing_period_start")
.first()
)
expected_count = (
self.accessor._get_db_obj_query(bill_table_name).filter(provider_id=self.azure_provider_uuid).count()
)
expected_line_item_count = (
self.accessor._get_db_obj_query(line_item_table_name)
.filter(cost_entry_bill__provider_id=self.azure_provider_uuid)
.count()
)
self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).count(), expected_count + 1)
self.assertEqual(
self.accessor._get_db_obj_query(line_item_table_name).count(), expected_line_item_count + 1
)
removed_data = cleaner.purge_expired_report_data(provider_uuid=self.azure_provider_uuid)
self.assertEqual(len(removed_data), expected_count)
self.assertEqual(removed_data[0].get("provider_uuid"), first_bill.provider_id)
self.assertIn(
str(first_bill.billing_period_start), [entry.get("billing_period_start") for entry in removed_data]
)
with schema_context(self.schema):
self.assertEqual(self.accessor._get_db_obj_query(bill_table_name).all().count(), 1)
self.assertEqual(self.accessor._get_db_obj_query(line_item_table_name).count(), 1)
def setUp(self):
"""Set up each test."""
super().setUp()
self.cleaner = AzureReportDBCleaner(
schema=self.schema)
def test_drop_report_partitions(self):
"""Test that cleaner drops report line item daily summary parititons"""
provider = Provider.objects.filter(
type__in=[Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL
]).first()
self.assertIsNotNone(provider)
partitioned_table_name = AZURE_REPORT_TABLE_MAP[
"line_item_daily_summary"]
report_period_table_name = AZURE_REPORT_TABLE_MAP["bill"]
partitioned_table_model = None
report_period_model = None
for model in django.apps.apps.get_models():
if model._meta.db_table == partitioned_table_name:
partitioned_table_model = model
elif model._meta.db_table == report_period_table_name:
report_period_model = model
if partitioned_table_model and report_period_model:
break
self.assertIsNotNone(partitioned_table_model)
self.assertIsNotNone(report_period_model)
with schema_context(self.schema):
test_part = PartitionedTable(
schema_name=self.schema,
table_name=f"{partitioned_table_name}_2016_01",
partition_of_table_name=partitioned_table_name,
partition_type=PartitionedTable.RANGE,
partition_col="usage_start",
partition_parameters={
"default": False,
"from": "2016-01-01",
"to": "2016-02-01"
},
active=True,
)
test_part.save()
self.assertTrue(table_exists(self.schema, test_part.table_name))
report_period_start = datetime.datetime(2016,
1,
1,
tzinfo=pytz.UTC)
report_period_end = datetime.datetime(2016, 1, 31, tzinfo=pytz.UTC)
report_period = report_period_model(
billing_period_start=report_period_start,
billing_period_end=report_period_end,
provider_id=provider.uuid,
)
report_period.save()
usage_date = datetime.date(2016, 1, 12)
lids_rec = partitioned_table_model(
usage_start=usage_date,
usage_end=usage_date,
cost_entry_bill_id=report_period.id,
uuid=uuid.uuid4())
lids_rec.save()
cutoff_date = datetime.datetime(2016, 12, 31, tzinfo=pytz.UTC)
cleaner = AzureReportDBCleaner(self.schema)
removed_data = cleaner.purge_expired_report_data(cutoff_date,
simulate=False)
self.assertEqual(len(removed_data), 1)
self.assertFalse(table_exists(self.schema, test_part.table_name))
