def test_check_cluster_infrastructure(self):
"""Check that we correctly associate a cluster with infrastructure provider."""
start_date = self.dh.this_month_start.date()
end_date = self.dh.this_month_end.date()
new_ocp_provider = Provider(name="new_test_ocp",
type=Provider.PROVIDER_OCP)
new_ocp_provider.save()
# mock_trino.return_value = {str(new_ocp_provider.uuid): (self.aws_provider_uuid, Provider.PROVIDER_AWS_LOCAL)}
with patch.object(OCPReportDBAccessor,
"get_ocp_infrastructure_map_trino") as mock_trino:
mock_trino.return_value = {
str(new_ocp_provider.uuid):
(self.aws_provider_uuid, Provider.PROVIDER_AWS_LOCAL)
}
updater = OCPReportParquetSummaryUpdater(self.schema,
new_ocp_provider,
self.manifest)
updater.check_cluster_infrastructure(start_date, end_date)
result = Provider.objects.get(
uuid=new_ocp_provider.uuid
).infrastructure.infrastructure_provider_id
self.assertEqual(result, self.aws_provider.uuid)
def setUp(self):
"""Set up tests."""
super().setUp()
self.test_account = "10001"
user_data = self._create_user_data()
customer = self._create_customer_data(account=self.test_account)
self.request_context = self._create_request_context(customer, user_data, create_customer=True, is_admin=True)
self.test_source_id = 1
name = "Test Azure Source"
customer_obj = Customer.objects.get(account_id=customer.get("account_id"))
self.azure_provider = Provider(name=name, type=Provider.PROVIDER_AZURE, customer=customer_obj)
self.azure_provider.save()
self.azure_obj = Sources(
source_id=self.test_source_id,
auth_header=self.request_context["request"].META,
account_id=customer.get("account_id"),
offset=1,
source_type=Provider.PROVIDER_AZURE,
name=name,
authentication={
"credentials": {"client_id": "test_client", "tenant_id": "test_tenant", "client_secret": "test_secret"}
},
source_uuid=self.azure_provider.uuid,
)
self.azure_obj.save()
mock_url = PropertyMock(return_value="http://www.sourcesclient.com/api/v1/sources/")
SourcesViewSet.url = mock_url
def test_execute_update_sql(self):
"""Test that execute_update_sql runs the update SQL"""
# Add a bogus provider
p = Provider(
uuid=uuid.uuid4(), name="eek_provider_2", type=Provider.PROVIDER_OCP, setup_complete=False, active=True
)
p.save()
# Update it with SQL
udt_count = kdb.execute_update_sql(Provider.objects.filter(pk=p.pk), active=False)
self.assertEqual(udt_count, 1)
self.assertEqual(Provider.objects.get(pk=p.pk).active, False)
def test_execute_delete_sql(self):
"""Test that execute_delete_sql runs the delete SQL"""
# Add a bogus provider
p = Provider(
uuid=uuid.uuid4(), name="eek_provider_1", type=Provider.PROVIDER_OCP, setup_complete=False, active=True
)
p.save()
# Delete it with SQL
del_count = kdb.execute_delete_sql(Provider.objects.filter(pk=p.pk))
self.assertEqual(del_count, 1)
self.assertIsNone(Provider.objects.filter(pk=p.pk).first())
def test_cascade_delete_with_skip(self):
"""Test that cascade_delete can walk relations abd skip specified relations"""
action_ts = datetime.now().replace(tzinfo=UTC)
# Add a bogus customer
c = Customer(
date_created=action_ts,
date_updated=action_ts,
uuid=uuid.uuid4(),
account_id="68461385",
schema_name="acct68461385",
)
c.save()
# Create a customer tenant
t = Tenant(schema_name=c.schema_name)
t.save()
t.create_schema()
# Add some bogus providers
pocp = Provider(
uuid=uuid.uuid4(),
name="eek_ocp_provider_30",
type=Provider.PROVIDER_OCP,
setup_complete=False,
active=True,
customer=c,
)
pocp.save()
expected1 = "DEBUG:koku.database:Level 1: delete records from OCPUsageReportPeriod"
expected2 = "DEBUG:koku.database:SKIPPING RELATION OCPUsageLineItemDailySummary by directive"
expected3 = "DEBUG:koku.database:SKIPPING RELATION OCPUsageLineItem by directive"
skip_models = [
kdb.get_model("OCPUsageLineItemDailySummary"),
kdb.get_model("OCPUsageLineItem")
]
query = Provider.objects.filter(pk=pocp.pk)
with self.assertLogs("koku.database", level="DEBUG") as _logger:
with schema_context(c.schema_name):
kdb.cascade_delete(Provider, query, skip_relations=skip_models)
self.assertIn(expected1, _logger.output)
self.assertIn(expected2, _logger.output)
self.assertIn(expected3, _logger.output)
with schema_context(c.schema_name):
self.assertEqual(
OCPUsageReportPeriod.objects.filter(pk=pocp.pk).count(), 0)
self.assertEqual(Provider.objects.filter(pk=pocp.pk).count(), 0)
def save(self, force_insert=False, force_update=False, commit=True):
try:
obj = Provider.objects.get(name=self.data['name'],
email=self.data['email'],
phone_no=self.data['phone_no'],
language=self.data['language'],
currency=self.data['currency'])
except:
obj = Provider(name=self.data['name'],
email=self.data['email'],
phone_no=self.data['phone_no'],
language=self.data['language'],
currency=self.data['currency'])
obj.save()
def test_cascade_delete_immediate_constraint(self):
"""Test that cascade_delete can set constraints to execute immediately"""
action_ts = datetime.now().replace(tzinfo=UTC)
# Add a bogus customer
c = Customer(
date_created=action_ts,
date_updated=action_ts,
uuid=uuid.uuid4(),
account_id="918273",
schema_name="acct918273",
)
c.save()
# Create a customer tenant
t = Tenant(schema_name=c.schema_name)
t.save()
t.create_schema()
# Add some bogus providers
paws = Provider(
uuid=uuid.uuid4(),
name="eek_aws_provider_4",
type=Provider.PROVIDER_AWS,
setup_complete=False,
active=True,
customer=c,
)
paws.save()
# Create billing period stuff for each provider
period_start = datetime(2020, 1, 1, tzinfo=UTC)
period_end = datetime(2020, 2, 1, tzinfo=UTC)
awsceb = AWSCostEntryBill(
billing_resource="6846351687354184651",
billing_period_start=period_start,
billing_period_end=period_end,
provider=paws,
)
with schema_context(c.schema_name):
awsceb.save()
expected = "DEBUG:koku.database:Setting constaints to execute immediately"
with self.assertLogs("koku.database", level="DEBUG") as _logger:
paws.delete()
self.assertIn(expected, _logger.output)
with schema_context(c.schema_name):
self.assertEqual(
AWSCostEntryBill.objects.filter(pk=awsceb.pk).count(), 0)
self.assertEqual(Provider.objects.filter(pk=paws.pk).count(), 0)
class SourcesViewTests(IamTestCase):
"""Test Cases for the sources endpoint."""
def setUp(self):
"""Set up tests."""
super().setUp()
self.test_account = "10001"
user_data = self._create_user_data()
customer = self._create_customer_data(account=self.test_account)
self.request_context = self._create_request_context(
customer, user_data, create_customer=True, is_admin=True)
self.test_source_id = 1
name = "Test Azure Source"
customer_obj = Customer.objects.get(
account_id=customer.get("account_id"))
self.azure_provider = Provider(name=name,
type=Provider.PROVIDER_AZURE,
customer=customer_obj)
self.azure_provider.save()
self.azure_obj = Sources(
source_id=self.test_source_id,
auth_header=self.request_context["request"].META,
account_id=customer.get("account_id"),
offset=1,
source_type=Provider.PROVIDER_AZURE,
name=name,
authentication={
"credentials": {
"client_id": "test_client",
"tenant_id": "test_tenant",
"client_secret": "test_secret"
}
},
source_uuid=self.azure_provider.uuid,
)
self.azure_obj.save()
mock_url = PropertyMock(
return_value="http://www.sourcesclient.com/api/v1/sources/")
SourcesViewSet.url = mock_url
def test_source_patch(self):
"""Test the PATCH endpoint."""
credentials = {"subscription_id": "subscription-uuid"}
with requests_mock.mock() as m:
m.patch(
f"http://www.sourcesclient.com/api/v1/sources/{self.test_source_id}/",
status_code=200,
json={"credentials": credentials},
)
params = '{"credentials: blah}'
url = reverse("sources-detail", kwargs={"pk": self.test_source_id})
response = self.client.patch(
url,
params,
content_type="application/json",
**self.request_context["request"].META)
self.assertEqual(response.status_code, 405)
def test_source_put(self):
"""Test the PUT endpoint."""
credentials = {"subscription_id": "subscription-uuid"}
with requests_mock.mock() as m:
m.put(
f"http://www.sourcesclient.com/api/v1/sources/{self.test_source_id}/",
status_code=200,
json={"credentials": credentials},
)
params = {"credentials": credentials}
url = reverse("sources-detail", kwargs={"pk": self.test_source_id})
response = self.client.put(url,
json.dumps(params),
content_type="application/json",
**self.request_context["request"].META)
self.assertEqual(response.status_code, 405)
def test_source_list(self):
"""Test the LIST endpoint."""
with requests_mock.mock() as m:
m.get("http://www.sourcesclient.com/api/v1/sources/",
status_code=200)
url = reverse("sources-list")
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertEqual(body.get("meta").get("count"), 1)
def test_source_list_other_header(self):
"""Test the LIST endpoint with other auth header not matching test data."""
user_data = self._create_user_data()
other_account = "10002"
customer = self._create_customer_data(account=other_account)
IdentityHeaderMiddleware.create_customer(other_account)
request_context = self._create_request_context(customer,
user_data,
create_customer=True,
is_admin=True)
with requests_mock.mock() as m:
m.get("http://www.sourcesclient.com/api/v1/sources/",
status_code=200)
url = reverse("sources-list")
response = self.client.get(url,
content_type="application/json",
**request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertEqual(body.get("meta").get("count"), 0)
def test_source_get(self):
"""Test the GET endpoint."""
with requests_mock.mock() as m:
m.get(
f"http://www.sourcesclient.com/api/v1/sources/{self.test_source_id}/",
status_code=200,
headers={"Content-Type": "application/json"},
)
url = reverse("sources-detail", kwargs={"pk": self.test_source_id})
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIsNotNone(body)
def test_source_get_other_header(self):
"""Test the GET endpoint other header not matching test data."""
user_data = self._create_user_data()
other_account = "10002"
customer = self._create_customer_data(account=other_account)
IdentityHeaderMiddleware.create_customer(other_account)
request_context = self._create_request_context(customer,
user_data,
create_customer=True,
is_admin=True)
with requests_mock.mock() as m:
m.get(
f"http://www.sourcesclient.com/api/v1/sources/{self.test_source_id}/",
status_code=200,
headers={"Content-Type": "application/json"},
)
url = reverse("sources-detail", kwargs={"pk": self.test_source_id})
response = self.client.get(url,
content_type="application/json",
**request_context["request"].META)
self.assertEqual(response.status_code, 404)
def test_source_destroy_not_allowed(self):
"""Test access to the destroy endpoint."""
url = reverse("sources-detail", kwargs={"pk": self.test_source_id})
response = self.client.delete(url,
content_type="application/json",
**self.request_context["request"].META)
self.assertEqual(response.status_code, 405)
@patch("sources.api.view.ProviderManager.provider_statistics",
return_value={})
def test_source_get_stats(self, _):
"""Test the GET status endpoint."""
url = reverse("sources-stats", kwargs={"pk": self.test_source_id})
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIsNotNone(body)
@patch("sources.api.view.ProviderManager",
side_effect=ProviderManagerError("test error"))
def test_source_list_zerror(self, _):
"""Test provider_linked is False in list when Provider does not exist."""
cache.clear()
url = reverse("sources-list")
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIsNotNone(body)
self.assertTrue(body.get("data"))
self.assertFalse(body.get("data")[0]["provider_linked"])
@patch("sources.api.view.ProviderManager")
def test_source_list_provider_success(self, mock_provider_manager):
"""Test provider_linked is True in list when Provider exists."""
provider_manager = ProviderManager(self.azure_provider.uuid)
mock_provider_manager.return_value = provider_manager
url = reverse("sources-list")
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIsNotNone(body)
self.assertTrue(body.get("data"))
self.assertTrue(body.get("data")[0]["provider_linked"])
self.assertTrue(body.get("data")[0]["active"])
@patch("sources.api.view.ProviderManager",
side_effect=ProviderManagerError("test error"))
def test_source_retrieve_error(self, _):
"""Test provider_linked is False in Source when Provider does not exist."""
url = reverse("sources-detail", kwargs={"pk": self.test_source_id})
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIsNotNone(body)
self.assertFalse(body["provider_linked"])
self.assertFalse(body["active"])
@patch("sources.api.view.ProviderManager",
side_effect=ProviderManagerError("test error"))
def test_source_get_stats_error(self, _):
"""Test provider_linked is False in source-stats when Provider does not exist."""
url = reverse("sources-stats", kwargs={"pk": self.test_source_id})
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 200)
self.assertIsNotNone(body)
self.assertFalse(body["provider_linked"])
def test_source_get_random_int(self):
"""Test the GET endpoint with non-existent source int id."""
source_id = randint(20, 100)
with requests_mock.mock() as m:
m.get(
f"http://www.sourcesclient.com/api/v1/sources/{source_id}/",
status_code=404,
headers={"Content-Type": "application/json"},
)
url = reverse("sources-detail", kwargs={"pk": source_id})
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 404)
self.assertIsNotNone(body)
def test_source_get_random_uuid(self):
"""Test the GET endpoint with non-existent source uuid."""
source_id = uuid4()
with requests_mock.mock() as m:
m.get(
f"http://www.sourcesclient.com/api/v1/sources/{source_id}/",
status_code=404,
headers={"Content-Type": "application/json"},
)
url = reverse("sources-detail", kwargs={"pk": source_id})
response = self.client.get(url,
content_type="application/json",
**self.request_context["request"].META)
body = response.json()
self.assertEqual(response.status_code, 404)
self.assertIsNotNone(body)
def test_sources_access(self):
"""Test the limiting of source type visibility."""
mock_user = Mock(admin=True)
request = Mock(user=mock_user)
excluded = SourcesViewSet.get_excludes(request)
self.assertEqual(excluded, [])
mock_user = Mock(admin=False, access=None)
request = Mock(user=mock_user)
excluded = SourcesViewSet.get_excludes(request)
expected = []
for resource_type in RESOURCE_TYPE_MAP.keys():
expected.extend(RESOURCE_TYPE_MAP.get(resource_type))
self.assertEqual(excluded, expected)
permissions = {AwsAccessPermission.resource_type: {"read": []}}
mock_user = Mock(admin=False, access=permissions)
request = Mock(user=mock_user)
excluded = SourcesViewSet.get_excludes(request)
expected = []
for resource_type in RESOURCE_TYPE_MAP.keys():
expected.extend(RESOURCE_TYPE_MAP.get(resource_type))
self.assertEqual(excluded, expected)
permissions = {AwsAccessPermission.resource_type: {"read": ["*"]}}
mock_user = Mock(admin=False, access=permissions)
request = Mock(user=mock_user)
excluded = SourcesViewSet.get_excludes(request)
expected = [
Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL,
Provider.PROVIDER_OCP,
Provider.PROVIDER_GCP,
Provider.PROVIDER_GCP_LOCAL,
]
self.assertEqual(excluded, expected)
def test_cascade_delete(self):
"""Test that cascade_delete can walk relations to delete FK constraint matched records"""
action_ts = datetime.now().replace(tzinfo=UTC)
# Add a bogus customer
c = Customer(
date_created=action_ts,
date_updated=action_ts,
uuid=uuid.uuid4(),
account_id="918273",
schema_name="acct918273",
)
c.save()
# Create a customer tenant
t = Tenant(schema_name=c.schema_name)
t.save()
t.create_schema()
# Add some bogus providers
paws = Provider(
uuid=uuid.uuid4(),
name="eek_aws_provider_3",
type=Provider.PROVIDER_AWS,
setup_complete=False,
active=True,
customer=c,
)
paws.save()
pazure = Provider(
uuid=uuid.uuid4(),
name="eek_azure_provider_3",
type=Provider.PROVIDER_AZURE,
setup_complete=False,
active=True,
customer=c,
)
pazure.save()
pgcp = Provider(
uuid=uuid.uuid4(),
name="eek_gcp_provider_3",
type=Provider.PROVIDER_GCP,
setup_complete=False,
active=True,
customer=c,
)
pgcp.save()
pocp = Provider(
uuid=uuid.uuid4(),
name="eek_ocp_provider_3",
type=Provider.PROVIDER_OCP,
setup_complete=False,
active=True,
customer=c,
)
pocp.save()
# Create billing period stuff for each provider
period_start = datetime(2020, 1, 1, tzinfo=UTC)
period_end = datetime(2020, 2, 1, tzinfo=UTC)
awsceb = AWSCostEntryBill(
billing_resource="6846351687354184651",
billing_period_start=period_start,
billing_period_end=period_end,
provider=paws,
)
azureceb = AzureCostEntryBill(
billing_period_start=period_start, billing_period_end=period_end, provider=pazure
)
gcpceb = GCPCostEntryBill(billing_period_start=period_start, billing_period_end=period_end, provider=pgcp)
ocpurp = OCPUsageReportPeriod(
cluster_id="584634154687685", report_period_start=period_start, report_period_end=period_end, provider=pocp
)
with schema_context(c.schema_name):
awsceb.save()
azureceb.save()
gcpceb.save()
ocpurp.save()
expected = "INFO:koku.database:Level 1: delete records from AWSCostEntryBill"
with self.assertLogs("koku.database", level="INFO") as _logger:
paws.delete()
self.assertIn(expected, _logger.output)
expected = "INFO:koku.database:Level 1: delete records from AzureCostEntryBill"
with self.assertLogs("koku.database", level="INFO") as _logger:
pazure.delete()
self.assertIn(expected, _logger.output)
expected = "INFO:koku.database:Level 1: delete records from GCPCostEntryBill"
with self.assertLogs("koku.database", level="INFO") as _logger:
pgcp.delete()
self.assertIn(expected, _logger.output)
expected = "INFO:koku.database:Level 1: delete records from OCPUsageReportPeriod"
with self.assertLogs("koku.database", level="INFO") as _logger:
pocp.delete()
self.assertIn(expected, _logger.output)
with schema_context(c.schema_name):
self.assertEqual(AWSCostEntryBill.objects.filter(pk=awsceb.pk).count(), 0)
self.assertEqual(AzureCostEntryBill.objects.filter(pk=azureceb.pk).count(), 0)
self.assertEqual(GCPCostEntryBill.objects.filter(pk=gcpceb.pk).count(), 0)
self.assertEqual(OCPUsageReportPeriod.objects.filter(pk=ocpurp.pk).count(), 0)
self.assertEqual(Provider.objects.filter(pk__in=(paws.pk, pazure.pk, pgcp.pk, pocp.pk)).count(), 0)
def test_cascade_delete_skip(self):
"""Test that cascade_delete can skip relations"""
action_ts = datetime.now().replace(tzinfo=UTC)
# Add a bogus customer
c = Customer(
date_created=action_ts,
date_updated=action_ts,
uuid=uuid.uuid4(),
account_id="918273",
schema_name="acct918273",
)
c.save()
# Create a customer tenant
t = Tenant(schema_name=c.schema_name)
t.save()
t.create_schema()
# Add some bogus providers
paws = Provider(
uuid=uuid.uuid4(),
name="eek_aws_provider_4",
type=Provider.PROVIDER_AWS,
setup_complete=False,
active=True,
customer=c,
)
paws.save()
# Create billing period stuff for each provider
period_start = datetime(2020, 1, 1, tzinfo=UTC)
period_end = datetime(2020, 2, 1, tzinfo=UTC)
AWSCostEntry = kdb.get_model("AWSCostEntry")
AWSCostEntryLineItem = kdb.get_model("AWSCostEntryLineItem")
with schema_context(c.schema_name):
awsceb = AWSCostEntryBill(
billing_resource="6846351687354184651",
billing_period_start=period_start,
billing_period_end=period_end,
provider=paws,
)
awsceb.save()
awsce = AWSCostEntry(interval_start=period_start,
interval_end=period_end,
bill=awsceb)
awsce.save()
awsceli = AWSCostEntryLineItem(
line_item_type="test",
usage_account_id="test",
usage_start=period_start,
usage_end=period_start,
product_code="test",
currency_code="USD",
cost_entry=awsce,
cost_entry_bill=awsceb,
)
awsceli.save()
expected = "DEBUG:koku.database:SKIPPING RELATION AWSCostEntryLineItem by directive"
expected2 = "DEBUG:koku.database:SKIPPING RELATION GCPCostEntryLineItem by directive"
expected3 = "DEBUG:koku.database:SKIPPING RELATION OCPUsageLineItem by directive"
expected4 = "DEBUG:koku.database:SKIPPING RELATION OCPStorageLineItem by directive"
expected5 = "DEBUG:koku.database:SKIPPING RELATION OCPNodeLabelLineItem by directive"
with self.assertLogs("koku.database", level="DEBUG") as _logger:
paws.delete()
self.assertIn(expected, _logger.output)
self.assertIn(expected2, _logger.output)
self.assertIn(expected3, _logger.output)
self.assertIn(expected4, _logger.output)
self.assertIn(expected5, _logger.output)
with schema_context(c.schema_name):
self.assertEqual(
AWSCostEntryBill.objects.filter(pk=awsceb.pk).count(), 0)
self.assertEqual(
AWSCostEntry.objects.filter(pk=awsce.pk).count(), 0)
self.assertNotEqual(
AWSCostEntryLineItem.objects.filter(pk=awsceli.pk).count(), 0)
self.assertEqual(Provider.objects.filter(pk=paws.pk).count(), 0)