def test_azure_source_billing_source_update(self, _):
"""Test the updating azure billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_resource_group = "TESTRG"
test_storage_account = "testsa"
validated_data = {
"billing_source": {
"data_source": {
"resource_group": test_resource_group,
"storage_account": test_storage_account
}
}
}
with patch("sources.api.serializers.ServerProxy") as mock_client:
mock_sources_client = MockSourcesClient(
"http://mock-soures-client")
mock_client.return_value.__enter__.return_value = mock_sources_client
instance = serializer.update(self.azure_obj, validated_data)
self.assertIn("data_source", instance.billing_source.keys())
self.assertEqual(
test_resource_group,
instance.billing_source.get("data_source").get("resource_group"))
self.assertEqual(
test_storage_account,
instance.billing_source.get("data_source").get("storage_account"))
def test_aws_source_billing_source_update_missing_bucket(self):
"""Test the updating aws billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_bucket = None
validated_data = {'billing_source': {'bucket': test_bucket}}
with self.assertRaises(SourcesStorageError):
serializer.update(self.aws_obj, validated_data)
def test_azure_source_update_missing_credential(self, _):
"""Test the update azure source with missing credentials."""
self.azure_obj.authentication = {}
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {
"authentication": {
"credentials": {
"subscription_id": "subscription-uuid"
}
}
}
with patch("sources.api.serializers.ServerProxy") as mock_client:
mock_sources_client = MockSourcesClient(
"http://mock-soures-client")
mock_client.return_value.__enter__.return_value = mock_sources_client
instance = serializer.update(self.azure_obj, validated_data)
self.assertEqual(
"subscription-uuid",
instance.authentication.get("credentials").get(
"subscription_id"))
for field in ("client_id", "tenant_id", "client_secret"):
self.assertNotIn(field,
instance.authentication.get("credentials").keys())
def test_azure_source_billing_source_update_missing_resource_group(self, _):
"""Test the updating azure billing_source with missing resource group."""
serializer = SourcesSerializer(context=self.request_context)
test_storage_account = "testsa"
validated_data = {"billing_source": {"data_source": {"storage_account": test_storage_account}}}
with self.assertRaises(SourcesStorageError):
serializer.update(self.azure_obj, validated_data)
def test_azure_source_billing_source_update_missing_storage_account(self):
"""Test the updating azure billing_source with missing storage account."""
serializer = SourcesSerializer(context=self.request_context)
test_resource_group = 'TESTRG'
validated_data = {'billing_source': {'data_source': {'resource_group': test_resource_group}}}
with self.assertRaises(SourcesStorageError):
serializer.update(self.azure_obj, validated_data)
def test_aws_source_billing_source_update(self):
"""Test the updating aws billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_bucket = 'test-bucket'
validated_data = {'billing_source': {'bucket': test_bucket}}
instance = serializer.update(self.aws_obj, validated_data)
self.assertIn('bucket', instance.billing_source.keys())
self.assertEqual(test_bucket, instance.billing_source.get('bucket'))
def test_aws_source_billing_source_update(self):
"""Test the updating aws billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_bucket = "test-bucket"
validated_data = {"billing_source": {"bucket": test_bucket}}
instance = serializer.update(self.aws_obj, validated_data)
self.assertIn("bucket", instance.billing_source.keys())
self.assertEqual(test_bucket, instance.billing_source.get("bucket"))
def test_azure_source_update_wrong_type(self):
"""Test the updating azure source with wrong source type."""
self.azure_obj.source_type = Provider.PROVIDER_AWS
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {'authentication': {'credentials': {'subscription_id': 'subscription-uuid'}}}
with self.assertRaises(SourcesStorageError):
serializer.update(self.azure_obj, validated_data)
def test_ocp_source_billing_source_update(self):
"""Test the updating billing_source for invalid OCP source."""
self.aws_obj.instance_type = Provider.PROVIDER_OCP
self.aws_obj.save()
test_bucket = 'test-bucket'
serializer = SourcesSerializer(context=self.request_context)
test_bucket = None
validated_data = {'billing_source': {'bucket': test_bucket}}
with self.assertRaises(SourcesStorageError):
serializer.update(self.aws_obj, validated_data)
def test_azure_source_authentication_update(self):
"""Test the updating azure subscription id."""
serializer = SourcesSerializer(context=self.request_context)
validated_data = {'authentication': {'credentials': {'subscription_id': 'subscription-uuid'}}}
instance = serializer.update(self.azure_obj, validated_data)
self.assertIn('credentials', instance.authentication.keys())
self.assertIn('client_id', instance.authentication.get('credentials').keys())
self.assertIn('tenant_id', instance.authentication.get('credentials').keys())
self.assertIn('subscription_id', instance.authentication.get('credentials').keys())
self.assertEqual('subscription-uuid', instance.authentication.get('credentials').get('subscription_id'))
def test_azure_source_update_with_koku_uuid(self):
"""Test the updating azure subscription id with a koku_uuid."""
self.azure_obj.koku_uuid = fake.uuid4()
self.azure_obj.pending_update = False
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {'authentication': {'credentials': {'subscription_id': 'subscription-uuid'}}}
instance = serializer.update(self.azure_obj, validated_data)
self.assertTrue(instance.pending_update)
def test_azure_source_billing_source_update(self):
"""Test the updating azure billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_resource_group = 'TESTRG'
test_storage_account = 'testsa'
validated_data = {'billing_source': {'data_source': {'resource_group': test_resource_group,
'storage_account': test_storage_account}}}
instance = serializer.update(self.azure_obj, validated_data)
self.assertIn('data_source', instance.billing_source.keys())
self.assertEqual(test_resource_group, instance.billing_source.get('data_source').get('resource_group'))
self.assertEqual(test_storage_account, instance.billing_source.get('data_source').get('storage_account'))
def test_aws_source_billing_source_update(self, mock_delay):
"""Test the updating aws billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_bucket = "some-new-bucket"
validated_data = {"billing_source": {"bucket": test_bucket}}
with patch.object(ProviderAccessor,
"cost_usage_source_ready",
returns=True):
instance = serializer.update(self.aws_obj, validated_data)
self.assertIn("bucket", instance.billing_source.keys())
self.assertEqual(test_bucket, instance.billing_source.get("bucket"))
def test_azure_source_update_missing_credential(self):
"""Test the updating azure source with invalid authentication."""
self.azure_obj.authentication = {}
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {'authentication': {'credentials': {'subscription_id': 'subscription-uuid'}}}
instance = serializer.update(self.azure_obj, validated_data)
self.assertEqual('subscription-uuid', instance.authentication.get('credentials').get('subscription_id'))
for field in ('client_id', 'tenant_id', 'client_secret'):
self.assertNotIn(field, instance.authentication.get('credentials').keys())
def test_azure_source_update_wrong_type(self, _):
"""Test the updating azure source with wrong source type."""
self.azure_obj.source_type = Provider.PROVIDER_AWS
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {"authentication": {"credentials": {"subscription_id": "subscription-uuid"}}}
with self.assertRaises(SourcesStorageError):
with patch("sources.api.serializers.ServerProxy") as mock_client:
mock_sources_client = MockSourcesClient("http://mock-soures-client")
mock_client.return_value.__enter__.return_value = mock_sources_client
serializer.update(self.azure_obj, validated_data)
def test_aws_source_billing_source_update(self, _):
"""Test the updating aws billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_bucket = "some-new-bucket"
validated_data = {"billing_source": {"bucket": test_bucket}}
with patch("sources.api.serializers.ServerProxy") as mock_client:
with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True):
mock_sources_client = MockSourcesClient("http://mock-soures-client")
mock_client.return_value.__enter__.return_value = mock_sources_client
instance = serializer.update(self.aws_obj, validated_data)
self.assertIn("bucket", instance.billing_source.keys())
self.assertEqual(test_bucket, instance.billing_source.get("bucket"))
def test_azure_source_billing_source_update_with_koku_uuid(self):
"""Test the updating azure billing_source with koku_uuid."""
self.azure_obj.koku_uuid = fake.uuid4()
self.azure_obj.pending_update = False
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
test_resource_group = 'TESTRG'
test_storage_account = 'testsa'
validated_data = {'billing_source': {'data_source': {'resource_group': test_resource_group,
'storage_account': test_storage_account}}}
instance = serializer.update(self.azure_obj, validated_data)
self.assertTrue(instance.pending_update)
def test_patch_unavailable_sources_client(self, _):
serializer = SourcesSerializer(context=self.request_context)
with patch("sources.api.serializers.ServerProxy") as mock_client:
mock_client.side_effect = ConnectionRefusedError
with self.assertRaises(SourcesDependencyError):
validated_data = {
"billing_source": {
"data_source": {
"bucket": "some-new-bucket"
}
}
}
serializer.update(self.aws_obj, validated_data)
mock_client.side_effect = gaierror
with self.assertRaises(SourcesDependencyError):
validated_data = {
"billing_source": {
"data_source": {
"bucket": "some-new-bucket"
}
}
}
serializer.update(self.aws_obj, validated_data)
# catch ProtocolError
with self.assertRaises(SourcesDependencyError):
validated_data = {
"billing_source": {
"data_source": {
"bucket": "some-new-bucket"
}
}
}
serializer.update(self.aws_obj, validated_data)
def test_azure_source_update_missing_credential(self):
"""Test the updating azure source with invalid authentication."""
self.azure_obj.authentication = {}
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {
'authentication': {
'credentials': {
'subscription_id': 'subscription-uuid'
}
}
}
with self.assertRaises(SourcesStorageError):
serializer.update(self.azure_obj, validated_data)
def test_provider_create(self, mock_header, mock_request_info, mock_delay):
mock_request_info.return_value = self.User, self.Customer
serializer = AdminSourcesSerializer(context=self.request_context)
source = {
"source_id": 10,
"name": "ProviderAWS",
"source_type": "AWS",
"authentication": {
"resource_name":
"arn:aws:iam::111111111111:role/CostManagement"
},
"billing_source": {
"bucket": "first-bucket"
},
"auth_header": Config.SOURCES_FAKE_HEADER,
"account_id": "acct10001",
"offset": 10,
}
with patch.object(ProviderAccessor,
"cost_usage_source_ready",
returns=True):
instance = serializer.create(source)
self.assertEqual(instance.billing_source.get("bucket"), "first-bucket")
serializer = SourcesSerializer(context=self.request_context)
validated = {"billing_source": {"bucket": "second-bucket"}}
with patch.object(ProviderAccessor,
"cost_usage_source_ready",
returns=True):
instance2 = serializer.update(instance, validated)
self.assertEqual(instance2.billing_source.get("bucket"),
"second-bucket")
def test_source_update_rabbit_down(self, mock_delay):
"""Test the updating a source with rabbit down."""
self.azure_obj.source_type = Provider.PROVIDER_AZURE
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {
"authentication": {
"credentials": {
"subscription_id": "subscription-uuid"
}
}
}
mock_side_effect = SourcesDependencyError("Where's Rabbit")
mock_delay.side_effect = mock_side_effect
with self.assertRaises(SourcesDependencyError):
serializer.update(self.azure_obj, validated_data)
def test_azure_source_billing_source_update_with_koku_uuid(self, _):
"""Test the updating azure billing_source with source_uuid."""
self.azure_obj.source_uuid = fake.uuid4()
self.azure_obj.pending_update = False
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
test_resource_group = "TESTRG"
test_storage_account = "testsa"
validated_data = {
"billing_source": {
"data_source": {"resource_group": test_resource_group, "storage_account": test_storage_account}
}
}
with patch("sources.api.serializers.ServerProxy") as mock_client:
mock_sources_client = MockSourcesClient("http://mock-soures-client")
mock_client.return_value.__enter__.return_value = mock_sources_client
instance = serializer.update(self.azure_obj, validated_data)
self.assertTrue(instance.pending_update)
def test_azure_source_update_missing_credential(self):
"""Test the updating azure source with invalid authentication."""
self.azure_obj.authentication = {}
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
validated_data = {
"authentication": {
"credentials": {
"subscription_id": "subscription-uuid"
}
}
}
instance = serializer.update(self.azure_obj, validated_data)
self.assertEqual(
"subscription-uuid",
instance.authentication.get("credentials").get("subscription_id"))
for field in ("client_id", "tenant_id", "client_secret"):
self.assertNotIn(field,
instance.authentication.get("credentials").keys())
def test_azure_source_billing_source_update_with_koku_uuid(
self, mock_delay):
"""Test the updating azure billing_source with source_uuid."""
self.azure_obj.source_uuid = fake.uuid4()
self.azure_obj.pending_update = False
self.azure_obj.save()
serializer = SourcesSerializer(context=self.request_context)
test_resource_group = "TESTRG"
test_storage_account = "testsa"
validated_data = {
"billing_source": {
"data_source": {
"resource_group": test_resource_group,
"storage_account": test_storage_account
}
}
}
instance = serializer.update(self.azure_obj, validated_data)
self.assertTrue(instance.pending_update)
def test_azure_source_authentication_update(self):
"""Test the updating azure subscription id."""
serializer = SourcesSerializer(context=self.request_context)
validated_data = {
"authentication": {
"credentials": {
"subscription_id": "subscription-uuid"
}
}
}
instance = serializer.update(self.azure_obj, validated_data)
self.assertIn("credentials", instance.authentication.keys())
self.assertIn("client_id",
instance.authentication.get("credentials").keys())
self.assertIn("tenant_id",
instance.authentication.get("credentials").keys())
self.assertIn("subscription_id",
instance.authentication.get("credentials").keys())
self.assertEqual(
"subscription-uuid",
instance.authentication.get("credentials").get("subscription_id"))
def test_azure_source_billing_source_update(self, mock_delay):
"""Test the updating azure billing_source."""
serializer = SourcesSerializer(context=self.request_context)
test_resource_group = "TESTRG"
test_storage_account = "testsa"
validated_data = {
"billing_source": {
"data_source": {
"resource_group": test_resource_group,
"storage_account": test_storage_account
}
}
}
instance = serializer.update(self.azure_obj, validated_data)
self.assertIn("data_source", instance.billing_source.keys())
self.assertEqual(
test_resource_group,
instance.billing_source.get("data_source").get("resource_group"))
self.assertEqual(
test_storage_account,
instance.billing_source.get("data_source").get("storage_account"))
def test_provider_create(self, mock_header, mock_request_info, _):
mock_request_info.return_value = self.User, self.Customer
serializer = AdminSourcesSerializer(context=self.request_context)
source = {
"source_id": 10,
"name": "ProviderAWS",
"source_type": "AWS",
"authentication": {
"credentials": {
"role_arn": "arn:aws:iam::111111111111:role/CostManagement"
}
},
"billing_source": {
"data_source": {
"bucket": "first-bucket"
}
},
"auth_header": Config.SOURCES_FAKE_HEADER,
"account_id": "acct10001",
"offset": 10,
}
with patch.object(ProviderAccessor,
"cost_usage_source_ready",
returns=True):
instance = serializer.create(source)
self.assertEqual(
instance.billing_source.get("data_source", {}).get("bucket"),
"first-bucket")
serializer = SourcesSerializer(context=self.request_context)
validated = {
"billing_source": {
"data_source": {
"bucket": "second-bucket"
}
}
}
with patch.object(ProviderAccessor,
"cost_usage_source_ready",
returns=True):
with patch("sources.api.serializers.ServerProxy") as mock_client:
mock_sources_client = MockSourcesClient(
"http://mock-soures-client")
mock_client.return_value.__enter__.return_value = mock_sources_client
instance2 = serializer.update(instance, validated)
self.assertEqual(
instance2.billing_source.get("data_source", {}).get("bucket"),
"second-bucket")
def test_update_aws_billing_source(self, _):
"""Test to validate that the billing source dictionary is updated."""
aws_instance = self.aws_obj
aws_instance.billing_source = PROVIDERS[Provider.PROVIDER_AWS].get(
"billing_source")
aws_instance.save()
test_matrix = [
{
"instance": aws_instance,
"billing_source": {
"bucket": "test-bucket"
},
"expected": {
"data_source": {
"bucket": "test-bucket"
}
},
},
{
"instance": aws_instance,
"billing_source": {
"data_source": {
"bucket": "test-bucket"
}
},
"expected": {
"data_source": {
"bucket": "test-bucket"
}
},
},
]
for test in test_matrix:
with self.subTest(test=test):
try:
new_billing = SourcesSerializer()._update_billing_source(
aws_instance, test.get("billing_source"))
self.assertEqual(new_billing, test.get("expected"))
except Exception as error:
self.fail(str(error))
def test_update_azure_billing_source(self, _):
"""Test to validate that the billing source dictionary is updated."""
azure_instance = self.azure_obj
azure_instance.billing_source = {
"data_source": {
"resource_group": "original-1",
"storage_account": "original-2"
}
}
azure_instance.save()
test_matrix = [
{
"instance": azure_instance,
"billing_source": {
"data_source": {
"resource_group": "foo",
"storage_account": "bar"
}
},
"expected": {
"data_source": {
"resource_group": "foo",
"storage_account": "bar"
}
},
},
{
"instance": azure_instance,
"billing_source": {
"data_source": {
"resource_group": "foo"
}
},
"expected": {
"data_source": {
"resource_group": "foo",
"storage_account": "original-2"
}
},
},
{
"instance": azure_instance,
"billing_source": {
"data_source": {
"storage_account": "bar"
}
},
"expected": {
"data_source": {
"resource_group": "original-1",
"storage_account": "bar"
}
},
},
]
for test in test_matrix:
with self.subTest(test=test):
try:
new_billing = SourcesSerializer()._update_billing_source(
azure_instance, test.get("billing_source"))
self.assertEqual(new_billing, test.get("expected"))
except Exception as error:
self.fail(str(error))
def test_validate_billing_source(self, _):
"""Test to validate that the billing source dictionary is valid."""
test_matrix = [
{
"provider_type": Provider.PROVIDER_AWS,
"billing_source": {
"bucket": "test-bucket"
},
"exception": False
},
{
"provider_type": Provider.PROVIDER_AWS,
"billing_source": {
"data_source": {
"bucket": "test-bucket"
}
},
"exception": False,
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {
"data_source": {
"resource_group": "foo",
"storage_account": "bar"
}
},
"exception": False,
},
{
"provider_type": Provider.PROVIDER_AWS,
"billing_source": {
"data_source": {
"nobucket": "test-bucket"
}
},
"exception": True,
},
{
"provider_type": Provider.PROVIDER_AWS,
"billing_source": {
"nobucket": "test-bucket"
},
"exception": True
},
{
"provider_type": Provider.PROVIDER_AWS,
"billing_source": {
"data_source": {}
},
"exception": True
},
{
"provider_type": Provider.PROVIDER_AWS,
"billing_source": {},
"exception": True
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {},
"exception": True
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {
"nodata_source": {
"resource_group": "foo",
"storage_account": "bar"
}
},
"exception": True,
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {
"data_source": {
"noresource_group": "foo",
"storage_account": "bar"
}
},
"exception": True,
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {
"data_source": {
"resource_group": "foo",
"nostorage_account": "bar"
}
},
"exception": True,
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {
"data_source": {
"resource_group": "foo"
}
},
"exception": True,
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {
"data_source": {
"storage_account": "bar"
}
},
"exception": True,
},
{
"provider_type": Provider.PROVIDER_GCP,
"billing_source": {
"data_source": {
"dataset": "test_dataset",
"table_id": "test_table_id"
}
},
"exception": False,
},
{
"provider_type": Provider.PROVIDER_GCP,
"billing_source": {
"data_source": {
"dataset": "test_dataset"
}
},
"exception": False,
},
{
"provider_type": Provider.PROVIDER_GCP,
"billing_source": {
"data_source": {
"table_id": "test_table_id"
}
},
"exception": True,
},
{
"provider_type": Provider.PROVIDER_GCP,
"billing_source": {},
"exception": True
},
]
for test in test_matrix:
with self.subTest(test=test):
if test.get("exception"):
with self.assertRaises(SourcesStorageError):
SourcesSerializer()._validate_billing_source(
test.get("provider_type"),
test.get("billing_source"))
else:
try:
SourcesSerializer()._validate_billing_source(
test.get("provider_type"),
test.get("billing_source"))
except Exception as error:
self.fail(str(error))
