def test_remove_users(self):
"""Remove users associated with a customer."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
customer_uuid = customer.uuid
# Add another user
group = Group.objects.get(name=customer.name)
new_user_dict = self.gen_user_data()
user_serializer = UserSerializer(data=new_user_dict)
new_user = None
if user_serializer.is_valid(raise_exception=True):
new_user = user_serializer.save()
group.user_set.add(new_user)
manager = CustomerManager(customer_uuid)
# Attempt to remove as customer owner
with self.assertRaises(CustomerManagerPermissionError):
manager.remove_users(customer.owner)
# Attempt to remove as regular user
with self.assertRaises(CustomerManagerPermissionError):
manager.remove_users(new_user)
# Attempt to remove as super user
superuser = User.objects.filter(is_superuser=True).first()
manager.remove_users(superuser)
self.assertFalse(manager.get_users_for_customer())
def test_get_providers_queryset_for_customer(self):
"""Verify all providers returned by a customer."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
# Verify no providers are returned
self.assertFalse(ProviderManager.get_providers_queryset_for_customer(customer).exists())
# Create Providers
provider_1 = Provider.objects.create(name='provider1', created_by=customer.owner, customer=customer)
provider_2 = Provider.objects.create(name='provider2', created_by=customer.owner, customer=customer)
providers = ProviderManager.get_providers_queryset_for_customer(customer)
# Verify providers are returned
provider_1_found = False
provider_2_found = False
for provider in providers:
if provider.uuid == provider_1.uuid:
provider_1_found = True
elif provider.uuid == provider_2.uuid:
provider_2_found = True
self.assertTrue(provider_1_found)
self.assertTrue(provider_2_found)
self.assertEqual((len(providers)), 2)
def test_is_removable_by_user(self):
"""Can current user remove provider."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
# Create Provider
provider = Provider.objects.create(name='providername', created_by=customer.owner, customer=customer)
provider_uuid = provider.uuid
# Create another user for negative tests
# Add another user
group = Group.objects.get(name=customer.name)
new_user_dict = self.gen_user_data()
user_serializer = UserSerializer(data=new_user_dict)
new_user = None
if user_serializer.is_valid(raise_exception=True):
new_user = user_serializer.save()
group.user_set.add(new_user)
manager = ProviderManager(provider_uuid)
self.assertTrue(manager.is_removable_by_user(customer.owner))
self.assertFalse(manager.is_removable_by_user(new_user))
superuser = User.objects.filter(is_superuser=True).first()
self.assertFalse(manager.is_removable_by_user(superuser))
def test_remove(self):
"""Remove provider."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
# Create Provider
provider = Provider.objects.create(name='providername', created_by=customer.owner, customer=customer)
provider_uuid = provider.uuid
# Create another user for negative tests
# Add another user
group = Group.objects.get(name=customer.name)
new_user_dict = self.gen_user_data()
user_serializer = UserSerializer(data=new_user_dict)
other_user = None
if user_serializer.is_valid(raise_exception=True):
other_user = user_serializer.save()
group.user_set.add(other_user)
manager = ProviderManager(provider_uuid)
with self.assertRaises(ProviderManagerError):
self.assertFalse(manager.remove(other_user))
manager.remove(customer.owner)
provider_query = Provider.objects.all().filter(uuid=provider_uuid)
self.assertFalse(provider_query)
def test_get_tenant(self):
"""Get the tenant object for a customer."""
# Create customer.
customer_name = 'test_customer_tenant'
customer_json = {'name': customer_name,
'owner': self.gen_user_data()}
customer = None
serializer = CustomerSerializer(data=customer_json)
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
customer_uuid = customer.uuid
# Create tenant
customer_obj = Customer.objects.filter(name=customer_name).get()
tenant = Tenant(schema_name=customer_obj.schema_name)
tenant.save()
# Get manager
manager = CustomerManager(customer_uuid)
customer_obj = manager.get_model()
customer_schema_name = customer_obj.schema_name
# Verify tenant is returned from manager
tenant_obj = Tenant.objects.filter(schema_name=customer_schema_name).get()
self.assertEqual(manager.get_tenant(), tenant_obj)
def test_create_provider_with_exception(self):
"""Test creating a provider with a provider exception."""
iam_arn = 'arn:aws:s3:::my_s3_bucket'
bucket_name = 'my_s3_bucket'
provider = {
'name': 'test_provider',
'type': Provider.PROVIDER_AWS,
'authentication': {
'provider_resource_name': iam_arn
},
'billing_source': {
'bucket': bucket_name
}
}
new_cust = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
new_cust = serializer.save()
request = Mock()
request.user = new_cust.owner
context = {'request': request}
with patch.object(ProviderAccessor,
'cost_usage_source_ready',
side_effect=serializers.ValidationError):
ProviderSerializer(data=provider, context=context)
def test_removing_tenant(self):
"""Remove the tenant object for a customer."""
# Create customer.
customer_name = 'test_customer_tenant'
customer_json = {'name': customer_name,
'owner': self.gen_user_data()}
customer = None
serializer = CustomerSerializer(data=customer_json)
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
customer_uuid = customer.uuid
# Create tenant
customer_obj = Customer.objects.filter(name=customer_name).get()
tenant = Tenant(schema_name=customer_obj.schema_name)
tenant.save()
# Get manager
manager = CustomerManager(customer_uuid)
customer_obj = manager.get_model()
customer_schema_name = customer_obj.schema_name
# Verify tenant is returned from manager
tenant_obj = Tenant.objects.filter(schema_name=customer_schema_name).get()
self.assertEqual(manager.get_tenant(), tenant_obj)
# Attempt to remove as customer owner
with self.assertRaises(CustomerManagerPermissionError):
manager.remove_tenant(customer_obj.owner)
# Attempt to remove tenant as super user
superuser = User.objects.filter(is_superuser=True).first()
manager.remove_tenant(superuser)
def test_get_users_for_customer(self):
"""Can current user remove provider."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
customer_uuid = customer.uuid
# Add another user
group = Group.objects.get(name=customer.name)
new_user_dict = self.gen_user_data()
user_serializer = UserSerializer(data=new_user_dict)
new_user = None
if user_serializer.is_valid(raise_exception=True):
new_user = user_serializer.save()
group.user_set.add(new_user)
manager = CustomerManager(customer_uuid)
users = manager.get_users_for_customer()
self.assertEqual(len(users), 2)
new_user_found = False
owner_found = False
for found_user in users:
if found_user.username == new_user_dict['username']:
new_user_found = True
elif found_user.username == customer.owner.username:
owner_found = True
self.assertTrue(new_user_found)
self.assertTrue(owner_found)
def test_uuid_field(self):
"""Test that a uuid is generated."""
# create the customer
instance = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
instance = serializer.save()
self.assertIsInstance(instance.uuid, uuid.UUID)
def test_get_name(self):
"""Can the customer name be returned."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
customer_uuid = customer.uuid
manager = CustomerManager(customer_uuid)
self.assertEqual(manager.get_name(), self.customer_data[0]['name'])
def test_create_customer(self):
"""Test creating a customer."""
# create the customers
for customer in self.customer_data:
instance = None
serializer = CustomerSerializer(data=customer)
if serializer.is_valid(raise_exception=True):
instance = serializer.save()
self.assertEqual(customer['name'], instance.name)
self.assertTrue(
ResetToken.objects.filter(user=instance.owner).exists)
def test_provider_org_fail(self, check_org_access):
"""Test creating a provider with AWS org access failure."""
check_org_access.return_value = False
iam_arn = 'arn:aws:s3:::my_s3_bucket'
bucket_name = 'my_s3_bucket'
access_key_id, secret_access_key, session_token = _get_sts_access(
iam_arn)
s3_resource = boto3.resource(
's3',
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
aws_session_token=session_token,
)
s3_resource.create_bucket(Bucket=bucket_name)
provider = {'name': 'test_provider',
'type': Provider.PROVIDER_AWS,
'authentication': {
'provider_resource_name': iam_arn
},
'billing_source': {
'bucket': bucket_name
}}
new_cust = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
new_cust = serializer.save()
request = Mock()
request.user = new_cust.owner
context = {'request': request}
serializer = ProviderSerializer(data=provider, context=context)
if serializer.is_valid(raise_exception=True):
serializer.save()
check_org_access.assert_called_once()
def test_provider_s3_fail(self, check_s3_access):
"""Test creating a provider with AWS s3 bucket doesn't exist."""
check_s3_access.return_value = False
iam_arn = 'arn:aws:s3:::my_s3_bucket'
bucket_name = 'my_s3_bucket'
provider = {'name': 'test_provider',
'type': Provider.PROVIDER_AWS,
'authentication': {
'provider_resource_name': iam_arn
},
'billing_source': {
'bucket': bucket_name
}}
new_cust = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
new_cust = serializer.save()
request = Mock()
request.user = new_cust.owner
context = {'request': request}
serializer = ProviderSerializer(data=provider, context=context)
if serializer.is_valid(raise_exception=True):
with self.assertRaises(serializers.ValidationError):
serializer.save()
check_s3_access.assert_called_once(iam_arn)
def test_create_customer(self):
"""Test creating a customer."""
# create the customers
customer = self._create_customer_data()
instance = None
serializer = CustomerSerializer(data=customer)
if serializer.is_valid(raise_exception=True):
instance = serializer.save()
schema_name = serializer.data.get('schema_name')
self.assertIsNone(schema_name)
self.assertFalse('schema_name' in serializer.data)
self.assertEqual(customer['account_id'], instance.account_id)
self.assertIsInstance(instance.uuid, uuid.UUID)
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256, required=True,
allow_null=False, allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
authentication = ProviderAuthenticationSerializer()
billing_source = ProviderBillingSourceSerializer()
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by')
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
authentication = validated_data.pop('authentication')
billing_source = validated_data.pop('billing_source')
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = User.objects.get(pk=request.user.id)
if user.groups.count() == 1:
group = user.groups.first()
customer = Customer.objects.get(pk=group.id)
else:
key = 'customer'
message = 'Group for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
provider_resource_name = authentication.get('provider_resource_name')
bucket = billing_source.get('bucket')
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
interface.cost_usage_source_ready(provider_resource_name, bucket)
auth = ProviderAuthentication.objects.create(**authentication)
bill = ProviderBillingSource.objects.create(**billing_source)
auth.save()
bill.save()
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider
def test_get_tenant_not_found(self):
"""Try to get a missing tenant object for a customer."""
# Create customer.
customer_name = 'test_customer_tenant'
customer_json = {'name': customer_name, 'owner': self.gen_user_data()}
customer = None
serializer = CustomerSerializer(data=customer_json)
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
customer_uuid = customer.uuid
# Get manager
manager = CustomerManager(customer_uuid)
self.assertEqual(manager.get_tenant(), None)
def test_create_provider(self):
"""Test creating a provider."""
iam_arn = 'arn:aws:s3:::my_s3_bucket'
bucket_name = 'my_s3_bucket'
provider = {'name': 'test_provider',
'type': Provider.PROVIDER_AWS,
'authentication': {
'provider_resource_name': iam_arn
},
'billing_source': {
'bucket': bucket_name
}}
new_cust = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
new_cust = serializer.save()
request = Mock()
request.user = new_cust.owner
context = {'request': request}
instance = None
with patch.object(ProviderAccessor, 'cost_usage_source_ready', returns=True):
serializer = ProviderSerializer(data=provider, context=context)
if serializer.is_valid(raise_exception=True):
instance = serializer.save()
schema_name = serializer.data['customer'].get('schema_name')
self.assertIsInstance(instance.uuid, uuid.UUID)
self.assertIsNone(schema_name)
self.assertFalse('schema_name' in serializer.data['customer'])
def test_get_name(self):
"""Can the provider name be returned."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
# Create Provider
provider_name = 'sample_provider'
provider = Provider.objects.create(name=provider_name, created_by=customer.owner, customer=customer)
# Get Provider UUID
provider_uuid = provider.uuid
# Get Provider Manager
manager = ProviderManager(provider_uuid)
self.assertEqual(manager.get_name(), provider_name)
def test_remove(self):
"""Remove provider."""
# Create Customer
customer = None
serializer = CustomerSerializer(data=self.customer_data[0])
if serializer.is_valid(raise_exception=True):
customer = serializer.save()
# Create Provider
provider_authentication = ProviderAuthentication.objects.create(provider_resource_name='arn:aws:iam::2:role/mg')
provider_billing = ProviderBillingSource.objects.create(bucket='my_s3_bucket')
provider = Provider.objects.create(name='providername',
created_by=customer.owner,
customer=customer,
authentication=provider_authentication,
billing_source=provider_billing)
provider_uuid = provider.uuid
# Create another user for negative tests
# Add another user
group = Group.objects.get(name=customer.name)
new_user_dict = self.gen_user_data()
user_serializer = UserSerializer(data=new_user_dict)
other_user = None
if user_serializer.is_valid(raise_exception=True):
other_user = user_serializer.save()
group.user_set.add(other_user)
manager = ProviderManager(provider_uuid)
with self.assertRaises(ProviderManagerError):
self.assertFalse(manager.remove(other_user))
manager.remove(customer.owner)
provider_query = Provider.objects.all().filter(uuid=provider_uuid)
self.assertFalse(provider_query)
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256, required=True,
allow_null=False, allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
created_timestamp = serializers.DateTimeField(read_only=True)
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
# pylint: disable=too-few-public-methods
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by', 'created_timestamp')
def __init__(self, instance=None, data=empty, **kwargs):
"""Initialize the Provider Serializer.
Here we ensure we use the appropriate serializer to validate the
authentication and billing_source parameters.
"""
super().__init__(instance, data, **kwargs)
provider_type = None
if data and data != empty:
provider_type = data.get('type')
if provider_type and provider_type.lower() not in PROVIDER_CHOICE_LIST:
key = 'type'
message = f'{provider_type} is not a valid source type.'
raise serializers.ValidationError(error_obj(key, message))
if provider_type:
self.fields['authentication'] = AUTHENTICATION_SERIALIZERS.get(provider_type)()
self.fields['billing_source'] = BILLING_SOURCE_SERIALIZERS.get(provider_type)(
default={'bucket': '', 'data_source': {'bucket': ''}}
)
else:
self.fields['authentication'] = ProviderAuthenticationSerializer()
self.fields['billing_source'] = ProviderBillingSourceSerializer()
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = request.user
if user.customer:
customer = user.customer
else:
key = 'customer'
message = 'Customer for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
if 'billing_source' in validated_data:
billing_source = validated_data.pop('billing_source')
data_source = billing_source.get('data_source', {})
bucket = data_source.get('bucket')
else:
# Because of a unique together constraint, this is done
# to allow for this field to be non-required for OCP
# but will still have a blank no-op entry in the DB
billing_source = {'bucket': '', 'data_source': {}}
data_source = None
authentication = validated_data.pop('authentication')
credentials = authentication.get('credentials')
provider_resource_name = credentials.get('provider_resource_name')
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
if credentials and data_source and provider_type not in ['AWS', 'OCP']:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name, bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(**authentication)
# We can re-use a billing source or a auth, but not the same combination.
unique_count = Provider.objects.filter(authentication=auth)\
.filter(billing_source=bill).count()
if unique_count != 0:
existing_provider = Provider.objects.filter(authentication=auth)\
.filter(billing_source=bill).first()
if existing_provider.type in ('AWS', 'OCP'):
sources_auth = {'resource_name': provider_resource_name}
elif existing_provider.type in ('AZURE', ):
sources_auth = {'credentials': auth.credentials}
else:
sources_auth = {}
source_query = Sources.objects.filter(authentication=sources_auth)
if source_query.exists():
source_obj = source_query.first()
source_obj.koku_uuid = existing_provider.uuid
source_obj.save()
return existing_provider
error = {'Error': 'A Provider already exists with that Authentication and Billing Source'}
raise serializers.ValidationError(error)
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider
@transaction.atomic
def update(self, instance, validated_data):
"""Update a Provider instance from validated data."""
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop('authentication')
credentials = authentication.get('credentials')
provider_resource_name = credentials.get('provider_resource_name')
billing_source = validated_data.pop('billing_source')
data_source = billing_source.get('data_source')
bucket = billing_source.get('bucket')
if credentials and data_source and provider_type not in ['AWS', 'OCP']:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name, bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(**authentication)
for key in validated_data.keys():
setattr(instance, key, validated_data[key])
instance.authentication = auth
instance.billing_source = bill
try:
instance.save()
except IntegrityError:
error = {'Error': 'A Provider already exists with that Authentication and Billing Source'}
raise serializers.ValidationError(error)
return instance
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(allow_null=True, required=False)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=LCASE_PROVIDER_CHOICE_LIST)
created_timestamp = serializers.DateTimeField(read_only=True)
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
active = serializers.BooleanField(read_only=True)
paused = serializers.BooleanField(required=False)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = (
"uuid",
"name",
"type",
"authentication",
"billing_source",
"customer",
"created_by",
"created_timestamp",
"active",
"paused",
)
def __init__(self, instance=None, data=empty, **kwargs):
"""Initialize the Provider Serializer.
Here we ensure we use the appropriate serializer to validate the
authentication and billing_source parameters.
"""
super().__init__(instance, data, **kwargs)
provider_type = None
if data and data != empty:
provider_type = data.get("type")
if provider_type and provider_type.lower(
) not in LCASE_PROVIDER_CHOICE_LIST:
key = "type"
message = f"{provider_type} is not a valid source type."
raise serializers.ValidationError(error_obj(key, message))
if provider_type:
provider_type = provider_type.lower()
self.fields["authentication"] = AUTHENTICATION_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))()
self.fields["billing_source"] = BILLING_SOURCE_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))()
else:
self.fields["authentication"] = ProviderAuthenticationSerializer()
self.fields["billing_source"] = ProviderBillingSourceSerializer()
@property
def demo_credentials(self):
"""Build formatted credentials for our nise-populator demo accounts."""
creds_by_source_type = defaultdict(list)
for account, cred_dict in settings.DEMO_ACCOUNTS.items():
for cred, info in cred_dict.items():
if info.get("source_type") == Provider.PROVIDER_AWS:
creds_by_source_type[Provider.PROVIDER_AWS].append(
{"role_arn": cred})
elif info.get("source_type") == Provider.PROVIDER_AZURE:
creds_by_source_type[Provider.PROVIDER_AZURE].append(
{"client_id": cred})
elif info.get("source_type") == Provider.PROVIDER_GCP:
creds_by_source_type[Provider.PROVIDER_GCP].append(
{"project_id": cred})
return creds_by_source_type
def get_request_info(self):
"""Obtain request information like user and customer context."""
user = self.context.get("user")
customer = self.context.get("customer")
if user and customer:
return user, customer
request = self.context.get("request")
if request and hasattr(request, "user"):
user = request.user
if user.customer:
customer = user.customer
else:
key = "customer"
message = "Customer for requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
else:
key = "created_by"
message = "Requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
return user, customer
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
user, customer = self.get_request_info()
provider_type = validated_data["type"].lower()
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source")
if self._is_demo_account(provider_type, credentials):
LOG.info(
"Customer account is a DEMO account. Skipping cost_usage_source_ready check."
)
else:
interface.cost_usage_source_ready(credentials, data_source)
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
# We can re-use a billing source or a auth, but not the same combination.
dup_queryset = (Provider.objects.filter(authentication=auth).filter(
billing_source=bill).filter(customer=customer))
if dup_queryset.count() != 0:
conflict_provider = dup_queryset.first()
message = (
f"Cost management does not allow duplicate accounts. "
f"{conflict_provider.name} already exists. Edit source settings to configure a new source."
)
LOG.warn(message)
raise serializers.ValidationError(
error_obj(ProviderErrors.DUPLICATE_AUTH, message))
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.active = True
provider.save()
customer.date_updated = DateHelper().now_utc
customer.save()
return provider
def update(self, instance, validated_data):
"""Update a Provider instance from validated data."""
_, customer = self.get_request_info()
provider_type = validated_data["type"].lower()
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source")
# updating `paused` must happen regardless of Provider availabilty
instance.paused = validated_data.pop("paused", instance.paused)
try:
if self._is_demo_account(provider_type, credentials):
LOG.info(
"Customer account is a DEMO account. Skipping cost_usage_source_ready check."
)
else:
interface.cost_usage_source_ready(credentials, data_source)
except serializers.ValidationError as validation_error:
instance.active = False
instance.save()
raise validation_error
with transaction.atomic():
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
if instance.billing_source != bill or instance.authentication != auth:
dup_queryset = (Provider.objects.filter(
authentication=auth).filter(billing_source=bill).filter(
customer=customer))
if dup_queryset.count() != 0:
conflict_provder = dup_queryset.first()
message = (
f"Cost management does not allow duplicate accounts. "
f"{conflict_provder.name} already exists. Edit source settings to configure a new source."
)
LOG.warn(message)
raise serializers.ValidationError(
error_obj(ProviderErrors.DUPLICATE_AUTH, message))
for key in validated_data.keys():
setattr(instance, key, validated_data[key])
instance.authentication = auth
instance.billing_source = bill
instance.active = True
instance.save()
customer.date_updated = DateHelper().now_utc
customer.save()
return instance
def _is_demo_account(self, provider_type, credentials):
"""Test whether this source is a demo account."""
key_types = {
Provider.PROVIDER_AWS: "role_arn",
Provider.PROVIDER_AZURE: "client_id",
Provider.PROVIDER_GCP: "project_id",
}
key_to_check = key_types.get(provider_type, "")
creds_to_check = self.demo_credentials.get(provider_type, [])
for cred in creds_to_check:
if credentials.get(key_to_check,
True) == cred.get(key_to_check, False):
return True
return False
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(allow_null=True, required=False)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=LCASE_PROVIDER_CHOICE_LIST)
created_timestamp = serializers.DateTimeField(read_only=True)
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
active = serializers.BooleanField(read_only=True)
# pylint: disable=too-few-public-methods
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = (
"uuid",
"name",
"type",
"authentication",
"billing_source",
"customer",
"created_by",
"created_timestamp",
"active",
)
def __init__(self, instance=None, data=empty, **kwargs):
"""Initialize the Provider Serializer.
Here we ensure we use the appropriate serializer to validate the
authentication and billing_source parameters.
"""
super().__init__(instance, data, **kwargs)
provider_type = None
if data and data != empty:
provider_type = data.get("type")
if provider_type and provider_type.lower(
) not in LCASE_PROVIDER_CHOICE_LIST:
key = "type"
message = f"{provider_type} is not a valid source type."
raise serializers.ValidationError(error_obj(key, message))
if provider_type:
provider_type = provider_type.lower()
self.fields["authentication"] = AUTHENTICATION_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))()
self.fields["billing_source"] = BILLING_SOURCE_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))(default={
"bucket": "",
"data_source": {
"bucket": ""
}
})
else:
self.fields["authentication"] = ProviderAuthenticationSerializer()
self.fields["billing_source"] = ProviderBillingSourceSerializer()
def get_request_info(self):
"""Obtain request information like user and customer context."""
user = None
customer = None
request = self.context.get("request")
if request and hasattr(request, "user"):
user = request.user
if user.customer:
customer = user.customer
else:
key = "customer"
message = "Customer for requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
else:
key = "created_by"
message = "Requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
return request, user, customer
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
_, user, customer = self.get_request_info()
if "billing_source" in validated_data:
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source", {})
bucket = data_source.get("bucket")
else:
# Because of a unique together constraint, this is done
# to allow for this field to be non-required for OCP
# but will still have a blank no-op entry in the DB
billing_source = {"bucket": "", "data_source": {}}
data_source = None
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
provider_resource_name = credentials.get("provider_resource_name")
provider_type = validated_data["type"]
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
interface = ProviderAccessor(provider_type)
if customer.account_id not in settings.DEMO_ACCOUNTS:
if credentials and data_source and provider_type not in [
Provider.PROVIDER_AWS, Provider.PROVIDER_OCP
]:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name,
bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
# We can re-use a billing source or a auth, but not the same combination.
unique_count = (Provider.objects.filter(authentication=auth).filter(
billing_source=bill).filter(customer=customer).count())
if unique_count != 0:
error = {
"Error":
"A Provider already exists with that Authentication and Billing Source"
}
raise serializers.ValidationError(error)
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.active = True
provider.save()
return provider
def update(self, instance, validated_data):
"""Update a Provider instance from validated data."""
_, _, customer = self.get_request_info()
provider_type = validated_data["type"].lower()
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
if instance.type != provider_type:
error = {
"Error":
"The Provider Type cannot be changed with a PUT request."
}
raise serializers.ValidationError(error)
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
provider_resource_name = credentials.get("provider_resource_name")
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source")
bucket = billing_source.get("bucket")
try:
if customer.account_id not in settings.DEMO_ACCOUNTS:
if credentials and data_source and provider_type not in [
Provider.PROVIDER_AWS, Provider.PROVIDER_OCP
]:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name,
bucket)
except serializers.ValidationError as validation_error:
instance.active = False
instance.save()
raise validation_error
with transaction.atomic():
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
for key in validated_data.keys():
setattr(instance, key, validated_data[key])
instance.authentication = auth
instance.billing_source = bill
instance.active = True
try:
instance.save()
except IntegrityError:
error = {
"Error":
"A Provider already exists with that Authentication and Billing Source"
}
raise serializers.ValidationError(error)
return instance
def get_users_for_customer(self):
"""List of users that belong to the customer."""
group = CustomerSerializer.get_authentication_group_for_customer(
self.model)
users = CustomerSerializer.get_users_for_group(group)
return users
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
authentication = ProviderAuthenticationSerializer()
billing_source = ProviderBillingSourceSerializer(default={'bucket': ''})
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by')
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = request.user
if user.customer:
customer = user.customer
else:
key = 'customer'
message = 'Customer for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
if 'billing_source' in validated_data:
billing_source = validated_data.pop('billing_source')
bucket = billing_source.get('bucket')
else:
# Because of a unique together constraint, this is done
# to allow for this field to be non-required for OCP
# but will still have a blank no-op entry in the DB
billing_source = {'bucket': ''}
authentication = validated_data.pop('authentication')
provider_resource_name = authentication.get('provider_resource_name')
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
interface.cost_usage_source_ready(provider_resource_name, bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
# We can re-use a billing source or a auth, but not the same combination.
unique_count = Provider.objects.filter(authentication=auth)\
.filter(billing_source=bill).count()
if unique_count != 0:
error = {
'Error':
'A Provider already exists with that Authentication and Billing Source'
}
raise serializers.ValidationError(error)
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
authentication = ProviderAuthenticationSerializer()
billing_source = ProviderBillingSourceSerializer()
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by')
@transaction.atomic
def create(self, validated_data):
"""Create a user from validated data."""
authentication = validated_data.pop('authentication')
billing_source = validated_data.pop('billing_source')
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = User.objects.get(pk=request.user.id)
if user.groups.count() == 1:
group = user.groups.first()
customer = Customer.objects.get(pk=group.id)
else:
key = 'customer'
message = 'Group for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
provider_resource_name = authentication.get('provider_resource_name')
bucket = billing_source.get('bucket')
access_key_id, secret_access_key, session_token = _get_sts_access(
provider_resource_name)
if (access_key_id is None or access_key_id is None
or session_token is None):
key = 'provider_resource_name'
message = 'Unable to obtain credentials with using {}.'.format(
provider_resource_name)
raise serializers.ValidationError(error_obj(key, message))
s3_exists = _check_s3_access(access_key_id, secret_access_key,
session_token, bucket)
if not s3_exists:
key = 'bucket'
message = 'Bucket {} could not be found with {}.'.format(
bucket, provider_resource_name)
raise serializers.ValidationError(error_obj(key, message))
org_access = _check_org_access(access_key_id, secret_access_key,
session_token)
if not org_access:
key = 'provider_resource_name'
message = 'Unable to obtain organization data with {}.'.format(
provider_resource_name)
LOG.info(message)
auth = ProviderAuthentication.objects.create(**authentication)
bill = ProviderBillingSource.objects.create(**billing_source)
auth.save()
bill.save()
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider
