def test_user_preference_locale(self):
"""Test that valid locales are saved."""
user = None
serializer = UserSerializer(data=self.user_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
kwargs = {'context': {'user': user}}
data = {'name': 'locale',
'description': self.fake.text(),
'preference': {'locale': 'tg_TJ.KOI8-C'}}
pref = list(UserPreference.objects.filter(user=user, name='locale')).pop()
self.assertIsNotNone(pref)
self.assertIsInstance(pref, UserPreference)
serializer = UserPreferenceSerializer(pref, data=data, **kwargs)
if serializer.is_valid(raise_exception=True):
serializer.save()
self.assertEqual(pref.name, data.get('name'))
self.assertEqual(pref.description, data.get('description'))
self.assertEqual(pref.preference, data.get('preference'))
def setUp(self):
"""Set up the tests."""
super().setUp()
request = self.request_context['request']
serializer = UserSerializer(data=self.user_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
request.user = user
provider_data = {
'name': 'test_provider',
'type': Provider.PROVIDER_OCP,
'authentication': {
'provider_resource_name': self.fake.word()
}
}
serializer = ProviderSerializer(data=provider_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
self.provider = serializer.save()
def test_user_preference_currency(self):
"""Test that valid currency codes are saved."""
user = None
serializer = UserSerializer(data=self.user_data)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
kwargs = {'context': {'user': user}}
data = {'name': 'currency',
'description': self.fake.text(),
'preference': {'currency': random.choice(_currency_symbols())}}
pref = list(UserPreference.objects.filter(user=user, name='currency')).pop()
self.assertIsNotNone(pref)
self.assertIsInstance(pref, UserPreference)
serializer = UserPreferenceSerializer(pref, data=data, **kwargs)
if serializer.is_valid(raise_exception=True):
pref = serializer.save()
self.assertEqual(pref.name, data.get('name'))
self.assertEqual(pref.description, data.get('description'))
self.assertEqual(pref.preference, data.get('preference'))
def test_user_preference_timezone(self):
"""Test that valid timezones are saved."""
user = None
serializer = UserSerializer(data=self.user_data)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
kwargs = {'context': {'user': user}}
data = {'name': 'timezone',
'description': self.fake.text(),
'preference': {'timezone': 'Antarctica/Troll'}}
pref = list(UserPreference.objects.filter(user=user, name='timezone')).pop()
self.assertIsNotNone(pref)
self.assertIsInstance(pref, UserPreference)
serializer = UserPreferenceSerializer(pref, data=data, **kwargs)
if serializer.is_valid(raise_exception=True):
pref = serializer.save()
self.assertEqual(pref.name, data.get('name'))
self.assertEqual(pref.description, data.get('description'))
self.assertEqual(pref.preference, data.get('preference'))
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
authentication = ProviderAuthenticationSerializer()
billing_source = ProviderBillingSourceSerializer()
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by')
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
authentication = validated_data.pop('authentication')
billing_source = validated_data.pop('billing_source')
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = User.objects.get(pk=request.user.id)
if user.groups.count() == 1:
group = user.groups.first()
customer = Customer.objects.get(pk=group.id)
else:
key = 'customer'
message = 'Group for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
provider_resource_name = authentication.get('provider_resource_name')
bucket = billing_source.get('bucket')
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
interface.cost_usage_source_ready(provider_resource_name, bucket)
auth = ProviderAuthentication.objects.create(**authentication)
bill = ProviderBillingSource.objects.create(**billing_source)
auth.save()
bill.save()
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(allow_null=True, required=False)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=LCASE_PROVIDER_CHOICE_LIST)
created_timestamp = serializers.DateTimeField(read_only=True)
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
active = serializers.BooleanField(read_only=True)
# pylint: disable=too-few-public-methods
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = (
"uuid",
"name",
"type",
"authentication",
"billing_source",
"customer",
"created_by",
"created_timestamp",
"active",
)
def __init__(self, instance=None, data=empty, **kwargs):
"""Initialize the Provider Serializer.
Here we ensure we use the appropriate serializer to validate the
authentication and billing_source parameters.
"""
super().__init__(instance, data, **kwargs)
provider_type = None
if data and data != empty:
provider_type = data.get("type")
if provider_type and provider_type.lower(
) not in LCASE_PROVIDER_CHOICE_LIST:
key = "type"
message = f"{provider_type} is not a valid source type."
raise serializers.ValidationError(error_obj(key, message))
if provider_type:
provider_type = provider_type.lower()
self.fields["authentication"] = AUTHENTICATION_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))()
self.fields["billing_source"] = BILLING_SOURCE_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))(default={
"bucket": "",
"data_source": {
"bucket": ""
}
})
else:
self.fields["authentication"] = ProviderAuthenticationSerializer()
self.fields["billing_source"] = ProviderBillingSourceSerializer()
def get_request_info(self):
"""Obtain request information like user and customer context."""
user = None
customer = None
request = self.context.get("request")
if request and hasattr(request, "user"):
user = request.user
if user.customer:
customer = user.customer
else:
key = "customer"
message = "Customer for requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
else:
key = "created_by"
message = "Requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
return request, user, customer
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
_, user, customer = self.get_request_info()
if "billing_source" in validated_data:
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source", {})
bucket = data_source.get("bucket")
else:
# Because of a unique together constraint, this is done
# to allow for this field to be non-required for OCP
# but will still have a blank no-op entry in the DB
billing_source = {"bucket": "", "data_source": {}}
data_source = None
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
provider_resource_name = credentials.get("provider_resource_name")
provider_type = validated_data["type"]
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
interface = ProviderAccessor(provider_type)
if customer.account_id not in settings.DEMO_ACCOUNTS:
if credentials and data_source and provider_type not in [
Provider.PROVIDER_AWS, Provider.PROVIDER_OCP
]:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name,
bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
# We can re-use a billing source or a auth, but not the same combination.
unique_count = (Provider.objects.filter(authentication=auth).filter(
billing_source=bill).filter(customer=customer).count())
if unique_count != 0:
error = {
"Error":
"A Provider already exists with that Authentication and Billing Source"
}
raise serializers.ValidationError(error)
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.active = True
provider.save()
return provider
def update(self, instance, validated_data):
"""Update a Provider instance from validated data."""
_, _, customer = self.get_request_info()
provider_type = validated_data["type"].lower()
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
if instance.type != provider_type:
error = {
"Error":
"The Provider Type cannot be changed with a PUT request."
}
raise serializers.ValidationError(error)
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
provider_resource_name = credentials.get("provider_resource_name")
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source")
bucket = billing_source.get("bucket")
try:
if customer.account_id not in settings.DEMO_ACCOUNTS:
if credentials and data_source and provider_type not in [
Provider.PROVIDER_AWS, Provider.PROVIDER_OCP
]:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name,
bucket)
except serializers.ValidationError as validation_error:
instance.active = False
instance.save()
raise validation_error
with transaction.atomic():
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
for key in validated_data.keys():
setattr(instance, key, validated_data[key])
instance.authentication = auth
instance.billing_source = bill
instance.active = True
try:
instance.save()
except IntegrityError:
error = {
"Error":
"A Provider already exists with that Authentication and Billing Source"
}
raise serializers.ValidationError(error)
return instance
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
serializer = UserSerializer(data=self.user_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
self.report = {
'group_by': {
'account': ['*']
},
'filter': {
'resolution': 'monthly',
'time_scope_value': -1,
'time_scope_units': 'month',
'resource_scope': []
},
'data': [{
'date':
'2018-07',
'accounts': [{
'account':
'4418636104713',
'values': [{
'date': '2018-07',
'units': 'GB-Mo',
'account': '4418636104713',
'total': 1826.74238146924
}]
}, {
'account':
'8577742690384',
'values': [{
'date': '2018-07',
'units': 'GB-Mo',
'account': '8577742690384',
'total': 1137.74036198065
}]
}, {
'account':
'3474227945050',
'values': [{
'date': '2018-07',
'units': 'GB-Mo',
'account': '3474227945050',
'total': 1045.80659412797
}]
}, {
'account':
'7249815104968',
'values': [{
'date': '2018-07',
'units': 'GB-Mo',
'account': '7249815104968',
'total': 807.326470618818
}]
}, {
'account':
'9420673783214',
'values': [{
'date': '2018-07',
'units': 'GB-Mo',
'account': '9420673783214',
'total': 658.306642830709
}]
}]
}],
'total': {
'value': 5475.922451027388,
'units': 'GB-Mo'
}
}
def setUp(self):
"""Set up the rate view tests."""
super().setUp()
request = self.request_context['request']
serializer = UserSerializer(data=self.user_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
request.user = user
provider_data = {
'name': 'test_provider',
'type': Provider.PROVIDER_OCP,
'authentication': {
'provider_resource_name': self.fake.word()
}
}
serializer = ProviderSerializer(data=provider_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
self.provider = serializer.save()
self.fake_data = {
'provider_uuid':
self.provider.uuid,
'metric':
Rate.METRIC_MEM_GB_USAGE_HOUR,
'tiered_rate': [{
'value': round(Decimal(random.random()), 6),
'unit': 'USD',
'usage_start': None,
'usage_end': None
}]
}
with tenant_context(self.tenant):
serializer = RateSerializer(data=self.fake_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
serializer = UserSerializer(data=self.user_data, context=self.request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
self.report = {
'group_by': {
'subscription_guid': ['*']
},
'filter': {
'resolution': 'monthly',
'time_scope_value': -1,
'time_scope_units': 'month',
'resource_scope': []
},
'data': [
{
'date': '2018-07',
'subscription_guids': [
{
'subscription_guid': '00000000-0000-0000-0000-000000000000',
'values': [
{
'date': '2018-07',
'units': 'GB-Mo',
'subscription_guid': '00000000-0000-0000-0000-000000000000',
'total': 1826.74238146924
}
]
},
{
'subscription_guid': '11111111-1111-1111-1111-111111111111',
'values': [
{
'date': '2018-07',
'units': 'GB-Mo',
'subscription_guid': '11111111-1111-1111-1111-111111111111',
'total': 1137.74036198065
}
]
},
{
'subscription_guid': '22222222-2222-2222-2222-222222222222',
'values': [
{
'date': '2018-07',
'units': 'GB-Mo',
'subscription_guid': '22222222-2222-2222-2222-222222222222',
'total': 1045.80659412797
}
]
},
{
'subscription_guid': '33333333-3333-3333-3333-333333333333',
'values': [
{
'date': '2018-07',
'units': 'GB-Mo',
'subscription_guid': '33333333-3333-3333-3333-333333333333',
'total': 807.326470618818
}
]
},
{
'subscription_guid': '44444444-4444-4444-4444-444444444444',
'values': [
{
'date': '2018-07',
'units': 'GB-Mo',
'subscription_guid': '44444444-4444-4444-4444-444444444444',
'total': 658.306642830709
}
]
}
]
}
],
'total': {
'value': 5475.922451027388,
'units': 'GB-Mo'
}
}
def initialize_request(self, context=None):
"""Initialize model data."""
if context:
request_context = context.get("request_context")
request = request_context["request"]
serializer = UserSerializer(data=context.get("user_data"),
context=request_context)
else:
request_context = self.request_context
request = request_context["request"]
serializer = UserSerializer(data=self.user_data,
context=request_context)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
request.user = user
provider_data = {
"name": "test_provider",
"type": Provider.PROVIDER_OCP.lower(),
"authentication": {
"provider_resource_name": self.fake.word()
},
}
serializer = ProviderSerializer(data=provider_data,
context=request_context)
if serializer.is_valid(raise_exception=True):
self.provider = serializer.save()
self.ocp_metric = CostModelMetricsMap.OCP_METRIC_CPU_CORE_USAGE_HOUR
self.ocp_source_type = Provider.PROVIDER_OCP
tiered_rates = [{
"value": round(Decimal(random.random()), 6),
"unit": "USD",
"usage": {
"usage_start": None,
"usage_end": None
},
}]
self.fake_data = {
"name":
"Test Cost Model",
"description":
"Test",
"source_type":
self.ocp_source_type,
"provider_uuids": [self.provider.uuid],
"rates": [{
"metric": {
"name": self.ocp_metric
},
"tiered_rates": tiered_rates
}],
}
with tenant_context(self.tenant):
serializer = CostModelSerializer(data=self.fake_data,
context=request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
def test_user_preference_duplicate(self):
"""Test that we fail to create arbitrary preference if it already exits."""
user = None
serializer = UserSerializer(data=self.user_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
kwargs = {"context": {"user": user}}
test_pref = {"foo": ["a", [1, 2, 3], {"b": "c"}]}
data = {
"name": self.fake.word(),
"description": self.fake.text(),
"preference": test_pref
}
serializer = UserPreferenceSerializer(data=data, **kwargs)
if serializer.is_valid(raise_exception=True):
serializer.save()
with self.assertRaises(IntegrityError):
serializer = UserPreferenceSerializer(data=data, **kwargs)
if serializer.is_valid(raise_exception=True):
serializer.save()
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256, required=True,
allow_null=False, allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
created_timestamp = serializers.DateTimeField(read_only=True)
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
# pylint: disable=too-few-public-methods
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by', 'created_timestamp')
def __init__(self, instance=None, data=empty, **kwargs):
"""Initialize the Provider Serializer.
Here we ensure we use the appropriate serializer to validate the
authentication and billing_source parameters.
"""
super().__init__(instance, data, **kwargs)
provider_type = None
if data and data != empty:
provider_type = data.get('type')
if provider_type and provider_type.lower() not in PROVIDER_CHOICE_LIST:
key = 'type'
message = f'{provider_type} is not a valid source type.'
raise serializers.ValidationError(error_obj(key, message))
if provider_type:
self.fields['authentication'] = AUTHENTICATION_SERIALIZERS.get(provider_type)()
self.fields['billing_source'] = BILLING_SOURCE_SERIALIZERS.get(provider_type)(
default={'bucket': '', 'data_source': {'bucket': ''}}
)
else:
self.fields['authentication'] = ProviderAuthenticationSerializer()
self.fields['billing_source'] = ProviderBillingSourceSerializer()
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = request.user
if user.customer:
customer = user.customer
else:
key = 'customer'
message = 'Customer for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
if 'billing_source' in validated_data:
billing_source = validated_data.pop('billing_source')
data_source = billing_source.get('data_source', {})
bucket = data_source.get('bucket')
else:
# Because of a unique together constraint, this is done
# to allow for this field to be non-required for OCP
# but will still have a blank no-op entry in the DB
billing_source = {'bucket': '', 'data_source': {}}
data_source = None
authentication = validated_data.pop('authentication')
credentials = authentication.get('credentials')
provider_resource_name = credentials.get('provider_resource_name')
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
if credentials and data_source and provider_type not in ['AWS', 'OCP']:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name, bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(**authentication)
# We can re-use a billing source or a auth, but not the same combination.
unique_count = Provider.objects.filter(authentication=auth)\
.filter(billing_source=bill).count()
if unique_count != 0:
existing_provider = Provider.objects.filter(authentication=auth)\
.filter(billing_source=bill).first()
if existing_provider.type in ('AWS', 'OCP'):
sources_auth = {'resource_name': provider_resource_name}
elif existing_provider.type in ('AZURE', ):
sources_auth = {'credentials': auth.credentials}
else:
sources_auth = {}
source_query = Sources.objects.filter(authentication=sources_auth)
if source_query.exists():
source_obj = source_query.first()
source_obj.koku_uuid = existing_provider.uuid
source_obj.save()
return existing_provider
error = {'Error': 'A Provider already exists with that Authentication and Billing Source'}
raise serializers.ValidationError(error)
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider
@transaction.atomic
def update(self, instance, validated_data):
"""Update a Provider instance from validated data."""
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop('authentication')
credentials = authentication.get('credentials')
provider_resource_name = credentials.get('provider_resource_name')
billing_source = validated_data.pop('billing_source')
data_source = billing_source.get('data_source')
bucket = billing_source.get('bucket')
if credentials and data_source and provider_type not in ['AWS', 'OCP']:
interface.cost_usage_source_ready(credentials, data_source)
else:
interface.cost_usage_source_ready(provider_resource_name, bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(**authentication)
for key in validated_data.keys():
setattr(instance, key, validated_data[key])
instance.authentication = auth
instance.billing_source = bill
try:
instance.save()
except IntegrityError:
error = {'Error': 'A Provider already exists with that Authentication and Billing Source'}
raise serializers.ValidationError(error)
return instance
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
self.data_generator = OCPReportDataGenerator(self.tenant)
self.data_generator.add_data_to_tenant()
serializer = UserSerializer(data=self.user_data, context=self.request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
self.report_ocp_cpu = {
'group_by': {
'project': [
'*'
]
},
'filter': {
'resolution': 'monthly',
'time_scope_value': '-1',
'time_scope_units': 'month'
},
'data': [
{
'date': '2018-10',
'projects': [
{
'project': 'default',
'values': [
{
'date': '2018-10',
'project': 'default',
'limit': 'null',
'usage': 0.119385,
'request': 9.506666
}
]
},
{
'project': 'metering',
'values': [
{
'date': '2018-10',
'project': 'metering',
'limit': 'null',
'usage': 4.464511,
'request': 53.985832
}
]
},
{
'project': 'monitoring',
'values': [
{
'date': '2018-10',
'project': 'monitoring',
'limit': 'null',
'usage': 7.861343,
'request': 17.920067
}
]
},
{
'project': 'openshift-web-console',
'values': [
{
'date': '2018-10',
'project': 'openshift-web-console',
'limit': 'null',
'usage': 0.862687,
'request': 4.753333
}
]
}
]
}
],
'total': {
'pod_usage_cpu_core_hours': 13.307928,
'pod_request_cpu_core_hours': 86.165898
}
}
self.report_ocp_mem = {
'group_by': {
'project': [
'*'
]
},
'filter': {
'resolution': 'monthly',
'time_scope_value': '-1',
'time_scope_units': 'month'
},
'data': [
{
'date': '2018-10',
'projects': [
{
'project': 'default',
'values': [
{
'date': '2018-10',
'project': 'default',
'memory_usage_gigabytes': 0.162249,
'memory_requests_gigabytes': 1.063302
}
]
},
{
'project': 'metering',
'values': [
{
'date': '2018-10',
'project': 'metering',
'memory_usage_gigabytes': 5.899788,
'memory_requests_gigabytes': 7.007081
}
]
},
{
'project': 'monitoring',
'values': [
{
'date': '2018-10',
'project': 'monitoring',
'memory_usage_gigabytes': 3.178287,
'memory_requests_gigabytes': 4.153526
}
]
},
{
'project': 'openshift-web-console',
'values': [
{
'date': '2018-10',
'project': 'openshift-web-console',
'memory_usage_gigabytes': 0.068988,
'memory_requests_gigabytes': 0.207677
}
]
}
]
}
],
'total': {
'pod_usage_memory_gigabytes': 9.309312,
'pod_request_memory_gigabytes': 12.431585
}
}
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
serializer = UserSerializer(data=self.user_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
self.report = {
"group_by": {
"subscription_guid": ["*"]
},
"filter": {
"resolution": "monthly",
"time_scope_value": -1,
"time_scope_units": "month",
"resource_scope": [],
},
"data": [{
"date":
"2018-07",
"subscription_guids": [
{
"subscription_guid":
"00000000-0000-0000-0000-000000000000",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"subscription_guid":
"00000000-0000-0000-0000-000000000000",
"total": 1826.74238146924,
}],
},
{
"subscription_guid":
"11111111-1111-1111-1111-111111111111",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"subscription_guid":
"11111111-1111-1111-1111-111111111111",
"total": 1137.74036198065,
}],
},
{
"subscription_guid":
"22222222-2222-2222-2222-222222222222",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"subscription_guid":
"22222222-2222-2222-2222-222222222222",
"total": 1045.80659412797,
}],
},
{
"subscription_guid":
"33333333-3333-3333-3333-333333333333",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"subscription_guid":
"33333333-3333-3333-3333-333333333333",
"total": 807.326470618818,
}],
},
{
"subscription_guid":
"44444444-4444-4444-4444-444444444444",
"values": [{
"date": "2018-07",
"units": "GB-Mo",
"subscription_guid":
"44444444-4444-4444-4444-444444444444",
"total": 658.306642830709,
}],
},
],
}],
"total": {
"value": 5475.922451027388,
"units": "GB-Mo"
},
}
def setUp(self):
"""Set up the customer view tests."""
super().setUp()
serializer = UserSerializer(data=self.user_data, context=self.request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
self.client = APIClient()
self.report = {
"group_by": {"account": ["*"]},
"filter": {
"resolution": "monthly",
"time_scope_value": -1,
"time_scope_units": "month",
"resource_scope": [],
},
"data": [
{
"date": "2018-07",
"accounts": [
{
"account": "4418636104713",
"values": [
{
"date": "2018-07",
"units": "GB-Mo",
"account": "4418636104713",
"total": 1826.74238146924,
}
],
},
{
"account": "8577742690384",
"values": [
{
"date": "2018-07",
"units": "GB-Mo",
"account": "8577742690384",
"total": 1137.74036198065,
}
],
},
{
"account": "3474227945050",
"values": [
{
"date": "2018-07",
"units": "GB-Mo",
"account": "3474227945050",
"total": 1045.80659412797,
}
],
},
{
"account": "7249815104968",
"values": [
{
"date": "2018-07",
"units": "GB-Mo",
"account": "7249815104968",
"total": 807.326470618818,
}
],
},
{
"account": "9420673783214",
"values": [
{
"date": "2018-07",
"units": "GB-Mo",
"account": "9420673783214",
"total": 658.306642830709,
}
],
},
],
}
],
"total": {"value": 5475.922451027388, "units": "GB-Mo"},
}
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(allow_null=True, required=False)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=LCASE_PROVIDER_CHOICE_LIST)
created_timestamp = serializers.DateTimeField(read_only=True)
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
active = serializers.BooleanField(read_only=True)
paused = serializers.BooleanField(required=False)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = (
"uuid",
"name",
"type",
"authentication",
"billing_source",
"customer",
"created_by",
"created_timestamp",
"active",
"paused",
)
def __init__(self, instance=None, data=empty, **kwargs):
"""Initialize the Provider Serializer.
Here we ensure we use the appropriate serializer to validate the
authentication and billing_source parameters.
"""
super().__init__(instance, data, **kwargs)
provider_type = None
if data and data != empty:
provider_type = data.get("type")
if provider_type and provider_type.lower(
) not in LCASE_PROVIDER_CHOICE_LIST:
key = "type"
message = f"{provider_type} is not a valid source type."
raise serializers.ValidationError(error_obj(key, message))
if provider_type:
provider_type = provider_type.lower()
self.fields["authentication"] = AUTHENTICATION_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))()
self.fields["billing_source"] = BILLING_SOURCE_SERIALIZERS.get(
Provider.PROVIDER_CASE_MAPPING.get(provider_type))()
else:
self.fields["authentication"] = ProviderAuthenticationSerializer()
self.fields["billing_source"] = ProviderBillingSourceSerializer()
@property
def demo_credentials(self):
"""Build formatted credentials for our nise-populator demo accounts."""
creds_by_source_type = defaultdict(list)
for account, cred_dict in settings.DEMO_ACCOUNTS.items():
for cred, info in cred_dict.items():
if info.get("source_type") == Provider.PROVIDER_AWS:
creds_by_source_type[Provider.PROVIDER_AWS].append(
{"role_arn": cred})
elif info.get("source_type") == Provider.PROVIDER_AZURE:
creds_by_source_type[Provider.PROVIDER_AZURE].append(
{"client_id": cred})
elif info.get("source_type") == Provider.PROVIDER_GCP:
creds_by_source_type[Provider.PROVIDER_GCP].append(
{"project_id": cred})
return creds_by_source_type
def get_request_info(self):
"""Obtain request information like user and customer context."""
user = self.context.get("user")
customer = self.context.get("customer")
if user and customer:
return user, customer
request = self.context.get("request")
if request and hasattr(request, "user"):
user = request.user
if user.customer:
customer = user.customer
else:
key = "customer"
message = "Customer for requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
else:
key = "created_by"
message = "Requesting user could not be found."
raise serializers.ValidationError(error_obj(key, message))
return user, customer
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
user, customer = self.get_request_info()
provider_type = validated_data["type"].lower()
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source")
if self._is_demo_account(provider_type, credentials):
LOG.info(
"Customer account is a DEMO account. Skipping cost_usage_source_ready check."
)
else:
interface.cost_usage_source_ready(credentials, data_source)
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
# We can re-use a billing source or a auth, but not the same combination.
dup_queryset = (Provider.objects.filter(authentication=auth).filter(
billing_source=bill).filter(customer=customer))
if dup_queryset.count() != 0:
conflict_provider = dup_queryset.first()
message = (
f"Cost management does not allow duplicate accounts. "
f"{conflict_provider.name} already exists. Edit source settings to configure a new source."
)
LOG.warn(message)
raise serializers.ValidationError(
error_obj(ProviderErrors.DUPLICATE_AUTH, message))
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.active = True
provider.save()
customer.date_updated = DateHelper().now_utc
customer.save()
return provider
def update(self, instance, validated_data):
"""Update a Provider instance from validated data."""
_, customer = self.get_request_info()
provider_type = validated_data["type"].lower()
provider_type = Provider.PROVIDER_CASE_MAPPING.get(provider_type)
validated_data["type"] = provider_type
interface = ProviderAccessor(provider_type)
authentication = validated_data.pop("authentication")
credentials = authentication.get("credentials")
billing_source = validated_data.pop("billing_source")
data_source = billing_source.get("data_source")
# updating `paused` must happen regardless of Provider availabilty
instance.paused = validated_data.pop("paused", instance.paused)
try:
if self._is_demo_account(provider_type, credentials):
LOG.info(
"Customer account is a DEMO account. Skipping cost_usage_source_ready check."
)
else:
interface.cost_usage_source_ready(credentials, data_source)
except serializers.ValidationError as validation_error:
instance.active = False
instance.save()
raise validation_error
with transaction.atomic():
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
if instance.billing_source != bill or instance.authentication != auth:
dup_queryset = (Provider.objects.filter(
authentication=auth).filter(billing_source=bill).filter(
customer=customer))
if dup_queryset.count() != 0:
conflict_provder = dup_queryset.first()
message = (
f"Cost management does not allow duplicate accounts. "
f"{conflict_provder.name} already exists. Edit source settings to configure a new source."
)
LOG.warn(message)
raise serializers.ValidationError(
error_obj(ProviderErrors.DUPLICATE_AUTH, message))
for key in validated_data.keys():
setattr(instance, key, validated_data[key])
instance.authentication = auth
instance.billing_source = bill
instance.active = True
instance.save()
customer.date_updated = DateHelper().now_utc
customer.save()
return instance
def _is_demo_account(self, provider_type, credentials):
"""Test whether this source is a demo account."""
key_types = {
Provider.PROVIDER_AWS: "role_arn",
Provider.PROVIDER_AZURE: "client_id",
Provider.PROVIDER_GCP: "project_id",
}
key_to_check = key_types.get(provider_type, "")
creds_to_check = self.demo_credentials.get(provider_type, [])
for cred in creds_to_check:
if credentials.get(key_to_check,
True) == cred.get(key_to_check, False):
return True
return False
def initialize_request(self, context=None):
"""Initialize model data."""
if context:
request_context = context.get('request_context')
request = request_context['request']
serializer = UserSerializer(data=context.get('user_data'),
context=request_context)
else:
request_context = self.request_context
request = request_context['request']
serializer = UserSerializer(data=self.user_data,
context=request_context)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
request.user = user
provider_data = {
'name': 'test_provider',
'type': Provider.PROVIDER_OCP,
'authentication': {
'provider_resource_name': self.fake.word()
}
}
serializer = ProviderSerializer(data=provider_data,
context=request_context)
if serializer.is_valid(raise_exception=True):
self.provider = serializer.save()
self.ocp_metric = CostModelMetricsMap.OCP_METRIC_CPU_CORE_USAGE_HOUR
self.ocp_source_type = 'OCP'
tiered_rates = [{
'value': round(Decimal(random.random()), 6),
'unit': 'USD',
'usage': {
'usage_start': None,
'usage_end': None
}
}]
self.fake_data = {
'name':
'Test Cost Model',
'description':
'Test',
'source_type':
self.ocp_source_type,
'provider_uuids': [self.provider.uuid],
'rates': [{
'metric': {
'name': self.ocp_metric
},
'tiered_rates': tiered_rates
}]
}
with tenant_context(self.tenant):
serializer = CostModelSerializer(data=self.fake_data,
context=request_context)
if serializer.is_valid(raise_exception=True):
serializer.save()
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
authentication = ProviderAuthenticationSerializer()
billing_source = ProviderBillingSourceSerializer(default={'bucket': ''})
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by')
@transaction.atomic
def create(self, validated_data):
"""Create a provider from validated data."""
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = request.user
if user.customer:
customer = user.customer
else:
key = 'customer'
message = 'Customer for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
if 'billing_source' in validated_data:
billing_source = validated_data.pop('billing_source')
bucket = billing_source.get('bucket')
else:
# Because of a unique together constraint, this is done
# to allow for this field to be non-required for OCP
# but will still have a blank no-op entry in the DB
billing_source = {'bucket': ''}
authentication = validated_data.pop('authentication')
provider_resource_name = authentication.get('provider_resource_name')
provider_type = validated_data['type']
interface = ProviderAccessor(provider_type)
interface.cost_usage_source_ready(provider_resource_name, bucket)
bill, __ = ProviderBillingSource.objects.get_or_create(
**billing_source)
auth, __ = ProviderAuthentication.objects.get_or_create(
**authentication)
# We can re-use a billing source or a auth, but not the same combination.
unique_count = Provider.objects.filter(authentication=auth)\
.filter(billing_source=bill).count()
if unique_count != 0:
error = {
'Error':
'A Provider already exists with that Authentication and Billing Source'
}
raise serializers.ValidationError(error)
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider
def test_user_preference_duplicate(self):
"""Test that we fail to create arbitrary preference if it already exits."""
user = None
serializer = UserSerializer(data=self.user_data,
context=self.request_context)
if serializer.is_valid(raise_exception=True):
user = serializer.save()
kwargs = {'context': {'user': user}}
test_pref = {'foo': ['a', [1, 2, 3], {'b': 'c'}]}
data = {
'name': self.fake.word(),
'description': self.fake.text(),
'preference': test_pref
}
serializer = UserPreferenceSerializer(data=data, **kwargs)
if serializer.is_valid(raise_exception=True):
serializer.save()
with self.assertRaises(IntegrityError):
serializer = UserPreferenceSerializer(data=data, **kwargs)
if serializer.is_valid(raise_exception=True):
serializer.save()
class ProviderSerializer(serializers.ModelSerializer):
"""Serializer for the Provider model."""
uuid = serializers.UUIDField(read_only=True)
name = serializers.CharField(max_length=256,
required=True,
allow_null=False,
allow_blank=False)
type = serializers.ChoiceField(choices=Provider.PROVIDER_CHOICES)
authentication = ProviderAuthenticationSerializer()
billing_source = ProviderBillingSourceSerializer()
customer = CustomerSerializer(read_only=True)
created_by = UserSerializer(read_only=True)
class Meta:
"""Metadata for the serializer."""
model = Provider
fields = ('uuid', 'name', 'type', 'authentication', 'billing_source',
'customer', 'created_by')
@transaction.atomic
def create(self, validated_data):
"""Create a user from validated data."""
authentication = validated_data.pop('authentication')
billing_source = validated_data.pop('billing_source')
user = None
customer = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = User.objects.get(pk=request.user.id)
if user.groups.count() == 1:
group = user.groups.first()
customer = Customer.objects.get(pk=group.id)
else:
key = 'customer'
message = 'Group for requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
else:
key = 'created_by'
message = 'Requesting user could not be found.'
raise serializers.ValidationError(error_obj(key, message))
provider_resource_name = authentication.get('provider_resource_name')
bucket = billing_source.get('bucket')
access_key_id, secret_access_key, session_token = _get_sts_access(
provider_resource_name)
if (access_key_id is None or access_key_id is None
or session_token is None):
key = 'provider_resource_name'
message = 'Unable to obtain credentials with using {}.'.format(
provider_resource_name)
raise serializers.ValidationError(error_obj(key, message))
s3_exists = _check_s3_access(access_key_id, secret_access_key,
session_token, bucket)
if not s3_exists:
key = 'bucket'
message = 'Bucket {} could not be found with {}.'.format(
bucket, provider_resource_name)
raise serializers.ValidationError(error_obj(key, message))
org_access = _check_org_access(access_key_id, secret_access_key,
session_token)
if not org_access:
key = 'provider_resource_name'
message = 'Unable to obtain organization data with {}.'.format(
provider_resource_name)
LOG.info(message)
auth = ProviderAuthentication.objects.create(**authentication)
bill = ProviderBillingSource.objects.create(**billing_source)
auth.save()
bill.save()
provider = Provider.objects.create(**validated_data)
provider.customer = customer
provider.created_by = user
provider.authentication = auth
provider.billing_source = bill
provider.save()
return provider