def test_create_gcp_provider_validate_no_data_source_bucket(self): """Test the data_source.bucket validation for GCP provider.""" provider = { "name": "test_provider_val_data_source", "type": Provider.PROVIDER_GCP.lower(), "authentication": {"credentials": {"project_id": "gcp_project"}}, "billing_source": {"data_source": {"potato": ""}}, } with self.assertRaises(ValidationError) as e: serializer = ProviderSerializer(data=provider, context=self.request_context) serializer.is_valid(raise_exception=True) self.assertEqual(e.exception.status_code, 400) self.assertEqual(str(e.exception.detail["billing_source"]["data_source.bucket"][0]), "This field is required.")
def test_create_provider_with_bucket_and_data_source(self): """Test creating a provider with data_source and bucket fields should fail.""" bucket_name = "my_s3_bucket" provider = { "name": "test_provider", "type": Provider.PROVIDER_AWS.lower(), "authentication": {"credentials": {"one": "two", "three": "four"}}, "billing_source": {"data_source": {"foo": "bar"}, "bucket": bucket_name}, } request = self.request_context["request"] request.user.customer = None serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def test_create_provider_fails_user(self): """Test creating a provider fails with no user.""" provider = { "name": "test_provider", "type": Provider.PROVIDER_AWS.lower(), "authentication": { "provider_resource_name": "arn:aws:s3:::my_s3_bucket" }, "billing_source": { "bucket": "my_s3_bucket" }, } serializer = ProviderSerializer(data=provider) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def test_create_provider_invalid_type(self): """Test that an invalid provider type is not validated.""" iam_arn = "arn:aws:s3:::my_s3_bucket" bucket_name = "my_s3_bucket" provider = { "name": "test_provider", "type": "Bad", "authentication": {"credentials": {"role_arn": iam_arn}}, "billing_source": {"data_source": {"bucket": bucket_name}}, } with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): with self.assertRaises(ValidationError): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): serializer.save()
def test_create_provider_fails_user(self): """Test creating a provider fails with no user.""" provider = { 'name': 'test_provider', 'type': Provider.PROVIDER_AWS, 'authentication': { 'provider_resource_name': 'arn:aws:s3:::my_s3_bucket' }, 'billing_source': { 'bucket': 'my_s3_bucket' } } serializer = ProviderSerializer(data=provider) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def test_create_provider_fails_customer(self): # pylint: disable=C0103 """Test creating a provider where customer is not found for user.""" provider = {'name': 'test_provider', 'type': Provider.PROVIDER_AWS, 'authentication': { 'provider_resource_name': 'arn:aws:s3:::my_s3_bucket' }, 'billing_source': { 'bucket': 'my_s3_bucket' }} request = self.request_context['request'] request.user.customer = None serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def setUp(self): """Set up the tests.""" super().setUp() request = self.request_context['request'] serializer = UserSerializer(data=self.user_data, context=self.request_context) if serializer.is_valid(raise_exception=True): user = serializer.save() request.user = user provider_data = {'name': 'test_provider', 'type': Provider.PROVIDER_OCP, 'authentication': { 'provider_resource_name': self.fake.word() }} serializer = ProviderSerializer(data=provider_data, context=self.request_context) if serializer.is_valid(raise_exception=True): self.provider = serializer.save() ocp_metric = CostModelMetricsMap.OCP_METRIC_CPU_CORE_USAGE_HOUR ocp_source_type = 'OCP' tiered_rates = [{'unit': 'USD', 'value': 0.22}] self.ocp_data = { 'name': 'Test Cost Model', 'description': 'Test', 'source_type': ocp_source_type, 'provider_uuids': [self.provider.uuid], 'rates': [ { 'metric': {'name': ocp_metric}, 'tiered_rates': tiered_rates } ] }
def test_create_provider(self): """Test creating a provider.""" iam_arn = 'arn:aws:s3:::my_s3_bucket' bucket_name = 'my_s3_bucket' provider = {'name': 'test_provider', 'type': Provider.PROVIDER_AWS, 'authentication': { 'provider_resource_name': iam_arn }, 'billing_source': { 'bucket': bucket_name }} new_cust = None serializer = CustomerSerializer(data=self.customer_data[0]) if serializer.is_valid(raise_exception=True): new_cust = serializer.save() request = Mock() request.user = new_cust.owner context = {'request': request} instance = None with patch.object(ProviderAccessor, 'cost_usage_source_ready', returns=True): serializer = ProviderSerializer(data=provider, context=context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data['customer'].get('schema_name') self.assertIsInstance(instance.uuid, uuid.UUID) self.assertIsNone(schema_name) self.assertFalse('schema_name' in serializer.data['customer'])
def test_create_provider_with_exception(self): """Test creating a provider with a provider exception.""" iam_arn = 'arn:aws:s3:::my_s3_bucket' bucket_name = 'my_s3_bucket' provider = { 'name': 'test_provider', 'type': Provider.PROVIDER_AWS, 'authentication': { 'provider_resource_name': iam_arn }, 'billing_source': { 'bucket': bucket_name } } new_cust = None serializer = CustomerSerializer(data=self.customer_data[0]) if serializer.is_valid(raise_exception=True): new_cust = serializer.save() request = Mock() request.user = new_cust.owner context = {'request': request} with patch.object(ProviderAccessor, 'cost_usage_source_ready', side_effect=serializers.ValidationError): ProviderSerializer(data=provider, context=context)
def test_provider_org_fail(self, check_org_access): """Test creating a provider with AWS org access failure.""" check_org_access.return_value = False iam_arn = 'arn:aws:s3:::my_s3_bucket' bucket_name = 'my_s3_bucket' access_key_id, secret_access_key, session_token = _get_sts_access( iam_arn) s3_resource = boto3.resource( 's3', aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, aws_session_token=session_token, ) s3_resource.create_bucket(Bucket=bucket_name) provider = {'name': 'test_provider', 'type': Provider.PROVIDER_AWS, 'authentication': { 'provider_resource_name': iam_arn }, 'billing_source': { 'bucket': bucket_name }} new_cust = None serializer = CustomerSerializer(data=self.customer_data[0]) if serializer.is_valid(raise_exception=True): new_cust = serializer.save() request = Mock() request.user = new_cust.owner context = {'request': request} serializer = ProviderSerializer(data=provider, context=context) if serializer.is_valid(raise_exception=True): serializer.save() check_org_access.assert_called_once()
def test_provider_s3_fail(self, check_s3_access): """Test creating a provider with AWS s3 bucket doesn't exist.""" check_s3_access.return_value = False iam_arn = 'arn:aws:s3:::my_s3_bucket' bucket_name = 'my_s3_bucket' provider = {'name': 'test_provider', 'type': Provider.PROVIDER_AWS, 'authentication': { 'provider_resource_name': iam_arn }, 'billing_source': { 'bucket': bucket_name }} new_cust = None serializer = CustomerSerializer(data=self.customer_data[0]) if serializer.is_valid(raise_exception=True): new_cust = serializer.save() request = Mock() request.user = new_cust.owner context = {'request': request} serializer = ProviderSerializer(data=provider, context=context) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save() check_s3_access.assert_called_once(iam_arn)
def initialize_request(self, context=None): """Initialize model data.""" if context: request_context = context.get('request_context') request = request_context['request'] serializer = UserSerializer(data=context.get('user_data'), context=request_context) else: request_context = self.request_context request = request_context['request'] serializer = UserSerializer(data=self.user_data, context=request_context) if serializer.is_valid(raise_exception=True): user = serializer.save() request.user = user provider_data = {'name': 'test_provider', 'type': Provider.PROVIDER_OCP, 'authentication': { 'provider_resource_name': self.fake.word() }} serializer = ProviderSerializer(data=provider_data, context=request_context) if serializer.is_valid(raise_exception=True): self.provider = serializer.save() self.fake_data = {'provider_uuids': [self.provider.uuid], 'metric': {'name': CostModelMetricsMap.OCP_METRIC_MEM_GB_USAGE_HOUR}, 'tiered_rate': [{ 'value': round(Decimal(random.random()), 6), 'unit': 'USD', 'usage': {'usage_start': None, 'usage_end': None} }] } with tenant_context(self.tenant): serializer = RateSerializer(data=self.fake_data, context=request_context) if serializer.is_valid(raise_exception=True): serializer.save()
def setUp(self): """Set up the tests.""" super().setUp() request = self.request_context["request"] serializer = UserSerializer(data=self.user_data, context=self.request_context) if serializer.is_valid(raise_exception=True): user = serializer.save() request.user = user provider_data = { "name": "test_provider", "type": Provider.PROVIDER_OCP.lower(), "authentication": {"provider_resource_name": self.fake.word()}, } serializer = ProviderSerializer(data=provider_data, context=self.request_context) if serializer.is_valid(raise_exception=True): self.provider = serializer.save() ocp_metric = CostModelMetricsMap.OCP_METRIC_CPU_CORE_USAGE_HOUR ocp_source_type = Provider.PROVIDER_OCP tiered_rates = [{"unit": "USD", "value": 0.22}] self.ocp_data = { "name": "Test Cost Model", "description": "Test", "source_type": ocp_source_type, "providers": [{"uuid": self.provider.uuid, "name": self.provider.name}], "markup": {"value": 10, "unit": "percent"}, "rates": [{"metric": {"name": ocp_metric}, "tiered_rates": tiered_rates}], }
def initialize_request(self, context=None): """Initialize model data.""" if context: request_context = context.get("request_context") request = request_context["request"] serializer = UserSerializer(data=context.get("user_data"), context=request_context) else: request_context = self.request_context request = request_context["request"] serializer = UserSerializer(data=self.user_data, context=request_context) if serializer.is_valid(raise_exception=True): user = serializer.save() request.user = user provider_data = { "name": "test_provider", "type": Provider.PROVIDER_OCP.lower(), "authentication": { "provider_resource_name": self.fake.word() }, } serializer = ProviderSerializer(data=provider_data, context=request_context) if serializer.is_valid(raise_exception=True): self.provider = serializer.save() self.ocp_metric = CostModelMetricsMap.OCP_METRIC_CPU_CORE_USAGE_HOUR self.ocp_source_type = Provider.PROVIDER_OCP tiered_rates = [{ "value": round(Decimal(random.random()), 6), "unit": "USD", "usage": { "usage_start": None, "usage_end": None }, }] self.fake_data = { "name": "Test Cost Model", "description": "Test", "source_type": self.ocp_source_type, "provider_uuids": [self.provider.uuid], "rates": [{ "metric": { "name": self.ocp_metric }, "tiered_rates": tiered_rates }], } with tenant_context(self.tenant): serializer = CostModelSerializer(data=self.fake_data, context=request_context) if serializer.is_valid(raise_exception=True): serializer.save()
def test_create_gcp_provider_duplicate_bucket(self): """Test that the same blank billing entry is used for all OCP providers.""" provider = { "name": "test_provider_one", "type": Provider.PROVIDER_GCP.lower(), "authentication": {"credentials": {"project_id": "gcp_project"}}, "billing_source": {"data_source": {"bucket": "test_bucket"}}, } with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): serializer.save() with self.assertRaises(ValidationError): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): serializer.save()
def test_create_gcp_provider(self): """Test that the same blank billing entry is used for all OCP providers.""" provider = { "name": "test_provider_one", "type": Provider.PROVIDER_GCP.lower(), "authentication": {"credentials": {"project_id": "gcp_project"}}, "billing_source": {"data_source": {"bucket": "test_bucket", "report_prefix": "precious"}}, } with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data["customer"].get("schema_name") self.assertIsInstance(instance.uuid, uuid.UUID) self.assertTrue(instance.active) self.assertIsNone(schema_name) self.assertFalse("schema_name" in serializer.data["customer"])
def test_create_provider_fails_customer(self): """Test creating a provider where customer is not found for user.""" provider = { "name": "test_provider", "type": Provider.PROVIDER_AWS.lower(), "authentication": {"credentials": {"role_arn": "arn:aws:s3:::my_s3_bucket"}}, "billing_source": {"data_source": {"bucket": "my_s3_bucket"}}, } user_data = self._create_user_data() alt_request_context = self._create_request_context( self.create_mock_customer_data(), user_data, create_tenant=True ) request = alt_request_context["request"] request.user.customer = None serializer = ProviderSerializer(data=provider, context=alt_request_context) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def test_create_provider_with_credentials_and_provider_resource_name(self): """Test creating a provider with credentials and provider_resource_name fields should fail.""" iam_arn = "arn:aws:s3:::my_s3_bucket" provider = { "name": "test_provider", "type": Provider.PROVIDER_AWS.lower(), "authentication": {"credentials": {"one": "two", "three": "four"}, "provider_resource_name": iam_arn}, "billing_source": {"data_source": {"foo": "bar"}}, } user_data = self._create_user_data() alt_request_context = self._create_request_context( self.create_mock_customer_data(), user_data, create_tenant=True ) request = alt_request_context["request"] request.user.customer = None serializer = ProviderSerializer(data=provider, context=alt_request_context) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def test_create_ocp_provider(self): """Test creating a provider.""" cluster_id = 'my-ocp-cluster-1' provider = {'name': 'test_provider', 'type': Provider.PROVIDER_OCP, 'authentication': { 'provider_resource_name': cluster_id }} instance = None with patch.object(ProviderAccessor, 'cost_usage_source_ready', returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data['customer'].get('schema_name') self.assertIsInstance(instance.uuid, uuid.UUID) self.assertIsNone(schema_name) self.assertFalse('schema_name' in serializer.data['customer'])
def test_error_providers_with_same_auth_or_billing_source(self): """Test that the errors are wrapped correctly.""" provider = self.generic_providers[Provider.PROVIDER_OCP] with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): serializer.save() with self.assertRaises(ValidationError) as excCtx: serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): serializer.save() validationErr = excCtx.exception.detail[ ProviderErrors.DUPLICATE_AUTH][0] self.assertTrue("Cost management does not allow duplicate accounts" in str(validationErr))
def test_create_provider_with_credentials_and_data_source(self): """Test creating a provider with data_source field instead of bucket.""" provider = { "name": "test_provider", "type": Provider.PROVIDER_AWS.lower(), "authentication": {"credentials": {"role_arn": "four"}}, "billing_source": {"data_source": {"bucket": "bar"}}, } instance = None with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data["customer"].get("schema_name") self.assertIsInstance(instance.uuid, uuid.UUID) self.assertTrue(instance.active) self.assertIsNone(schema_name) self.assertFalse("schema_name" in serializer.data["customer"])
def test_create_ocp_provider(self): """Test creating an OCP provider.""" cluster_id = "my-ocp-cluster-1" provider = { "name": "test_provider", "type": Provider.PROVIDER_OCP.lower(), "authentication": {"provider_resource_name": cluster_id}, } instance = None with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data["customer"].get("schema_name") self.assertIsInstance(instance.uuid, uuid.UUID) self.assertTrue(instance.active) self.assertIsNone(schema_name) self.assertFalse("schema_name" in serializer.data["customer"])
def test_create_ocp_source_with_existing_provider(self): """Test creating an OCP Source when the provider already exists.""" cluster_id = "my-ocp-cluster-1" provider = { "name": "test_provider", "type": Provider.PROVIDER_OCP.lower(), "authentication": { "credentials": { "cluster_id": cluster_id } }, "billing_source": {}, } instance = None with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data["customer"].get("schema_name") self.assertIsInstance(instance.uuid, uuid.UUID) self.assertTrue(instance.active) self.assertIsNone(schema_name) self.assertFalse("schema_name" in serializer.data["customer"]) # Add Source without provider uuid sources = Sources.objects.create( source_id=1, auth_header="testheader", offset=1, authentication={"cluster_id": cluster_id}) sources.save() # Verify ValidationError is raised when another source is added with an existing # provider. with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def initialize_request(self, context=None): """Initialize model data.""" if context: request_context = context.get("request_context") else: request_context = self.request_context provider_data = { "name": "test_provider", "type": Provider.PROVIDER_OCP.lower(), "authentication": { "credentials": { "cluster_id": self.fake.word() } }, "billing_source": {}, } serializer = ProviderSerializer(data=provider_data, context=request_context) if serializer.is_valid(raise_exception=True): self.provider = serializer.save() self.ocp_metric = metric_constants.OCP_METRIC_CPU_CORE_USAGE_HOUR self.ocp_source_type = Provider.PROVIDER_OCP tiered_rates = [{ "value": round(Decimal(random.random()), 6), "unit": "USD", "usage": { "usage_start": None, "usage_end": None }, }] self.fake_data = { "name": "Test Cost Model", "description": "Test", "source_type": self.ocp_source_type, "source_uuids": [self.provider.uuid], "rates": [{ "metric": { "name": self.ocp_metric }, "cost_type": "Infrastructure", "tiered_rates": tiered_rates }], } with tenant_context(self.tenant): serializer = CostModelSerializer(data=self.fake_data, context=request_context) if serializer.is_valid(raise_exception=True): with patch("cost_models.cost_model_manager.chain"): instance = serializer.save() self.fake_data_cost_model_uuid = instance.uuid
def test_create_provider_with_exception(self): """Test creating a provider with a provider exception.""" iam_arn = "arn:aws:s3:::my_s3_bucket" bucket_name = "my_s3_bucket" provider = { "name": "test_provider", "type": Provider.PROVIDER_AWS, "authentication": {"provider_resource_name": iam_arn}, "billing_source": {"bucket": bucket_name}, } with patch.object(ProviderAccessor, "cost_usage_source_ready", side_effect=serializers.ValidationError): ProviderSerializer(data=provider, context=self.request_context)
def update_provider_from_source(self, source): """Call to update provider.""" connection.set_schema_to_public() context, customer, _ = self._create_context() tenant = Tenant.objects.get(schema_name=customer.schema_name) provider_type = source.source_type json_data = { "name": source.name, "type": provider_type.lower(), "authentication": self._build_credentials_auth(source.authentication), "billing_source": self.get_billing_source_for_provider(provider_type, source.billing_source), } connection.set_tenant(tenant) instance = Provider.objects.get(uuid=source.koku_uuid) serializer = ProviderSerializer(instance=instance, data=json_data, partial=False, context=context) serializer.is_valid(raise_exception=True) serializer.save() connection.set_schema_to_public() return instance
def test_create_aws_provider(self): """Test creating a provider.""" iam_arn = "arn:aws:s3:::my_s3_bucket" bucket_name = "my_s3_bucket" provider = { "name": "test_provider", "type": Provider.PROVIDER_AWS.lower(), "authentication": {"provider_resource_name": iam_arn}, "billing_source": {"bucket": bucket_name}, } instance = None with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data["customer"].get("schema_name") self.assertIsInstance(instance.uuid, uuid.UUID) self.assertTrue(instance.active) self.assertIsNone(schema_name) self.assertFalse("schema_name" in serializer.data["customer"])
def test_missing_source_parameters_exception(self): """Test that ValidationError is raised when there are missing parameters.""" fields = ["resource_group", "storage_account"] credentials = { "subscription_id": FAKE.uuid4(), "tenant_id": FAKE.uuid4(), "client_id": FAKE.uuid4(), "client_secret": FAKE.word(), } source_name = {"resource_group": FAKE.word(), "storage_account": FAKE.word()} del source_name[random.choice(fields)] provider = { "name": FAKE.word(), "type": Provider.PROVIDER_AZURE.lower(), "authentication": credentials, "billing_source": {"data_source": source_name}, } with self.assertRaises(ValidationError): serializer = ProviderSerializer(data=provider, context=self.request_context) serializer.is_valid(raise_exception=True)
def test_create_ocp_source_with_existing_provider(self): """Test creating an OCP Source when the provider already exists.""" cluster_id = 'my-ocp-cluster-1' provider = { 'name': 'test_provider', 'type': Provider.PROVIDER_OCP, 'authentication': { 'provider_resource_name': cluster_id } } instance = None with patch.object(ProviderAccessor, 'cost_usage_source_ready', returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): instance = serializer.save() schema_name = serializer.data['customer'].get('schema_name') self.assertIsInstance(instance.uuid, uuid.UUID) self.assertTrue(instance.active) self.assertIsNone(schema_name) self.assertFalse('schema_name' in serializer.data['customer']) # Add Source without provider uuid sources = Sources.objects.create( source_id=1, auth_header='testheader', offset=1, authentication={'resource_name': cluster_id}) sources.save() # Verify ValidationError is raised when another source is added with an existing # provider. with patch.object(ProviderAccessor, 'cost_usage_source_ready', returns=True): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): with self.assertRaises(serializers.ValidationError): serializer.save()
def test_create_provider_invalid_type(self): """Test that an invalid provider type is not validated.""" iam_arn = 'arn:aws:s3:::my_s3_bucket' bucket_name = 'my_s3_bucket' provider = { 'name': 'test_provider', 'type': 'Bad', 'authentication': { 'provider_resource_name': iam_arn }, 'billing_source': { 'bucket': bucket_name } } with patch.object(ProviderAccessor, 'cost_usage_source_ready', returns=True): with self.assertRaises(ValidationError): serializer = ProviderSerializer(data=provider, context=self.request_context) if serializer.is_valid(raise_exception=True): serializer.save()