def test_get_source_type_name_non_200(self): """Test to get source type name from type id with bad response.""" source_type_id = 3 mock_source_name = "fakesource" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}", status_code=404, json={"data": [{ "name": mock_source_name }]}, ) with self.assertRaises(SourceNotFoundError): client.get_source_type_name(source_type_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}", status_code=401, json={"data": [{ "name": mock_source_name }]}, ) with self.assertRaises(SourcesHTTPClientError): client.get_source_type_name(source_type_id)
def test_get_aws_credentials_from_app_auth(self): """Test to get AWS Role ARN from authentication service for Application authentication.""" resource_id = 2 authentication_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}", status_code=200, json={"data": [{ "id": resource_id }]}, ) m.get( (f"http://www.sources.com/api/v1.0/authentications?" f"[authtype]=arn&[resource_id]={resource_id}"), status_code=200, json={"data": [{ "id": authentication_id }]}, ) m.get( (f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}" f"?expose_encrypted_attribute[]=password"), status_code=200, json={"password": self.authentication}, ) response = client.get_aws_credentials() self.assertEqual(response, {"role_arn": self.authentication})
def test_get_aws_role_arn(self): """Test to get AWS Role ARN from authentication service.""" resource_id = 2 authentication_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}', status_code=200, json={'data': [{ 'id': resource_id }]}) m.get(( f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint' f'&[authtype]=arn&[resource_id]={resource_id}'), status_code=200, json={'data': [{ 'id': authentication_id }]}) m.get(( f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}' f'?expose_encrypted_attribute[]=password'), status_code=200, json={'password': self.authentication}) response = client.get_aws_role_arn() self.assertEqual(response, self.authentication)
def test_get_azure_credentials_no_auth(self): """Test to get Azure credentials from authentication service with auth not ready.""" resource_id = 2 authentication_id = 3 authentication = 'testclientcreds' client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}', status_code=200, json={'data': [{ 'id': resource_id }]}) m.get(( f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint' f'&[authtype]=tenant_id_client_id_client_secret&[resource_id]={resource_id}' ), status_code=200, json={'data': []}) m.get(( f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}' f'?expose_encrypted_attribute[]=password'), status_code=200, json={'password': authentication}) with self.assertRaises(SourcesHTTPClientError): client.get_azure_credentials()
def __init__(self, source_id): """Initialize source id.""" self.source_id = source_id self.source = Sources.objects.get(source_id=source_id) if not source_settings_complete(self.source) or self.source.pending_delete: raise ObjectDoesNotExist(f"Source ID: {self.source_id} not ready for status") self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_application_type_is_cost_management_misconfigured(self): """Test to get application_type_id from source_id with route not found.""" application_type_id = 2 source_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=source_id) responses.add( responses.GET, f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources", json={"data": [{ "name": "test-source" }]}, status=404, ) responses.add( responses.GET, f"http://www.sources.com/api/v1.0/application_types", json={"data": [{ "id": self.application_type }]}, status=200, ) with self.assertRaises(SourceNotFoundError): client.get_application_type_is_cost_management(source_id)
def __init__(self, request, source_id): """Initialize source id.""" self.request = request self.user = request.user self.source_id = source_id self.source = Sources.objects.get(source_id=source_id) self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_application_type_is_cost_management(self): """Test to get application_type_id from source_id.""" application_type_id = 2 source_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=source_id) responses.add( responses.GET, f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources", json={"data": [{ "name": "test-source" }]}, status=200, ) responses.add( responses.GET, f"http://www.sources.com/api/v1.0/application_types", json={"data": [{ "id": self.application_type }]}, status=200, ) response = client.get_application_type_is_cost_management(source_id) self.assertTrue(response)
def test_set_source_status_patch_fail(self): """Test to set source status where the patch fails.""" test_source_id = 1 application_type_id = 2 application_id = 3 status = "unavailable" error_msg = "my error" source = Sources.objects.create(source_id=test_source_id, offset=42, source_type="AWS") source.save() client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=test_source_id) with requests_mock.mock() as m: m.get( (f"http://www.sources.com/api/v1.0/applications?" f"filter[application_type_id]={application_type_id}&filter[source_id]={test_source_id}" ), status_code=200, json={"data": [{ "id": application_id }]}, ) m.patch( f"http://www.sources.com/api/v1.0/applications/{application_id}", status_code=400, json={ "availability_status": status, "availability_status_error": str(error_msg) }, ) with self.assertRaises(SourcesHTTPClientError): client.set_source_status(error_msg, application_type_id)
def test_get_aws_role_arn_no_auth(self): """Test to get AWS Role ARN from authentication service with auth not ready.""" resource_id = 2 authentication_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}", status_code=200, json={"data": [{ "id": resource_id }]}, ) m.get( (f"http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint" f"&[authtype]=arn&[resource_id]={resource_id}"), status_code=200, json={"data": []}, ) m.get( (f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}" f"?expose_encrypted_attribute[]=password"), status_code=200, json={"password": self.authentication}, ) with self.assertRaises(SourcesHTTPClientError): client.get_aws_role_arn()
def test_get_source_details_unsuccessful(self): """Test to get source details unsuccessfully.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=404) with self.assertRaises(SourceNotFoundError): client.get_source_details()
def test_set_source_status_unexpected_header(self): """Test to set source status with missing account in header.""" test_source_id = 1 application_type_id = 2 error_msg = "my error" malformed_identity_header = { "not_identity": { "type": "User", "user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True}, } } json_malformed_identity = json_dumps(malformed_identity_header) malformed_internal_header = b64encode(json_malformed_identity.encode("utf-8")) malformed_auth_header = malformed_internal_header.decode("utf-8") missing_account_header = { "identity": { "type": "User", "user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True}, } } missing_account_identity = json_dumps(missing_account_header) missing_account_internal_header = b64encode(missing_account_identity.encode("utf-8")) missing_account_auth_header = missing_account_internal_header.decode("utf-8") test_headers = [malformed_auth_header, missing_account_auth_header] source = Sources.objects.create(source_id=test_source_id, offset=42, source_type="AWS") source.save() for header in test_headers: client = SourcesHTTPClient(auth_header=header, source_id=test_source_id) response = client.set_source_status(error_msg, application_type_id) self.assertFalse(response)
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f'Source ID not found for ID: {source_id}') return try: if source_type == 'OCP': source_details = sources_network.get_source_details() # Check for imported to maintain temporary backwards compatibility # until the Sources Front End creates 'imported' entry with OCP Cluster ID. if source_details.get('source_ref'): authentication = { 'resource_name': source_details.get('source_ref') } else: uid = source_details.get('uid') LOG.info( f'OCP is using fallback Source UID ({str(uid)} for authentication.' ' Update frontend to add Cluster ID to the source_ref field on the Source.' ) authentication = {'resource_name': uid} elif source_type == 'AWS': authentication = { 'resource_name': sources_network.get_aws_role_arn() } elif source_type == 'AZURE': authentication = { 'credentials': sources_network.get_azure_credentials() } else: LOG.error(f'Unexpected source type: {source_type}') return storage.add_provider_sources_auth_info(source_id, authentication) except SourcesHTTPClientError: LOG.info( f'Authentication info not available for Source ID: {source_id}')
def __init__(self, auth_header, source_id): sources_network = SourcesHTTPClient(auth_header, source_id) details = sources_network.get_source_details() self.name = details.get("name") self.source_type_id = int(details.get("source_type_id")) self.source_uuid = details.get("uid") self.source_type_name = sources_network.get_source_type_name(self.source_type_id) self.source_type = SOURCE_PROVIDER_MAP.get(self.source_type_name)
def test_get_network_response_exception(self): """Test get network response with request exception.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(url=MOCK_URL, exc=RequestException) with self.assertRaises(SourcesHTTPClientError): client._get_network_response(MOCK_URL, "test error")
def test_get_network_response_success(self): """Test get network response succeeds.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(url=MOCK_URL, json={"data": "valid json"}) resp = client._get_network_response(MOCK_URL, "test error") self.assertEqual(resp.get("data"), "valid json")
def execute_koku_provider_op(msg, cost_management_type_id): """ Execute the 'create' or 'destroy Koku-Provider operations. 'create' operations: Koku POST /providers is executed along with updating the Sources database table with the Koku Provider uuid. 'destroy' operations: Koku DELETE /providers is executed along with removing the Sources database entry. Two types of exceptions are handled for Koku HTTP operations. Recoverable client and Non-Recoverable client errors. If the error is recoverable the calling function (synchronize_sources) will re-queue the operation. Args: msg (Asyncio msg): Dictionary messages containing operation, provider and offset. example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3} cost_management_type_id (Integer): Cost Management Type Identifier Returns: None """ provider = msg.get("provider") operation = msg.get("operation") try: if operation == "create": task = create_or_update_provider.delay(provider.source_id) LOG.info( f"Creating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}" ) elif operation == "update": task = create_or_update_provider.delay(provider.source_id) LOG.info( f"Updating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}" ) elif operation == "destroy": task = delete_source_and_provider.delay(provider.source_id, provider.source_uuid, provider.auth_header) LOG.info( f"Deleting Koku Provider/Source for Source ID: {str(provider.source_id)} in task: {task.id}" ) else: LOG.error(f"unknown operation: {operation}") except RabbitOperationalError: err_msg = f"RabbitmQ unavailable. Unable to {operation} Source ID: {provider.source_id}" LOG.error(err_msg) sources_network = SourcesHTTPClient(provider.auth_header, provider.source_id) sources_network.set_source_status( err_msg, cost_management_type_id=cost_management_type_id) # Re-raise exception so it can be re-queued in synchronize_sources raise RabbitOperationalError(err_msg)
def check_sources_connection(): """Check sources-backend connection.""" try: cost_management_type_id = SourcesHTTPClient( SourcesConfig.SOURCES_FAKE_HEADER ).get_cost_management_application_type_id() return cost_management_type_id except (SourcesHTTPClientError, SourceNotFoundError): return
def test_get_cost_management_application_type_id_error(self): """Test to get application type id with error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER) with requests_mock.mock() as m: m.get( f'http://www.sources.com/api/v1.0/application_types?filter[name]=/insights/platform/cost-management', exc=requests.exceptions.RequestException) with self.assertRaises(SourcesHTTPClientError): client.get_cost_management_application_type_id()
def test_get_source_details_connection_error(self): """Test to get source details with connection error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}", exc=RequestException) with self.assertRaises(SourcesHTTPClientError): client.get_source_details()
def execute_koku_provider_op(msg, cost_management_type_id): """ Execute the 'create' or 'destroy Koku-Provider operations. 'create' operations: Koku POST /providers is executed along with updating the Sources database table with the Koku Provider uuid. 'destroy' operations: Koku DELETE /providers is executed along with removing the Sources database entry. Two types of exceptions are handled for Koku HTTP operations. Recoverable client and Non-Recoverable client errors. If the error is recoverable the calling function (synchronize_sources) will re-queue the operation. Args: msg (Asyncio msg): Dictionary messages containing operation, provider and offset. example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3} cost_management_type_id (Integer): Cost Management Type Identifier Returns: None """ provider = msg.get('provider') operation = msg.get('operation') koku_client = KokuHTTPClient(provider.auth_header) sources_client = SourcesHTTPClient(provider.auth_header, provider.source_id) try: if operation == 'create': LOG.info(f'Creating Koku Provider for Source ID: {str(provider.source_id)}') koku_details = koku_client.create_provider(provider.name, provider.source_type, provider.authentication, provider.billing_source, provider.source_uuid) LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} assigned to Source ID {str(provider.source_id)}.') storage.add_provider_koku_uuid(provider.source_id, koku_details.get('uuid')) elif operation == 'destroy': if provider.koku_uuid: try: response = koku_client.destroy_provider(provider.koku_uuid) LOG.info( f'Koku Provider UUID ({provider.koku_uuid}) Removal Status Code: {str(response.status_code)}') except KokuHTTPClientNonRecoverableError: LOG.info(f'Koku Provider already removed. Remove Source ID: {str(provider.source_id)}.') storage.destroy_provider_event(provider.source_id) elif operation == 'update': koku_details = koku_client.update_provider(provider.koku_uuid, provider.name, provider.source_type, provider.authentication, provider.billing_source) storage.clear_update_flag(provider.source_id) LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} with Source ID {str(provider.source_id)} updated.') sources_client.set_source_status(None, cost_management_type_id) except KokuHTTPClientError as koku_error: raise SourcesIntegrationError('Koku provider error: ', str(koku_error)) except KokuHTTPClientNonRecoverableError as koku_error: err_msg = f'Unable to {operation} provider for Source ID: {str(provider.source_id)}. Reason: {str(koku_error)}' LOG.error(err_msg) sources_client.set_source_status(str(koku_error), cost_management_type_id)
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f"Source ID not found for ID: {source_id}") return try: if source_type == Provider.PROVIDER_OCP: source_details = sources_network.get_source_details() if source_details.get("source_ref"): authentication = { "resource_name": source_details.get("source_ref") } else: raise SourcesHTTPClientError("Unable to find Cluster ID") elif source_type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL): authentication = { "resource_name": sources_network.get_aws_role_arn() } elif source_type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL): authentication = { "credentials": sources_network.get_azure_credentials() } else: LOG.error(f"Unexpected source type: {source_type}") return storage.add_provider_sources_auth_info(source_id, authentication) storage.clear_update_flag(source_id) LOG.info(f"Authentication attached to Source ID: {source_id}") except SourcesHTTPClientError as error: LOG.info( f"Authentication info not available for Source ID: {source_id}") sources_network.set_source_status(str(error))
def test_get_source_details(self): """Test to get source details.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=200, json={"name": self.name} ) response = client.get_source_details() self.assertEqual(response.get("name"), self.name)
def test_get_source_id_from_applications_id_server_error(self): """Test to get source ID from application resource_id with server error.""" resource_id = 2 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(f"http://www.sources.com/api/v1.0/applications?filter[id]={resource_id}", status_code=400) with self.assertRaises(SourcesHTTPClientError): client.get_source_id_from_applications_id(resource_id)
def test__get_ocp_credentials_missing_cluster_id(self): """Test to get ocp cluster-id with missing cluster-id raises exception.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": None} ) with self.assertRaises(SourcesHTTPClientError): client._get_ocp_credentials()
def sources_backend(self): """Return Sources Backend connection status.""" try: cost_management_type_id = SourcesHTTPClient(SourcesConfig.SOURCES_FAKE_HEADER).\ get_cost_management_application_type_id() return f'Cost Management Application ID: {cost_management_type_id}' except SourcesHTTPClientError: return 'Not connected'
def test_get_endpoint_ids_connection_error(self): """Test to get endpoint id with connection error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}", exc=RequestException) with self.assertRaises(SourcesHTTPClientError): client.get_endpoint_id()
def test_get_azure_credentials_connection_error(self): """Test to get Azure credentials from authentication service with connection error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}", exc=RequestException, ) with self.assertRaises(SourcesHTTPClientError): client.get_azure_credentials()
def test_get_network_response_status_exception(self): """Test get network response with invalid status responses.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) table = [{"status": 404, "expected": SourceNotFoundError}, {"status": 403, "expected": SourcesHTTPClientError}] for test in table: with self.subTest(test=test): with requests_mock.mock() as m: m.get(url=MOCK_URL, status_code=test.get("status"), exc=test.get("exc")) with self.assertRaises(test.get("expected")): client._get_network_response(MOCK_URL, "test error")
def test__get_ocp_credentials(self): """Test to get ocp cluster-id.""" uuid = str(uuid4()) client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": uuid} ) creds = client._get_ocp_credentials() self.assertEqual(creds.get("cluster_id"), uuid)