class SourceStatus: """Source Status.""" def __init__(self, request, source_id): """Initialize source id.""" self.request = request self.user = request.user self.source_id = source_id self.source = Sources.objects.get(source_id=source_id) self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id) @property def sources_response(self): return self.sources_client.build_source_status(self.status()) def _set_provider_active_status(self, active_status): """Set provider active status.""" if self.source.koku_uuid: try: provider = Provider.objects.get(uuid=self.source.koku_uuid) provider.active = active_status provider.save() except Provider.DoesNotExist: LOG.info( f"No provider found for Source ID: {self.source.source_id}" ) def determine_status(self, provider_type, source_authentication, source_billing_source): """Check cloud configuration status.""" interface = ProviderAccessor(provider_type) error_obj = None try: interface.cost_usage_source_ready(source_authentication, source_billing_source) self._set_provider_active_status(True) except ValidationError as validation_error: self._set_provider_active_status(False) error_obj = validation_error return error_obj def status(self): """Find the source's availability status.""" source_billing_source = self.source.billing_source.get( "data_source") or {} source_authentication = self.source.authentication.get( "credentials") or {} provider_type = self.source.source_type return self.determine_status(provider_type, source_authentication, source_billing_source) def push_status(self): """Push status_msg to platform sources.""" try: status_obj = self.status() self.sources_client.set_source_status(status_obj) except SourcesHTTPClientError as error: err_msg = "Unable to push source status. Reason: {}".format( str(error)) LOG.warning(err_msg)
def __init__(self, source_id): """Initialize source id.""" self.source_id = source_id self.source = Sources.objects.get(source_id=source_id) if not source_settings_complete(self.source) or self.source.pending_delete: raise ObjectDoesNotExist(f"Source ID: {self.source_id} not ready for status") self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_network_response_success(self): """Test get network response succeeds.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(url=MOCK_URL, json={"data": "valid json"}) resp = client._get_network_response(MOCK_URL, "test error") self.assertEqual(resp.get("data"), "valid json")
def test_set_source_status_unexpected_header(self): """Test to set source status with missing account in header.""" test_source_id = 1 application_type_id = 2 error_msg = "my error" malformed_identity_header = { "not_identity": { "type": "User", "user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True}, } } json_malformed_identity = json_dumps(malformed_identity_header) malformed_internal_header = b64encode(json_malformed_identity.encode("utf-8")) malformed_auth_header = malformed_internal_header.decode("utf-8") missing_account_header = { "identity": { "type": "User", "user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True}, } } missing_account_identity = json_dumps(missing_account_header) missing_account_internal_header = b64encode(missing_account_identity.encode("utf-8")) missing_account_auth_header = missing_account_internal_header.decode("utf-8") test_headers = [malformed_auth_header, missing_account_auth_header] source = Sources.objects.create(source_id=test_source_id, offset=42, source_type="AWS") source.save() for header in test_headers: client = SourcesHTTPClient(auth_header=header, source_id=test_source_id) response = client.set_source_status(error_msg, application_type_id) self.assertFalse(response)
def test_get_aws_role_arn_no_auth(self): """Test to get AWS Role ARN from authentication service with auth not ready.""" resource_id = 2 authentication_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}", status_code=200, json={"data": [{"id": resource_id}]}, ) m.get( ( f"http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint" f"&[authtype]=arn&[resource_id]={resource_id}" ), status_code=200, json={"data": []}, ) m.get( ( f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}" f"?expose_encrypted_attribute[]=password" ), status_code=200, json={"password": self.authentication}, ) with self.assertRaises(SourcesHTTPClientError): client.get_aws_role_arn()
def __init__(self, request, source_id): """Initialize source id.""" self.request = request self.user = request.user self.source_id = source_id self.source = Sources.objects.get(source_id=source_id) self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_network_response_exception(self): """Test get network response with request exception.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(url=MOCK_URL, exc=RequestException) with self.assertRaises(SourcesHTTPClientError): client._get_network_response(MOCK_URL, "test error")
def test_get_aws_credentials_from_app_auth(self): """Test to get AWS Role ARN from authentication service for Application authentication.""" resource_id = 2 authentication_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}", status_code=200, json={"data": [{"id": resource_id}]}, ) m.get( (f"http://www.sources.com/api/v1.0/authentications?" f"[authtype]=arn&[resource_id]={resource_id}"), status_code=200, json={"data": [{"id": authentication_id}]}, ) m.get( ( f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}" f"?expose_encrypted_attribute[]=password" ), status_code=200, json={"password": self.authentication}, ) response = client.get_aws_credentials() self.assertEqual(response, {"role_arn": self.authentication})
def test_get_application_type_is_cost_management_misconfigured(self): """Test to get application_type_id from source_id with route not found.""" application_type_id = 2 source_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=source_id) responses.add( responses.GET, f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources", json={"data": [{ "name": "test-source" }]}, status=404, ) responses.add( responses.GET, "http://www.sources.com/api/v1.0/application_types", json={"data": [{ "id": self.application_type }]}, status=200, ) with self.assertRaises(SourceNotFoundError): client.get_application_type_is_cost_management(source_id)
def test_set_source_status_patch_fail(self): """Test to set source status where the patch fails.""" test_source_id = 1 application_type_id = 2 application_id = 3 status = 'available' error_msg = 'my error' client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=test_source_id) with requests_mock.mock() as m: m.get(( f'http://www.sources.com/api/v1.0/applications?' f'filter[application_type_id]={application_type_id}&filter[source_id]={test_source_id}' ), status_code=200, json={'data': [{ 'id': application_id }]}) m.patch( f'http://www.sources.com/api/v1.0/applications/{application_id}', status_code=400, json={ 'availability_status': status, 'availability_status_error': str(error_msg) }) with self.assertRaises(SourcesHTTPClientError): client.set_source_status(error_msg, application_type_id)
def test_get_azure_credentials_no_auth(self): """Test to get Azure credentials from authentication service with auth not ready.""" resource_id = 2 authentication_id = 3 authentication = "testclientcreds" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}", status_code=200, json={"data": [{"id": resource_id}]}, ) m.get( ( f"http://www.sources.com/api/v1.0/authentications?" f"[authtype]=tenant_id_client_id_client_secret&[resource_id]={resource_id}" ), status_code=200, json={"data": []}, ) m.get( ( f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}" f"?expose_encrypted_attribute[]=password" ), status_code=200, json={"password": authentication}, ) with self.assertRaises(SourcesHTTPClientError): client.get_azure_credentials()
def test_get_aws_role_arn(self): """Test to get AWS Role ARN from authentication service.""" resource_id = 2 authentication_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}', status_code=200, json={'data': [{ 'id': resource_id }]}) m.get(( f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint' f'&[authtype]=arn&[resource_id]={resource_id}'), status_code=200, json={'data': [{ 'id': authentication_id }]}) m.get(( f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}' f'?expose_encrypted_attribute[]=password'), status_code=200, json={'password': self.authentication}) response = client.get_aws_role_arn() self.assertEqual(response, self.authentication)
def test_get_azure_credentials_no_auth(self): """Test to get Azure credentials from authentication service with auth not ready.""" resource_id = 2 authentication_id = 3 authentication = 'testclientcreds' client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}', status_code=200, json={'data': [{ 'id': resource_id }]}) m.get(( f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint' f'&[authtype]=tenant_id_client_id_client_secret&[resource_id]={resource_id}' ), status_code=200, json={'data': []}) m.get(( f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}' f'?expose_encrypted_attribute[]=password'), status_code=200, json={'password': authentication}) with self.assertRaises(SourcesHTTPClientError): client.get_azure_credentials()
def test_set_source_status(self): """Test to set source status.""" test_source_id = 1 application_type_id = 2 application_id = 3 status = "unavailable" error_msg = "my error" source = Sources.objects.create(source_id=test_source_id, offset=42, source_type="AWS") source.save() client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=test_source_id) with requests_mock.mock() as m: m.get( ( f"http://www.sources.com/api/v1.0/applications?" f"filter[application_type_id]={application_type_id}&filter[source_id]={test_source_id}" ), status_code=200, json={"data": [{"id": application_id}]}, ) m.patch( f"http://www.sources.com/api/v1.0/applications/{application_id}", status_code=204, json={"availability_status": status, "availability_status_error": str(error_msg)}, ) response = client.set_source_status(error_msg, application_type_id) self.assertTrue(response)
def cost_mgmt_msg_filter(msg_data): """Verify that message is for cost management.""" event_type = msg_data.get("event_type") auth_header = msg_data.get("auth_header") if event_type in (KAFKA_APPLICATION_DESTROY, KAFKA_SOURCE_DESTROY): return msg_data if event_type in (KAFKA_AUTHENTICATION_CREATE, KAFKA_AUTHENTICATION_UPDATE): sources_network = SourcesHTTPClient(auth_header) if msg_data.get("resource_type") == "Application": source_id = sources_network.get_source_id_from_applications_id( msg_data.get("resource_id")) msg_data["source_id"] = source_id if not sources_network.get_application_type_is_cost_management( source_id): LOG.info( f"Resource id {msg_data.get('resource_id')} not associated with cost-management." ) return None else: source_id = msg_data.get("source_id") return msg_data
def test_get_source_details_unsuccessful(self): """Test to get source details unsuccessfully.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=404) with self.assertRaises(SourcesHTTPClientError): client.get_source_details()
def test_get_application_type_is_cost_management(self): """Test to get application_type_id from source_id.""" application_type_id = 2 source_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=source_id) responses.add( responses.GET, f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources", json={"data": [{ "name": "test-source" }]}, status=200, ) responses.add( responses.GET, "http://www.sources.com/api/v1.0/application_types", json={"data": [{ "id": self.application_type }]}, status=200, ) response = client.get_application_type_is_cost_management(source_id) self.assertTrue(response)
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f'Source ID not found for ID: {source_id}') return try: if source_type == 'OCP': source_details = sources_network.get_source_details() # Check for imported to maintain temporary backwards compatibility # until the Sources Front End creates 'imported' entry with OCP Cluster ID. if source_details.get('source_ref'): authentication = { 'resource_name': source_details.get('source_ref') } else: uid = source_details.get('uid') LOG.info( f'OCP is using fallback Source UID ({str(uid)} for authentication.' ' Update frontend to add Cluster ID to the source_ref field on the Source.' ) authentication = {'resource_name': uid} elif source_type == 'AWS': authentication = { 'resource_name': sources_network.get_aws_role_arn() } elif source_type == 'AZURE': authentication = { 'credentials': sources_network.get_azure_credentials() } else: LOG.error(f'Unexpected source type: {source_type}') return storage.add_provider_sources_auth_info(source_id, authentication) except SourcesHTTPClientError: LOG.info( f'Authentication info not available for Source ID: {source_id}')
def __init__(self, auth_header, source_id): sources_network = SourcesHTTPClient(auth_header, source_id) details = sources_network.get_source_details() self.name = details.get("name") self.source_type_id = int(details.get("source_type_id")) self.source_uuid = details.get("uid") self.source_type_name = sources_network.get_source_type_name(self.source_type_id) self.source_type = SOURCE_PROVIDER_MAP.get(self.source_type_name)
def execute_koku_provider_op(msg, cost_management_type_id): """ Execute the 'create' or 'destroy Koku-Provider operations. 'create' operations: Koku POST /providers is executed along with updating the Sources database table with the Koku Provider uuid. 'destroy' operations: Koku DELETE /providers is executed along with removing the Sources database entry. Two types of exceptions are handled for Koku HTTP operations. Recoverable client and Non-Recoverable client errors. If the error is recoverable the calling function (synchronize_sources) will re-queue the operation. Args: msg (Asyncio msg): Dictionary messages containing operation, provider and offset. example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3} cost_management_type_id (Integer): Cost Management Type Identifier Returns: None """ provider = msg.get("provider") operation = msg.get("operation") try: if operation == "create": task = create_or_update_provider.delay(provider.source_id) LOG.info( f"Creating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}" ) elif operation == "update": task = create_or_update_provider.delay(provider.source_id) LOG.info( f"Updating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}" ) elif operation == "destroy": task = delete_source_and_provider.delay(provider.source_id, provider.source_uuid, provider.auth_header) LOG.info( f"Deleting Koku Provider/Source for Source ID: {str(provider.source_id)} in task: {task.id}" ) else: LOG.error(f"unknown operation: {operation}") except RabbitOperationalError: err_msg = f"RabbitmQ unavailable. Unable to {operation} Source ID: {provider.source_id}" LOG.error(err_msg) sources_network = SourcesHTTPClient(provider.auth_header, provider.source_id) sources_network.set_source_status( err_msg, cost_management_type_id=cost_management_type_id) # Re-raise exception so it can be re-queued in synchronize_sources raise RabbitOperationalError(err_msg)
def test_get_source_details_connection_error(self): """Test to get source details with connection error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}", exc=RequestException) with self.assertRaises(SourcesHTTPClientError): client.get_source_details()
def test_get_source_id_from_applications_id_server_error(self): """Test to get source ID from application resource_id with server error.""" resource_id = 2 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get(f"http://www.sources.com/api/v1.0/applications?filter[id]={resource_id}", status_code=400) with self.assertRaises(SourcesHTTPClientError): client.get_source_id_from_applications_id(resource_id)
def test_get_source_details(self): """Test to get source details.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=200, json={"name": self.name} ) response = client.get_source_details() self.assertEqual(response.get("name"), self.name)
def test_get_cost_management_application_type_id_error(self): """Test to get application type id with error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER) with requests_mock.mock() as m: m.get( f'http://www.sources.com/api/v1.0/application_types?filter[name]=/insights/platform/cost-management', exc=requests.exceptions.RequestException) with self.assertRaises(SourcesHTTPClientError): client.get_cost_management_application_type_id()
def test__get_ocp_credentials_missing_cluster_id(self): """Test to get ocp cluster-id with missing cluster-id raises exception.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": None} ) with self.assertRaises(SourcesHTTPClientError): client._get_ocp_credentials()
def test_get_network_response_status_exception(self): """Test get network response with invalid status responses.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) table = [{"status": 404, "expected": SourceNotFoundError}, {"status": 403, "expected": SourcesHTTPClientError}] for test in table: with self.subTest(test=test): with requests_mock.mock() as m: m.get(url=MOCK_URL, status_code=test.get("status"), exc=test.get("exc")) with self.assertRaises(test.get("expected")): client._get_network_response(MOCK_URL, "test error")
def test_get_endpoint_ids_connection_error(self): """Test to get endpoint id with connection error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}", exc=RequestException) with self.assertRaises(SourcesHTTPClientError): client.get_endpoint_id()
def test_get_source_type_name_error(self): """Test to get source type name from type id with error.""" source_type_id = 3 client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER) with requests_mock.mock() as m: m.get( f'http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}', exc=requests.exceptions.RequestException) with self.assertRaises(SourcesHTTPClientError): client.get_source_type_name(source_type_id)
def test_get_azure_credentials_connection_error(self): """Test to get Azure credentials from authentication service with connection error.""" client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}", exc=RequestException, ) with self.assertRaises(SourcesHTTPClientError): client.get_azure_credentials()
def test__get_ocp_credentials(self): """Test to get ocp cluster-id.""" uuid = str(uuid4()) client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id) with requests_mock.mock() as m: m.get( f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": uuid} ) creds = client._get_ocp_credentials() self.assertEqual(creds.get("cluster_id"), uuid)