class SourceStatus:
"""Source Status."""
def __init__(self, request, source_id):
"""Initialize source id."""
self.request = request
self.user = request.user
self.source_id = source_id
self.source = Sources.objects.get(source_id=source_id)
self.sources_client = SourcesHTTPClient(self.source.auth_header,
source_id=source_id)
@property
def sources_response(self):
return self.sources_client.build_source_status(self.status())
def _set_provider_active_status(self, active_status):
"""Set provider active status."""
if self.source.koku_uuid:
try:
provider = Provider.objects.get(uuid=self.source.koku_uuid)
provider.active = active_status
provider.save()
except Provider.DoesNotExist:
LOG.info(
f"No provider found for Source ID: {self.source.source_id}"
)
def determine_status(self, provider_type, source_authentication,
source_billing_source):
"""Check cloud configuration status."""
interface = ProviderAccessor(provider_type)
error_obj = None
try:
interface.cost_usage_source_ready(source_authentication,
source_billing_source)
self._set_provider_active_status(True)
except ValidationError as validation_error:
self._set_provider_active_status(False)
error_obj = validation_error
return error_obj
def status(self):
"""Find the source's availability status."""
source_billing_source = self.source.billing_source.get(
"data_source") or {}
source_authentication = self.source.authentication.get(
"credentials") or {}
provider_type = self.source.source_type
return self.determine_status(provider_type, source_authentication,
source_billing_source)
def push_status(self):
"""Push status_msg to platform sources."""
try:
status_obj = self.status()
self.sources_client.set_source_status(status_obj)
except SourcesHTTPClientError as error:
err_msg = "Unable to push source status. Reason: {}".format(
str(error))
LOG.warning(err_msg)
def __init__(self, source_id):
"""Initialize source id."""
self.source_id = source_id
self.source = Sources.objects.get(source_id=source_id)
if not source_settings_complete(self.source) or self.source.pending_delete:
raise ObjectDoesNotExist(f"Source ID: {self.source_id} not ready for status")
self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_network_response_success(self):
"""Test get network response succeeds."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(url=MOCK_URL, json={"data": "valid json"})
resp = client._get_network_response(MOCK_URL, "test error")
self.assertEqual(resp.get("data"), "valid json")
def test_set_source_status_unexpected_header(self):
"""Test to set source status with missing account in header."""
test_source_id = 1
application_type_id = 2
error_msg = "my error"
malformed_identity_header = {
"not_identity": {
"type": "User",
"user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True},
}
}
json_malformed_identity = json_dumps(malformed_identity_header)
malformed_internal_header = b64encode(json_malformed_identity.encode("utf-8"))
malformed_auth_header = malformed_internal_header.decode("utf-8")
missing_account_header = {
"identity": {
"type": "User",
"user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True},
}
}
missing_account_identity = json_dumps(missing_account_header)
missing_account_internal_header = b64encode(missing_account_identity.encode("utf-8"))
missing_account_auth_header = missing_account_internal_header.decode("utf-8")
test_headers = [malformed_auth_header, missing_account_auth_header]
source = Sources.objects.create(source_id=test_source_id, offset=42, source_type="AWS")
source.save()
for header in test_headers:
client = SourcesHTTPClient(auth_header=header, source_id=test_source_id)
response = client.set_source_status(error_msg, application_type_id)
self.assertFalse(response)
def test_get_aws_role_arn_no_auth(self):
"""Test to get AWS Role ARN from authentication service with auth not ready."""
resource_id = 2
authentication_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}",
status_code=200,
json={"data": [{"id": resource_id}]},
)
m.get(
(
f"http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint"
f"&[authtype]=arn&[resource_id]={resource_id}"
),
status_code=200,
json={"data": []},
)
m.get(
(
f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}"
f"?expose_encrypted_attribute[]=password"
),
status_code=200,
json={"password": self.authentication},
)
with self.assertRaises(SourcesHTTPClientError):
client.get_aws_role_arn()
def __init__(self, request, source_id):
"""Initialize source id."""
self.request = request
self.user = request.user
self.source_id = source_id
self.source = Sources.objects.get(source_id=source_id)
self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_network_response_exception(self):
"""Test get network response with request exception."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(url=MOCK_URL, exc=RequestException)
with self.assertRaises(SourcesHTTPClientError):
client._get_network_response(MOCK_URL, "test error")
def test_get_aws_credentials_from_app_auth(self):
"""Test to get AWS Role ARN from authentication service for Application authentication."""
resource_id = 2
authentication_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}",
status_code=200,
json={"data": [{"id": resource_id}]},
)
m.get(
(f"http://www.sources.com/api/v1.0/authentications?" f"[authtype]=arn&[resource_id]={resource_id}"),
status_code=200,
json={"data": [{"id": authentication_id}]},
)
m.get(
(
f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}"
f"?expose_encrypted_attribute[]=password"
),
status_code=200,
json={"password": self.authentication},
)
response = client.get_aws_credentials()
self.assertEqual(response, {"role_arn": self.authentication})
def test_get_application_type_is_cost_management_misconfigured(self):
"""Test to get application_type_id from source_id with route not found."""
application_type_id = 2
source_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=source_id)
responses.add(
responses.GET,
f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources",
json={"data": [{
"name": "test-source"
}]},
status=404,
)
responses.add(
responses.GET,
"http://www.sources.com/api/v1.0/application_types",
json={"data": [{
"id": self.application_type
}]},
status=200,
)
with self.assertRaises(SourceNotFoundError):
client.get_application_type_is_cost_management(source_id)
def test_set_source_status_patch_fail(self):
"""Test to set source status where the patch fails."""
test_source_id = 1
application_type_id = 2
application_id = 3
status = 'available'
error_msg = 'my error'
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=test_source_id)
with requests_mock.mock() as m:
m.get((
f'http://www.sources.com/api/v1.0/applications?'
f'filter[application_type_id]={application_type_id}&filter[source_id]={test_source_id}'
),
status_code=200,
json={'data': [{
'id': application_id
}]})
m.patch(
f'http://www.sources.com/api/v1.0/applications/{application_id}',
status_code=400,
json={
'availability_status': status,
'availability_status_error': str(error_msg)
})
with self.assertRaises(SourcesHTTPClientError):
client.set_source_status(error_msg, application_type_id)
def test_get_azure_credentials_no_auth(self):
"""Test to get Azure credentials from authentication service with auth not ready."""
resource_id = 2
authentication_id = 3
authentication = "testclientcreds"
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}",
status_code=200,
json={"data": [{"id": resource_id}]},
)
m.get(
(
f"http://www.sources.com/api/v1.0/authentications?"
f"[authtype]=tenant_id_client_id_client_secret&[resource_id]={resource_id}"
),
status_code=200,
json={"data": []},
)
m.get(
(
f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}"
f"?expose_encrypted_attribute[]=password"
),
status_code=200,
json={"password": authentication},
)
with self.assertRaises(SourcesHTTPClientError):
client.get_azure_credentials()
def test_get_aws_role_arn(self):
"""Test to get AWS Role ARN from authentication service."""
resource_id = 2
authentication_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}',
status_code=200,
json={'data': [{
'id': resource_id
}]})
m.get((
f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint'
f'&[authtype]=arn&[resource_id]={resource_id}'),
status_code=200,
json={'data': [{
'id': authentication_id
}]})
m.get((
f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}'
f'?expose_encrypted_attribute[]=password'),
status_code=200,
json={'password': self.authentication})
response = client.get_aws_role_arn()
self.assertEqual(response, self.authentication)
def test_get_azure_credentials_no_auth(self):
"""Test to get Azure credentials from authentication service with auth not ready."""
resource_id = 2
authentication_id = 3
authentication = 'testclientcreds'
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}',
status_code=200,
json={'data': [{
'id': resource_id
}]})
m.get((
f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint'
f'&[authtype]=tenant_id_client_id_client_secret&[resource_id]={resource_id}'
),
status_code=200,
json={'data': []})
m.get((
f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}'
f'?expose_encrypted_attribute[]=password'),
status_code=200,
json={'password': authentication})
with self.assertRaises(SourcesHTTPClientError):
client.get_azure_credentials()
def test_set_source_status(self):
"""Test to set source status."""
test_source_id = 1
application_type_id = 2
application_id = 3
status = "unavailable"
error_msg = "my error"
source = Sources.objects.create(source_id=test_source_id, offset=42, source_type="AWS")
source.save()
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=test_source_id)
with requests_mock.mock() as m:
m.get(
(
f"http://www.sources.com/api/v1.0/applications?"
f"filter[application_type_id]={application_type_id}&filter[source_id]={test_source_id}"
),
status_code=200,
json={"data": [{"id": application_id}]},
)
m.patch(
f"http://www.sources.com/api/v1.0/applications/{application_id}",
status_code=204,
json={"availability_status": status, "availability_status_error": str(error_msg)},
)
response = client.set_source_status(error_msg, application_type_id)
self.assertTrue(response)
def cost_mgmt_msg_filter(msg_data):
"""Verify that message is for cost management."""
event_type = msg_data.get("event_type")
auth_header = msg_data.get("auth_header")
if event_type in (KAFKA_APPLICATION_DESTROY, KAFKA_SOURCE_DESTROY):
return msg_data
if event_type in (KAFKA_AUTHENTICATION_CREATE,
KAFKA_AUTHENTICATION_UPDATE):
sources_network = SourcesHTTPClient(auth_header)
if msg_data.get("resource_type") == "Application":
source_id = sources_network.get_source_id_from_applications_id(
msg_data.get("resource_id"))
msg_data["source_id"] = source_id
if not sources_network.get_application_type_is_cost_management(
source_id):
LOG.info(
f"Resource id {msg_data.get('resource_id')} not associated with cost-management."
)
return None
else:
source_id = msg_data.get("source_id")
return msg_data
def test_get_source_details_unsuccessful(self):
"""Test to get source details unsuccessfully."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=404)
with self.assertRaises(SourcesHTTPClientError):
client.get_source_details()
def test_get_application_type_is_cost_management(self):
"""Test to get application_type_id from source_id."""
application_type_id = 2
source_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=source_id)
responses.add(
responses.GET,
f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources",
json={"data": [{
"name": "test-source"
}]},
status=200,
)
responses.add(
responses.GET,
"http://www.sources.com/api/v1.0/application_types",
json={"data": [{
"id": self.application_type
}]},
status=200,
)
response = client.get_application_type_is_cost_management(source_id)
self.assertTrue(response)
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f'Source ID not found for ID: {source_id}')
return
try:
if source_type == 'OCP':
source_details = sources_network.get_source_details()
# Check for imported to maintain temporary backwards compatibility
# until the Sources Front End creates 'imported' entry with OCP Cluster ID.
if source_details.get('source_ref'):
authentication = {
'resource_name': source_details.get('source_ref')
}
else:
uid = source_details.get('uid')
LOG.info(
f'OCP is using fallback Source UID ({str(uid)} for authentication.'
' Update frontend to add Cluster ID to the source_ref field on the Source.'
)
authentication = {'resource_name': uid}
elif source_type == 'AWS':
authentication = {
'resource_name': sources_network.get_aws_role_arn()
}
elif source_type == 'AZURE':
authentication = {
'credentials': sources_network.get_azure_credentials()
}
else:
LOG.error(f'Unexpected source type: {source_type}')
return
storage.add_provider_sources_auth_info(source_id, authentication)
except SourcesHTTPClientError:
LOG.info(
f'Authentication info not available for Source ID: {source_id}')
def __init__(self, auth_header, source_id):
sources_network = SourcesHTTPClient(auth_header, source_id)
details = sources_network.get_source_details()
self.name = details.get("name")
self.source_type_id = int(details.get("source_type_id"))
self.source_uuid = details.get("uid")
self.source_type_name = sources_network.get_source_type_name(self.source_type_id)
self.source_type = SOURCE_PROVIDER_MAP.get(self.source_type_name)
def execute_koku_provider_op(msg, cost_management_type_id):
"""
Execute the 'create' or 'destroy Koku-Provider operations.
'create' operations:
Koku POST /providers is executed along with updating the Sources database table with
the Koku Provider uuid.
'destroy' operations:
Koku DELETE /providers is executed along with removing the Sources database entry.
Two types of exceptions are handled for Koku HTTP operations. Recoverable client and
Non-Recoverable client errors. If the error is recoverable the calling function
(synchronize_sources) will re-queue the operation.
Args:
msg (Asyncio msg): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
cost_management_type_id (Integer): Cost Management Type Identifier
Returns:
None
"""
provider = msg.get("provider")
operation = msg.get("operation")
try:
if operation == "create":
task = create_or_update_provider.delay(provider.source_id)
LOG.info(
f"Creating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}"
)
elif operation == "update":
task = create_or_update_provider.delay(provider.source_id)
LOG.info(
f"Updating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}"
)
elif operation == "destroy":
task = delete_source_and_provider.delay(provider.source_id,
provider.source_uuid,
provider.auth_header)
LOG.info(
f"Deleting Koku Provider/Source for Source ID: {str(provider.source_id)} in task: {task.id}"
)
else:
LOG.error(f"unknown operation: {operation}")
except RabbitOperationalError:
err_msg = f"RabbitmQ unavailable. Unable to {operation} Source ID: {provider.source_id}"
LOG.error(err_msg)
sources_network = SourcesHTTPClient(provider.auth_header,
provider.source_id)
sources_network.set_source_status(
err_msg, cost_management_type_id=cost_management_type_id)
# Re-raise exception so it can be re-queued in synchronize_sources
raise RabbitOperationalError(err_msg)
def test_get_source_details_connection_error(self):
"""Test to get source details with connection error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}",
exc=RequestException)
with self.assertRaises(SourcesHTTPClientError):
client.get_source_details()
def test_get_source_id_from_applications_id_server_error(self):
"""Test to get source ID from application resource_id with server error."""
resource_id = 2
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(f"http://www.sources.com/api/v1.0/applications?filter[id]={resource_id}", status_code=400)
with self.assertRaises(SourcesHTTPClientError):
client.get_source_id_from_applications_id(resource_id)
def test_get_source_details(self):
"""Test to get source details."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=200, json={"name": self.name}
)
response = client.get_source_details()
self.assertEqual(response.get("name"), self.name)
def test_get_cost_management_application_type_id_error(self):
"""Test to get application type id with error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
with requests_mock.mock() as m:
m.get(
f'http://www.sources.com/api/v1.0/application_types?filter[name]=/insights/platform/cost-management',
exc=requests.exceptions.RequestException)
with self.assertRaises(SourcesHTTPClientError):
client.get_cost_management_application_type_id()
def test__get_ocp_credentials_missing_cluster_id(self):
"""Test to get ocp cluster-id with missing cluster-id raises exception."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": None}
)
with self.assertRaises(SourcesHTTPClientError):
client._get_ocp_credentials()
def test_get_network_response_status_exception(self):
"""Test get network response with invalid status responses."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
table = [{"status": 404, "expected": SourceNotFoundError}, {"status": 403, "expected": SourcesHTTPClientError}]
for test in table:
with self.subTest(test=test):
with requests_mock.mock() as m:
m.get(url=MOCK_URL, status_code=test.get("status"), exc=test.get("exc"))
with self.assertRaises(test.get("expected")):
client._get_network_response(MOCK_URL, "test error")
def test_get_endpoint_ids_connection_error(self):
"""Test to get endpoint id with connection error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}",
exc=RequestException)
with self.assertRaises(SourcesHTTPClientError):
client.get_endpoint_id()
def test_get_source_type_name_error(self):
"""Test to get source type name from type id with error."""
source_type_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
with requests_mock.mock() as m:
m.get(
f'http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}',
exc=requests.exceptions.RequestException)
with self.assertRaises(SourcesHTTPClientError):
client.get_source_type_name(source_type_id)
def test_get_azure_credentials_connection_error(self):
"""Test to get Azure credentials from authentication service with connection error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}",
exc=RequestException,
)
with self.assertRaises(SourcesHTTPClientError):
client.get_azure_credentials()
def test__get_ocp_credentials(self):
"""Test to get ocp cluster-id."""
uuid = str(uuid4())
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": uuid}
)
creds = client._get_ocp_credentials()
self.assertEqual(creds.get("cluster_id"), uuid)
