def test_get_source_type_name_non_200(self):
"""Test to get source type name from type id with bad response."""
source_type_id = 3
mock_source_name = "fakesource"
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}",
status_code=404,
json={"data": [{
"name": mock_source_name
}]},
)
with self.assertRaises(SourceNotFoundError):
client.get_source_type_name(source_type_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}",
status_code=401,
json={"data": [{
"name": mock_source_name
}]},
)
with self.assertRaises(SourcesHTTPClientError):
client.get_source_type_name(source_type_id)
def test_get_aws_credentials_from_app_auth(self):
"""Test to get AWS Role ARN from authentication service for Application authentication."""
resource_id = 2
authentication_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}",
status_code=200,
json={"data": [{
"id": resource_id
}]},
)
m.get(
(f"http://www.sources.com/api/v1.0/authentications?"
f"[authtype]=arn&[resource_id]={resource_id}"),
status_code=200,
json={"data": [{
"id": authentication_id
}]},
)
m.get(
(f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}"
f"?expose_encrypted_attribute[]=password"),
status_code=200,
json={"password": self.authentication},
)
response = client.get_aws_credentials()
self.assertEqual(response, {"role_arn": self.authentication})
def test_get_aws_role_arn(self):
"""Test to get AWS Role ARN from authentication service."""
resource_id = 2
authentication_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}',
status_code=200,
json={'data': [{
'id': resource_id
}]})
m.get((
f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint'
f'&[authtype]=arn&[resource_id]={resource_id}'),
status_code=200,
json={'data': [{
'id': authentication_id
}]})
m.get((
f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}'
f'?expose_encrypted_attribute[]=password'),
status_code=200,
json={'password': self.authentication})
response = client.get_aws_role_arn()
self.assertEqual(response, self.authentication)
def test_get_azure_credentials_no_auth(self):
"""Test to get Azure credentials from authentication service with auth not ready."""
resource_id = 2
authentication_id = 3
authentication = 'testclientcreds'
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}',
status_code=200,
json={'data': [{
'id': resource_id
}]})
m.get((
f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint'
f'&[authtype]=tenant_id_client_id_client_secret&[resource_id]={resource_id}'
),
status_code=200,
json={'data': []})
m.get((
f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}'
f'?expose_encrypted_attribute[]=password'),
status_code=200,
json={'password': authentication})
with self.assertRaises(SourcesHTTPClientError):
client.get_azure_credentials()
def __init__(self, source_id):
"""Initialize source id."""
self.source_id = source_id
self.source = Sources.objects.get(source_id=source_id)
if not source_settings_complete(self.source) or self.source.pending_delete:
raise ObjectDoesNotExist(f"Source ID: {self.source_id} not ready for status")
self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_application_type_is_cost_management_misconfigured(self):
"""Test to get application_type_id from source_id with route not found."""
application_type_id = 2
source_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=source_id)
responses.add(
responses.GET,
f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources",
json={"data": [{
"name": "test-source"
}]},
status=404,
)
responses.add(
responses.GET,
f"http://www.sources.com/api/v1.0/application_types",
json={"data": [{
"id": self.application_type
}]},
status=200,
)
with self.assertRaises(SourceNotFoundError):
client.get_application_type_is_cost_management(source_id)
def __init__(self, request, source_id):
"""Initialize source id."""
self.request = request
self.user = request.user
self.source_id = source_id
self.source = Sources.objects.get(source_id=source_id)
self.sources_client = SourcesHTTPClient(self.source.auth_header, source_id=source_id)
def test_get_application_type_is_cost_management(self):
"""Test to get application_type_id from source_id."""
application_type_id = 2
source_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=source_id)
responses.add(
responses.GET,
f"http://www.sources.com/api/v1.0/application_types/{application_type_id}/sources",
json={"data": [{
"name": "test-source"
}]},
status=200,
)
responses.add(
responses.GET,
f"http://www.sources.com/api/v1.0/application_types",
json={"data": [{
"id": self.application_type
}]},
status=200,
)
response = client.get_application_type_is_cost_management(source_id)
self.assertTrue(response)
def test_set_source_status_patch_fail(self):
"""Test to set source status where the patch fails."""
test_source_id = 1
application_type_id = 2
application_id = 3
status = "unavailable"
error_msg = "my error"
source = Sources.objects.create(source_id=test_source_id,
offset=42,
source_type="AWS")
source.save()
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=test_source_id)
with requests_mock.mock() as m:
m.get(
(f"http://www.sources.com/api/v1.0/applications?"
f"filter[application_type_id]={application_type_id}&filter[source_id]={test_source_id}"
),
status_code=200,
json={"data": [{
"id": application_id
}]},
)
m.patch(
f"http://www.sources.com/api/v1.0/applications/{application_id}",
status_code=400,
json={
"availability_status": status,
"availability_status_error": str(error_msg)
},
)
with self.assertRaises(SourcesHTTPClientError):
client.set_source_status(error_msg, application_type_id)
def test_get_aws_role_arn_no_auth(self):
"""Test to get AWS Role ARN from authentication service with auth not ready."""
resource_id = 2
authentication_id = 3
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}",
status_code=200,
json={"data": [{
"id": resource_id
}]},
)
m.get(
(f"http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint"
f"&[authtype]=arn&[resource_id]={resource_id}"),
status_code=200,
json={"data": []},
)
m.get(
(f"http://www.sources.com/internal/v1.0/authentications/{authentication_id}"
f"?expose_encrypted_attribute[]=password"),
status_code=200,
json={"password": self.authentication},
)
with self.assertRaises(SourcesHTTPClientError):
client.get_aws_role_arn()
def test_get_source_details_unsuccessful(self):
"""Test to get source details unsuccessfully."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=404)
with self.assertRaises(SourceNotFoundError):
client.get_source_details()
def test_set_source_status_unexpected_header(self):
"""Test to set source status with missing account in header."""
test_source_id = 1
application_type_id = 2
error_msg = "my error"
malformed_identity_header = {
"not_identity": {
"type": "User",
"user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True},
}
}
json_malformed_identity = json_dumps(malformed_identity_header)
malformed_internal_header = b64encode(json_malformed_identity.encode("utf-8"))
malformed_auth_header = malformed_internal_header.decode("utf-8")
missing_account_header = {
"identity": {
"type": "User",
"user": {"username": "******", "email": "*****@*****.**", "is_org_admin": True},
}
}
missing_account_identity = json_dumps(missing_account_header)
missing_account_internal_header = b64encode(missing_account_identity.encode("utf-8"))
missing_account_auth_header = missing_account_internal_header.decode("utf-8")
test_headers = [malformed_auth_header, missing_account_auth_header]
source = Sources.objects.create(source_id=test_source_id, offset=42, source_type="AWS")
source.save()
for header in test_headers:
client = SourcesHTTPClient(auth_header=header, source_id=test_source_id)
response = client.set_source_status(error_msg, application_type_id)
self.assertFalse(response)
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f'Source ID not found for ID: {source_id}')
return
try:
if source_type == 'OCP':
source_details = sources_network.get_source_details()
# Check for imported to maintain temporary backwards compatibility
# until the Sources Front End creates 'imported' entry with OCP Cluster ID.
if source_details.get('source_ref'):
authentication = {
'resource_name': source_details.get('source_ref')
}
else:
uid = source_details.get('uid')
LOG.info(
f'OCP is using fallback Source UID ({str(uid)} for authentication.'
' Update frontend to add Cluster ID to the source_ref field on the Source.'
)
authentication = {'resource_name': uid}
elif source_type == 'AWS':
authentication = {
'resource_name': sources_network.get_aws_role_arn()
}
elif source_type == 'AZURE':
authentication = {
'credentials': sources_network.get_azure_credentials()
}
else:
LOG.error(f'Unexpected source type: {source_type}')
return
storage.add_provider_sources_auth_info(source_id, authentication)
except SourcesHTTPClientError:
LOG.info(
f'Authentication info not available for Source ID: {source_id}')
def __init__(self, auth_header, source_id):
sources_network = SourcesHTTPClient(auth_header, source_id)
details = sources_network.get_source_details()
self.name = details.get("name")
self.source_type_id = int(details.get("source_type_id"))
self.source_uuid = details.get("uid")
self.source_type_name = sources_network.get_source_type_name(self.source_type_id)
self.source_type = SOURCE_PROVIDER_MAP.get(self.source_type_name)
def test_get_network_response_exception(self):
"""Test get network response with request exception."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(url=MOCK_URL, exc=RequestException)
with self.assertRaises(SourcesHTTPClientError):
client._get_network_response(MOCK_URL, "test error")
def test_get_network_response_success(self):
"""Test get network response succeeds."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(url=MOCK_URL, json={"data": "valid json"})
resp = client._get_network_response(MOCK_URL, "test error")
self.assertEqual(resp.get("data"), "valid json")
def execute_koku_provider_op(msg, cost_management_type_id):
"""
Execute the 'create' or 'destroy Koku-Provider operations.
'create' operations:
Koku POST /providers is executed along with updating the Sources database table with
the Koku Provider uuid.
'destroy' operations:
Koku DELETE /providers is executed along with removing the Sources database entry.
Two types of exceptions are handled for Koku HTTP operations. Recoverable client and
Non-Recoverable client errors. If the error is recoverable the calling function
(synchronize_sources) will re-queue the operation.
Args:
msg (Asyncio msg): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
cost_management_type_id (Integer): Cost Management Type Identifier
Returns:
None
"""
provider = msg.get("provider")
operation = msg.get("operation")
try:
if operation == "create":
task = create_or_update_provider.delay(provider.source_id)
LOG.info(
f"Creating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}"
)
elif operation == "update":
task = create_or_update_provider.delay(provider.source_id)
LOG.info(
f"Updating Koku Provider for Source ID: {str(provider.source_id)} in task: {task.id}"
)
elif operation == "destroy":
task = delete_source_and_provider.delay(provider.source_id,
provider.source_uuid,
provider.auth_header)
LOG.info(
f"Deleting Koku Provider/Source for Source ID: {str(provider.source_id)} in task: {task.id}"
)
else:
LOG.error(f"unknown operation: {operation}")
except RabbitOperationalError:
err_msg = f"RabbitmQ unavailable. Unable to {operation} Source ID: {provider.source_id}"
LOG.error(err_msg)
sources_network = SourcesHTTPClient(provider.auth_header,
provider.source_id)
sources_network.set_source_status(
err_msg, cost_management_type_id=cost_management_type_id)
# Re-raise exception so it can be re-queued in synchronize_sources
raise RabbitOperationalError(err_msg)
def check_sources_connection():
"""Check sources-backend connection."""
try:
cost_management_type_id = SourcesHTTPClient(
SourcesConfig.SOURCES_FAKE_HEADER
).get_cost_management_application_type_id()
return cost_management_type_id
except (SourcesHTTPClientError, SourceNotFoundError):
return
def test_get_cost_management_application_type_id_error(self):
"""Test to get application type id with error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
with requests_mock.mock() as m:
m.get(
f'http://www.sources.com/api/v1.0/application_types?filter[name]=/insights/platform/cost-management',
exc=requests.exceptions.RequestException)
with self.assertRaises(SourcesHTTPClientError):
client.get_cost_management_application_type_id()
def test_get_source_details_connection_error(self):
"""Test to get source details with connection error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(f"http://www.sources.com/api/v1.0/sources/{self.source_id}",
exc=RequestException)
with self.assertRaises(SourcesHTTPClientError):
client.get_source_details()
def execute_koku_provider_op(msg, cost_management_type_id):
"""
Execute the 'create' or 'destroy Koku-Provider operations.
'create' operations:
Koku POST /providers is executed along with updating the Sources database table with
the Koku Provider uuid.
'destroy' operations:
Koku DELETE /providers is executed along with removing the Sources database entry.
Two types of exceptions are handled for Koku HTTP operations. Recoverable client and
Non-Recoverable client errors. If the error is recoverable the calling function
(synchronize_sources) will re-queue the operation.
Args:
msg (Asyncio msg): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
cost_management_type_id (Integer): Cost Management Type Identifier
Returns:
None
"""
provider = msg.get('provider')
operation = msg.get('operation')
koku_client = KokuHTTPClient(provider.auth_header)
sources_client = SourcesHTTPClient(provider.auth_header, provider.source_id)
try:
if operation == 'create':
LOG.info(f'Creating Koku Provider for Source ID: {str(provider.source_id)}')
koku_details = koku_client.create_provider(provider.name, provider.source_type, provider.authentication,
provider.billing_source, provider.source_uuid)
LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} assigned to Source ID {str(provider.source_id)}.')
storage.add_provider_koku_uuid(provider.source_id, koku_details.get('uuid'))
elif operation == 'destroy':
if provider.koku_uuid:
try:
response = koku_client.destroy_provider(provider.koku_uuid)
LOG.info(
f'Koku Provider UUID ({provider.koku_uuid}) Removal Status Code: {str(response.status_code)}')
except KokuHTTPClientNonRecoverableError:
LOG.info(f'Koku Provider already removed. Remove Source ID: {str(provider.source_id)}.')
storage.destroy_provider_event(provider.source_id)
elif operation == 'update':
koku_details = koku_client.update_provider(provider.koku_uuid, provider.name, provider.source_type,
provider.authentication, provider.billing_source)
storage.clear_update_flag(provider.source_id)
LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} with Source ID {str(provider.source_id)} updated.')
sources_client.set_source_status(None, cost_management_type_id)
except KokuHTTPClientError as koku_error:
raise SourcesIntegrationError('Koku provider error: ', str(koku_error))
except KokuHTTPClientNonRecoverableError as koku_error:
err_msg = f'Unable to {operation} provider for Source ID: {str(provider.source_id)}. Reason: {str(koku_error)}'
LOG.error(err_msg)
sources_client.set_source_status(str(koku_error), cost_management_type_id)
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f"Source ID not found for ID: {source_id}")
return
try:
if source_type == Provider.PROVIDER_OCP:
source_details = sources_network.get_source_details()
if source_details.get("source_ref"):
authentication = {
"resource_name": source_details.get("source_ref")
}
else:
raise SourcesHTTPClientError("Unable to find Cluster ID")
elif source_type in (Provider.PROVIDER_AWS,
Provider.PROVIDER_AWS_LOCAL):
authentication = {
"resource_name": sources_network.get_aws_role_arn()
}
elif source_type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
authentication = {
"credentials": sources_network.get_azure_credentials()
}
else:
LOG.error(f"Unexpected source type: {source_type}")
return
storage.add_provider_sources_auth_info(source_id, authentication)
storage.clear_update_flag(source_id)
LOG.info(f"Authentication attached to Source ID: {source_id}")
except SourcesHTTPClientError as error:
LOG.info(
f"Authentication info not available for Source ID: {source_id}")
sources_network.set_source_status(str(error))
def test_get_source_details(self):
"""Test to get source details."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/sources/{self.source_id}", status_code=200, json={"name": self.name}
)
response = client.get_source_details()
self.assertEqual(response.get("name"), self.name)
def test_get_source_id_from_applications_id_server_error(self):
"""Test to get source ID from application resource_id with server error."""
resource_id = 2
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(f"http://www.sources.com/api/v1.0/applications?filter[id]={resource_id}", status_code=400)
with self.assertRaises(SourcesHTTPClientError):
client.get_source_id_from_applications_id(resource_id)
def test__get_ocp_credentials_missing_cluster_id(self):
"""Test to get ocp cluster-id with missing cluster-id raises exception."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": None}
)
with self.assertRaises(SourcesHTTPClientError):
client._get_ocp_credentials()
def sources_backend(self):
"""Return Sources Backend connection status."""
try:
cost_management_type_id = SourcesHTTPClient(SourcesConfig.SOURCES_FAKE_HEADER).\
get_cost_management_application_type_id()
return f'Cost Management Application ID: {cost_management_type_id}'
except SourcesHTTPClientError:
return 'Not connected'
def test_get_endpoint_ids_connection_error(self):
"""Test to get endpoint id with connection error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER,
source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={self.source_id}",
exc=RequestException)
with self.assertRaises(SourcesHTTPClientError):
client.get_endpoint_id()
def test_get_azure_credentials_connection_error(self):
"""Test to get Azure credentials from authentication service with connection error."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/applications?filter[source_id]={self.source_id}",
exc=RequestException,
)
with self.assertRaises(SourcesHTTPClientError):
client.get_azure_credentials()
def test_get_network_response_status_exception(self):
"""Test get network response with invalid status responses."""
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
table = [{"status": 404, "expected": SourceNotFoundError}, {"status": 403, "expected": SourcesHTTPClientError}]
for test in table:
with self.subTest(test=test):
with requests_mock.mock() as m:
m.get(url=MOCK_URL, status_code=test.get("status"), exc=test.get("exc"))
with self.assertRaises(test.get("expected")):
client._get_network_response(MOCK_URL, "test error")
def test__get_ocp_credentials(self):
"""Test to get ocp cluster-id."""
uuid = str(uuid4())
client = SourcesHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER, source_id=self.source_id)
with requests_mock.mock() as m:
m.get(
f"{MOCK_URL}/api/v1.0/{ENDPOINT_SOURCES}/{self.source_id}", status_code=200, json={"source_ref": uuid}
)
creds = client._get_ocp_credentials()
self.assertEqual(creds.get("cluster_id"), uuid)
