def test_get_authentication_for_provider_errors(self):
"""Test to build Koku Provider authentication json obj with errors."""
test_matrix = [{
'provider_type': 'AWS',
'authentication': {
'resource_namez': 'arn:fake'
},
'expected_response': KokuHTTPClientError
}, {
'provider_type': 'OCP',
'authentication': {
'resource_namez': 'test-cluster-id'
},
'expected_response': KokuHTTPClientError
}, {
'provider_type': 'AZURE',
'authentication': {
'credentialz': {
'foo': 'bar'
}
},
'expected_response': KokuHTTPClientError
}]
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
for test in test_matrix:
with self.assertRaises(test.get('expected_response')):
client.get_authentication_for_provider(
test.get('provider_type'), test.get('authentication'))
def test_get_billing_source_for_provider_error(self):
"""Test to build Koku Provider billing_source json obj with errors."""
test_matrix = [
{
"provider_type": Provider.PROVIDER_AWS,
"billing_source": {
"data_source": "test-bucket"
},
"expected_response": KokuHTTPClientError,
},
{
"provider_type": Provider.PROVIDER_AZURE,
"billing_source": {
"bucket": {
"foo": "bar"
}
},
"expected_response": KokuHTTPClientError,
},
]
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
for test in test_matrix:
with self.assertRaises(test.get("expected_response")):
client.get_billing_source_for_provider(
test.get("provider_type"), test.get("billing_source"))
def test_get_authentication_for_provider_errors(self):
"""Test to build Koku Provider authentication json obj with errors."""
test_matrix = [
{
"provider_type": Provider.PROVIDER_AWS,
"authentication": {
"resource_namez": "arn:fake"
},
"expected_response": KokuHTTPClientError,
},
{
"provider_type": Provider.PROVIDER_OCP,
"authentication": {
"resource_namez": "test-cluster-id"
},
"expected_response": KokuHTTPClientError,
},
{
"provider_type": Provider.PROVIDER_AZURE,
"authentication": {
"credentialz": {
"foo": "bar"
}
},
"expected_response": KokuHTTPClientError,
},
]
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
for test in test_matrix:
with self.assertRaises(test.get("expected_response")):
client.get_authentication_for_provider(
test.get("provider_type"), test.get("authentication"))
def test_create_provider_connection_error(self):
"""Test to create a provider with a connection error."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
with requests_mock.mock() as m:
m.post('http://www.koku.com/api/cost-management/v1/providers/',
exc=requests.exceptions.RequestException)
with self.assertRaises(KokuHTTPClientError):
client.create_provider(self.name, self.provider_type, self.authentication, self.billing_source)
def test_destroy_provider_not_found_error(self):
"""Test to destroy a provider with a koku server not found error."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.delete(f'http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/',
status_code=404, json={})
with self.assertRaises(KokuHTTPClientNonRecoverableError):
client.destroy_provider(expected_uuid)
def test_destroy_provider_exception(self):
"""Test to destroy a provider with a connection error."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.delete(f'http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/',
exc=requests.exceptions.RequestException)
with self.assertRaises(KokuHTTPClientError):
client.destroy_provider(expected_uuid)
def test_destroy_provider(self):
"""Test to destroy a provider."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.delete(f'http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/',
status_code=204)
response = client.destroy_provider(expected_uuid)
self.assertEqual(response.status_code, 204)
def test_create_provider_bad_permissions(self):
"""Test to create a provider with a bad permissions ."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
with requests_mock.mock() as m:
m.post('http://www.koku.com/api/cost-management/v1/providers/',
status_code=401,
json={'uuid': faker.uuid4()})
with self.assertRaises(KokuHTTPClientNonRecoverableError):
client.create_provider(self.name, self.provider_type, self.authentication, self.billing_source)
def execute_koku_provider_op(msg, cost_management_type_id):
"""
Execute the 'create' or 'destroy Koku-Provider operations.
'create' operations:
Koku POST /providers is executed along with updating the Sources database table with
the Koku Provider uuid.
'destroy' operations:
Koku DELETE /providers is executed along with removing the Sources database entry.
Two types of exceptions are handled for Koku HTTP operations. Recoverable client and
Non-Recoverable client errors. If the error is recoverable the calling function
(synchronize_sources) will re-queue the operation.
Args:
msg (Asyncio msg): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
cost_management_type_id (Integer): Cost Management Type Identifier
Returns:
None
"""
provider = msg.get('provider')
operation = msg.get('operation')
koku_client = KokuHTTPClient(provider.auth_header)
sources_client = SourcesHTTPClient(provider.auth_header, provider.source_id)
try:
if operation == 'create':
LOG.info(f'Creating Koku Provider for Source ID: {str(provider.source_id)}')
koku_details = koku_client.create_provider(provider.name, provider.source_type, provider.authentication,
provider.billing_source, provider.source_uuid)
LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} assigned to Source ID {str(provider.source_id)}.')
storage.add_provider_koku_uuid(provider.source_id, koku_details.get('uuid'))
elif operation == 'destroy':
if provider.koku_uuid:
try:
response = koku_client.destroy_provider(provider.koku_uuid)
LOG.info(
f'Koku Provider UUID ({provider.koku_uuid}) Removal Status Code: {str(response.status_code)}')
except KokuHTTPClientNonRecoverableError:
LOG.info(f'Koku Provider already removed. Remove Source ID: {str(provider.source_id)}.')
storage.destroy_provider_event(provider.source_id)
elif operation == 'update':
koku_details = koku_client.update_provider(provider.koku_uuid, provider.name, provider.source_type,
provider.authentication, provider.billing_source)
storage.clear_update_flag(provider.source_id)
LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} with Source ID {str(provider.source_id)} updated.')
sources_client.set_source_status(None, cost_management_type_id)
except KokuHTTPClientError as koku_error:
raise SourcesIntegrationError('Koku provider error: ', str(koku_error))
except KokuHTTPClientNonRecoverableError as koku_error:
err_msg = f'Unable to {operation} provider for Source ID: {str(provider.source_id)}. Reason: {str(koku_error)}'
LOG.error(err_msg)
sources_client.set_source_status(str(koku_error), cost_management_type_id)
def test_update_provider_not_found_error(self):
"""Test to update a provider with a koku server not found error."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.put(f'http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/',
status_code=404, json={})
with self.assertRaises(KokuHTTPClientNonRecoverableError):
client.update_provider(expected_uuid, 'Aws Test', 'AWS',
{'resource_name': 'arn:test'}, {'bucket': 'bucket'})
def test_update_provider(self):
"""Test to update a provider."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.put(f'http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/',
status_code=200, json={})
response = client.update_provider(expected_uuid, 'Aws Test', 'AWS',
{'resource_name': 'arn:test'}, {'bucket': 'bucket'})
self.assertEqual(response, {})
def test_update_provider_exception(self):
"""Test to update a provider with a connection error."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.put(f'http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/',
exc=requests.exceptions.RequestException)
with self.assertRaises(KokuHTTPClientError):
client.update_provider(expected_uuid, 'Aws Test', 'AWS',
{'resource_name': 'arn:test'}, {'bucket': 'bucket'})
def test_create_provider(self):
"""Test to create a provider."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.post('http://www.koku.com/api/cost-management/v1/providers/',
status_code=201,
json={'uuid': expected_uuid})
response = client.create_provider(self.name, self.provider_type, self.authentication, self.billing_source)
self.assertEqual(response.get('uuid'), expected_uuid)
def test_get_billing_source_for_provider_error(self):
"""Test to build Koku Provider billing_source json obj with errors."""
test_matrix = [{'provider_type': 'AWS', 'billing_source': {'data_source': 'test-bucket'},
'expected_response': KokuHTTPClientError},
{'provider_type': 'AZURE', 'billing_source': {'bucket': {'foo': 'bar'}},
'expected_response': KokuHTTPClientError}
]
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
for test in test_matrix:
with self.assertRaises(test.get('expected_response')):
client.get_billing_source_for_provider(test.get('provider_type'), test.get('billing_source'))
def test_update_provider(self):
"""Test to update a provider."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.put(
f"http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/",
status_code=200,
json={})
response = client.update_provider(expected_uuid, "Aws Test",
Provider.PROVIDER_AWS,
{"resource_name": "arn:test"},
{"bucket": "bucket"})
self.assertEqual(response, {})
def test_get_billing_source_for_provider(self):
"""Test to build Koku Provider billing_source json obj."""
test_matrix = [{'provider_type': 'AWS', 'billing_source': {'bucket': 'test-bucket'},
'expected_response': {'bucket': 'test-bucket'}},
{'provider_type': 'OCP', 'billing_source': {'bucket': ''},
'expected_response': {'bucket': ''}},
{'provider_type': 'AZURE', 'billing_source': {'data_source': {'foo': 'bar'}},
'expected_response': {'data_source': {'foo': 'bar'}}}
]
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
for test in test_matrix:
response = client.get_billing_source_for_provider(test.get('provider_type'), test.get('billing_source'))
self.assertEqual(response, test.get('expected_response'))
def test_update_provider_error(self):
"""Test to update a provider with a koku server error."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.put(
f"http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/",
status_code=400,
json={})
with self.assertRaises(KokuHTTPClientNonRecoverableError):
client.update_provider(
expected_uuid,
"Aws Test",
Provider.PROVIDER_AWS,
{"resource_name": "arn:test"},
{"bucket": "bucket"},
)
def test_update_provider_exception(self):
"""Test to update a provider with a connection error."""
client = KokuHTTPClient(auth_header=Config.SOURCES_FAKE_HEADER)
expected_uuid = faker.uuid4()
with requests_mock.mock() as m:
m.put(
f"http://www.koku.com/api/cost-management/v1/providers/{expected_uuid}/",
exc=requests.exceptions.RequestException,
)
with self.assertRaises(KokuHTTPClientError):
client.update_provider(
expected_uuid,
"Aws Test",
Provider.PROVIDER_AWS,
{"resource_name": "arn:test"},
{"bucket": "bucket"},
)