def test_clear_update_flag(self):
"""Test for clearing source update flag."""
test_matrix = [{
'koku_uuid': None,
'pending_update': False,
'expected_pending_update': False
}, {
'koku_uuid': faker.uuid4(),
'pending_update': False,
'expected_pending_update': False
}, {
'koku_uuid': faker.uuid4(),
'pending_update': True,
'expected_pending_update': False
}]
test_source_id = 3
for test in test_matrix:
aws_obj = Sources(source_id=test_source_id,
auth_header=self.test_header,
koku_uuid=test.get('koku_uuid'),
pending_update=test.get('pending_update'),
offset=3,
endpoint_id=4,
source_type=Provider.PROVIDER_AWS,
name='Test AWS Source',
billing_source={'bucket': 'test-bucket'})
aws_obj.save()
storage.clear_update_flag(test_source_id)
response = Sources.objects.get(source_id=test_source_id)
self.assertEquals(test.get('expected_pending_update'),
response.pending_update)
test_source_id += 1
async def process_synchronize_sources_msg(msg_tuple,
process_queue,
cost_management_type_id,
loop=EVENT_LOOP):
"""
Synchronize Platform Sources with Koku Providers.
Task will process the process_queue which contains filtered
events (Cost Management Platform-Sources).
The items on the queue are Koku-Provider 'create' or 'destroy
events. If the Koku-Provider operation fails the event will
be re-queued until the operation is successful.
Args:
process_queue (Asyncio.Queue): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
cost_management_type_id (Integer): Cost Management Type Identifier
Returns:
None
"""
priority, msg = msg_tuple
LOG.info(f'Koku provider operation to execute: {msg.get("operation")} '
f'for Source ID: {str(msg.get("provider").source_id)}')
try:
with concurrent.futures.ThreadPoolExecutor() as pool:
await loop.run_in_executor(pool, execute_koku_provider_op, msg,
cost_management_type_id)
LOG.info(
f'Koku provider operation to execute: {msg.get("operation")} '
f'for Source ID: {str(msg.get("provider").source_id)} complete.')
if msg.get("operation") != "destroy":
storage.clear_update_flag(msg.get("provider").source_id)
except (InterfaceError, OperationalError) as error:
connection.close()
LOG.warning(
f"[synchronize_sources] Closing DB connection and re-queueing failed operation."
f" Encountered {type(error).__name__}: {error}")
await _requeue_provider_sync_message(priority, msg, process_queue)
except RabbitOperationalError as error:
LOG.warning(
f"[synchronize_sources] RabbitMQ is down and re-queueing failed operation."
f" Encountered {type(error).__name__}: {error}")
await _requeue_provider_sync_message(priority, msg, process_queue)
except Exception as error:
# The reason for catching all exceptions is to ensure that the event
# loop remains active in the event that provider synchronization fails unexpectedly.
provider = msg.get("provider")
source_id = provider.source_id if provider else "unknown"
LOG.error(
f"[synchronize_sources] Unexpected synchronization error for Source ID {source_id} "
f"encountered: {type(error).__name__}: {error}",
exc_info=True,
)
def test_clear_update_flag(self):
"""Test for clearing source update flag."""
test_matrix = [
{"koku_uuid": None, "pending_update": False, "expected_pending_update": False},
{"koku_uuid": faker.uuid4(), "pending_update": False, "expected_pending_update": False},
{"koku_uuid": faker.uuid4(), "pending_update": True, "expected_pending_update": False},
]
test_source_id = 3
for test in test_matrix:
aws_obj = Sources(
source_id=test_source_id,
auth_header=self.test_header,
koku_uuid=test.get("koku_uuid"),
pending_update=test.get("pending_update"),
offset=3,
source_type=Provider.PROVIDER_AWS,
name="Test AWS Source",
billing_source={"bucket": "test-bucket"},
)
aws_obj.save()
storage.clear_update_flag(test_source_id)
response = Sources.objects.get(source_id=test_source_id)
self.assertEquals(test.get("expected_pending_update"), response.pending_update)
test_source_id += 1
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f"Source ID not found for ID: {source_id}")
return
try:
if source_type == Provider.PROVIDER_OCP:
source_details = sources_network.get_source_details()
if source_details.get("source_ref"):
authentication = {
"resource_name": source_details.get("source_ref")
}
else:
raise SourcesHTTPClientError("Unable to find Cluster ID")
elif source_type in (Provider.PROVIDER_AWS,
Provider.PROVIDER_AWS_LOCAL):
authentication = {
"resource_name": sources_network.get_aws_role_arn()
}
elif source_type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
authentication = {
"credentials": sources_network.get_azure_credentials()
}
else:
LOG.error(f"Unexpected source type: {source_type}")
return
storage.add_provider_sources_auth_info(source_id, authentication)
storage.clear_update_flag(source_id)
LOG.info(f"Authentication attached to Source ID: {source_id}")
except SourcesHTTPClientError as error:
LOG.info(
f"Authentication info not available for Source ID: {source_id}")
sources_network.set_source_status(str(error))
def execute_koku_provider_op(msg, cost_management_type_id):
"""
Execute the 'create' or 'destroy Koku-Provider operations.
'create' operations:
Koku POST /providers is executed along with updating the Sources database table with
the Koku Provider uuid.
'destroy' operations:
Koku DELETE /providers is executed along with removing the Sources database entry.
Two types of exceptions are handled for Koku HTTP operations. Recoverable client and
Non-Recoverable client errors. If the error is recoverable the calling function
(synchronize_sources) will re-queue the operation.
Args:
msg (Asyncio msg): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
cost_management_type_id (Integer): Cost Management Type Identifier
Returns:
None
"""
provider = msg.get('provider')
operation = msg.get('operation')
koku_client = KokuHTTPClient(provider.auth_header)
sources_client = SourcesHTTPClient(provider.auth_header, provider.source_id)
try:
if operation == 'create':
LOG.info(f'Creating Koku Provider for Source ID: {str(provider.source_id)}')
koku_details = koku_client.create_provider(provider.name, provider.source_type, provider.authentication,
provider.billing_source, provider.source_uuid)
LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} assigned to Source ID {str(provider.source_id)}.')
storage.add_provider_koku_uuid(provider.source_id, koku_details.get('uuid'))
elif operation == 'destroy':
if provider.koku_uuid:
try:
response = koku_client.destroy_provider(provider.koku_uuid)
LOG.info(
f'Koku Provider UUID ({provider.koku_uuid}) Removal Status Code: {str(response.status_code)}')
except KokuHTTPClientNonRecoverableError:
LOG.info(f'Koku Provider already removed. Remove Source ID: {str(provider.source_id)}.')
storage.destroy_provider_event(provider.source_id)
elif operation == 'update':
koku_details = koku_client.update_provider(provider.koku_uuid, provider.name, provider.source_type,
provider.authentication, provider.billing_source)
storage.clear_update_flag(provider.source_id)
LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} with Source ID {str(provider.source_id)} updated.')
sources_client.set_source_status(None, cost_management_type_id)
except KokuHTTPClientError as koku_error:
raise SourcesIntegrationError('Koku provider error: ', str(koku_error))
except KokuHTTPClientNonRecoverableError as koku_error:
err_msg = f'Unable to {operation} provider for Source ID: {str(provider.source_id)}. Reason: {str(koku_error)}'
LOG.error(err_msg)
sources_client.set_source_status(str(koku_error), cost_management_type_id)
def update_account(self, source):
"""Call to update provider."""
try:
LOG.info(f"Updating Provider for Source ID: {str(self._source_id)}")
provider = self._provider_builder.update_provider_from_source(source)
clear_update_flag(self._source_id)
except ProviderBuilderError as provider_err:
raise SourcesProviderCoordinatorError(str(provider_err))
return provider
def update_account(self, provider_uuid, name, provider_type,
authentication, billing_source):
"""Call to update provider."""
try:
provider = self._provider_builder.update_provider(
provider_uuid, name, provider_type, authentication,
billing_source)
clear_update_flag(self._source_id)
except ProviderBuilderError as provider_err:
raise SourcesProviderCoordinatorError(str(provider_err))
return provider
async def synchronize_sources(process_queue,
cost_management_type_id): # pragma: no cover
"""
Synchronize Platform Sources with Koku Providers.
Task will process the process_queue which contains filtered
events (Cost Management Platform-Sources).
The items on the queue are Koku-Provider 'create' or 'destroy
events. If the Koku-Provider operation fails the event will
be re-queued until the operation is successful.
Args:
process_queue (Asyncio.Queue): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
cost_management_type_id (Integer): Cost Management Type Identifier
Returns:
None
"""
LOG.info('Processing koku provider events...')
while True:
msg = await process_queue.get()
LOG.info(f'Koku provider operation to execute: {msg.get("operation")} '
f'for Source ID: {str(msg.get("provider").source_id)}')
try:
with concurrent.futures.ThreadPoolExecutor() as pool:
await EVENT_LOOP.run_in_executor(pool,
execute_koku_provider_op, msg,
cost_management_type_id)
LOG.info(
f'Koku provider operation to execute: {msg.get("operation")} '
f'for Source ID: {str(msg.get("provider").source_id)} complete.'
)
storage.clear_update_flag(msg.get('provider').source_id)
except SourcesIntegrationError as error:
LOG.error('Re-queueing failed operation. Error: %s', str(error))
await asyncio.sleep(Config.RETRY_SECONDS)
_log_process_queue_event(process_queue, msg)
await process_queue.put(msg)
LOG.info(
f'Requeue of failed operation: {msg.get("operation")} '
f'for Source ID: {str(msg.get("provider").source_id)} complete.'
)
except Exception as error:
# The reason for catching all exceptions is to ensure that the event
# loop remains active in the event that provider synchronization fails unexpectedly.
provider = msg.get('provider')
source_id = provider.source_id if provider else 'unknown'
LOG.error(
f'Source {source_id} Unexpected synchronization error: {str(error)}'
)
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f'Source ID not found for ID: {source_id}')
return
try:
if source_type == 'OCP':
source_details = sources_network.get_source_details()
# Check for imported to maintain temporary backwards compatibility
# until the Sources Front End creates 'imported' entry with OCP Cluster ID.
if source_details.get('source_ref'):
authentication = {'resource_name': source_details.get('source_ref')}
else:
uid = source_details.get('uid')
LOG.info(f'OCP is using fallback Source UID ({str(uid)} for authentication.'
' Update frontend to add Cluster ID to the source_ref field on the Source.')
authentication = {'resource_name': uid}
elif source_type == 'AWS':
authentication = {'resource_name': sources_network.get_aws_role_arn()}
elif source_type == 'AZURE':
authentication = {'credentials': sources_network.get_azure_credentials()}
else:
LOG.error(f'Unexpected source type: {source_type}')
return
storage.add_provider_sources_auth_info(source_id, authentication)
storage.clear_update_flag(source_id)
except SourcesHTTPClientError:
LOG.info(f'Authentication info not available for Source ID: {source_id}')
def set_source_status(self, error_msg, cost_management_type_id=None):
"""Set the source status with error message."""
if storage.is_known_source(self._source_id):
storage.clear_update_flag(self._source_id)
status_header = self.build_status_header()
if not status_header:
return False
if not cost_management_type_id:
cost_management_type_id = self.get_cost_management_application_type_id(
)
application_query_url = (
f"{self._base_url}/{ENDPOINT_APPLICATIONS}"
f"?filter[application_type_id]={cost_management_type_id}&filter[source_id]={self._source_id}"
)
application_query_response = self._get_network_response(
application_query_url,
"[set_source_status] unable to get application")
response_data = (application_query_response.get("data") or [None])[0]
if response_data:
application_id = response_data.get("id")
application_url = f"{self._base_url}/{ENDPOINT_APPLICATIONS}/{application_id}"
json_data = self.build_source_status(error_msg)
if storage.save_status(self._source_id, json_data):
LOG.info(
f"[set_source_status] source_id: {self._source_id}: {json_data}"
)
application_response = requests.patch(application_url,
json=json_data,
headers=status_header)
error_message = (
f"[set_source_status] error: Status code: "
f"{application_response.status_code}. Response: {application_response.text}."
)
if application_response.status_code != 204:
if application_response.status_code != 404:
raise SourcesHTTPClientError(error_message)
else:
LOG.info(error_message)
return True
return False
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if not source_type:
LOG.info(f"Source ID not found for ID: {source_id}")
return
sources_network = SourcesHTTPClient(auth_header, source_id)
try:
authentication = get_authentication(source_type, sources_network)
except SourcesHTTPClientError as error:
LOG.info(
f"Authentication info not available for Source ID: {source_id}")
sources_network.set_source_status(error)
else:
if not authentication:
return
storage.add_provider_sources_auth_info(source_id, authentication)
storage.clear_update_flag(source_id)
LOG.info(f"Authentication attached to Source ID: {source_id}")
def test_clear_update_flag_unknown_id(self):
"""Test to clear update flag for an unknown id."""
self.test_obj.pending_update = True
self.test_obj.save()
storage.clear_update_flag(self.test_source_id + 1)
self.assertTrue(self.test_obj.pending_update)
def set_status_helper(*args, **kwargs):
"""helper to clear update flag."""
storage.clear_update_flag(source_id)
