def test_clear_update_flag(self): """Test for clearing source update flag.""" test_matrix = [{ 'koku_uuid': None, 'pending_update': False, 'expected_pending_update': False }, { 'koku_uuid': faker.uuid4(), 'pending_update': False, 'expected_pending_update': False }, { 'koku_uuid': faker.uuid4(), 'pending_update': True, 'expected_pending_update': False }] test_source_id = 3 for test in test_matrix: aws_obj = Sources(source_id=test_source_id, auth_header=self.test_header, koku_uuid=test.get('koku_uuid'), pending_update=test.get('pending_update'), offset=3, endpoint_id=4, source_type=Provider.PROVIDER_AWS, name='Test AWS Source', billing_source={'bucket': 'test-bucket'}) aws_obj.save() storage.clear_update_flag(test_source_id) response = Sources.objects.get(source_id=test_source_id) self.assertEquals(test.get('expected_pending_update'), response.pending_update) test_source_id += 1
async def process_synchronize_sources_msg(msg_tuple, process_queue, cost_management_type_id, loop=EVENT_LOOP): """ Synchronize Platform Sources with Koku Providers. Task will process the process_queue which contains filtered events (Cost Management Platform-Sources). The items on the queue are Koku-Provider 'create' or 'destroy events. If the Koku-Provider operation fails the event will be re-queued until the operation is successful. Args: process_queue (Asyncio.Queue): Dictionary messages containing operation, provider and offset. example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3} cost_management_type_id (Integer): Cost Management Type Identifier Returns: None """ priority, msg = msg_tuple LOG.info(f'Koku provider operation to execute: {msg.get("operation")} ' f'for Source ID: {str(msg.get("provider").source_id)}') try: with concurrent.futures.ThreadPoolExecutor() as pool: await loop.run_in_executor(pool, execute_koku_provider_op, msg, cost_management_type_id) LOG.info( f'Koku provider operation to execute: {msg.get("operation")} ' f'for Source ID: {str(msg.get("provider").source_id)} complete.') if msg.get("operation") != "destroy": storage.clear_update_flag(msg.get("provider").source_id) except (InterfaceError, OperationalError) as error: connection.close() LOG.warning( f"[synchronize_sources] Closing DB connection and re-queueing failed operation." f" Encountered {type(error).__name__}: {error}") await _requeue_provider_sync_message(priority, msg, process_queue) except RabbitOperationalError as error: LOG.warning( f"[synchronize_sources] RabbitMQ is down and re-queueing failed operation." f" Encountered {type(error).__name__}: {error}") await _requeue_provider_sync_message(priority, msg, process_queue) except Exception as error: # The reason for catching all exceptions is to ensure that the event # loop remains active in the event that provider synchronization fails unexpectedly. provider = msg.get("provider") source_id = provider.source_id if provider else "unknown" LOG.error( f"[synchronize_sources] Unexpected synchronization error for Source ID {source_id} " f"encountered: {type(error).__name__}: {error}", exc_info=True, )
def test_clear_update_flag(self): """Test for clearing source update flag.""" test_matrix = [ {"koku_uuid": None, "pending_update": False, "expected_pending_update": False}, {"koku_uuid": faker.uuid4(), "pending_update": False, "expected_pending_update": False}, {"koku_uuid": faker.uuid4(), "pending_update": True, "expected_pending_update": False}, ] test_source_id = 3 for test in test_matrix: aws_obj = Sources( source_id=test_source_id, auth_header=self.test_header, koku_uuid=test.get("koku_uuid"), pending_update=test.get("pending_update"), offset=3, source_type=Provider.PROVIDER_AWS, name="Test AWS Source", billing_source={"bucket": "test-bucket"}, ) aws_obj.save() storage.clear_update_flag(test_source_id) response = Sources.objects.get(source_id=test_source_id) self.assertEquals(test.get("expected_pending_update"), response.pending_update) test_source_id += 1
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f"Source ID not found for ID: {source_id}") return try: if source_type == Provider.PROVIDER_OCP: source_details = sources_network.get_source_details() if source_details.get("source_ref"): authentication = { "resource_name": source_details.get("source_ref") } else: raise SourcesHTTPClientError("Unable to find Cluster ID") elif source_type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL): authentication = { "resource_name": sources_network.get_aws_role_arn() } elif source_type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL): authentication = { "credentials": sources_network.get_azure_credentials() } else: LOG.error(f"Unexpected source type: {source_type}") return storage.add_provider_sources_auth_info(source_id, authentication) storage.clear_update_flag(source_id) LOG.info(f"Authentication attached to Source ID: {source_id}") except SourcesHTTPClientError as error: LOG.info( f"Authentication info not available for Source ID: {source_id}") sources_network.set_source_status(str(error))
def execute_koku_provider_op(msg, cost_management_type_id): """ Execute the 'create' or 'destroy Koku-Provider operations. 'create' operations: Koku POST /providers is executed along with updating the Sources database table with the Koku Provider uuid. 'destroy' operations: Koku DELETE /providers is executed along with removing the Sources database entry. Two types of exceptions are handled for Koku HTTP operations. Recoverable client and Non-Recoverable client errors. If the error is recoverable the calling function (synchronize_sources) will re-queue the operation. Args: msg (Asyncio msg): Dictionary messages containing operation, provider and offset. example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3} cost_management_type_id (Integer): Cost Management Type Identifier Returns: None """ provider = msg.get('provider') operation = msg.get('operation') koku_client = KokuHTTPClient(provider.auth_header) sources_client = SourcesHTTPClient(provider.auth_header, provider.source_id) try: if operation == 'create': LOG.info(f'Creating Koku Provider for Source ID: {str(provider.source_id)}') koku_details = koku_client.create_provider(provider.name, provider.source_type, provider.authentication, provider.billing_source, provider.source_uuid) LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} assigned to Source ID {str(provider.source_id)}.') storage.add_provider_koku_uuid(provider.source_id, koku_details.get('uuid')) elif operation == 'destroy': if provider.koku_uuid: try: response = koku_client.destroy_provider(provider.koku_uuid) LOG.info( f'Koku Provider UUID ({provider.koku_uuid}) Removal Status Code: {str(response.status_code)}') except KokuHTTPClientNonRecoverableError: LOG.info(f'Koku Provider already removed. Remove Source ID: {str(provider.source_id)}.') storage.destroy_provider_event(provider.source_id) elif operation == 'update': koku_details = koku_client.update_provider(provider.koku_uuid, provider.name, provider.source_type, provider.authentication, provider.billing_source) storage.clear_update_flag(provider.source_id) LOG.info(f'Koku Provider UUID {koku_details.get("uuid")} with Source ID {str(provider.source_id)} updated.') sources_client.set_source_status(None, cost_management_type_id) except KokuHTTPClientError as koku_error: raise SourcesIntegrationError('Koku provider error: ', str(koku_error)) except KokuHTTPClientNonRecoverableError as koku_error: err_msg = f'Unable to {operation} provider for Source ID: {str(provider.source_id)}. Reason: {str(koku_error)}' LOG.error(err_msg) sources_client.set_source_status(str(koku_error), cost_management_type_id)
def update_account(self, source): """Call to update provider.""" try: LOG.info(f"Updating Provider for Source ID: {str(self._source_id)}") provider = self._provider_builder.update_provider_from_source(source) clear_update_flag(self._source_id) except ProviderBuilderError as provider_err: raise SourcesProviderCoordinatorError(str(provider_err)) return provider
def update_account(self, provider_uuid, name, provider_type, authentication, billing_source): """Call to update provider.""" try: provider = self._provider_builder.update_provider( provider_uuid, name, provider_type, authentication, billing_source) clear_update_flag(self._source_id) except ProviderBuilderError as provider_err: raise SourcesProviderCoordinatorError(str(provider_err)) return provider
async def synchronize_sources(process_queue, cost_management_type_id): # pragma: no cover """ Synchronize Platform Sources with Koku Providers. Task will process the process_queue which contains filtered events (Cost Management Platform-Sources). The items on the queue are Koku-Provider 'create' or 'destroy events. If the Koku-Provider operation fails the event will be re-queued until the operation is successful. Args: process_queue (Asyncio.Queue): Dictionary messages containing operation, provider and offset. example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3} cost_management_type_id (Integer): Cost Management Type Identifier Returns: None """ LOG.info('Processing koku provider events...') while True: msg = await process_queue.get() LOG.info(f'Koku provider operation to execute: {msg.get("operation")} ' f'for Source ID: {str(msg.get("provider").source_id)}') try: with concurrent.futures.ThreadPoolExecutor() as pool: await EVENT_LOOP.run_in_executor(pool, execute_koku_provider_op, msg, cost_management_type_id) LOG.info( f'Koku provider operation to execute: {msg.get("operation")} ' f'for Source ID: {str(msg.get("provider").source_id)} complete.' ) storage.clear_update_flag(msg.get('provider').source_id) except SourcesIntegrationError as error: LOG.error('Re-queueing failed operation. Error: %s', str(error)) await asyncio.sleep(Config.RETRY_SECONDS) _log_process_queue_event(process_queue, msg) await process_queue.put(msg) LOG.info( f'Requeue of failed operation: {msg.get("operation")} ' f'for Source ID: {str(msg.get("provider").source_id)} complete.' ) except Exception as error: # The reason for catching all exceptions is to ensure that the event # loop remains active in the event that provider synchronization fails unexpectedly. provider = msg.get('provider') source_id = provider.source_id if provider else 'unknown' LOG.error( f'Source {source_id} Unexpected synchronization error: {str(error)}' )
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f'Source ID not found for ID: {source_id}') return try: if source_type == 'OCP': source_details = sources_network.get_source_details() # Check for imported to maintain temporary backwards compatibility # until the Sources Front End creates 'imported' entry with OCP Cluster ID. if source_details.get('source_ref'): authentication = {'resource_name': source_details.get('source_ref')} else: uid = source_details.get('uid') LOG.info(f'OCP is using fallback Source UID ({str(uid)} for authentication.' ' Update frontend to add Cluster ID to the source_ref field on the Source.') authentication = {'resource_name': uid} elif source_type == 'AWS': authentication = {'resource_name': sources_network.get_aws_role_arn()} elif source_type == 'AZURE': authentication = {'credentials': sources_network.get_azure_credentials()} else: LOG.error(f'Unexpected source type: {source_type}') return storage.add_provider_sources_auth_info(source_id, authentication) storage.clear_update_flag(source_id) except SourcesHTTPClientError: LOG.info(f'Authentication info not available for Source ID: {source_id}')
def set_source_status(self, error_msg, cost_management_type_id=None): """Set the source status with error message.""" if storage.is_known_source(self._source_id): storage.clear_update_flag(self._source_id) status_header = self.build_status_header() if not status_header: return False if not cost_management_type_id: cost_management_type_id = self.get_cost_management_application_type_id( ) application_query_url = ( f"{self._base_url}/{ENDPOINT_APPLICATIONS}" f"?filter[application_type_id]={cost_management_type_id}&filter[source_id]={self._source_id}" ) application_query_response = self._get_network_response( application_query_url, "[set_source_status] unable to get application") response_data = (application_query_response.get("data") or [None])[0] if response_data: application_id = response_data.get("id") application_url = f"{self._base_url}/{ENDPOINT_APPLICATIONS}/{application_id}" json_data = self.build_source_status(error_msg) if storage.save_status(self._source_id, json_data): LOG.info( f"[set_source_status] source_id: {self._source_id}: {json_data}" ) application_response = requests.patch(application_url, json=json_data, headers=status_header) error_message = ( f"[set_source_status] error: Status code: " f"{application_response.status_code}. Response: {application_response.text}." ) if application_response.status_code != 204: if application_response.status_code != 404: raise SourcesHTTPClientError(error_message) else: LOG.info(error_message) return True return False
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if not source_type: LOG.info(f"Source ID not found for ID: {source_id}") return sources_network = SourcesHTTPClient(auth_header, source_id) try: authentication = get_authentication(source_type, sources_network) except SourcesHTTPClientError as error: LOG.info( f"Authentication info not available for Source ID: {source_id}") sources_network.set_source_status(error) else: if not authentication: return storage.add_provider_sources_auth_info(source_id, authentication) storage.clear_update_flag(source_id) LOG.info(f"Authentication attached to Source ID: {source_id}")
def test_clear_update_flag_unknown_id(self): """Test to clear update flag for an unknown id.""" self.test_obj.pending_update = True self.test_obj.save() storage.clear_update_flag(self.test_source_id + 1) self.assertTrue(self.test_obj.pending_update)
def set_status_helper(*args, **kwargs): """helper to clear update flag.""" storage.clear_update_flag(source_id)