def process_message(app_type_id, msg): # noqa: C901 """ Process message from Platform-Sources kafka service. Handler for various application/source create and delete events. 'create' events: Issues a Sources REST API call to get additional context for the Platform-Sources kafka event. This information is stored in the Sources database table. 'destroy' events: Enqueues a source delete event which will be processed in the synchronize_sources method. Args: app_type_id - application type identifier msg - kafka message Returns: None """ LOG.info(f"Processing Event: {msg}") msg_data = None try: msg_data = cost_mgmt_msg_filter(msg) except SourceNotFoundError: LOG.warning(f"Source not found in platform sources. Skipping msg: {msg}") return if not msg_data: LOG.debug(f"Message not intended for cost management: {msg}") return if msg_data.get("event_type") in (KAFKA_APPLICATION_CREATE,): storage.create_source_event(msg_data.get("source_id"), msg_data.get("auth_header"), msg_data.get("offset")) if storage.is_known_source(msg_data.get("source_id")): sources_network_info(msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_AUTHENTICATION_CREATE, KAFKA_AUTHENTICATION_UPDATE): if msg_data.get("event_type") in (KAFKA_AUTHENTICATION_CREATE,): storage.create_source_event( # this will create source _only_ if it does not exist. msg_data.get("source_id"), msg_data.get("auth_header"), msg_data.get("offset") ) save_auth_info(msg_data.get("auth_header"), msg_data.get("source_id")) elif msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE,): if storage.is_known_source(msg_data.get("source_id")) is False: LOG.info("Update event for unknown source id, skipping...") return sources_network_info(msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_APPLICATION_DESTROY,): storage.enqueue_source_delete(msg_data.get("source_id"), msg_data.get("offset"), allow_out_of_order=True) elif msg_data.get("event_type") in (KAFKA_SOURCE_DESTROY,): storage.enqueue_source_delete(msg_data.get("source_id"), msg_data.get("offset")) if msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE, KAFKA_AUTHENTICATION_UPDATE): storage.enqueue_source_update(msg_data.get("source_id"))
async def process_messages(msg_pending_queue): # pragma: no cover """ Process messages from Platform-Sources kafka service. Handler for various application/source create and delete events. 'create' events: Issues a Sources REST API call to get additional context for the Platform-Sources kafka event. This information is stored in the Sources database table. 'destroy' events: Enqueues a source delete event which will be processed in the synchronize_sources method. Args: msg_pending_queue (Asyncio queue): Queue to hold kafka messages to be filtered Returns: None """ LOG.info('Waiting to process incoming kafka messages...') while True: msg_data = await msg_pending_queue.get() LOG.info(f'Processing Event: {str(msg_data)}') try: if msg_data.get('event_type') in (KAFKA_APPLICATION_CREATE, KAFKA_SOURCE_UPDATE): storage.create_provider_event(msg_data.get('source_id'), msg_data.get('auth_header'), msg_data.get('offset')) with concurrent.futures.ThreadPoolExecutor() as pool: await EVENT_LOOP.run_in_executor( pool, sources_network_info, msg_data.get('source_id'), msg_data.get('auth_header')) elif msg_data.get('event_type') in (KAFKA_AUTHENTICATION_CREATE, KAFKA_AUTHENTICATION_UPDATE): with concurrent.futures.ThreadPoolExecutor() as pool: await EVENT_LOOP.run_in_executor( pool, sources_network_auth_info, msg_data.get('resource_id'), msg_data.get('auth_header')) msg_data['source_id'] = storage.get_source_from_endpoint( msg_data.get('resource_id')) elif msg_data.get('event_type') in (KAFKA_APPLICATION_DESTROY, KAFKA_SOURCE_DESTROY): storage.enqueue_source_delete(msg_data.get('source_id')) if msg_data.get('event_type') in (KAFKA_SOURCE_UPDATE, KAFKA_AUTHENTICATION_UPDATE): storage.enqueue_source_update(msg_data.get('source_id')) except Exception as error: # The reason for catching all exceptions is to ensure that the event # loop remains active in the event that message processing fails unexpectedly. source_id = str(msg_data.get('source_id', 'unknown')) LOG.error( f'Source {source_id} Unexpected message processing error: {str(error)}' )
def test_enqueue_source_update(self): """Test for enqueuing source updating.""" test_matrix = [ { "koku_uuid": None, "pending_delete": False, "pending_update": False, "expected_pending_update": False }, { "koku_uuid": None, "pending_delete": True, "pending_update": False, "expected_pending_update": False }, { "koku_uuid": faker.uuid4(), "pending_delete": True, "pending_update": False, "expected_pending_update": False, }, { "koku_uuid": faker.uuid4(), "pending_delete": False, "pending_update": False, "expected_pending_update": True, }, { "koku_uuid": faker.uuid4(), "pending_delete": False, "pending_update": True, "expected_pending_update": True, }, ] test_source_id = 3 for test in test_matrix: aws_obj = Sources( source_id=test_source_id, auth_header=self.test_header, koku_uuid=test.get("koku_uuid"), pending_delete=test.get("pending_delete"), pending_update=test.get("pending_update"), offset=3, endpoint_id=4, source_type=Provider.PROVIDER_AWS, name="Test AWS Source", billing_source={"bucket": "test-bucket"}, ) aws_obj.save() storage.enqueue_source_update(test_source_id) response = Sources.objects.get(source_id=test_source_id) self.assertEquals(test.get("expected_pending_update"), response.pending_update) test_source_id += 1
def test_enqueue_source_update(self): """Test for enqueuing source updating.""" test_matrix = [{ 'koku_uuid': None, 'pending_delete': False, 'pending_update': False, 'expected_pending_update': False }, { 'koku_uuid': None, 'pending_delete': True, 'pending_update': False, 'expected_pending_update': False }, { 'koku_uuid': faker.uuid4(), 'pending_delete': True, 'pending_update': False, 'expected_pending_update': False }, { 'koku_uuid': faker.uuid4(), 'pending_delete': False, 'pending_update': False, 'expected_pending_update': True }, { 'koku_uuid': faker.uuid4(), 'pending_delete': False, 'pending_update': True, 'expected_pending_update': True }] test_source_id = 3 for test in test_matrix: aws_obj = Sources(source_id=test_source_id, auth_header=self.test_header, koku_uuid=test.get('koku_uuid'), pending_delete=test.get('pending_delete'), pending_update=test.get('pending_update'), offset=3, endpoint_id=4, source_type=Provider.PROVIDER_AWS, name='Test AWS Source', billing_source={'bucket': 'test-bucket'}) aws_obj.save() storage.enqueue_source_update(test_source_id) response = Sources.objects.get(source_id=test_source_id) self.assertEquals(test.get('expected_pending_update'), response.pending_update) test_source_id += 1
def test_enqueue_source_update_unknown_source(self): """Test to enqueue a source update for an unknown source.""" self.test_obj.koku_uuid = faker.uuid4() storage.enqueue_source_update(self.test_source_id + 1) self.assertFalse(self.test_obj.pending_update)
async def process_messages(msg_pending_queue): # noqa: C901; pragma: no cover """ Process messages from Platform-Sources kafka service. Handler for various application/source create and delete events. 'create' events: Issues a Sources REST API call to get additional context for the Platform-Sources kafka event. This information is stored in the Sources database table. 'destroy' events: Enqueues a source delete event which will be processed in the synchronize_sources method. Args: msg_pending_queue (Asyncio queue): Queue to hold kafka messages to be filtered Returns: None """ LOG.info("Waiting to process incoming kafka messages...") while True: msg_data = await msg_pending_queue.get() LOG.info(f"Processing Event: {str(msg_data)}") try: if msg_data.get("event_type") in (KAFKA_APPLICATION_CREATE, KAFKA_AUTHENTICATION_CREATE): if msg_data.get("event_type") == KAFKA_AUTHENTICATION_CREATE: sources_network = SourcesHTTPClient( msg_data.get("auth_header")) msg_data[ "source_id"] = sources_network.get_source_id_from_endpoint_id( msg_data.get("resource_id")) storage.create_source_event(msg_data.get("source_id"), msg_data.get("auth_header"), msg_data.get("offset")) with concurrent.futures.ThreadPoolExecutor() as pool: await EVENT_LOOP.run_in_executor( pool, sources_network_info, msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE, ): with concurrent.futures.ThreadPoolExecutor() as pool: if storage.is_known_source( msg_data.get("source_id")) is False: LOG.info( f"Update event for unknown source id, skipping...") continue await EVENT_LOOP.run_in_executor( pool, sources_network_info, msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_AUTHENTICATION_UPDATE, ): msg_data["source_id"] = storage.get_source_from_endpoint( msg_data.get("resource_id")) with concurrent.futures.ThreadPoolExecutor() as pool: await EVENT_LOOP.run_in_executor( pool, save_auth_info, msg_data.get("auth_header"), msg_data.get("source_id")) elif msg_data.get("event_type") in (KAFKA_APPLICATION_DESTROY, KAFKA_SOURCE_DESTROY): storage.enqueue_source_delete(msg_data.get("source_id")) if msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE, KAFKA_AUTHENTICATION_UPDATE): storage.enqueue_source_update(msg_data.get("source_id")) except (InterfaceError, OperationalError) as error: LOG.error( f"[process_messages] Closing DB connection and re-queueing failed operation." f" Encountered {type(error).__name__}: {error}") connection.close() await asyncio.sleep(Config.RETRY_SECONDS) await msg_pending_queue.put(msg_data) LOG.info( f'Requeued failed operation: {msg_data.get("event_type")} ' f'for Source ID: {str(msg_data.get("source_id"))}.') except Exception as error: # The reason for catching all exceptions is to ensure that the event # loop remains active in the event that message processing fails unexpectedly. source_id = str(msg_data.get("source_id", "unknown")) LOG.error( f"Source {source_id} Unexpected message processing error: {str(error)}", exc_info=True)