def test_create_source_event_invalid_auth_header(self): """Tests creating a source db record with invalid auth_header.""" test_source_id = 2 test_offset = 3 storage.create_source_event(test_source_id, "bad", test_offset) with self.assertRaises(Sources.DoesNotExist): Sources.objects.get(source_id=test_source_id)
def process(self): """Process the message.""" if self.event_type in (KAFKA_APPLICATION_CREATE,): storage.create_source_event(self.source_id, self.auth_header, self.offset) if storage.is_known_source(self.source_id): if self.event_type in (KAFKA_APPLICATION_CREATE,): self.save_sources_details() self.save_source_info(bill=True) # _Authentication_ messages are responsible for saving credentials. # However, OCP does not send an Auth message. Therefore, we need # to run the following branch for OCP which completes the source # creation cycle for an OCP source. if storage.get_source_type(self.source_id) == Provider.PROVIDER_OCP: self.save_source_info(auth=True) if self.event_type in (KAFKA_APPLICATION_UPDATE,): if storage.get_source_type(self.source_id) == Provider.PROVIDER_AZURE: # Because azure auth is split in Sources backend, we need to check both # auth and billing when we recieve either auth update or app update event updated = self.save_source_info(auth=True, bill=True) else: updated = self.save_source_info(bill=True) if updated: LOG.info(f"[ApplicationMsgProcessor] source_id {self.source_id} updated") storage.enqueue_source_create_or_update(self.source_id) else: LOG.info(f"[ApplicationMsgProcessor] source_id {self.source_id} not updated. No changes detected.") if self.event_type in (KAFKA_APPLICATION_DESTROY,): storage.enqueue_source_delete(self.source_id, self.offset, allow_out_of_order=True)
def process(self): """Process the message.""" if self.event_type in (KAFKA_AUTHENTICATION_CREATE): LOG.debug( f"[AuthenticationMsgProcessor] creating source for source_id: {self.source_id}" ) storage.create_source_event(self.source_id, self.account_number, self.auth_header, self.offset) if storage.is_known_source(self.source_id): if self.event_type in (KAFKA_AUTHENTICATION_CREATE): self.save_source_info(auth=True) if self.event_type in (KAFKA_AUTHENTICATION_UPDATE): if storage.get_source_type( self.source_id) == Provider.PROVIDER_AZURE: # Because azure auth is split in Sources backend, we need to check both # auth and billing when we recieve either auth update or app update event updated = self.save_source_info(auth=True, bill=True) else: updated = self.save_source_info(auth=True) if updated: LOG.info( f"[AuthenticationMsgProcessor] source_id {self.source_id} updated" ) storage.enqueue_source_create_or_update(self.source_id) else: LOG.info( f"[AuthenticationMsgProcessor] source_id {self.source_id} not updated. No changes detected." )
def process_message(app_type_id, msg): # noqa: C901 """ Process message from Platform-Sources kafka service. Handler for various application/source create and delete events. 'create' events: Issues a Sources REST API call to get additional context for the Platform-Sources kafka event. This information is stored in the Sources database table. 'destroy' events: Enqueues a source delete event which will be processed in the synchronize_sources method. Args: app_type_id - application type identifier msg - kafka message Returns: None """ LOG.info(f"Processing Event: {msg}") msg_data = None try: msg_data = cost_mgmt_msg_filter(msg) except SourceNotFoundError: LOG.warning(f"Source not found in platform sources. Skipping msg: {msg}") return if not msg_data: LOG.debug(f"Message not intended for cost management: {msg}") return if msg_data.get("event_type") in (KAFKA_APPLICATION_CREATE,): storage.create_source_event(msg_data.get("source_id"), msg_data.get("auth_header"), msg_data.get("offset")) if storage.is_known_source(msg_data.get("source_id")): sources_network_info(msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_AUTHENTICATION_CREATE, KAFKA_AUTHENTICATION_UPDATE): if msg_data.get("event_type") in (KAFKA_AUTHENTICATION_CREATE,): storage.create_source_event( # this will create source _only_ if it does not exist. msg_data.get("source_id"), msg_data.get("auth_header"), msg_data.get("offset") ) save_auth_info(msg_data.get("auth_header"), msg_data.get("source_id")) elif msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE,): if storage.is_known_source(msg_data.get("source_id")) is False: LOG.info("Update event for unknown source id, skipping...") return sources_network_info(msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_APPLICATION_DESTROY,): storage.enqueue_source_delete(msg_data.get("source_id"), msg_data.get("offset"), allow_out_of_order=True) elif msg_data.get("event_type") in (KAFKA_SOURCE_DESTROY,): storage.enqueue_source_delete(msg_data.get("source_id"), msg_data.get("offset")) if msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE, KAFKA_AUTHENTICATION_UPDATE): storage.enqueue_source_update(msg_data.get("source_id"))
def test_create_source_event_out_of_order_unexpected_entry(self): """Tests that a source entry is not cleaned up when unexpected entry is found.""" test_source_id = 3 test_offset = 4 test_obj = Sources(source_id=test_source_id, offset=3) test_obj.save() storage.create_source_event(test_source_id, Config.SOURCES_FAKE_HEADER, test_offset) self.assertTrue(Sources.objects.filter(source_id=test_source_id).exists())
def test_create_source_event_out_of_order(self): """Tests that a source entry is cleaned up when following an out of order destroy.""" test_source_id = 3 test_offset = 4 test_obj = Sources(source_id=test_source_id, offset=3, out_of_order_delete=True) test_obj.save() storage.create_source_event(test_source_id, Config.SOURCES_FAKE_HEADER, test_offset) self.assertFalse(Sources.objects.filter(source_id=test_source_id).exists())
def test_create_source_event_db_down(self): """Tests creating a source db record with invalid auth_header.""" test_source_id = 2 test_offset = 3 with patch("sources.storage.Sources.objects") as mock_objects: mock_objects.get.side_effect = InterfaceError("Test exception") with self.assertRaises(InterfaceError): storage.create_source_event(test_source_id, Config.SOURCES_FAKE_HEADER, test_offset)
def test_create_source_event(self): """Tests that a source can be created.""" test_source_id = 2 test_offset = 3 storage.create_source_event(test_source_id, Config.SOURCES_FAKE_HEADER, test_offset) db_obj = Sources.objects.get(source_id=test_source_id) self.assertEqual(db_obj.source_id, test_source_id) self.assertEqual(db_obj.auth_header, Config.SOURCES_FAKE_HEADER) self.assertEqual(db_obj.offset, test_offset) self.assertEqual(db_obj.account_id, self.account_id)
def test_create_source_event_db_down(self): """Tests creating a source db record with invalid auth_header.""" test_source_id = 2 test_offset = 3 ocp_obj = Sources(source_id=test_source_id, offset=3, out_of_order_delete=True, pending_delete=False) ocp_obj.save() with patch.object(Sources, "delete") as mock_object: mock_object.side_effect = InterfaceError("Error") with self.assertRaises(InterfaceError): storage.create_source_event(test_source_id, Config.SOURCES_FAKE_HEADER, test_offset)
async def process_messages(msg_pending_queue): # noqa: C901; pragma: no cover """ Process messages from Platform-Sources kafka service. Handler for various application/source create and delete events. 'create' events: Issues a Sources REST API call to get additional context for the Platform-Sources kafka event. This information is stored in the Sources database table. 'destroy' events: Enqueues a source delete event which will be processed in the synchronize_sources method. Args: msg_pending_queue (Asyncio queue): Queue to hold kafka messages to be filtered Returns: None """ LOG.info("Waiting to process incoming kafka messages...") while True: msg_data = await msg_pending_queue.get() LOG.info(f"Processing Event: {str(msg_data)}") try: if msg_data.get("event_type") in (KAFKA_APPLICATION_CREATE, KAFKA_AUTHENTICATION_CREATE): if msg_data.get("event_type") == KAFKA_AUTHENTICATION_CREATE: sources_network = SourcesHTTPClient( msg_data.get("auth_header")) msg_data[ "source_id"] = sources_network.get_source_id_from_endpoint_id( msg_data.get("resource_id")) storage.create_source_event(msg_data.get("source_id"), msg_data.get("auth_header"), msg_data.get("offset")) with concurrent.futures.ThreadPoolExecutor() as pool: await EVENT_LOOP.run_in_executor( pool, sources_network_info, msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE, ): with concurrent.futures.ThreadPoolExecutor() as pool: if storage.is_known_source( msg_data.get("source_id")) is False: LOG.info( f"Update event for unknown source id, skipping...") continue await EVENT_LOOP.run_in_executor( pool, sources_network_info, msg_data.get("source_id"), msg_data.get("auth_header")) elif msg_data.get("event_type") in (KAFKA_AUTHENTICATION_UPDATE, ): msg_data["source_id"] = storage.get_source_from_endpoint( msg_data.get("resource_id")) with concurrent.futures.ThreadPoolExecutor() as pool: await EVENT_LOOP.run_in_executor( pool, save_auth_info, msg_data.get("auth_header"), msg_data.get("source_id")) elif msg_data.get("event_type") in (KAFKA_APPLICATION_DESTROY, KAFKA_SOURCE_DESTROY): storage.enqueue_source_delete(msg_data.get("source_id")) if msg_data.get("event_type") in (KAFKA_SOURCE_UPDATE, KAFKA_AUTHENTICATION_UPDATE): storage.enqueue_source_update(msg_data.get("source_id")) except (InterfaceError, OperationalError) as error: LOG.error( f"[process_messages] Closing DB connection and re-queueing failed operation." f" Encountered {type(error).__name__}: {error}") connection.close() await asyncio.sleep(Config.RETRY_SECONDS) await msg_pending_queue.put(msg_data) LOG.info( f'Requeued failed operation: {msg_data.get("event_type")} ' f'for Source ID: {str(msg_data.get("source_id"))}.') except Exception as error: # The reason for catching all exceptions is to ensure that the event # loop remains active in the event that message processing fails unexpectedly. source_id = str(msg_data.get("source_id", "unknown")) LOG.error( f"Source {source_id} Unexpected message processing error: {str(error)}", exc_info=True)