def test_create_msg_processor_various_headers_failures(self): """Test create_msg_processor various kafka headers.""" header_missing_account_number = "eyJpZGVudGl0eSI6IHsidXNlciI6IHsiaXNfb3JnX2FkbWluIjogImZhbHNlIiwgInVzZXJuYW1lIjogInNvdXJjZXMiLCAiZW1haWwiOiAic291cmNlc0Bzb3VyY2VzLmlvIn0sICJpbnRlcm5hbCI6IHsib3JnX2lkIjogIjU0MzIxIn19fQ==" # noqa: E501 event = KAFKA_APPLICATION_CREATE account_id = "10002" table = [ { "header_list": ( ("event_type", bytes(event, encoding="utf-8")), ("x-rh-identity", bytes(header_missing_account_number, encoding="utf-8")), ) }, { "header_list": ( ("event_type", bytes(event, encoding="utf-8")), ("x-rh-sources-account-number", bytes(account_id, encoding="utf-8")), ) }, { "header_list": ( ("event_type", bytes(event, encoding="utf-8")), ("x-rh-identity", bytes(header_missing_account_number, encoding="utf-8")), ("x-rh-sources-account-number", bytes()), ) }, ] for test in table: with self.subTest(test=test): msg = msg_generator(event_type=event) msg._headers = test.get("header_list") with self.assertRaises(SourcesMessageError): create_msg_processor(msg, COST_MGMT_APP_TYPE_ID)
def test_create_msg_processor(self): """Test create_msg_processor returns the correct processor based on msg event.""" test_mtx = [ { "event_type": KAFKA_APPLICATION_CREATE, "expected": ApplicationMsgProcessor }, { "event_type": KAFKA_APPLICATION_UPDATE, "expected": ApplicationMsgProcessor }, { "event_type": KAFKA_APPLICATION_DESTROY, "expected": ApplicationMsgProcessor }, { "event_type": KAFKA_APPLICATION_PAUSE, "expected": ApplicationMsgProcessor }, { "event_type": KAFKA_APPLICATION_UNPAUSE, "expected": ApplicationMsgProcessor }, { "event_type": KAFKA_AUTHENTICATION_CREATE, "expected": AuthenticationMsgProcessor }, { "event_type": KAFKA_AUTHENTICATION_UPDATE, "expected": AuthenticationMsgProcessor }, { "event_type": KAFKA_SOURCE_UPDATE, "expected": SourceMsgProcessor }, { "event_type": KAFKA_SOURCE_DESTROY, "expected": NoneType }, { "event_type": "Source.create", "expected": NoneType }, { "event_type": KAFKA_APPLICATION_CREATE, "test_topic": "unknown", "expected": NoneType }, ] for test in test_mtx: with self.subTest(test=test): msg = msg_generator(event_type=test.get("event_type"), topic=test.get("test_topic")) self.assertIsInstance( create_msg_processor(msg, COST_MGMT_APP_TYPE_ID), test.get("expected"))
def test_create_msg_processor_various_headers_success(self): """Test create_msg_processor various kafka headers.""" event = KAFKA_APPLICATION_CREATE account_id = "10002" table = [ { "header_list": ( ("event_type", bytes(event, encoding="utf-8")), ("x-rh-identity", bytes(Config.SOURCES_FAKE_HEADER, encoding="utf-8")), ("x-rh-sources-account-number", bytes(account_id, encoding="utf-8")), ), "expected": { "account_number": account_id, "auth_header": Config.SOURCES_FAKE_HEADER }, }, { "header_list": ( ("event_type", bytes(event, encoding="utf-8")), ("x-rh-identity", bytes(Config.SOURCES_FAKE_HEADER, encoding="utf-8")), ), "expected": { "account_number": "12345", "auth_header": Config.SOURCES_FAKE_HEADER }, }, ] for test in table: with self.subTest(test=test): msg = msg_generator(event_type=event) msg._headers = test.get("header_list") result = create_msg_processor(msg, COST_MGMT_APP_TYPE_ID) for k, v in test.get("expected").items(): self.assertEqual(getattr(result, k), v)
def test_msg_for_cost_mgmt(self): """Test msg_for_cost_mgmt true or false.""" test_app_value_is_cost = { "id": 1, "source_id": 1, "application_type_id": COST_MGMT_APP_TYPE_ID } test_app_value_is_not_cost = { "id": 1, "source_id": 1, "application_type_id": COST_MGMT_APP_TYPE_ID + 1 } test_auth_value_valid = { "id": 1, "source_id": 1, "authtype": choice(list(AUTH_TYPES.values())) } test_auth_value_invalid = { "id": 1, "source_id": 1, "authtype": "access_key_secret_key" } table = [ # Source events { "processor": KafkaMessageProcessor, "event-type": "Source.create", "expected": False }, { "processor": KafkaMessageProcessor, "event-type": KAFKA_SOURCE_UPDATE, "expected": True }, { "processor": KafkaMessageProcessor, "event-type": KAFKA_SOURCE_DESTROY, "expected": False }, # Application events { "event-type": KAFKA_APPLICATION_CREATE, "expected": True, "value": test_app_value_is_cost }, { "event-type": KAFKA_APPLICATION_CREATE, "expected": False, "value": test_app_value_is_not_cost }, { "event-type": KAFKA_APPLICATION_UPDATE, "expected": True, "value": test_app_value_is_cost }, { "event-type": KAFKA_APPLICATION_UPDATE, "expected": False, "value": test_app_value_is_not_cost }, { "event-type": KAFKA_APPLICATION_DESTROY, "expected": True, "value": test_app_value_is_cost }, { "event-type": KAFKA_APPLICATION_DESTROY, "expected": False, "value": test_app_value_is_not_cost }, { "event-type": KAFKA_APPLICATION_PAUSE, "expected": True, "value": test_app_value_is_cost }, { "event-type": KAFKA_APPLICATION_PAUSE, "expected": False, "value": test_app_value_is_not_cost }, { "event-type": KAFKA_APPLICATION_UNPAUSE, "expected": True, "value": test_app_value_is_cost }, { "event-type": KAFKA_APPLICATION_UNPAUSE, "expected": False, "value": test_app_value_is_not_cost }, # Authentication events { "event-type": KAFKA_AUTHENTICATION_CREATE, "expected": True, "patch": True, "value": test_auth_value_valid, }, { "event-type": KAFKA_AUTHENTICATION_CREATE, "expected": False, "patch": False, "value": test_auth_value_valid, }, { "event-type": KAFKA_AUTHENTICATION_CREATE, "expected": False, "value": test_auth_value_invalid }, { "event-type": KAFKA_AUTHENTICATION_UPDATE, "expected": True, "patch": True, "value": test_auth_value_valid, }, { "event-type": KAFKA_AUTHENTICATION_UPDATE, "expected": False, "patch": False, "value": test_auth_value_valid, }, { "event-type": KAFKA_AUTHENTICATION_UPDATE, "expected": False, "value": test_auth_value_invalid }, ] for test in table: with self.subTest(test=test): with patch.object(SourcesHTTPClient, "get_application_type_is_cost_management", return_value=test.get("patch")): msg = msg_generator(event_type=test.get("event-type"), value=test.get("value")) if test.get("processor"): processor = test.get("processor")( msg, test.get("event-type"), COST_MGMT_APP_TYPE_ID) else: processor = create_msg_processor( msg, COST_MGMT_APP_TYPE_ID) self.assertEqual(processor.msg_for_cost_mgmt(), test.get("expected"))
def test_create_msg_processor_missing_header(self): """Test create_msg_processor on a message missing kafka headers.""" msg = msg_generator(event_type=KAFKA_APPLICATION_CREATE) msg._headers = {} # override the generator headers self.assertIsInstance(create_msg_processor(msg, COST_MGMT_APP_TYPE_ID), NoneType)
def test_create_msg_processor_missing_auth_and_account_id(self): """Test that KafkaMessageProcessor raises SourcesMessageError on missing info.""" event = KAFKA_APPLICATION_CREATE msg = msg_generator(event_type=event, header=None) with self.assertRaises(SourcesMessageError): create_msg_processor(msg, COST_MGMT_APP_TYPE_ID)
def listen_for_messages(kaf_msg, consumer, application_source_id): # noqa: C901 """ Listen for Platform-Sources kafka messages. Args: consumer (Consumer): Kafka consumer object application_source_id (Integer): Cost Management's current Application Source ID. Used for kafka message filtering. Returns: None """ try: try: msg_processor = create_msg_processor(kaf_msg, application_source_id) if msg_processor and msg_processor.source_id and msg_processor.auth_header: tp = TopicPartition(Config.SOURCES_TOPIC, msg_processor.partition, msg_processor.offset) if not msg_processor.msg_for_cost_mgmt(): LOG.info("Event not associated with cost-management.") consumer.commit() return LOG.info(f"processing cost-mgmt message: {msg_processor}") with transaction.atomic(): msg_processor.process() except (InterfaceError, OperationalError) as error: close_and_set_db_connection() LOG.error( f"[listen_for_messages] Database error. Error: {type(error).__name__}: {error}. Retrying..." ) rewind_consumer_to_retry(consumer, tp) except IntegrityError as error: LOG.error( f"[listen_for_messages] {type(error).__name__}: {error}. Retrying...", exc_info=True) rewind_consumer_to_retry(consumer, tp) except SourcesHTTPClientError as err: LOG.warning( f"[listen_for_messages] {type(err).__name__}: {err}. Retrying..." ) SOURCES_HTTP_CLIENT_ERROR_COUNTER.inc() rewind_consumer_to_retry(consumer, tp) except (SourcesMessageError, SourceNotFoundError) as error: LOG.warning( f"[listen_for_messages] {type(error).__name__}: {error}. Skipping msg: {kaf_msg.value()}" ) consumer.commit() else: consumer.commit() except KafkaError as error: LOG.error( f"[listen_for_messages] Kafka error encountered: {type(error).__name__}: {error}", exc_info=True) except Exception as error: LOG.error( f"[listen_for_messages] UNKNOWN error encountered: {type(error).__name__}: {error}", exc_info=True)