def test_create_msg_processor_various_headers_failures(self):
"""Test create_msg_processor various kafka headers."""
header_missing_account_number = "eyJpZGVudGl0eSI6IHsidXNlciI6IHsiaXNfb3JnX2FkbWluIjogImZhbHNlIiwgInVzZXJuYW1lIjogInNvdXJjZXMiLCAiZW1haWwiOiAic291cmNlc0Bzb3VyY2VzLmlvIn0sICJpbnRlcm5hbCI6IHsib3JnX2lkIjogIjU0MzIxIn19fQ==" # noqa: E501
event = KAFKA_APPLICATION_CREATE
account_id = "10002"
table = [
{
"header_list": (
("event_type", bytes(event, encoding="utf-8")),
("x-rh-identity",
bytes(header_missing_account_number, encoding="utf-8")),
)
},
{
"header_list": (
("event_type", bytes(event, encoding="utf-8")),
("x-rh-sources-account-number",
bytes(account_id, encoding="utf-8")),
)
},
{
"header_list": (
("event_type", bytes(event, encoding="utf-8")),
("x-rh-identity",
bytes(header_missing_account_number, encoding="utf-8")),
("x-rh-sources-account-number", bytes()),
)
},
]
for test in table:
with self.subTest(test=test):
msg = msg_generator(event_type=event)
msg._headers = test.get("header_list")
with self.assertRaises(SourcesMessageError):
create_msg_processor(msg, COST_MGMT_APP_TYPE_ID)
def test_create_msg_processor(self):
"""Test create_msg_processor returns the correct processor based on msg event."""
test_mtx = [
{
"event_type": KAFKA_APPLICATION_CREATE,
"expected": ApplicationMsgProcessor
},
{
"event_type": KAFKA_APPLICATION_UPDATE,
"expected": ApplicationMsgProcessor
},
{
"event_type": KAFKA_APPLICATION_DESTROY,
"expected": ApplicationMsgProcessor
},
{
"event_type": KAFKA_APPLICATION_PAUSE,
"expected": ApplicationMsgProcessor
},
{
"event_type": KAFKA_APPLICATION_UNPAUSE,
"expected": ApplicationMsgProcessor
},
{
"event_type": KAFKA_AUTHENTICATION_CREATE,
"expected": AuthenticationMsgProcessor
},
{
"event_type": KAFKA_AUTHENTICATION_UPDATE,
"expected": AuthenticationMsgProcessor
},
{
"event_type": KAFKA_SOURCE_UPDATE,
"expected": SourceMsgProcessor
},
{
"event_type": KAFKA_SOURCE_DESTROY,
"expected": NoneType
},
{
"event_type": "Source.create",
"expected": NoneType
},
{
"event_type": KAFKA_APPLICATION_CREATE,
"test_topic": "unknown",
"expected": NoneType
},
]
for test in test_mtx:
with self.subTest(test=test):
msg = msg_generator(event_type=test.get("event_type"),
topic=test.get("test_topic"))
self.assertIsInstance(
create_msg_processor(msg, COST_MGMT_APP_TYPE_ID),
test.get("expected"))
def test_create_msg_processor_various_headers_success(self):
"""Test create_msg_processor various kafka headers."""
event = KAFKA_APPLICATION_CREATE
account_id = "10002"
table = [
{
"header_list": (
("event_type", bytes(event, encoding="utf-8")),
("x-rh-identity",
bytes(Config.SOURCES_FAKE_HEADER, encoding="utf-8")),
("x-rh-sources-account-number",
bytes(account_id, encoding="utf-8")),
),
"expected": {
"account_number": account_id,
"auth_header": Config.SOURCES_FAKE_HEADER
},
},
{
"header_list": (
("event_type", bytes(event, encoding="utf-8")),
("x-rh-identity",
bytes(Config.SOURCES_FAKE_HEADER, encoding="utf-8")),
),
"expected": {
"account_number": "12345",
"auth_header": Config.SOURCES_FAKE_HEADER
},
},
]
for test in table:
with self.subTest(test=test):
msg = msg_generator(event_type=event)
msg._headers = test.get("header_list")
result = create_msg_processor(msg, COST_MGMT_APP_TYPE_ID)
for k, v in test.get("expected").items():
self.assertEqual(getattr(result, k), v)
def test_msg_for_cost_mgmt(self):
"""Test msg_for_cost_mgmt true or false."""
test_app_value_is_cost = {
"id": 1,
"source_id": 1,
"application_type_id": COST_MGMT_APP_TYPE_ID
}
test_app_value_is_not_cost = {
"id": 1,
"source_id": 1,
"application_type_id": COST_MGMT_APP_TYPE_ID + 1
}
test_auth_value_valid = {
"id": 1,
"source_id": 1,
"authtype": choice(list(AUTH_TYPES.values()))
}
test_auth_value_invalid = {
"id": 1,
"source_id": 1,
"authtype": "access_key_secret_key"
}
table = [
# Source events
{
"processor": KafkaMessageProcessor,
"event-type": "Source.create",
"expected": False
},
{
"processor": KafkaMessageProcessor,
"event-type": KAFKA_SOURCE_UPDATE,
"expected": True
},
{
"processor": KafkaMessageProcessor,
"event-type": KAFKA_SOURCE_DESTROY,
"expected": False
},
# Application events
{
"event-type": KAFKA_APPLICATION_CREATE,
"expected": True,
"value": test_app_value_is_cost
},
{
"event-type": KAFKA_APPLICATION_CREATE,
"expected": False,
"value": test_app_value_is_not_cost
},
{
"event-type": KAFKA_APPLICATION_UPDATE,
"expected": True,
"value": test_app_value_is_cost
},
{
"event-type": KAFKA_APPLICATION_UPDATE,
"expected": False,
"value": test_app_value_is_not_cost
},
{
"event-type": KAFKA_APPLICATION_DESTROY,
"expected": True,
"value": test_app_value_is_cost
},
{
"event-type": KAFKA_APPLICATION_DESTROY,
"expected": False,
"value": test_app_value_is_not_cost
},
{
"event-type": KAFKA_APPLICATION_PAUSE,
"expected": True,
"value": test_app_value_is_cost
},
{
"event-type": KAFKA_APPLICATION_PAUSE,
"expected": False,
"value": test_app_value_is_not_cost
},
{
"event-type": KAFKA_APPLICATION_UNPAUSE,
"expected": True,
"value": test_app_value_is_cost
},
{
"event-type": KAFKA_APPLICATION_UNPAUSE,
"expected": False,
"value": test_app_value_is_not_cost
},
# Authentication events
{
"event-type": KAFKA_AUTHENTICATION_CREATE,
"expected": True,
"patch": True,
"value": test_auth_value_valid,
},
{
"event-type": KAFKA_AUTHENTICATION_CREATE,
"expected": False,
"patch": False,
"value": test_auth_value_valid,
},
{
"event-type": KAFKA_AUTHENTICATION_CREATE,
"expected": False,
"value": test_auth_value_invalid
},
{
"event-type": KAFKA_AUTHENTICATION_UPDATE,
"expected": True,
"patch": True,
"value": test_auth_value_valid,
},
{
"event-type": KAFKA_AUTHENTICATION_UPDATE,
"expected": False,
"patch": False,
"value": test_auth_value_valid,
},
{
"event-type": KAFKA_AUTHENTICATION_UPDATE,
"expected": False,
"value": test_auth_value_invalid
},
]
for test in table:
with self.subTest(test=test):
with patch.object(SourcesHTTPClient,
"get_application_type_is_cost_management",
return_value=test.get("patch")):
msg = msg_generator(event_type=test.get("event-type"),
value=test.get("value"))
if test.get("processor"):
processor = test.get("processor")(
msg, test.get("event-type"), COST_MGMT_APP_TYPE_ID)
else:
processor = create_msg_processor(
msg, COST_MGMT_APP_TYPE_ID)
self.assertEqual(processor.msg_for_cost_mgmt(),
test.get("expected"))
def test_create_msg_processor_missing_header(self):
"""Test create_msg_processor on a message missing kafka headers."""
msg = msg_generator(event_type=KAFKA_APPLICATION_CREATE)
msg._headers = {} # override the generator headers
self.assertIsInstance(create_msg_processor(msg, COST_MGMT_APP_TYPE_ID),
NoneType)
def test_create_msg_processor_missing_auth_and_account_id(self):
"""Test that KafkaMessageProcessor raises SourcesMessageError on missing info."""
event = KAFKA_APPLICATION_CREATE
msg = msg_generator(event_type=event, header=None)
with self.assertRaises(SourcesMessageError):
create_msg_processor(msg, COST_MGMT_APP_TYPE_ID)
def listen_for_messages(kaf_msg, consumer,
application_source_id): # noqa: C901
"""
Listen for Platform-Sources kafka messages.
Args:
consumer (Consumer): Kafka consumer object
application_source_id (Integer): Cost Management's current Application Source ID. Used for
kafka message filtering.
Returns:
None
"""
try:
try:
msg_processor = create_msg_processor(kaf_msg,
application_source_id)
if msg_processor and msg_processor.source_id and msg_processor.auth_header:
tp = TopicPartition(Config.SOURCES_TOPIC,
msg_processor.partition,
msg_processor.offset)
if not msg_processor.msg_for_cost_mgmt():
LOG.info("Event not associated with cost-management.")
consumer.commit()
return
LOG.info(f"processing cost-mgmt message: {msg_processor}")
with transaction.atomic():
msg_processor.process()
except (InterfaceError, OperationalError) as error:
close_and_set_db_connection()
LOG.error(
f"[listen_for_messages] Database error. Error: {type(error).__name__}: {error}. Retrying..."
)
rewind_consumer_to_retry(consumer, tp)
except IntegrityError as error:
LOG.error(
f"[listen_for_messages] {type(error).__name__}: {error}. Retrying...",
exc_info=True)
rewind_consumer_to_retry(consumer, tp)
except SourcesHTTPClientError as err:
LOG.warning(
f"[listen_for_messages] {type(err).__name__}: {err}. Retrying..."
)
SOURCES_HTTP_CLIENT_ERROR_COUNTER.inc()
rewind_consumer_to_retry(consumer, tp)
except (SourcesMessageError, SourceNotFoundError) as error:
LOG.warning(
f"[listen_for_messages] {type(error).__name__}: {error}. Skipping msg: {kaf_msg.value()}"
)
consumer.commit()
else:
consumer.commit()
except KafkaError as error:
LOG.error(
f"[listen_for_messages] Kafka error encountered: {type(error).__name__}: {error}",
exc_info=True)
except Exception as error:
LOG.error(
f"[listen_for_messages] UNKNOWN error encountered: {type(error).__name__}: {error}",
exc_info=True)