async def test_kafka_configuration(self):
kafka_config = salkafka.KafkaConfiguration(**self.config_kwargs)
for name, value in self.config_kwargs.items():
self.assertEqual(getattr(kafka_config, name), value)
for name, bad_value in (
("wait_for_ack", 2),
("wait_for_ack", -1),
("wait_for_ack", None),
("wait_for_ack", "some"),
):
bad_config_kwargs = self.config_kwargs.copy()
bad_config_kwargs[name] = bad_value
with self.assertRaises(ValueError):
salkafka.KafkaConfiguration(**bad_config_kwargs)
async def test_make_producer(self):
topic_name = "lsst.sal.Test.foo"
config = salkafka.KafkaConfiguration(**self.config_kwargs)
log = logging.getLogger()
async with salkafka.KafkaProducerFactory(
config=config,
log=log,
) as kafka_factory:
avro_schema = {
"name": topic_name,
"type": "record",
"fields": [{
"name": "TestID",
"type": "long"
}],
}
producer = await kafka_factory.make_producer(
avro_schema=avro_schema)
self.assertEqual(producer.bootstrap_servers, config.broker_url)
self.assertEqual(producer.sent_data, [])
expected_name_value_list = []
for i in range(10):
value = {"TestID": i}
expected_name_value_list.append((topic_name, value))
await producer.send_and_wait(topic_name, value=value)
sent_name_value_list = [item[0:2] for item in producer.sent_data]
self.assertEqual(sent_name_value_list, expected_name_value_list)
for item in producer.sent_data:
self.assertIsInstance(item[2], bytes)
invalid_value = {"unexpected_key": "some_value"}
with self.assertRaises(Exception):
# the actual exception is not documented
await producer.send_and_wait(topic_name, value=invalid_value)
async def make_component_producer(self):
"""Make a CSC and component producer for the Test SAL component.
Attributes
----------
csc : `lsst.ts.salobj.TestCsc`
A Test CSC you can use to produce data.
component_producer : `ComponentProducer`
The component producer.
"""
# Use an arbitrary non-zero index for the producer to test that
# the topic producer can see it.
self.csc = salobj.TestCsc(index=5)
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)
broker_url = "test.kafka:9000"
registry_url = "https://registry.test.kafka/"
partitions = 2
replication_factor = 3
wait_for_ack = 1
kafka_config = salkafka.KafkaConfiguration(
broker_url=broker_url,
registry_url=registry_url,
partitions=partitions,
replication_factor=replication_factor,
wait_for_ack=wait_for_ack,
)
kafka_factory = salkafka.KafkaProducerFactory(
config=kafka_config,
log=log,
)
self.component_producer = salkafka.ComponentProducer(
domain=self.csc.domain,
component="Test",
kafka_factory=kafka_factory)
await asyncio.gather(
self.component_producer.start_task,
kafka_factory.start_task,
self.csc.start_task,
)
try:
yield
finally:
await self.csc.close()
await kafka_factory.close()
await self.component_producer.close()
async def test_make_kafka_topics(self):
config = salkafka.KafkaConfiguration(**self.config_kwargs)
log = logging.getLogger()
async with salkafka.KafkaProducerFactory(config=config,
log=log) as kafka_factory:
existing_topic_names = ["old_topic", "another_old_topic"]
kafka_factory.broker_client.set_existing_topic_names(
existing_topic_names)
new_topic_names = ["new_topic", "another_new_topic"]
all_topic_names = existing_topic_names + new_topic_names
created_topic_names = kafka_factory.make_kafka_topics(
all_topic_names)
self.assertEqual(set(new_topic_names), set(created_topic_names))
def setUp(self):
salobj.set_random_lsst_dds_partition_prefix()
self.data_dir = pathlib.Path(
__file__).parent / "data" / "topic_names_sets"
broker_url = "test.kafka:9000"
registry_url = "https://registry.test.kafka/"
partitions = 2
replication_factor = 3
wait_for_ack = 1
self.kafka_config = salkafka.KafkaConfiguration(
broker_url=broker_url,
registry_url=registry_url,
partitions=partitions,
replication_factor=replication_factor,
wait_for_ack=wait_for_ack,
)
async def make_producer(self, topic_name, sal_prefix):
"""Make a CSC and topic producer for a specified topic of the
Test SAL component.
Parameters
----------
topic_name : `str`
Topic name, without any prefix.
For example specify summaryState for logevent_summaryState.
sal_prefix : `str`
SAL prefix for the topic.
For example specify logevent_ for logevent_summaryState.
Attributes
----------
csc : `lsst.ts.salobj.TestCsc`
A Test CSC you can use to produce data.
topic_producer : `TopicProducer`
The topic producer.
"""
# Use an arbitrary non-zero index for the producer to test that
# the topic producer can see it.
self.csc = salobj.TestCsc(index=5)
# Always use an index of 0 for the TopicProducer's read topic
# (which we get from this SalInfo).
read_salinfo = salobj.SalInfo(domain=self.csc.domain,
name="Test",
index=0)
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)
kafka_config = salkafka.KafkaConfiguration(
broker_url="test.kafka:9000",
registry_url="https://registry.test.kafka/",
partitions=2,
replication_factor=3,
wait_for_ack=1,
)
kafka_factory = salkafka.KafkaProducerFactory(
config=kafka_config,
log=log,
)
# We are not sure we can construct the topic producer
# because the topic data may be invalid),
# so initialize it to None for proper cleanup.
self.topic_producer = None
try:
read_topic = salobj.topics.ReadTopic(
salinfo=read_salinfo,
name=topic_name,
sal_prefix=sal_prefix,
max_history=0,
filter_ackcmd=False,
)
self.topic_producer = salkafka.TopicProducer(
kafka_factory=kafka_factory, topic=read_topic, log=log)
await asyncio.gather(
self.topic_producer.start_task,
kafka_factory.start_task,
read_salinfo.start(),
self.csc.start_task,
)
# Run the unit test
yield
finally:
if self.topic_producer is not None:
await self.topic_producer.close()
await kafka_factory.close()
await read_salinfo.close()
await self.csc.close()