async def test_kafka_configuration(self): kafka_config = salkafka.KafkaConfiguration(**self.config_kwargs) for name, value in self.config_kwargs.items(): self.assertEqual(getattr(kafka_config, name), value) for name, bad_value in ( ("wait_for_ack", 2), ("wait_for_ack", -1), ("wait_for_ack", None), ("wait_for_ack", "some"), ): bad_config_kwargs = self.config_kwargs.copy() bad_config_kwargs[name] = bad_value with self.assertRaises(ValueError): salkafka.KafkaConfiguration(**bad_config_kwargs)
async def test_make_producer(self): topic_name = "lsst.sal.Test.foo" config = salkafka.KafkaConfiguration(**self.config_kwargs) log = logging.getLogger() async with salkafka.KafkaProducerFactory( config=config, log=log, ) as kafka_factory: avro_schema = { "name": topic_name, "type": "record", "fields": [{ "name": "TestID", "type": "long" }], } producer = await kafka_factory.make_producer( avro_schema=avro_schema) self.assertEqual(producer.bootstrap_servers, config.broker_url) self.assertEqual(producer.sent_data, []) expected_name_value_list = [] for i in range(10): value = {"TestID": i} expected_name_value_list.append((topic_name, value)) await producer.send_and_wait(topic_name, value=value) sent_name_value_list = [item[0:2] for item in producer.sent_data] self.assertEqual(sent_name_value_list, expected_name_value_list) for item in producer.sent_data: self.assertIsInstance(item[2], bytes) invalid_value = {"unexpected_key": "some_value"} with self.assertRaises(Exception): # the actual exception is not documented await producer.send_and_wait(topic_name, value=invalid_value)
async def make_component_producer(self): """Make a CSC and component producer for the Test SAL component. Attributes ---------- csc : `lsst.ts.salobj.TestCsc` A Test CSC you can use to produce data. component_producer : `ComponentProducer` The component producer. """ # Use an arbitrary non-zero index for the producer to test that # the topic producer can see it. self.csc = salobj.TestCsc(index=5) log = logging.getLogger() log.addHandler(logging.StreamHandler()) log.setLevel(logging.INFO) broker_url = "test.kafka:9000" registry_url = "https://registry.test.kafka/" partitions = 2 replication_factor = 3 wait_for_ack = 1 kafka_config = salkafka.KafkaConfiguration( broker_url=broker_url, registry_url=registry_url, partitions=partitions, replication_factor=replication_factor, wait_for_ack=wait_for_ack, ) kafka_factory = salkafka.KafkaProducerFactory( config=kafka_config, log=log, ) self.component_producer = salkafka.ComponentProducer( domain=self.csc.domain, component="Test", kafka_factory=kafka_factory) await asyncio.gather( self.component_producer.start_task, kafka_factory.start_task, self.csc.start_task, ) try: yield finally: await self.csc.close() await kafka_factory.close() await self.component_producer.close()
async def test_make_kafka_topics(self): config = salkafka.KafkaConfiguration(**self.config_kwargs) log = logging.getLogger() async with salkafka.KafkaProducerFactory(config=config, log=log) as kafka_factory: existing_topic_names = ["old_topic", "another_old_topic"] kafka_factory.broker_client.set_existing_topic_names( existing_topic_names) new_topic_names = ["new_topic", "another_new_topic"] all_topic_names = existing_topic_names + new_topic_names created_topic_names = kafka_factory.make_kafka_topics( all_topic_names) self.assertEqual(set(new_topic_names), set(created_topic_names))
def setUp(self): salobj.set_random_lsst_dds_partition_prefix() self.data_dir = pathlib.Path( __file__).parent / "data" / "topic_names_sets" broker_url = "test.kafka:9000" registry_url = "https://registry.test.kafka/" partitions = 2 replication_factor = 3 wait_for_ack = 1 self.kafka_config = salkafka.KafkaConfiguration( broker_url=broker_url, registry_url=registry_url, partitions=partitions, replication_factor=replication_factor, wait_for_ack=wait_for_ack, )
async def make_producer(self, topic_name, sal_prefix): """Make a CSC and topic producer for a specified topic of the Test SAL component. Parameters ---------- topic_name : `str` Topic name, without any prefix. For example specify summaryState for logevent_summaryState. sal_prefix : `str` SAL prefix for the topic. For example specify logevent_ for logevent_summaryState. Attributes ---------- csc : `lsst.ts.salobj.TestCsc` A Test CSC you can use to produce data. topic_producer : `TopicProducer` The topic producer. """ # Use an arbitrary non-zero index for the producer to test that # the topic producer can see it. self.csc = salobj.TestCsc(index=5) # Always use an index of 0 for the TopicProducer's read topic # (which we get from this SalInfo). read_salinfo = salobj.SalInfo(domain=self.csc.domain, name="Test", index=0) log = logging.getLogger() log.addHandler(logging.StreamHandler()) log.setLevel(logging.INFO) kafka_config = salkafka.KafkaConfiguration( broker_url="test.kafka:9000", registry_url="https://registry.test.kafka/", partitions=2, replication_factor=3, wait_for_ack=1, ) kafka_factory = salkafka.KafkaProducerFactory( config=kafka_config, log=log, ) # We are not sure we can construct the topic producer # because the topic data may be invalid), # so initialize it to None for proper cleanup. self.topic_producer = None try: read_topic = salobj.topics.ReadTopic( salinfo=read_salinfo, name=topic_name, sal_prefix=sal_prefix, max_history=0, filter_ackcmd=False, ) self.topic_producer = salkafka.TopicProducer( kafka_factory=kafka_factory, topic=read_topic, log=log) await asyncio.gather( self.topic_producer.start_task, kafka_factory.start_task, read_salinfo.start(), self.csc.start_task, ) # Run the unit test yield finally: if self.topic_producer is not None: await self.topic_producer.close() await kafka_factory.close() await read_salinfo.close() await self.csc.close()