def ping(state: State, times: int, wait: int): """Test the connection to the kafka cluster. Ping the kafka cluster by writing messages to and reading messages from it. After the specified number of "pings", return the minimum, maximum, and average time for the round trip. """ topic_controller = state.cluster.topic_controller deltas = [] try: try: topic_controller.create_topics([Topic(PING_TOPIC)]) except TopicAlreadyExistsException: pass producer = PingProducer(PING_TOPIC) consumer = ConsumerFactory().create_ping_consumer(group_id=PING_GROUP_ID, topic_name=PING_TOPIC) click.echo(f"Pinging with {state.cluster.bootstrap_servers}.") for i in range(times): producer.produce() _, delta = consumer.consume() deltas.append(delta) click.echo(f"m_seq={i} time={delta:.2f}ms") sleep(wait) except KeyboardInterrupt: return topic_controller.delete_topic(Topic(PING_TOPIC)) click.echo("--- statistics ---") click.echo(f"{len(deltas)} messages sent/received.") click.echo(f"min/avg/max = {min(deltas):.2f}/{(sum(deltas) / len(deltas)):.2f}/{max(deltas):.2f} ms")
def edit_consumer_group_offsets( self, consumer_id: str, offset_plan: List[ConsumerGroupOffsetPlan]): """ Commit consumergroup offsets to specific values :param consumer_id: ID of the consumer group :param offset_plan: List of ConsumerGroupOffsetPlan objects denoting the offsets for each partition in different topics :return: """ consumer = ConsumerFactory().create_consumer( group_id=consumer_id, topic_name=None, output_directory=None, last=False, avro=False, initialize_default_output_directory=False, match=None, enable_auto_commit=False, ) offsets = [ TopicPartition(topic=plan_element.topic_name, partition=plan_element.partition_id, offset=plan_element.proposed_offset) for plan_element in offset_plan if not plan_element.offset_equal ] consumer.commit(offsets=offsets)
def get_offsets_closest_to_timestamp( self, topic_name: str, timestamp_limit: pendulum) -> Dict[int, int]: topic = self.get_cluster_topic(topic_name=topic_name) partition_offsets = { partition.partition_id: 0 for partition in topic.partitions } consumers = [] factory = ConsumerFactory() partitions = partition_offsets.keys() group_id = "group_for_" + topic_name + "_" + str( int(round(time.time() * 1000))) for partition in partitions: consumer = factory.create_consumer( group_id=group_id, topic_name=None, output_directory=None, avro=False, match=None, last=False, initialize_default_output_directory=False, ) consumer.assign_specific_partitions(topic_name, [partition]) consumers.append(consumer) for partition_counter in range(0, len(consumers)): max_retry_count = 5 keep_polling_current_partition = True while keep_polling_current_partition: try: message = consumers[ partition_counter].consume_single_message(timeout=10) except MessageEmptyException: # a possible timeout due to a network issue, retry (but not more than max_retry_count attempts) max_retry_count -= 1 if max_retry_count <= 0: keep_polling_current_partition = False except EndOfPartitionReachedException: keep_polling_current_partition = False else: if (datetime.datetime.fromtimestamp( int(message.timestamp()[1] / 1000.0), timezone.utc) < timestamp_limit): partition_offsets[ message.partition()] = message.offset() + 1 return partition_offsets
def test_avro_consume_to_file(consumer_group, avro_producer: AvroProducer, source_topic: Tuple[str, int], tmpdir_factory): source_topic_id, _ = source_topic output_directory = tmpdir_factory.mktemp("output_directory") produced_messages = produce_test_messages_with_avro( avro_producer, source_topic) file_consumer = ConsumerFactory().create_consumer(consumer_group, source_topic_id, output_directory, False, avro=True) number_of_consumer_messages = file_consumer.consume(10) consumed_messages = get_consumed_messages(output_directory, True) assert number_of_consumer_messages == 10 assert produced_messages == consumed_messages
def test_plain_text_consume_to_file(consumer_group, producer: ConfluenceProducer, source_topic: Tuple[str, int], tmpdir_factory): source_topic_id, _ = source_topic output_directory = tmpdir_factory.mktemp("output_directory") produced_messages = produce_test_messages(producer, source_topic) file_consumer = ConsumerFactory().create_consumer(consumer_group, source_topic_id, output_directory, False, avro=False) number_of_consumer_messages = file_consumer.consume(10) consumed_messages = get_consumed_messages(output_directory, False) assert number_of_consumer_messages == 10 assert produced_messages == consumed_messages
def test_set_offsets_offset_to_delta( topic: str, produced_messages_same_partition, interactive_cli_runner, producer: ConfluenceProducer, consumer_group, consumergroup_controller, tmpdir_factory, ): produced_messages_same_partition(topic, producer) vanilla_consumer = ConsumerFactory().create_consumer( group_id=consumer_group, topic_name=None, output_directory=None, last=False, avro=False, initialize_default_output_directory=False, match=None, enable_auto_commit=True, ) vanilla_consumer.subscribe([topic]) vanilla_consumer.consume(10) vanilla_consumer.close() del vanilla_consumer consumergroup_desc_before = consumergroup_controller.get_consumergroup( consumer_id=consumer_group).describe(verbose=True) interactive_cli_runner.invoke( set_offsets, args=[ consumer_group, "--topic-name", topic, "--offset-by-delta", "-2" ], input="y\n", catch_exceptions=True, ) # Check assertions: consumergroup_desc_after = consumergroup_controller.get_consumergroup( consumer_id=consumer_group).describe(verbose=True) assert consumergroup_desc_before["offsets"][topic.encode( "UTF-8")][0]["consumer_offset"] == 10 assert consumergroup_desc_after["offsets"][topic.encode( "UTF-8")][0]["consumer_offset"] == 8
def test_plain_text_consume_and_produce_newly_created_topic( consumer_group: str, interactive_cli_runner: CliRunner, producer: ConfluenceProducer, topic: str, topic_id: str, tmpdir_factory: TempdirFactory, ): source_topic_id = topic target_topic_id = topic_id output_directory = tmpdir_factory.mktemp("output_directory") produced_messages = produce_test_messages(producer, (source_topic_id, 1)) file_consumer = ConsumerFactory().create_consumer(consumer_group, source_topic_id, output_directory, last=False, avro=False) file_consumer.consume(10) result = interactive_cli_runner.invoke( produce, args=["-d", output_directory, target_topic_id], input="y\n", catch_exceptions=False) assert result.exit_code == 0 # Check assertions: assertion_check_directory = tmpdir_factory.mktemp( "assertion_check_directory") file_consumer = ConsumerFactory().create_consumer( (consumer_group + "assertion_check"), target_topic_id, assertion_check_directory, last=False, avro=False) file_consumer.consume(10) consumed_messages = get_consumed_messages(assertion_check_directory, False) assert produced_messages == consumed_messages
def test_avro_consume_and_produce( consumer_group, avro_producer: AvroProducer, source_topic: Tuple[str, int], target_topic: Tuple[str, int], tmpdir_factory, ): source_topic_id, _ = source_topic target_topic_id, _ = target_topic output_directory = tmpdir_factory.mktemp("output_directory") produced_messages = produce_test_messages_with_avro( avro_producer, source_topic) file_consumer = ConsumerFactory().create_consumer(consumer_group, source_topic_id, output_directory, False, avro=True) file_consumer.consume(10) producer = ProducerFactory().create_producer( topic_name=target_topic_id, input_directory=output_directory, avro=True) producer.produce() # Check assertions: assertion_check_directory = tmpdir_factory.mktemp( "assertion_check_directory") file_consumer = ConsumerFactory().create_consumer( consumer_group + "assertion_check", target_topic_id, assertion_check_directory, False, avro=True) file_consumer.consume(10) consumed_messages = get_consumed_messages(assertion_check_directory, True) assert produced_messages == consumed_messages
def test_plain_text_consume_and_produce( consumer_group, producer: ConfluenceProducer, source_topic: Tuple[str, int], target_topic: Tuple[str, int], tmpdir_factory, ): source_topic_id, _ = source_topic target_topic_id, _ = target_topic output_directory = tmpdir_factory.mktemp("output_directory") produced_messages = produce_test_messages(producer, source_topic) file_consumer = ConsumerFactory().create_consumer(consumer_group, source_topic_id, output_directory, False, avro=False) file_consumer.consume(10) producer = ProducerFactory().create_producer(target_topic_id, output_directory, avro=False) producer.produce() # Check assertions: assertion_check_directory = tmpdir_factory.mktemp( "assertion_check_directory") file_consumer = ConsumerFactory().create_consumer( (consumer_group + "assertion_check"), target_topic_id, assertion_check_directory, False, avro=False) file_consumer.consume(10) consumed_messages = get_consumed_messages(assertion_check_directory, False) assert produced_messages == consumed_messages