def create_producer(self, topic, send_timeout_millis=30000, compression_type=CompressionType.NONE, max_pending_messages=1000, block_if_queue_full=False, batching_enabled=False, batching_max_messages=1000, batching_max_allowed_size_in_bytes=128 * 1024, batching_max_publish_delay_ms=10): """ Create a new producer on a given topic. **Args** * `topic`: The topic name **Options** * `send_timeout_seconds`: If a message is not acknowledged by the server before the `send_timeout` expires, an error will be reported. * `compression_type`: Set the compression type for the producer. By default, message payloads are not compressed. Supported compression types are `CompressionType.LZ4` and `CompressionType.ZLib`. * `max_pending_messages`: Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. * `block_if_queue_full`: Set whether `send_async` operations should block when the outgoing message queue is full. * `message_routing_mode`: Set the message routing mode for the partitioned producer. """ conf = _pulsar.ProducerConfiguration() conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) conf.batching_max_allowed_size_in_bytes( batching_max_allowed_size_in_bytes) conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms) p = Producer() p._producer = self._client.create_producer(topic, conf) return p
def create_producer( self, topic, producer_name=None, schema=schema.BytesSchema(), initial_sequence_id=None, send_timeout_millis=30000, compression_type=CompressionType.NONE, max_pending_messages=1000, max_pending_messages_across_partitions=50000, block_if_queue_full=False, batching_enabled=False, batching_max_messages=1000, batching_max_allowed_size_in_bytes=128 * 1024, batching_max_publish_delay_ms=10, message_routing_mode=PartitionsRoutingMode.RoundRobinDistribution, properties=None, ): """ Create a new producer on a given topic. **Args** * `topic`: The topic name **Options** * `producer_name`: Specify a name for the producer. If not assigned, the system will generate a globally unique name which can be accessed with `Producer.producer_name()`. When specifying a name, it is app to the user to ensure that, for a given topic, the producer name is unique across all Pulsar's clusters. * `schema`: Define the schema of the data that will be published by this producer. The schema will be used for two purposes: - Validate the data format against the topic defined schema - Perform serialization/deserialization between data and objects An example for this parameter would be to pass `schema=JsonSchema(MyRecordClass)`. * `initial_sequence_id`: Set the baseline for the sequence ids for messages published by the producer. First message will be using `(initialSequenceId + 1)`` as its sequence id and subsequent messages will be assigned incremental sequence ids, if not otherwise specified. * `send_timeout_seconds`: If a message is not acknowledged by the server before the `send_timeout` expires, an error will be reported. * `compression_type`: Set the compression type for the producer. By default, message payloads are not compressed. Supported compression types are `CompressionType.LZ4`, `CompressionType.ZLib`, `CompressionType.ZSTD` and `CompressionType.SNAPPY`. ZSTD is supported since Pulsar 2.3. Consumers will need to be at least at that release in order to be able to receive messages compressed with ZSTD. SNAPPY is supported since Pulsar 2.4. Consumers will need to be at least at that release in order to be able to receive messages compressed with SNAPPY. * `max_pending_messages`: Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. * `max_pending_messages_across_partitions`: Set the max size of the queue holding the messages pending to receive an acknowledgment across partitions from the broker. * `block_if_queue_full`: Set whether `send_async` operations should block when the outgoing message queue is full. * `message_routing_mode`: Set the message routing mode for the partitioned producer. Default is `PartitionsRoutingMode.RoundRobinDistribution`, other option is `PartitionsRoutingMode.UseSinglePartition` * `properties`: Sets the properties for the producer. The properties associated with a producer can be used for identify a producer at broker side. """ _check_type(str, topic, 'topic') _check_type_or_none(str, producer_name, 'producer_name') _check_type(_schema.Schema, schema, 'schema') _check_type_or_none(int, initial_sequence_id, 'initial_sequence_id') _check_type(int, send_timeout_millis, 'send_timeout_millis') _check_type(CompressionType, compression_type, 'compression_type') _check_type(int, max_pending_messages, 'max_pending_messages') _check_type(int, max_pending_messages_across_partitions, 'max_pending_messages_across_partitions') _check_type(bool, block_if_queue_full, 'block_if_queue_full') _check_type(bool, batching_enabled, 'batching_enabled') _check_type(int, batching_max_messages, 'batching_max_messages') _check_type(int, batching_max_allowed_size_in_bytes, 'batching_max_allowed_size_in_bytes') _check_type(int, batching_max_publish_delay_ms, 'batching_max_publish_delay_ms') _check_type_or_none(dict, properties, 'properties') conf = _pulsar.ProducerConfiguration() conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) conf.max_pending_messages_across_partitions( max_pending_messages_across_partitions) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) conf.batching_max_allowed_size_in_bytes( batching_max_allowed_size_in_bytes) conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms) conf.partitions_routing_mode(message_routing_mode) if producer_name: conf.producer_name(producer_name) if initial_sequence_id: conf.initial_sequence_id(initial_sequence_id) if properties: for k, v in properties.items(): conf.property(k, v) conf.schema(schema.schema_info()) p = Producer() p._producer = self._client.create_producer(topic, conf) p._schema = schema return p
def create_producer(self, topic, producer_name=None, initial_sequence_id=None, send_timeout_millis=30000, compression_type=CompressionType.NONE, max_pending_messages=1000, block_if_queue_full=False, batching_enabled=False, batching_max_messages=1000, batching_max_allowed_size_in_bytes=128 * 1024, batching_max_publish_delay_ms=10): """ Create a new producer on a given topic. **Args** * `topic`: The topic name **Options** * `producer_name`: Specify a name for the producer. If not assigned, the system will generate a globally unique name which can be accessed with `Producer.producer_name()`. When specifying a name, it is app to the user to ensure that, for a given topic, the producer name is unique across all Pulsar's clusters. * `initial_sequence_id`: Set the baseline for the sequence ids for messages published by the producer. First message will be using `(initialSequenceId + 1)`` as its sequence id and subsequent messages will be assigned incremental sequence ids, if not otherwise specified. * `send_timeout_seconds`: If a message is not acknowledged by the server before the `send_timeout` expires, an error will be reported. * `compression_type`: Set the compression type for the producer. By default, message payloads are not compressed. Supported compression types are `CompressionType.LZ4` and `CompressionType.ZLib`. * `max_pending_messages`: Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. * `block_if_queue_full`: Set whether `send_async` operations should block when the outgoing message queue is full. * `message_routing_mode`: Set the message routing mode for the partitioned producer. """ conf = _pulsar.ProducerConfiguration() conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) conf.batching_max_allowed_size_in_bytes( batching_max_allowed_size_in_bytes) conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms) if producer_name: conf.producer_name(producer_name) if initial_sequence_id: conf.initial_sequence_id(initial_sequence_id) p = Producer() p._producer = self._client.create_producer(topic, conf) return p
* `send_timeout_seconds`: If a message is not acknowledged by the server before the `send_timeout` expires, an error will be reported. * `compression_type`: Set the compression type for the producer. By default, message payloads are not compressed. Supported compression types are `CompressionType.LZ4` and `CompressionType.ZLib`. * `max_pending_messages`: Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. * `block_if_queue_full`: Set whether `send_async` operations should block when the outgoing message queue is full. * `message_routing_mode`: Set the message routing mode for the partitioned producer. """ conf = _pulsar.ProducerConfiguration() conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) conf.batching_max_allowed_size_in_bytes( batching_max_allowed_size_in_bytes) conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms) p = Producer() p._producer = self._client.create_producer(topic, conf) return p def subscribe(self, topic,