def create_producer(self,
topic,
send_timeout_millis=30000,
compression_type=CompressionType.NONE,
max_pending_messages=1000,
block_if_queue_full=False,
batching_enabled=False,
batching_max_messages=1000,
batching_max_allowed_size_in_bytes=128 * 1024,
batching_max_publish_delay_ms=10):
"""
Create a new producer on a given topic.
**Args**
* `topic`:
The topic name
**Options**
* `send_timeout_seconds`:
If a message is not acknowledged by the server before the
`send_timeout` expires, an error will be reported.
* `compression_type`:
Set the compression type for the producer. By default, message
payloads are not compressed. Supported compression types are
`CompressionType.LZ4` and `CompressionType.ZLib`.
* `max_pending_messages`:
Set the max size of the queue holding the messages pending to receive
an acknowledgment from the broker.
* `block_if_queue_full`: Set whether `send_async` operations should
block when the outgoing message queue is full.
* `message_routing_mode`:
Set the message routing mode for the partitioned producer.
"""
conf = _pulsar.ProducerConfiguration()
conf.send_timeout_millis(send_timeout_millis)
conf.compression_type(compression_type)
conf.max_pending_messages(max_pending_messages)
conf.block_if_queue_full(block_if_queue_full)
conf.batching_enabled(batching_enabled)
conf.batching_max_messages(batching_max_messages)
conf.batching_max_allowed_size_in_bytes(
batching_max_allowed_size_in_bytes)
conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms)
p = Producer()
p._producer = self._client.create_producer(topic, conf)
return p
def create_producer(
self,
topic,
producer_name=None,
schema=schema.BytesSchema(),
initial_sequence_id=None,
send_timeout_millis=30000,
compression_type=CompressionType.NONE,
max_pending_messages=1000,
max_pending_messages_across_partitions=50000,
block_if_queue_full=False,
batching_enabled=False,
batching_max_messages=1000,
batching_max_allowed_size_in_bytes=128 * 1024,
batching_max_publish_delay_ms=10,
message_routing_mode=PartitionsRoutingMode.RoundRobinDistribution,
properties=None,
):
"""
Create a new producer on a given topic.
**Args**
* `topic`:
The topic name
**Options**
* `producer_name`:
Specify a name for the producer. If not assigned,
the system will generate a globally unique name which can be accessed
with `Producer.producer_name()`. When specifying a name, it is app to
the user to ensure that, for a given topic, the producer name is unique
across all Pulsar's clusters.
* `schema`:
Define the schema of the data that will be published by this producer.
The schema will be used for two purposes:
- Validate the data format against the topic defined schema
- Perform serialization/deserialization between data and objects
An example for this parameter would be to pass `schema=JsonSchema(MyRecordClass)`.
* `initial_sequence_id`:
Set the baseline for the sequence ids for messages
published by the producer. First message will be using
`(initialSequenceId + 1)`` as its sequence id and subsequent messages will
be assigned incremental sequence ids, if not otherwise specified.
* `send_timeout_seconds`:
If a message is not acknowledged by the server before the
`send_timeout` expires, an error will be reported.
* `compression_type`:
Set the compression type for the producer. By default, message
payloads are not compressed. Supported compression types are
`CompressionType.LZ4`, `CompressionType.ZLib`, `CompressionType.ZSTD` and `CompressionType.SNAPPY`.
ZSTD is supported since Pulsar 2.3. Consumers will need to be at least at that
release in order to be able to receive messages compressed with ZSTD.
SNAPPY is supported since Pulsar 2.4. Consumers will need to be at least at that
release in order to be able to receive messages compressed with SNAPPY.
* `max_pending_messages`:
Set the max size of the queue holding the messages pending to receive
an acknowledgment from the broker.
* `max_pending_messages_across_partitions`:
Set the max size of the queue holding the messages pending to receive
an acknowledgment across partitions from the broker.
* `block_if_queue_full`: Set whether `send_async` operations should
block when the outgoing message queue is full.
* `message_routing_mode`:
Set the message routing mode for the partitioned producer. Default is `PartitionsRoutingMode.RoundRobinDistribution`,
other option is `PartitionsRoutingMode.UseSinglePartition`
* `properties`:
Sets the properties for the producer. The properties associated with a producer
can be used for identify a producer at broker side.
"""
_check_type(str, topic, 'topic')
_check_type_or_none(str, producer_name, 'producer_name')
_check_type(_schema.Schema, schema, 'schema')
_check_type_or_none(int, initial_sequence_id, 'initial_sequence_id')
_check_type(int, send_timeout_millis, 'send_timeout_millis')
_check_type(CompressionType, compression_type, 'compression_type')
_check_type(int, max_pending_messages, 'max_pending_messages')
_check_type(int, max_pending_messages_across_partitions,
'max_pending_messages_across_partitions')
_check_type(bool, block_if_queue_full, 'block_if_queue_full')
_check_type(bool, batching_enabled, 'batching_enabled')
_check_type(int, batching_max_messages, 'batching_max_messages')
_check_type(int, batching_max_allowed_size_in_bytes,
'batching_max_allowed_size_in_bytes')
_check_type(int, batching_max_publish_delay_ms,
'batching_max_publish_delay_ms')
_check_type_or_none(dict, properties, 'properties')
conf = _pulsar.ProducerConfiguration()
conf.send_timeout_millis(send_timeout_millis)
conf.compression_type(compression_type)
conf.max_pending_messages(max_pending_messages)
conf.max_pending_messages_across_partitions(
max_pending_messages_across_partitions)
conf.block_if_queue_full(block_if_queue_full)
conf.batching_enabled(batching_enabled)
conf.batching_max_messages(batching_max_messages)
conf.batching_max_allowed_size_in_bytes(
batching_max_allowed_size_in_bytes)
conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms)
conf.partitions_routing_mode(message_routing_mode)
if producer_name:
conf.producer_name(producer_name)
if initial_sequence_id:
conf.initial_sequence_id(initial_sequence_id)
if properties:
for k, v in properties.items():
conf.property(k, v)
conf.schema(schema.schema_info())
p = Producer()
p._producer = self._client.create_producer(topic, conf)
p._schema = schema
return p
def create_producer(self,
topic,
producer_name=None,
initial_sequence_id=None,
send_timeout_millis=30000,
compression_type=CompressionType.NONE,
max_pending_messages=1000,
block_if_queue_full=False,
batching_enabled=False,
batching_max_messages=1000,
batching_max_allowed_size_in_bytes=128 * 1024,
batching_max_publish_delay_ms=10):
"""
Create a new producer on a given topic.
**Args**
* `topic`:
The topic name
**Options**
* `producer_name`:
Specify a name for the producer. If not assigned,
the system will generate a globally unique name which can be accessed
with `Producer.producer_name()`. When specifying a name, it is app to
the user to ensure that, for a given topic, the producer name is unique
across all Pulsar's clusters.
* `initial_sequence_id`:
Set the baseline for the sequence ids for messages
published by the producer. First message will be using
`(initialSequenceId + 1)`` as its sequence id and subsequent messages will
be assigned incremental sequence ids, if not otherwise specified.
* `send_timeout_seconds`:
If a message is not acknowledged by the server before the
`send_timeout` expires, an error will be reported.
* `compression_type`:
Set the compression type for the producer. By default, message
payloads are not compressed. Supported compression types are
`CompressionType.LZ4` and `CompressionType.ZLib`.
* `max_pending_messages`:
Set the max size of the queue holding the messages pending to receive
an acknowledgment from the broker.
* `block_if_queue_full`: Set whether `send_async` operations should
block when the outgoing message queue is full.
* `message_routing_mode`:
Set the message routing mode for the partitioned producer.
"""
conf = _pulsar.ProducerConfiguration()
conf.send_timeout_millis(send_timeout_millis)
conf.compression_type(compression_type)
conf.max_pending_messages(max_pending_messages)
conf.block_if_queue_full(block_if_queue_full)
conf.batching_enabled(batching_enabled)
conf.batching_max_messages(batching_max_messages)
conf.batching_max_allowed_size_in_bytes(
batching_max_allowed_size_in_bytes)
conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms)
if producer_name:
conf.producer_name(producer_name)
if initial_sequence_id:
conf.initial_sequence_id(initial_sequence_id)
p = Producer()
p._producer = self._client.create_producer(topic, conf)
return p
* `send_timeout_seconds`:
If a message is not acknowledged by the server before the
`send_timeout` expires, an error will be reported.
* `compression_type`:
Set the compression type for the producer. By default, message
payloads are not compressed. Supported compression types are
`CompressionType.LZ4` and `CompressionType.ZLib`.
* `max_pending_messages`:
Set the max size of the queue holding the messages pending to receive
an acknowledgment from the broker.
* `block_if_queue_full`: Set whether `send_async` operations should
block when the outgoing message queue is full.
* `message_routing_mode`:
Set the message routing mode for the partitioned producer.
"""
conf = _pulsar.ProducerConfiguration()
conf.send_timeout_millis(send_timeout_millis)
conf.compression_type(compression_type)
conf.max_pending_messages(max_pending_messages)
conf.block_if_queue_full(block_if_queue_full)
conf.batching_enabled(batching_enabled)
conf.batching_max_messages(batching_max_messages)
conf.batching_max_allowed_size_in_bytes(
batching_max_allowed_size_in_bytes)
conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms)
p = Producer()
p._producer = self._client.create_producer(topic, conf)
return p
def subscribe(self,
topic,
