def test_tls_auth3(self):
certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/'
if not os.path.exists(certs_dir):
certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/"
authPlugin = "tls"
authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir, certs_dir)
client = Client(self.serviceUrlTls,
tls_trust_certs_file_path=certs_dir + 'cacert.pem',
tls_allow_insecure_connection=False,
authentication=Authentication(authPlugin, authParams))
consumer = client.subscribe('persistent://property/cluster/namespace/my-python-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://property/cluster/namespace/my-python-topic-producer-consumer')
producer.send('hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_reader_has_message_available(self):
# create client, producer, reader
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available')
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available',
MessageId.latest)
# before produce data, expected not has message available
self.assertFalse(reader.has_message_available());
for i in range(10):
producer.send('hello-%d' % i)
# produced data, expected has message available
self.assertTrue(reader.has_message_available());
for i in range(10):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
# consumed all data, expected not has message available
self.assertFalse(reader.has_message_available());
for i in range(10, 20):
producer.send('hello-%d' % i)
# produced data again, expected has message available
self.assertTrue(reader.has_message_available());
reader.close()
producer.close()
client.close()
def test_producer_deduplication(self):
# Enable deduplication on namespace
doHttpPost(self.adminUrl + '/admin/namespaces/sample/standalone/ns1/deduplication',
'true')
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer-deduplication-' + str(time.time())
producer = client.create_producer(topic, producer_name='my-producer-name')
self.assertEqual(producer.last_sequence_id(), -1)
consumer = client.subscribe(topic, 'my-sub')
producer.send('hello-0', sequence_id=0)
producer.send('hello-1', sequence_id=1)
producer.send('hello-2', sequence_id=2)
self.assertEqual(producer.last_sequence_id(), 2)
# Repeat the messages and verify they're not received by consumer
producer.send('hello-1', sequence_id=1)
producer.send('hello-2', sequence_id=2)
self.assertEqual(producer.last_sequence_id(), 2)
for i in range(3):
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-%d' % i)
consumer.acknowledge(msg)
try:
# No other messages should be received
consumer.receive(timeout_millis=1000)
self.assertTrue(False)
except:
# Exception is expected
pass
producer.close()
producer = client.create_producer(topic, producer_name='my-producer-name')
self.assertEqual(producer.last_sequence_id(), 2)
# Repeat the messages and verify they're not received by consumer
producer.send('hello-1', sequence_id=1)
producer.send('hello-2', sequence_id=2)
self.assertEqual(producer.last_sequence_id(), 2)
try:
# No other messages should be received
consumer.receive(timeout_millis=1000)
self.assertTrue(False)
except:
# Exception is expected
pass
def test_consumer_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
sub_name = 'my-sub-name'
self._check_value_error(lambda: client.subscribe(None, sub_name))
self._check_value_error(lambda: client.subscribe(topic, None))
self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_type=None))
self._check_value_error(lambda: client.subscribe(topic, sub_name, receiver_queue_size='test'))
self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_name=5))
self._check_value_error(lambda: client.subscribe(topic, sub_name, unacked_messages_timeout_ms='test'))
self._check_value_error(lambda: client.subscribe(topic, sub_name, broker_consumer_stats_cache_time_ms='test'))
client.close()
def test_message_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
producer = client.create_producer(topic)
content = 'test'.encode('utf-8')
self._check_value_error(lambda: producer.send(5))
self._check_value_error(lambda: producer.send(content, properties='test'))
self._check_value_error(lambda: producer.send(content, partition_key=5))
self._check_value_error(lambda: producer.send(content, sequence_id='test'))
self._check_value_error(lambda: producer.send(content, replication_clusters=5))
self._check_value_error(lambda: producer.send(content, disable_replication='test'))
client.close()
def test_reader_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
# This should not raise exception
client.create_reader(topic, MessageId.earliest)
self._check_value_error(lambda: client.create_reader(None, MessageId.earliest))
self._check_value_error(lambda: client.create_reader(topic, None))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test'))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5))
client.close()
def test_auth_junk_params(self):
certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/'
if not os.path.exists(certs_dir):
certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/"
authPlugin = "someoldjunk.so"
authParams = "blah"
client = Client(self.serviceUrlTls,
tls_trust_certs_file_path=certs_dir + 'cacert.pem',
tls_allow_insecure_connection=False,
authentication=Authentication(authPlugin, authParams))
try:
client.subscribe('persistent://property/cluster/namespace/my-python-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
except:
pass # Exception is expected
def test_producer_send_async(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic')
sent_messages = []
def send_callback(producer, msg):
sent_messages.append(msg)
producer.send_async('hello', send_callback)
producer.send_async('hello', send_callback)
producer.send_async('hello', send_callback)
time.sleep(0.1)
self.assertEqual(len(sent_messages), 3)
client.close()
def test_producer_consumer(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('persistent://sample/standalone/ns/my-python-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-producer-consumer')
producer.send('hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_reader_on_last_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message')
for i in range(10):
producer.send('hello-%d' % i)
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message',
MessageId.latest)
for i in range(10, 20):
producer.send('hello-%d' % i)
for i in range(10, 20):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader.close()
client.close()
def _v2_topics(self, url):
client = Client(url)
consumer = client.subscribe('my-v2-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-v2-topic-producer-consumer')
producer.send('hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
consumer.acknowledge(msg)
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-simple',
MessageId.earliest)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-simple')
producer.send('hello')
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
reader.close()
client.close()
def test_seek(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('persistent://sample/standalone/ns/my-python-topic-seek',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-seek')
for i in range(100):
producer.send('hello-%d' % i)
for i in range(100):
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-%d' % i)
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-0')
client.close()
def test_reader_on_specific_message_with_batches(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
batching_enabled=True,
batching_max_publish_delay_ms=1000)
for i in range(10):
producer.send_async('hello-%d' % i, None)
# Send one sync message to make sure everything was published
producer.send('hello-10')
reader1 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next()
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
last_msg_id)
for i in range(5, 11):
msg = reader2.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_reader_on_specific_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message')
for i in range(10):
producer.send('hello-%d' % i)
reader1 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next()
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
last_msg_id)
for i in range(5, 10):
msg = reader2.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_message_listener(self):
client = Client(self.serviceUrl)
received_messages = []
def listener(consumer, msg):
print("Got message: %s" % msg)
received_messages.append(msg)
consumer.acknowledge(msg)
client.subscribe('persistent://sample/standalone/ns/my-python-topic-listener',
'my-sub',
consumer_type=ConsumerType.Exclusive,
message_listener=listener)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-listener')
producer.send('hello-1')
producer.send('hello-2')
producer.send('hello-3')
time.sleep(0.1)
self.assertEqual(len(received_messages), 3)
self.assertEqual(received_messages[0].data(), b"hello-1")
self.assertEqual(received_messages[1].data(), b"hello-2")
self.assertEqual(received_messages[2].data(), b"hello-3")
client.close()
def test_producer_sequence_after_reconnection(self):
# Enable deduplication on namespace
doHttpPost(self.adminUrl + '/admin/namespaces/sample/standalone/ns1/deduplication',
'true')
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer-sequence-after-reconnection-' + str(time.time())
producer = client.create_producer(topic, producer_name='my-producer-name')
self.assertEqual(producer.last_sequence_id(), -1)
for i in range(10):
producer.send('hello-%d' % i)
self.assertEqual(producer.last_sequence_id(), i)
producer.close()
producer = client.create_producer(topic, producer_name='my-producer-name')
self.assertEqual(producer.last_sequence_id(), 9)
for i in range(10, 20):
producer.send('hello-%d' % i)
self.assertEqual(producer.last_sequence_id(), i)
def test_topics_pattern_consumer(self):
import re
client = Client(self.serviceUrl)
topics_pattern = 'persistent://sample/standalone/ns/my-python-pattern-consumer.*'
topic1 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-1'
topic2 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-2'
topic3 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-3'
url1 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-1/partitions'
url2 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-2/partitions'
url3 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-3/partitions'
doHttpPut(url1, '2')
doHttpPut(url2, '3')
doHttpPut(url3, '4')
producer1 = client.create_producer(topic1)
producer2 = client.create_producer(topic2)
producer3 = client.create_producer(topic3)
consumer = client.subscribe(re.compile(topics_pattern),
'my-pattern-consumer-sub',
consumer_type = ConsumerType.Shared,
receiver_queue_size = 10,
pattern_auto_discovery_period = 1
)
# wait enough time to trigger auto discovery
time.sleep(2)
for i in range(100):
producer1.send('hello-1-%d' % i)
for i in range(100):
producer2.send('hello-2-%d' % i)
for i in range(100):
producer3.send('hello-3-%d' % i)
for i in range(300):
msg = consumer.receive()
consumer.acknowledge(msg)
try:
# No other messages should be received
consumer.receive(timeout_millis=500)
self.assertTrue(False)
except:
# Exception is expected
pass
client.close()
def test_publish_compact_and_consume(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test_publish_compact_and_consume'
producer = client.create_producer(topic, producer_name='my-producer-name', batching_enabled=False)
self.assertEqual(producer.last_sequence_id(), -1)
consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
consumer.close()
consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False)
# producer create 2 messages with same key.
producer.send('hello-0', partition_key='key0')
producer.send('hello-1', partition_key='key0')
producer.close()
# issue compact command, and wait success
url=self.adminUrl + '/admin/persistent/sample/standalone/ns1/my-python-test_publish_compact_and_consume/compaction'
doHttpPut(url, '')
while True:
req = urllib2.Request(url)
response = urllib2.urlopen(req)
s=response.read()
if 'RUNNING' in s:
print("Compact still running")
print(s)
time.sleep(0.2)
else:
self.assertTrue('SUCCESS' in s)
print("Compact Complete now")
print(s)
break
# after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key.
consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
msg0 = consumer1.receive()
self.assertEqual(msg0.data(), b'hello-1')
consumer1.acknowledge(msg0)
consumer1.close()
# after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key.
msg0 = consumer2.receive()
self.assertEqual(msg0.data(), b'hello-0')
consumer2.acknowledge(msg0)
msg1 = consumer2.receive()
self.assertEqual(msg1.data(), b'hello-1')
consumer2.acknowledge(msg1)
consumer2.close()
client.close()
def test_topics_consumer(self):
client = Client(self.serviceUrl)
topic1 = 'persistent://sample/standalone/ns/my-python-topics-consumer-1'
topic2 = 'persistent://sample/standalone/ns/my-python-topics-consumer-2'
topic3 = 'persistent://sample/standalone/ns/my-python-topics-consumer-3'
topics = [topic1, topic2, topic3]
url1 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-1/partitions'
url2 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-2/partitions'
url3 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-3/partitions'
doHttpPut(url1, '2')
doHttpPut(url2, '3')
doHttpPut(url3, '4')
producer1 = client.create_producer(topic1)
producer2 = client.create_producer(topic2)
producer3 = client.create_producer(topic3)
consumer = client.subscribe(topics,
'my-topics-consumer-sub',
consumer_type=ConsumerType.Shared,
receiver_queue_size=10
)
for i in range(100):
producer1.send('hello-1-%d' % i)
for i in range(100):
producer2.send('hello-2-%d' % i)
for i in range(100):
producer3.send('hello-3-%d' % i)
for i in range(300):
msg = consumer.receive()
consumer.acknowledge(msg)
try:
# No other messages should be received
consumer.receive(timeout_millis=500)
self.assertTrue(False)
except:
# Exception is expected
pass
client.close()
def test_get_topics_partitions(self):
client = Client(self.serviceUrl)
topic_partitioned = "persistent://public/default/test_get_topics_partitions"
topic_non_partitioned = "persistent://public/default/test_get_topics_not-partitioned"
url1 = self.adminUrl + "/admin/v2/persistent/public/default/test_get_topics_partitions/partitions"
doHttpPut(url1, "3")
self.assertEqual(
client.get_topic_partitions(topic_partitioned),
[
"persistent://public/default/test_get_topics_partitions-partition-0",
"persistent://public/default/test_get_topics_partitions-partition-1",
"persistent://public/default/test_get_topics_partitions-partition-2",
],
)
self.assertEqual(client.get_topic_partitions(topic_non_partitioned),
[topic_non_partitioned])
client.close()
def test_producer_consumer(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-producer-consumer')
producer.send(b'hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
consumer.unsubscribe()
client.close()
def _v2_topics(self, url):
client = Client(url)
consumer = client.subscribe('my-v2-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-v2-topic-producer-consumer')
producer.send(b'hello')
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
consumer.acknowledge(msg)
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader('my-python-topic-reader-simple',
MessageId.earliest)
producer = client.create_producer('my-python-topic-reader-simple')
producer.send(b'hello')
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
reader.close()
client.close()
def test_token_auth(self):
with open('/tmp/pulsar-test-data/tokens/token.txt') as tf:
token = tf.read().strip()
# Use adminUrl to test both HTTP request and binary protocol
client = Client(self.adminUrl,
authentication=AuthenticationToken(token))
consumer = client.subscribe(
'persistent://private/auth/my-python-topic-token-auth',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer(
'persistent://private/auth/my-python-topic-token-auth')
producer.send(b'hello')
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
client.close()
def test_deliver_after(self):
client = Client(self.serviceUrl)
consumer = client.subscribe("my-python-topic-deliver-after",
"my-sub",
consumer_type=ConsumerType.Shared)
producer = client.create_producer("my-python-topic-deliver-after")
# Delay message in 1.1s
producer.send(b"hello", deliver_after=timedelta(milliseconds=1100))
# Message should not be available in the next second
with self.assertRaises(pulsar.Timeout):
consumer.receive(1000)
# Message should be published in the next 500ms
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b"hello")
consumer.unsubscribe()
producer.close()
client.close()
def test_reader_on_last_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message')
for i in range(10):
producer.send('hello-%d' % i)
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message',
MessageId.latest)
for i in range(10, 20):
producer.send('hello-%d' % i)
for i in range(10, 20):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader.close()
client.close()
def test_deliver_at(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-deliver-at',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-deliver-at')
# Delay message in 1.1s
producer.send(b'hello',
deliver_at=int(round(time.time() * 1000)) + 1100)
# Message should not be available in the next second
with self.assertRaises(pulsar.Timeout):
consumer.receive(1000)
# Message should be published now
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
consumer.unsubscribe()
producer.close()
client.close()
def test_reader_on_last_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
"my-python-topic-reader-on-last-message")
for i in range(10):
producer.send(b"hello-%d" % i)
reader = client.create_reader("my-python-topic-reader-on-last-message",
MessageId.latest)
for i in range(10, 20):
producer.send(b"hello-%d" % i)
for i in range(10, 20):
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b"hello-%d" % i)
reader.close()
client.close()
def test_seek(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-seek',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-seek')
for i in range(100):
producer.send(b'hello-%d' % i)
for i in range(100):
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-%d' % i)
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-0')
client.close()
def test_producer_consumer(self):
client = Client('pulsar://localhost:6650/')
consumer = client.subscribe(
'persistent://sample/standalone/ns/my-python-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic-producer-consumer'
)
producer.send('hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), 'hello')
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_redelivery_count(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-redelivery-count',
'my-sub',
consumer_type=ConsumerType.Shared,
negative_ack_redelivery_delay_ms=500)
producer = client.create_producer('my-python-topic-redelivery-count')
producer.send(b'hello')
redelivery_count = 0
for i in range(4):
msg = consumer.receive(TM)
print("Received message %s" % msg.data())
consumer.negative_acknowledge(msg)
redelivery_count = msg.redelivery_count()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
self.assertEqual(3, redelivery_count)
consumer.unsubscribe()
producer.close()
client.close()
def test_deliver_after(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-deliver-after',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-deliver-after')
# Delay message in 1.1s
producer.send(b'hello', deliver_after=timedelta(milliseconds=1100))
# Message should not be available in the next second
try:
msg = consumer.receive(1000)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
# Message should be published in the next 500ms
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
consumer.unsubscribe()
producer.close()
client.close()
def test_tls_auth(self):
certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/'
if not os.path.exists(certs_dir):
certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/"
client = Client(self.serviceUrlTls,
tls_trust_certs_file_path=certs_dir + 'cacert.pem',
tls_allow_insecure_connection=False,
authentication=AuthenticationTLS(certs_dir + 'client-cert.pem', certs_dir + 'client-key.pem'))
consumer = client.subscribe('my-python-topic-tls-auth',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-tls-auth')
producer.send(b'hello')
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
with self.assertRaises(pulsar.Timeout):
consumer.receive(100)
client.close()
def test_message_properties(self):
client = Client(self.serviceUrl)
topic = 'my-python-test-message-properties'
consumer = client.subscribe(topic=topic,
subscription_name='my-subscription',
schema=pulsar.schema.StringSchema())
producer = client.create_producer(topic=topic,
schema=StringSchema())
producer.send('hello',
properties={
'a': '1',
'b': '2'
})
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.value(), 'hello')
self.assertEqual(msg.properties(), {
'a': '1',
'b': '2'
})
consumer.unsubscribe()
client.close()
def test_producer_argument_errors(self):
client = Client(self.serviceUrl)
self._check_value_error(lambda: client.create_producer(None))
topic = 'my-python-test-producer'
self._check_value_error(
lambda: client.create_producer(topic, producer_name=5))
self._check_value_error(
lambda: client.create_producer(topic, initial_sequence_id='test'))
self._check_value_error(
lambda: client.create_producer(topic, send_timeout_millis='test'))
self._check_value_error(
lambda: client.create_producer(topic, compression_type=None))
self._check_value_error(
lambda: client.create_producer(topic, max_pending_messages='test'))
self._check_value_error(
lambda: client.create_producer(topic, block_if_queue_full='test'))
self._check_value_error(
lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(
lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(lambda: client.create_producer(
topic, batching_max_allowed_size_in_bytes='test'))
self._check_value_error(lambda: client.create_producer(
topic, batching_max_publish_delay_ms='test'))
client.close()
def test_simple_producer(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic')
producer.send('hello')
producer.close()
client.close()
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
from pulsar import Client, AuthenticationToken
client = Client(os.environ.get('SERVICE_URL'),
authentication=AuthenticationToken(
os.environ.get('AUTH_PARAMS')))
client.close()
def test_producer_argument_errors(self):
client = Client(self.serviceUrl)
self._check_value_error(lambda: client.create_producer(None))
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
self._check_value_error(lambda: client.create_producer(topic, producer_name=5))
self._check_value_error(lambda: client.create_producer(topic, initial_sequence_id='test'))
self._check_value_error(lambda: client.create_producer(topic, send_timeout_millis='test'))
self._check_value_error(lambda: client.create_producer(topic, compression_type=None))
self._check_value_error(lambda: client.create_producer(topic, max_pending_messages='test'))
self._check_value_error(lambda: client.create_producer(topic, block_if_queue_full='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_max_allowed_size_in_bytes='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_max_publish_delay_ms='test'))
client.close()
def test_producer_routing_mode(self):
client = Client(self.serviceUrl)
producer = client.create_producer('my-python-test-producer',
message_routing_mode=PartitionsRoutingMode.UseSinglePartition)
producer.send(b'test')
client.close()
def test_client_logger(self):
logger = logging.getLogger("pulsar")
Client(self.serviceUrl, logger=logger)
def test_seek(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-seek',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-seek')
for i in range(100):
if i > 0:
time.sleep(0.02)
producer.send(b'hello-%d' % i)
ids = []
timestamps = []
for i in range(100):
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-%d' % i)
ids.append(msg.message_id())
timestamps.append(msg.publish_timestamp())
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-0')
# seek on messageId
consumer.seek(ids[50])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-50')
# ditto, but seek on timestamp
consumer.seek(timestamps[42])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-42')
# repeat with reader
reader = client.create_reader('my-python-topic-seek', MessageId.latest)
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
# earliest
reader.seek(MessageId.earliest)
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-0')
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-1')
# seek on messageId
reader.seek(ids[33])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-33')
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-34')
# seek on timestamp
reader.seek(timestamps[79])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-79')
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-80')
reader.close()
client.close()
def get_producer():
cl = Client(self.serviceUrl)
return cl.create_producer(topic="foobar")
def test_connect_error(self):
with self.assertRaises(pulsar.ConnectError):
client = Client("fakeServiceUrl")
client.create_producer("connect-error-topic")
client.close()
"""
This example shows how to configure negative acknowledgement.
"""
from pulsar import Client, schema
client = Client('pulsar://localhost:6650')
consumer = client.subscribe('negative_acks',
'test',
schema=schema.StringSchema())
producer = client.create_producer('negative_acks',
schema=schema.StringSchema())
for i in range(10):
print('send msg "hello-%d"' % i)
producer.send_async('hello-%d' % i, callback=None)
# Flush all the messages buffered in the client and
# wait until all messages have been successfully persisted
producer.flush()
for i in range(10):
msg = consumer.receive()
consumer.negative_acknowledge(msg)
print('receive and nack msg "%s"' % msg.data().decode("utf-8"))
for i in range(10):
msg = consumer.receive()
consumer.acknowledge(msg)
print('receive and ack msg "%s"' % msg.data().decode("utf-8"))
try:
# No more messages expected
def __init__(self, args):
self.args = args
self.client = Client(args.service_url,
authentication=AuthenticationOauth2(
args.auth_params))
def test_consumer_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'my-python-test-producer'
sub_name = 'my-sub-name'
self._check_value_error(lambda: client.subscribe(None, sub_name))
self._check_value_error(lambda: client.subscribe(topic, None))
self._check_value_error(
lambda: client.subscribe(topic, sub_name, consumer_type=None))
self._check_value_error(lambda: client.subscribe(
topic, sub_name, receiver_queue_size='test'))
self._check_value_error(
lambda: client.subscribe(topic, sub_name, consumer_name=5))
self._check_value_error(lambda: client.subscribe(
topic, sub_name, unacked_messages_timeout_ms='test'))
self._check_value_error(lambda: client.subscribe(
topic, sub_name, broker_consumer_stats_cache_time_ms='test'))
client.close()
def test_producer_routing_mode(self):
client = Client(self.serviceUrl)
producer = client.create_producer('my-python-test-producer',
message_routing_mode=PartitionsRoutingMode.UseSinglePartition)
producer.send(b'test')
client.close()
def test_publish_compact_and_consume(self):
client = Client(self.serviceUrl)
topic = 'compaction_%s' % (uuid.uuid4())
producer = client.create_producer(topic,
producer_name='my-producer-name',
batching_enabled=False)
self.assertEqual(producer.last_sequence_id(), -1)
consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
consumer.close()
consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False)
# producer create 2 messages with same key.
producer.send(b'hello-0', partition_key='key0')
producer.send(b'hello-1', partition_key='key0')
producer.close()
# issue compact command, and wait success
url = '%s/admin/v2/persistent/public/default/%s/compaction' % (
self.adminUrl, topic)
doHttpPut(url, '')
while True:
s = doHttpGet(url).decode('utf-8')
if 'RUNNING' in s:
print(s)
print("Compact still running")
time.sleep(0.2)
else:
print(s)
print("Compact Complete now")
self.assertTrue('SUCCESS' in s)
break
# after compaction completes the compacted ledger is recorded
# as a property of a cursor. As persisting the cursor is async
# and we don't wait for the acknowledgement of the acknowledgement,
# there may be a race if we try to read the compacted ledger immediately.
# therefore wait a second to allow the compacted ledger to be updated on
# the broker.
time.sleep(1.0)
# after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key.
consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
msg0 = consumer1.receive(TM)
self.assertEqual(msg0.data(), b'hello-1')
consumer1.acknowledge(msg0)
consumer1.close()
# ditto for reader
reader1 = client.create_reader(topic,
MessageId.earliest,
is_read_compacted=True)
msg0 = reader1.read_next(TM)
self.assertEqual(msg0.data(), b'hello-1')
reader1.close()
# after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key.
msg0 = consumer2.receive(TM)
self.assertEqual(msg0.data(), b'hello-0')
consumer2.acknowledge(msg0)
msg1 = consumer2.receive(TM)
self.assertEqual(msg1.data(), b'hello-1')
consumer2.acknowledge(msg1)
consumer2.close()
# ditto for reader
reader2 = client.create_reader(topic,
MessageId.earliest,
is_read_compacted=False)
msg0 = reader2.read_next(TM)
self.assertEqual(msg0.data(), b'hello-0')
msg1 = reader2.read_next(TM)
self.assertEqual(msg1.data(), b'hello-1')
reader2.close()
client.close()
def test_seek(self):
client = Client(self.serviceUrl)
topic = "my-python-topic-seek-" + str(time.time())
consumer = client.subscribe(topic,
"my-sub",
consumer_type=ConsumerType.Shared)
producer = client.create_producer(topic)
for i in range(100):
if i > 0:
time.sleep(0.02)
producer.send(b"hello-%d" % i)
ids = []
timestamps = []
for i in range(100):
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-%d" % i)
ids.append(msg.message_id())
timestamps.append(msg.publish_timestamp())
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-0")
# seek on messageId
consumer.seek(ids[50])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-50")
# ditto, but seek on timestamp
consumer.seek(timestamps[42])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-42")
# repeat with reader
reader = client.create_reader(topic, MessageId.latest)
with self.assertRaises(pulsar.Timeout):
reader.read_next(100)
# earliest
reader.seek(MessageId.earliest)
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-0")
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-1")
# seek on messageId
reader.seek(ids[33])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-33")
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-34")
# seek on timestamp
reader.seek(timestamps[79])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-79")
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-80")
reader.close()
client.close()
def test_simple_producer(self):
client = Client(self.serviceUrl)
producer = client.create_producer('my-python-topic')
producer.send(b'hello')
producer.close()
client.close()
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
from pulsar import Client, AuthenticationToken
client = Client(os.environ.get('SERVICE_URL'),
authentication=AuthenticationToken(
os.environ.get('AUTH_PARAMS')))
producer = client.create_producer('my-topic')
for i in range(10):
producer.send(('Hello-%d' % i).encode('utf-8'))
print('send msg "hello-%d"' % i)
client.close()
