def test_reader_has_message_available(self):
# create client, producer, reader
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available')
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available',
MessageId.latest)
# before produce data, expected not has message available
self.assertFalse(reader.has_message_available());
for i in range(10):
producer.send('hello-%d' % i)
# produced data, expected has message available
self.assertTrue(reader.has_message_available());
for i in range(10):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
# consumed all data, expected not has message available
self.assertFalse(reader.has_message_available());
for i in range(10, 20):
producer.send('hello-%d' % i)
# produced data again, expected has message available
self.assertTrue(reader.has_message_available());
reader.close()
producer.close()
client.close()
def test_message_listener(self):
client = Client(self.serviceUrl)
received_messages = []
def listener(consumer, msg):
print("Got message: %s" % msg)
received_messages.append(msg)
consumer.acknowledge(msg)
client.subscribe('persistent://sample/standalone/ns/my-python-topic-listener',
'my-sub',
consumer_type=ConsumerType.Exclusive,
message_listener=listener)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-listener')
producer.send('hello-1')
producer.send('hello-2')
producer.send('hello-3')
time.sleep(0.1)
self.assertEqual(len(received_messages), 3)
self.assertEqual(received_messages[0].data(), b"hello-1")
self.assertEqual(received_messages[1].data(), b"hello-2")
self.assertEqual(received_messages[2].data(), b"hello-3")
client.close()
def test_reader_on_specific_message_with_batches(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
batching_enabled=True,
batching_max_publish_delay_ms=1000)
for i in range(10):
producer.send_async('hello-%d' % i, None)
# Send one sync message to make sure everything was published
producer.send('hello-10')
reader1 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next()
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
last_msg_id)
for i in range(5, 11):
msg = reader2.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_reader_on_specific_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message')
for i in range(10):
producer.send('hello-%d' % i)
reader1 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next()
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
last_msg_id)
for i in range(5, 10):
msg = reader2.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_tls_auth3(self):
certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/'
if not os.path.exists(certs_dir):
certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/"
authPlugin = "tls"
authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir, certs_dir)
client = Client(self.serviceUrlTls,
tls_trust_certs_file_path=certs_dir + 'cacert.pem',
tls_allow_insecure_connection=False,
authentication=Authentication(authPlugin, authParams))
consumer = client.subscribe('persistent://property/cluster/namespace/my-python-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://property/cluster/namespace/my-python-topic-producer-consumer')
producer.send('hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_topics_pattern_consumer(self):
import re
client = Client(self.serviceUrl)
topics_pattern = 'persistent://sample/standalone/ns/my-python-pattern-consumer.*'
topic1 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-1'
topic2 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-2'
topic3 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-3'
url1 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-1/partitions'
url2 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-2/partitions'
url3 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-3/partitions'
doHttpPut(url1, '2')
doHttpPut(url2, '3')
doHttpPut(url3, '4')
producer1 = client.create_producer(topic1)
producer2 = client.create_producer(topic2)
producer3 = client.create_producer(topic3)
consumer = client.subscribe(re.compile(topics_pattern),
'my-pattern-consumer-sub',
consumer_type = ConsumerType.Shared,
receiver_queue_size = 10,
pattern_auto_discovery_period = 1
)
# wait enough time to trigger auto discovery
time.sleep(2)
for i in range(100):
producer1.send('hello-1-%d' % i)
for i in range(100):
producer2.send('hello-2-%d' % i)
for i in range(100):
producer3.send('hello-3-%d' % i)
for i in range(300):
msg = consumer.receive()
consumer.acknowledge(msg)
try:
# No other messages should be received
consumer.receive(timeout_millis=500)
self.assertTrue(False)
except:
# Exception is expected
pass
client.close()
def test_reader_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
# This should not raise exception
client.create_reader(topic, MessageId.earliest)
self._check_value_error(lambda: client.create_reader(None, MessageId.earliest))
self._check_value_error(lambda: client.create_reader(topic, None))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test'))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5))
client.close()
def test_consumer_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
sub_name = 'my-sub-name'
self._check_value_error(lambda: client.subscribe(None, sub_name))
self._check_value_error(lambda: client.subscribe(topic, None))
self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_type=None))
self._check_value_error(lambda: client.subscribe(topic, sub_name, receiver_queue_size='test'))
self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_name=5))
self._check_value_error(lambda: client.subscribe(topic, sub_name, unacked_messages_timeout_ms='test'))
self._check_value_error(lambda: client.subscribe(topic, sub_name, broker_consumer_stats_cache_time_ms='test'))
client.close()
def test_message_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
producer = client.create_producer(topic)
content = 'test'.encode('utf-8')
self._check_value_error(lambda: producer.send(5))
self._check_value_error(lambda: producer.send(content, properties='test'))
self._check_value_error(lambda: producer.send(content, partition_key=5))
self._check_value_error(lambda: producer.send(content, sequence_id='test'))
self._check_value_error(lambda: producer.send(content, replication_clusters=5))
self._check_value_error(lambda: producer.send(content, disable_replication='test'))
client.close()
def test_producer_send_async(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic')
sent_messages = []
def send_callback(producer, msg):
sent_messages.append(msg)
producer.send_async('hello', send_callback)
producer.send_async('hello', send_callback)
producer.send_async('hello', send_callback)
time.sleep(0.1)
self.assertEqual(len(sent_messages), 3)
client.close()
def test_publish_compact_and_consume(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test_publish_compact_and_consume'
producer = client.create_producer(topic, producer_name='my-producer-name', batching_enabled=False)
self.assertEqual(producer.last_sequence_id(), -1)
consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
consumer.close()
consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False)
# producer create 2 messages with same key.
producer.send('hello-0', partition_key='key0')
producer.send('hello-1', partition_key='key0')
producer.close()
# issue compact command, and wait success
url=self.adminUrl + '/admin/persistent/sample/standalone/ns1/my-python-test_publish_compact_and_consume/compaction'
doHttpPut(url, '')
while True:
req = urllib2.Request(url)
response = urllib2.urlopen(req)
s=response.read()
if 'RUNNING' in s:
print("Compact still running")
print(s)
time.sleep(0.2)
else:
self.assertTrue('SUCCESS' in s)
print("Compact Complete now")
print(s)
break
# after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key.
consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
msg0 = consumer1.receive()
self.assertEqual(msg0.data(), b'hello-1')
consumer1.acknowledge(msg0)
consumer1.close()
# after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key.
msg0 = consumer2.receive()
self.assertEqual(msg0.data(), b'hello-0')
consumer2.acknowledge(msg0)
msg1 = consumer2.receive()
self.assertEqual(msg1.data(), b'hello-1')
consumer2.acknowledge(msg1)
consumer2.close()
client.close()
def test_topics_consumer(self):
client = Client(self.serviceUrl)
topic1 = 'persistent://sample/standalone/ns/my-python-topics-consumer-1'
topic2 = 'persistent://sample/standalone/ns/my-python-topics-consumer-2'
topic3 = 'persistent://sample/standalone/ns/my-python-topics-consumer-3'
topics = [topic1, topic2, topic3]
url1 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-1/partitions'
url2 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-2/partitions'
url3 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-3/partitions'
doHttpPut(url1, '2')
doHttpPut(url2, '3')
doHttpPut(url3, '4')
producer1 = client.create_producer(topic1)
producer2 = client.create_producer(topic2)
producer3 = client.create_producer(topic3)
consumer = client.subscribe(topics,
'my-topics-consumer-sub',
consumer_type=ConsumerType.Shared,
receiver_queue_size=10
)
for i in range(100):
producer1.send('hello-1-%d' % i)
for i in range(100):
producer2.send('hello-2-%d' % i)
for i in range(100):
producer3.send('hello-3-%d' % i)
for i in range(300):
msg = consumer.receive()
consumer.acknowledge(msg)
try:
# No other messages should be received
consumer.receive(timeout_millis=500)
self.assertTrue(False)
except:
# Exception is expected
pass
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader('my-python-topic-reader-simple',
MessageId.earliest)
producer = client.create_producer('my-python-topic-reader-simple')
producer.send(b'hello')
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
with self.assertRaises(pulsar.Timeout):
reader.read_next(100)
reader.close()
client.close()
def test_message_properties(self):
client = Client(self.serviceUrl)
topic = "my-python-test-message-properties"
consumer = client.subscribe(topic=topic,
subscription_name="my-subscription",
schema=pulsar.schema.StringSchema())
producer = client.create_producer(topic=topic,
schema=pulsar.schema.StringSchema())
producer.send("hello", properties={"a": "1", "b": "2"})
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.value(), "hello")
self.assertEqual(msg.properties(), {"a": "1", "b": "2"})
consumer.unsubscribe()
client.close()
def test_producer_consumer(self):
client = Client(self.serviceUrl)
consumer = client.subscribe("my-python-topic-producer-consumer",
"my-sub",
consumer_type=ConsumerType.Shared)
producer = client.create_producer("my-python-topic-producer-consumer")
producer.send(b"hello")
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b"hello")
with self.assertRaises(pulsar.Timeout):
consumer.receive(100)
consumer.unsubscribe()
client.close()
def test_token_auth_supplier(self):
def read_token():
with open('/tmp/pulsar-test-data/tokens/token.txt') as tf:
return tf.read().strip()
client = Client(self.serviceUrl,
authentication=AuthenticationToken(read_token))
consumer = client.subscribe('persistent://private/auth/my-python-topic-token-auth',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://private/auth/my-python-topic-token-auth')
producer.send(b'hello')
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
client.close()
def _v2_topics(self, url):
client = Client(url)
consumer = client.subscribe("my-v2-topic-producer-consumer",
"my-sub",
consumer_type=ConsumerType.Shared)
producer = client.create_producer("my-v2-topic-producer-consumer")
producer.send(b"hello")
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b"hello")
consumer.acknowledge(msg)
with self.assertRaises(pulsar.Timeout):
consumer.receive(100)
client.close()
def test_message_properties(self):
client = Client(self.serviceUrl)
topic = 'my-python-test-message-properties'
consumer = client.subscribe(topic=topic,
subscription_name='my-subscription',
schema=pulsar.schema.StringSchema())
producer = client.create_producer(topic=topic,
schema=pulsar.schema.StringSchema())
producer.send('hello', properties={'a': '1', 'b': '2'})
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.value(), 'hello')
self.assertEqual(msg.properties(), {'a': '1', 'b': '2'})
consumer.unsubscribe()
client.close()
def test_get_topics_partitions(self):
client = Client(self.serviceUrl)
topic_partitioned = 'persistent://public/default/test_get_topics_partitions'
topic_non_partitioned = 'persistent://public/default/test_get_topics_not-partitioned'
url1 = self.adminUrl + '/admin/v2/persistent/public/default/test_get_topics_partitions/partitions'
doHttpPut(url1, '3')
self.assertEqual(client.get_topic_partitions(topic_partitioned), [
'persistent://public/default/test_get_topics_partitions-partition-0',
'persistent://public/default/test_get_topics_partitions-partition-1',
'persistent://public/default/test_get_topics_partitions-partition-2'
])
self.assertEqual(client.get_topic_partitions(topic_non_partitioned),
[topic_non_partitioned])
client.close()
def test_message_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'my-python-test-producer'
producer = client.create_producer(topic)
content = 'test'.encode('utf-8')
self._check_type_error(lambda: producer.send(5))
self._check_value_error(lambda: producer.send(content, properties='test'))
self._check_value_error(lambda: producer.send(content, partition_key=5))
self._check_value_error(lambda: producer.send(content, sequence_id='test'))
self._check_value_error(lambda: producer.send(content, replication_clusters=5))
self._check_value_error(lambda: producer.send(content, disable_replication='test'))
self._check_value_error(lambda: producer.send(content, event_timestamp='test'))
self._check_value_error(lambda: producer.send(content, deliver_at='test'))
self._check_value_error(lambda: producer.send(content, deliver_after='test'))
client.close()
def test_producer_send_async(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic')
sent_messages = []
def send_callback(producer, msg):
sent_messages.append(msg)
producer.send_async('hello', send_callback)
producer.send_async('hello', send_callback)
producer.send_async('hello', send_callback)
time.sleep(0.1)
self.assertEqual(len(sent_messages), 3)
client.close()
def test_publish_compact_and_consume(self):
client = Client(self.serviceUrl)
topic = 'my-python-test_publish_compact_and_consume'
producer = client.create_producer(topic, producer_name='my-producer-name', batching_enabled=False)
self.assertEqual(producer.last_sequence_id(), -1)
consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
consumer.close()
consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False)
# producer create 2 messages with same key.
producer.send(b'hello-0', partition_key='key0')
producer.send(b'hello-1', partition_key='key0')
producer.close()
# issue compact command, and wait success
url=self.adminUrl + '/admin/v2/persistent/public/default/my-python-test_publish_compact_and_consume/compaction'
doHttpPut(url, '')
while True:
s=doHttpGet(url).decode('utf-8')
if 'RUNNING' in s:
print("Compact still running")
print(s)
time.sleep(0.2)
else:
self.assertTrue('SUCCESS' in s)
print("Compact Complete now")
print(s)
break
# after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key.
consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
msg0 = consumer1.receive()
self.assertEqual(msg0.data(), b'hello-1')
consumer1.acknowledge(msg0)
consumer1.close()
# after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key.
msg0 = consumer2.receive()
self.assertEqual(msg0.data(), b'hello-0')
consumer2.acknowledge(msg0)
msg1 = consumer2.receive()
self.assertEqual(msg1.data(), b'hello-1')
consumer2.acknowledge(msg1)
consumer2.close()
client.close()
def test_token_auth(self):
with open('/tmp/pulsar-test-data/tokens/token.txt') as tf:
token = tf.read().strip()
# Use adminUrl to test both HTTP request and binary protocol
client = Client(self.adminUrl,
authentication=AuthenticationToken(token))
consumer = client.subscribe('persistent://private/auth/my-python-topic-token-auth',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://private/auth/my-python-topic-token-auth')
producer.send(b'hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
client.close()
def test_consumer_queue_size_is_zero(self):
client = Client(self.serviceUrl)
consumer = client.subscribe(
'my-python-topic-consumer-init-queue-size-is-zero',
'my-sub',
consumer_type=ConsumerType.Shared,
receiver_queue_size=0,
initial_position=InitialPosition.Earliest)
producer = client.create_producer(
'my-python-topic-consumer-init-queue-size-is-zero')
producer.send(b'hello')
time.sleep(0.1)
msg = consumer.receive()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
consumer.unsubscribe()
client.close()
def test_producer_argument_errors(self):
client = Client(self.serviceUrl)
self._check_value_error(lambda: client.create_producer(None))
topic = 'my-python-test-producer'
self._check_value_error(lambda: client.create_producer(topic, producer_name=5))
self._check_value_error(lambda: client.create_producer(topic, initial_sequence_id='test'))
self._check_value_error(lambda: client.create_producer(topic, send_timeout_millis='test'))
self._check_value_error(lambda: client.create_producer(topic, compression_type=None))
self._check_value_error(lambda: client.create_producer(topic, max_pending_messages='test'))
self._check_value_error(lambda: client.create_producer(topic, block_if_queue_full='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_max_allowed_size_in_bytes='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_max_publish_delay_ms='test'))
client.close()
def test_producer_argument_errors(self):
client = Client(self.serviceUrl)
self._check_value_error(lambda: client.create_producer(None))
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
self._check_value_error(lambda: client.create_producer(topic, producer_name=5))
self._check_value_error(lambda: client.create_producer(topic, initial_sequence_id='test'))
self._check_value_error(lambda: client.create_producer(topic, send_timeout_millis='test'))
self._check_value_error(lambda: client.create_producer(topic, compression_type=None))
self._check_value_error(lambda: client.create_producer(topic, max_pending_messages='test'))
self._check_value_error(lambda: client.create_producer(topic, block_if_queue_full='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_max_allowed_size_in_bytes='test'))
self._check_value_error(lambda: client.create_producer(topic, batching_max_publish_delay_ms='test'))
client.close()
def test_encryption(self):
publicKeyPath = "/pulsar//pulsar-broker/src/test/resources/certificate/public-key.client-rsa.pem"
privateKeyPath = "/pulsar/pulsar-broker/src/test/resources/certificate/private-key.client-rsa.pem"
crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath)
client = Client(self.serviceUrl)
topic = 'my-python-test-end-to-end-encryption'
consumer = client.subscribe(topic=topic,
subscription_name='my-subscription',
crypto_key_reader=crypto_key_reader)
producer = client.create_producer(topic=topic,
encryption_key="client-rsa.pem",
crypto_key_reader=crypto_key_reader)
producer.send('hello')
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.value(), 'hello')
consumer.unsubscribe()
client.close()
def test_producer_consumer_zstd(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-producer-consumer-zstd',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-producer-consumer-zstd',
compression_type=CompressionType.ZSTD)
producer.send(b'hello')
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
with self.assertRaises(pulsar.Timeout):
consumer.receive(100)
consumer.unsubscribe()
client.close()
class OAuth2Producer:
def __init__(self, args):
self.args = args
self.client = Client(args.service_url,
authentication=AuthenticationOauth2(
args.auth_params))
def produce(self):
p = self.client.create_producer(args.topic)
num = args.number
while num > 0:
msg = 'message {} from oauth2 producer'.format(args.number - num)
p.send(msg.encode('utf-8'))
print('Produce message \'{}\' to the pulsar service successfully.'.
format(msg))
num -= 1
def close(self):
self.client.close()
def test_async_func_with_custom_logger(self):
# boost::python::call may fail in C++ destructors, even worse, calls
# to PyErr_Print could corrupt the Python interpreter.
# See https://github.com/boostorg/python/issues/374 for details.
# This test is to verify these functions won't be called in C++ destructors
# so that Python's async function works well.
client = Client(self.serviceUrl,
logger=logging.getLogger('custom-logger'))
async def async_get(value):
consumer = client.subscribe('test_async_get', 'sub')
consumer.close()
return value
value = 'foo'
result = asyncio.run(async_get(value))
self.assertEqual(value, result)
client.close()
def test_producer_consumer(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('persistent://sample/standalone/ns/my-python-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-producer-consumer')
producer.send('hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_consumer_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'my-python-test-producer'
sub_name = 'my-sub-name'
self._check_value_error(lambda: client.subscribe(None, sub_name))
self._check_value_error(lambda: client.subscribe(topic, None))
self._check_value_error(
lambda: client.subscribe(topic, sub_name, consumer_type=None))
self._check_value_error(lambda: client.subscribe(
topic, sub_name, receiver_queue_size='test'))
self._check_value_error(
lambda: client.subscribe(topic, sub_name, consumer_name=5))
self._check_value_error(lambda: client.subscribe(
topic, sub_name, unacked_messages_timeout_ms='test'))
self._check_value_error(lambda: client.subscribe(
topic, sub_name, broker_consumer_stats_cache_time_ms='test'))
client.close()
def test_producer_send_async(self):
client = Client(self.serviceUrl)
producer = client.create_producer('my-python-topic')
sent_messages = []
def send_callback(producer, msg):
sent_messages.append(msg)
producer.send_async(b'hello', send_callback)
producer.send_async(b'hello', send_callback)
producer.send_async(b'hello', send_callback)
i = 0
while len(sent_messages) < 3 and i < 100:
time.sleep(0.1)
i += 1
self.assertEqual(len(sent_messages), 3)
client.close()
def test_topics_consumer(self):
client = Client(self.serviceUrl)
topic1 = 'persistent://public/default/my-python-topics-consumer-1'
topic2 = 'persistent://public/default/my-python-topics-consumer-2'
topic3 = 'persistent://public/default-2/my-python-topics-consumer-3' # topic from different namespace
topics = [topic1, topic2, topic3]
url1 = self.adminUrl + '/admin/v2/persistent/public/default/my-python-topics-consumer-1/partitions'
url2 = self.adminUrl + '/admin/v2/persistent/public/default/my-python-topics-consumer-2/partitions'
url3 = self.adminUrl + '/admin/v2/persistent/public/default-2/my-python-topics-consumer-3/partitions'
doHttpPut(url1, '2')
doHttpPut(url2, '3')
doHttpPut(url3, '4')
producer1 = client.create_producer(topic1)
producer2 = client.create_producer(topic2)
producer3 = client.create_producer(topic3)
consumer = client.subscribe(topics,
'my-topics-consumer-sub',
consumer_type=ConsumerType.Shared,
receiver_queue_size=10
)
for i in range(100):
producer1.send(b'hello-1-%d' % i)
for i in range(100):
producer2.send(b'hello-2-%d' % i)
for i in range(100):
producer3.send(b'hello-3-%d' % i)
for i in range(300):
msg = consumer.receive(TM)
consumer.acknowledge(msg)
with self.assertRaises(pulsar.Timeout):
consumer.receive(100)
client.close()
def test_deliver_at(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-deliver-at',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-deliver-at')
# Delay message in 1.1s
producer.send(b'hello', deliver_at=int(round(time.time() * 1000)) + 1100)
# Message should not be available in the next second
with self.assertRaises(pulsar.Timeout):
consumer.receive(1000)
# Message should be published now
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
consumer.unsubscribe()
producer.close()
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader('my-python-topic-reader-simple',
MessageId.earliest)
producer = client.create_producer('my-python-topic-reader-simple')
producer.send(b'hello')
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
reader.close()
client.close()
def test_get_partitioned_topic_name(self):
client = Client(self.serviceUrl)
url1 = self.adminUrl + '/admin/v2/persistent/public/default/partitioned_topic_name_test/partitions'
doHttpPut(url1, '3')
partitions = ['persistent://public/default/partitioned_topic_name_test-partition-0',
'persistent://public/default/partitioned_topic_name_test-partition-1',
'persistent://public/default/partitioned_topic_name_test-partition-2']
self.assertEqual(client.get_topic_partitions('persistent://public/default/partitioned_topic_name_test'),
partitions)
consumer = client.subscribe('persistent://public/default/partitioned_topic_name_test',
'partitioned_topic_name_test_sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://public/default/partitioned_topic_name_test')
producer.send(b'hello')
msg = consumer.receive(1000)
self.assertTrue(msg.topic_name() in partitions)
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-simple',
MessageId.earliest)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-simple')
producer.send('hello')
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
reader.close()
client.close()
def test_reader_on_last_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message')
for i in range(10):
producer.send('hello-%d' % i)
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message',
MessageId.latest)
for i in range(10, 20):
producer.send('hello-%d' % i)
for i in range(10, 20):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader.close()
client.close()
def test_reader_on_last_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer('my-python-topic-reader-on-last-message')
for i in range(10):
producer.send(b'hello-%d' % i)
reader = client.create_reader('my-python-topic-reader-on-last-message',
MessageId.latest)
for i in range(10, 20):
producer.send(b'hello-%d' % i)
for i in range(10, 20):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader.close()
client.close()
def test_deliver_after(self):
client = Client(self.serviceUrl)
consumer = client.subscribe("my-python-topic-deliver-after",
"my-sub",
consumer_type=ConsumerType.Shared)
producer = client.create_producer("my-python-topic-deliver-after")
# Delay message in 1.1s
producer.send(b"hello", deliver_after=timedelta(milliseconds=1100))
# Message should not be available in the next second
with self.assertRaises(pulsar.Timeout):
consumer.receive(1000)
# Message should be published in the next 500ms
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b"hello")
consumer.unsubscribe()
producer.close()
client.close()
def test_get_topics_partitions(self):
client = Client(self.serviceUrl)
topic_partitioned = "persistent://public/default/test_get_topics_partitions"
topic_non_partitioned = "persistent://public/default/test_get_topics_not-partitioned"
url1 = self.adminUrl + "/admin/v2/persistent/public/default/test_get_topics_partitions/partitions"
doHttpPut(url1, "3")
self.assertEqual(
client.get_topic_partitions(topic_partitioned),
[
"persistent://public/default/test_get_topics_partitions-partition-0",
"persistent://public/default/test_get_topics_partitions-partition-1",
"persistent://public/default/test_get_topics_partitions-partition-2",
],
)
self.assertEqual(client.get_topic_partitions(topic_non_partitioned),
[topic_non_partitioned])
client.close()
def _v2_topics(self, url):
client = Client(url)
consumer = client.subscribe('my-v2-topic-producer-consumer',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-v2-topic-producer-consumer')
producer.send('hello')
msg = consumer.receive(1000)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
consumer.acknowledge(msg)
try:
msg = consumer.receive(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
client.close()
def test_seek(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('persistent://sample/standalone/ns/my-python-topic-seek',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-seek')
for i in range(100):
producer.send('hello-%d' % i)
for i in range(100):
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-%d' % i)
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-0')
client.close()
def test_seek(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-seek',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-seek')
for i in range(100):
producer.send(b'hello-%d' % i)
for i in range(100):
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-%d' % i)
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive()
self.assertEqual(msg.data(), b'hello-0')
client.close()
def test_simple_producer(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic')
producer.send('hello')
producer.close()
client.close()
def test_producer_routing_mode(self):
client = Client(self.serviceUrl)
producer = client.create_producer('my-python-test-producer',
message_routing_mode=PartitionsRoutingMode.UseSinglePartition)
producer.send(b'test')
client.close()
