def test_reader_on_specific_message_with_batches(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
batching_enabled=True,
batching_max_publish_delay_ms=1000)
for i in range(10):
producer.send_async('hello-%d' % i, None)
# Send one sync message to make sure everything was published
producer.send('hello-10')
reader1 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next()
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
last_msg_id)
for i in range(5, 11):
msg = reader2.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_reader_on_specific_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message')
for i in range(10):
producer.send('hello-%d' % i)
reader1 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next()
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
last_msg_id)
for i in range(5, 10):
msg = reader2.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_reader_on_specific_message(self):
num_of_msgs = 10
client = Client(self.serviceUrl)
producer = client.create_producer(
'my-python-topic-reader-on-specific-message')
for i in range(num_of_msgs):
producer.send(b'hello-%d' % i)
reader1 = client.create_reader(
'my-python-topic-reader-on-specific-message', MessageId.earliest)
for i in range(num_of_msgs // 2):
msg = reader1.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
last_msg_id = msg.message_id()
last_msg_idx = i
reader2 = client.create_reader(
'my-python-topic-reader-on-specific-message', last_msg_id)
# The reset would be effectively done on the next position relative to reset.
# When available, we should test this behaviour with `startMessageIdInclusive` opt.
from_msg_idx = last_msg_idx
for i in range(from_msg_idx, num_of_msgs):
msg = reader2.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_reader_on_specific_message_with_batches(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'my-python-topic-reader-on-specific-message-with-batches',
batching_enabled=True,
batching_max_publish_delay_ms=1000)
for i in range(10):
producer.send_async(b'hello-%d' % i, None)
# Send one sync message to make sure everything was published
producer.send(b'hello-10')
reader1 = client.create_reader(
'my-python-topic-reader-on-specific-message-with-batches',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next(TM)
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'my-python-topic-reader-on-specific-message-with-batches',
last_msg_id)
for i in range(5, 11):
msg = reader2.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_reader_on_specific_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer(
'my-python-topic-reader-on-specific-message')
for i in range(10):
producer.send(b'hello-%d' % i)
reader1 = client.create_reader(
'my-python-topic-reader-on-specific-message',
MessageId.earliest)
for i in range(5):
msg = reader1.read_next()
last_msg_id = msg.message_id()
reader2 = client.create_reader(
'my-python-topic-reader-on-specific-message',
last_msg_id)
for i in range(5, 10):
msg = reader2.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader1.close()
reader2.close()
client.close()
def test_reader_has_message_available(self):
# create client, producer, reader
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available')
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available',
MessageId.latest)
# before produce data, expected not has message available
self.assertFalse(reader.has_message_available());
for i in range(10):
producer.send('hello-%d' % i)
# produced data, expected has message available
self.assertTrue(reader.has_message_available());
for i in range(10):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
# consumed all data, expected not has message available
self.assertFalse(reader.has_message_available());
for i in range(10, 20):
producer.send('hello-%d' % i)
# produced data again, expected has message available
self.assertTrue(reader.has_message_available());
reader.close()
producer.close()
client.close()
def test_reader_has_message_available(self):
# create client, producer, reader
client = Client(self.serviceUrl)
producer = client.create_producer(
'my-python-topic-reader-has-message-available')
reader = client.create_reader(
'my-python-topic-reader-has-message-available', MessageId.latest)
# before produce data, expected not has message available
self.assertFalse(reader.has_message_available())
for i in range(10):
producer.send(b'hello-%d' % i)
# produced data, expected has message available
self.assertTrue(reader.has_message_available())
for i in range(10):
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
# consumed all data, expected not has message available
self.assertFalse(reader.has_message_available())
for i in range(10, 20):
producer.send(b'hello-%d' % i)
# produced data again, expected has message available
self.assertTrue(reader.has_message_available())
reader.close()
producer.close()
client.close()
def test_encryption(self):
publicKeyPath = "/pulsar//pulsar-broker/src/test/resources/certificate/public-key.client-rsa.pem"
privateKeyPath = "/pulsar/pulsar-broker/src/test/resources/certificate/private-key.client-rsa.pem"
crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath)
client = Client(self.serviceUrl)
topic = "my-python-test-end-to-end-encryption"
consumer = client.subscribe(topic=topic,
subscription_name="my-subscription",
crypto_key_reader=crypto_key_reader)
producer = client.create_producer(topic=topic,
encryption_key="client-rsa.pem",
crypto_key_reader=crypto_key_reader)
reader = client.create_reader(topic=topic,
start_message_id=MessageId.earliest,
crypto_key_reader=crypto_key_reader)
producer.send(b"hello")
msg = consumer.receive(TM)
self.assertTrue(msg)
self.assertEqual(msg.value(), b"hello")
consumer.unsubscribe()
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b"hello")
with self.assertRaises(pulsar.Timeout):
reader.read_next(100)
reader.close()
client.close()
def test_reader_is_connected(self):
client = Client(self.serviceUrl)
topic = "test_reader_is_connected"
reader = client.create_reader(topic, MessageId.earliest)
self.assertTrue(reader.is_connected())
reader.close()
self.assertFalse(reader.is_connected())
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader("my-python-topic-reader-simple",
MessageId.earliest)
producer = client.create_producer("my-python-topic-reader-simple")
producer.send(b"hello")
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b"hello")
with self.assertRaises(pulsar.Timeout):
reader.read_next(100)
reader.close()
client.close()
def test_reader_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
# This should not raise exception
client.create_reader(topic, MessageId.earliest)
self._check_value_error(lambda: client.create_reader(None, MessageId.earliest))
self._check_value_error(lambda: client.create_reader(topic, None))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test'))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5))
client.close()
def test_reader_argument_errors(self):
client = Client(self.serviceUrl)
topic = 'my-python-test-producer'
# This should not raise exception
client.create_reader(topic, MessageId.earliest)
self._check_value_error(lambda: client.create_reader(None, MessageId.earliest))
self._check_value_error(lambda: client.create_reader(topic, None))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test'))
self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5))
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-simple',
MessageId.earliest)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-simple')
producer.send('hello')
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
reader.close()
client.close()
def test_reader_on_last_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message')
for i in range(10):
producer.send('hello-%d' % i)
reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message',
MessageId.latest)
for i in range(10, 20):
producer.send('hello-%d' % i)
for i in range(10, 20):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader.close()
client.close()
def test_reader_simple(self):
client = Client(self.serviceUrl)
reader = client.create_reader('my-python-topic-reader-simple',
MessageId.earliest)
producer = client.create_producer('my-python-topic-reader-simple')
producer.send(b'hello')
msg = reader.read_next(TM)
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello')
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
reader.close()
client.close()
def test_reader_on_last_message(self):
client = Client(self.serviceUrl)
producer = client.create_producer('my-python-topic-reader-on-last-message')
for i in range(10):
producer.send(b'hello-%d' % i)
reader = client.create_reader('my-python-topic-reader-on-last-message',
MessageId.latest)
for i in range(10, 20):
producer.send(b'hello-%d' % i)
for i in range(10, 20):
msg = reader.read_next()
self.assertTrue(msg)
self.assertEqual(msg.data(), b'hello-%d' % i)
reader.close()
client.close()
def test_publish_compact_and_consume(self):
client = Client(self.serviceUrl)
topic = 'compaction_%s' % (uuid.uuid4())
producer = client.create_producer(topic,
producer_name='my-producer-name',
batching_enabled=False)
self.assertEqual(producer.last_sequence_id(), -1)
consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
consumer.close()
consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False)
# producer create 2 messages with same key.
producer.send(b'hello-0', partition_key='key0')
producer.send(b'hello-1', partition_key='key0')
producer.close()
# issue compact command, and wait success
url = '%s/admin/v2/persistent/public/default/%s/compaction' % (
self.adminUrl, topic)
doHttpPut(url, '')
while True:
s = doHttpGet(url).decode('utf-8')
if 'RUNNING' in s:
print(s)
print("Compact still running")
time.sleep(0.2)
else:
print(s)
print("Compact Complete now")
self.assertTrue('SUCCESS' in s)
break
# after compaction completes the compacted ledger is recorded
# as a property of a cursor. As persisting the cursor is async
# and we don't wait for the acknowledgement of the acknowledgement,
# there may be a race if we try to read the compacted ledger immediately.
# therefore wait a second to allow the compacted ledger to be updated on
# the broker.
time.sleep(1.0)
# after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key.
consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
msg0 = consumer1.receive(TM)
self.assertEqual(msg0.data(), b'hello-1')
consumer1.acknowledge(msg0)
consumer1.close()
# ditto for reader
reader1 = client.create_reader(topic,
MessageId.earliest,
is_read_compacted=True)
msg0 = reader1.read_next(TM)
self.assertEqual(msg0.data(), b'hello-1')
reader1.close()
# after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key.
msg0 = consumer2.receive(TM)
self.assertEqual(msg0.data(), b'hello-0')
consumer2.acknowledge(msg0)
msg1 = consumer2.receive(TM)
self.assertEqual(msg1.data(), b'hello-1')
consumer2.acknowledge(msg1)
consumer2.close()
# ditto for reader
reader2 = client.create_reader(topic,
MessageId.earliest,
is_read_compacted=False)
msg0 = reader2.read_next(TM)
self.assertEqual(msg0.data(), b'hello-0')
msg1 = reader2.read_next(TM)
self.assertEqual(msg1.data(), b'hello-1')
reader2.close()
client.close()
def test_seek(self):
client = Client(self.serviceUrl)
consumer = client.subscribe('my-python-topic-seek',
'my-sub',
consumer_type=ConsumerType.Shared)
producer = client.create_producer('my-python-topic-seek')
for i in range(100):
if i > 0:
time.sleep(0.02)
producer.send(b'hello-%d' % i)
ids = []
timestamps = []
for i in range(100):
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-%d' % i)
ids.append(msg.message_id())
timestamps.append(msg.publish_timestamp())
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-0')
# seek on messageId
consumer.seek(ids[50])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-50')
# ditto, but seek on timestamp
consumer.seek(timestamps[42])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b'hello-42')
# repeat with reader
reader = client.create_reader('my-python-topic-seek', MessageId.latest)
try:
msg = reader.read_next(100)
self.assertTrue(False) # Should not reach this point
except:
pass # Exception is expected
# earliest
reader.seek(MessageId.earliest)
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-0')
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-1')
# seek on messageId
reader.seek(ids[33])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-33')
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-34')
# seek on timestamp
reader.seek(timestamps[79])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-79')
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b'hello-80')
reader.close()
client.close()
def test_seek(self):
client = Client(self.serviceUrl)
topic = "my-python-topic-seek-" + str(time.time())
consumer = client.subscribe(topic,
"my-sub",
consumer_type=ConsumerType.Shared)
producer = client.create_producer(topic)
for i in range(100):
if i > 0:
time.sleep(0.02)
producer.send(b"hello-%d" % i)
ids = []
timestamps = []
for i in range(100):
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-%d" % i)
ids.append(msg.message_id())
timestamps.append(msg.publish_timestamp())
consumer.acknowledge(msg)
# seek, and after reconnect, expected receive first message.
consumer.seek(MessageId.earliest)
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-0")
# seek on messageId
consumer.seek(ids[50])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-50")
# ditto, but seek on timestamp
consumer.seek(timestamps[42])
time.sleep(0.5)
msg = consumer.receive(TM)
self.assertEqual(msg.data(), b"hello-42")
# repeat with reader
reader = client.create_reader(topic, MessageId.latest)
with self.assertRaises(pulsar.Timeout):
reader.read_next(100)
# earliest
reader.seek(MessageId.earliest)
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-0")
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-1")
# seek on messageId
reader.seek(ids[33])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-33")
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-34")
# seek on timestamp
reader.seek(timestamps[79])
time.sleep(0.5)
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-79")
msg = reader.read_next(TM)
self.assertEqual(msg.data(), b"hello-80")
reader.close()
client.close()
