def test_reader_on_specific_message_with_batches(self): client = Client(self.serviceUrl) producer = client.create_producer( 'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches', batching_enabled=True, batching_max_publish_delay_ms=1000) for i in range(10): producer.send_async('hello-%d' % i, None) # Send one sync message to make sure everything was published producer.send('hello-10') reader1 = client.create_reader( 'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches', MessageId.earliest) for i in range(5): msg = reader1.read_next() last_msg_id = msg.message_id() reader2 = client.create_reader( 'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches', last_msg_id) for i in range(5, 11): msg = reader2.read_next() self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) reader1.close() reader2.close() client.close()
def test_reader_on_specific_message(self): client = Client(self.serviceUrl) producer = client.create_producer( 'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message') for i in range(10): producer.send('hello-%d' % i) reader1 = client.create_reader( 'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message', MessageId.earliest) for i in range(5): msg = reader1.read_next() last_msg_id = msg.message_id() reader2 = client.create_reader( 'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message', last_msg_id) for i in range(5, 10): msg = reader2.read_next() self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) reader1.close() reader2.close() client.close()
def test_reader_on_specific_message(self): num_of_msgs = 10 client = Client(self.serviceUrl) producer = client.create_producer( 'my-python-topic-reader-on-specific-message') for i in range(num_of_msgs): producer.send(b'hello-%d' % i) reader1 = client.create_reader( 'my-python-topic-reader-on-specific-message', MessageId.earliest) for i in range(num_of_msgs // 2): msg = reader1.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) last_msg_id = msg.message_id() last_msg_idx = i reader2 = client.create_reader( 'my-python-topic-reader-on-specific-message', last_msg_id) # The reset would be effectively done on the next position relative to reset. # When available, we should test this behaviour with `startMessageIdInclusive` opt. from_msg_idx = last_msg_idx for i in range(from_msg_idx, num_of_msgs): msg = reader2.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) reader1.close() reader2.close() client.close()
def test_reader_on_specific_message_with_batches(self): client = Client(self.serviceUrl) producer = client.create_producer( 'my-python-topic-reader-on-specific-message-with-batches', batching_enabled=True, batching_max_publish_delay_ms=1000) for i in range(10): producer.send_async(b'hello-%d' % i, None) # Send one sync message to make sure everything was published producer.send(b'hello-10') reader1 = client.create_reader( 'my-python-topic-reader-on-specific-message-with-batches', MessageId.earliest) for i in range(5): msg = reader1.read_next(TM) last_msg_id = msg.message_id() reader2 = client.create_reader( 'my-python-topic-reader-on-specific-message-with-batches', last_msg_id) for i in range(5, 11): msg = reader2.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) reader1.close() reader2.close() client.close()
def test_reader_on_specific_message(self): client = Client(self.serviceUrl) producer = client.create_producer( 'my-python-topic-reader-on-specific-message') for i in range(10): producer.send(b'hello-%d' % i) reader1 = client.create_reader( 'my-python-topic-reader-on-specific-message', MessageId.earliest) for i in range(5): msg = reader1.read_next() last_msg_id = msg.message_id() reader2 = client.create_reader( 'my-python-topic-reader-on-specific-message', last_msg_id) for i in range(5, 10): msg = reader2.read_next() self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) reader1.close() reader2.close() client.close()
def test_reader_has_message_available(self): # create client, producer, reader client = Client(self.serviceUrl) producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available') reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available', MessageId.latest) # before produce data, expected not has message available self.assertFalse(reader.has_message_available()); for i in range(10): producer.send('hello-%d' % i) # produced data, expected has message available self.assertTrue(reader.has_message_available()); for i in range(10): msg = reader.read_next() self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) # consumed all data, expected not has message available self.assertFalse(reader.has_message_available()); for i in range(10, 20): producer.send('hello-%d' % i) # produced data again, expected has message available self.assertTrue(reader.has_message_available()); reader.close() producer.close() client.close()
def test_reader_has_message_available(self): # create client, producer, reader client = Client(self.serviceUrl) producer = client.create_producer( 'my-python-topic-reader-has-message-available') reader = client.create_reader( 'my-python-topic-reader-has-message-available', MessageId.latest) # before produce data, expected not has message available self.assertFalse(reader.has_message_available()) for i in range(10): producer.send(b'hello-%d' % i) # produced data, expected has message available self.assertTrue(reader.has_message_available()) for i in range(10): msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) # consumed all data, expected not has message available self.assertFalse(reader.has_message_available()) for i in range(10, 20): producer.send(b'hello-%d' % i) # produced data again, expected has message available self.assertTrue(reader.has_message_available()) reader.close() producer.close() client.close()
def test_encryption(self): publicKeyPath = "/pulsar//pulsar-broker/src/test/resources/certificate/public-key.client-rsa.pem" privateKeyPath = "/pulsar/pulsar-broker/src/test/resources/certificate/private-key.client-rsa.pem" crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath) client = Client(self.serviceUrl) topic = "my-python-test-end-to-end-encryption" consumer = client.subscribe(topic=topic, subscription_name="my-subscription", crypto_key_reader=crypto_key_reader) producer = client.create_producer(topic=topic, encryption_key="client-rsa.pem", crypto_key_reader=crypto_key_reader) reader = client.create_reader(topic=topic, start_message_id=MessageId.earliest, crypto_key_reader=crypto_key_reader) producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.value(), b"hello") consumer.unsubscribe() msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): reader.read_next(100) reader.close() client.close()
def test_reader_is_connected(self): client = Client(self.serviceUrl) topic = "test_reader_is_connected" reader = client.create_reader(topic, MessageId.earliest) self.assertTrue(reader.is_connected()) reader.close() self.assertFalse(reader.is_connected()) client.close()
def test_reader_simple(self): client = Client(self.serviceUrl) reader = client.create_reader("my-python-topic-reader-simple", MessageId.earliest) producer = client.create_producer("my-python-topic-reader-simple") producer.send(b"hello") msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): reader.read_next(100) reader.close() client.close()
def test_reader_argument_errors(self): client = Client(self.serviceUrl) topic = 'persistent://sample/standalone/ns1/my-python-test-producer' # This should not raise exception client.create_reader(topic, MessageId.earliest) self._check_value_error(lambda: client.create_reader(None, MessageId.earliest)) self._check_value_error(lambda: client.create_reader(topic, None)) self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test')) self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5)) client.close()
def test_reader_argument_errors(self): client = Client(self.serviceUrl) topic = 'my-python-test-producer' # This should not raise exception client.create_reader(topic, MessageId.earliest) self._check_value_error(lambda: client.create_reader(None, MessageId.earliest)) self._check_value_error(lambda: client.create_reader(topic, None)) self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test')) self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5)) client.close()
def test_reader_simple(self): client = Client(self.serviceUrl) reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-simple', MessageId.earliest) producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-simple') producer.send('hello') msg = reader.read_next() self.assertTrue(msg) self.assertEqual(msg.data(), b'hello') try: msg = reader.read_next(100) self.assertTrue(False) # Should not reach this point except: pass # Exception is expected reader.close() client.close()
def test_reader_on_last_message(self): client = Client(self.serviceUrl) producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message') for i in range(10): producer.send('hello-%d' % i) reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message', MessageId.latest) for i in range(10, 20): producer.send('hello-%d' % i) for i in range(10, 20): msg = reader.read_next() self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) reader.close() client.close()
def test_reader_simple(self): client = Client(self.serviceUrl) reader = client.create_reader('my-python-topic-reader-simple', MessageId.earliest) producer = client.create_producer('my-python-topic-reader-simple') producer.send(b'hello') msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b'hello') try: msg = reader.read_next(100) self.assertTrue(False) # Should not reach this point except: pass # Exception is expected reader.close() client.close()
def test_reader_on_last_message(self): client = Client(self.serviceUrl) producer = client.create_producer('my-python-topic-reader-on-last-message') for i in range(10): producer.send(b'hello-%d' % i) reader = client.create_reader('my-python-topic-reader-on-last-message', MessageId.latest) for i in range(10, 20): producer.send(b'hello-%d' % i) for i in range(10, 20): msg = reader.read_next() self.assertTrue(msg) self.assertEqual(msg.data(), b'hello-%d' % i) reader.close() client.close()
def test_publish_compact_and_consume(self): client = Client(self.serviceUrl) topic = 'compaction_%s' % (uuid.uuid4()) producer = client.create_producer(topic, producer_name='my-producer-name', batching_enabled=False) self.assertEqual(producer.last_sequence_id(), -1) consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True) consumer.close() consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False) # producer create 2 messages with same key. producer.send(b'hello-0', partition_key='key0') producer.send(b'hello-1', partition_key='key0') producer.close() # issue compact command, and wait success url = '%s/admin/v2/persistent/public/default/%s/compaction' % ( self.adminUrl, topic) doHttpPut(url, '') while True: s = doHttpGet(url).decode('utf-8') if 'RUNNING' in s: print(s) print("Compact still running") time.sleep(0.2) else: print(s) print("Compact Complete now") self.assertTrue('SUCCESS' in s) break # after compaction completes the compacted ledger is recorded # as a property of a cursor. As persisting the cursor is async # and we don't wait for the acknowledgement of the acknowledgement, # there may be a race if we try to read the compacted ledger immediately. # therefore wait a second to allow the compacted ledger to be updated on # the broker. time.sleep(1.0) # after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key. consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True) msg0 = consumer1.receive(TM) self.assertEqual(msg0.data(), b'hello-1') consumer1.acknowledge(msg0) consumer1.close() # ditto for reader reader1 = client.create_reader(topic, MessageId.earliest, is_read_compacted=True) msg0 = reader1.read_next(TM) self.assertEqual(msg0.data(), b'hello-1') reader1.close() # after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key. msg0 = consumer2.receive(TM) self.assertEqual(msg0.data(), b'hello-0') consumer2.acknowledge(msg0) msg1 = consumer2.receive(TM) self.assertEqual(msg1.data(), b'hello-1') consumer2.acknowledge(msg1) consumer2.close() # ditto for reader reader2 = client.create_reader(topic, MessageId.earliest, is_read_compacted=False) msg0 = reader2.read_next(TM) self.assertEqual(msg0.data(), b'hello-0') msg1 = reader2.read_next(TM) self.assertEqual(msg1.data(), b'hello-1') reader2.close() client.close()
def test_seek(self): client = Client(self.serviceUrl) consumer = client.subscribe('my-python-topic-seek', 'my-sub', consumer_type=ConsumerType.Shared) producer = client.create_producer('my-python-topic-seek') for i in range(100): if i > 0: time.sleep(0.02) producer.send(b'hello-%d' % i) ids = [] timestamps = [] for i in range(100): msg = consumer.receive(TM) self.assertEqual(msg.data(), b'hello-%d' % i) ids.append(msg.message_id()) timestamps.append(msg.publish_timestamp()) consumer.acknowledge(msg) # seek, and after reconnect, expected receive first message. consumer.seek(MessageId.earliest) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b'hello-0') # seek on messageId consumer.seek(ids[50]) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b'hello-50') # ditto, but seek on timestamp consumer.seek(timestamps[42]) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b'hello-42') # repeat with reader reader = client.create_reader('my-python-topic-seek', MessageId.latest) try: msg = reader.read_next(100) self.assertTrue(False) # Should not reach this point except: pass # Exception is expected # earliest reader.seek(MessageId.earliest) time.sleep(0.5) msg = reader.read_next(TM) self.assertEqual(msg.data(), b'hello-0') msg = reader.read_next(TM) self.assertEqual(msg.data(), b'hello-1') # seek on messageId reader.seek(ids[33]) time.sleep(0.5) msg = reader.read_next(TM) self.assertEqual(msg.data(), b'hello-33') msg = reader.read_next(TM) self.assertEqual(msg.data(), b'hello-34') # seek on timestamp reader.seek(timestamps[79]) time.sleep(0.5) msg = reader.read_next(TM) self.assertEqual(msg.data(), b'hello-79') msg = reader.read_next(TM) self.assertEqual(msg.data(), b'hello-80') reader.close() client.close()
def test_seek(self): client = Client(self.serviceUrl) topic = "my-python-topic-seek-" + str(time.time()) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) for i in range(100): if i > 0: time.sleep(0.02) producer.send(b"hello-%d" % i) ids = [] timestamps = [] for i in range(100): msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-%d" % i) ids.append(msg.message_id()) timestamps.append(msg.publish_timestamp()) consumer.acknowledge(msg) # seek, and after reconnect, expected receive first message. consumer.seek(MessageId.earliest) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-0") # seek on messageId consumer.seek(ids[50]) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-50") # ditto, but seek on timestamp consumer.seek(timestamps[42]) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-42") # repeat with reader reader = client.create_reader(topic, MessageId.latest) with self.assertRaises(pulsar.Timeout): reader.read_next(100) # earliest reader.seek(MessageId.earliest) time.sleep(0.5) msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-0") msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-1") # seek on messageId reader.seek(ids[33]) time.sleep(0.5) msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-33") msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-34") # seek on timestamp reader.seek(timestamps[79]) time.sleep(0.5) msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-79") msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-80") reader.close() client.close()