Ejemplo n.º 1
0
    def test_tls_auth3(self):
        certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/'
        if not os.path.exists(certs_dir):
            certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/"
        authPlugin = "tls"
        authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (certs_dir, certs_dir)

        client = Client(self.serviceUrlTls,
                        tls_trust_certs_file_path=certs_dir + 'cacert.pem',
                        tls_allow_insecure_connection=False,
                        authentication=Authentication(authPlugin, authParams))

        consumer = client.subscribe('persistent://property/cluster/namespace/my-python-topic-producer-consumer',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('persistent://property/cluster/namespace/my-python-topic-producer-consumer')
        producer.send('hello')

        msg = consumer.receive(1000)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')

        try:
            msg = consumer.receive(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        client.close()
Ejemplo n.º 2
0
    def test_reader_has_message_available(self):
        # create client, producer, reader
        client = Client(self.serviceUrl)
        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available')
        reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-has-message-available',
                                      MessageId.latest)

        # before produce data, expected not has message available
        self.assertFalse(reader.has_message_available());

        for i in range(10):
            producer.send('hello-%d' % i)

        # produced data, expected has message available
        self.assertTrue(reader.has_message_available());

        for i in range(10):
            msg = reader.read_next()
            self.assertTrue(msg)
            self.assertEqual(msg.data(), b'hello-%d' % i)

        # consumed all data, expected not has message available
        self.assertFalse(reader.has_message_available());

        for i in range(10, 20):
            producer.send('hello-%d' % i)

        # produced data again, expected has message available
        self.assertTrue(reader.has_message_available());
        reader.close()
        producer.close()
        client.close()
Ejemplo n.º 3
0
    def test_producer_deduplication(self):
        # Enable deduplication on namespace
        doHttpPost(self.adminUrl + '/admin/namespaces/sample/standalone/ns1/deduplication',
                   'true')
        client = Client(self.serviceUrl)

        topic = 'persistent://sample/standalone/ns1/my-python-test-producer-deduplication-' + str(time.time())

        producer = client.create_producer(topic, producer_name='my-producer-name')
        self.assertEqual(producer.last_sequence_id(), -1)

        consumer = client.subscribe(topic, 'my-sub')

        producer.send('hello-0', sequence_id=0)
        producer.send('hello-1', sequence_id=1)
        producer.send('hello-2', sequence_id=2)
        self.assertEqual(producer.last_sequence_id(), 2)

        # Repeat the messages and verify they're not received by consumer
        producer.send('hello-1', sequence_id=1)
        producer.send('hello-2', sequence_id=2)
        self.assertEqual(producer.last_sequence_id(), 2)

        for i in range(3):
            msg = consumer.receive()
            self.assertEqual(msg.data(), b'hello-%d' % i)
            consumer.acknowledge(msg)

        try:
            # No other messages should be received
            consumer.receive(timeout_millis=1000)
            self.assertTrue(False)
        except:
            # Exception is expected
            pass

        producer.close()

        producer = client.create_producer(topic, producer_name='my-producer-name')
        self.assertEqual(producer.last_sequence_id(), 2)

        # Repeat the messages and verify they're not received by consumer
        producer.send('hello-1', sequence_id=1)
        producer.send('hello-2', sequence_id=2)
        self.assertEqual(producer.last_sequence_id(), 2)

        try:
            # No other messages should be received
            consumer.receive(timeout_millis=1000)
            self.assertTrue(False)
        except:
            # Exception is expected
            pass
Ejemplo n.º 4
0
    def test_consumer_argument_errors(self):
        client = Client(self.serviceUrl)

        topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
        sub_name = 'my-sub-name'

        self._check_value_error(lambda: client.subscribe(None, sub_name))
        self._check_value_error(lambda: client.subscribe(topic, None))
        self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_type=None))
        self._check_value_error(lambda: client.subscribe(topic, sub_name, receiver_queue_size='test'))
        self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_name=5))
        self._check_value_error(lambda: client.subscribe(topic, sub_name, unacked_messages_timeout_ms='test'))
        self._check_value_error(lambda: client.subscribe(topic, sub_name, broker_consumer_stats_cache_time_ms='test'))
        client.close()
Ejemplo n.º 5
0
    def test_message_argument_errors(self):
        client = Client(self.serviceUrl)
        topic = 'persistent://sample/standalone/ns1/my-python-test-producer'
        producer = client.create_producer(topic)

        content = 'test'.encode('utf-8')

        self._check_value_error(lambda: producer.send(5))
        self._check_value_error(lambda: producer.send(content, properties='test'))
        self._check_value_error(lambda: producer.send(content, partition_key=5))
        self._check_value_error(lambda: producer.send(content, sequence_id='test'))
        self._check_value_error(lambda: producer.send(content, replication_clusters=5))
        self._check_value_error(lambda: producer.send(content, disable_replication='test'))
        client.close()
Ejemplo n.º 6
0
    def test_reader_argument_errors(self):
        client = Client(self.serviceUrl)
        topic = 'persistent://sample/standalone/ns1/my-python-test-producer'

        # This should not raise exception
        client.create_reader(topic, MessageId.earliest)

        self._check_value_error(lambda: client.create_reader(None, MessageId.earliest))
        self._check_value_error(lambda: client.create_reader(topic, None))
        self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size='test'))
        self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5))
        client.close()
Ejemplo n.º 7
0
 def test_auth_junk_params(self):
     certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/'
     if not os.path.exists(certs_dir):
         certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/"
     authPlugin = "someoldjunk.so"
     authParams = "blah"
     client = Client(self.serviceUrlTls,
                     tls_trust_certs_file_path=certs_dir + 'cacert.pem',
                     tls_allow_insecure_connection=False,
                     authentication=Authentication(authPlugin, authParams))
     try:
         client.subscribe('persistent://property/cluster/namespace/my-python-topic-producer-consumer',
                          'my-sub',
                          consumer_type=ConsumerType.Shared)
     except:
         pass  # Exception is expected
Ejemplo n.º 8
0
    def test_producer_send_async(self):
        client = Client(self.serviceUrl)
        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic')

        sent_messages = []

        def send_callback(producer, msg):
            sent_messages.append(msg)

        producer.send_async('hello', send_callback)
        producer.send_async('hello', send_callback)
        producer.send_async('hello', send_callback)

        time.sleep(0.1)
        self.assertEqual(len(sent_messages), 3)
        client.close()
Ejemplo n.º 9
0
    def test_producer_consumer(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('persistent://sample/standalone/ns/my-python-topic-producer-consumer',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-producer-consumer')
        producer.send('hello')

        msg = consumer.receive(1000)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')

        try:
            msg = consumer.receive(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        client.close()
Ejemplo n.º 10
0
    def test_reader_on_last_message(self):
        client = Client(self.serviceUrl)
        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message')

        for i in range(10):
            producer.send('hello-%d' % i)

        reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message',
                                      MessageId.latest)

        for i in range(10, 20):
            producer.send('hello-%d' % i)

        for i in range(10, 20):
            msg = reader.read_next()
            self.assertTrue(msg)
            self.assertEqual(msg.data(), b'hello-%d' % i)

        reader.close()
        client.close()
Ejemplo n.º 11
0
    def _v2_topics(self, url):
        client = Client(url)
        consumer = client.subscribe('my-v2-topic-producer-consumer',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-v2-topic-producer-consumer')
        producer.send('hello')

        msg = consumer.receive(1000)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')
        consumer.acknowledge(msg)

        try:
            msg = consumer.receive(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        client.close()
Ejemplo n.º 12
0
    def test_reader_simple(self):
        client = Client(self.serviceUrl)
        reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-simple',
                                      MessageId.earliest)

        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-simple')
        producer.send('hello')

        msg = reader.read_next()
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')

        try:
            msg = reader.read_next(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        reader.close()
        client.close()
Ejemplo n.º 13
0
    def test_seek(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('persistent://sample/standalone/ns/my-python-topic-seek',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-seek')

        for i in range(100):
            producer.send('hello-%d' % i)

        for i in range(100):
            msg = consumer.receive()
            self.assertEqual(msg.data(), b'hello-%d' % i)
            consumer.acknowledge(msg)

        # seek, and after reconnect, expected receive first message.
        consumer.seek(MessageId.earliest)
        time.sleep(0.5)
        msg = consumer.receive()
        self.assertEqual(msg.data(), b'hello-0')
        client.close()
Ejemplo n.º 14
0
    def test_reader_on_specific_message_with_batches(self):
        client = Client(self.serviceUrl)
        producer = client.create_producer(
            'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
            batching_enabled=True,
            batching_max_publish_delay_ms=1000)

        for i in range(10):
            producer.send_async('hello-%d' % i, None)

        # Send one sync message to make sure everything was published
        producer.send('hello-10')

        reader1 = client.create_reader(
                'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
                MessageId.earliest)

        for i in range(5):
            msg = reader1.read_next()
            last_msg_id = msg.message_id()

        reader2 = client.create_reader(
                'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message-with-batches',
                last_msg_id)

        for i in range(5, 11):
            msg = reader2.read_next()
            self.assertTrue(msg)
            self.assertEqual(msg.data(), b'hello-%d' % i)

        reader1.close()
        reader2.close()
        client.close()
Ejemplo n.º 15
0
    def test_reader_on_specific_message(self):
        client = Client(self.serviceUrl)
        producer = client.create_producer(
            'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message')

        for i in range(10):
            producer.send('hello-%d' % i)

        reader1 = client.create_reader(
                'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
                MessageId.earliest)

        for i in range(5):
            msg = reader1.read_next()
            last_msg_id = msg.message_id()

        reader2 = client.create_reader(
                'persistent://sample/standalone/ns/my-python-topic-reader-on-specific-message',
                last_msg_id)

        for i in range(5, 10):
            msg = reader2.read_next()
            self.assertTrue(msg)
            self.assertEqual(msg.data(), b'hello-%d' % i)

        reader1.close()
        reader2.close()
        client.close()
Ejemplo n.º 16
0
    def test_message_listener(self):
        client = Client(self.serviceUrl)

        received_messages = []

        def listener(consumer, msg):
            print("Got message: %s" % msg)
            received_messages.append(msg)
            consumer.acknowledge(msg)

        client.subscribe('persistent://sample/standalone/ns/my-python-topic-listener',
                         'my-sub',
                         consumer_type=ConsumerType.Exclusive,
                         message_listener=listener)
        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-listener')
        producer.send('hello-1')
        producer.send('hello-2')
        producer.send('hello-3')

        time.sleep(0.1)
        self.assertEqual(len(received_messages), 3)
        self.assertEqual(received_messages[0].data(), b"hello-1")
        self.assertEqual(received_messages[1].data(), b"hello-2")
        self.assertEqual(received_messages[2].data(), b"hello-3")
        client.close()
Ejemplo n.º 17
0
    def test_producer_sequence_after_reconnection(self):
        # Enable deduplication on namespace
        doHttpPost(self.adminUrl + '/admin/namespaces/sample/standalone/ns1/deduplication',
                   'true')
        client = Client(self.serviceUrl)

        topic = 'persistent://sample/standalone/ns1/my-python-test-producer-sequence-after-reconnection-' + str(time.time())

        producer = client.create_producer(topic, producer_name='my-producer-name')
        self.assertEqual(producer.last_sequence_id(), -1)

        for i in range(10):
            producer.send('hello-%d' % i)
            self.assertEqual(producer.last_sequence_id(), i)

        producer.close()

        producer = client.create_producer(topic, producer_name='my-producer-name')
        self.assertEqual(producer.last_sequence_id(), 9)

        for i in range(10, 20):
            producer.send('hello-%d' % i)
            self.assertEqual(producer.last_sequence_id(), i)
Ejemplo n.º 18
0
    def test_topics_pattern_consumer(self):
        import re
        client = Client(self.serviceUrl)

        topics_pattern = 'persistent://sample/standalone/ns/my-python-pattern-consumer.*'

        topic1 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-1'
        topic2 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-2'
        topic3 = 'persistent://sample/standalone/ns/my-python-pattern-consumer-3'

        url1 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-1/partitions'
        url2 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-2/partitions'
        url3 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-pattern-consumer-3/partitions'

        doHttpPut(url1, '2')
        doHttpPut(url2, '3')
        doHttpPut(url3, '4')

        producer1 = client.create_producer(topic1)
        producer2 = client.create_producer(topic2)
        producer3 = client.create_producer(topic3)

        consumer = client.subscribe(re.compile(topics_pattern),
                                    'my-pattern-consumer-sub',
                                    consumer_type = ConsumerType.Shared,
                                    receiver_queue_size = 10,
                                    pattern_auto_discovery_period = 1
                                   )

        # wait enough time to trigger auto discovery
        time.sleep(2)

        for i in range(100):
            producer1.send('hello-1-%d' % i)

        for i in range(100):
            producer2.send('hello-2-%d' % i)

        for i in range(100):
            producer3.send('hello-3-%d' % i)


        for i in range(300):
            msg = consumer.receive()
            consumer.acknowledge(msg)

        try:
            # No other messages should be received
            consumer.receive(timeout_millis=500)
            self.assertTrue(False)
        except:
            # Exception is expected
            pass
        client.close()
Ejemplo n.º 19
0
    def test_publish_compact_and_consume(self):
        client = Client(self.serviceUrl)
        topic = 'persistent://sample/standalone/ns1/my-python-test_publish_compact_and_consume'
        producer = client.create_producer(topic, producer_name='my-producer-name', batching_enabled=False)
        self.assertEqual(producer.last_sequence_id(), -1)
        consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
        consumer.close()
        consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False)

        # producer create 2 messages with same key.
        producer.send('hello-0', partition_key='key0')
        producer.send('hello-1', partition_key='key0')
        producer.close()

        # issue compact command, and wait success
        url=self.adminUrl + '/admin/persistent/sample/standalone/ns1/my-python-test_publish_compact_and_consume/compaction'
        doHttpPut(url, '')
        while True:
            req = urllib2.Request(url)
            response = urllib2.urlopen(req)
            s=response.read()
            if 'RUNNING' in s:
                print("Compact still running")
                print(s)
                time.sleep(0.2)
            else:
                self.assertTrue('SUCCESS' in s)
                print("Compact Complete now")
                print(s)
                break

        # after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key.
        consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
        msg0 = consumer1.receive()
        self.assertEqual(msg0.data(), b'hello-1')
        consumer1.acknowledge(msg0)
        consumer1.close()

        # after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key.
        msg0 = consumer2.receive()
        self.assertEqual(msg0.data(), b'hello-0')
        consumer2.acknowledge(msg0)
        msg1 = consumer2.receive()
        self.assertEqual(msg1.data(), b'hello-1')
        consumer2.acknowledge(msg1)
        consumer2.close()
        client.close()
Ejemplo n.º 20
0
    def test_topics_consumer(self):
        client = Client(self.serviceUrl)
        topic1 = 'persistent://sample/standalone/ns/my-python-topics-consumer-1'
        topic2 = 'persistent://sample/standalone/ns/my-python-topics-consumer-2'
        topic3 = 'persistent://sample/standalone/ns/my-python-topics-consumer-3'
        topics = [topic1, topic2, topic3]

        url1 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-1/partitions'
        url2 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-2/partitions'
        url3 = self.adminUrl + '/admin/persistent/sample/standalone/ns/my-python-topics-consumer-3/partitions'

        doHttpPut(url1, '2')
        doHttpPut(url2, '3')
        doHttpPut(url3, '4')

        producer1 = client.create_producer(topic1)
        producer2 = client.create_producer(topic2)
        producer3 = client.create_producer(topic3)

        consumer = client.subscribe(topics,
                                    'my-topics-consumer-sub',
                                    consumer_type=ConsumerType.Shared,
                                    receiver_queue_size=10
                                    )

        for i in range(100):
            producer1.send('hello-1-%d' % i)

        for i in range(100):
            producer2.send('hello-2-%d' % i)

        for i in range(100):
            producer3.send('hello-3-%d' % i)


        for i in range(300):
            msg = consumer.receive()
            consumer.acknowledge(msg)

        try:
        # No other messages should be received
            consumer.receive(timeout_millis=500)
            self.assertTrue(False)
        except:
            # Exception is expected
            pass
        client.close()
Ejemplo n.º 21
0
    def test_get_topics_partitions(self):
        client = Client(self.serviceUrl)
        topic_partitioned = "persistent://public/default/test_get_topics_partitions"
        topic_non_partitioned = "persistent://public/default/test_get_topics_not-partitioned"

        url1 = self.adminUrl + "/admin/v2/persistent/public/default/test_get_topics_partitions/partitions"
        doHttpPut(url1, "3")

        self.assertEqual(
            client.get_topic_partitions(topic_partitioned),
            [
                "persistent://public/default/test_get_topics_partitions-partition-0",
                "persistent://public/default/test_get_topics_partitions-partition-1",
                "persistent://public/default/test_get_topics_partitions-partition-2",
            ],
        )

        self.assertEqual(client.get_topic_partitions(topic_non_partitioned),
                         [topic_non_partitioned])
        client.close()
Ejemplo n.º 22
0
    def test_producer_consumer(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('my-python-topic-producer-consumer',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-python-topic-producer-consumer')
        producer.send(b'hello')

        msg = consumer.receive(1000)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')

        try:
            msg = consumer.receive(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        consumer.unsubscribe()
        client.close()
Ejemplo n.º 23
0
    def _v2_topics(self, url):
        client = Client(url)
        consumer = client.subscribe('my-v2-topic-producer-consumer',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-v2-topic-producer-consumer')
        producer.send(b'hello')

        msg = consumer.receive(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')
        consumer.acknowledge(msg)

        try:
            msg = consumer.receive(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        client.close()
Ejemplo n.º 24
0
    def test_reader_simple(self):
        client = Client(self.serviceUrl)
        reader = client.create_reader('my-python-topic-reader-simple',
                                      MessageId.earliest)

        producer = client.create_producer('my-python-topic-reader-simple')
        producer.send(b'hello')

        msg = reader.read_next(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')

        try:
            msg = reader.read_next(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        reader.close()
        client.close()
Ejemplo n.º 25
0
    def test_token_auth(self):
        with open('/tmp/pulsar-test-data/tokens/token.txt') as tf:
            token = tf.read().strip()

        # Use adminUrl to test both HTTP request and binary protocol
        client = Client(self.adminUrl,
                        authentication=AuthenticationToken(token))

        consumer = client.subscribe(
            'persistent://private/auth/my-python-topic-token-auth',
            'my-sub',
            consumer_type=ConsumerType.Shared)
        producer = client.create_producer(
            'persistent://private/auth/my-python-topic-token-auth')
        producer.send(b'hello')

        msg = consumer.receive(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')
        client.close()
Ejemplo n.º 26
0
    def test_deliver_after(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe("my-python-topic-deliver-after",
                                    "my-sub",
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer("my-python-topic-deliver-after")
        # Delay message in 1.1s
        producer.send(b"hello", deliver_after=timedelta(milliseconds=1100))

        # Message should not be available in the next second
        with self.assertRaises(pulsar.Timeout):
            consumer.receive(1000)

        # Message should be published in the next 500ms
        msg = consumer.receive(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b"hello")
        consumer.unsubscribe()
        producer.close()
        client.close()
Ejemplo n.º 27
0
    def test_reader_on_last_message(self):
        client = Client(self.serviceUrl)
        producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message')

        for i in range(10):
            producer.send('hello-%d' % i)

        reader = client.create_reader('persistent://sample/standalone/ns/my-python-topic-reader-on-last-message',
                                      MessageId.latest)

        for i in range(10, 20):
            producer.send('hello-%d' % i)

        for i in range(10, 20):
            msg = reader.read_next()
            self.assertTrue(msg)
            self.assertEqual(msg.data(), b'hello-%d' % i)

        reader.close()
        client.close()
Ejemplo n.º 28
0
    def test_deliver_at(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('my-python-topic-deliver-at',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-python-topic-deliver-at')
        # Delay message in 1.1s
        producer.send(b'hello',
                      deliver_at=int(round(time.time() * 1000)) + 1100)

        # Message should not be available in the next second
        with self.assertRaises(pulsar.Timeout):
            consumer.receive(1000)

        # Message should be published now
        msg = consumer.receive(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')
        consumer.unsubscribe()
        producer.close()
        client.close()
Ejemplo n.º 29
0
    def test_reader_on_last_message(self):
        client = Client(self.serviceUrl)
        producer = client.create_producer(
            "my-python-topic-reader-on-last-message")

        for i in range(10):
            producer.send(b"hello-%d" % i)

        reader = client.create_reader("my-python-topic-reader-on-last-message",
                                      MessageId.latest)

        for i in range(10, 20):
            producer.send(b"hello-%d" % i)

        for i in range(10, 20):
            msg = reader.read_next(TM)
            self.assertTrue(msg)
            self.assertEqual(msg.data(), b"hello-%d" % i)

        reader.close()
        client.close()
Ejemplo n.º 30
0
    def test_seek(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('my-python-topic-seek',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-python-topic-seek')

        for i in range(100):
            producer.send(b'hello-%d' % i)

        for i in range(100):
            msg = consumer.receive()
            self.assertEqual(msg.data(), b'hello-%d' % i)
            consumer.acknowledge(msg)

        # seek, and after reconnect, expected receive first message.
        consumer.seek(MessageId.earliest)
        time.sleep(0.5)
        msg = consumer.receive()
        self.assertEqual(msg.data(), b'hello-0')
        client.close()
Ejemplo n.º 31
0
    def test_producer_consumer(self):
        client = Client('pulsar://localhost:6650/')
        consumer = client.subscribe(
            'persistent://sample/standalone/ns/my-python-topic-producer-consumer',
            'my-sub',
            consumer_type=ConsumerType.Shared)
        producer = client.create_producer(
            'persistent://sample/standalone/ns/my-python-topic-producer-consumer'
        )
        producer.send('hello')

        msg = consumer.receive(1000)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), 'hello')

        try:
            msg = consumer.receive(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        client.close()
Ejemplo n.º 32
0
    def test_redelivery_count(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('my-python-topic-redelivery-count',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared,
                                    negative_ack_redelivery_delay_ms=500)
        producer = client.create_producer('my-python-topic-redelivery-count')
        producer.send(b'hello')

        redelivery_count = 0
        for i in range(4):
            msg = consumer.receive(TM)
            print("Received message %s" % msg.data())
            consumer.negative_acknowledge(msg)
            redelivery_count = msg.redelivery_count()

        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')
        self.assertEqual(3, redelivery_count)
        consumer.unsubscribe()
        producer.close()
        client.close()
Ejemplo n.º 33
0
    def test_deliver_after(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('my-python-topic-deliver-after',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-python-topic-deliver-after')
        # Delay message in 1.1s
        producer.send(b'hello', deliver_after=timedelta(milliseconds=1100))

        # Message should not be available in the next second
        try:
            msg = consumer.receive(1000)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        # Message should be published in the next 500ms
        msg = consumer.receive(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')
        consumer.unsubscribe()
        producer.close()
        client.close()
Ejemplo n.º 34
0
    def test_tls_auth(self):
        certs_dir = '/pulsar/pulsar-broker/src/test/resources/authentication/tls/'
        if not os.path.exists(certs_dir):
            certs_dir = "../../pulsar-broker/src/test/resources/authentication/tls/"
        client = Client(self.serviceUrlTls,
                        tls_trust_certs_file_path=certs_dir + 'cacert.pem',
                        tls_allow_insecure_connection=False,
                        authentication=AuthenticationTLS(certs_dir + 'client-cert.pem', certs_dir + 'client-key.pem'))

        consumer = client.subscribe('my-python-topic-tls-auth',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-python-topic-tls-auth')
        producer.send(b'hello')

        msg = consumer.receive(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.data(), b'hello')

        with self.assertRaises(pulsar.Timeout):
            consumer.receive(100)

        client.close()
Ejemplo n.º 35
0
    def test_message_properties(self):
        client = Client(self.serviceUrl)
        topic = 'my-python-test-message-properties'
        consumer = client.subscribe(topic=topic,
                                    subscription_name='my-subscription',
                                    schema=pulsar.schema.StringSchema())
        producer = client.create_producer(topic=topic,
                                          schema=StringSchema())
        producer.send('hello',
                      properties={
                          'a': '1',
                          'b': '2'
                      })

        msg = consumer.receive(TM)
        self.assertTrue(msg)
        self.assertEqual(msg.value(), 'hello')
        self.assertEqual(msg.properties(), {
                          'a': '1',
                          'b': '2'
                      })

        consumer.unsubscribe()
        client.close()
Ejemplo n.º 36
0
    def test_producer_argument_errors(self):
        client = Client(self.serviceUrl)

        self._check_value_error(lambda: client.create_producer(None))

        topic = 'my-python-test-producer'

        self._check_value_error(
            lambda: client.create_producer(topic, producer_name=5))
        self._check_value_error(
            lambda: client.create_producer(topic, initial_sequence_id='test'))
        self._check_value_error(
            lambda: client.create_producer(topic, send_timeout_millis='test'))
        self._check_value_error(
            lambda: client.create_producer(topic, compression_type=None))
        self._check_value_error(
            lambda: client.create_producer(topic, max_pending_messages='test'))
        self._check_value_error(
            lambda: client.create_producer(topic, block_if_queue_full='test'))
        self._check_value_error(
            lambda: client.create_producer(topic, batching_enabled='test'))
        self._check_value_error(
            lambda: client.create_producer(topic, batching_enabled='test'))
        self._check_value_error(lambda: client.create_producer(
            topic, batching_max_allowed_size_in_bytes='test'))
        self._check_value_error(lambda: client.create_producer(
            topic, batching_max_publish_delay_ms='test'))
        client.close()
Ejemplo n.º 37
0
 def test_simple_producer(self):
     client = Client(self.serviceUrl)
     producer = client.create_producer('persistent://sample/standalone/ns/my-python-topic')
     producer.send('hello')
     producer.close()
     client.close()
Ejemplo n.º 38
0
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

import os
from pulsar import Client, AuthenticationToken

client = Client(os.environ.get('SERVICE_URL'),
                authentication=AuthenticationToken(
                    os.environ.get('AUTH_PARAMS')))

client.close()
Ejemplo n.º 39
0
    def test_producer_argument_errors(self):
        client = Client(self.serviceUrl)

        self._check_value_error(lambda: client.create_producer(None))

        topic = 'persistent://sample/standalone/ns1/my-python-test-producer'

        self._check_value_error(lambda: client.create_producer(topic, producer_name=5))
        self._check_value_error(lambda: client.create_producer(topic, initial_sequence_id='test'))
        self._check_value_error(lambda: client.create_producer(topic, send_timeout_millis='test'))
        self._check_value_error(lambda: client.create_producer(topic, compression_type=None))
        self._check_value_error(lambda: client.create_producer(topic, max_pending_messages='test'))
        self._check_value_error(lambda: client.create_producer(topic, block_if_queue_full='test'))
        self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
        self._check_value_error(lambda: client.create_producer(topic, batching_enabled='test'))
        self._check_value_error(lambda: client.create_producer(topic, batching_max_allowed_size_in_bytes='test'))
        self._check_value_error(lambda: client.create_producer(topic, batching_max_publish_delay_ms='test'))
        client.close()
Ejemplo n.º 40
0
 def test_producer_routing_mode(self):
     client = Client(self.serviceUrl)
     producer = client.create_producer('my-python-test-producer',
                                       message_routing_mode=PartitionsRoutingMode.UseSinglePartition)
     producer.send(b'test')
     client.close()
Ejemplo n.º 41
0
 def test_client_logger(self):
     logger = logging.getLogger("pulsar")
     Client(self.serviceUrl, logger=logger)
Ejemplo n.º 42
0
    def test_seek(self):
        client = Client(self.serviceUrl)
        consumer = client.subscribe('my-python-topic-seek',
                                    'my-sub',
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer('my-python-topic-seek')

        for i in range(100):
            if i > 0:
                time.sleep(0.02)
            producer.send(b'hello-%d' % i)

        ids = []
        timestamps = []
        for i in range(100):
            msg = consumer.receive(TM)
            self.assertEqual(msg.data(), b'hello-%d' % i)
            ids.append(msg.message_id())
            timestamps.append(msg.publish_timestamp())
            consumer.acknowledge(msg)

        # seek, and after reconnect, expected receive first message.
        consumer.seek(MessageId.earliest)
        time.sleep(0.5)
        msg = consumer.receive(TM)
        self.assertEqual(msg.data(), b'hello-0')

        # seek on messageId
        consumer.seek(ids[50])
        time.sleep(0.5)
        msg = consumer.receive(TM)
        self.assertEqual(msg.data(), b'hello-50')

        # ditto, but seek on timestamp
        consumer.seek(timestamps[42])
        time.sleep(0.5)
        msg = consumer.receive(TM)
        self.assertEqual(msg.data(), b'hello-42')

        # repeat with reader
        reader = client.create_reader('my-python-topic-seek', MessageId.latest)
        try:
            msg = reader.read_next(100)
            self.assertTrue(False)  # Should not reach this point
        except:
            pass  # Exception is expected

        # earliest
        reader.seek(MessageId.earliest)
        time.sleep(0.5)
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b'hello-0')
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b'hello-1')

        # seek on messageId
        reader.seek(ids[33])
        time.sleep(0.5)
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b'hello-33')
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b'hello-34')

        # seek on timestamp
        reader.seek(timestamps[79])
        time.sleep(0.5)
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b'hello-79')
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b'hello-80')

        reader.close()
        client.close()
Ejemplo n.º 43
0
 def get_producer():
     cl = Client(self.serviceUrl)
     return cl.create_producer(topic="foobar")
Ejemplo n.º 44
0
 def test_connect_error(self):
     with self.assertRaises(pulsar.ConnectError):
         client = Client("fakeServiceUrl")
         client.create_producer("connect-error-topic")
         client.close()
Ejemplo n.º 45
0
"""
This example shows how to configure negative acknowledgement.
"""
from pulsar import Client, schema

client = Client('pulsar://localhost:6650')
consumer = client.subscribe('negative_acks',
                            'test',
                            schema=schema.StringSchema())
producer = client.create_producer('negative_acks',
                                  schema=schema.StringSchema())

for i in range(10):
    print('send msg "hello-%d"' % i)
    producer.send_async('hello-%d' % i, callback=None)
# Flush all the messages buffered in the client and
# wait until all messages have been successfully persisted
producer.flush()

for i in range(10):
    msg = consumer.receive()
    consumer.negative_acknowledge(msg)
    print('receive and nack msg "%s"' % msg.data().decode("utf-8"))

for i in range(10):
    msg = consumer.receive()
    consumer.acknowledge(msg)
    print('receive and ack msg "%s"' % msg.data().decode("utf-8"))

try:
    # No more messages expected
Ejemplo n.º 46
0
 def __init__(self, args):
     self.args = args
     self.client = Client(args.service_url,
                          authentication=AuthenticationOauth2(
                              args.auth_params))
Ejemplo n.º 47
0
    def test_consumer_argument_errors(self):
        client = Client(self.serviceUrl)

        topic = 'my-python-test-producer'
        sub_name = 'my-sub-name'

        self._check_value_error(lambda: client.subscribe(None, sub_name))
        self._check_value_error(lambda: client.subscribe(topic, None))
        self._check_value_error(
            lambda: client.subscribe(topic, sub_name, consumer_type=None))
        self._check_value_error(lambda: client.subscribe(
            topic, sub_name, receiver_queue_size='test'))
        self._check_value_error(
            lambda: client.subscribe(topic, sub_name, consumer_name=5))
        self._check_value_error(lambda: client.subscribe(
            topic, sub_name, unacked_messages_timeout_ms='test'))
        self._check_value_error(lambda: client.subscribe(
            topic, sub_name, broker_consumer_stats_cache_time_ms='test'))
        client.close()
Ejemplo n.º 48
0
 def test_producer_routing_mode(self):
     client = Client(self.serviceUrl)
     producer = client.create_producer('my-python-test-producer',
                                       message_routing_mode=PartitionsRoutingMode.UseSinglePartition)
     producer.send(b'test')
     client.close()
Ejemplo n.º 49
0
    def test_publish_compact_and_consume(self):
        client = Client(self.serviceUrl)
        topic = 'compaction_%s' % (uuid.uuid4())
        producer = client.create_producer(topic,
                                          producer_name='my-producer-name',
                                          batching_enabled=False)
        self.assertEqual(producer.last_sequence_id(), -1)
        consumer = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
        consumer.close()
        consumer2 = client.subscribe(topic, 'my-sub2', is_read_compacted=False)

        # producer create 2 messages with same key.
        producer.send(b'hello-0', partition_key='key0')
        producer.send(b'hello-1', partition_key='key0')
        producer.close()

        # issue compact command, and wait success
        url = '%s/admin/v2/persistent/public/default/%s/compaction' % (
            self.adminUrl, topic)
        doHttpPut(url, '')
        while True:
            s = doHttpGet(url).decode('utf-8')
            if 'RUNNING' in s:
                print(s)
                print("Compact still running")
                time.sleep(0.2)
            else:
                print(s)
                print("Compact Complete now")
                self.assertTrue('SUCCESS' in s)
                break

        # after compaction completes the compacted ledger is recorded
        # as a property of a cursor. As persisting the cursor is async
        # and we don't wait for the acknowledgement of the acknowledgement,
        # there may be a race if we try to read the compacted ledger immediately.
        # therefore wait a second to allow the compacted ledger to be updated on
        # the broker.
        time.sleep(1.0)

        # after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key.
        consumer1 = client.subscribe(topic, 'my-sub1', is_read_compacted=True)
        msg0 = consumer1.receive(TM)
        self.assertEqual(msg0.data(), b'hello-1')
        consumer1.acknowledge(msg0)
        consumer1.close()

        # ditto for reader
        reader1 = client.create_reader(topic,
                                       MessageId.earliest,
                                       is_read_compacted=True)
        msg0 = reader1.read_next(TM)
        self.assertEqual(msg0.data(), b'hello-1')
        reader1.close()

        # after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key.
        msg0 = consumer2.receive(TM)
        self.assertEqual(msg0.data(), b'hello-0')
        consumer2.acknowledge(msg0)
        msg1 = consumer2.receive(TM)
        self.assertEqual(msg1.data(), b'hello-1')
        consumer2.acknowledge(msg1)
        consumer2.close()

        # ditto for reader
        reader2 = client.create_reader(topic,
                                       MessageId.earliest,
                                       is_read_compacted=False)
        msg0 = reader2.read_next(TM)
        self.assertEqual(msg0.data(), b'hello-0')
        msg1 = reader2.read_next(TM)
        self.assertEqual(msg1.data(), b'hello-1')
        reader2.close()
        client.close()
Ejemplo n.º 50
0
    def test_seek(self):
        client = Client(self.serviceUrl)
        topic = "my-python-topic-seek-" + str(time.time())
        consumer = client.subscribe(topic,
                                    "my-sub",
                                    consumer_type=ConsumerType.Shared)
        producer = client.create_producer(topic)

        for i in range(100):
            if i > 0:
                time.sleep(0.02)
            producer.send(b"hello-%d" % i)

        ids = []
        timestamps = []
        for i in range(100):
            msg = consumer.receive(TM)
            self.assertEqual(msg.data(), b"hello-%d" % i)
            ids.append(msg.message_id())
            timestamps.append(msg.publish_timestamp())
            consumer.acknowledge(msg)

        # seek, and after reconnect, expected receive first message.
        consumer.seek(MessageId.earliest)
        time.sleep(0.5)
        msg = consumer.receive(TM)
        self.assertEqual(msg.data(), b"hello-0")

        # seek on messageId
        consumer.seek(ids[50])
        time.sleep(0.5)
        msg = consumer.receive(TM)
        self.assertEqual(msg.data(), b"hello-50")

        # ditto, but seek on timestamp
        consumer.seek(timestamps[42])
        time.sleep(0.5)
        msg = consumer.receive(TM)
        self.assertEqual(msg.data(), b"hello-42")

        # repeat with reader
        reader = client.create_reader(topic, MessageId.latest)
        with self.assertRaises(pulsar.Timeout):
            reader.read_next(100)

        # earliest
        reader.seek(MessageId.earliest)
        time.sleep(0.5)
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b"hello-0")
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b"hello-1")

        # seek on messageId
        reader.seek(ids[33])
        time.sleep(0.5)
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b"hello-33")
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b"hello-34")

        # seek on timestamp
        reader.seek(timestamps[79])
        time.sleep(0.5)
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b"hello-79")
        msg = reader.read_next(TM)
        self.assertEqual(msg.data(), b"hello-80")

        reader.close()
        client.close()
Ejemplo n.º 51
0
 def test_simple_producer(self):
     client = Client(self.serviceUrl)
     producer = client.create_producer('my-python-topic')
     producer.send(b'hello')
     producer.close()
     client.close()
Ejemplo n.º 52
0
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

import os
from pulsar import Client, AuthenticationToken

client = Client(os.environ.get('SERVICE_URL'),
                authentication=AuthenticationToken(
                    os.environ.get('AUTH_PARAMS')))

producer = client.create_producer('my-topic')

for i in range(10):
    producer.send(('Hello-%d' % i).encode('utf-8'))
    print('send msg "hello-%d"' % i)

client.close()