def produce_test(topic, message):
    client = pulsar.Client('pulsar://localhost:6650')
    producer = client.create_producer(topic)
    for i in range(16):
        producer.send((message).encode('utf-8'))
    time.sleep(1000)
    client.close()
Пример #2
0
def main(argv):
    # Load the categories and the neighborhood
    categories = load_csv(argv[1])
    neighborhoods = load_csv(argv[2])
    category_pool = cycle(categories)
    neighborhood_pool = cycle(neighborhoods)

    # Create a Pulsar client instance.
    client = pulsar.Client('pulsar://localhost:6650')

    # Create a producer on the topic. If the topic doesn't exist
    # it will be automatically created
    producer = client.create_producer(
        'persistent://sample/standalone/ns1/allemails')
    message_count = 0

    # Keep publishing the messages forever
    while True:
        # For simulation purposes, we are just joining a random neighborhood
        # with the toy category type
        generated_email = ' '.join(
            [next(neighborhood_pool),
             next(category_pool)])
        producer.send(generated_email)

        message_count += 1
        if message_count % 100 == 0:
            # For the sake of demo, sleep 1 second
            print('Produced %d messages' % message_count)
            time.sleep(1)

    # Close the client connection
    client.close()
Пример #3
0
def main(args):
    logging.info('Connecting to Pulsar...')

    # Create a pulsar client instance with reference to the broker
    client = pulsar.Client('pulsar://localhost:6650')

    # Build a producer instance on a specific topic
    producer = client.create_producer(SENTENCES_TOPIC)
    logging.info('Connected to Pulsar')

    # Collection of sentences to serve as random input sequence
    sentences = random_cycle([
        "the cow jumped over the moon", "an apple a day keeps the doctor away",
        "four score and seven years ago", "snow white and the seven dwarfs",
        "i am at two with nature"
    ])

    logging.info('Sending sentences to the word count topology...')

    for sentence in sentences:
        sleep(0.05)  # Throttle messages with a 50 ms delay
        logging.info('Sending sentence: %s ', sentence)
        producer.send(sentence)  # Publish randomly selected sentence to Pulsar

    client.close()
Пример #4
0
    def test_bytes_schema(self):
        client = pulsar.Client(self.serviceUrl)
        producer = client.create_producer(
                        'my-bytes-python-topic',
                        schema=BytesSchema())

        # Validate that incompatible schema is rejected
        try:
            class Example(Record):
                a = Integer()
                b = Integer()

            client.create_producer('my-bytes-python-topic',
                             schema=JsonSchema(Example))
            self.fail('Should have failed')
        except Exception as e:
            pass  # Expected

        consumer = client.subscribe('my-bytes-python-topic', 'sub-1',
                                    schema=BytesSchema())

        producer.send(b"Hello")

        msg = consumer.receive()

        self.assertEqual(b"Hello", msg.value())
        client.close()
Пример #5
0
    def __init__(
        self,
        broker_service_url=None,
        consumer=None,
        producer=None,
    ):
        self._consumer = None
        self._producer = None

        if consumer and type(consumer) is not Consumer:
            raise TypeError('Input should be of type Consumer')
        if producer and type(producer) is not Producer:
            raise TypeError('Input should be of type Producer')

        self.client = pulsar.Client(broker_service_url)

        if consumer:
            self._consumer = self.client.subscribe(
                consumer.topic,
                consumer.subscription_name,
                consumer_type=_pulsar.ConsumerType.Shared,
                negative_ack_redelivery_delay_ms=500,
                schema=AvroSchema(consumer.schema_class))

        if producer:
            self._producer = self.client.create_producer(
                producer.topic,
                batching_enabled=True,
                batching_max_publish_delay_ms=10,
                schema=AvroSchema(producer.schema_class))
Пример #6
0
def test_message(message,
                 broker='pulsar://localhost:6650',
                 topic='test',
                 max_records=100000,
                 batching=True,
                 max_pending=10000):

    message_size = sys.getsizeof(message)
    logger.info('Throughput test message: %s  (%d bytes)', message,
                message_size)
    message_size /= 1048576.
    message = message.encode('utf-8')
    client = pulsar.Client(broker)
    producer = client.create_producer(topic,
                                      block_if_queue_full=True,
                                      batching_enabled=batching,
                                      max_pending_messages=max_pending)
    handler = CallbackHandler()
    t0 = time.time()
    for i in range(max_records):
        producer.send_async(message, callback=handler.callback)
    producer.flush()
    dt = time.time() - t0
    logger.info("Message rate: %10.5f records/s", max_records / dt)
    if handler.dropped:
        logger.info("Dropped messages:  %d", handler.dropped)
        logger.info("Last result:       %s", handler.result)
    logger.info("Throughput:   %10.5f MB/s", max_records * message_size / dt)
    client.close()
Пример #7
0
    def test_avro_enum(self):
        class MyEnum(Enum):
            A = 1
            B = 2
            C = 3

        class Example(Record):
            name = String()
            v = MyEnum

        topic = 'my-avro-enum-topic'

        client = pulsar.Client(self.serviceUrl)
        producer = client.create_producer(
                        topic=topic,
                        schema=AvroSchema(Example))

        consumer = client.subscribe(topic, 'test',
                                    schema=AvroSchema(Example))

        r = Example(name='test', v=MyEnum.C)
        producer.send(r)

        msg = consumer.receive()
        msg.value()
        self.assertEqual(MyEnum.C, msg.value().v)
        client.close()
Пример #8
0
    def _get_or_create_consumer(self):
        if self._check_consumer_alive() != True:
            try:
                self._consumer_conn = pulsar.Client(
                    service_url="pulsar://{}:{}".format(
                        self._host, self._port),
                    operation_timeout_seconds=30,
                )
            except Exception:
                self._consumer_conn = None

            try:
                self._consumer_receive = self._consumer_conn.subscribe(
                    TOPIC_PREFIX.format(self._tenant, self._namespace,
                                        self._receive_topic),
                    subscription_name=DEFAULT_SUBSCRIPTION_NAME,
                    consumer_name=UNIQUE_CONSUMER_NAME,
                    initial_position=pulsar.InitialPosition.Earliest,
                    replicate_subscription_state_enabled=True,
                    **self._subscription_config,
                )

                # set cursor to latest confirmed
                if self._latest_confirmed is not None:
                    self._consumer_receive.seek(self._latest_confirmed)

            except Exception as e:
                LOGGER.debug(
                    f"catch exception {e} in creating pulsar consumer")
                self._consumer_conn.close()
                self._consumer_conn = None
Пример #9
0
def main():
    # Create a Pulsar client instance. The instance can be shared across multiple
    # producers and consumers
    client = pulsar.Client('pulsar://localhost:6650',
                           log_conf_file_path='./log4j.cxx')

    # Subscribe to the emails topic. Note that we are subscribing in
    # a Exclusive mode, which means that all of the lego requests
    # are handled by this super efficient lego elf
    consumer = client.subscribe(
        'persistent://sample/standalone/ns1/categorized-emails-Lego',
        'postmaster',
        consumer_type=pulsar.ConsumerType.Exclusive)

    while True:
        # try and receive messages with a timeout of 10 seconds
        msg = consumer.receive()
        print("Lego Elf received lego request from neighborhood " + msg.data())

        # Create the toy
        lego_toy = create_lego()

        # Route the toy to the right neighborhood
        republish_toys(client, msg.data(), lego_toy)

        # Acknowledge processing of message so that it can be deleted
        consumer.acknowledge(msg)

    # Close the client connection
    client.close()
Пример #10
0
def test():
    SERVICE_URL = "pulsar://pulsar.aimango.net:6650"
    REST_URL = "http://pulsar.aimango.net:8080"
    cli = pulsar.Client(SERVICE_URL)
    produ = cli.create_producer('persistent://public/default/liuchaozhi',
                                'xxx')
    produ.send('xxx'.encode('utf-8'))
    pulsar_client = mango_pulsar_client.MangoPulsarClient(
        SERVICE_URL, REST_URL)
    pro = pulsar_client.create_producer(
        'persistent://public/default/liuchaozhi')
    pro.send('xxx'.encode('utf-8'))
    topics = pulsar_client.admin.persistent_topic.get_the_list_of_topics_under_namespace(
        'public', 'default')
    print(topics)
    topic = 'persistent://public/default/xxxx'
    res = pulsar_client.admin.persistent_topic.create_non_partitioned_topic(
        'public', 'default', 'xxxx')
    print(res['content'])
    res = pulsar_client.admin.clusters.get_the_list_of_all_the_Pulsar_clusters(
    )
    print(res)
    producer = pulsar_client.create_producer(topic, 'xxx-producer')
    producer.send('xxxx'.encode('utf-8'))
    producer.flush()
    pulsar_client.close()
Пример #11
0
def publish_companies(
        corps: list,
        pub_topic: str = "search_filings-8-K",
        pulsar_connection_string: str = "pulsar://localhost:6650"):
    """

    :param corps:
    :param pub_topic:
    :param pulsar_connection_string:
    :return:
    """

    client = pulsar.Client(pulsar_connection_string)
    producer = client.create_producer(topic=pub_topic,
                                      block_if_queue_full=True,
                                      batching_enabled=True,
                                      send_timeout_millis=300000,
                                      batching_max_publish_delay_ms=120000)

    i = 1
    for corp in corps:
        _logger.info("Publishing {0} of {1}".format(i, len(corps)))
        msg = json.dumps(corp).encode('utf-8')
        producer.send(msg)
        i += 1
Пример #12
0
    def test_default_value(self):
        class MyRecord(Record):
            A = Integer()
            B = String()
            C = Boolean(default=True, required=True)
            D = Double(default=6.4)

        topic = "my-default-value-topic"

        client = pulsar.Client(self.serviceUrl)
        producer = client.create_producer(
                    topic=topic,
                    schema=JsonSchema(MyRecord))

        consumer = client.subscribe(topic, 'test', schema=JsonSchema(MyRecord))

        r = MyRecord(A=5, B="text")
        producer.send(r)

        msg = consumer.receive()
        self.assertEqual(msg.value().A, 5)
        self.assertEqual(msg.value().B, u'text')
        self.assertEqual(msg.value().C, True)
        self.assertEqual(msg.value().D, 6.4)

        producer.close()
        consumer.close()
        client.close()
Пример #13
0
def produce_test(partition, key, message):
    client = pulsar.Client('pulsar://localhost:6650')
    producer = client.create_producer(partition)
    for i in range(16):
        message_mix = str(int(i)) + message
        producer.send((message_mix).encode('utf-8'), partition_key=key)
    client.close()
Пример #14
0
    def test_avro_enum(self):
        class MyEnum(Enum):
            A = 1
            B = 2
            C = 3

        class Example(Record):
            name = String()
            v = MyEnum
            w = CustomEnum(MyEnum)
            x = CustomEnum(MyEnum,
                           required=True,
                           default=MyEnum.B,
                           required_default=True)

        topic = 'my-avro-enum-topic'

        client = pulsar.Client(self.serviceUrl)
        producer = client.create_producer(topic=topic,
                                          schema=AvroSchema(Example))

        consumer = client.subscribe(topic, 'test', schema=AvroSchema(Example))

        r = Example(name='test', v=MyEnum.C, w=MyEnum.A)
        producer.send(r)

        msg = consumer.receive()
        msg.value()
        self.assertEqual(MyEnum.C, msg.value().v)
        self.assertEqual(MyEnum.A, MyEnum(msg.value().w))
        self.assertEqual(MyEnum.B, MyEnum(msg.value().x))
        client.close()
Пример #15
0
async def test():

    # client = pulsar.Client('pulsar://pulsar.aimango.net:6650',authentication=AuthenticationToken('FLFyW0oLJ2Fi22KKCm21J18mblAT5ucEKU='))
    #
    # consumer_type = pulsar.ConsumerType.Exclusive
    # consumer = client.subscribe('topic-1', 'sub-1', consumer_type=consumer_type, schema=pulsar.schema.BytesSchema())
    # print(consumer.topic())
    # while True:
    #     msg = consumer.receive()
    #
    #     print(msg)
    #     consumer.acknowledge(msg)
    SERVICE_URL = "pulsar://47.94.142.82:6650"
    TOPIC = "persistent://public/default/liuchaozhi"
    SUBSCRIPTION = "sub-1"
    PUBLISH = "pub-1"

    client = pulsar.Client(SERVICE_URL)

    producer = client.create_producer(TOPIC, PUBLISH)
    consumer = client.subscribe(
        TOPIC,
        SUBSCRIPTION,
        # 如果要限制接收器队列大小
        receiver_queue_size=10,
        consumer_type=pulsar.ConsumerType.Shared)
    for index in range(10):
        producer.send("sdddfasdfdsfds".encode('utf-8'))
def main():
    global __running

    logger.info("Init py-producer")

    client = pulsar.Client(__PULSAR_URL)
    producer = client.create_producer(__TOPIC_URL)

    rate = 1.0 / float(__PRODUCER_RATE)

    sent_msgs = 0

    while __running:
        json_data = generate_random_data()

        data = json.dumps(json_data).encode('utf-8')

        try:
            if sent_msgs % 1000 == 0:
                logger.info("Already sent %d messages", sent_msgs)

            producer.send(data)
            sent_msgs = sent_msgs + 1
        except Exception:
            logger.error("Error sending data to Apache Pulsar")
            logging.error(traceback.format_exc())

        time.sleep(rate)

    producer.close()

    logger.info("Finished py-producer")
Пример #17
0
def send_data(sensorPath, sensorID, pulsarIP, pulsarTopic):
    # Pulsar producer configuration
    client = pulsar.Client('pulsar://' + pulsarIP + ':6650')
    producer = client.create_producer(pulsarTopic)

    # Read sensor data from CSV file
    with open(sensorPath + sensorID + '.csv') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            # Get actual timestamp (Event time)
            eventTime = str(time.time()).split(".")[0]

            # Create sensor data as CSV
            csvToSend = eventTime + ','\
                        + sensorID + ','\
                        + row['V'] + ','\
                        + row['I'] + ','\
                        + row['f'] + ','\
                        + row['DPF'] + ','\
                        + row['APF'] + ','\
                        + row['P'] + ','\
                        + row['Pt'] + ','\
                        + row['Q'] + ','\
                        + row['Qt'] + ',' \
                        + row['S'] + ','\
                        + row['St']
            print(csvToSend)

            # Produce messages to pulsar
            producer.send(csvToSend.encode('utf-8'),
                          event_timestamp=int(eventTime))
            time.sleep(1)  # interval between events is 1 second
    client.close()
Пример #18
0
    def test_schema_version(self):
        class Example(Record):
            a = Integer()
            b = Integer()

        client = pulsar.Client(self.serviceUrl)
        producer = client.create_producer(
                        'my-avro-python-schema-version-topic',
                        schema=AvroSchema(Example))

        consumer = client.subscribe('my-avro-python-schema-version-topic', 'sub-1',
                                    schema=AvroSchema(Example))

        r = Example(a=1, b=2)
        producer.send(r)

        msg = consumer.receive()

        self.assertIsNotNone(msg.schema_version())

        self.assertEquals(b'\x00\x00\x00\x00\x00\x00\x00\x00', msg.schema_version().encode())

        self.assertEqual(r, msg.value())

        client.close()
Пример #19
0
    def _get_or_create_producer(self):
        if self._check_producer_alive() != True:
            # if self._producer_conn is None:
            try:
                self._producer_conn = pulsar.Client(
                    service_url="pulsar://{}:{}".format(
                        self._host, self._port),
                    operation_timeout_seconds=30,
                )
            except Exception as e:
                self._producer_conn = None

            # alway used current client to fetch producer
            try:
                self._producer_send = self._producer_conn.create_producer(
                    TOPIC_PREFIX.format(self._tenant, self._namespace,
                                        self._send_topic),
                    producer_name=UNIQUE_PRODUCER_NAME,
                    send_timeout_millis=60000,
                    max_pending_messages=500,
                    compression_type=pulsar.CompressionType.LZ4,
                    **self._producer_config,
                )
            except Exception as e:
                LOGGER.debug(
                    f"catch exception {e} in creating pulsar producer")
                self._producer_conn = None
Пример #20
0
def main():
    # Create a Pulsar client instance. The instance can be shared across multiple
    # producers and consumers
    client = pulsar.Client('pulsar://localhost:6650',
                           log_conf_file_path='./log4j.cxx')

    # Subscribe to the emails topic. Note that we are subscribing in
    # a shared mode, which means I can easily run multiple distributors
    # to parallely consume all the emails
    consumer = client.subscribe('persistent://sample/standalone/ns1/allemails',
                                'postmaster',
                                consumer_type=pulsar.ConsumerType.Shared)

    while True:
        # try and receive messages with a timeout of 10 seconds
        msg = consumer.receive()

        # Extract the category of the toy desired from the message by parsing it
        category = extract_category(msg.data())

        # Extract the neighboor hood of the kid who sent the email
        neighborhood = extract_neighborhood(msg.data())

        print('Received message for ' + category +\
              ' from neighborhood ' + neighborhood)

        # republish
        republish(client, category, neighborhood)

        # Acknowledge processing of message so that it can be deleted
        consumer.acknowledge(msg)

    # Close the client connection
    client.close()
Пример #21
0
def main():
    client = pulsar.Client(protocol_url)
    consumer = client.subscribe('my-topicc', 'my-subscription')

    try:
        consumer_routine(consumer)
    except KeyboardInterrupt:
        client.close()
Пример #22
0
 def __init__(self, url=None):
     if url is None:
         config = configparser.ConfigParser()
         config.read("config.ini")
         url_ = config["urls"]["UrlPulsar"]
     else:
         url_ = url
     self._pulsar_client = pulsar.Client(url_)
Пример #23
0
    def __init__(self,
        BrokerURL='pulsar://localhost:6650', # The Apache Pulsar broker URL
        AuthToken=None, # The Apache Pulsar authentication token to use
        logger=None
        ):
        
        # connect to the broker
        if AuthToken is not None:
            self._client = pulsar.Client(
                service_url=BrokerURL,
                authentication=pulsar.AuthenticationToken(AuthToken)
                )
        else:
            self._client = pulsar.Client(
                service_url=BrokerURL
                )

        self._broker_url = BrokerURL
Пример #24
0
    def __init__(self, url=None, token=None):
        if not isinstance(url, str) or not isinstance(token, str):
            assert 0

        from pulsar import AuthenticationToken

        self._client = pulsar.Client(url,
                                     authentication=AuthenticationToken(token))
        self._client_id = generate_client_id()
Пример #25
0
def test_producer():
    client = pulsar.Client('pulsar://47.94.142.82:6650')

    producer = client.create_producer('my-topic')

    for i in range(10):
        producer.send(('Hello-%d' % i).encode('utf-8'))

    client.close()
Пример #26
0
    def produceByTime(self):
        try:
            client = pulsar.Client(self.url,
                                use_tls=True,
                                tls_allow_insecure_connection=True,
                                authentication=self.authToken
                            )

            producer = client.create_producer(self.topic)
        except Exception as e:
            print(e)
            raise SystemExit

        # Publish our topic name if we are doing topicFromTopic
        if self.topicFromTopic:
            print("####################### GEN TOPIC #########################")
            try:
                s = producer.send(self.topicFromTopic.encode('utf-8'))
                print("Created topic " + self.topicFromTopic + " published into " + self.topic)
                self.topic = self.topicFromTopic
                if self.sendAsync:
                  producer = client.create_producer(
                               self.topic,
                               block_if_queue_full=True,
                               batching_enabled=True,
                               batching_max_publish_delay_ms=100
                             )
                else:
                  producer = client.create_producer(self.topic)
            except Exception as e:
                print("Failed to send topic name: %s", e)
                raise SystemExit

        if self.runForever:
            print("Running FOREVER!!!!")
            endTime = 0
        else:
            print("Running for " + str(self.runTime) + "seconds.")
            endTime = self.runTime + time.time()

        msg = self.genMsg()

        while (time.time() < endTime) or (self.runForever):
            if self.uniq:
                msgToSend = msg + str(time.time()).encode()
            else:
                msgToSend = msg
            try:
                if self.sendAsync:
                    producer.send_async(msgToSend, self.sendAsyncCallback)
                else:
                    producer.send(msgToSend)
            except Exception as e:
                print("Failed to send message: %s", e)
            if self.delay:
                time.sleep(self.delay/1000)
Пример #27
0
    def __init__(self, cfg):

        self.cfg = cfg

        self.pulsar_client = pulsar.Client(cfg.pulsar_url)

        if not helper.waiting4namespace(self.cfg):
            self.ledger_producer = self.pulsar_client.create_producer(
                topic=f"persistent://{cfg.tenant}/{cfg.market_uuid}/ledger",
                schema=pulsar.schema.AvroSchema(MV2.schema.LedgerSchema))
Пример #28
0
def flush_topic():
    """A consumer that will read everything from the pulsar topic to 'reset it.'"""
    topic = "testTopic"
    client = pulsar.Client('pulsar://localhost:6650')
    consumer = client.subscribe(topic, 'testSubscription')
    while True:
        msg = consumer.receive()
        print("Received message '{}' id='{}'".format(msg.data(),
                                                     msg.message_id()))
        consumer.acknowledge(msg)
def main(argv):
    host = '192.168.50.233'
    port = 6650
    topic = 'test_channel'
    client = pulsar.Client('pulsar://{}:{}'.format(host, port))
    producer = client.create_producer(topic)
    for i in range(10):
        producer.send(("hello-%s" % i).encode("utf-8"))
    print("Finished produce message.")
    client.close()
Пример #30
0
    def __init__(self, broker=None, topic=None):

        if broker == None:
            broker = os.getenv("PULSAR_BROKER", "pulsar://localhost:6650")

        if topic == None:
            routing_key = os.getenv("PULSAR_PRODUCER_TOPIC")

        self.client = pulsar.Client(broker)
        self.producer = self.client.create_producer(topic)