def produce_test(topic, message):
client = pulsar.Client('pulsar://localhost:6650')
producer = client.create_producer(topic)
for i in range(16):
producer.send((message).encode('utf-8'))
time.sleep(1000)
client.close()
def main(argv):
# Load the categories and the neighborhood
categories = load_csv(argv[1])
neighborhoods = load_csv(argv[2])
category_pool = cycle(categories)
neighborhood_pool = cycle(neighborhoods)
# Create a Pulsar client instance.
client = pulsar.Client('pulsar://localhost:6650')
# Create a producer on the topic. If the topic doesn't exist
# it will be automatically created
producer = client.create_producer(
'persistent://sample/standalone/ns1/allemails')
message_count = 0
# Keep publishing the messages forever
while True:
# For simulation purposes, we are just joining a random neighborhood
# with the toy category type
generated_email = ' '.join(
[next(neighborhood_pool),
next(category_pool)])
producer.send(generated_email)
message_count += 1
if message_count % 100 == 0:
# For the sake of demo, sleep 1 second
print('Produced %d messages' % message_count)
time.sleep(1)
# Close the client connection
client.close()
def main(args):
logging.info('Connecting to Pulsar...')
# Create a pulsar client instance with reference to the broker
client = pulsar.Client('pulsar://localhost:6650')
# Build a producer instance on a specific topic
producer = client.create_producer(SENTENCES_TOPIC)
logging.info('Connected to Pulsar')
# Collection of sentences to serve as random input sequence
sentences = random_cycle([
"the cow jumped over the moon", "an apple a day keeps the doctor away",
"four score and seven years ago", "snow white and the seven dwarfs",
"i am at two with nature"
])
logging.info('Sending sentences to the word count topology...')
for sentence in sentences:
sleep(0.05) # Throttle messages with a 50 ms delay
logging.info('Sending sentence: %s ', sentence)
producer.send(sentence) # Publish randomly selected sentence to Pulsar
client.close()
def test_bytes_schema(self):
client = pulsar.Client(self.serviceUrl)
producer = client.create_producer(
'my-bytes-python-topic',
schema=BytesSchema())
# Validate that incompatible schema is rejected
try:
class Example(Record):
a = Integer()
b = Integer()
client.create_producer('my-bytes-python-topic',
schema=JsonSchema(Example))
self.fail('Should have failed')
except Exception as e:
pass # Expected
consumer = client.subscribe('my-bytes-python-topic', 'sub-1',
schema=BytesSchema())
producer.send(b"Hello")
msg = consumer.receive()
self.assertEqual(b"Hello", msg.value())
client.close()
def __init__(
self,
broker_service_url=None,
consumer=None,
producer=None,
):
self._consumer = None
self._producer = None
if consumer and type(consumer) is not Consumer:
raise TypeError('Input should be of type Consumer')
if producer and type(producer) is not Producer:
raise TypeError('Input should be of type Producer')
self.client = pulsar.Client(broker_service_url)
if consumer:
self._consumer = self.client.subscribe(
consumer.topic,
consumer.subscription_name,
consumer_type=_pulsar.ConsumerType.Shared,
negative_ack_redelivery_delay_ms=500,
schema=AvroSchema(consumer.schema_class))
if producer:
self._producer = self.client.create_producer(
producer.topic,
batching_enabled=True,
batching_max_publish_delay_ms=10,
schema=AvroSchema(producer.schema_class))
def test_message(message,
broker='pulsar://localhost:6650',
topic='test',
max_records=100000,
batching=True,
max_pending=10000):
message_size = sys.getsizeof(message)
logger.info('Throughput test message: %s (%d bytes)', message,
message_size)
message_size /= 1048576.
message = message.encode('utf-8')
client = pulsar.Client(broker)
producer = client.create_producer(topic,
block_if_queue_full=True,
batching_enabled=batching,
max_pending_messages=max_pending)
handler = CallbackHandler()
t0 = time.time()
for i in range(max_records):
producer.send_async(message, callback=handler.callback)
producer.flush()
dt = time.time() - t0
logger.info("Message rate: %10.5f records/s", max_records / dt)
if handler.dropped:
logger.info("Dropped messages: %d", handler.dropped)
logger.info("Last result: %s", handler.result)
logger.info("Throughput: %10.5f MB/s", max_records * message_size / dt)
client.close()
def test_avro_enum(self):
class MyEnum(Enum):
A = 1
B = 2
C = 3
class Example(Record):
name = String()
v = MyEnum
topic = 'my-avro-enum-topic'
client = pulsar.Client(self.serviceUrl)
producer = client.create_producer(
topic=topic,
schema=AvroSchema(Example))
consumer = client.subscribe(topic, 'test',
schema=AvroSchema(Example))
r = Example(name='test', v=MyEnum.C)
producer.send(r)
msg = consumer.receive()
msg.value()
self.assertEqual(MyEnum.C, msg.value().v)
client.close()
def _get_or_create_consumer(self):
if self._check_consumer_alive() != True:
try:
self._consumer_conn = pulsar.Client(
service_url="pulsar://{}:{}".format(
self._host, self._port),
operation_timeout_seconds=30,
)
except Exception:
self._consumer_conn = None
try:
self._consumer_receive = self._consumer_conn.subscribe(
TOPIC_PREFIX.format(self._tenant, self._namespace,
self._receive_topic),
subscription_name=DEFAULT_SUBSCRIPTION_NAME,
consumer_name=UNIQUE_CONSUMER_NAME,
initial_position=pulsar.InitialPosition.Earliest,
replicate_subscription_state_enabled=True,
**self._subscription_config,
)
# set cursor to latest confirmed
if self._latest_confirmed is not None:
self._consumer_receive.seek(self._latest_confirmed)
except Exception as e:
LOGGER.debug(
f"catch exception {e} in creating pulsar consumer")
self._consumer_conn.close()
self._consumer_conn = None
def main():
# Create a Pulsar client instance. The instance can be shared across multiple
# producers and consumers
client = pulsar.Client('pulsar://localhost:6650',
log_conf_file_path='./log4j.cxx')
# Subscribe to the emails topic. Note that we are subscribing in
# a Exclusive mode, which means that all of the lego requests
# are handled by this super efficient lego elf
consumer = client.subscribe(
'persistent://sample/standalone/ns1/categorized-emails-Lego',
'postmaster',
consumer_type=pulsar.ConsumerType.Exclusive)
while True:
# try and receive messages with a timeout of 10 seconds
msg = consumer.receive()
print("Lego Elf received lego request from neighborhood " + msg.data())
# Create the toy
lego_toy = create_lego()
# Route the toy to the right neighborhood
republish_toys(client, msg.data(), lego_toy)
# Acknowledge processing of message so that it can be deleted
consumer.acknowledge(msg)
# Close the client connection
client.close()
def test():
SERVICE_URL = "pulsar://pulsar.aimango.net:6650"
REST_URL = "http://pulsar.aimango.net:8080"
cli = pulsar.Client(SERVICE_URL)
produ = cli.create_producer('persistent://public/default/liuchaozhi',
'xxx')
produ.send('xxx'.encode('utf-8'))
pulsar_client = mango_pulsar_client.MangoPulsarClient(
SERVICE_URL, REST_URL)
pro = pulsar_client.create_producer(
'persistent://public/default/liuchaozhi')
pro.send('xxx'.encode('utf-8'))
topics = pulsar_client.admin.persistent_topic.get_the_list_of_topics_under_namespace(
'public', 'default')
print(topics)
topic = 'persistent://public/default/xxxx'
res = pulsar_client.admin.persistent_topic.create_non_partitioned_topic(
'public', 'default', 'xxxx')
print(res['content'])
res = pulsar_client.admin.clusters.get_the_list_of_all_the_Pulsar_clusters(
)
print(res)
producer = pulsar_client.create_producer(topic, 'xxx-producer')
producer.send('xxxx'.encode('utf-8'))
producer.flush()
pulsar_client.close()
def publish_companies(
corps: list,
pub_topic: str = "search_filings-8-K",
pulsar_connection_string: str = "pulsar://localhost:6650"):
"""
:param corps:
:param pub_topic:
:param pulsar_connection_string:
:return:
"""
client = pulsar.Client(pulsar_connection_string)
producer = client.create_producer(topic=pub_topic,
block_if_queue_full=True,
batching_enabled=True,
send_timeout_millis=300000,
batching_max_publish_delay_ms=120000)
i = 1
for corp in corps:
_logger.info("Publishing {0} of {1}".format(i, len(corps)))
msg = json.dumps(corp).encode('utf-8')
producer.send(msg)
i += 1
def test_default_value(self):
class MyRecord(Record):
A = Integer()
B = String()
C = Boolean(default=True, required=True)
D = Double(default=6.4)
topic = "my-default-value-topic"
client = pulsar.Client(self.serviceUrl)
producer = client.create_producer(
topic=topic,
schema=JsonSchema(MyRecord))
consumer = client.subscribe(topic, 'test', schema=JsonSchema(MyRecord))
r = MyRecord(A=5, B="text")
producer.send(r)
msg = consumer.receive()
self.assertEqual(msg.value().A, 5)
self.assertEqual(msg.value().B, u'text')
self.assertEqual(msg.value().C, True)
self.assertEqual(msg.value().D, 6.4)
producer.close()
consumer.close()
client.close()
def produce_test(partition, key, message):
client = pulsar.Client('pulsar://localhost:6650')
producer = client.create_producer(partition)
for i in range(16):
message_mix = str(int(i)) + message
producer.send((message_mix).encode('utf-8'), partition_key=key)
client.close()
def test_avro_enum(self):
class MyEnum(Enum):
A = 1
B = 2
C = 3
class Example(Record):
name = String()
v = MyEnum
w = CustomEnum(MyEnum)
x = CustomEnum(MyEnum,
required=True,
default=MyEnum.B,
required_default=True)
topic = 'my-avro-enum-topic'
client = pulsar.Client(self.serviceUrl)
producer = client.create_producer(topic=topic,
schema=AvroSchema(Example))
consumer = client.subscribe(topic, 'test', schema=AvroSchema(Example))
r = Example(name='test', v=MyEnum.C, w=MyEnum.A)
producer.send(r)
msg = consumer.receive()
msg.value()
self.assertEqual(MyEnum.C, msg.value().v)
self.assertEqual(MyEnum.A, MyEnum(msg.value().w))
self.assertEqual(MyEnum.B, MyEnum(msg.value().x))
client.close()
async def test():
# client = pulsar.Client('pulsar://pulsar.aimango.net:6650',authentication=AuthenticationToken('FLFyW0oLJ2Fi22KKCm21J18mblAT5ucEKU='))
#
# consumer_type = pulsar.ConsumerType.Exclusive
# consumer = client.subscribe('topic-1', 'sub-1', consumer_type=consumer_type, schema=pulsar.schema.BytesSchema())
# print(consumer.topic())
# while True:
# msg = consumer.receive()
#
# print(msg)
# consumer.acknowledge(msg)
SERVICE_URL = "pulsar://47.94.142.82:6650"
TOPIC = "persistent://public/default/liuchaozhi"
SUBSCRIPTION = "sub-1"
PUBLISH = "pub-1"
client = pulsar.Client(SERVICE_URL)
producer = client.create_producer(TOPIC, PUBLISH)
consumer = client.subscribe(
TOPIC,
SUBSCRIPTION,
# 如果要限制接收器队列大小
receiver_queue_size=10,
consumer_type=pulsar.ConsumerType.Shared)
for index in range(10):
producer.send("sdddfasdfdsfds".encode('utf-8'))
def main():
global __running
logger.info("Init py-producer")
client = pulsar.Client(__PULSAR_URL)
producer = client.create_producer(__TOPIC_URL)
rate = 1.0 / float(__PRODUCER_RATE)
sent_msgs = 0
while __running:
json_data = generate_random_data()
data = json.dumps(json_data).encode('utf-8')
try:
if sent_msgs % 1000 == 0:
logger.info("Already sent %d messages", sent_msgs)
producer.send(data)
sent_msgs = sent_msgs + 1
except Exception:
logger.error("Error sending data to Apache Pulsar")
logging.error(traceback.format_exc())
time.sleep(rate)
producer.close()
logger.info("Finished py-producer")
def send_data(sensorPath, sensorID, pulsarIP, pulsarTopic):
# Pulsar producer configuration
client = pulsar.Client('pulsar://' + pulsarIP + ':6650')
producer = client.create_producer(pulsarTopic)
# Read sensor data from CSV file
with open(sensorPath + sensorID + '.csv') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
# Get actual timestamp (Event time)
eventTime = str(time.time()).split(".")[0]
# Create sensor data as CSV
csvToSend = eventTime + ','\
+ sensorID + ','\
+ row['V'] + ','\
+ row['I'] + ','\
+ row['f'] + ','\
+ row['DPF'] + ','\
+ row['APF'] + ','\
+ row['P'] + ','\
+ row['Pt'] + ','\
+ row['Q'] + ','\
+ row['Qt'] + ',' \
+ row['S'] + ','\
+ row['St']
print(csvToSend)
# Produce messages to pulsar
producer.send(csvToSend.encode('utf-8'),
event_timestamp=int(eventTime))
time.sleep(1) # interval between events is 1 second
client.close()
def test_schema_version(self):
class Example(Record):
a = Integer()
b = Integer()
client = pulsar.Client(self.serviceUrl)
producer = client.create_producer(
'my-avro-python-schema-version-topic',
schema=AvroSchema(Example))
consumer = client.subscribe('my-avro-python-schema-version-topic', 'sub-1',
schema=AvroSchema(Example))
r = Example(a=1, b=2)
producer.send(r)
msg = consumer.receive()
self.assertIsNotNone(msg.schema_version())
self.assertEquals(b'\x00\x00\x00\x00\x00\x00\x00\x00', msg.schema_version().encode())
self.assertEqual(r, msg.value())
client.close()
def _get_or_create_producer(self):
if self._check_producer_alive() != True:
# if self._producer_conn is None:
try:
self._producer_conn = pulsar.Client(
service_url="pulsar://{}:{}".format(
self._host, self._port),
operation_timeout_seconds=30,
)
except Exception as e:
self._producer_conn = None
# alway used current client to fetch producer
try:
self._producer_send = self._producer_conn.create_producer(
TOPIC_PREFIX.format(self._tenant, self._namespace,
self._send_topic),
producer_name=UNIQUE_PRODUCER_NAME,
send_timeout_millis=60000,
max_pending_messages=500,
compression_type=pulsar.CompressionType.LZ4,
**self._producer_config,
)
except Exception as e:
LOGGER.debug(
f"catch exception {e} in creating pulsar producer")
self._producer_conn = None
def main():
# Create a Pulsar client instance. The instance can be shared across multiple
# producers and consumers
client = pulsar.Client('pulsar://localhost:6650',
log_conf_file_path='./log4j.cxx')
# Subscribe to the emails topic. Note that we are subscribing in
# a shared mode, which means I can easily run multiple distributors
# to parallely consume all the emails
consumer = client.subscribe('persistent://sample/standalone/ns1/allemails',
'postmaster',
consumer_type=pulsar.ConsumerType.Shared)
while True:
# try and receive messages with a timeout of 10 seconds
msg = consumer.receive()
# Extract the category of the toy desired from the message by parsing it
category = extract_category(msg.data())
# Extract the neighboor hood of the kid who sent the email
neighborhood = extract_neighborhood(msg.data())
print('Received message for ' + category +\
' from neighborhood ' + neighborhood)
# republish
republish(client, category, neighborhood)
# Acknowledge processing of message so that it can be deleted
consumer.acknowledge(msg)
# Close the client connection
client.close()
def main():
client = pulsar.Client(protocol_url)
consumer = client.subscribe('my-topicc', 'my-subscription')
try:
consumer_routine(consumer)
except KeyboardInterrupt:
client.close()
def __init__(self, url=None):
if url is None:
config = configparser.ConfigParser()
config.read("config.ini")
url_ = config["urls"]["UrlPulsar"]
else:
url_ = url
self._pulsar_client = pulsar.Client(url_)
def __init__(self,
BrokerURL='pulsar://localhost:6650', # The Apache Pulsar broker URL
AuthToken=None, # The Apache Pulsar authentication token to use
logger=None
):
# connect to the broker
if AuthToken is not None:
self._client = pulsar.Client(
service_url=BrokerURL,
authentication=pulsar.AuthenticationToken(AuthToken)
)
else:
self._client = pulsar.Client(
service_url=BrokerURL
)
self._broker_url = BrokerURL
def __init__(self, url=None, token=None):
if not isinstance(url, str) or not isinstance(token, str):
assert 0
from pulsar import AuthenticationToken
self._client = pulsar.Client(url,
authentication=AuthenticationToken(token))
self._client_id = generate_client_id()
def test_producer():
client = pulsar.Client('pulsar://47.94.142.82:6650')
producer = client.create_producer('my-topic')
for i in range(10):
producer.send(('Hello-%d' % i).encode('utf-8'))
client.close()
def produceByTime(self):
try:
client = pulsar.Client(self.url,
use_tls=True,
tls_allow_insecure_connection=True,
authentication=self.authToken
)
producer = client.create_producer(self.topic)
except Exception as e:
print(e)
raise SystemExit
# Publish our topic name if we are doing topicFromTopic
if self.topicFromTopic:
print("####################### GEN TOPIC #########################")
try:
s = producer.send(self.topicFromTopic.encode('utf-8'))
print("Created topic " + self.topicFromTopic + " published into " + self.topic)
self.topic = self.topicFromTopic
if self.sendAsync:
producer = client.create_producer(
self.topic,
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=100
)
else:
producer = client.create_producer(self.topic)
except Exception as e:
print("Failed to send topic name: %s", e)
raise SystemExit
if self.runForever:
print("Running FOREVER!!!!")
endTime = 0
else:
print("Running for " + str(self.runTime) + "seconds.")
endTime = self.runTime + time.time()
msg = self.genMsg()
while (time.time() < endTime) or (self.runForever):
if self.uniq:
msgToSend = msg + str(time.time()).encode()
else:
msgToSend = msg
try:
if self.sendAsync:
producer.send_async(msgToSend, self.sendAsyncCallback)
else:
producer.send(msgToSend)
except Exception as e:
print("Failed to send message: %s", e)
if self.delay:
time.sleep(self.delay/1000)
def __init__(self, cfg):
self.cfg = cfg
self.pulsar_client = pulsar.Client(cfg.pulsar_url)
if not helper.waiting4namespace(self.cfg):
self.ledger_producer = self.pulsar_client.create_producer(
topic=f"persistent://{cfg.tenant}/{cfg.market_uuid}/ledger",
schema=pulsar.schema.AvroSchema(MV2.schema.LedgerSchema))
def flush_topic():
"""A consumer that will read everything from the pulsar topic to 'reset it.'"""
topic = "testTopic"
client = pulsar.Client('pulsar://localhost:6650')
consumer = client.subscribe(topic, 'testSubscription')
while True:
msg = consumer.receive()
print("Received message '{}' id='{}'".format(msg.data(),
msg.message_id()))
consumer.acknowledge(msg)
def main(argv):
host = '192.168.50.233'
port = 6650
topic = 'test_channel'
client = pulsar.Client('pulsar://{}:{}'.format(host, port))
producer = client.create_producer(topic)
for i in range(10):
producer.send(("hello-%s" % i).encode("utf-8"))
print("Finished produce message.")
client.close()
def __init__(self, broker=None, topic=None):
if broker == None:
broker = os.getenv("PULSAR_BROKER", "pulsar://localhost:6650")
if topic == None:
routing_key = os.getenv("PULSAR_PRODUCER_TOPIC")
self.client = pulsar.Client(broker)
self.producer = self.client.create_producer(topic)