def __init__(
self,
broker_service_url=None,
consumer=None,
producer=None,
):
self._consumer = None
self._producer = None
if consumer and type(consumer) is not Consumer:
raise TypeError('Input should be of type Consumer')
if producer and type(producer) is not Producer:
raise TypeError('Input should be of type Producer')
self.client = pulsar.Client(broker_service_url)
if consumer:
self._consumer = self.client.subscribe(
consumer.topic,
consumer.subscription_name,
consumer_type=_pulsar.ConsumerType.Shared,
negative_ack_redelivery_delay_ms=500,
schema=AvroSchema(consumer.schema_class))
if producer:
self._producer = self.client.create_producer(
producer.topic,
batching_enabled=True,
batching_max_publish_delay_ms=10,
schema=AvroSchema(producer.schema_class))
def redeliver_message(self):
client = pulsar.Client(
service_url=self._url,
authentication=AuthenticationToken(token=self._token))
consumer = client.subscribe(topic=self._topics,
subscription_name=self._subscription_name,
schema=AvroSchema(MilvusRecord),
consumer_type=ConsumerType.KeyShared)
delete_producers = []
insert_producers = []
for topic in self._topics:
d_and_u_topic = topic + "-delete"
delete_producers.append(
client.create_producer(topic=d_and_u_topic,
schema=AvroSchema(MilvusRecord)))
i_and_q_topic = topic + "-insert"
insert_producers.append(
client.create_producer(topic=i_and_q_topic,
schema=AvroSchema(MilvusRecord)))
print("Service start successful !!!")
while True:
msg = consumer.receive()
message_topic_name = msg.topic_name()
for i, topic in enumerate(self._topics):
if message_topic_name.find(topic) != -1:
print(i)
if msg.value().op in [Op.insert, Op.query]:
insert_producers[i].send(
content=msg.value(),
event_timestamp=int(time.time() * 1000),
partition_key=str(msg.value().id))
else:
delete_producers[i].send(
content=msg.value(),
event_timestamp=int(time.time() * 1000),
partition_key=str(msg.value().id))
break
consumer.acknowledge(msg)
def send(self, vectors, ids=None):
from pulsar.schema import AvroSchema
producer = self._client.create_producer(self._topic, schema=AvroSchema(MilvusRecord))
if ids is None:
ids = generate_ids()
assert (len(ids) >= len(vectors))
# TODO: batch sending
for i in range(len(vectors)):
producer.send(MilvusRecord(client_id=self._client_id,
id=ids[i],
op=self._op,
vector=vectors[i]),
partition_key=str(ids[i]))
def consume(topic):
url = 'pulsar://localhost:6650'
token = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyLTEifQ.8oAwbbd8dd3ZYhCWKAiShP4Kd0nHSwvQbTMX7Iat_o0'
client = pulsar.Client(service_url=url,
authentication=AuthenticationToken(token=token))
consumer = client.subscribe(topic=topic,
subscription_name=topic,
schema=AvroSchema(MilvusRecord),
consumer_type=ConsumerType.Shared)
while True:
msg = consumer.receive()
ex = msg.value()
try:
print("Received message id={} op={} topic_name={}".format(
ex.id, ex.op, msg.topic_name()))
# Acknowledge successful processing of the message
consumer.acknowledge(msg)
except:
# Message failed to be processed
consumer.negative_acknowledge(msg)
import pulsar
from pulsar.schema import AvroSchema
from example03_nested_schema_with_doc_namespace.FieldLocationNested import FieldLocationNested
client = pulsar.Client('pulsar://localhost:6650')
consumer = client.subscribe(
topic='persistent://climate/field-service/nested-field-locations',
subscription_name='my-subscription',
schema=AvroSchema(FieldLocationNested))
while True:
msg = consumer.receive()
locationEvent = msg.value()
print("Event is: ", locationEvent)
print("Userdata is: ", locationEvent.Userdata)
# print(locationEvent.latitude)
# print(locationEvent.longitude)
# print(locationEvent.Userdata.fieldId)
# print(locationEvent.Userdata.action)
consumer.acknowledge(msg)
client.close()
import pulsar
from pulsar.schema import AvroSchema
from experimental.example04_simplest_schema import Example
client = pulsar.Client('pulsar://localhost:6650')
topicSchema = AvroSchema(Example)
print("Schema info is: " + topicSchema.schema_info().schema())
# Replace with your topic name. However, the namespace for the topic should have these schema compatibility settings:
# "schema_auto_update_compatibility_strategy" : "Full",
# "schema_compatibility_strategy" : "ALWAYS_COMPATIBLE",
# "is_allow_auto_update_schema" : true,
# "schema_validation_enforced" : true
producer = client.create_producer(
topic='persistent://climate/field-service/simplest-schema',
schema=topicSchema)
producer.send(Example({'name': 'Python producer'}))
client.close()
print("Successfully sent message")
import pulsar
from pulsar.schema import AvroSchema
from example07_two_schema_versions_no_default_values.SchemaV0 import NoNestingNoDefaultValues
client = pulsar.Client('pulsar://localhost:6650')
topicSchema = AvroSchema(NoNestingNoDefaultValues)
print("Schema info is: " + topicSchema.schema_info().schema())
# Replace with your topic name. However, the namespace for the topic should have these schema compatibility settings:
# "schema_auto_update_compatibility_strategy" : "Full",
# "schema_compatibility_strategy" : "ALWAYS_COMPATIBLE",
# "is_allow_auto_update_schema" : true,
# "schema_validation_enforced" : true
producer = client.create_producer(
topic='persistent://climate/field-service/two-schemas-no-defaults',
schema=topicSchema)
producer.send(NoNestingNoDefaultValues(name="Python producer"))
client.close()
print("Successfully sent message")
import pulsar
from pulsar.schema import AvroSchema
from example04_simplest_schema_java_compatible.SimpleSchema import Example
client = pulsar.Client('pulsar://localhost:6650')
consumer = client.subscribe(
topic='persistent://climate/field-service/simplest-schema',
subscription_name='my-subscription',
schema=AvroSchema(Example))
while True:
msg = consumer.receive()
exampleEvent = msg.value()
print("Example event is %s" % exampleEvent)
consumer.acknowledge(msg)
client.close()
import pulsar
from pulsar.schema import AvroSchema
from example06_default_values_no_nulls.SchemaV2 import DefaultValuesNoNulls, DefaultValuesNoNullsUserdataSchema
client = pulsar.Client('pulsar://localhost:6650')
topicSchema = AvroSchema(DefaultValuesNoNulls)
print("Schema info is: " + topicSchema.schema_info().schema())
# Replace with your topic name. However, the namespace for the topic should have these schema compatibility settings:
# "schema_auto_update_compatibility_strategy" : "Full",
# "schema_compatibility_strategy" : "ALWAYS_COMPATIBLE",
# "is_allow_auto_update_schema" : true,
# "schema_validation_enforced" : true
producer = client.create_producer(
topic='persistent://climate/field-service/default-values-nested-no-nulls',
schema=topicSchema)
event = DefaultValuesNoNulls(
name="Python producer",
floatValue=1.23,
DefaultValuesNoNullsUserdata=DefaultValuesNoNullsUserdataSchema())
producer.send(event)
client.close()
print("Successfully sent message")
import pulsar
from pulsar.schema import AvroSchema
from example06_default_values_no_nulls.SchemaV1 import DefaultValuesNoNulls
client = pulsar.Client('pulsar://localhost:6650')
consumer = client.subscribe(
topic='persistent://climate/field-service/default-values-nested-no-nulls',
subscription_name='my-subscription',
schema=AvroSchema(DefaultValuesNoNulls))
while True:
msg = consumer.receive()
exampleEvent = msg.value()
print("Example event is %s"%exampleEvent)
consumer.acknowledge(msg)
client.close()
import pulsar
from pulsar.schema import AvroSchema
from example03_nested_schema_with_doc_namespace.FieldLocationNested import FieldLocationNested, UserdataSchema
client = pulsar.Client('pulsar://localhost:6650')
topicSchema = AvroSchema(FieldLocationNested)
print("Schema info is: " + topicSchema.schema_info().schema())
# Replace with your topic name. However, the namespace for the topic should have these schema compatibility settings:
# "schema_auto_update_compatibility_strategy" : "AutoUpdateDisabled",
# "schema_compatibility_strategy" : "FORWARD_TRANSITIVE",
# "is_allow_auto_update_schema" : false,
# "schema_validation_enforced" : true
producer = client.create_producer(
topic='persistent://climate/field-service/nested-field-locations',
schema=topicSchema)
userdata = UserdataSchema(fieldId="Python", action="action")
fieldLocationNested = FieldLocationNested(latitude=1.0,
longitude=2.0,
Userdata=userdata)
producer.send(fieldLocationNested)
client.close()
print("\nSuccessfully sent message\n")
import pulsar
from pulsar.schema import AvroSchema
from example07_two_schema_versions_no_default_values.SchemaV1 import NoNestingNoDefaultValues
client = pulsar.Client('pulsar://localhost:6650')
consumer = client.subscribe(
topic='persistent://climate/field-service/two-schemas-no-defaults',
subscription_name='my-subscription',
schema=AvroSchema(NoNestingNoDefaultValues))
while True:
msg = consumer.receive()
exampleEvent = msg.value()
print("Example event is %s" % exampleEvent)
consumer.acknowledge(msg)
client.close()
