def main(args):
topic = args.topic
delimiter = args.delimiter
producer_conf = {'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': StringSerializer('utf_8')}
producer_conf.update(sasl_conf(args))
producer = SerializingProducer(producer_conf)
print("Producing records to topic {}. ^C to exit.".format(topic))
while True:
# Serve on_delivery callbacks from previous calls to produce()
producer.poll(0.0)
try:
msg_data = input(">")
msg = msg_data.split(delimiter)
if len(msg) == 2:
producer.produce(topic=topic, key=msg[0], value=msg[1],
on_delivery=delivery_report)
else:
producer.produce(topic=topic, value=msg[0],
on_delivery=delivery_report)
except KeyboardInterrupt:
break
print("\nFlushing {} records...".format(len(producer)))
producer.flush()
def __new__(cls):
# Producer configuration. Must match Stimzi/Kafka configuration.
config = {
'bootstrap.servers': "jizt-cluster-kafka-bootstrap:9092",
'client.id': socket.gethostname(),
'key.serializer': StringSerializer('utf_8'),
'value.serializer': StringSerializer('utf_8')
}
return SerializingProducer(config)
def __init__(self, config: KafkaEmitterConfig):
self.config = config
schema_registry_conf = {
"url": self.config.connection.schema_registry_url,
**self.config.connection.schema_registry_config,
}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
def convert_mce_to_dict(mce: MetadataChangeEvent,
ctx: SerializationContext) -> dict:
tuple_encoding = mce.to_obj(tuples=True)
return tuple_encoding
mce_avro_serializer = AvroSerializer(
schema_str=getMetadataChangeEventSchema(),
schema_registry_client=schema_registry_client,
to_dict=convert_mce_to_dict,
)
def convert_mcp_to_dict(
mcp: Union[MetadataChangeProposal, MetadataChangeProposalWrapper],
ctx: SerializationContext,
) -> dict:
tuple_encoding = mcp.to_obj(tuples=True)
return tuple_encoding
mcp_avro_serializer = AvroSerializer(
schema_str=getMetadataChangeProposalSchema(),
schema_registry_client=schema_registry_client,
to_dict=convert_mcp_to_dict,
)
# We maintain a map of producers for each kind of event
producers_config = {
MCE_KEY: {
"bootstrap.servers": self.config.connection.bootstrap,
"key.serializer": StringSerializer("utf_8"),
"value.serializer": mce_avro_serializer,
**self.config.connection.producer_config,
},
MCP_KEY: {
"bootstrap.servers": self.config.connection.bootstrap,
"key.serializer": StringSerializer("utf_8"),
"value.serializer": mcp_avro_serializer,
**self.config.connection.producer_config,
},
}
self.producers = {
key: SerializingProducer(value)
for (key, value) in producers_config.items()
}
def producer_config(args):
logger = logging.getLogger(__name__)
return {
'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': StringSerializer('utf_8'),
'security.protocol': 'sasl_plaintext',
'sasl.mechanisms': 'OAUTHBEARER',
# sasl.oauthbearer.config can be used to pass argument to your oauth_cb
# It is not used in this example since we are passing all the arguments
# from command line
# 'sasl.oauthbearer.config': 'not-used',
'oauth_cb': functools.partial(_get_token, args),
'logger': logger,
}
def send_event_to_message_bus(**kwargs): # pragma: no cover
"""
Forward a SUBSCRIPTION_LICENSE_MODIFIED event to the settings.LICENSE_TOPIC_NAME queue on the event bus
:param kwargs: event data sent by signal
"""
license_data = kwargs.get('license', None)
if not license_data or not isinstance(license_data,
SubscriptionLicenseData):
logger.error(
"Received null or incorrect data from SUBSCRIPTION_LICENSE_MODIFIED"
)
return
try:
license_event_producer = ProducerFactory.get_or_create_event_producer(
settings.LICENSE_TOPIC_NAME, StringSerializer('utf-8'),
SubscriptionLicenseEventSerializer.get_serializer())
license_event_data = {"license": license_data}
message_key = license_data.enterprise_customer_uuid
event_type = kwargs['signal'].event_type
license_event_producer.produce(
settings.LICENSE_TOPIC_NAME,
key=message_key,
value=license_event_data,
on_delivery=verify_event,
headers={EVENT_TYPE_HEADER_KEY: event_type})
license_event_producer.poll()
except ValueSerializationError as vse:
logger.exception(vse)
except Exception: # pylint: disable=broad-except
logger.exception("Unknown error sending license event to event bus")
def __init__(self, config: KafkaSinkConfig, ctx):
super().__init__(ctx)
self.config = config
self.report = SinkReport()
schema_registry_conf = {
'url': self.config.connection.schema_registry_url,
**self.config.connection.schema_registry_config,
}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
def convert_mce_to_dict(mce: MetadataChangeEvent, ctx):
tuple_encoding = mce.to_obj(tuples=True)
return tuple_encoding
avro_serializer = AvroSerializer(SCHEMA_JSON_STR,
schema_registry_client,
to_dict=convert_mce_to_dict)
producer_config = {
"bootstrap.servers": self.config.connection.bootstrap,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': avro_serializer,
**self.config.connection.producer_config,
}
self.producer = SerializingProducer(producer_config)
def getConfigs():
sr_client_props = {
'url': '<CCLOUD_SR_DNS>',
'basic.auth.user.info': '<CCLOUD_SR_KEY>:<CCLOUD_SR_SECRET>'
}
sr_client = SchemaRegistryClient(sr_client_props)
value_serializer = ProtobufSerializer(FoodPreferences_pb2.PersonFood,
sr_client)
configs = {
'bootstrap.servers': '<CCLOUD_DNS>',
'security.protocol': 'SASL_SSL',
'sasl.mechanism': 'PLAIN',
'sasl.username': '******',
'sasl.password': '******',
'client.id': 'pythonProduction',
'compression.type': 'zstd',
'retries': '10',
'linger.ms': '5',
'key.serializer': StringSerializer('utf_8'),
'value.serializer': value_serializer
}
return configs
def __init__(self,
producer_name,
value_schema,
groupID='KafkaAvroProducer'):
# Consumer name for logging purposes
self.logging_prefix = '[' + producer_name + '][KafkaAvroProducer]'
# Schema Registry configuration
self.schema_registry_conf = EventBackboneConfig.getSchemaRegistryConf()
# Schema Registry Client
self.schema_registry_client = SchemaRegistryClient(
self.schema_registry_conf)
# String Serializer for the key
self.key_serializer = StringSerializer('utf_8')
# Avro Serializer for the value
self.value_serializer = AvroSerializer(value_schema,
self.schema_registry_client)
# Get the producer configuration
self.producer_conf = EventBackboneConfig.getProducerConfiguration(
groupID, self.key_serializer, self.value_serializer)
EventBackboneConfig.printProducerConfiguration(
self.logging_prefix, self.producer_conf,
self.schema_registry_conf['url'])
# Create the producer
self.producer = SerializingProducer(self.producer_conf)
def write_to_kafka(bootstrap_servers, schema_registry_url, topic_name, data):
print("Kafka Version : ", confluent_kafka.version(),confluent_kafka.libversion())
schema_registry_conf = {'url': schema_registry_url}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
value_avro_serializer = AvroSerializer(schemas.weather_source_schema, schema_registry_client)
string_serializer = StringSerializer('utf-8')
conf = {'bootstrap.servers': bootstrap_servers,
'client.id': socket.gethostname(),
'on_delivery': delivery_report,
'key.serializer': string_serializer,
'value.serializer': value_avro_serializer
}
avroProducer = SerializingProducer(conf)
key=datetime.date.today() + '~' + str(data['lat']) + '~' + str(data['lon'])
message = json.dumps(data, cls=DatetimeEncoder)
print("Key Type : ", type(key))
print("Value Type : ", type(json.loads(message)))
avroProducer.produce(topic=topic_name, key=key, value=json.loads(message))
avroProducer.flush()
def from_file(cls, config_file_path, **kwargs):
"""
config_file_path = path to the config file to use in defining this producer
Possible keyword arguments:
logger = the logger object to use
!!!!! any other keyword arguments will be added to the configuration (with underscores replaced with dots) !!!!!
"""
parser = ConfigFileParser(config_file_path, **kwargs)
configs = parser.get_config_dict_for_groups(['cluster', 'producer'])
for argname, arg in kwargs.items():
if argname == 'logger':
continue
configs[argname.replace('_', '.')] = arg
#if one of several recognized serializers have been given as config paramenters for the key/value serializer, replace them with the actual class
names_to_classes = {
'DoubleSerializer': DoubleSerializer(),
'IntegerSerializer': IntegerSerializer(),
'StringSerializer': StringSerializer(),
'DataFileChunkSerializer': DataFileChunkSerializer(),
}
configs_to_check = ['key.serializer', 'value.serializer']
for cfg in configs_to_check:
if cfg in configs.keys():
if configs[cfg] in names_to_classes:
configs[cfg] = names_to_classes[configs[cfg]]
return cls(configs)
def __init__(self, producer_name, value_schema, groupID = 'KafkaAvroProducer',
kafka_brokers = "",
kafka_user = "",
kafka_pwd = "",
kafka_cacert = "",
kafka_sasl_mechanism = "",
topic_name = ""):
self.kafka_brokers = kafka_brokers
self.kafka_user = kafka_user
self.kafka_pwd = kafka_pwd
self.kafka_sasl_mechanism = kafka_sasl_mechanism
self.kafka_cacert = kafka_cacert
self.topic_name = topic_name
# Consumer name for logging purposes
self.logging_prefix = '['+ producer_name + '][KafkaAvroProducer]'
# Schema Registry configuration
self.schema_registry_conf = {'url': config.SCHEMA_REGISTRY_URL}
# Schema Registry Client
self.schema_registry_client = SchemaRegistryClient(self.schema_registry_conf)
# String Serializer for the key
self.key_serializer = StringSerializer('utf_8')
# Avro Serializer for the value
print(value_schema)
print(type(value_schema))
value_schema=value_schema.strip()
self.value_serializer = AvroSerializer(value_schema, self.schema_registry_client)
# Get the producer configuration
self.producer_conf = self.getProducerConfiguration(groupID,
self.key_serializer,
self.value_serializer)
# Create the producer
self.producer = SerializingProducer(self.producer_conf)
def __init__(self, config: KafkaEmitterConfig):
self.config = config
schema_registry_conf = {
"url": self.config.connection.schema_registry_url,
**self.config.connection.schema_registry_config,
}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
def convert_mce_to_dict(mce: MetadataChangeEvent, ctx):
tuple_encoding = mce.to_obj(tuples=True)
return tuple_encoding
avro_serializer = AvroSerializer(
schema_str=SCHEMA_JSON_STR,
schema_registry_client=schema_registry_client,
to_dict=convert_mce_to_dict,
)
producer_config = {
"bootstrap.servers": self.config.connection.bootstrap,
"key.serializer": StringSerializer("utf_8"),
"value.serializer": avro_serializer,
**self.config.connection.producer_config,
}
self.producer = SerializingProducer(producer_config)
def test_string_serialization(kafka_cluster, data, codec):
"""
Tests basic unicode serialization/deserialization functionality
Args:
kafka_cluster (KafkaClusterFixture): cluster fixture
data (unicode): input data
codec (str): encoding type
"""
topic = kafka_cluster.create_topic("serialization-string")
producer = kafka_cluster.producer(value_serializer=StringSerializer(codec))
producer.produce(topic, value=data)
producer.flush()
consumer = kafka_cluster.consumer(
value_deserializer=StringDeserializer(codec))
consumer.subscribe([topic])
msg = consumer.poll()
assert msg.value() == data
consumer.close()
class Configuration:
commit_log_topic: str
snapshot_topic: str
bootstrap_servers: str
group_id: str
batch_timeout_sec: int = 5
messages_per_transaction: int = 2000
store_consumer: dict = field(
default_factory=lambda: {
'bootstrap.servers': None,
'group.id': None,
'auto.offset.reset': 'earliest',
'enable.auto.commit': False,
'enable.auto.offset.store': False,
'enable.partition.eof': True,
'key.deserializer': StringDeserializer(),
'value.deserializer': JSONDeserializer(),
# 'stats_cb': publish_statistics,
# 'statistics.interval.ms': 15000,
})
consumer: dict = field(
default_factory=lambda: {
'bootstrap.servers': None,
'group.id': None,
'auto.offset.reset': 'earliest',
'enable.auto.commit': False,
'enable.auto.offset.store': False,
'enable.partition.eof': False,
'key.deserializer': StringDeserializer(),
'value.deserializer': JSONDeserializer(),
# 'value.deserializer': DummyDeserializer(),
# 'stats_cb': publish_statistics,
# 'statistics.interval.ms': 15000,
})
producer: dict = field(
default_factory=lambda: {
'bootstrap.servers': None,
'transactional.id': None,
'transaction.timeout.ms': 60000,
'enable.idempotence': True,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': JSONSerializer(),
'debug': 'broker,eos',
})
def __post_init__(self):
self.store_consumer['bootstrap.servers'] = \
self.consumer['bootstrap.servers'] = \
self.producer['bootstrap.servers'] = \
self.bootstrap_servers
self.store_consumer['group.id'] = \
self.consumer['group.id'] = \
self.producer['transactional.id'] = \
self.group_id
def main(args):
topic = args.topic
outputtopic = args.outputtopic
schema_str = EventSchema
schema_enriched_event_str = EnrichedEventSchema
sr_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(sr_conf)
avro_deserializer = AvroDeserializer(schema_str, schema_registry_client)
string_deserializer = StringDeserializer('utf_8')
avro_serializer = AvroSerializer(schema_enriched_event_str,
schema_registry_client)
consumer_conf = {
'bootstrap.servers': args.bootstrap_servers,
'key.deserializer': string_deserializer,
'value.deserializer': avro_deserializer,
'group.id': args.group + str(random.Random()),
'auto.offset.reset': "latest"
}
consumer = DeserializingConsumer(consumer_conf)
consumer.subscribe([topic])
cluster = Cluster([args.host])
session = cluster.connect("datascience")
session.row_factory = dict_factory
producer_conf = {
'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': avro_serializer
}
producer = SerializingProducer(producer_conf)
while True:
try:
# SIGINT can't be handled when polling, limit timeout to 1 second.
start = time.time()
msg = consumer.poll(1.0)
if msg is None:
continue
evt = msg.value()
enrich(evt, session, producer, outputtopic)
except Exception:
print('Exception', sys.exc_info()[0])
continue
consumer.close()
def main(args):
topic = args.topic
schema_str = """
{
"namespace": "confluent.io.examples.serialization.avro",
"name": "User",
"type": "record",
"fields": [
{"name": "name", "type": "string"},
{"name": "favorite_number", "type": "int"},
{"name": "favorite_color", "type": "string"}
]
}
"""
schema_registry_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
avro_serializer = AvroSerializer(schema_registry_client, schema_str,
user_to_dict)
producer_conf = {
'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': avro_serializer
}
producer = SerializingProducer(producer_conf)
print("Producing user records to topic {}. ^C to exit.".format(topic))
while True:
# Serve on_delivery callbacks from previous calls to produce()
producer.poll(0.0)
try:
user_name = input("Enter name: ")
user_address = input("Enter address: ")
user_favorite_number = int(input("Enter favorite number: "))
user_favorite_color = input("Enter favorite color: ")
user = User(name=user_name,
address=user_address,
favorite_color=user_favorite_color,
favorite_number=user_favorite_number)
producer.produce(topic=topic,
key=str(uuid4()),
value=user,
on_delivery=delivery_report)
except KeyboardInterrupt:
break
except ValueError:
print("Invalid input, discarding record...")
continue
print("\nFlushing records...")
producer.flush()
def avro_messages_producer(schema):
schema_registry_conf = {'url': config['kafka']['schema_registry']}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
avro_serializer = AvroSerializer(schema, schema_registry_client)
string_serializer = StringSerializer('utf-8')
producer_conf = {
'bootstrap.servers': config['kafka']['servers'],
'key.serializer': string_serializer,
'value.serializer': avro_serializer
}
return SerializingProducer(producer_conf)
def send_record(args):
""" Sends Record using a SerializingProducer & AvroSerializer """
topic = args.topic.rstrip()
schema_registry_config = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(schema_registry_config)
avro_serializer = AvroSerializer(schema_registry_client, DATA_SCHEMA,
data_to_dict)
producer_config = {
"bootstrap.servers": args.bootstrap_servers,
"key.serializer": StringSerializer('utf_8'),
"value.serializer": avro_serializer
}
producer = SerializingProducer(producer_config)
split_incoming_data = args.record_value.split(',')
if not len(split_incoming_data) == 7: # Data Format Check
print('** Error: Insufficient Incoming Data: ', split_incoming_data)
raise Exception
try: # Data Format Check
incoming_data = {
'envId': int(split_incoming_data[0]),
'whenCollected': str(split_incoming_data[1]),
'timeLightOnMins': int(split_incoming_data[2]),
'humidity': int(split_incoming_data[3]),
'soilMoisture': int(split_incoming_data[4]),
'temperature': int(split_incoming_data[5]),
'waterConsumption': int(split_incoming_data[6])
}
except Exception as error:
print('** Error Creating Dict of Data: ', error)
print(f'Producing data records to topic {topic}. ^C to exit.')
producer.poll(1)
try:
key = args.record_key if args.record_key else str(uuid4())
data_object = Data(incoming_data)
print('\t-Producing Avro record. . .')
producer.produce(topic=topic,
key=key,
value=data_object,
on_delivery=delivery_report)
except ValueError:
print('\t-Invalid input, discarding record. . .')
print('\nFlushing records. . .')
producer.flush()
def __init__(self, value_schema, groupID='KafkaAvroProducer'):
# Schema Registry configuration
self.schema_registry_conf = self.getSchemaRegistryConf()
# Schema Registry Client
self.schema_registry_client = SchemaRegistryClient(
self.schema_registry_conf)
# String Serializer for the key
self.key_serializer = StringSerializer('utf_8')
# Avro Serializer for the value
self.value_serializer = AvroSerializer(value_schema,
self.schema_registry_client)
# Get the producer configuration
self.producer_conf = self.getProducerConfiguration(groupID)
# Create the producer
self.producer = SerializingProducer(self.producer_conf)
def test_consume_error(kafka_cluster):
"""
Tests to ensure librdkafka errors are propagated as
an instance of ConsumeError.
"""
topic = kafka_cluster.create_topic("test_commit_transaction")
consumer_conf = {'enable.partition.eof': True}
producer = kafka_cluster.producer()
producer.produce(topic=topic, value="a")
producer.flush()
consumer = kafka_cluster.consumer(consumer_conf,
value_deserializer=StringSerializer())
consumer.assign([TopicPartition(topic, 0, OFFSET_END)])
with pytest.raises(ConsumeError, match="No more messages"):
# Trigger EOF error
consumer.poll()
def main(args):
topic = args.topic
schema_registry_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
protobuf_serializer = ProtobufSerializer(user_pb2.User,
schema_registry_client,
{'use.deprecated.format': True})
producer_conf = {
'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': protobuf_serializer
}
producer = SerializingProducer(producer_conf)
print("Producing user records to topic {}. ^C to exit.".format(topic))
while True:
# Serve on_delivery callbacks from previous calls to produce()
producer.poll(0.0)
try:
user_name = input("Enter name: ")
user_favorite_number = int(input("Enter favorite number: "))
user_favorite_color = input("Enter favorite color: ")
user = user_pb2.User(name=user_name,
favorite_color=user_favorite_color,
favorite_number=user_favorite_number)
producer.produce(topic=topic,
partition=0,
key=str(uuid4()),
value=user,
on_delivery=delivery_report)
except (KeyboardInterrupt, EOFError):
break
except ValueError:
print("Invalid input, discarding record...")
continue
print("\nFlushing records...")
producer.flush()
def __init__(self, config_env):
self.config = config_env
self.topic_name = self.config["kafka_produce_topic"]
conf = {
'bootstrap.servers':
self.config["bootstrap_servers"],
'message.max.bytes':
self.config["kafkaMaxMessageBytes"],
'queue.buffering.max.ms':
self.config["queue.buffering.max.ms"],
'queue.buffering.max.messages':
self.config["queue.buffering.max.messages"],
'key.serializer':
StringSerializer('utf_8'),
'value.serializer':
self.__protobuf_serializer()
}
self.producer = SerializingProducer(conf)
def test_consume_error(kafka_cluster):
"""
Tests to ensure librdkafka errors are propagated as
an instance of ConsumeError.
"""
topic = kafka_cluster.create_topic("test_commit_transaction")
consumer_conf = {'group.id': 'pytest', 'enable.partition.eof': True}
producer = kafka_cluster.producer()
producer.produce(topic=topic, value="a")
producer.flush()
consumer = kafka_cluster.consumer(consumer_conf,
value_deserializer=StringSerializer())
consumer.assign([TopicPartition(topic, 0, OFFSET_END)])
with pytest.raises(ConsumeError) as exc_info:
# Trigger EOF error
consumer.poll()
assert exc_info.value.args[0].code() == KafkaError._PARTITION_EOF, \
"Expected _PARTITION_EOF, not {}".format(exc_info)
def __init__(self,
bootstrap_servers,
rss_feeds,
topic='crawl-queue',
time_checkpoint_fn_base='scheduler_checkpoint'):
self.bootstrap_servers = bootstrap_servers
self.topic = topic
self.feeds = rss_feeds
self.time_checkpoints = dict()
for spider_name in self.feeds.values():
fn = f'{time_checkpoint_fn_base}_{spider_name}.txt'
fn = fn.replace('/', '_') # we don't want / in our pathnames.
self.time_checkpoints[spider_name] = TimeCheckpoint(fn=fn)
producer_conf = {
'bootstrap.servers': self.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': _json_serializer_wrapper
}
self.producer = SerializingProducer(producer_conf)
def produce(self, count: int):
def increment(err, msg):
assert err is None
assert msg is not None
assert msg.offset() == self.acked
self.logger.debug("Acked offset %d", msg.offset())
self.acked += 1
producer = SerializingProducer({
'bootstrap.servers':
self.brokers,
'key.serializer':
StringSerializer('utf_8'),
'value.serializer':
self._make_serializer()
})
self.logger.info("Producing %d %s records to topic %s", count,
self.schema_type.name, self.topic)
for i in range(count):
# Prevent overflow of buffer
while len(producer) > 50000:
# Serve on_delivery callbacks from previous calls to produce()
producer.poll(0.1)
producer.produce(topic=self.topic,
key=str(uuid4()),
value=self._make_payload(i),
on_delivery=increment)
self.produced += 1
self.logger.info("Flushing records...")
producer.flush()
self.logger.info("Records flushed: %d", self.produced)
while self.acked < count:
producer.poll(0.01)
self.logger.info("Records acked: %d", self.acked)
def main(args):
topic = args.topic
schema_registry_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
avro_serializer = AvroSerializer(User.avro_schema(),
schema_registry_client,
user_to_dict)
producer_conf = {'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': avro_serializer}
producer = SerializingProducer(producer_conf)
print(f"Producing user records to topic {topic}. ^C to exit.")
while True:
producer.poll(0.0)
try:
user_name = input("Enter name: ")
user_favorite_number = int(input("Enter favorite number: "))
user_favorite_color = input("Enter favorite color: ")
user = User(name=user_name,
favorite_color=user_favorite_color,
favorite_number=user_favorite_number)
producer.produce(topic=topic, key=str(uuid4()), value=user,
on_delivery=delivery_report)
except KeyboardInterrupt:
break
except ValueError:
print("Invalid input, discarding record...")
continue
print("\nFlushing records...")
producer.flush()
def main(args):
topic = args.topic
schema_str = EventSchema
schema_registry_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
avro_serializer = AvroSerializer(schema_str, schema_registry_client)
producer_conf = {
'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': avro_serializer
}
producer = SerializingProducer(producer_conf)
list_type = [{
"grilleIdent": "Numero 123T",
"codeRetourServiceMetier": "code 23432543",
"referer": "1qsd",
"browserVersion": "qsdqsd",
"androidUDID": "qsdqsdqsd",
"iosIDFA": "qdqsdqsd",
"appVersion": "qsdqsdqsdqsd",
"idTmx": "qsdqsdqsd"
}, {
"numeroCompteBeneficiaire": "Numero 123T",
"codePaysResidence": "code 23432543",
"codePaysResidenceIso": "code 23432543",
"adresseBeneficiaire": "code 23432543",
"nomCompletBeneficiaire": "code 23432543",
"idListeBeneficiaire": "code 23432543",
"idBeneficiaire": "code 23432543",
"modeValidation": 34,
"bicBeneficiaire": "code 23432543",
"idTmx": "code 23432543"
}]
while True:
x = random.choice([0, 1])
eventHeader = {
"eventId": str(uuid4()),
"dateTimeRef": 1589364605654,
"nomenclatureEv": "Event Header",
"canal": 1,
"media": 2,
"schemaVersion": "v0",
"headerVersion": "v2",
"serveur": "s1",
"acteurDeclencheur": {
"adresseIP": "127.0.0.1",
"idTelematique": str(uuid4()),
"idPersonne": "zahir"
}
}
value = {
"EventHeader": eventHeader,
"EventBusinessContext": list_type[x]
}
print(value)
producer.produce(topic=topic,
key=str(uuid4()),
value=value,
on_delivery=delivery_report)
producer.flush()
time.sleep(0.1)
# Create it by running
# protoc -I=. --python_out=. ./meal.proto
import meal_pb2
from confluent_kafka import SerializingProducer
from confluent_kafka.serialization import StringSerializer
from confluent_kafka.schema_registry import SchemaRegistryClient
from confluent_kafka.schema_registry.protobuf import ProtobufSerializer
topic = 'meal'
schema_registry_client = SchemaRegistryClient({'url': 'http://t620.lan:8081'})
protobuf_serializer = ProtobufSerializer(meal_pb2.Meal, schema_registry_client)
producer_conf = {
'bootstrap.servers': 't620.lan:9092',
'key.serializer': StringSerializer('utf_8'),
'value.serializer': protobuf_serializer
}
producer = SerializingProducer(producer_conf)
producer.poll(0.0)
mybeer = meal_pb2.Meal.DrinkItems(drink_name="beer")
mywine = meal_pb2.Meal.DrinkItems(drink_name="wine")
meal = meal_pb2.Meal(name='pizza', drink=[mybeer, mywine])
# Less meal (for testing kafkajs which seems to miss the drinks
#meal = meal_pb2.Meal(name='pizza', drink=[])
producer.produce(topic=topic, key=str(uuid4()), value=meal)
def main(args):
topic = args.topic
schema_str = """
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "User",
"description": "A Confluent Kafka Python User",
"type": "object",
"properties": {
"name": {
"description": "User's name",
"type": "string"
},
"favorite_number": {
"description": "User's favorite number",
"type": "number",
"exclusiveMinimum": 0
},
"favorite_color": {
"description": "User's favorite color",
"type": "string"
}
},
"required": [ "name", "favorite_number", "favorite_color" ]
}
"""
schema_registry_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(schema_registry_conf)
json_serializer = JSONSerializer(schema_registry_client, schema_str,
user_to_dict)
producer_conf = {
'bootstrap.servers': args.bootstrap_servers,
'key.serializer': StringSerializer('utf_8'),
'value.serializer': json_serializer
}
producer = SerializingProducer(producer_conf)
print("Producing user records to topic {}. ^C to exit.".format(topic))
while True:
# Serve on_delivery callbacks from previous calls to produce()
producer.poll(0.0)
try:
user_name = input("Enter name: ")
user_address = input("Enter address: ")
user_favorite_number = int(input("Enter favorite number: "))
user_favorite_color = input("Enter favorite color: ")
user = User(name=user_name,
address=user_address,
favorite_color=user_favorite_color,
favorite_number=user_favorite_number)
producer.produce(topic=topic,
key=str(uuid4()),
value=user,
on_delivery=delivery_report)
except KeyboardInterrupt:
break
except ValueError:
print("Invalid input, discarding record...")
continue
print("\nFlushing records...")
producer.flush()
from datetime import datetime
import toml
def acked(err, msg):
if err is not None:
print("Failed to deliver message: %s: %s" % (str(msg), str(err)))
else:
print(
'Message {} successfully produced to {} [{}] at offset {}'.format(
msg.key(), msg.topic(), msg.partition(), msg.offset()))
# Load HopsWorks Kafka configuration
conf = toml.load('config.toml')
# Initialize a simple String serializer for the key
string_serializer = StringSerializer('utf_8')
producer_conf = {
'bootstrap.servers': conf['hops']['url'] + ':' + conf['kafka']['port'],
'security.protocol': 'SSL',
'ssl.ca.location': conf['project']['ca_file'],
'ssl.certificate.location': conf['project']['certificate_file'],
'ssl.key.location': conf['project']['key_file'],
'ssl.key.password': conf['project']['key_password'],
'key.serializer': string_serializer,
'value.serializer': string_serializer,
'client.id': socket.gethostname()
}
print(producer_conf)