def __init__(self, configFile, topic):
self.conf = ccloud_lib.read_ccloud_config(configFile)
self.producer_conf = ccloud_lib.pop_schema_registry_params_from_config(
self.conf)
self.producer = Producer(self.producer_conf)
self.topic = topic
ccloud_lib.create_topic(self.conf, topic)
def main(config_path, should_random=False):
topic = "inclass-2"
config = ccloud.read_ccloud_config(config_path)
# set log file path
log_path = "./log"
os.makedirs(log_path, exist_ok=True)
log_path = os.path.join(log_path, "{}_producer.log".format(date.today()))
# set loggingfile
logging.basicConfig(
level=logging.INFO,
format="%(message)s",
handlers=[
# logging.FileHandler(log_path),
logging.StreamHandler(sys.stdout),
],
)
# load, and produce json data.
data_list = "bcsample.json"
logging.info(data_list)
with open(data_list) as json_file:
data = json.load(json_file)
# normal running
produce(config, topic, data, should_random)
def __init__(self, configFile):
self.conf = ccloud_lib.read_ccloud_config(configFile)
self.consumer_conf = ccloud_lib.pop_schema_registry_params_from_config(
self.conf)
self.consumer_conf['group.id'] = 'python_example_group_1'
self.consumer_conf['auto.offset.reset'] = 'earliest'
self.consumer = Consumer(self.consumer_conf)
def create_producer(config_file, topic):
conf = ccloud_lib.read_ccloud_config(config_file)
producer = Producer({
'bootstrap.servers': conf['bootstrap.servers'],
'sasl.mechanisms': conf['sasl.mechanisms'],
'security.protocol': conf['security.protocol'],
'sasl.username': conf['sasl.username'],
'sasl.password': conf['sasl.password'],
})
# Create topic if needed
ccloud_lib.create_topic(conf, topic)
return producer
def create_consumer(config_file, group_id):
conf = ccloud_lib.read_ccloud_config(config_file)
# Create Consumer instance
# 'auto.offset.reset=earliest' to start reading from the beginning of the
# topic if no committed offsets exist
consumer = Consumer({
'bootstrap.servers': conf['bootstrap.servers'],
'sasl.mechanisms': conf['sasl.mechanisms'],
'security.protocol': conf['security.protocol'],
'sasl.username': conf['sasl.username'],
'sasl.password': conf['sasl.password'],
'group.id': group_id,
'auto.offset.reset': 'earliest',
})
return consumer
def main(config_path, handle_key, group_id="activity"):
topic = "inclass-2"
config = ccloud.read_ccloud_config(config_path)
# set log file path
log_path = "./log"
os.makedirs(log_path, exist_ok=True)
log_path = os.path.join(log_path, "{}_consumer.log".format(date.today()))
# set loggingfile
logging.basicConfig(
level=logging.INFO,
format="%(message)s",
handlers=[
# logging.FileHandler(log_path),
logging.StreamHandler(sys.stdout),
],
)
consume(config=config,
topic=topic,
group_id=group_id,
handle_key=handle_key)
from confluent_kafka import Consumer
import json
import ccloud_lib
from e import load
if __name__ == '__main__':
# Read arguments and configurations and initialize
args = ccloud_lib.parse_args()
config_file = args.config_file
topic = args.topic
conf = ccloud_lib.read_ccloud_config(config_file)
# Create Consumer instance
# 'auto.offset.reset=earliest' to start reading from the beginning of the
# topic if no committed offsets exist
consumer = Consumer({
'bootstrap.servers': conf['bootstrap.servers'],
'sasl.mechanisms': conf['sasl.mechanisms'],
'security.protocol': conf['security.protocol'],
'sasl.username': conf['sasl.username'],
'sasl.password': conf['sasl.password'],
'group.id': 'python_example_group_1',
'auto.offset.reset': 'earliest',
})
# Subscribe to topic
# Using Confluent Python Client for Apache Kafka
#
# =============================================================================
from confluent_kafka import Consumer
import json
import ccloud_lib
if __name__ == '__main__':
# Initialization
args = ccloud_lib.parse_args()
config_file = args.config_file
topic = args.topic
conf = ccloud_lib.read_ccloud_config(config_file)
# Create Consumer instance
# 'auto.offset.reset=earliest' to start reading from the beginning of the
# topic if no committed offsets exist
c = Consumer({
'bootstrap.servers': conf['bootstrap.servers'],
'sasl.mechanisms': 'PLAIN',
'security.protocol': 'SASL_SSL',
'sasl.username': conf['sasl.username'],
'sasl.password': conf['sasl.password'],
'group.id': 'python_example_group_1',
'auto.offset.reset': 'earliest'
})
# Subscribe to topic
#!/usr/bin/env python3
from confluent_kafka import Consumer
import ccloud_lib as ccloud
if __name__ == "__main__":
config = ccloud.read_ccloud_config("confluent.config")
config["group.id"] = "activity"
config["auto.offset.reset"] = "earliest"
consumer = Consumer(config)
consumer.subscribe(["inclass-2"])
while True:
msg = consumer.poll(1)
if msg is None:
print("wait")
continue
elif msg.error():
print("error: {}".format(msg.error()))
else:
print("consume")
consumer.close()
