Python read_ccloud_config示例

示例#1

文件： classProducer.py 项目： hpclabdecom/BCC447_2020_02

 def __init__(self, configFile, topic):
     self.conf = ccloud_lib.read_ccloud_config(configFile)
     self.producer_conf = ccloud_lib.pop_schema_registry_params_from_config(
         self.conf)
     self.producer = Producer(self.producer_conf)
     self.topic = topic
     ccloud_lib.create_topic(self.conf, topic)

示例#2

文件： producer.py 项目： xingchengyusi/cs410-inclass

def main(config_path, should_random=False):
    topic = "inclass-2"
    config = ccloud.read_ccloud_config(config_path)

    # set log file path
    log_path = "./log"
    os.makedirs(log_path, exist_ok=True)
    log_path = os.path.join(log_path, "{}_producer.log".format(date.today()))

    # set loggingfile
    logging.basicConfig(
        level=logging.INFO,
        format="%(message)s",
        handlers=[
            # logging.FileHandler(log_path),
            logging.StreamHandler(sys.stdout),
        ],
    )

    # load, and produce json data.
    data_list = "bcsample.json"
    logging.info(data_list)
    with open(data_list) as json_file:
        data = json.load(json_file)

    # normal running
    produce(config, topic, data, should_random)

示例#3

文件： classConsumer.py 项目： hpclabdecom/BCC447_2020_02

 def __init__(self, configFile):
     self.conf = ccloud_lib.read_ccloud_config(configFile)
     self.consumer_conf = ccloud_lib.pop_schema_registry_params_from_config(
         self.conf)
     self.consumer_conf['group.id'] = 'python_example_group_1'
     self.consumer_conf['auto.offset.reset'] = 'earliest'
     self.consumer = Consumer(self.consumer_conf)

示例#4

def create_producer(config_file, topic):
    conf = ccloud_lib.read_ccloud_config(config_file)
    producer = Producer({
        'bootstrap.servers': conf['bootstrap.servers'],
        'sasl.mechanisms': conf['sasl.mechanisms'],
        'security.protocol': conf['security.protocol'],
        'sasl.username': conf['sasl.username'],
        'sasl.password': conf['sasl.password'],
    })
    # Create topic if needed
    ccloud_lib.create_topic(conf, topic)
    return producer

示例#5

def create_consumer(config_file, group_id):
    conf = ccloud_lib.read_ccloud_config(config_file)
    # Create Consumer instance
    # 'auto.offset.reset=earliest' to start reading from the beginning of the
    #   topic if no committed offsets exist
    consumer = Consumer({
        'bootstrap.servers': conf['bootstrap.servers'],
        'sasl.mechanisms': conf['sasl.mechanisms'],
        'security.protocol': conf['security.protocol'],
        'sasl.username': conf['sasl.username'],
        'sasl.password': conf['sasl.password'],
        'group.id': group_id,
        'auto.offset.reset': 'earliest',
    })
    return consumer

示例#6

def main(config_path, handle_key, group_id="activity"):
    topic = "inclass-2"
    config = ccloud.read_ccloud_config(config_path)

    # set log file path
    log_path = "./log"
    os.makedirs(log_path, exist_ok=True)
    log_path = os.path.join(log_path, "{}_consumer.log".format(date.today()))

    # set loggingfile
    logging.basicConfig(
        level=logging.INFO,
        format="%(message)s",
        handlers=[
            # logging.FileHandler(log_path),
            logging.StreamHandler(sys.stdout),
        ],
    )

    consume(config=config,
            topic=topic,
            group_id=group_id,
            handle_key=handle_key)

示例#7

文件： consumer2.py 项目： priyankaseethamraju/Data-Engineering

from confluent_kafka import Consumer
import json
import ccloud_lib

from e import load



if __name__ == '__main__':

    # Read arguments and configurations and initialize
    args = ccloud_lib.parse_args()
    config_file = args.config_file
    topic = args.topic
    conf = ccloud_lib.read_ccloud_config(config_file)

    # Create Consumer instance
    # 'auto.offset.reset=earliest' to start reading from the beginning of the
    #   topic if no committed offsets exist
    consumer = Consumer({
        'bootstrap.servers': conf['bootstrap.servers'],
        'sasl.mechanisms': conf['sasl.mechanisms'],
        'security.protocol': conf['security.protocol'],
        'sasl.username': conf['sasl.username'],
        'sasl.password': conf['sasl.password'],
        'group.id': 'python_example_group_1',
        'auto.offset.reset': 'earliest',
    })

    # Subscribe to topic

示例#8

文件： consumer.py 项目： omkreddy/examples

# Using Confluent Python Client for Apache Kafka
#
# =============================================================================

from confluent_kafka import Consumer
import json
import ccloud_lib


if __name__ == '__main__':

    # Initialization
    args = ccloud_lib.parse_args()
    config_file = args.config_file
    topic = args.topic
    conf = ccloud_lib.read_ccloud_config(config_file)

    # Create Consumer instance
    # 'auto.offset.reset=earliest' to start reading from the beginning of the
    #   topic if no committed offsets exist
    c = Consumer({
        'bootstrap.servers': conf['bootstrap.servers'],
        'sasl.mechanisms': 'PLAIN',
        'security.protocol': 'SASL_SSL',
        'sasl.username': conf['sasl.username'],
        'sasl.password': conf['sasl.password'],
        'group.id': 'python_example_group_1',
        'auto.offset.reset': 'earliest'
    })

    # Subscribe to topic

示例#9

文件： topic_clean.py 项目： xingchengyusi/cs410-inclass

#!/usr/bin/env python3
from confluent_kafka import Consumer
import ccloud_lib as ccloud


if __name__ == "__main__":
    config = ccloud.read_ccloud_config("confluent.config")
    config["group.id"] = "activity"
    config["auto.offset.reset"] = "earliest"

    consumer = Consumer(config)
    consumer.subscribe(["inclass-2"])

    while True:
        msg = consumer.poll(1)

        if msg is None:
            print("wait")
            continue
        elif msg.error():
            print("error: {}".format(msg.error()))
        else:
            print("consume")

    consumer.close()