def __init__(self, config): # pragma: no cover """ Streaming client implementation based on Kafka. Configuration keys: KAFKA_ADDRESS KAFKA_CONSUMER_GROUP KAFKA_TOPIC TIMEOUT EVENTHUB_KAFKA_CONNECTION_STRING """ self.logger = Logger() self.topic = config.get("KAFKA_TOPIC") if config.get("TIMEOUT"): try: self.timeout = int(config.get("TIMEOUT")) except ValueError: self.timeout = None else: self.timeout = None kafka_config = self.create_kafka_config(config) self.admin = admin.AdminClient(kafka_config) if config.get("KAFKA_CONSUMER_GROUP") is None: self.logger.info('Creating Producer') self.producer = Producer(kafka_config) else: self.logger.info('Creating Consumer') self.consumer = Consumer(kafka_config)
def delete_kafka_topics(kafka_broker, topic_regex): # test valid broker, it will throw in case of invalid broker registered_topics = get_topics(kafka_broker) kba = admin.AdminClient({'bootstrap.servers': kafka_broker}) """ delete topics """ # Call delete_topics to asynchronously delete topics, a future is returned. # By default this operation on the broker returns immediately while # topics are deleted in the background. But here we give it some time (30s) # to propagate in the cluster before returning. # # Returns a dict of <topic,future>. topics = [] regtopic = re.compile(topic_regex) for artop in registered_topics[1]: if regtopic.match(artop): topics.append(artop) if not topics: return [] fs = kba.delete_topics(topics) # Wait for operation to finish. for topic, f in fs.items(): # it will throw if topic can not be deleted f.result() # The result itself is None print("Topic {} deleted".format(topic)) return topics
def __init__(self, config): """ Class to create a KafkaStreamingClient instance. :param config: Dictionary file with all the relevant parameters. :param topic: A string kafka topic. """ self.topic = config.get("KAFKA_TOPIC") if config.get("TIMEOUT"): try: self.timeout = int(config.get("TIMEOUT")) except ValueError: self.timeout = None else: self.timeout = None kafka_config = self.create_kafka_config(config) self.admin = admin.AdminClient(kafka_config) if config.get("KAFKA_CONSUMER_GROUP") is None: self.producer = Producer(kafka_config) else: self.app_host = config.get("APP_HOST") self.app_port = config.get("APP_PORT") self.consumer = Consumer(kafka_config)
def initialize_admin(self): """ Creates an kafka admin object :return: kafka admin object """ adm = admin.AdminClient(self.__config) return adm
def create_kafka_admin(self): try: print("Connecting to Kafka Server....") admin_client = admin.AdminClient( {'bootstrap.servers': self.broker}) print("Successfully Connected!") return admin_client except Exception as ex: print('Exception while connecting Kafka') raise
def __init__(self, app): self.app = app self.system_topics = None self.bootstrap_server = app.config["KAFKA_BOOTSTRAP_SERVER"] # Create kafka admin client if there is a connection if not self.bootstrap_server: self.k_admin_client = None self.k_admin_client = kafka_admin.AdminClient( {'bootstrap.servers': self.bootstrap_server}) # the system_topics are updated in the get_connection method self.get_connection()
def create_topic(cfg: config.Config): """Ensure that the required Kafka topic exists.""" # based on code in https://github.com/confluentinc/confluent-kafka-python client = admin.AdminClient(cfg.get_kafka_config()) topic = admin.NewTopic(cfg.kafka_topic, num_partitions=3, replication_factor=1) topic_map = client.create_topics([topic]) for (topic, future) in topic_map.items(): try: future.result() logger.info('Topic %s created', topic) except Exception as err: if err.args[0].code() != kafka.KafkaError.TOPIC_ALREADY_EXISTS: raise logger.debug('Topic %s already exists', topic)
def setConnection(self, connectParamsFile='Vinnig/KafkaConnectionSettings.json'): try: home = os.path.expanduser("~") filenameC = os.path.join(home, connectParamsFile) with open(filenameC) as fc: jsC = json.load(fc) self.connectParams = jsC['connectParams'] self.topicBasename = jsC['topicBasename'] self.producer = Producer( self.connectParams) # producer only used for log self.producer.poll(0.1) self.kafka_admin = admin.AdminClient(self.connectParams) self.topics = self.kafka_admin.list_topics().topics except Exception as e: print('Problem connecting to Kafka: {}'.format(e))
def check_kafka(timeout=30): end_time = time.time() + timeout client = admin.AdminClient({"bootstrap.servers": KAFKA_BROKER_URL}) print(f"Waiting for kafka") while time.time() < end_time: try: cluster_meta_data = client.list_topics(timeout=1) if KAFKA_TOPIC in cluster_meta_data.topics.keys(): print("Kafka is up!") return time.sleep(0.3) except KafkaException: time.sleep(0.3) continue else: raise Exception(f"Could not fin {KAFKA_TOPIC} in topics")
def __init__(self, connectParamsFile='Vinnig/KafkaConnectionSettings.json', connectParams=None, topicBasename=''): if ( connectParams == None ): # then read the configuration file. Location is relative to home directory home = os.path.expanduser("~") filename = os.path.join(home, connectParamsFile) with open(filename) as f: js = json.load(f) connectParams = js['connectParams'] topicBasename = js['topicBasename'] self.kafka_admin = admin.AdminClient(connectParams) self.topics = self.kafka_admin.list_topics().topics self.connectParams = connectParams self.topicBasename = topicBasename self.device = None self.deviceName = 'Unknown'
def __init__(self, config): # pragma: no cover """ Streaming client implementation based on Kafka. Configuration keys: KAFKA_ADDRESS KAFKA_CONSUMER_GROUP KAFKA_TOPIC TIMEOUT EVENT_HUB_KAFKA_CONNECTION_STRING """ self.logger = Logger() self.topic = config.get("KAFKA_TOPIC") if not self.topic: raise ValueError("KAFKA_TOPIC is not set in the config object.") if not config.get("KAFKA_ADDRESS"): raise ValueError("KAFKA_ADDRESS is not set in the config object.") if config.get("TIMEOUT"): try: self.timeout = int(config.get("TIMEOUT")) except ValueError: self.timeout = None else: self.timeout = None kafka_config = self.create_kafka_config(config) self.admin = admin.AdminClient(kafka_config) if config.get("KAFKA_CONSUMER_GROUP") is None: self.logger.info('Creating Producer') self.producer = Producer(kafka_config) self.run = False else: self.logger.info('Creating Consumer') self.consumer = Consumer(kafka_config) self.run = True signal.signal(signal.SIGTERM, self.exit_gracefully)
def verify_admin(): """ Verify Admin API """ a = admin.AdminClient({'bootstrap.servers': bootstrap_servers}) our_topic = topic + '_admin_' + str(uuid.uuid4()) num_partitions = 2 topic_config = {"compression.type": "gzip"} # # First iteration: validate our_topic creation. # Second iteration: create topic. # for validate in (True, False): fs = a.create_topics([ admin.NewTopic(our_topic, num_partitions=num_partitions, config=topic_config, replication_factor=1) ], validate_only=validate, operation_timeout=10.0) for topic2, f in fs.items(): f.result() # trigger exception if there was an error # # Find the topic in list_topics # verify_topic_metadata(a, {our_topic: num_partitions}) # # Increase the partition count # num_partitions += 3 fs = a.create_partitions( [admin.NewPartitions(our_topic, new_total_count=num_partitions)], operation_timeout=10.0) for topic2, f in fs.items(): f.result() # trigger exception if there was an error # # Verify with list_topics. # verify_topic_metadata(a, {our_topic: num_partitions}) def verify_config(expconfig, configs): """ Verify that the config key,values in expconfig are found and matches the ConfigEntry in configs. """ for key, expvalue in expconfig.items(): entry = configs.get(key, None) assert entry is not None, "Config {} not found in returned configs".format( key) assert entry.value == str(expvalue), \ "Config {} with value {} does not match expected value {}".format(key, entry, expvalue) # # Get current topic config # resource = admin.ConfigResource(admin.RESOURCE_TOPIC, our_topic) fs = a.describe_configs([resource]) configs = fs[resource].result() # will raise exception on failure # Verify config matches our expectations verify_config(topic_config, configs) # # Now change the config. # topic_config["file.delete.delay.ms"] = 12345 topic_config["compression.type"] = "snappy" for key, value in topic_config.items(): resource.set_config(key, value) fs = a.alter_configs([resource]) fs[resource].result() # will raise exception on failure # # Read the config back again and verify. # fs = a.describe_configs([resource]) configs = fs[resource].result() # will raise exception on failure # Verify config matches our expectations verify_config(topic_config, configs) # # Delete the topic # fs = a.delete_topics([our_topic]) fs[our_topic].result() # will raise exception on failure print("Topic {} marked for deletion".format(our_topic))
def __init__(self, conf): self.conf = conf kafka_conf = {'bootstrap.servers': conf.KAFKA_BROKERS} kafka_admin = kad.AdminClient(kafka_conf) self.admin = kafka_admin
data = res.json() return data with open(os.path.join(WORKING_DIRECTORY, 'city.list.json')) as json_file: cities = json.load(json_file) chile = [] for index, city in enumerate(cities): if city["country"] == "CL": chile.append(city) logging.info(f"Total cities {len(chile)}") calls = 0 kafka_client = admin.AdminClient(KAFKA_CONFIG) new_topic = admin.NewTopic("weather", 1, 1) kafka_client.create_topics([new_topic]) producer = Producer(KAFKA_CONFIG) while True: for index, city in enumerate(chile): calls += 1 query = current_city(city['id'], api_key) if query["cod"] != 200: logging.error(f"Error in {index} {city}") break else: logging.info(f"calls: {calls} city: {city['name']}") producer.produce("weather", pickle.dumps(query))
EVENT_FILE = "test_events.json" # file of the records, not a json itself, but each row is QUANTITIES = ["actSpeed_C11", "vaTorque_C11"] RES_QUANTITY = "vaPower_C11" MAX_JOIN_CNT = None # maximum of 1500 rows MAX_BATCH_SIZE = 100 TRANSACTION_TIME = 0.1 VERBOSE = False if sys.platform.startswith("win"): pytest.skip("skipping unix-only tests", allow_module_level=True) # Create a kafka producer and consumer instance and subscribe to the topics print("Create Kafka instances.") # create a Kafka admin client k_admin_client = kafka_admin.AdminClient({'bootstrap.servers': KAFKA_BOOTSTRAP_SERVERS}) kafka_consumer = Consumer({ 'bootstrap.servers': KAFKA_BOOTSTRAP_SERVERS, 'group.id': f"kafka-eof_{str(uuid.uuid4())}", 'auto.offset.reset': 'earliest', 'enable.auto.commit': False, 'enable.auto.offset.store': False, 'enable.partition.eof': False }) kafka_consumer.subscribe([KAFKA_TOPIC_IN_0, KAFKA_TOPIC_IN_1]) # kafka_consumer.assign([TopicPartition(KAFKA_TOPIC_IN_0), TopicPartition(KAFKA_TOPIC_IN_1)]) # create a Kafka producer kafka_producer = Producer({'bootstrap.servers': KAFKA_BOOTSTRAP_SERVERS,
def enqueue(self, filename): """ This method tries to post a message to the pre-defined kafka topic. :param filename: :return status False or True: """ status = False if filename is None or len(filename) == 0: logging_to_console_and_syslog( "ConfluentKafkaMsgQAPI: filename is None or invalid") return status if self.producer_instance is None: logging_to_console_and_syslog( "KafkaMsgQAPI: Producer instance is None. Trying to create one.." ) if not self.__producer_connect(): logging_to_console_and_syslog( "Unable to create producer instance.") return status if not self.is_topic_created: try: if self.producer_instance.list_topics(self.topic, timeout=1.0): logging_to_console_and_syslog( "Found topic name = {} in the zookeeper.".format( self.topic)) self.is_topic_created = True except KafkaException: kafka_admin_client = admin.AdminClient(self.producer_conf) logging_to_console_and_syslog("Creating topic {}.".format( self.topic)) ret = kafka_admin_client.create_topics(new_topics=[ admin.NewTopic(topic=self.topic, num_partitions=1) ], operation_timeout=1.0) logging_to_console_and_syslog("ret = {}".format(ret)) # Asynchronously produce a message, the delivery report callback # will be triggered from poll() above, or flush() below, when the message has # been successfully delivered or failed permanently. logging_to_console_and_syslog( "ConfluentKafkaMsgQAPI: Posting filename={} into " "kafka broker={}, topic={}".format(filename, self.broker_name, self.topic)) value = filename.encode('utf-8') try: # Produce line (without newline) self.producer_instance.produce( self.topic, value, callback=ConfluentKafkaMsgQAPI.delivery_callback) status = True except BufferError: sys.stderr.write('%% Local producer queue is full ' '(%d messages awaiting delivery): try again\n' % len(self.producer_instance)) status = False except: print("ConfluentKafkaMsgQAPI: Exception in user code:") print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60) status = False else: event = "ConfluentKafkaMsgQAPI: Posting filename={} into " \ "kafka broker={}, topic={}." \ .format(filename, self.broker_name, self.topic) logging_to_console_and_syslog(event) # Wait for any outstanding messages to be delivered and delivery report # callbacks to be triggered. # Serve delivery callback queue. # NOTE: Since produce() is an asynchronous API this poll() call # will most likely not serve the delivery callback for the # last produce()d message. self.producer_instance.poll(timeout=0.1) # Wait until all messages have been delivered # sys.stderr.write('%% Waiting for %d deliveries\n' % len(self.producer_instance)) self.producer_instance.flush(timeout=0.1) return status
def create_admin_client(config): global adminClient adminClient = ckAdmin.AdminClient(config)
def verify_admin(): """ Verify Admin API """ a = admin.AdminClient({'bootstrap.servers': bootstrap_servers}) our_topic = topic + '_admin_' + str(uuid.uuid4()) num_partitions = 2 topic_config = {"compression.type": "gzip"} # # First iteration: validate our_topic creation. # Second iteration: create topic. # for validate in (True, False): fs = a.create_topics([ admin.NewTopic(our_topic, num_partitions=num_partitions, config=topic_config, replication_factor=1) ], validate_only=validate, operation_timeout=10.0) for topic2, f in fs.items(): f.result() # trigger exception if there was an error # # Find the topic in list_topics # verify_topic_metadata(a, {our_topic: num_partitions}) # # Increase the partition count # num_partitions += 3 fs = a.create_partitions( [admin.NewPartitions(our_topic, new_total_count=num_partitions)], operation_timeout=10.0) for topic2, f in fs.items(): f.result() # trigger exception if there was an error # # Verify with list_topics. # verify_topic_metadata(a, {our_topic: num_partitions}) # # Verify with list_groups. # # Produce some messages p = confluent_kafka.Producer({"bootstrap.servers": bootstrap_servers}) p.produce(our_topic, 'Hello Python!', headers=produce_headers) p.produce(our_topic, key='Just a key and headers', headers=produce_headers) def consume_messages(group_id): # Consume messages conf = { 'bootstrap.servers': bootstrap_servers, 'group.id': group_id, 'session.timeout.ms': 6000, 'enable.auto.commit': False, 'on_commit': print_commit_result, 'error_cb': error_cb, 'auto.offset.reset': 'earliest', 'enable.partition.eof': True } c = confluent_kafka.Consumer(conf) c.subscribe([our_topic]) eof_reached = dict() while True: msg = c.poll() if msg is None: raise Exception( 'Got timeout from poll() without a timeout set: %s' % msg) if msg.error(): if msg.error().code( ) == confluent_kafka.KafkaError._PARTITION_EOF: print('Reached end of %s [%d] at offset %d' % (msg.topic(), msg.partition(), msg.offset())) eof_reached[(msg.topic(), msg.partition())] = True if len(eof_reached) == len(c.assignment()): print( 'EOF reached for all assigned partitions: exiting') break else: print('Consumer error: %s: ignoring' % msg.error()) break # Commit offset c.commit(msg, asynchronous=False) group1 = 'test-group-1' group2 = 'test-group-2' consume_messages(group1) consume_messages(group2) # list_groups without group argument groups = set(group.id for group in a.list_groups(timeout=10)) assert group1 in groups, "Consumer group {} not found".format(group1) assert group2 in groups, "Consumer group {} not found".format(group2) # list_groups with group argument groups = set(group.id for group in a.list_groups(group1)) assert group1 in groups, "Consumer group {} not found".format(group1) groups = set(group.id for group in a.list_groups(group2)) assert group2 in groups, "Consumer group {} not found".format(group2) def verify_config(expconfig, configs): """ Verify that the config key,values in expconfig are found and matches the ConfigEntry in configs. """ for key, expvalue in expconfig.items(): entry = configs.get(key, None) assert entry is not None, "Config {} not found in returned configs".format( key) assert entry.value == str(expvalue), \ "Config {} with value {} does not match expected value {}".format(key, entry, expvalue) # # Get current topic config # resource = admin.ConfigResource(admin.RESOURCE_TOPIC, our_topic) fs = a.describe_configs([resource]) configs = fs[resource].result() # will raise exception on failure # Verify config matches our expectations verify_config(topic_config, configs) # # Now change the config. # topic_config["file.delete.delay.ms"] = 12345 topic_config["compression.type"] = "snappy" for key, value in topic_config.items(): resource.set_config(key, value) fs = a.alter_configs([resource]) fs[resource].result() # will raise exception on failure # # Read the config back again and verify. # fs = a.describe_configs([resource]) configs = fs[resource].result() # will raise exception on failure # Verify config matches our expectations verify_config(topic_config, configs) # # Delete the topic # fs = a.delete_topics([our_topic]) fs[our_topic].result() # will raise exception on failure print("Topic {} marked for deletion".format(our_topic))
# Kafka Details kafkaBrokerServer = hostname kafkaBrokerPort = 9092 kafkaBroker = kafkaBrokerServer + ":" + str(kafkaBrokerPort) zookeeperServer = hostname zookeeperPort = 2185 zookeeper = zookeeperServer + ":" + str(zookeeperPort) topic = 'mytopic' print """\nINFO: Kakfa Connection Details: Kafka Broker : %s Zookeeper : %s Topic : %s """ % (kafkaBroker, zookeeper, topic) print "\nINFO: Creating connection obj for Admin Task" connection = admin.AdminClient({'bootstrap.servers': kafkaBroker}) print "\nINFO: Check if Topic %s exists. Else Create it" % topic check_topic_existence(connection, topic) conf = {'bootstrap.servers': kafkaBroker} print "\nINFO: Create Client obj for Kafka Connection" client = Producer(**conf) produce_messages(client, int(num_mesg), topic)
def check_kafka(): client = admin.AdminClient({"bootstrap.servers": KAFKA_BOOTSTRAP_SERVERS}) cluster_meta_data = client.list_topics(timeout=1) # fail if custom topic is not present return WEATHER_STATUS in cluster_meta_data.topics.keys()
def check_ksql(): client = admin.AdminClient({"bootstrap.servers": KAFKA_BOOTSTRAP_SERVERS}) cluster_meta_data = client.list_topics(timeout=1) # fail if custom topic is not present return TURNSTILE_ENTRIES_TABLE in cluster_meta_data.topics.keys()
def example_delete_topics(a, topics): """ delete topics """ # Call delete_topics to asynchronously delete topics, a future is returned. # By default this operation on the broker returns immediately while # topics are deleted in the background. But here we give it some time (30s) # to propagate in the cluster before returning. # # Returns a dict of <topic,future>. fs = a.delete_topics(topics, operation_timeout=30) # Wait for operation to finish. for topic, f in fs.items(): try: f.result() # The result itself is None print("Topic {} deleted".format(topic)) except Exception as e: print("Failed to delete topic {}: {}".format(topic, e)) conf = { 'bootstrap.servers': 'localhost:9092', 'client.id': 'test', 'default.topic.config': { 'acks': 'all' } } a = admin.AdminClient(conf) example_delete_topics(a, ['test-topic', 'orders-topic'])