def ensure_topic(topic, num_partitions, replication_factor, logger, timeout_ms=3000, brokers='localhost'): adminclient = KafkaAdminClient(bootstrap_servers=brokers, client_id='ansible-rm') topic_list = [] topic_list.append( NewTopic(name=topic, num_partitions=1, replication_factor=1)) try: adminclient.create_topics(new_topics=topic_list, validate_only=False) # adminclient.delete_topics(topic_list) logger.info('kafka topic ' + topic + ' created') except TopicAlreadyExistsError as e: logger.info('kafka topic ' + topic + ' exists') except Exception as e: logger.error('error creating kafka topic ' + topic) raise Exception( 'Unknown error code during creation of topic `{}`: {}'.format( topic, str(e)))
class KafkaAdminGroupReader: def __init__(self, kafka_config): self.log = logging.getLogger(__name__) self.admin_client = KafkaAdminClient( bootstrap_servers=kafka_config.broker_list, ) def read_group(self, groupid): topics = set() group_offsets = self.admin_client.list_consumer_group_offsets(groupid) for tp in six.iterkeys(group_offsets): topics.add(tp.topic) return list(topics) def read_groups(self, groupids=None, list_only=False): if groupids is None: groupids = self._list_groups() if list_only: return {groupid: [] for groupid in groupids} groups = {} for groupid in groupids: topics = self.read_group(groupid) groups[groupid] = topics return groups def _list_groups(self): groups_and_protocol_types = self.admin_client.list_consumer_groups() return [gpt[0] for gpt in groups_and_protocol_types]
def __init__(self, id: int, name: str): if (id <= 0): raise ValueError("ID must be a positive integer") super().__init__() self._daemon: bool = True self._id: int = id self._name: str = name self._status_up_time: datetime = datetime.now() self._actual_status: Status = self.__set_status() self._consumer = KafkaConsumer(f"get_status_{self.id}", bootstrap_servers=["localhost:9092"]) self._reg_consumer = KafkaConsumer( f"reg_response_{self.id}", bootstrap_servers=["localhost:9092"], consumer_timeout_ms=5000) try: admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") admin_client.create_topics([ NewTopic(f"reg_response_{self.id}", 1, 1), NewTopic(f"get_status_{self.id}", 1, 1) ]) except: pass self.__make_reg(id)
def delete_topic(username, title): # Deletes a kafak topic, based on username and title admin_client = KafkaAdminClient(bootstrap_servers='localhost:9092') topic_lst = [] topic_name = "{}-{}".format(username, title) topic_lst.append(topic_name) admin_client.delete_topics(topic_lst)
def alternative_main(input_path, bootstrap_server, topic_name): admin_client = KafkaAdminClient(bootstrap_servers="localhost:7092", client_id='test') topic_list = [] topic_list.append( NewTopic(name="test", num_partitions=1, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) #Starting session spark = SparkSession.builder.appName('BigData1').getOrCreate() spark.sparkContext.setLogLevel("ERROR") joined_raw = spark.read.parquet(input_path) #print(joined_raw.show()) # Write key-value data from a DataFrame to a specific Kafka topic specified in an option query = joined_raw.select( F.col('time').cast("string").alias("key"), F.to_json(F.struct("duration", "visibility")).alias("value")) print(query.show()) query = joined_raw \ .select(F.col('time').cast("string").alias("key"), F.to_json(F.struct("duration", "visibility")).alias("value")) \ .write \ .format("kafka") \ .option("kafka.bootstrap.servers", bootstrap_server) \ .option("topic", "test") \ .save()
def init_admin_client(self): start_time = time.monotonic() wait_time = constants.MINUTE while True: if time.monotonic() - start_time > wait_time: raise Timeout( f"Timeout ({wait_time}) on creating admin client") try: self.admin_client = KafkaAdminClient( api_version_auto_timeout_ms=constants. API_VERSION_AUTO_TIMEOUT_MS, bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], client_factory=KarapaceKafkaClient, ) break except (NodeNotReadyError, NoBrokersAvailable, AssertionError): self.log.warning( "No Brokers available yet, retrying init_admin_client()") except: # pylint: disable=bare-except self.log.exception( "Failed to initialize admin client, retrying init_admin_client()" ) time.sleep(2.0)
def connect(self): """Make a connection""" logger.info(f"connect kafka: {self.connection_conf}") if self.admin_client: try: self.admin_client.close() except: pass self.admin_client = KafkaAdminClient(bootstrap_servers=self.connection_conf) topic_list = [] topic_list.append( NewTopic(name=self.topic, num_partitions=self.num_partitions, replication_factor=self.replication_factor)) try: self.admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception as e: logger.error(e) if self.producer: self.producer.close() self.producer = KafkaProducer(bootstrap_servers=self.connection_conf) if self.consumer: self.consumer.close() self.consumer = KafkaConsumer(self.topic, group_id=f"{self.topic}.spider.consumer", bootstrap_servers=self.connection_conf)
def get(self, topicName): """ Get Topic Detail. """ app.logger.info( "Request to get details for topic {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) result = admin.describe_topics([topicName]) except UnknownTopicOrPartitionError as e: api.abort(500, e.description) except Exception as e: api.abort(500, str(e.args)) finally: admin.close() app.logger.debug(result) if result[0]['error_code'] == 0: return { 'partitions': len(result[0]['partitions']), 'replicas': len(result[0]['partitions'][0]['replicas']) } else: api.abort(400, "Bad Request(Wrong Topic Name)")
def __init__( self , kafka_endpoint , access_key , access_certificate , ca_certificate ): self.kafka_endpoint = kafka_endpoint self.kafka_consumer = KafkaConsumer( bootstrap_servers = self.kafka_endpoint , value_deserializer = lambda m: json.loads( m.decode('utf-8') ) , ssl_keyfile = access_key , ssl_certfile = access_certificate , ssl_cafile = ca_certificate , security_protocol = "SSL" , api_version_auto_timeout_ms = 10000 ) topics = self.kafka_consumer.topics() if 'os_stats' not in topics: print( "os_stats topic not found ... creating" ) admin_client = KafkaAdminClient( bootstrap_servers = self.kafka_endpoint , ssl_keyfile = access_key , ssl_certfile = access_certificate , ssl_cafile = ca_certificate , security_protocol = "SSL" , api_version_auto_timeout_ms = 10000 ) topic_list = [] topic_list.append(NewTopic( name = "os_stats" , num_partitions = 1 , replication_factor = 1 ) ) admin_client.create_topics( new_topics = topic_list , validate_only = False ) self.kafka_consumer.subscribe( 'os_stats' )
def get_consumer(self): if self.err_var is not None: self.err_var.set("") self.ui.update() set_content(self) from kafka.admin import KafkaAdminClient adminClient = KafkaAdminClient(bootstrap_servers=self.cur_broker) data = adminClient.list_consumer_group_offsets(self.cur_consumer) self.content_text.config(state=NORMAL) self.content_text.delete(1.0, END) self.content_text.insert( END, u"{} {} :\n\n".format(self.cur_broker, self.cur_consumer)) self.content_text.insert( END, u"{:20s} {:10s} {:12s} {:12s} {}\n".format("topic", "partition", "offset", "newest", "lag")) for key in data.keys(): num = self.get_topic_single_partiton_offset( self.cur_broker, key.topic, key.partition) lag = int(num) - int(data[key].offset) self.content_text.insert( END, u"{:20s} {:10s} {:12s} {:12s} {}\n".format( key.topic, str(key.partition), str(data[key].offset), str(num), lag)) msg = "offset:{} lag:{}".format(num, lag) self.write_file(msg, self.cur_consumer) if len(data.keys()) == 0: self.content_text.insert(END, u"没有查到数据") self.content_text.config(state=DISABLED)
def test_validate_number_in_interval_double(self): prod, cons = app.run(Config.K_MONITOR_TEST_TOPIC, Config.PS_DATABASE_NAME, Config.PS_TEST_WEBSITE_TABLE_NAME, "tests/t_monitor_heavy_test.yml") interval = File.read_time_interval("tests/t_monitor_heavy_test.yml") time.sleep(interval * 2) app.stop_monitor(prod, cons) admin_client = KafkaAdminClient( bootstrap_servers=[Config.K_HOST + ':' + Config.K_PORT], security_protocol=Config.K_SECURITY_PROTOCOL, ssl_cafile=Config.K_SSL_CAT_FILE, ssl_certfile=Config.K_SSL_CERT_FILE, ssl_keyfile=Config.K_SSL_KEY_FILE) admin_client.delete_topics([Config.K_MONITOR_TEST_TOPIC]) monitors = File.read_monitors("tests/t_monitor_heavy_test.yml") #send messages equals total urls count in 2 cycle is double the urls size self.assertEqual(prod.get_message_count(), len(monitors) * 2)
def get(self, topicName): """ Get Topic Configuration. """ app.logger.info( "Request to get Configuration for topic {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) config_list = [] config = ConfigResource(ConfigResourceType.TOPIC, topicName) topic_configs = admin.describe_configs([config]) topic_config = topic_configs[0].resources[0] for c in topic_config[4]: config_list.append({'key': c[0], 'value': c[1]}) return config_list except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close()
def create_group(phone_no): if get_jwt_identity() == phone_no: try: group_id = request.json['group_id'] group_name = request.json['group_name'] created_at = request.json['created_at'] participants = request.json['participants'] # Instantiate kafka admin client to create a topic client = KafkaAdminClient(bootstrap_servers="localhost:9092", client_id=phone_no) topics = [] topics.append( NewTopic(name=group_id, num_partitions=1, replication_factor=1)) client.create_topics(new_topics=topics, validate_only=False) group = Groups(group_id=group_id, group_name=group_name, created_at=created_at, participants=participants) group.save() return make_response( jsonify({'success': 'Group created successfully'}), 200) except KeyError: abort(400) else: abort(401)
def get(self, serviceName): """ Get Service State. """ app.logger.info( "Request to get details for service {0}.".format(serviceName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) state_list = [] if serviceName.lower() == 'kafka': for n in admin.describe_cluster()['brokers']: h = n['host'] s = remote_execute(h, serviceName.lower(), 'get') state_list.append({'host': h, 'state': s}) return state_list except Exception as e: ns_service.abort(500, str(e.args)) finally: admin.close()
def create_topic(self): """ Connects to cloudharness Events and creates a new topic Return: True if topic was created correctly, False otherwise. """ ## Connect to kafka admin_client = KafkaAdminClient( bootstrap_servers=self._get_bootstrap_servers(), client_id=self._get_client_id()) # ## Create topic new_topic = NewTopic(name=self.topic_id, num_partitions=1, replication_factor=1) try: result = admin_client.create_topics(new_topics=[new_topic], validate_only=False) log.info(f"Created new topic {self.topic_id}") return result except TopicAlreadyExistsError as e: # topic already exists "no worries", proceed return True except Exception as e: log.error(f"Error creating the new Topics --> {e}", exc_info=True) raise EventGeneralException from e
def main(): topic_name= sys.argv[1] try: num_partitions= int(sys.argv[2]) except: num_partitions= 1 try: replication_factor= int(sys.argv[3]) except: replication_factor= 1 try: bootstrap_servers=sys.argv[4] logging.info('bootstrap_servers: ' + bootstrap_servers) except: logging.error('bootstrap_servers not specified') os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers, client_id='test') topic_list = [] topic_list.append(NewTopic(name= topic_name , num_partitions=num_partitions, replication_factor=replication_factor)) admin_client.create_topics(new_topics=topic_list, validate_only=False) logging.info('Topic {}, with num_partitions {} and replication_factor {} has been created'.format(topic_name,str(num_partitions),str(replication_factor)))
def put(self, topicName): """ Update Topic Configuration. """ ckey = request.json['key'] cvalue = request.json['value'] app.logger.info( "Request to update configuration for topic {0} for key {1} and value {2}." .format(topicName, ckey, cvalue)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) new_config = ConfigResource(ConfigResourceType.TOPIC, topicName, {ckey: cvalue}) result = admin.alter_configs([new_config]) except UnknownTopicOrPartitionError as e: api.abort(400, e.description) except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close() if result.resources[0][0] == 0: return {"configured": topicName} else: api.abort(400, "Bad Request(" + result.resources[0][1] + ")")
def get(self, topicName): """ Get Topic ACL. """ app.logger.info("Request to get ACL for topic {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) acl_filter = ACLFilter(principal=None, host="*", operation=ACLOperation.ANY, permission_type=ACLPermissionType.ANY, resource_pattern=ResourcePattern( ResourceType.TOPIC, topicName)) acls, error = admin.describe_acls(acl_filter) acl_list = [] for a in acls: princpl = a.principal oprtn = a.operation.name acl_list.append({'user': princpl, 'type': oprtn}) return acl_list except Exception as e: ns_acl.abort(500, str(e.args)) finally: admin.close()
def delete(self, topicName): """ Delete Topic. """ app.logger.info( "Request to delete topic witn name {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) result = admin.delete_topics([topicName]) except UnknownTopicOrPartitionError as e: api.abort(400, e.description) except Exception as e: api.abort(500, str(e.args)) finally: admin.close() app.logger.debug(result) if result.topic_error_codes[0][1] == 0: return {"deleted": topicName} else: api.abort(400, "Bad Request(Topic Deletion Failed)")
def init_admin_client(self): try: self.admin_client = KafkaAdminClient( api_version_auto_timeout_ms=constants. API_VERSION_AUTO_TIMEOUT_MS, bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], sasl_mechanism=self.config["sasl_mechanism"], sasl_plain_username=self.config["sasl_plain_username"], sasl_plain_password=self.config["sasl_plain_password"], ) return True except (NodeNotReadyError, NoBrokersAvailable, AssertionError): self.log.warning( "No Brokers available yet, retrying init_admin_client()") time.sleep(2.0) except: # pylint: disable=bare-except self.log.exception( "Failed to initialize admin client, retrying init_admin_client()" ) time.sleep(2.0) return False
def create_topic(name): admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092", client_id='test') topic_list = [] topic_list.append( NewTopic(name=str(name), num_partitions=1, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False)
class AdminClient: def __init__(self, record): ''' record: not null, a list of input needed to create kafka topic ''' self.topic_name = record[0] self.replication_factor = record[1] self.partition = record[2] self.zoo_conn = record[3] self.client = KafkaAdminClient(bootstrap_servers=zoo_conn) def create_kafka_topic(self): try: self.client.creat_topics(NewTopic( name=self.topic_name, num_partitions=self.partition, replication_factor=self.replication_factor), validate_only=False) except Exception as ex: print("Error creating Kafka Topic \n") print(str(ex)) def delete_kafka_topic(self, topic_name): try: self.client.delete_topics(NewTopic(name=topic_name)) print(topic_name + ' is successfully deleted') except Exception as ex: print('error deleting topic') print(str(ex))
def run2(): admin_client = KafkaAdminClient( bootstrap_servers="kafka:9092", ) topic_list = [] topic_list.append(NewTopic(name="api_topic", num_partitions=1, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False)
def _connect(self): """Connect to kafka admin client""" try: self._client = KafkaAdminClient(**self.kwargs) logger.info('Connected to Kafka Admin client!') except Exception as ex: logger.error('Unable to connect to Kafka Admin client. %s', str(ex))
def get_consumers(self, brokers): from kafka.admin import KafkaAdminClient adminClient = KafkaAdminClient(bootstrap_servers=brokers) consumer_data = adminClient.list_consumer_groups() consumers = [] for consumer in consumer_data: consumers.append(consumer[0]) return consumers
def add_topic(username, title): # Creates a new kafka topic, based on username and title admin_client = KafkaAdminClient(bootstrap_servers='localhost:9092') topic_lst = [] topic_name = "{}-{}".format(username, title) topic_lst.append( NewTopic(name=topic_name, num_partitions=1, replication_factor=1)) admin_client.create_topics(topic_lst)
def CreateKafkaTopic(topic_name): admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092", client_id='test') topic_list = [ NewTopic(name=topic_name, num_partitions=1, replication_factor=1) ] admin_client.create_topics(new_topics=topic_list)
def createKafkaTopic(topicName) : try: admin_client = KafkaAdminClient(bootstrap_servers=["localhost:9092"]) admin_client.create_topics(new_topics=topic_list, validate_only=False) topic_list = [] topic_list.append(NewTopic(name=topicName, num_partitions=1, replication_factor=1)) except: pass
def create_topic(self): kafka_admin_client = KafkaAdminClient( bootstrap_servers=[settings.KAFKA['SERVER']]) topic = NewTopic(name=topic, num_partitions=3, topic_configs={'retention.ms': RETENTION_TIME}, replication_factor=1) kafka_admin_client.create_topics([topic])
def lambda_handler(event, context): responseData = {} responseStatus = cfnresponse.SUCCESS print("Request body is:", event) if event['RequestType'] == 'Create': bootstrap_uri = "Bootstrap servers not provided in env" if 'BOOTSTRAP_SERVERS' in os.environ: bootstrap_uri = os.environ['BOOTSTRAP_SERVERS'] print(bootstrap_uri) else: print(bootstrap_uri) responseData[ 'cause'] = "Bootstrap servers not mentioned in env variables" cfnresponse.send(event, context, cfnresponse.FAILED, responseData) try: admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_uri, client_id='lambda') except Exception as e: responseData[ 'status'] = "Failed to make KafkaAdmin Client, posssible reasons:bootstrap server name not resolvable, \ bootsrap servers not reacheable, MSK cluster not running" print(e) cfnresponse.send(event, context, cfnresponse.FAILED, responseData) if 'KafkaTopic' in event['ResourceProperties']: topic_list = [] topic_list.append( NewTopic( name=event['ResourceProperties']['KafkaTopic']['name'], num_partitions=int(event['ResourceProperties'] ['KafkaTopic']['num_partitions']), replication_factor=int( event['ResourceProperties']['KafkaTopic'] ['replication_factor']))) try: admin_client.create_topics(new_topics=topic_list, validate_only=False) responseData['status'] = "Topic created successfully" except Exception as e: print("Failed to create topic:", e) responseData['status'] = 'FAILED' responseData['cause'] = e cfnresponse.send(event, context, cfnresponse.FAILED, responseData) else: responseData[ 'cause'] = 'Failed to create topics, no KafkaTopic provided' responseStatus = cfnresponse.FAILED else: #if event['RequestType'] == 'Delete' or event['RequestType'] == 'Update': responseData[ 'cause'] = 'CloudFormation Delete and Update method not implemented, just return cnfresponse=SUCCSESS without any job/modifications' responseStatus = cfnresponse.SUCCESS cfnresponse.send(event, context, responseStatus, responseData)