def delete_kafka_topic(): logger.info("Request received - DELETE /delete_kafka_topic") if not request.is_json: logger.warning("Format not valid") return 'Format not valid', 400 try: admin_client = KafkaAdminClient(bootstrap_servers=kafka_ip_port, client_id='delete_kafka_topic') # Parse JSON data = request.get_json() logger.info("Data received: %s", data) topic = data["topic"] logger.info("Deleting topic %s in Kafka", topic) topic_list = [] topic_list.append(topic) admin_client.delete_topics(topics=topic_list) admin_client.close() except Exception as e: logger.error("Error while parsing request") logger.exception(e) return str(e), 400 return '', 201
def get(self, topicName): """ Get Topic Configuration. """ app.logger.info( "Request to get Configuration for topic {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) config_list = [] config = ConfigResource(ConfigResourceType.TOPIC, topicName) topic_configs = admin.describe_configs([config]) topic_config = topic_configs[0].resources[0] for c in topic_config[4]: config_list.append({'key': c[0], 'value': c[1]}) return config_list except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close()
def create_topic(self, topic): """ Creates a topic at Kafka. Arguments: topic (str): Name of the topic to create """ try: admin_client = KafkaAdminClient( bootstrap_servers=self.bootstrap_servers, security_protocol=self.security_protocol, ssl_cafile=self.ssl_cafile, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile, ) topic_list = [] topic_list.append( NewTopic(topic, num_partitions=1, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) admin_client.close() except TopicAlreadyExistsError: # we are fine if the Topic already exists logger.info("Topic already exists") pass except Exception as ex: logger.error("Could not create topic %s: %s", topic, str(ex)) raise Exception(ex)
def __init__(self, topicName): admin_client = KafkaAdminClient( bootstrap_servers=['kafka1:9092', 'kafka2:9092', 'kafka3:9092'], client_id='disco_eventproducer_admin') try: topic_list = [ NewTopic(name=topicName, num_partitions=1, replication_factor=2, topic_configs={'retention.ms': 30758400000}) ] admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception as e: logging.warning(str(e)) pass finally: admin_client.close() self.producer = KafkaProducer( bootstrap_servers='localhost:9092', value_serializer=lambda v: msgpack.packb(v, use_bin_type=True), compression_type='snappy') self.topicName = topicName
def setup(topic_name): # First, check if the topic already exists in kafka kafka_client = KafkaClient(bootstrap_servers=KAFKA_SERVER, api_version=(2, 5, 0)) future = kafka_client.cluster.request_update() kafka_client.poll(future=future) metadata = kafka_client.cluster current_topics = metadata.topics() kafka_client.close() print('Active topics:', current_topics) if topic_name not in current_topics: print(f'Creating topic {topic_name}...') kafka_admin_client = KafkaAdminClient(bootstrap_servers=KAFKA_SERVER, api_version=(2, 5, 0)) topic_list = [ NewTopic(name=topic_name, num_partitions=1, replication_factor=1) ] kafka_admin_client.create_topics(new_topics=topic_list, validate_only=False) kafka_admin_client.close() else: print(f'Topic {topic_name} exists')
def put(self, topicName): """ Update Topic Configuration. """ ckey = request.json['key'] cvalue = request.json['value'] app.logger.info( "Request to update configuration for topic {0} for key {1} and value {2}." .format(topicName, ckey, cvalue)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) new_config = ConfigResource(ConfigResourceType.TOPIC, topicName, {ckey: cvalue}) result = admin.alter_configs([new_config]) except UnknownTopicOrPartitionError as e: api.abort(400, e.description) except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close() if result.resources[0][0] == 0: return {"configured": topicName} else: api.abort(400, "Bad Request(" + result.resources[0][1] + ")")
def get(self, serviceName): """ Get Service State. """ app.logger.info( "Request to get details for service {0}.".format(serviceName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) state_list = [] if serviceName.lower() == 'kafka': for n in admin.describe_cluster()['brokers']: h = n['host'] s = remote_execute(h, serviceName.lower(), 'get') state_list.append({'host': h, 'state': s}) return state_list except Exception as e: ns_service.abort(500, str(e.args)) finally: admin.close()
def get(self, topicName): """ Get Topic ACL. """ app.logger.info("Request to get ACL for topic {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) acl_filter = ACLFilter(principal=None, host="*", operation=ACLOperation.ANY, permission_type=ACLPermissionType.ANY, resource_pattern=ResourcePattern( ResourceType.TOPIC, topicName)) acls, error = admin.describe_acls(acl_filter) acl_list = [] for a in acls: princpl = a.principal oprtn = a.operation.name acl_list.append({'user': princpl, 'type': oprtn}) return acl_list except Exception as e: ns_acl.abort(500, str(e.args)) finally: admin.close()
def handle(event, context): if event.method == 'POST': data = json.loads(event.body) if "topic" not in data: return { "statusCode": 400, "body": "Format not valid" } try: topic = data["topic"] admin_client = KafkaAdminClient(bootstrap_servers="kafka.deployment8:9092", client_id='create_kafka_topic') topic_list = [] topic_list.append(NewTopic(name=topic, num_partitions=1, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) admin_client.close() except Exception as e: return { "statusCode": 400, "body": "".format(e) } return { "statusCode": 200, "body": "OK" } else: return { "statusCode": 200, "body": "No action for this endpoint" }
def delete(self, topicName): """ Delete Topic. """ app.logger.info( "Request to delete topic witn name {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) result = admin.delete_topics([topicName]) except UnknownTopicOrPartitionError as e: api.abort(400, e.description) except Exception as e: api.abort(500, str(e.args)) finally: admin.close() app.logger.debug(result) if result.topic_error_codes[0][1] == 0: return {"deleted": topicName} else: api.abort(400, "Bad Request(Topic Deletion Failed)")
def get(self, topicName): """ Get Topic Detail. """ app.logger.info( "Request to get details for topic {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) result = admin.describe_topics([topicName]) except UnknownTopicOrPartitionError as e: api.abort(500, e.description) except Exception as e: api.abort(500, str(e.args)) finally: admin.close() app.logger.debug(result) if result[0]['error_code'] == 0: return { 'partitions': len(result[0]['partitions']), 'replicas': len(result[0]['partitions'][0]['replicas']) } else: api.abort(400, "Bad Request(Wrong Topic Name)")
def create_kafka_topic(): logger.info("Request received - POST /create_kafka_topic") if not request.is_json: logger.warning("Format not valid") return 'Format not valid', 400 try: admin_client = KafkaAdminClient(bootstrap_servers=kafka_ip_port, client_id='create_kafka_topic') # Parse JSON data = request.get_json() logger.info("Data received: %s", data) topic = data["topic"] logger.info("Creating topic %s in Kafka", topic) topic_list = [] topic_list.append( NewTopic(name=topic, num_partitions=1, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) admin_client.close() except Exception as e: logger.error("Error while parsing request") logger.exception(e) return str(e), 400 return '', 201
def __init__(self, threshold, startTime, endTime, timeWindow, probeData, topicIn, topicOut, slideStep=3600): self.threshold = threshold self.startTime = startTime self.endTime = endTime self.timeWindow = timeWindow # Slide the window by this amount self.slideWindow = slideStep # Ignore disconnection events followed by a reconnect within # discoProbesWindow seconds # Also report only probes disconnected discoProbesWindow # seconds before/after the burst starting time self.discoProbesWindow = 300 self.probeData = probeData self.eventData = defaultdict(list) self.numTotalProbes = {} self.initNumProbes() self.disconnectedProbes = {} admin_client = KafkaAdminClient( bootstrap_servers=['kafka1:9092', 'kafka2:9092', 'kafka3:9092'], client_id='disco_disco_admin') try: topic_list = [ NewTopic(name=topicOut, num_partitions=1, replication_factor=2, topic_configs={'retention.ms': 30758400000}) ] admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception as e: logging.warning(str(e)) pass finally: admin_client.close() self.producer = KafkaProducer( bootstrap_servers='localhost:9092', value_serializer=lambda v: msgpack.packb(v, use_bin_type=True), compression_type='snappy') self.topicIn = topicIn self.topicOut = topicOut self.executor = ProcessPoolExecutor(max_workers=10)
def create_topic(bootstrap_ip_port, kafka_topic, num_parts, repli_factor): admin_client = KafkaAdminClient(bootstrap_servers=[bootstrap_ip_port]) topic_list = [] topic_list.append( NewTopic(name=kafka_topic, num_partitions=int(num_parts), replication_factor=int(repli_factor))) try: admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception as ex: return ex finally: admin_client.close() return 1
def get_all_loader_topics(admin_client: KafkaAdminClient) -> List[str]: # TODO: Make this method use a regex on the topic name pattern instead LOG.info("Fetching topic names") topics: Set[str] = admin_client._client.cluster.topics( exclude_internal_topics=True) LOG.debug("Creating Kafka Admin Client") admin_client.close() return [ topic for topic in topics if not any([True for excluded in EXCLUDED_TOPICS if excluded in topic]) ]
def __init__(self, streamType, streamName, startTime, disconnectedProbes, level, topicIn, topicOut): # Logging FORMAT = '%(asctime)s %(processName)s %(message)s' logging.basicConfig(format=FORMAT, filename='disco-probetracker.log', level=logging.WARN, datefmt='%Y-%m-%d %H:%M:%S') logging.info("Probe tracker started: {} {} {} {}".format( streamType, streamName, startTime, disconnectedProbes)) self.streamType = streamType self.streamName = streamName self.startTime = startTime self.level = level self.topicIn = topicIn self.disconnectedProbes = disconnectedProbes self.nbProbesThreshold = len(disconnectedProbes) / 2 self.reconnectedProbes = {} self.topicName = topicOut admin_client = KafkaAdminClient( bootstrap_servers=['kafka1:9092', 'kafka2:9092', 'kafka3:9092'], client_id='disco_disco_admin') try: topic_list = [ NewTopic(name=self.topicName, num_partitions=1, replication_factor=2, topic_configs={'retention.ms': 30758400000}) ] admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception as e: pass finally: admin_client.close() self.producer = KafkaProducer( bootstrap_servers='localhost:9092', value_serializer=lambda v: msgpack.packb(v, use_bin_type=True), compression_type='snappy') self.consumer = KafkaConsumer( enable_auto_commit=False, bootstrap_servers=['localhost:9092'], value_deserializer=lambda v: msgpack.unpackb(v, raw=False)) self.consumer.subscribe(topicIn)
def pushRIBData(AF, collector, startts, endts): stream = getBGPStream("ribs", AF, [collector], startts, endts) topicName = "ihr_bgp_" + collector + "_rib" admin_client = KafkaAdminClient( bootstrap_servers=['kafka1:9092', 'kafka2:9092', 'kafka3:9092'], client_id='bgp_producer_admin') try: topic_list = [ NewTopic(name=topicName, num_partitions=1, replication_factor=1) ] admin_client.create_topics(new_topics=topic_list, validate_only=False) except: pass admin_client.close() stream.start() producer = KafkaProducer( bootstrap_servers=['kafka1:9092', 'kafka2:9092', 'kafka3:9092'], # acks=0, value_serializer=lambda v: msgpack.packb(v, use_bin_type=True), linger_ms=1000, request_timeout_ms=300000, compression_type='snappy') rec = BGPRecord() while stream and stream.get_next_record(rec): completeRecord = {} completeRecord["rec"] = getRecordDict(rec) completeRecord["elements"] = [] recordTimeStamp = rec.time recordTimeStamp = int(recordTimeStamp) * 1000 elem = rec.get_next_elem() while (elem): elementDict = getElementDict(elem) completeRecord["elements"].append(elementDict) elem = rec.get_next_elem() producer.send(topicName, completeRecord, timestamp_ms=recordTimeStamp) producer.close()
def handle(event, context): if event.method == "POST": try: data = json.loads(event.body) topic = data["topic"] admin_client = KafkaAdminClient( bootstrap_servers="kafka.deployment8:9092", client_id="delete_kafka_topic", ) admin_client.delete_topics(topics=[topic]) admin_client.close() return {"statusCode": 201, "body": "No Content"} except Exception as e: return {"statusCode": 400, "body": f"Error parsing request: {e}"} else: return {"statusCode": 200, "body": "No action for this endpoint"}
def get(self): """ Get List of Topics. """ app.logger.info("Request to get list of topics.") try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) return admin.list_topics() except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close()
def create_topic_if_needed(self, messaging_properties, topic_config_properties): if topic_config_properties.auto_create: # KafkaAdminClient is picky about which keyword arguments are passed in, so build the parameters from KafkaAdminClient.DEFAULT_CONFIG config = {key:messaging_properties.config.get(key, None) for key in KafkaAdminClient.DEFAULT_CONFIG if messaging_properties.config.get(key, None) is not None } config['bootstrap_servers'] = messaging_properties.connection_address config['client_id'] ='ignition' admin_client = KafkaAdminClient(**config) try: logger.info("Creating topic {0} with replication factor {1}, partitions {2} and config {3}".format(topic_config_properties.name, topic_config_properties.replication_factor, topic_config_properties.num_partitions, topic_config_properties.config)) topic_list = [NewTopic(name=topic_config_properties.name, num_partitions=topic_config_properties.num_partitions, replication_factor=topic_config_properties.replication_factor, topic_configs=topic_config_properties.config)] admin_client.create_topics(new_topics=topic_list, validate_only=False) except TopicAlreadyExistsError as _: logger.info("Topic {0} already exists, not creating".format(topic_config_properties.name)) finally: try: admin_client.close() except Exception as e: logger.debug("Exception closing Kafka admin client {0}".format(str(e))) else: logger.info("Not creating job queue topic {0}".format(topic_config_properties.name))
def init(self): if not 'INIT_KAFKA_CREATE_CHANNELS' in self.config: return if self.reconnect() is None: return admin_client = KafkaAdminClient(bootstrap_servers=self.url_out) # group_id=self.id_group channels = self.config['INIT_KAFKA_CREATE_CHANNELS'] achs = channels.split(';') topic_list = [] for channel in achs: if self.verbose: print("DBG: createTopic in Kafka '%s': topic='%s'" % (self.url_out, channel)) topic_list.append(NewTopic(name=channel, num_partitions=1, replication_factor=1)) res = None try: res = admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception as e: print("ERR: createTopic in Kafka '%s': err='%s'" % (self.url_out, str(e))) admin_client.close()
class KafkaClient: def __init__(self, client, url="localhost:9092"): try: self.client = KafkaAdminClient(bootstrap_servers=url, client_id=client) except Exception as excp: print(f"[ERROR] - could not connect to kafka: {str(excp)}") def create_topics(self, topics): topics_list = [ NewTopic(name=topic, num_partitions=1, replication_factor=1) for topic in topics ] self.client.create_topics(new_topics=topics_list, validate_only=False) def close_topics(self, topics): self.client.delete_topics(topics) def close(self): self.client.close() print("[INFO] - kafka connection closed")
def __delTopic(topic): try: admin_client = KafkaAdminClient( bootstrap_servers=current_app.config["KFKA_URL"], ) # logging.info(f"admin_client={admin_client}") if admin_client == None: return "admin_client is none", RespCode.ARGS_ERROR topic_list = [] topic_list.append(topic) res = admin_client.delete_topics(topic_list) admin_client.close() # logging.info(f"NewTopic res={res}") with MsgCount.get_lock(): MsgCount.value = 0 return "" except KafkaError as ke: admin_client.close() # logging.info(f"except ={dir(ke)}") # logging.info(f"except ={ke}") # logging.info(f"except ke.errno={ke.errno}") # logging.info(f"except ke.message={ke.message}") return f"{ke.errno}-{ke.message}", RespCode.ARGS_ERROR except Exception as e: admin_client.close() logging.info(f"exception ={e}") # logging.info(f"exception ={dir(e)}") return "exception ", RespCode.ARGS_ERROR
def __createTopic(topic): try: admin_client = KafkaAdminClient( bootstrap_servers=current_app.config["KFKA_URL"], ) # logging.info(f"admin_client={admin_client}") if admin_client == None: return "admin_client is none", RespCode.ARGS_ERROR topic_list = [] topic_list.append( NewTopic(name=topic, num_partitions=3, replication_factor=2)) res = admin_client.create_topics(new_topics=topic_list, validate_only=False) # logging.info(f"NewTopic res={res}") admin_client.close() return "" except KafkaError as ke: # logging.info(f"except ={dir(ke)}") # logging.info(f"except ={ke}") # logging.info(f"except ke.errno={ke.errno}") # logging.info(f"except ke.message={ke.message}") admin_client.close() return f"{ke.errno}-{ke.message}", RespCode.ARGS_ERROR except Exception as e: admin_client.close() return "exception ", RespCode.ARGS_ERROR
def post(self, topicName): """ Create New Topic. """ partitions = request.json['partitions'] replicas = request.json['replicas'] app.logger.info( "Request to create topic witn name {0} and {1} partitions and {2} replicas." .format(topicName, partitions, replicas)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) newTopic = NewTopic(topicName, partitions, replicas) result = admin.create_topics([newTopic]) except TopicAlreadyExistsError as e: api.abort(400, e.description) except IllegalArgumentError as e: api.abort(400, e.description) except InvalidReplicationFactorError as e: api.abort(400, e.description) except Exception as e: api.abort(500, str(e.args)) finally: admin.close() app.logger.debug(result) if result.topic_errors[0][1] == 0: return {"created": topicName} else: api.abort(400, "Bad Request(Topic Creation Failed)")
def delete(self, topicName): """ Delete Topic ACL. """ args = acl_detail_parse.parse_args() auser = args['user'] atype = args['type'] acl_user = "******" + auser app.logger.info( "Request to delete ACL for topic {0} for user {1} and access type {2}." .format(topicName, auser, atype)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) results = admin.delete_acls([ ACLFilter(principal=acl_user, host="*", operation=ACLOperation.ANY, permission_type=ACLPermissionType.ANY, resource_pattern=ResourcePattern( ResourceType.TOPIC, topicName)) ]) except Exception as e: ns_acl.abort(500, str(e.args)) finally: admin.close() if len(results[0][1]) > 0: return {"delete": "sucess"} else: ns_acl.abort(500, "Internal Error(Cannot delete any ACL)")
class KafkaSchemaReader(Thread): def __init__(self, config, master_coordinator=None): Thread.__init__(self) self.master_coordinator = master_coordinator self.log = logging.getLogger("KafkaSchemaReader") self.timeout_ms = 200 self.config = config self.subjects = {} self.schemas: Dict[int, TypedSchema] = {} self.global_schema_id = 0 self.offset = 0 self.admin_client = None self.schema_topic = None self.topic_replication_factor = self.config["replication_factor"] self.consumer = None self.queue = Queue() self.ready = False self.running = True self.id_lock = Lock() sentry_config = config.get("sentry", {"dsn": None}).copy() if "tags" not in sentry_config: sentry_config["tags"] = {} self.stats = StatsClient(sentry_config=sentry_config) def init_consumer(self): # Group not set on purpose, all consumers read the same data session_timeout_ms = self.config["session_timeout_ms"] request_timeout_ms = max( session_timeout_ms, KafkaConsumer.DEFAULT_CONFIG["request_timeout_ms"]) self.consumer = KafkaConsumer( self.config["topic_name"], enable_auto_commit=False, api_version=(1, 0, 0), bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], auto_offset_reset="earliest", session_timeout_ms=session_timeout_ms, request_timeout_ms=request_timeout_ms, kafka_client=KarapaceKafkaClient, metadata_max_age_ms=self.config["metadata_max_age_ms"], ) def init_admin_client(self): try: self.admin_client = KafkaAdminClient( api_version_auto_timeout_ms=constants. API_VERSION_AUTO_TIMEOUT_MS, bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], ) return True except (NodeNotReadyError, NoBrokersAvailable, AssertionError): self.log.warning( "No Brokers available yet, retrying init_admin_client()") time.sleep(2.0) except: # pylint: disable=bare-except self.log.exception( "Failed to initialize admin client, retrying init_admin_client()" ) time.sleep(2.0) return False @staticmethod def get_new_schema_topic(config): return NewTopic(name=config["topic_name"], num_partitions=constants.SCHEMA_TOPIC_NUM_PARTITIONS, replication_factor=config["replication_factor"], topic_configs={"cleanup.policy": "compact"}) def create_schema_topic(self): schema_topic = self.get_new_schema_topic(self.config) try: self.log.info("Creating topic: %r", schema_topic) self.admin_client.create_topics( [schema_topic], timeout_ms=constants.TOPIC_CREATION_TIMEOUT_MS) self.log.info("Topic: %r created successfully", self.config["topic_name"]) self.schema_topic = schema_topic return True except TopicAlreadyExistsError: self.log.warning("Topic: %r already exists", self.config["topic_name"]) self.schema_topic = schema_topic return True except: # pylint: disable=bare-except self.log.exception( "Failed to create topic: %r, retrying create_schema_topic()", self.config["topic_name"]) time.sleep(5) return False def get_schema_id(self, new_schema): with self.id_lock: schemas = self.schemas.items() for schema_id, schema in schemas: if schema == new_schema: return schema_id with self.id_lock: self.global_schema_id += 1 return self.global_schema_id def close(self): self.log.info("Closing schema_reader") self.running = False def run(self): while self.running: try: if not self.admin_client: if self.init_admin_client() is False: continue if not self.schema_topic: if self.create_schema_topic() is False: continue if not self.consumer: self.init_consumer() self.handle_messages() except Exception as e: # pylint: disable=broad-except if self.stats: self.stats.unexpected_exception(ex=e, where="schema_reader_loop") self.log.exception( "Unexpected exception in schema reader loop") try: if self.admin_client: self.admin_client.close() if self.consumer: self.consumer.close() except Exception as e: # pylint: disable=broad-except if self.stats: self.stats.unexpected_exception(ex=e, where="schema_reader_exit") self.log.exception("Unexpected exception closing schema reader") def handle_messages(self): raw_msgs = self.consumer.poll(timeout_ms=self.timeout_ms) if self.ready is False and raw_msgs == {}: self.ready = True add_offsets = False if self.master_coordinator is not None: master, _ = self.master_coordinator.get_master_info() # keep old behavior for True. When master is False, then we are a follower, so we should not accept direct # writes anyway. When master is None, then this particular node is waiting for a stable value, so any # messages off the topic are writes performed by another node if master is True: add_offsets = True for _, msgs in raw_msgs.items(): for msg in msgs: try: key = json.loads(msg.key.decode("utf8")) except json.JSONDecodeError: self.log.exception( "Invalid JSON in msg.key: %r, value: %r", msg.key, msg.value) continue value = None if msg.value: try: value = json.loads(msg.value.decode("utf8")) except json.JSONDecodeError: self.log.exception( "Invalid JSON in msg.value: %r, key: %r", msg.value, msg.key) continue self.log.info( "Read new record: key: %r, value: %r, offset: %r", key, value, msg.offset) self.handle_msg(key, value) self.offset = msg.offset self.log.info("Handled message, current offset: %r", self.offset) if self.ready and add_offsets: self.queue.put(self.offset) def handle_msg(self, key: dict, value: dict): if key["keytype"] == "CONFIG": if "subject" in key and key["subject"] is not None: if not value: self.log.info( "Deleting compatibility config completely for subject: %r", key["subject"]) self.subjects[key["subject"]].pop("compatibility", None) return self.log.info("Setting subject: %r config to: %r, value: %r", key["subject"], value["compatibilityLevel"], value) if not key["subject"] in self.subjects: self.log.info( "Adding first version of subject: %r with no schemas", key["subject"]) self.subjects[key["subject"]] = {"schemas": {}} subject_data = self.subjects.get(key["subject"]) subject_data["compatibility"] = value["compatibilityLevel"] else: self.log.info("Setting global config to: %r, value: %r", value["compatibilityLevel"], value) self.config["compatibility"] = value["compatibilityLevel"] elif key["keytype"] == "SCHEMA": if not value: subject, version = key["subject"], key["version"] self.log.info("Deleting subject: %r version: %r completely", subject, version) if subject not in self.subjects: self.log.error("Subject %s did not exist, should have", subject) elif version not in self.subjects[subject]["schemas"]: self.log.error( "Version %d for subject %s did not exist, should have", version, subject) else: self.subjects[subject]["schemas"].pop(version, None) return schema_type = value.get("schemaType", "AVRO") schema_str = value["schema"] try: typed_schema = TypedSchema.parse( schema_type=SchemaType(schema_type), schema_str=schema_str) except InvalidSchema: try: schema_json = json.loads(schema_str) typed_schema = TypedSchema( schema_type=SchemaType(schema_type), schema=schema_json, schema_str=schema_str) except JSONDecodeError: self.log.error("Invalid json: %s", value["schema"]) return self.log.debug("Got typed schema %r", typed_schema) subject = value["subject"] if subject not in self.subjects: self.log.info("Adding first version of subject: %r, value: %r", subject, value) self.subjects[subject] = { "schemas": { value["version"]: { "schema": typed_schema, "version": value["version"], "id": value["id"], "deleted": value.get("deleted", False), } } } self.log.info("Setting schema_id: %r with schema: %r", value["id"], typed_schema) self.schemas[value["id"]] = typed_schema if value["id"] > self.global_schema_id: # Not an existing schema self.global_schema_id = value["id"] elif value.get("deleted", False) is True: self.log.info("Deleting subject: %r, version: %r", subject, value["version"]) if not value["version"] in self.subjects[subject]["schemas"]: self.schemas[value["id"]] = typed_schema else: self.subjects[subject]["schemas"][ value["version"]]["deleted"] = True elif value.get("deleted", False) is False: self.log.info("Adding new version of subject: %r, value: %r", subject, value) self.subjects[subject]["schemas"][value["version"]] = { "schema": typed_schema, "version": value["version"], "id": value["id"], "deleted": value.get("deleted", False), } self.log.info("Setting schema_id: %r with schema: %r", value["id"], value["schema"]) with self.id_lock: self.schemas[value["id"]] = typed_schema if value["id"] > self.global_schema_id: # Not an existing schema self.global_schema_id = value["id"] elif key["keytype"] == "DELETE_SUBJECT": self.log.info("Deleting subject: %r, value: %r", value["subject"], value) if not value["subject"] in self.subjects: self.log.error("Subject: %r did not exist, should have", value["subject"]) else: updated_schemas = { key: self._delete_schema_below_version(schema, value["version"]) for key, schema in self.subjects[ value["subject"]]["schemas"].items() } self.subjects[value["subject"]]["schemas"] = updated_schemas elif key["keytype"] == "NOOP": # for spec completeness pass @staticmethod def _delete_schema_below_version(schema, version): if schema["version"] <= version: schema["deleted"] = True return schema def get_schemas(self, subject): non_deleted_schemas = { key: val for key, val in self.subjects[subject]["schemas"].items() if val.get("deleted", False) is False } return non_deleted_schemas
class SchemaBackup: def __init__(self, config_path, backup_path, topic_option=None): self.config = KarapaceBase.read_config(config_path) self.backup_location = backup_path self.topic_name = topic_option or self.config["topic_name"] self.log = logging.getLogger("SchemaBackup") self.consumer = None self.producer = None self.admin_client = None self.timeout_ms = 1000 def init_consumer(self): self.consumer = KafkaConsumer( self.topic_name, enable_auto_commit=False, bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], auto_offset_reset="earliest", metadata_max_age_ms=self.config["metadata_max_age_ms"], client_factory=KarapaceKafkaClient, ) def init_producer(self): self.producer = KafkaProducer( bootstrap_servers=self.config["bootstrap_uri"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], client_factory=KarapaceKafkaClient, ) def init_admin_client(self): start_time = time.monotonic() wait_time = constants.MINUTE while True: if time.monotonic() - start_time > wait_time: raise Timeout( f"Timeout ({wait_time}) on creating admin client") try: self.admin_client = KafkaAdminClient( api_version_auto_timeout_ms=constants. API_VERSION_AUTO_TIMEOUT_MS, bootstrap_servers=self.config["bootstrap_uri"], client_id=self.config["client_id"], security_protocol=self.config["security_protocol"], ssl_cafile=self.config["ssl_cafile"], ssl_certfile=self.config["ssl_certfile"], ssl_keyfile=self.config["ssl_keyfile"], client_factory=KarapaceKafkaClient, ) break except (NodeNotReadyError, NoBrokersAvailable, AssertionError): self.log.warning( "No Brokers available yet, retrying init_admin_client()") except: # pylint: disable=bare-except self.log.exception( "Failed to initialize admin client, retrying init_admin_client()" ) time.sleep(2.0) def _create_schema_topic_if_needed(self): if self.topic_name != self.config["topic_name"]: self.log.info( "Topic name overridden, not creating a topic with schema configuration" ) return self.init_admin_client() start_time = time.monotonic() wait_time = constants.MINUTE while True: if time.monotonic() - start_time > wait_time: raise Timeout( f"Timeout ({wait_time}) on creating admin client") schema_topic = KafkaSchemaReader.get_new_schema_topic(self.config) try: self.log.info("Creating schema topic: %r", schema_topic) self.admin_client.create_topics( [schema_topic], timeout_ms=constants.TOPIC_CREATION_TIMEOUT_MS) self.log.info("Topic: %r created successfully", self.config["topic_name"]) break except TopicAlreadyExistsError: self.log.info("Topic: %r already exists", self.config["topic_name"]) break except: # pylint: disable=bare-except self.log.exception( "Failed to create topic: %r, retrying _create_schema_topic_if_needed()", self.config["topic_name"]) time.sleep(5) def close(self): self.log.info("Closing schema backup reader") if self.consumer: self.consumer.close() self.consumer = None if self.producer: self.producer.close() self.producer = None if self.admin_client: self.admin_client.close() self.admin_client = None def request_backup(self): if not self.consumer: self.init_consumer() self.log.info("Starting schema backup read for topic: %r", self.topic_name) values = [] raw_msg = self.consumer.poll(timeout_ms=self.timeout_ms) for _, messages in raw_msg.items(): for message in messages: key = message.key.decode("utf8") try: key = json.loads(key) except json.JSONDecodeError: self.log.debug( "Invalid JSON in message.key: %r, value: %r", message.key, message.value) value = None if message.value: value = message.value.decode("utf8") try: value = json.loads(value) except json.JSONDecodeError: self.log.debug( "Invalid JSON in message.value: %r, key: %r", message.value, message.key) values.append((key, value)) ser = json.dumps(values) if self.backup_location: with open(self.backup_location, "w") as fp: fp.write(ser) self.log.info("Schema backup written to %r", self.backup_location) else: print(ser) self.log.info("Schema backup written to stdout") self.close() def restore_backup(self): if not os.path.exists(self.backup_location): raise BackupError("Backup location doesn't exist") self._create_schema_topic_if_needed() if not self.producer: self.init_producer() self.log.info("Starting backup restore for topic: %r", self.topic_name) values = None with open(self.backup_location, "r") as fp: raw_msg = fp.read() values = json.loads(raw_msg) if not values: raise BackupError("Nothing to restore in %s" % self.backup_location) for item in values: key = encode_value(item[0]) value = encode_value(item[1]) future = self.producer.send(self.topic_name, key=key, value=value) self.producer.flush(timeout=self.timeout_ms) msg = future.get(self.timeout_ms) self.log.debug("Sent kafka msg key: %r, value: %r, offset: %r", key, value, msg.offset) self.close()
class Admin(object): def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs self.kwargs['bootstrap_servers'] = self.kwargs.get( 'bootstrap_servers', ['localhost:9092']) self._client = None self.connect() def __enter__(self): return self def __exit__(self, errortype, value, traceback): self.close() def _connect(self): """Connect to kafka admin client""" try: self._client = KafkaAdminClient(**self.kwargs) logger.info('Connected to Kafka Admin client!') except Exception as ex: logger.error('Unable to connect to Kafka Admin client. %s', str(ex)) def connect(self): """Connect to kafka admin""" if self._client is None: self._connect() def create_topics(self, topics: List[dict]): """Create topics in kafka""" topics_to_create = [] for topic in topics: _topic = NewTopic(name=topic['name'], num_partitions=topic.get('num_partitions', 1), replication_factor=topic.get( 'replication_factor', 1)) topics_to_create.append(_topic) try: self._client.create_topics(new_topics=topics_to_create, validate_only=False) logger.info('Topic {%s} creates successfully', topics) except TopicAlreadyExistsError as ex: logger.debug('Skipping to create topic as topic already exists %s', str(ex)) except Exception as ex: logger.error('Unable to create topics. %s', str(ex)) def delete_topics(self, topics: List[str]): """Delete topics""" try: self._client.delete_topics(topics) logger.info('Delete topics: {%s}', topics) except UnknownTopicOrPartitionError as ex: logger.debug('Skipping topic delete as topic dos not exists %s', str(ex)) except Exception as ex: logger.error('Unable to delete topics. %s', str(ex)) def close(self): if self._client is None: return try: self._client.close() logger.info('Closed Kafka Admin client connection') except Exception as ex: logger.error('Unable to close Kafka Admin client cnnection. %s', str(ex))
from kafka.admin import KafkaAdminClient from kafka import KafkaConsumer from config import * delete_topic = "mytopic" admin_client = KafkaAdminClient(bootstrap_servers=BOOTSTRAP_SERVERS) consumer = KafkaConsumer(bootstrap_servers=[BOOTSTRAP_SERVERS]) if delete_topic in consumer.topics(): admin_client.delete_topics(topics=[delete_topic]) if delete_topic not in consumer.topics(): print(f"Topic: {delete_topic} deleted successfully !!") else: print(f"Unable to delete topic: {delete_topic}") else: print(f"Topic: {delete_topic} not found !!") consumer.close() admin_client.close()