def _metadata(kafka_handle, all_topics=True, topic=None, timeout_ms=1000): """ Return dict with all information retrievable from librdkafka's Metadata API """ topic = ffi.NULL if topic is None else topic.cdata meta_dp = ffi.new("const rd_kafka_metadata_t **") err = lib.rd_kafka_metadata(kafka_handle.cdata, int(all_topics), topic, meta_dp, timeout_ms) if err != lib.RD_KAFKA_RESP_ERR_NO_ERROR: raise LibrdkafkaException(err2str(err)) d = {} meta = meta_dp[0] d['brokers'] = {} for i in range(meta.broker_cnt): b = meta.brokers[i] d['brokers'][b.id] = dict(host=ffi.string(b.host), port=b.port) d['topics'] = {} for i in range(meta.topic_cnt): t = meta.topics[i] d['topics'][ffi.string(t.topic)] = topic_d = {} topic_d['partitions'] = {} for j in range(t.partition_cnt): p = t.partitions[j] topic_d['partitions'][p.id] = dict( err=err2str(p.err), leader=p.leader, replicas=[p.replicas[r] for r in range(p.replica_cnt)], isrs=[p.isrs[r] for r in range(p.isr_cnt)]) topic_d['err'] = err2str(t.err) d['orig_broker_id'] = meta.orig_broker_id d['orig_broker_name'] = ffi.string(meta.orig_broker_name) lib.rd_kafka_metadata_destroy(meta_dp[0]) return d
def __init__(self, handle_type, config_dict): """ config_dict -- A dict with keys as per librdkafka's CONFIGURATION.md """ self.config_man = config_handles.ConfigManager(self, config_dict) errstr = mk_errstr() self.cdata = lib.rd_kafka_new(handle_type, self.config_man.pop_config(), errstr, len(errstr)) if self.cdata == ffi.NULL: raise LibrdkafkaException(ffi.string(errstr)) else: finaliser.register(self, lib.rd_kafka_destroy, self.cdata)
def __init__(self, name, kafka_handle, topic_config_dict): self.kafka_handle = kafka_handle self.conf_callbacks = [] # keeps callback handles alive conf = lib.rd_kafka_topic_conf_new() for k, v in topic_config_dict.items(): if k == "partitioner": self.conf_callbacks.append( config_handles.topic_conf_set_partitioner_cb(conf, v)) else: errstr = mk_errstr() res = lib.rd_kafka_topic_conf_set(conf, k, v, errstr, len(errstr)) if res != lib.RD_KAFKA_CONF_OK: raise LibrdkafkaException(ffi.string(errstr)) self.cdata = lib.rd_kafka_topic_new(self.kafka_handle.cdata, name, conf) if self.cdata == ffi.NULL: raise LibrdkafkaException(errno2str()) else: finaliser.register(self, lib.rd_kafka_topic_destroy, self.cdata)
def err2str(err): return ffi.string(lib.rd_kafka_err2str(err))
def name(self): return ffi.string(lib.rd_kafka_name(self.cdata))