def _metadata(kafka_handle, all_topics=True, topic=None, timeout_ms=1000):
"""
Return dict with all information retrievable from librdkafka's Metadata API
"""
topic = ffi.NULL if topic is None else topic.cdata
meta_dp = ffi.new("const rd_kafka_metadata_t **")
err = lib.rd_kafka_metadata(kafka_handle.cdata, int(all_topics), topic,
meta_dp, timeout_ms)
if err != lib.RD_KAFKA_RESP_ERR_NO_ERROR:
raise LibrdkafkaException(err2str(err))
d = {}
meta = meta_dp[0]
d['brokers'] = {}
for i in range(meta.broker_cnt):
b = meta.brokers[i]
d['brokers'][b.id] = dict(host=ffi.string(b.host), port=b.port)
d['topics'] = {}
for i in range(meta.topic_cnt):
t = meta.topics[i]
d['topics'][ffi.string(t.topic)] = topic_d = {}
topic_d['partitions'] = {}
for j in range(t.partition_cnt):
p = t.partitions[j]
topic_d['partitions'][p.id] = dict(
err=err2str(p.err),
leader=p.leader,
replicas=[p.replicas[r] for r in range(p.replica_cnt)],
isrs=[p.isrs[r] for r in range(p.isr_cnt)])
topic_d['err'] = err2str(t.err)
d['orig_broker_id'] = meta.orig_broker_id
d['orig_broker_name'] = ffi.string(meta.orig_broker_name)
lib.rd_kafka_metadata_destroy(meta_dp[0])
return d
def __init__(self, handle_type, config_dict):
"""
config_dict -- A dict with keys as per librdkafka's CONFIGURATION.md
"""
self.config_man = config_handles.ConfigManager(self, config_dict)
errstr = mk_errstr()
self.cdata = lib.rd_kafka_new(handle_type,
self.config_man.pop_config(), errstr,
len(errstr))
if self.cdata == ffi.NULL:
raise LibrdkafkaException(ffi.string(errstr))
else:
finaliser.register(self, lib.rd_kafka_destroy, self.cdata)
def __init__(self, name, kafka_handle, topic_config_dict):
self.kafka_handle = kafka_handle
self.conf_callbacks = [] # keeps callback handles alive
conf = lib.rd_kafka_topic_conf_new()
for k, v in topic_config_dict.items():
if k == "partitioner":
self.conf_callbacks.append(
config_handles.topic_conf_set_partitioner_cb(conf, v))
else:
errstr = mk_errstr()
res = lib.rd_kafka_topic_conf_set(conf, k, v, errstr,
len(errstr))
if res != lib.RD_KAFKA_CONF_OK:
raise LibrdkafkaException(ffi.string(errstr))
self.cdata = lib.rd_kafka_topic_new(self.kafka_handle.cdata, name,
conf)
if self.cdata == ffi.NULL:
raise LibrdkafkaException(errno2str())
else:
finaliser.register(self, lib.rd_kafka_topic_destroy, self.cdata)
def err2str(err):
return ffi.string(lib.rd_kafka_err2str(err))
def name(self):
return ffi.string(lib.rd_kafka_name(self.cdata))