def create_from_zookeeper(cls, zkconnect, default_retention=1, fetch_topics=True):
log.info("Connecting to zookeeper {0}".format(zkconnect))
try:
zk = KazooClient(zkconnect)
zk.start()
except Exception as e:
raise ZookeeperException("Cannot connect to Zookeeper: {0}".format(e))
# Get broker list
cluster = cls(retention=default_retention)
add_brokers_from_zk(cluster, zk)
# Get current partition state
if fetch_topics:
log.info("Getting partition list from Zookeeper")
for topic in zk.get_children("/brokers/topics"):
zdata, zstat = zk.get("/brokers/topics/{0}".format(topic))
add_topic_with_replicas(cluster, topic, json_loads(zdata))
set_topic_retention(cluster.topics[topic], zk)
if cluster.num_topics() == 0:
raise ZookeeperException("The cluster specified does not have any topics")
log.info("Closing connection to zookeeper")
zk.stop()
zk.close()
return cluster
def set_topic_retention(topic, zk):
try:
zdata, zstat = zk.get("/config/topics/{0}".format(topic.name))
tdata = json_loads(zdata)
topic.retention = int(tdata['config']['retention.ms'])
except (KeyError, ValueError, KazooException):
# If we can't get the config override for any reason, just stick with whatever the default is
pass
def create_from_json(cls, broker_id, jsondata):
data = json_loads(jsondata)
# These things are required, and we can't proceed if they're not there
try:
newbroker = cls(data['host'], id=broker_id, port=data['port'])
except KeyError:
raise ConfigurationException("Cannot parse broker data in zookeeper. This version of Kafka may not be supported.")
# These things are optional, and are pulled in for convenience or extra features
for attr in ['jmx_port', 'rack', 'version', 'timestamp']:
try:
setattr(newbroker, attr, data[attr])
except KeyError:
pass
# if the broker defines multiple endpoints,
newbroker._set_endpoints(data.get('endpoints', []))
return newbroker
def create_from_json(cls, broker_id, jsondata):
data = json_loads(jsondata)
# These things are required, and we can't proceed if they're not there
try:
newbroker = cls(data['host'], id=broker_id)
except KeyError:
raise ConfigurationException(
"Cannot parse broker data in zookeeper. This version of Kafka may not be supported."
)
# These things are optional, and are pulled in for convenience or extra features
for attr in [
'jmx_port', 'port', 'rack', 'version', 'endpoints', 'timestamp'
]:
try:
setattr(newbroker, attr, data[attr])
except KeyError:
pass
return newbroker
def test_json_loads_bytes(self):
assert json_loads(b'1') == 1
def test_json_loads(self):
assert json_loads('1') == 1
def test_json_loads_bytes(self):
assert json_loads(b'1') == 1
def test_json_loads(self):
assert json_loads('1') == 1