def _set_watches(self):
"""Set watches in zookeeper that will trigger rebalances.
Rebalances should be triggered whenever a broker, topic, or consumer
znode is changed in zookeeper. This ensures that the balance of the
consumer group remains up-to-date with the current state of the
cluster.
"""
self._setting_watches = True
# Set all our watches and then rebalance
broker_path = '/brokers/ids'
try:
self._broker_watcher = ChildrenWatch(
self._zookeeper, broker_path,
self._brokers_changed
)
except NoNodeException:
raise Exception(
'The broker_path "%s" does not exist in your '
'ZooKeeper cluster -- is your Kafka cluster running?'
% broker_path)
self._topics_watcher = ChildrenWatch(
self._zookeeper,
'/brokers/topics',
self._topics_changed
)
self._consumer_watcher = ChildrenWatch(
self._zookeeper, self._consumer_id_path,
self._consumers_changed
)
self._setting_watches = False
def _add_self(self):
"""Add this consumer to the zookeeper participants.
Ensures we don't add more participants than partitions
"""
for i in xrange(self.connect_retries):
time.sleep(i**2) # first run is 0, ensures we sleep before retry
participants = self._get_participants()
if len(self.topic.partitions) > len(participants):
break # some room to spare
else:
logger.debug("More consumers than partitions. "
"Waiting %is to retry" % (i+1) ** 2)
else:
raise NoAvailablePartitionsError("Couldn't acquire partition. "
"More consumers than partitions.")
path = '%s/%s' % (self.id_path, self.id)
self.cluster.zookeeper.create(
path, self.topic.name, ephemeral=True, makepath=True)
# Set all our watches and then rebalance
self._rebalancing = False
broker_path = '/brokers/ids'
try:
self._broker_watcher = ChildrenWatch(
self.cluster.zookeeper, broker_path,
self._brokers_changed
)
except NoNodeException:
raise ImproperlyConfiguredError(
'The broker_path "%s" does not exist in your '
'ZooKeeper cluster -- is your Kafka cluster running?'
% broker_path)
topics_path = '/brokers/topics'
self._topics_watcher = ChildrenWatch(
self.cluster.zookeeper,
'/brokers/topics',
self._topics_changed
)
self._rebalancing = True
# Final watch will trigger rebalance
self._consumer_watcher = ChildrenWatch(
self.cluster.zookeeper, self.id_path,
self._consumers_changed
)
def load(self):
# Check whether the pool exists
if not self.zk.exists(self.path):
raise PoolException("Pool with name {} does not exist!".format(
self.name))
# Next load the pool meta-data
self.meta, self.meta_stat = self.zk.get(self.path)
self.meta = json.loads(self.meta.decode())
# Finally, we need to keep track of resources and nodes
ChildrenWatch(self.zk, os.path.join(self.path, 'resources'),
self._on_resources_change)
ChildrenWatch(self.zk, os.path.join(self.path, 'nodes'),
self._on_nodes_change)
def setup(self):
logger.debug('Setting up directories and nodes')
self.zk.start()
# Initialize nodes
self.initialize_nodes()
# Register watches
self.on_running_droid = ChildrenWatch(self.zk,
'/droids/running',
self.on_running_droid,
send_event=True)
self.on_assigned_droid = ChildrenWatch(
self.zk,
'/droids/assigned',
self.on_assigned_droid,
)
def __init__(self, zookeeper_hosts, kafka_hosts):
self.groups_dict = {}
self.topics_dict = {}
self.brokers_list = []
self.consumer = KafkaConsumer(bootstrap_servers=kafka_hosts.split(','))
self.zk = KazooClient(hosts=zookeeper_hosts)
self.zk.add_listener(self.keep_start)
self.zk.start()
if self.zk.exists('/consumers') is None or self.zk.exists('/brokers') is None:
raise ValueError(zookeeper_hosts + 'is not zookeeper of kafka')
ChildrenWatch(self.zk, '/consumers', self.groups_watch)
ChildrenWatch(self.zk, '/brokers/topics', self.topics_watch)
ChildrenWatch(self.zk, '/brokers/ids/', self.brokers_watch)
t = threading.Thread(target=self.latest, name=kafka_hosts)
t.setDaemon(True)
t.start()
def __init__(self):
self.zk = KazooClient(
hosts="bigdata1:2181,bigdata2:2181,bigdata3:2181")
self.validator_children_watcher = ChildrenWatch(client=self.zk,
path="/servers",
func=self.watcher_func)
self.zk.start()
def register_watches(self):
logger.info('Registering watches')
DataWatch(self.zk, self.path.baseconf(), self.on_base_conf_change)
DataWatch(self.zk, self.path.stateconf(), self.on_state_conf_change)
ChildrenWatch(self.zk,
self.path.toolchain(),
self.on_toolchains,
send_event=True)
def __init__(self):
self.zk = KazooClient(
hosts='192.168.31.20:2181,192.168.31.223:2181,129.168.31.144:2181')
self.validator_children_watcher = ChildrenWatch(
client=self.zk,
path='/mproxy/validators',
func=self.validator_watcher_fun)
self.zk.start()
def __init__(self, path, value=b'', hosts='127.0.0.1:2181'):
self.zk = KazooClient(hosts)
self.zk.start()
self.children = {}
self.path = path
self.zk.ensure_path(self.path)
self.zk.set(self.path, value)
self.children_watch = ChildrenWatch(self.zk, path, self.change)
def start(self):
adjust_zk_logging_level()
self.zk.start()
try:
ChildrenWatch(self.zk, '', self.choose)
except ZookeeperError:
self.agent.onNoMasterDetectedMessage()
self.stop()
def _master_group_callback(self, data, stat, event):
if stat and not self._children_watch:
log.info(
"Master group %s exists. Starting to watch for election result"
% self._cluster.master_group)
self._children_watch = ChildrenWatch(self._client,
self._cluster.master_group,
func=self._child_callback)
def configure_watches(self):
self.joined_count = 0
self.finished_count = 0
self.joined = {k: 0 for k in self.dags.keys()}
self.finished = {k: 0 for k in self.dags.keys()}
def _joined_endpoint_listener(children, event):
if event and event.type == EventType.CHILD:
if 'joined' in event.path:
joined_dag = event.path.split('/')[-1]
self.joined[joined_dag] = self.joined[joined_dag] + 1
if (self.joined[joined_dag] == self.dags[joined_dag]
['vertices']):
self.joined_count += 1
print('DAGS:%d have joined' % (self.joined_count))
if (self.joined_count == len(self.dags)):
self.start_ts = int(time.time() * 1000)
print(
'All vertices have joined. Removing start barrier'
)
self.start_barrier.remove()
return False
def _finished_endpoint_listener(children, event):
if event and event.type == EventType.CHILD:
if 'finished' in event.path:
finished_dag = event.path.split('/')[-1]
self.finished[
finished_dag] = self.finished[finished_dag] + 1
if (self.finished[finished_dag] == self.dags[finished_dag]
['sinks']):
self.finished_count += 1
if (self.finished_count == len(self.dags)):
self.end_ts = int(time.time() * 1000)
self.end_barrier.remove()
return False
for graph_id in self.dags.keys():
ChildrenWatch(client=self._zk,\
path='/dom%d/joined/%s'%(self._domain_id,graph_id),\
func=_joined_endpoint_listener,send_event=True)
ChildrenWatch(client=self._zk,\
path='/dom%d/finished/%s'%(self._domain_id,graph_id),\
func=_finished_endpoint_listener,send_event=True)
def groups_watch(self, children):
for group in [group for group in self.groups_dict.keys() if group not in children]:
self.groups_dict.pop(group)
for group in [group for group in children if group not in self.groups_dict.keys()]:
owners_p = '/consumers/' + group + '/owners'
if self.zk.exists(owners_p) is None:
continue
g_o_t = GroupOwnersTopic()
self.groups_dict[group] = g_o_t
ChildrenWatch(self.zk, owners_p, g_o_t.g_topic_watch)
def register(self, topics):
self.my_client.start()
self.create_mw()
self.broker_address = self.my_client.get("%s/Leader" %
self.zk_root)[0].decode()
topics_strength = []
for topic in topics:
try:
c = self.my_client.get_children("%s/Topic/%s/Pub" %
(self.zk_root, topic['topic']))
except NoNodeError:
self.my_client.create("%s/Topic/%s/Pub" %
(self.zk_root, topic['topic']),
makepath=True,
ephemeral=False)
c = []
id = self.my_client.create("%s/Topic/%s/Pub/Pub" %
(self.zk_root, topic['topic']),
sequence=True,
makepath=True,
ephemeral=True)
#print (id)
strength = id[-3:]
#print (strength)
history = topic["history"]
s_h = ','.join([self.ip_address, strength, str(history)])
self.topic_strength[topic['topic']] = strength
self.my_client.set(id, s_h.encode())
self.compare_strength(topic, c)
topic_s = copy.deepcopy(topic)
topic_s["strength"] = strength
topics_strength.append(topic_s)
cw = ChildrenWatch(
self.my_client,
'%s/Topic/%s/Pub' % (self.zk_root, topic['topic']),
partial(self.watch_strength, topic))
self.logger.info('pub register to broker on %s. ip=%s, topic=%s' %
(self.broker_address, self.ip_address, topic_s))
node_url = "%s/Publisher/" % self.zk_root + self.pub_name
node_data = self.ip_address
try:
self.my_client.create(node_url,
node_data.encode(),
ephemeral=True,
makepath=True)
except NodeExistsError:
pass
self.pub_mw.register(topics_strength)
leader_watcher = DataWatch(self.my_client, '%s/Leader' % self.zk_root,
self.update_broker_ip_socket)
return 0
def start(self):
self.zk.start()
node = os.path.join(self.root, 'agents', self.hostname)
self.zk.ensure_path(node)
tasks_node = os.path.join(node, 'tasks')
self.zk.ensure_path(tasks_node)
self.zk.create(os.path.join(node, 'alive'),
str(datetime.datetime.now().timestamp()).encode(),
ephemeral=True)
ChildrenWatch(self.zk, tasks_node, self.watch)
threading.Thread(target=self.run, name='task-runner').start()
def _topics_changed(self, topics):
"""Watch for the topic we want to show up, then stop watch
"""
if self.topic.name in topics:
self._topic_watcher = ChildrenWatch(
self.cluster.zookeeper,
'/brokers/topics/%s' % self.topic.name,
self._topic_changed
)
return False # stop watch
def join(self):
path = self.zk.create(os.path.join(self.pool.path, 'nodes', ''), ephemeral=True, sequence=True)
self.path = path
self.id = path.rsplit('/', 1)[-1]
# Watch for leadership changes so we can possibly take over
ChildrenWatch(self.zk, os.path.join(self.pool.path, 'leaders'), self._on_leaders_change)
# Now that we've joined, lets see if there are any dangling resources we
# can take ownership of
gevent.spawn(self._check_for_takeover, delay=0)
def start(self):
self.zk.start()
self._create_znode()
self.flag = True
if int(self.config['mode']) == 1:
broker = BrokerType1(self.config)
else:
broker = BrokerType2(self.config)
self.broker = broker
self._register_to_zk()
pub_wather = ChildrenWatch(self.zk, '%s/Publisher'%self.zk_root, self._on_pub_change)
sub_wather = ChildrenWatch(self.zk, '%s/Subscriber'%self.zk_root, self._on_sub_change)
self.logger.info('broker started. mode=%s, port=%s, znode=%s'%(self.config['mode'],
self.config['port'], self._znode))
while self.flag:
try:
broker.handle_req()
except RuntimeError as e:
if e.args != ('again', ):
raise
def join(self):
path = self.zk.create(os.path.join(self.pool.path, 'nodes', ''),
ephemeral=True,
sequence=True)
self.path = path
self.id = path.rsplit('/', 1)[-1]
# Watch for leadership changes so we can possibly take over
ChildrenWatch(self.zk,
os.path.join(self.pool.path, 'leaders'),
self.on_leader_change,
send_event=True)
def start_job(self):
self.zk.start()
job_id = self.zk.create('MR/job/job_', sequence=True, makepath=True)
job_id = job_id.split('/')[-1]
self.job_id = job_id
print('job starting. job_id=%s'%job_id)
self.time_info['job_start'] = datetime.now().strftime(s)
worker_watcher = ChildrenWatch(self.zk, 'MR/worker', self.remove_from_wait)
self.next_task()
def main():
logging.basicConfig()
zk=KazooClient(hosts='129.59.107.59:2181')
zk.add_listener(connection_state_listener)
zk.start()
zk.ensure_path("/test/group-membership")
zk.ensure_path("/test/barriers/barrier")
global barrier
barrier=Barrier(client=zk,path="/test/barriers/barrier")
ChildrenWatch(client=zk,path="/test/group-membership",func=membership_watch,send_event=True)
while True:
time.sleep(20)
def watch_folder(self, path):
"""recursive nonsense"""
if "autoscaling.lock" in path:
return
self.log.debug("Adding watch on {}".format(path))
watcher = ChildrenWatch(self.zk, path, func=self.process_folder_event, send_event=True)
self.watchers[path] = watcher
children = watcher._prior_children
if children and ('instances' in children):
self.watch_node("{}/instances".format(path))
elif children:
for child in children:
self.watch_folder("{}/{}".format(path, child))
def install_watches(self):
def _joined_endpoint_listener(children,event):
if event and event.type==EventType.CHILD:
if 'joined' in event.path:
if (len(children)==self.vertex_count):
#send start command
self._socket.send_string('%s start'%(self.graph_id))
return False
elif 'sink' in event.path:
if (len(children)==self.sink_count):
self._exited=True
#send exit command
self._socket.send_string('%s exit'%(self.graph_id))
return False
ChildrenWatch(client=self._zk,\
path='%s/coord/joined'%(self.graph_id),\
func=_joined_endpoint_listener,send_event=True)
ChildrenWatch(client=self._zk,\
path='%s/coord/sink'%(self.graph_id),\
func=_joined_endpoint_listener,send_event=True)
def watchChildren():
try:
print("ANOTHER WATCHER")
# this zk watcher is used to watch the slave nodes,
# if one of the slaves crash the event would be captured by this watcher and a new slave will be spawned if necessary.
zk.ensure_path("/zoo/slave")
watcher = ChildrenWatch(zk,
'/zoo/slave',
func=childrenHandler,
send_event=True)
except:
print("ERROR 1:", sys.exc_info())
def watch(self):
"""
监听portal那里插入的新任务
:return:
"""
logger.info("scheduler watch start")
jobs_path = '{}/jobs'.format(self.root)
if not self.zk.exists(jobs_path):
self.zk.ensure_path(jobs_path)
signal_path = '{}/signal'.format(self.root)
if not self.zk.exists(signal_path):
self.zk.ensure_path(signal_path)
ChildrenWatch(self.zk, '{}/signal'.format(self.root),
self.handle_new_job)
def __init__(self):
super().__init__()
try:
Gateway.zk = KazooClient(hosts='172.25.0.101:2181')
Gateway.zk.start()
# print(Gateway.constants.SERVER_PREFIX + Gateway.constants.MESSAGE_CONNECTED + "with 127.0.0.1:2181")
Gateway.zk.add_listener(self.connection_listener)
ChildrenWatch(Gateway.zk,
'/nodes',
func=Gateway.handle_dbnodes_change)
except Exception as e:
Gateway.print_error(e)
self.add_myself_to_zookeeper()
self.dbnodes = []
def watch_folder(self, path, enqueue_children=False):
"""recursive nonsense"""
if "autoscaling.lock" in path:
return
if path.split('/')[-1] == 'instances':
self.watch_node(path, enqueue=enqueue_children)
return
self.log.info(f"Adding folder watch on {path}")
watcher = ChildrenWatch(self.zk, path, func=self.process_folder_event, send_event=True)
self.watchers[path] = watcher
children = watcher._client.get_children(watcher._path)
if children:
for child in children:
self.watch_folder(f"{path}/{child}", enqueue_children=enqueue_children)
def __init__(self, client, znode_data, storage_manager):
self._client = client
self._znode_data = znode_data
self._client.start()
self._client.add_listener(self.zoo_listener)
self.create_znodes()
self._storage_manager = storage_manager
ChildrenWatch(self._client,
"/storage",
func=self.storage_children_watcher,
allow_session_lost=True,
send_event=False)
def __init__(self, ip_port, score_board_size):
'''Initialize everyting for the watcher'''
logging.basicConfig()
self.score_board_size = score_board_size
self.is_dump = False
self.is_init_client = True
try:
# Create client
self.zk = KazooClient(hosts=ip_port, logger=logging)
self.zk.start()
except Exception as ex:
print(
'Error connecting the Zookeeper Service, Please make sure the service is up or the IP:PORT provided is correct'
)
sys.exit(-1)
# Ensure Paths
self.zk.ensure_path('/csjain_queue')
self.zk.ensure_path('/csjain_players')
# Create Data structures
self.score_queue = Queue(self.zk, '/csjain_queue')
self.party = Party(self.zk, '/csjain_players')
self.online_players = set(self.party)
if len(self.score_queue) == 0:
print('Most recent scores')
print('------------------')
print('\n')
print('Highest scores')
print('--------------')
# Create Watchers
_ = ChildrenWatch(self.zk, '/csjain_queue', self.process_score)
_ = ChildrenWatch(self.zk, '/csjain_players', self.process_client)
def __init__(self, cluster):
self.cluster = cluster
# The internal cache of all brokers available within the cluster.
self.__brokers = {}
self._node_path = '/brokers/ids'
try:
self._broker_watch = ChildrenWatch(self.cluster.zookeeper,
self._node_path,
self._configure)
except NoNodeException:
raise ImproperlyConfiguredError(
'The path "%s" does not exist in your '
'ZooKeeper cluster -- is your Kafka cluster running?' %
self._node_path)
