def _set_watches(self): """Set watches in zookeeper that will trigger rebalances. Rebalances should be triggered whenever a broker, topic, or consumer znode is changed in zookeeper. This ensures that the balance of the consumer group remains up-to-date with the current state of the cluster. """ self._setting_watches = True # Set all our watches and then rebalance broker_path = '/brokers/ids' try: self._broker_watcher = ChildrenWatch( self._zookeeper, broker_path, self._brokers_changed ) except NoNodeException: raise Exception( 'The broker_path "%s" does not exist in your ' 'ZooKeeper cluster -- is your Kafka cluster running?' % broker_path) self._topics_watcher = ChildrenWatch( self._zookeeper, '/brokers/topics', self._topics_changed ) self._consumer_watcher = ChildrenWatch( self._zookeeper, self._consumer_id_path, self._consumers_changed ) self._setting_watches = False
def _add_self(self): """Add this consumer to the zookeeper participants. Ensures we don't add more participants than partitions """ for i in xrange(self.connect_retries): time.sleep(i**2) # first run is 0, ensures we sleep before retry participants = self._get_participants() if len(self.topic.partitions) > len(participants): break # some room to spare else: logger.debug("More consumers than partitions. " "Waiting %is to retry" % (i+1) ** 2) else: raise NoAvailablePartitionsError("Couldn't acquire partition. " "More consumers than partitions.") path = '%s/%s' % (self.id_path, self.id) self.cluster.zookeeper.create( path, self.topic.name, ephemeral=True, makepath=True) # Set all our watches and then rebalance self._rebalancing = False broker_path = '/brokers/ids' try: self._broker_watcher = ChildrenWatch( self.cluster.zookeeper, broker_path, self._brokers_changed ) except NoNodeException: raise ImproperlyConfiguredError( 'The broker_path "%s" does not exist in your ' 'ZooKeeper cluster -- is your Kafka cluster running?' % broker_path) topics_path = '/brokers/topics' self._topics_watcher = ChildrenWatch( self.cluster.zookeeper, '/brokers/topics', self._topics_changed ) self._rebalancing = True # Final watch will trigger rebalance self._consumer_watcher = ChildrenWatch( self.cluster.zookeeper, self.id_path, self._consumers_changed )
def load(self): # Check whether the pool exists if not self.zk.exists(self.path): raise PoolException("Pool with name {} does not exist!".format( self.name)) # Next load the pool meta-data self.meta, self.meta_stat = self.zk.get(self.path) self.meta = json.loads(self.meta.decode()) # Finally, we need to keep track of resources and nodes ChildrenWatch(self.zk, os.path.join(self.path, 'resources'), self._on_resources_change) ChildrenWatch(self.zk, os.path.join(self.path, 'nodes'), self._on_nodes_change)
def setup(self): logger.debug('Setting up directories and nodes') self.zk.start() # Initialize nodes self.initialize_nodes() # Register watches self.on_running_droid = ChildrenWatch(self.zk, '/droids/running', self.on_running_droid, send_event=True) self.on_assigned_droid = ChildrenWatch( self.zk, '/droids/assigned', self.on_assigned_droid, )
def __init__(self, zookeeper_hosts, kafka_hosts): self.groups_dict = {} self.topics_dict = {} self.brokers_list = [] self.consumer = KafkaConsumer(bootstrap_servers=kafka_hosts.split(',')) self.zk = KazooClient(hosts=zookeeper_hosts) self.zk.add_listener(self.keep_start) self.zk.start() if self.zk.exists('/consumers') is None or self.zk.exists('/brokers') is None: raise ValueError(zookeeper_hosts + 'is not zookeeper of kafka') ChildrenWatch(self.zk, '/consumers', self.groups_watch) ChildrenWatch(self.zk, '/brokers/topics', self.topics_watch) ChildrenWatch(self.zk, '/brokers/ids/', self.brokers_watch) t = threading.Thread(target=self.latest, name=kafka_hosts) t.setDaemon(True) t.start()
def __init__(self): self.zk = KazooClient( hosts="bigdata1:2181,bigdata2:2181,bigdata3:2181") self.validator_children_watcher = ChildrenWatch(client=self.zk, path="/servers", func=self.watcher_func) self.zk.start()
def register_watches(self): logger.info('Registering watches') DataWatch(self.zk, self.path.baseconf(), self.on_base_conf_change) DataWatch(self.zk, self.path.stateconf(), self.on_state_conf_change) ChildrenWatch(self.zk, self.path.toolchain(), self.on_toolchains, send_event=True)
def __init__(self): self.zk = KazooClient( hosts='192.168.31.20:2181,192.168.31.223:2181,129.168.31.144:2181') self.validator_children_watcher = ChildrenWatch( client=self.zk, path='/mproxy/validators', func=self.validator_watcher_fun) self.zk.start()
def __init__(self, path, value=b'', hosts='127.0.0.1:2181'): self.zk = KazooClient(hosts) self.zk.start() self.children = {} self.path = path self.zk.ensure_path(self.path) self.zk.set(self.path, value) self.children_watch = ChildrenWatch(self.zk, path, self.change)
def start(self): adjust_zk_logging_level() self.zk.start() try: ChildrenWatch(self.zk, '', self.choose) except ZookeeperError: self.agent.onNoMasterDetectedMessage() self.stop()
def _master_group_callback(self, data, stat, event): if stat and not self._children_watch: log.info( "Master group %s exists. Starting to watch for election result" % self._cluster.master_group) self._children_watch = ChildrenWatch(self._client, self._cluster.master_group, func=self._child_callback)
def configure_watches(self): self.joined_count = 0 self.finished_count = 0 self.joined = {k: 0 for k in self.dags.keys()} self.finished = {k: 0 for k in self.dags.keys()} def _joined_endpoint_listener(children, event): if event and event.type == EventType.CHILD: if 'joined' in event.path: joined_dag = event.path.split('/')[-1] self.joined[joined_dag] = self.joined[joined_dag] + 1 if (self.joined[joined_dag] == self.dags[joined_dag] ['vertices']): self.joined_count += 1 print('DAGS:%d have joined' % (self.joined_count)) if (self.joined_count == len(self.dags)): self.start_ts = int(time.time() * 1000) print( 'All vertices have joined. Removing start barrier' ) self.start_barrier.remove() return False def _finished_endpoint_listener(children, event): if event and event.type == EventType.CHILD: if 'finished' in event.path: finished_dag = event.path.split('/')[-1] self.finished[ finished_dag] = self.finished[finished_dag] + 1 if (self.finished[finished_dag] == self.dags[finished_dag] ['sinks']): self.finished_count += 1 if (self.finished_count == len(self.dags)): self.end_ts = int(time.time() * 1000) self.end_barrier.remove() return False for graph_id in self.dags.keys(): ChildrenWatch(client=self._zk,\ path='/dom%d/joined/%s'%(self._domain_id,graph_id),\ func=_joined_endpoint_listener,send_event=True) ChildrenWatch(client=self._zk,\ path='/dom%d/finished/%s'%(self._domain_id,graph_id),\ func=_finished_endpoint_listener,send_event=True)
def groups_watch(self, children): for group in [group for group in self.groups_dict.keys() if group not in children]: self.groups_dict.pop(group) for group in [group for group in children if group not in self.groups_dict.keys()]: owners_p = '/consumers/' + group + '/owners' if self.zk.exists(owners_p) is None: continue g_o_t = GroupOwnersTopic() self.groups_dict[group] = g_o_t ChildrenWatch(self.zk, owners_p, g_o_t.g_topic_watch)
def register(self, topics): self.my_client.start() self.create_mw() self.broker_address = self.my_client.get("%s/Leader" % self.zk_root)[0].decode() topics_strength = [] for topic in topics: try: c = self.my_client.get_children("%s/Topic/%s/Pub" % (self.zk_root, topic['topic'])) except NoNodeError: self.my_client.create("%s/Topic/%s/Pub" % (self.zk_root, topic['topic']), makepath=True, ephemeral=False) c = [] id = self.my_client.create("%s/Topic/%s/Pub/Pub" % (self.zk_root, topic['topic']), sequence=True, makepath=True, ephemeral=True) #print (id) strength = id[-3:] #print (strength) history = topic["history"] s_h = ','.join([self.ip_address, strength, str(history)]) self.topic_strength[topic['topic']] = strength self.my_client.set(id, s_h.encode()) self.compare_strength(topic, c) topic_s = copy.deepcopy(topic) topic_s["strength"] = strength topics_strength.append(topic_s) cw = ChildrenWatch( self.my_client, '%s/Topic/%s/Pub' % (self.zk_root, topic['topic']), partial(self.watch_strength, topic)) self.logger.info('pub register to broker on %s. ip=%s, topic=%s' % (self.broker_address, self.ip_address, topic_s)) node_url = "%s/Publisher/" % self.zk_root + self.pub_name node_data = self.ip_address try: self.my_client.create(node_url, node_data.encode(), ephemeral=True, makepath=True) except NodeExistsError: pass self.pub_mw.register(topics_strength) leader_watcher = DataWatch(self.my_client, '%s/Leader' % self.zk_root, self.update_broker_ip_socket) return 0
def start(self): self.zk.start() node = os.path.join(self.root, 'agents', self.hostname) self.zk.ensure_path(node) tasks_node = os.path.join(node, 'tasks') self.zk.ensure_path(tasks_node) self.zk.create(os.path.join(node, 'alive'), str(datetime.datetime.now().timestamp()).encode(), ephemeral=True) ChildrenWatch(self.zk, tasks_node, self.watch) threading.Thread(target=self.run, name='task-runner').start()
def _topics_changed(self, topics): """Watch for the topic we want to show up, then stop watch """ if self.topic.name in topics: self._topic_watcher = ChildrenWatch( self.cluster.zookeeper, '/brokers/topics/%s' % self.topic.name, self._topic_changed ) return False # stop watch
def join(self): path = self.zk.create(os.path.join(self.pool.path, 'nodes', ''), ephemeral=True, sequence=True) self.path = path self.id = path.rsplit('/', 1)[-1] # Watch for leadership changes so we can possibly take over ChildrenWatch(self.zk, os.path.join(self.pool.path, 'leaders'), self._on_leaders_change) # Now that we've joined, lets see if there are any dangling resources we # can take ownership of gevent.spawn(self._check_for_takeover, delay=0)
def start(self): self.zk.start() self._create_znode() self.flag = True if int(self.config['mode']) == 1: broker = BrokerType1(self.config) else: broker = BrokerType2(self.config) self.broker = broker self._register_to_zk() pub_wather = ChildrenWatch(self.zk, '%s/Publisher'%self.zk_root, self._on_pub_change) sub_wather = ChildrenWatch(self.zk, '%s/Subscriber'%self.zk_root, self._on_sub_change) self.logger.info('broker started. mode=%s, port=%s, znode=%s'%(self.config['mode'], self.config['port'], self._znode)) while self.flag: try: broker.handle_req() except RuntimeError as e: if e.args != ('again', ): raise
def join(self): path = self.zk.create(os.path.join(self.pool.path, 'nodes', ''), ephemeral=True, sequence=True) self.path = path self.id = path.rsplit('/', 1)[-1] # Watch for leadership changes so we can possibly take over ChildrenWatch(self.zk, os.path.join(self.pool.path, 'leaders'), self.on_leader_change, send_event=True)
def start_job(self): self.zk.start() job_id = self.zk.create('MR/job/job_', sequence=True, makepath=True) job_id = job_id.split('/')[-1] self.job_id = job_id print('job starting. job_id=%s'%job_id) self.time_info['job_start'] = datetime.now().strftime(s) worker_watcher = ChildrenWatch(self.zk, 'MR/worker', self.remove_from_wait) self.next_task()
def main(): logging.basicConfig() zk=KazooClient(hosts='129.59.107.59:2181') zk.add_listener(connection_state_listener) zk.start() zk.ensure_path("/test/group-membership") zk.ensure_path("/test/barriers/barrier") global barrier barrier=Barrier(client=zk,path="/test/barriers/barrier") ChildrenWatch(client=zk,path="/test/group-membership",func=membership_watch,send_event=True) while True: time.sleep(20)
def watch_folder(self, path): """recursive nonsense""" if "autoscaling.lock" in path: return self.log.debug("Adding watch on {}".format(path)) watcher = ChildrenWatch(self.zk, path, func=self.process_folder_event, send_event=True) self.watchers[path] = watcher children = watcher._prior_children if children and ('instances' in children): self.watch_node("{}/instances".format(path)) elif children: for child in children: self.watch_folder("{}/{}".format(path, child))
def install_watches(self): def _joined_endpoint_listener(children,event): if event and event.type==EventType.CHILD: if 'joined' in event.path: if (len(children)==self.vertex_count): #send start command self._socket.send_string('%s start'%(self.graph_id)) return False elif 'sink' in event.path: if (len(children)==self.sink_count): self._exited=True #send exit command self._socket.send_string('%s exit'%(self.graph_id)) return False ChildrenWatch(client=self._zk,\ path='%s/coord/joined'%(self.graph_id),\ func=_joined_endpoint_listener,send_event=True) ChildrenWatch(client=self._zk,\ path='%s/coord/sink'%(self.graph_id),\ func=_joined_endpoint_listener,send_event=True)
def watchChildren(): try: print("ANOTHER WATCHER") # this zk watcher is used to watch the slave nodes, # if one of the slaves crash the event would be captured by this watcher and a new slave will be spawned if necessary. zk.ensure_path("/zoo/slave") watcher = ChildrenWatch(zk, '/zoo/slave', func=childrenHandler, send_event=True) except: print("ERROR 1:", sys.exc_info())
def watch(self): """ 监听portal那里插入的新任务 :return: """ logger.info("scheduler watch start") jobs_path = '{}/jobs'.format(self.root) if not self.zk.exists(jobs_path): self.zk.ensure_path(jobs_path) signal_path = '{}/signal'.format(self.root) if not self.zk.exists(signal_path): self.zk.ensure_path(signal_path) ChildrenWatch(self.zk, '{}/signal'.format(self.root), self.handle_new_job)
def __init__(self): super().__init__() try: Gateway.zk = KazooClient(hosts='172.25.0.101:2181') Gateway.zk.start() # print(Gateway.constants.SERVER_PREFIX + Gateway.constants.MESSAGE_CONNECTED + "with 127.0.0.1:2181") Gateway.zk.add_listener(self.connection_listener) ChildrenWatch(Gateway.zk, '/nodes', func=Gateway.handle_dbnodes_change) except Exception as e: Gateway.print_error(e) self.add_myself_to_zookeeper() self.dbnodes = []
def watch_folder(self, path, enqueue_children=False): """recursive nonsense""" if "autoscaling.lock" in path: return if path.split('/')[-1] == 'instances': self.watch_node(path, enqueue=enqueue_children) return self.log.info(f"Adding folder watch on {path}") watcher = ChildrenWatch(self.zk, path, func=self.process_folder_event, send_event=True) self.watchers[path] = watcher children = watcher._client.get_children(watcher._path) if children: for child in children: self.watch_folder(f"{path}/{child}", enqueue_children=enqueue_children)
def __init__(self, client, znode_data, storage_manager): self._client = client self._znode_data = znode_data self._client.start() self._client.add_listener(self.zoo_listener) self.create_znodes() self._storage_manager = storage_manager ChildrenWatch(self._client, "/storage", func=self.storage_children_watcher, allow_session_lost=True, send_event=False)
def __init__(self, ip_port, score_board_size): '''Initialize everyting for the watcher''' logging.basicConfig() self.score_board_size = score_board_size self.is_dump = False self.is_init_client = True try: # Create client self.zk = KazooClient(hosts=ip_port, logger=logging) self.zk.start() except Exception as ex: print( 'Error connecting the Zookeeper Service, Please make sure the service is up or the IP:PORT provided is correct' ) sys.exit(-1) # Ensure Paths self.zk.ensure_path('/csjain_queue') self.zk.ensure_path('/csjain_players') # Create Data structures self.score_queue = Queue(self.zk, '/csjain_queue') self.party = Party(self.zk, '/csjain_players') self.online_players = set(self.party) if len(self.score_queue) == 0: print('Most recent scores') print('------------------') print('\n') print('Highest scores') print('--------------') # Create Watchers _ = ChildrenWatch(self.zk, '/csjain_queue', self.process_score) _ = ChildrenWatch(self.zk, '/csjain_players', self.process_client)
def __init__(self, cluster): self.cluster = cluster # The internal cache of all brokers available within the cluster. self.__brokers = {} self._node_path = '/brokers/ids' try: self._broker_watch = ChildrenWatch(self.cluster.zookeeper, self._node_path, self._configure) except NoNodeException: raise ImproperlyConfiguredError( 'The path "%s" does not exist in your ' 'ZooKeeper cluster -- is your Kafka cluster running?' % self._node_path)