def register_watches(self): logger.info('Registering watches') DataWatch(self.zk, self.path.baseconf(), self.on_base_conf_change) DataWatch(self.zk, self.path.stateconf(), self.on_state_conf_change) ChildrenWatch(self.zk, self.path.toolchain(), self.on_toolchains, send_event=True)
def start(self): """ Start the listener to watch the master group. NOTE: The listener only starts watching master after the base ZNode for the group is created. """ DataWatch(self._client, self._cluster.cluster_path, func=self._cluster_path_callback) DataWatch(self._client, self._cluster.master_group, func=self._master_group_callback)
def watch_new_task(self, tasks): for task_id in set(tasks).difference(self.tasks): self.schedule(task_id) DataWatch(self.zk, '/{}/signal/{}'.format(self.root, task_id), partial(self.watch_exist_task, task_id=task_id)) self.tasks = tasks return not self.event.is_set()
def _try_takeover(self, resource, force=False): if self.max_resources and len(self.resources) >= self.max_resources: return False if not force and resource in self._resource_backoff: if time.time() - self._resource_backoff[resource] < 10: return False del self._resource_backoff[resource] if self._resources_acquiring.locked(): return False with self._resources_acquiring: path = os.path.join(self.pool.path, 'leaders', resource) try: self.zk.create(path, unicode.encode(self.id), ephemeral=True) except NodeExistsError: if not force: return False _, metadata = self.zk.get(path) transaction = self.zk.transaction() transaction.delete(path, version=metadata.version) transaction.create(path, unicode.encode(self.id), ephemeral=True) result = transaction.commit() if result[0] is not True or result[1] != path: return False DataWatch(self.zk, path, self._on_resource_leader_change) self.resources.add(resource) if callable(self.on_acquire_resource): self.on_acquire_resource(self, resource) return True
def setup_free_droids(self): running_droids = self.zk.get_children('/droids/running') for droid in running_droids: logger.debug('Setting up running droids') self.zk.create('/droids/free/{}'.format(droid)) DataWatch(self.zk, '/droids/running/{}'.format(droid), self.on_running_droid_change)
def main(): if len(sys.argv) < 5: print(help_msg, "\n") print(sys.argv[0], 'zookeeper_server broker_uri_1 broker_uri_2 broker_uri_3') print('Example:', sys.argv[0], 'localhost:2181 socket://localhost:10001/broker1 ' 'socket://localhost:10002/broker2 socket://localhost:10003/broker3') exit() zk_server = sys.argv[1] broker_uris = sys.argv[2:5] shift_cmd = shift_cmd_template.format(*broker_uris) print('Deploying', shift_cmd) zk_client = KazooClient(zk_server, timeout=10 * 60) print('Connecting to Zookeeper at', zk_server) zk_client.start() for uri in broker_uris: broker_status[uri] = False bid = get_id(uri) # make sure broker is free data, stats = zk_client.get(ZK_BROKER_OPS_STATUS_STR.format(bid)) op_status = OpStatus(data.decode('utf-8').upper()) if op_status not in [OpStatus.Null, OpStatus.Finished]: raise RuntimeError('Cannot start {}, {} is in {} state'.format(shift_cmd, bid, op_status.name)) # update broker's ops status zk_client.set(ZK_BROKER_OPS_STATUS_STR.format(bid), OpStatus.Null.value.encode('utf-8')) # write the cmd to the broker's ops zk_client.set(ZK_BROKER_OPS_STR.format(bid), shift_cmd.encode('utf-8')) # set watches for this broker's op status DataWatch(zk_client, ZK_BROKER_OPS_STATUS_STR.format(bid), func=get_broker_op_data_watcher(uri)) print('Waiting for brokers ...') while not all_done(): time.sleep(1)
def register(self, topics): self.zk_client.start() try: self.zk_client.create('%s/Subscriber/%s' % (self.zk_root, self.name), ('%s,%s' % (self.ip, '')).encode(), ephemeral=True, makepath=True) except NodeExistsError: pass self.create_middleware() for t in topics: try: c = self.zk_client.get_children("%s/Topic/%s/Sub"%(self.zk_root, t['topic'])) except NoNodeError: self.zk_client.create("%s/Topic/%s/Sub"%(self.zk_root, t['topic']), makepath=True, ephemeral=False) c = [] id = self.zk_client.create("%s/Topic/%s/Sub/Sub"%(self.zk_root, t['topic']), sequence=True, makepath=True, ephemeral=True) history = t["history"] s_h = ','.join([self.ip, str(history)]) self.zk_client.set(id, s_h.encode()) self.logger.info('sub register to broker on %s. ip=%s, topic=%s' % (self.ip_b, self.ip, t['topic'])) self.sub_mid.register(topics) DataWatch(self.zk_client, "%s/Leader"%self.zk_root, self.update) return 0
def register(self, topic): self.create_mw() self.sub_mid.register(topic) self.zk_client.create('%s/Subscriber/%s'%(self.zk_root, self.name), ('%s,%s'%(self.ip, topic)).encode(), ephemeral=True, makepath=True) DataWatch(self.zk_client, "%s/Leader"%self.zk_root, self.update) self.logger.info('sub register to broker on %s. ip=%s, topic=%s' % (self.ip_b, self.ip, topic)) return 0
def handle_new_callback(self, callbacks): for callback in set(callbacks).difference(self.callbacks): logger.info("handler new callback: callback={}".format(callback)) DataWatch(self.zk, '{}/callback/{}'.format(self.root, callback), partial(self.handle_exist_callback, callback=callback)) self.run_callback(callback) self.callbacks = callbacks return not self.event.is_set()
def topics_watch(self, children): for topic in [topic for topic in self.topics_dict.keys() if topic not in children]: self.topics_dict.pop(topic) for topic in [topic for topic in children if topic not in self.topics_dict.keys()]: t_v = TopicValue() self.topics_dict[topic] = t_v DataWatch(self.zk, '/brokers/topics/' + topic, t_v.topic_watch) t_v.topic_partition = [TopicPartition(topic, p) for p in self.consumer.partitions_for_topic(topic)]
def _setup_watcher(self): if self.zk_client is not None: self.zk_client.close() self.zk_client = KazooClient( hosts='{}:{}'.format(self._zk_ip, self._zk_port)) self.zk_client.start() DataWatch(client=self.zk_client, path=self._znode_path, func=self._update_config)
def _monitor(self): """Begins watching the ZK path for node changes. """ if not self._zk.exists(self._zk_path): self._log.warn('Path %s does not exist, waiting for it to be created.' % self._zk_path) # Data changed will notify node on creation / deletion via DataWatch(self._zk, self._zk_path, self._data_changed)
def watcher(self, zk_path): try: # 为所要监听的节点开启一个子节点监听器 # ChildrenWatch(client=self._zkc, path=zk_path, func=self._node_change, send_event=True) # 为所要监听的节点开启一个该节点值变化的监听器 DataWatch(client=self.zk, path=zk_path, func=self._data_change) except Exception as e: raise
def data_watch(zk, path): def change(data, stat): if data is None: # data is None equal node not exist, so return false to disable future data change call return False logger.info('Path %s Data Change From %s to %s' % (path, self.children[path.split('/')[-1]], data)) self.children[path.split('/')[-1]] = data return DataWatch(zk, path, change)
def choose(self, children): if not children: self.agent.onNoMasterDetectedMessage() return True masterSeq = max(children) if masterSeq == self.masterSeq: return True self.masterSeq = masterSeq DataWatch(self.zk, '/' + masterSeq, self.notify) return True
def _establishSession(self): if self._assignmentsWatcher is None: # add watch for assignment updates def watchAssignments(data, stat, event): self._currentAssignment = self._processAssignmentsData(data) logger.info('Assignment update received. | assignments= %s', self._currentAssignment) self._ensureAssignmentsPath() self._assignmentWatcher = DataWatch(self.zk, self._assignmentsPath, watchAssignments)
def choose(self, children): children = [child for child in children if child != 'log_replicas'] if not children: self.agent.onNoMasterDetectedMessage() return True masterSeq = min(children) if masterSeq == self.masterSeq: return True self.masterSeq = masterSeq DataWatch(self.zk, '/' + masterSeq, self.notify) return True
def __init__(self, cluster, topic): self.cluster = cluster self.topic = topic self.__brokers = {} self.topic_path = '/brokers/topics/%s' % self.topic.name self._topic_watcher = DataWatch(self.cluster.zookeeper, self.topic_path, self._topic_changed, allow_missing_node=True)
def register(self, topics): self.my_client.start() self.create_mw() self.broker_address = self.my_client.get("%s/Leader" % self.zk_root)[0].decode() topics_strength = [] for topic in topics: try: c = self.my_client.get_children("%s/Topic/%s/Pub" % (self.zk_root, topic['topic'])) except NoNodeError: self.my_client.create("%s/Topic/%s/Pub" % (self.zk_root, topic['topic']), makepath=True, ephemeral=False) c = [] id = self.my_client.create("%s/Topic/%s/Pub/Pub" % (self.zk_root, topic['topic']), sequence=True, makepath=True, ephemeral=True) #print (id) strength = id[-3:] #print (strength) history = topic["history"] s_h = ','.join([self.ip_address, strength, str(history)]) self.topic_strength[topic['topic']] = strength self.my_client.set(id, s_h.encode()) self.compare_strength(topic, c) topic_s = copy.deepcopy(topic) topic_s["strength"] = strength topics_strength.append(topic_s) cw = ChildrenWatch( self.my_client, '%s/Topic/%s/Pub' % (self.zk_root, topic['topic']), partial(self.watch_strength, topic)) self.logger.info('pub register to broker on %s. ip=%s, topic=%s' % (self.broker_address, self.ip_address, topic_s)) node_url = "%s/Publisher/" % self.zk_root + self.pub_name node_data = self.ip_address try: self.my_client.create(node_url, node_data.encode(), ephemeral=True, makepath=True) except NodeExistsError: pass self.pub_mw.register(topics_strength) leader_watcher = DataWatch(self.my_client, '%s/Leader' % self.zk_root, self.update_broker_ip_socket) return 0
def __init__(self, cluster, id_): self.cluster = cluster self.id = int(id_) self.__host = None self.__port = None self.is_dead = False self._node_path = '/brokers/ids/%s' % self.id self._config_watcher = DataWatch(self.cluster.zookeeper, self._node_path, self._configure)
def on_running_droid(self, children, event): if not event: return logger.debug('Registering watches on running droids') assigned_droids = self.zk.get_children('/droids/assigned') free_droids = set(children) - set(assigned_droids) for droid in free_droids: # Implies a new droid became available logger.debug('WooHoo! Another droid joins our ranks!') # Create corresponding node in /droids/free free_p = '/droids/free/{}'.format(droid) self.zk.ensure_path(free_p) droid_p = '/droids/running/{}'.format(droid) DataWatch(self.zk, droid_p, self.on_running_droid_change)
def _add_pub(self, req): result = self.table.add_pub(pub=req['ip'], topics=req['topic']) topics = req['topic'] if isinstance(req['topic'], list) else [req['topic']] for t in topics: t = t['topic'] if not self.zk.exists('%s/Topic/%s/Pub' % (self.zk_root, t)): self.zk.create('%s/Topic/%s/Pub' % (self.zk_root, t), makepath=True) if t not in self.watched_topics: self.watched_topics[t] = DataWatch( self.zk, '%s/Topic/%s/Pub' % (self.zk_root, t), partial(self._on_strengtheset_change, t)) return result
def __init__(self, zk, service, cb=None, hostname=None): self.service = service self.zk = zk self.cb = cb if not hostname: hostname = socket.getfqdn() self.hostname = hostname self.nodemap_path = "/services/%s/nodemaps" % service self.nodemap_watcher = DataWatch( self.zk, self.nodemap_path, self._nodemap_changed )
def _nodemap_changed(self, data, stat): """Called when the nodemap changes.""" if not stat: raise EnvironmentNotFoundException(self.nodemap_path) try: conf_path = self._deserialize_nodemap(data)[self.hostname] except KeyError: conf_path = '/services/%s/conf' % self.service self.config_watcher = DataWatch( self.zk, conf_path, self._config_changed )
def switch_enter(self, ev): dpid = ev.datapath.id self.sws[dpid] = True self.dps[dpid] = ev.datapath dpNode = self.zkConf['root'] + self.zkConf['swstat'] \ + '/' + dpid_to_str(dpid) self.zk.ensure_path(dpNode) if self.election(dpid): self.role = OFPCR_ROLE_MASTER else: self.role = OFPCR_ROLE_SLAVE self.countUp(dpid) self.roleRequest(dpid, self.role) mflag = dpNode + '/' + 'master' DataWatch(self.zk, mflag, self.masterWatcher)
def setup(self): logger.debug('Registering onto zookeeper') self.zk.start() config = get_config() value = { 'thrift_host': get_public_hostname(), 'thrift_port': config['thrift_port'], } path = self.zk.create('/droids/running/droid', value=json.dumps(value), sequence=True, makepath=True, ephemeral=True) self.nodename = path.rsplit('/', 1)[1] DataWatch(self.zk, '/droids/assigned/{}'.format(self.nodename), self.on_droid_release)
def register(self, topic): self.create_mw() self.broker_address = self.my_client.get("%s/Leader" % self.zk_root)[0].decode() self.pub_mw.register(topic) self.logger.info('pub register to bloker on %s. ip=%s, topic=%s' % (self.broker_address, self.ip_address, topic)) node_url = "%s/Publisher/" % self.zk_root + self.pub_name node_data = self.ip_address + "," + topic self.my_client.create(node_url, node_data.encode(), ephemeral=True, makepath=True) leader_watcher = DataWatch(self.my_client, '%s/Leader' % self.zk_root, self.update_broker_ip_socket) return 0
def __init__(self): # 获取Client实例。 self.zk = KazooClient( hosts="192.168.21.145:2181,192.168.21.144:2181,192.168.21.141:2181", timeout=2) # 设置节点监听和数值监听。 self.my_watch1 = ChildrenWatch( client=self.zk, path="/", func=self.validator_watcher_fun) # func= 设置监听函数,这个监听会一直有效。 self.my_watch2 = DataWatch(client=self.zk, path="/app7", func=self.data_watch_fun) # 初始化连接到zookeeper self.zk.start()
def setup_nodes(self): # Setup ephemeral nodes lock = self.zk.Lock(self.path.namelock()) with lock: used_names = self.zk.get_children(self.path.toolchain()) new_name = self.name_gen.generate() while new_name in used_names: new_name = self.name_gen.generate() self.name = new_name # Register watch DataWatch(self.zk, self.path.toolchain(self.name), self.on_sync) # Setup path for conf synchronization self.zk.create(self.path.toolchain(new_name), ephemeral=True) # Put information about node self.zk.create(self.path.node(self.name), value=self.orc_host, ephemeral=True)
def install_listener(self): def listener(data, stat, event): try: if (event and event.type == EventType.CHANGED): curr_ts = int(time.time()) if (self.start_ts > 0): elapsed_time = (curr_ts - self.start_ts) / (60.0) print( 'Current runid:%s. Previous experiment run took:%f mins' % (data, elapsed_time)) self.start_ts = curr_ts except Exception as e: print(e) DataWatch(client=self.zk, path='/runid', func=listener, send_event=True)