def test_ensure_path(self): client = self.client client.ensure_path("/1/2") self.assertTrue(client.exists("/1/2")) client.ensure_path("/1/2/3/4") self.assertTrue(client.exists("/1/2/3/4"))
def mutex(self, args, kwargs, delete=False): '''Creates the mutex locks and yields the mutex status.''' conf = self.app.conf global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None) items = getcallargs(self.run, *args, **kwargs) timeout = items.get('mutex_timeout') or global_timeout or 3600 success = False try: hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181') client = kazoo.client.KazooClient(hosts=hosts) client.start() lock_node = self._get_node(args, kwargs) if client.exists(lock_node): if time.time() - client.get(lock_node)[1].created > timeout: client.delete(lock_node) success = True else: success = True except kazoo.exceptions.KazooException: yield False else: if success: client.create(lock_node, makepath=True) yield True if delete: client.delete(lock_node) else: yield False finally: try: client.stop() client.close() except kazoo.exceptions.KazooException: pass
def get_or_create_node(self, path, value='', acl=None, ephemeral=False, sequence=False, makepath=False): client = None try: client = self.get_kazoo_client() client.start() if not client.exists(path): client.create(path, value=value, acl=acl, ephemeral=ephemeral, sequence=sequence, makepath=makepath) return client.get(path) except Exception as ex: raise ex finally: if client: client.stop() client.close()
def get_or_create_node(self, path, value='', acl=None, ephemeral=False, sequence=False, makepath=False): client = None hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181') try: client = kazoo.client.KazooClient(hosts=hosts) client.start() if not client.exists(path): client.create(path, value=value, acl=acl, ephemeral=ephemeral, sequence=sequence, makepath=makepath) return client.get(path) except Exception as ex: raise ex finally: if client: client.stop() client.close()
def list_controls(): '''View current locks/semaphores/mutexes''' client = kazoo.client.KazooClient('localhost:2181') client.start() # Locks lock_node = u'/zkcelery/locks' lock_header = False if client.exists(lock_node): for lock_name in client.get_children(lock_node): full_path = u'%s/%s' % (lock_node, lock_name) lock = client.Lock(full_path) contenders = lock.contenders() if contenders: if not lock_header: print 'Current Locks' lock_header = True print '\t%s - %s' % (full_path, contenders) # Semaphores semaphore_node = u'/zkcelery/semaphores' semaphore_header = False if client.exists(semaphore_node): for semaphore_name in client.get_children(semaphore_node): if '__lock__' not in semaphore_name: full_path = u'%s/%s' % (semaphore_node, semaphore_name) semaphore = client.Semaphore(full_path) lease_holders = semaphore.lease_holders() if lease_holders: if not semaphore_header: print 'Current Semaphores' semaphore_header = True print '\t%s - %s' % (full_path, lease_holders) # Mutex mutex_node = u'/zkcelery/mutexes' mutex_header = False if client.exists(mutex_node): for mutex_name in client.get_children(mutex_node): full_path = u'%s/%s' % (mutex_node, mutex_name) lock_time = time.localtime(client.get(full_path)[1].created) lock_str = time.strftime('Set at %Y-%m-%d %H:%M:%S', lock_time) if not mutex_header: print 'Current Mutexes' mutex_header = True print '\t%s - %s' % (full_path, lock_str) client.stop() client.close()
def now(self): '''Get the catch-up time or current time''' client = kazoo.client.KazooClient(hosts=self.hosts) client.start(timeout=1) if client.exists(self.now_node): ret = cPickle.loads(client.get(self.now_node)[0]) else: ret = celery.current_app._get_current_object().now() client.stop() client.close() return ret
def _remove_lock(self, args, kwargs): '''Remove mutex for given args and kwargs.''' client = None try: hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181') client = kazoo.client.KazooClient(hosts=hosts) client.start() lock_node = self._get_node(args, kwargs) if client.exists(lock_node): client.delete(lock_node) finally: if hasattr(client, 'stop'): client.stop() client.close()
def __init__(self, client, path, max_count=sys.maxint, default=0): super(ZookeeperCounter, self).__init__(client, path, default) self.max_count = max_count # Delete existing counter if it exists with stale data if client.exists(path): data = client.get(path)[0] if data != b'': try: self.default_type(data) except (TypeError, ValueError): client.delete(path) self._ensure_node()
def after_return(self, status, retval, task_id, args, kwargs, einfo): '''Delete lock node of task, regardles of status.''' if not (self.request.called_directly or self.request.is_eager): ## Only remove the lock if the job was not called locally client = None try: hosts = self._get_app().conf.ZOOKEEPER_HOSTS client = kazoo.client.KazooClient(hosts=hosts) client.start() lock_node = self._get_node(args, kwargs) if client.exists(lock_node): client.delete(lock_node) finally: if hasattr(client, 'stop'): client.stop() client.close()
def mutex(self, args, kwargs, delete=False): '''Creates the mutex locks and yields the mutex status.''' conf = self._get_app().conf global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None) items = inspect.getcallargs(self.run, *args, **kwargs) timeout = items.get('mutex_timeout') or global_timeout or 3600 success = False try: hosts = conf.ZOOKEEPER_HOSTS client = kazoo.client.KazooClient(hosts=hosts) client.start() lock_node = self._get_node(args, kwargs) if client.exists(lock_node): if time.time() - float(client.get(lock_node)[0]) > timeout: client.delete(lock_node) success = True else: success = True except kazoo.exceptions.KazooException as exc: logger.exception('Error stopping execution') yield False else: if success: client.create(lock_node, six.text_type(time.time()).encode(), makepath=True) yield True if delete: client.delete(lock_node) else: logger.debug('This task has been locked.') yield False finally: try: client.stop() client.close() except kazoo.exceptions.KazooException as exc: pass
def delete_zkpath(client, zk_path): zk_path_str = str(zk_path) if client.exists(zk_path_str): print("%s: deleting" % zk_path_str) client.delete(zk_path_str, recursive=True)
node_data = client.get(node_path)[0].decode('utf-8') try: node = json.loads(node_data) except: print("%s: decode failed" % node_path) if not node_data: print("Deleting empty %s" % node_path) client.delete(node_path, recursive=True) else: print(node_data) continue if node["state"] == "deleting": print("Deleting %s" % node_path) client.delete(node_path, recursive=True) else: print(node_path, node["state"]) elif args.force_delete: nodeid = args.force_delete base_path = "/nodepool/nodes" node_path = os.path.join(base_path, nodeid) if not client.exists(node_path): print("Node path does not exist") else: lock_path = os.path.join("/nodepool/nodes/lock", nodeid) print("Force-deleting node %s" % nodeid) client.delete(node_path, recursive=True) if client.exists(lock_path): print("Force-deleting lock path %s" % lock_path) client.delete(lock_path, recursive=True)