def test_ensure_path(self):
client = self.client
client.ensure_path("/1/2")
self.assertTrue(client.exists("/1/2"))
client.ensure_path("/1/2/3/4")
self.assertTrue(client.exists("/1/2/3/4"))
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self.app.conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - client.get(lock_node)[1].created > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException:
yield False
else:
if success:
client.create(lock_node, makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException:
pass
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self.app.conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - client.get(lock_node)[1].created > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException:
yield False
else:
if success:
client.create(lock_node, makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException:
pass
def get_or_create_node(self,
path,
value='',
acl=None,
ephemeral=False,
sequence=False,
makepath=False):
client = None
try:
client = self.get_kazoo_client()
client.start()
if not client.exists(path):
client.create(path,
value=value,
acl=acl,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
return client.get(path)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def get_or_create_node(self,
path,
value='',
acl=None,
ephemeral=False,
sequence=False,
makepath=False):
client = None
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
try:
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
if not client.exists(path):
client.create(path,
value=value,
acl=acl,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
return client.get(path)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def list_controls():
'''View current locks/semaphores/mutexes'''
client = kazoo.client.KazooClient('localhost:2181')
client.start()
# Locks
lock_node = u'/zkcelery/locks'
lock_header = False
if client.exists(lock_node):
for lock_name in client.get_children(lock_node):
full_path = u'%s/%s' % (lock_node, lock_name)
lock = client.Lock(full_path)
contenders = lock.contenders()
if contenders:
if not lock_header:
print 'Current Locks'
lock_header = True
print '\t%s - %s' % (full_path, contenders)
# Semaphores
semaphore_node = u'/zkcelery/semaphores'
semaphore_header = False
if client.exists(semaphore_node):
for semaphore_name in client.get_children(semaphore_node):
if '__lock__' not in semaphore_name:
full_path = u'%s/%s' % (semaphore_node, semaphore_name)
semaphore = client.Semaphore(full_path)
lease_holders = semaphore.lease_holders()
if lease_holders:
if not semaphore_header:
print 'Current Semaphores'
semaphore_header = True
print '\t%s - %s' % (full_path, lease_holders)
# Mutex
mutex_node = u'/zkcelery/mutexes'
mutex_header = False
if client.exists(mutex_node):
for mutex_name in client.get_children(mutex_node):
full_path = u'%s/%s' % (mutex_node, mutex_name)
lock_time = time.localtime(client.get(full_path)[1].created)
lock_str = time.strftime('Set at %Y-%m-%d %H:%M:%S', lock_time)
if not mutex_header:
print 'Current Mutexes'
mutex_header = True
print '\t%s - %s' % (full_path, lock_str)
client.stop()
client.close()
def list_controls():
'''View current locks/semaphores/mutexes'''
client = kazoo.client.KazooClient('localhost:2181')
client.start()
# Locks
lock_node = u'/zkcelery/locks'
lock_header = False
if client.exists(lock_node):
for lock_name in client.get_children(lock_node):
full_path = u'%s/%s' % (lock_node, lock_name)
lock = client.Lock(full_path)
contenders = lock.contenders()
if contenders:
if not lock_header:
print 'Current Locks'
lock_header = True
print '\t%s - %s' % (full_path, contenders)
# Semaphores
semaphore_node = u'/zkcelery/semaphores'
semaphore_header = False
if client.exists(semaphore_node):
for semaphore_name in client.get_children(semaphore_node):
if '__lock__' not in semaphore_name:
full_path = u'%s/%s' % (semaphore_node, semaphore_name)
semaphore = client.Semaphore(full_path)
lease_holders = semaphore.lease_holders()
if lease_holders:
if not semaphore_header:
print 'Current Semaphores'
semaphore_header = True
print '\t%s - %s' % (full_path, lease_holders)
# Mutex
mutex_node = u'/zkcelery/mutexes'
mutex_header = False
if client.exists(mutex_node):
for mutex_name in client.get_children(mutex_node):
full_path = u'%s/%s' % (mutex_node, mutex_name)
lock_time = time.localtime(client.get(full_path)[1].created)
lock_str = time.strftime('Set at %Y-%m-%d %H:%M:%S', lock_time)
if not mutex_header:
print 'Current Mutexes'
mutex_header = True
print '\t%s - %s' % (full_path, lock_str)
client.stop()
client.close()
def now(self):
'''Get the catch-up time or current time'''
client = kazoo.client.KazooClient(hosts=self.hosts)
client.start(timeout=1)
if client.exists(self.now_node):
ret = cPickle.loads(client.get(self.now_node)[0])
else:
ret = celery.current_app._get_current_object().now()
client.stop()
client.close()
return ret
def _remove_lock(self, args, kwargs):
'''Remove mutex for given args and kwargs.'''
client = None
try:
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
client.delete(lock_node)
finally:
if hasattr(client, 'stop'):
client.stop()
client.close()
def __init__(self, client, path, max_count=sys.maxint, default=0):
super(ZookeeperCounter, self).__init__(client, path, default)
self.max_count = max_count
# Delete existing counter if it exists with stale data
if client.exists(path):
data = client.get(path)[0]
if data != b'':
try:
self.default_type(data)
except (TypeError, ValueError):
client.delete(path)
self._ensure_node()
def _remove_lock(self, args, kwargs):
'''Remove mutex for given args and kwargs.'''
client = None
try:
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
client.delete(lock_node)
finally:
if hasattr(client, 'stop'):
client.stop()
client.close()
def __init__(self, client, path, max_count=sys.maxint, default=0):
super(ZookeeperCounter, self).__init__(client, path, default)
self.max_count = max_count
# Delete existing counter if it exists with stale data
if client.exists(path):
data = client.get(path)[0]
if data != b'':
try:
self.default_type(data)
except (TypeError, ValueError):
client.delete(path)
self._ensure_node()
def after_return(self, status, retval, task_id, args, kwargs, einfo):
'''Delete lock node of task, regardles of status.'''
if not (self.request.called_directly or self.request.is_eager):
## Only remove the lock if the job was not called locally
client = None
try:
hosts = self._get_app().conf.ZOOKEEPER_HOSTS
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
client.delete(lock_node)
finally:
if hasattr(client, 'stop'):
client.stop()
client.close()
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self._get_app().conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = inspect.getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = conf.ZOOKEEPER_HOSTS
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - float(client.get(lock_node)[0]) > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException as exc:
logger.exception('Error stopping execution')
yield False
else:
if success:
client.create(lock_node, six.text_type(time.time()).encode(), makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
logger.debug('This task has been locked.')
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException as exc:
pass
def delete_zkpath(client, zk_path):
zk_path_str = str(zk_path)
if client.exists(zk_path_str):
print("%s: deleting" % zk_path_str)
client.delete(zk_path_str, recursive=True)
node_data = client.get(node_path)[0].decode('utf-8')
try:
node = json.loads(node_data)
except:
print("%s: decode failed" % node_path)
if not node_data:
print("Deleting empty %s" % node_path)
client.delete(node_path, recursive=True)
else:
print(node_data)
continue
if node["state"] == "deleting":
print("Deleting %s" % node_path)
client.delete(node_path, recursive=True)
else:
print(node_path, node["state"])
elif args.force_delete:
nodeid = args.force_delete
base_path = "/nodepool/nodes"
node_path = os.path.join(base_path, nodeid)
if not client.exists(node_path):
print("Node path does not exist")
else:
lock_path = os.path.join("/nodepool/nodes/lock", nodeid)
print("Force-deleting node %s" % nodeid)
client.delete(node_path, recursive=True)
if client.exists(lock_path):
print("Force-deleting lock path %s" % lock_path)
client.delete(lock_path, recursive=True)