def get_or_create_node(self,
path,
value='',
acl=None,
ephemeral=False,
sequence=False,
makepath=False):
client = None
try:
client = self.get_kazoo_client()
client.start()
if not client.exists(path):
client.create(path,
value=value,
acl=acl,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
return client.get(path)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def delete(node):
'''Delete a node (useful for clearning a lock/semaphore)'''
client = kazoo.client.KazooClient('localhost:2181')
client.start()
client.delete(node, recursive=True)
client.stop()
client.close()
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self.app.conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - client.get(lock_node)[1].created > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException:
yield False
else:
if success:
client.create(lock_node, makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException:
pass
def delete(node):
'''Delete a node (useful for clearning a lock/semaphore)'''
client = kazoo.client.KazooClient('localhost:2181')
client.start()
client.delete(node, recursive=True)
client.stop()
client.close()
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self.app.conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - client.get(lock_node)[1].created > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException:
yield False
else:
if success:
client.create(lock_node, makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException:
pass
def get_or_create_node(self,
path,
value='',
acl=None,
ephemeral=False,
sequence=False,
makepath=False):
client = None
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
try:
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
if not client.exists(path):
client.create(path,
value=value,
acl=acl,
ephemeral=ephemeral,
sequence=sequence,
makepath=makepath)
return client.get(path)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def finalize_client(client):
"""Stops and closes a client, even if it wasn't started."""
client.stop()
try:
client.close()
except TypeError:
# NOTE(harlowja): https://github.com/python-zk/kazoo/issues/167
#
# This can be removed after that one is fixed/merged.
pass
def finalize_client(client):
"""Stops and closes a client, even if it wasn't started."""
client.stop()
try:
client.close()
except TypeError:
# NOTE(harlowja): https://github.com/python-zk/kazoo/issues/167
#
# This can be removed after that one is fixed/merged.
pass
def now(self):
'''Get the catch-up time or current time'''
client = kazoo.client.KazooClient(hosts=self.hosts)
client.start(timeout=1)
if client.exists(self.now_node):
ret = cPickle.loads(client.get(self.now_node)[0])
else:
ret = celery.current_app._get_current_object().now()
client.stop()
client.close()
return ret
def set_value(self, path, value):
client = None
try:
client = self.get_kazoo_client()
client.start()
client.set(path, value)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def set_value(self, path, value):
client = None
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
try:
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
client.set(path, value)
except Exception as ex:
raise ex
finally:
if client:
client.stop()
client.close()
def _remove_lock(self, args, kwargs):
'''Remove mutex for given args and kwargs.'''
client = None
try:
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
client.delete(lock_node)
finally:
if hasattr(client, 'stop'):
client.stop()
client.close()
def _remove_lock(self, args, kwargs):
'''Remove mutex for given args and kwargs.'''
client = None
try:
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
client.delete(lock_node)
finally:
if hasattr(client, 'stop'):
client.stop()
client.close()
def after_return(self, status, retval, task_id, args, kwargs, einfo):
'''Delete lock node of task, regardles of status.'''
if not (self.request.called_directly or self.request.is_eager):
## Only remove the lock if the job was not called locally
client = None
try:
hosts = self._get_app().conf.ZOOKEEPER_HOSTS
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
client.delete(lock_node)
finally:
if hasattr(client, 'stop'):
client.stop()
client.close()
def _lock(self, args, kwargs, use_lock=False):
'''Actual method for creating lock/semaphore.'''
max_leases = kwargs.pop('max_leases', 1)
identifier = kwargs.pop('identifier', None)
blocking = kwargs.pop('blocking', False)
retry = False if blocking else kwargs.pop('retry', False)
if not identifier:
identifier = '%s->%s' % (self.request.id, self.request.hostname)
node_name = 'locks' if use_lock else 'semaphores'
node_path = u'/zkcelery/%s/%s' % (node_name, self.name)
for value in (unicode(x) for x in args):
# This replace here converts a slash into a fraction-slash.
# They look the same but ZooKeeper uses slashes to denote a
# new node and since a value could contain a slash (e.g. a
# uri) we want to make it into a non-reserved character.
node_path += u'.%s' % (value.replace('/', u'\u2044'))
client = lock = None
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
try:
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
if use_lock:
lock = client.Lock(node_path, identifier=identifier)
else:
lock = client.Semaphore(node_path,
identifier=identifier,
max_leases=max_leases)
success = lock.acquire(blocking=blocking)
if retry:
if success:
yield
else:
self.retry(**kwargs)
else:
yield success
except kazoo.exceptions.KazooException:
if retry:
self.retry(**kwargs)
else:
yield False
finally:
if lock:
lock.release()
if client:
client.stop()
client.close()
def _lock(self, args, kwargs, use_lock=False):
'''Actual method for creating lock/semaphore.'''
max_leases = kwargs.pop('max_leases', 1)
identifier = kwargs.pop('identifier', None)
blocking = kwargs.pop('blocking', False)
retry = False if blocking else kwargs.pop('retry', False)
if not identifier:
identifier = '%s->%s' % (self.request.id, self.request.hostname)
node_name = 'locks' if use_lock else 'semaphores'
node_path = u'/zkcelery/%s/%s' % (node_name, self.name)
for value in (unicode(x) for x in args):
# This replace here converts a slash into a fraction-slash.
# They look the same but ZooKeeper uses slashes to denote a
# new node and since a value could contain a slash (e.g. a
# uri) we want to make it into a non-reserved character.
node_path += u'.%s' % (value.replace('/', u'\u2044'))
client = lock = None
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
try:
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
if use_lock:
lock = client.Lock(node_path, identifier=identifier)
else:
lock = client.Semaphore(node_path, identifier=identifier,
max_leases=max_leases)
success = lock.acquire(blocking=blocking)
if retry:
if success:
yield
else:
self.retry(**kwargs)
else:
yield success
except kazoo.exceptions.KazooException:
if retry:
self.retry(**kwargs)
else:
yield False
finally:
if lock:
lock.release()
if client:
client.stop()
client.close()
def list_controls():
'''View current locks/semaphores/mutexes'''
client = kazoo.client.KazooClient('localhost:2181')
client.start()
# Locks
lock_node = u'/zkcelery/locks'
lock_header = False
if client.exists(lock_node):
for lock_name in client.get_children(lock_node):
full_path = u'%s/%s' % (lock_node, lock_name)
lock = client.Lock(full_path)
contenders = lock.contenders()
if contenders:
if not lock_header:
print 'Current Locks'
lock_header = True
print '\t%s - %s' % (full_path, contenders)
# Semaphores
semaphore_node = u'/zkcelery/semaphores'
semaphore_header = False
if client.exists(semaphore_node):
for semaphore_name in client.get_children(semaphore_node):
if '__lock__' not in semaphore_name:
full_path = u'%s/%s' % (semaphore_node, semaphore_name)
semaphore = client.Semaphore(full_path)
lease_holders = semaphore.lease_holders()
if lease_holders:
if not semaphore_header:
print 'Current Semaphores'
semaphore_header = True
print '\t%s - %s' % (full_path, lease_holders)
# Mutex
mutex_node = u'/zkcelery/mutexes'
mutex_header = False
if client.exists(mutex_node):
for mutex_name in client.get_children(mutex_node):
full_path = u'%s/%s' % (mutex_node, mutex_name)
lock_time = time.localtime(client.get(full_path)[1].created)
lock_str = time.strftime('Set at %Y-%m-%d %H:%M:%S', lock_time)
if not mutex_header:
print 'Current Mutexes'
mutex_header = True
print '\t%s - %s' % (full_path, lock_str)
client.stop()
client.close()
def list_controls():
'''View current locks/semaphores/mutexes'''
client = kazoo.client.KazooClient('localhost:2181')
client.start()
# Locks
lock_node = u'/zkcelery/locks'
lock_header = False
if client.exists(lock_node):
for lock_name in client.get_children(lock_node):
full_path = u'%s/%s' % (lock_node, lock_name)
lock = client.Lock(full_path)
contenders = lock.contenders()
if contenders:
if not lock_header:
print 'Current Locks'
lock_header = True
print '\t%s - %s' % (full_path, contenders)
# Semaphores
semaphore_node = u'/zkcelery/semaphores'
semaphore_header = False
if client.exists(semaphore_node):
for semaphore_name in client.get_children(semaphore_node):
if '__lock__' not in semaphore_name:
full_path = u'%s/%s' % (semaphore_node, semaphore_name)
semaphore = client.Semaphore(full_path)
lease_holders = semaphore.lease_holders()
if lease_holders:
if not semaphore_header:
print 'Current Semaphores'
semaphore_header = True
print '\t%s - %s' % (full_path, lease_holders)
# Mutex
mutex_node = u'/zkcelery/mutexes'
mutex_header = False
if client.exists(mutex_node):
for mutex_name in client.get_children(mutex_node):
full_path = u'%s/%s' % (mutex_node, mutex_name)
lock_time = time.localtime(client.get(full_path)[1].created)
lock_str = time.strftime('Set at %Y-%m-%d %H:%M:%S', lock_time)
if not mutex_header:
print 'Current Mutexes'
mutex_header = True
print '\t%s - %s' % (full_path, lock_str)
client.stop()
client.close()
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
client = lock = None
try:
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
identifier = '%s->%s' % (self.request.id, self.request.hostname)
lock = client.Lock(lock_node, identifier=identifier)
mutex_acquired = lock.acquire(blocking=False)
except kazoo.exceptions.KazooException:
yield False
else:
yield mutex_acquired
finally:
if lock:
lock.release()
if client:
client.stop()
client.close()
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
client = lock = None
try:
hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
identifier = '%s->%s' % (self.request.id, self.request.hostname)
lock = client.Lock(lock_node, identifier=identifier)
mutex_acquired = lock.acquire(blocking=False)
except kazoo.exceptions.KazooException:
yield False
else:
yield mutex_acquired
finally:
if lock:
lock.release()
if client:
client.stop()
client.close()
def mutex(self, args, kwargs, delete=False):
'''Creates the mutex locks and yields the mutex status.'''
conf = self._get_app().conf
global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
items = inspect.getcallargs(self.run, *args, **kwargs)
timeout = items.get('mutex_timeout') or global_timeout or 3600
success = False
try:
hosts = conf.ZOOKEEPER_HOSTS
client = kazoo.client.KazooClient(hosts=hosts)
client.start()
lock_node = self._get_node(args, kwargs)
if client.exists(lock_node):
if time.time() - float(client.get(lock_node)[0]) > timeout:
client.delete(lock_node)
success = True
else:
success = True
except kazoo.exceptions.KazooException as exc:
logger.exception('Error stopping execution')
yield False
else:
if success:
client.create(lock_node, six.text_type(time.time()).encode(), makepath=True)
yield True
if delete:
client.delete(lock_node)
else:
logger.debug('This task has been locked.')
yield False
finally:
try:
client.stop()
client.close()
except kazoo.exceptions.KazooException as exc:
pass
def close(client):
client.stop()
client.close()
_logger.info("ZK client has been closed")
def finalize_client(client):
"""Stops and closes a client, even if it wasn't started."""
client.stop()
client.close()
def finalize_client(client):
"""Stops and closes a client, even if it wasn't started."""
client.stop()
client.close()
def close(client):
client.stop()
client.close()
_logger.info("ZK client has been closed")
