Exemple #1
0
def delete(node):
    '''Delete a node (useful for clearning a lock/semaphore)'''
    client = kazoo.client.KazooClient('localhost:2181')
    client.start()
    client.delete(node, recursive=True)
    client.stop()
    client.close()
Exemple #2
0
 def get_or_create_node(self,
                        path,
                        value='',
                        acl=None,
                        ephemeral=False,
                        sequence=False,
                        makepath=False):
     client = None
     try:
         client = self.get_kazoo_client()
         client.start()
         if not client.exists(path):
             client.create(path,
                           value=value,
                           acl=acl,
                           ephemeral=ephemeral,
                           sequence=sequence,
                           makepath=makepath)
         return client.get(path)
     except Exception as ex:
         raise ex
     finally:
         if client:
             client.stop()
             client.close()
Exemple #3
0
 def mutex(self, args, kwargs, delete=False):
     '''Creates the mutex locks and yields the mutex status.'''
     conf = self.app.conf
     global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
     items = getcallargs(self.run, *args, **kwargs)
     timeout = items.get('mutex_timeout') or global_timeout or 3600
     success = False
     try:
         hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         lock_node = self._get_node(args, kwargs)
         if client.exists(lock_node):
             if time.time() - client.get(lock_node)[1].created > timeout:
                 client.delete(lock_node)
                 success = True
         else:
             success = True
     except kazoo.exceptions.KazooException:
         yield False
     else:
         if success:
             client.create(lock_node, makepath=True)
             yield True
             if delete:
                 client.delete(lock_node)
         else:
             yield False
     finally:
         try:
             client.stop()
             client.close()
         except kazoo.exceptions.KazooException:
             pass
Exemple #4
0
def delete(node):
    '''Delete a node (useful for clearning a lock/semaphore)'''
    client = kazoo.client.KazooClient('localhost:2181')
    client.start()
    client.delete(node, recursive=True)
    client.stop()
    client.close()
Exemple #5
0
 def mutex(self, args, kwargs, delete=False):
     '''Creates the mutex locks and yields the mutex status.'''
     conf = self.app.conf
     global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
     items = getcallargs(self.run, *args, **kwargs)
     timeout = items.get('mutex_timeout') or global_timeout or 3600
     success = False
     try:
         hosts = getattr(conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         lock_node = self._get_node(args, kwargs)
         if client.exists(lock_node):
             if time.time() - client.get(lock_node)[1].created > timeout:
                 client.delete(lock_node)
                 success = True
         else:
             success = True
     except kazoo.exceptions.KazooException:
         yield False
     else:
         if success:
             client.create(lock_node, makepath=True)
             yield True
             if delete:
                 client.delete(lock_node)
         else:
             yield False
     finally:
         try:
             client.stop()
             client.close()
         except kazoo.exceptions.KazooException:
             pass
Exemple #6
0
 def get_or_create_node(self,
                        path,
                        value='',
                        acl=None,
                        ephemeral=False,
                        sequence=False,
                        makepath=False):
     client = None
     hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
     try:
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         if not client.exists(path):
             client.create(path,
                           value=value,
                           acl=acl,
                           ephemeral=ephemeral,
                           sequence=sequence,
                           makepath=makepath)
         return client.get(path)
     except Exception as ex:
         raise ex
     finally:
         if client:
             client.stop()
             client.close()
Exemple #7
0
 def test_bad_create_args(self):
     # We need a non-namespaced client for this test
     client = self._get_nonchroot_client()
     try:
         client.start()
         self.assertRaises(ValueError, client.create, "1/2", "val1")
     finally:
         client.stop()
Exemple #8
0
def finalize_client(client):
    """Stops and closes a client, even if it wasn't started."""
    client.stop()
    try:
        client.close()
    except TypeError:
        # NOTE(harlowja): https://github.com/python-zk/kazoo/issues/167
        #
        # This can be removed after that one is fixed/merged.
        pass
Exemple #9
0
def finalize_client(client):
    """Stops and closes a client, even if it wasn't started."""
    client.stop()
    try:
        client.close()
    except TypeError:
        # NOTE(harlowja): https://github.com/python-zk/kazoo/issues/167
        #
        # This can be removed after that one is fixed/merged.
        pass
Exemple #10
0
 def now(self):
     '''Get the catch-up time or current time'''
     client = kazoo.client.KazooClient(hosts=self.hosts)
     client.start(timeout=1)
     if client.exists(self.now_node):
         ret = cPickle.loads(client.get(self.now_node)[0])
     else:
         ret = celery.current_app._get_current_object().now()
     client.stop()
     client.close()
     return ret
Exemple #11
0
 def set_value(self, path, value):
     client = None
     try:
         client = self.get_kazoo_client()
         client.start()
         client.set(path, value)
     except Exception as ex:
         raise ex
     finally:
         if client:
             client.stop()
             client.close()
Exemple #12
0
 def set_value(self, path, value):
     client = None
     hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
     try:
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         client.set(path, value)
     except Exception as ex:
         raise ex
     finally:
         if client:
             client.stop()
             client.close()
Exemple #13
0
 def _remove_lock(self, args, kwargs):
     '''Remove mutex for given args and kwargs.'''
     client = None
     try:
         hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         lock_node = self._get_node(args, kwargs)
         if client.exists(lock_node):
             client.delete(lock_node)
     finally:
         if hasattr(client, 'stop'):
             client.stop()
             client.close()
Exemple #14
0
 def _remove_lock(self, args, kwargs):
     '''Remove mutex for given args and kwargs.'''
     client = None
     try:
         hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         lock_node = self._get_node(args, kwargs)
         if client.exists(lock_node):
             client.delete(lock_node)
     finally:
         if hasattr(client, 'stop'):
             client.stop()
             client.close()
Exemple #15
0
 def after_return(self, status, retval, task_id, args, kwargs, einfo):
     '''Delete lock node of task, regardles of status.'''
     if not (self.request.called_directly or self.request.is_eager):
         ## Only remove the lock if the job was not called locally
         client = None
         try:
             hosts = self._get_app().conf.ZOOKEEPER_HOSTS
             client = kazoo.client.KazooClient(hosts=hosts)
             client.start()
             lock_node = self._get_node(args, kwargs)
             if client.exists(lock_node):
                 client.delete(lock_node)
         finally:
             if hasattr(client, 'stop'):
                 client.stop()
                 client.close()
Exemple #16
0
 def _lock(self, args, kwargs, use_lock=False):
     '''Actual method for creating lock/semaphore.'''
     max_leases = kwargs.pop('max_leases', 1)
     identifier = kwargs.pop('identifier', None)
     blocking = kwargs.pop('blocking', False)
     retry = False if blocking else kwargs.pop('retry', False)
     if not identifier:
         identifier = '%s->%s' % (self.request.id, self.request.hostname)
     node_name = 'locks' if use_lock else 'semaphores'
     node_path = u'/zkcelery/%s/%s' % (node_name, self.name)
     for value in (unicode(x) for x in args):
         # This replace here converts a slash into a fraction-slash.
         # They look the same but ZooKeeper uses slashes to denote a
         # new node and since a value could contain a slash (e.g. a
         # uri) we want to make it into a non-reserved character.
         node_path += u'.%s' % (value.replace('/', u'\u2044'))
     client = lock = None
     hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
     try:
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         if use_lock:
             lock = client.Lock(node_path, identifier=identifier)
         else:
             lock = client.Semaphore(node_path,
                                     identifier=identifier,
                                     max_leases=max_leases)
         success = lock.acquire(blocking=blocking)
         if retry:
             if success:
                 yield
             else:
                 self.retry(**kwargs)
         else:
             yield success
     except kazoo.exceptions.KazooException:
         if retry:
             self.retry(**kwargs)
         else:
             yield False
     finally:
         if lock:
             lock.release()
         if client:
             client.stop()
             client.close()
Exemple #17
0
 def _lock(self, args, kwargs, use_lock=False):
     '''Actual method for creating lock/semaphore.'''
     max_leases = kwargs.pop('max_leases', 1)
     identifier = kwargs.pop('identifier', None)
     blocking = kwargs.pop('blocking', False)
     retry = False if blocking else kwargs.pop('retry', False)
     if not identifier:
         identifier = '%s->%s' % (self.request.id, self.request.hostname)
     node_name = 'locks' if use_lock else 'semaphores'
     node_path = u'/zkcelery/%s/%s' % (node_name, self.name)
     for value in (unicode(x) for x in args):
         # This replace here converts a slash into a fraction-slash.
         # They look the same but ZooKeeper uses slashes to denote a
         # new node and since a value could contain a slash (e.g. a
         # uri) we want to make it into a non-reserved character.
         node_path += u'.%s' % (value.replace('/', u'\u2044'))
     client = lock = None
     hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
     try:
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         if use_lock:
             lock = client.Lock(node_path, identifier=identifier)
         else:
             lock = client.Semaphore(node_path, identifier=identifier,
                                     max_leases=max_leases)
         success = lock.acquire(blocking=blocking)
         if retry:
             if success:
                 yield
             else:
                 self.retry(**kwargs)
         else:
             yield success
     except kazoo.exceptions.KazooException:
         if retry:
             self.retry(**kwargs)
         else:
             yield False
     finally:
         if lock:
             lock.release()
         if client:
             client.stop()
             client.close()
Exemple #18
0
def list_controls():
    '''View current locks/semaphores/mutexes'''
    client = kazoo.client.KazooClient('localhost:2181')
    client.start()
    # Locks
    lock_node = u'/zkcelery/locks'
    lock_header = False
    if client.exists(lock_node):
        for lock_name in client.get_children(lock_node):
            full_path = u'%s/%s' % (lock_node, lock_name)
            lock = client.Lock(full_path)
            contenders = lock.contenders()
            if contenders:
                if not lock_header:
                    print 'Current Locks'
                    lock_header = True
                print '\t%s - %s' % (full_path, contenders)
    # Semaphores
    semaphore_node = u'/zkcelery/semaphores'
    semaphore_header = False
    if client.exists(semaphore_node):
        for semaphore_name in client.get_children(semaphore_node):
            if '__lock__' not in semaphore_name:
                full_path = u'%s/%s' % (semaphore_node, semaphore_name)
                semaphore = client.Semaphore(full_path)
                lease_holders = semaphore.lease_holders()
                if lease_holders:
                    if not semaphore_header:
                        print 'Current Semaphores'
                        semaphore_header = True
                    print '\t%s - %s' % (full_path, lease_holders)
    # Mutex
    mutex_node = u'/zkcelery/mutexes'
    mutex_header = False
    if client.exists(mutex_node):
        for mutex_name in client.get_children(mutex_node):
            full_path = u'%s/%s' % (mutex_node, mutex_name)
            lock_time = time.localtime(client.get(full_path)[1].created)
            lock_str = time.strftime('Set at %Y-%m-%d %H:%M:%S', lock_time)
            if not mutex_header:
                print 'Current Mutexes'
                mutex_header = True
            print '\t%s - %s' % (full_path, lock_str)
    client.stop()
    client.close()
Exemple #19
0
def list_controls():
    '''View current locks/semaphores/mutexes'''
    client = kazoo.client.KazooClient('localhost:2181')
    client.start()
    # Locks
    lock_node = u'/zkcelery/locks'
    lock_header = False
    if client.exists(lock_node):
        for lock_name in client.get_children(lock_node):
            full_path = u'%s/%s' % (lock_node, lock_name)
            lock = client.Lock(full_path)
            contenders = lock.contenders()
            if contenders:
                if not lock_header:
                    print 'Current Locks'
                    lock_header = True
                print '\t%s - %s' % (full_path, contenders)
    # Semaphores
    semaphore_node = u'/zkcelery/semaphores'
    semaphore_header = False
    if client.exists(semaphore_node):
        for semaphore_name in client.get_children(semaphore_node):
            if '__lock__' not in semaphore_name:
                full_path = u'%s/%s' % (semaphore_node, semaphore_name)
                semaphore = client.Semaphore(full_path)
                lease_holders = semaphore.lease_holders()
                if lease_holders:
                    if not semaphore_header:
                        print 'Current Semaphores'
                        semaphore_header = True
                    print '\t%s - %s' % (full_path, lease_holders)
    # Mutex
    mutex_node = u'/zkcelery/mutexes'
    mutex_header = False
    if client.exists(mutex_node):
        for mutex_name in client.get_children(mutex_node):
            full_path = u'%s/%s' % (mutex_node, mutex_name)
            lock_time = time.localtime(client.get(full_path)[1].created)
            lock_str = time.strftime('Set at %Y-%m-%d %H:%M:%S', lock_time)
            if not mutex_header:
                print 'Current Mutexes'
                mutex_header = True
            print '\t%s - %s' % (full_path, lock_str)
    client.stop()
    client.close()
Exemple #20
0
 def mutex(self, args, kwargs, delete=False):
     '''Creates the mutex locks and yields the mutex status.'''
     client = lock = None
     try:
         hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         lock_node = self._get_node(args, kwargs)
         identifier = '%s->%s' % (self.request.id, self.request.hostname)
         lock = client.Lock(lock_node, identifier=identifier)
         mutex_acquired = lock.acquire(blocking=False)
     except kazoo.exceptions.KazooException:
         yield False
     else:
         yield mutex_acquired
     finally:
         if lock:
             lock.release()
         if client:
             client.stop()
             client.close()
Exemple #21
0
 def mutex(self, args, kwargs, delete=False):
     '''Creates the mutex locks and yields the mutex status.'''
     client = lock = None
     try:
         hosts = getattr(self.app.conf, 'ZOOKEEPER_HOSTS', '127.0.0.1:2181')
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         lock_node = self._get_node(args, kwargs)
         identifier = '%s->%s' % (self.request.id, self.request.hostname)
         lock = client.Lock(lock_node, identifier=identifier)
         mutex_acquired = lock.acquire(blocking=False)
     except kazoo.exceptions.KazooException:
         yield False
     else:
         yield mutex_acquired
     finally:
         if lock:
             lock.release()
         if client:
             client.stop()
             client.close()
Exemple #22
0
 def mutex(self, args, kwargs, delete=False):
     '''Creates the mutex locks and yields the mutex status.'''
     conf = self._get_app().conf
     global_timeout = getattr(conf, 'MUTEX_TIMEOUT', None)
     items = inspect.getcallargs(self.run, *args, **kwargs)
     timeout = items.get('mutex_timeout') or global_timeout or 3600
     success = False
     try:
         hosts = conf.ZOOKEEPER_HOSTS
         client = kazoo.client.KazooClient(hosts=hosts)
         client.start()
         lock_node = self._get_node(args, kwargs)
         if client.exists(lock_node):
             if time.time() - float(client.get(lock_node)[0]) > timeout:
                 client.delete(lock_node)
                 success = True
         else:
             success = True
     except kazoo.exceptions.KazooException as exc:
         logger.exception('Error stopping execution')
         yield False
     else:
         if success:
             client.create(lock_node, six.text_type(time.time()).encode(), makepath=True)
             yield True
             if delete:
                 client.delete(lock_node)
         else:
             logger.debug('This task has been locked.')
             yield False
     finally:
         try:
             client.stop()
             client.close()
         except kazoo.exceptions.KazooException as exc:
             pass
Exemple #23
0
def close(client):
    client.stop()
    client.close()
    _logger.info("ZK client has been closed")
def finalize_client(client):
    """Stops and closes a client, even if it wasn't started."""
    client.stop()
    client.close()
def finalize_client(client):
    """Stops and closes a client, even if it wasn't started."""
    client.stop()
    client.close()
Exemple #26
0
def close(client):
    client.stop()
    client.close()
    _logger.info("ZK client has been closed")