Example #1
0
 def __init__(self,
              board,
              name,
              client,
              path,
              uuid=None,
              details=None,
              book=None,
              book_data=None,
              created_on=None,
              backend=None,
              priority=base.JobPriority.NORMAL):
     super(ZookeeperJob, self).__init__(board,
                                        name,
                                        uuid=uuid,
                                        details=details,
                                        backend=backend,
                                        book=book,
                                        book_data=book_data)
     self._client = client
     self._path = k_paths.normpath(path)
     self._lock_path = self._path + board.LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(board.JOB_PREFIX):])
     self._priority = priority
Example #2
0
    def set(self, path, value, version=-1):
        self.verify()
        path = k_paths.normpath(path)
        with self.storage.lock:
            if version != -1:
                (_data, stat) = self.get(path)
                if stat and stat.version != version:
                    raise k_exceptions.BadVersionError("Version mismatch %s "
                                                       "!= %s" % (stat.version,
                                                                  version))
            try:
                self.storage[path]['data'] = str(value)
                self.storage[path]['version'] += 1
            except KeyError:
                raise k_exceptions.NoNodeError("No path %s" % (path))
            parents = sorted(six.iterkeys(self.storage.get_parents(path)))

        # Fire any attached watches.
        event = k_states.WatchedEvent(type=k_states.EventType.CHANGED,
                                      state=k_states.KeeperState.CONNECTED,
                                      path=path)
        self._fire_watches([path], event)
        event = k_states.WatchedEvent(type=k_states.EventType.CHILD,
                                      state=k_states.KeeperState.CONNECTED,
                                      path=path)
        self._fire_watches(parents, event)
Example #3
0
 def __init__(self,
              name,
              board,
              client,
              backend,
              path,
              uuid=None,
              details=None,
              book=None,
              book_data=None,
              created_on=None):
     super(ZookeeperJob, self).__init__(name, uuid=uuid, details=details)
     self._board = board
     self._book = book
     if not book_data:
         book_data = {}
     self._book_data = book_data
     self._client = client
     self._backend = backend
     if all((self._book, self._book_data)):
         raise ValueError("Only one of 'book_data' or 'book'"
                          " can be provided")
     self._path = k_paths.normpath(path)
     self._lock_path = path + LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(JOB_PREFIX):])
Example #4
0
def normpath(path, keep_trailing=False):
    """Really normalize the path by adding a missing leading slash."""
    new_path = k_paths.normpath(path)
    if keep_trailing and path.endswith("/") and not new_path.endswith("/"):
        new_path = new_path + "/"
    if not new_path.startswith('/'):
        return '/' + new_path
    return new_path
Example #5
0
 def ensure_path(self, path):
     self.verify()
     path = k_paths.normpath(path)
     path_pieces = partition_path(path)
     for p in path_pieces:
         try:
             self.create(p)
         except k_exceptions.NodeExistsError:
             pass
Example #6
0
 def get(self, path, watch=None):
     self.verify()
     path = k_paths.normpath(path)
     try:
         node = self.storage[path]
     except KeyError:
         raise k_exceptions.NoNodeError("No path %s" % (path))
     if watch:
         self._watches[path].append(watch)
     return (node['data'], self._make_znode(path, node))
Example #7
0
 def __init__(self, board, name, client, path,
              uuid=None, details=None, book=None, book_data=None,
              created_on=None, backend=None):
     super(ZookeeperJob, self).__init__(board, name,
                                        uuid=uuid, details=details,
                                        backend=backend,
                                        book=book, book_data=book_data)
     self._client = client
     self._path = k_paths.normpath(path)
     self._lock_path = self._path + board.LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(board.JOB_PREFIX):])
Example #8
0
 def delete(self, path, recursive=False):
     self.verify()
     path = k_paths.normpath(path)
     with self.storage.lock:
         if path not in self.storage:
             raise k_exceptions.NoNodeError("No path %s" % (path))
         if recursive:
             paths = [path]
             for p in six.iterkeys(self.storage.get_children(path)):
                 paths.append(p)
         else:
             paths = [path]
         for p in reversed(sorted(paths)):
             self.storage.pop(p)
             event = k_states.WatchedEvent(
                 type=k_states.EventType.DELETED,
                 state=k_states.KeeperState.CONNECTED,
                 path=p)
             self._fire_watches([p], event)
Example #9
0
    def create(self, path, value='', ephemeral=False, sequence=False):
        self.verify()
        path = k_paths.normpath(path)
        if sequence:
            raise NotImplementedError("Sequencing not currently supported")
        with self.storage.lock:
            if path in self.storage:
                raise k_exceptions.NodeExistsError("Node %s already there"
                                                   % path)
            parent_path = os.path.split(path)[0]
            if parent_path == path and path in self.storage:
                # This is "/" and it already exists.
                return
            elif parent_path == path:
                # This is "/" and it doesn't already exists.
                pass
            elif parent_path not in self.storage:
                raise k_exceptions.NoNodeError("No parent %s" % (parent_path))
            self.storage[path] = {
                # Kazoo clients expect in milliseconds
                'created_on': _millitime(),
                'updated_on': _millitime(),
                'version': 0,
                # Not supported for now...
                'aversion': -1,
                'cversion': -1,
                'data': value,
            }
            parents = sorted(six.iterkeys(self.storage.get_parents(path)))
        if not parents:
            return

        # Fire any attached watches.
        event = k_states.WatchedEvent(type=k_states.EventType.CREATED,
                                      state=k_states.KeeperState.CONNECTED,
                                      path=path)
        self._fire_watches([parents[-1]], event)
        event = k_states.WatchedEvent(type=k_states.EventType.CHILD,
                                      state=k_states.KeeperState.CONNECTED,
                                      path=path)
        self._fire_watches(parents[0:-1], event)
        return path
Example #10
0
    def get_children(self, path, watch=None, include_data=False):
        self.verify()

        def _clean(p):
            return p.strip("/")

        path = k_paths.normpath(path)
        paths = self.storage.get_children(path)
        if watch:
            self._watches[path].append(watch)
        if include_data:
            children_with_data = []
            for (p, data) in six.iteritems(paths):
                children_with_data.append(_clean(p[len(path):]), data)
            return children_with_data
        else:
            children = []
            for p in list(six.iterkeys(paths)):
                children.append(_clean(p[len(path):]))
            return children
Example #11
0
 def __init__(self, name, board, client, backend, path,
              uuid=None, details=None, book=None, book_data=None,
              created_on=None):
     super(ZookeeperJob, self).__init__(name, uuid=uuid, details=details)
     self._board = board
     self._book = book
     if not book_data:
         book_data = {}
     self._book_data = book_data
     self._client = client
     self._backend = backend
     if all((self._book, self._book_data)):
         raise ValueError("Only one of 'book_data' or 'book'"
                          " can be provided")
     self._path = k_paths.normpath(path)
     self._lock_path = path + LOCK_POSTFIX
     self._created_on = created_on
     self._node_not_found = False
     basename = k_paths.basename(self._path)
     self._root = self._path[0:-len(basename)]
     self._sequence = int(basename[len(JOB_PREFIX):])
Example #12
0
    def set_hosts(self, hosts, randomize_hosts=None):
        """ sets the list of hosts used by this client.

        This function accepts the same format hosts parameter as the init
        function and sets the client to use the new hosts the next time it
        needs to look up a set of hosts. This function does not affect the
        current connected status.

        It is not currently possible to change the chroot with this function,
        setting a host list with a new chroot will raise a ConfigurationError.

        :param hosts: see description in :meth:`KazooClient.__init__`
        :param randomize_hosts: override client default for host randomization
        :raises:
            :exc:`ConfigurationError` if the hosts argument changes the chroot

        .. versionadded:: 1.4

        .. warning::

            Using this function to point a client to a completely disparate
            zookeeper server cluster has undefined behavior.

        """

        if randomize_hosts is None:
            randomize_hosts = self.randomize_hosts

        self.hosts, chroot = collect_hosts(hosts, randomize_hosts)

        if chroot:
            new_chroot = normpath(chroot)
        else:
            new_chroot = ''

        if self.chroot is not None and new_chroot != self.chroot:
            raise ConfigurationError("Changing chroot at runtime is not "
                                     "currently supported")

        self.chroot = new_chroot
Example #13
0
    def set_hosts(self, hosts, randomize_hosts=None):
        """ sets the list of hosts used by this client.

        This function accepts the same format hosts parameter as the init
        function and sets the client to use the new hosts the next time it
        needs to look up a set of hosts. This function does not affect the
        current connected status.

        It is not currently possible to change the chroot with this function,
        setting a host list with a new chroot will raise a ConfigurationError.

        :param hosts: see description in :meth:`KazooClient.__init__`
        :param randomize_hosts: override client default for host randomization
        :raises:
            :exc:`ConfigurationError` if the hosts argument changes the chroot

        .. versionadded:: 1.4

        .. warning::

            Using this function to point a client to a completely disparate
            zookeeper server cluster has undefined behavior.

        """

        if randomize_hosts is None:
            randomize_hosts = self.randomize_hosts

        self.hosts, chroot = collect_hosts(hosts, randomize_hosts)

        if chroot:
            new_chroot = normpath(chroot)
        else:
            new_chroot = ''

        if self.chroot is not None and new_chroot != self.chroot:
            raise ConfigurationError("Changing chroot at runtime is not "
                                     "currently supported")

        self.chroot = new_chroot
Example #14
0
    def __init__(self, hosts='127.0.0.1:2181',
                 timeout=10.0, client_id=None, handler=None,
                 default_acl=None, auth_data=None, read_only=None,
                 randomize_hosts=True, connection_retry=None,
                 command_retry=None, logger=None, **kwargs):
        """Create a :class:`KazooClient` instance. All time arguments
        are in seconds.

        :param hosts: Comma-separated list of hosts to connect to
                      (e.g. 127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
        :param timeout: The longest to wait for a Zookeeper connection.
        :param client_id: A Zookeeper client id, used when
                          re-establishing a prior session connection.
        :param handler: An instance of a class implementing the
                        :class:`~kazoo.interfaces.IHandler` interface
                        for callback handling.
        :param default_acl: A default ACL used on node creation.
        :param auth_data:
            A list of authentication credentials to use for the
            connection. Should be a list of (scheme, credential)
            tuples as :meth:`add_auth` takes.
        :param read_only: Allow connections to read only servers.
        :param randomize_hosts: By default randomize host selection.
        :param connection_retry:
            A :class:`kazoo.retry.KazooRetry` object to use for
            retrying the connection to Zookeeper. Also can be a dict of
            options which will be used for creating one.
        :param command_retry:
            A :class:`kazoo.retry.KazooRetry` object to use for
            the :meth:`KazooClient.retry` method. Also can be a dict of
            options which will be used for creating one.
        :param logger: A custom logger to use instead of the module
            global `log` instance.

        Basic Example:

        .. code-block:: python

            zk = KazooClient()
            zk.start()
            children = zk.get_children('/')
            zk.stop()

        As a convenience all recipe classes are available as attributes
        and get automatically bound to the client. For example::

            zk = KazooClient()
            zk.start()
            lock = zk.Lock('/lock_path')

        .. versionadded:: 0.6
            The read_only option. Requires Zookeeper 3.4+

        .. versionadded:: 0.6
            The retry_max_delay option.

        .. versionadded:: 0.6
            The randomize_hosts option.

        .. versionchanged:: 0.8
            Removed the unused watcher argument (was second argument).

        .. versionadded:: 1.2
            The connection_retry, command_retry and logger options.

        """
        self.logger = logger or log

        # Record the handler strategy used
        self.handler = handler if handler else SequentialThreadingHandler()
        if inspect.isclass(self.handler):
            raise ConfigurationError("Handler must be an instance of a class, "
                                     "not the class: %s" % self.handler)

        self.auth_data = auth_data if auth_data else set([])
        self.default_acl = default_acl
        self.randomize_hosts = randomize_hosts
        self.hosts, chroot = collect_hosts(hosts, randomize_hosts)
        if chroot:
            self.chroot = normpath(chroot)
        else:
            self.chroot = ''

        # Curator like simplified state tracking, and listeners for
        # state transitions
        self._state = KeeperState.CLOSED
        self.state = KazooState.LOST
        self.state_listeners = set()

        self._reset()
        self.read_only = read_only

        if client_id:
            self._session_id = client_id[0]
            self._session_passwd = client_id[1]
        else:
            self._reset_session()

        # ZK uses milliseconds
        self._session_timeout = int(timeout * 1000)

        # We use events like twitter's client to track current and
        # desired state (connected, and whether to shutdown)
        self._live = self.handler.event_object()
        self._writer_stopped = self.handler.event_object()
        self._stopped = self.handler.event_object()
        self._stopped.set()
        self._writer_stopped.set()

        self.retry = self._conn_retry = None

        if connection_retry is not None:
            self._conn_retry = connection_retry
            if self.handler.sleep_func != self._conn_retry.sleep_func:
                raise ConfigurationError("Retry handler and event handler "
                                         " must use the same sleep func")

        if command_retry is not None:
            self.retry = command_retry
            if self.handler.sleep_func != self.retry.sleep_func:
                raise ConfigurationError("Command retry handler and event handler "
                                         " must use the same sleep func")

        if self.retry is None or self._conn_retry is None:
            old_retry_keys = dict(_RETRY_COMPAT_DEFAULTS)
            for key in old_retry_keys:
                try:
                    old_retry_keys[key] = kwargs.pop(key)
                    warnings.warn('Passing retry configuration param %s to the'
                            ' client directly is deprecated, please pass a'
                            ' configured retry object (using param %s)' % (
                                key, _RETRY_COMPAT_MAPPING[key]),
                            DeprecationWarning, stacklevel=2)
                except KeyError:
                    pass

            retry_keys = {}
            for oldname, value in old_retry_keys.items():
                retry_keys[_RETRY_COMPAT_MAPPING[oldname]] = value

            if self._conn_retry is None:
                self._conn_retry = KazooRetry(
                    sleep_func=self.handler.sleep_func,
                    **retry_keys)
            if self.retry is None:
                self.retry = KazooRetry(
                    sleep_func=self.handler.sleep_func,
                    **retry_keys)

        self._conn_retry.interrupt = lambda: self._stopped.is_set()
        self._connection = ConnectionHandler(self, self._conn_retry.copy(),
            logger=self.logger)

        self.Barrier = partial(Barrier, self)
        self.Counter = partial(Counter, self)
        self.DoubleBarrier = partial(DoubleBarrier, self)
        self.ChildrenWatch = partial(ChildrenWatch, self)
        self.DataWatch = partial(DataWatch, self)
        self.Election = partial(Election, self)
        self.Lock = partial(Lock, self)
        self.Party = partial(Party, self)
        self.Queue = partial(Queue, self)
        self.LockingQueue = partial(LockingQueue, self)
        self.SetPartitioner = partial(SetPartitioner, self)
        self.Semaphore = partial(Semaphore, self)
        self.ShallowParty = partial(ShallowParty, self)

         # If we got any unhandled keywords, complain like python would
        if kwargs:
            raise TypeError('__init__() got unexpected keyword arguments: %s'
                            % (kwargs.keys(),))
Example #15
0
    def __init__(self, hosts='127.0.0.1:2181',
                 timeout=10.0, client_id=None, max_retries=None,
                 retry_delay=0.1, retry_backoff=2, retry_jitter=0.8,
                 retry_max_delay=3600, handler=None, default_acl=None,
                 auth_data=None, read_only=None, randomize_hosts=True):
        """Create a :class:`KazooClient` instance. All time arguments
        are in seconds.

        :param hosts: Comma-separated list of hosts to connect to
                      (e.g. 127.0.0.1:2181,127.0.0.1:2182).
        :param timeout: The longest to wait for a Zookeeper connection.
        :param client_id: A Zookeeper client id, used when
                          re-establishing a prior session connection.
        :param max_retries: Maximum retries when using the
                            :meth:`KazooClient.retry` method.
        :param retry_delay: Initial delay when retrying a call.
        :param retry_backoff:
            Backoff multiplier between retry attempts. Defaults to 2
            for exponential back-off.
        :param retry_jitter:
            How much jitter delay to introduce per call. An amount of
            time up to this will be added per retry call to avoid
            hammering the server.
        :param retry_max_delay:
            Maximum delay in seconds, regardless of other backoff
            settings. Defaults to one hour.
        :param handler: An instance of a class implementing the
                        :class:`~kazoo.interfaces.IHandler` interface
                        for callback handling.
        :param default_acl: A default ACL used on node creation.
        :param auth_data:
            A list of authentication credentials to use for the
            connection. Should be a list of (scheme, credential)
            tuples as :meth:`add_auth` takes.
        :param read_only: Allow connections to read only servers.
        :param randomize_hosts: By default randomize host selection.

        Retry parameters will be used for connection establishment
        attempts and reconnects.

        Basic Example:

        .. code-block:: python

            zk = KazooClient()
            zk.start()
            children = zk.get_children('/')
            zk.stop()

        As a convenience all recipe classes are available as attributes
        and get automatically bound to the client. For example::

            zk = KazooClient()
            zk.start()
            lock = zk.Lock('/lock_path')

        .. versionadded:: 0.6
            The read_only option. Requires Zookeeper 3.4+

        .. versionadded:: 0.6
            The retry_max_delay option.

        .. versionadded:: 0.6
            The randomize_hosts option.

        .. versionchanged:: 0.8
            Removed the unused watcher argument (was second argument).

        """
        self.log_debug = logging.DEBUG >= log.getEffectiveLevel()

        # Record the handler strategy used
        self.handler = handler if handler else SequentialThreadingHandler()
        if inspect.isclass(self.handler):
            raise ConfigurationError("Handler must be an instance of a class, "
                                     "not the class: %s" % self.handler)

        self.auth_data = auth_data if auth_data else set([])
        self.default_acl = default_acl
        self.randomize_hosts = randomize_hosts
        self.hosts, chroot = collect_hosts(hosts, randomize_hosts)
        if chroot:
            self.chroot = normpath(chroot)
        else:
            self.chroot = ''

        # Curator like simplified state tracking, and listeners for
        # state transitions
        self._state = KeeperState.CLOSED
        self.state = KazooState.LOST
        self.state_listeners = set()

        self._reset()
        self.read_only = read_only

        if client_id:
            self._session_id = client_id[0]
            self._session_passwd = client_id[1]
        else:
            self._reset_session()

        # ZK uses milliseconds
        self._session_timeout = int(timeout * 1000)

        # We use events like twitter's client to track current and
        # desired state (connected, and whether to shutdown)
        self._live = self.handler.event_object()
        self._writer_stopped = self.handler.event_object()
        self._stopped = self.handler.event_object()
        self._stopped.set()
        self._writer_stopped.set()

        self.retry = KazooRetry(
            max_tries=max_retries,
            delay=retry_delay,
            backoff=retry_backoff,
            max_jitter=retry_jitter,
            max_delay=retry_max_delay,
            sleep_func=self.handler.sleep_func
        )
        self.retry_sleeper = self.retry.retry_sleeper.copy()

        self._connection = ConnectionHandler(
            self, self.retry.retry_sleeper.copy(), log_debug=self.log_debug)

        # convenience API
        from kazoo.recipe.barrier import Barrier
        from kazoo.recipe.barrier import DoubleBarrier
        from kazoo.recipe.counter import Counter
        from kazoo.recipe.election import Election
        from kazoo.recipe.lock import Lock
        from kazoo.recipe.lock import Semaphore
        from kazoo.recipe.partitioner import SetPartitioner
        from kazoo.recipe.party import Party
        from kazoo.recipe.party import ShallowParty
        from kazoo.recipe.queue import Queue
        from kazoo.recipe.queue import LockingQueue
        from kazoo.recipe.watchers import ChildrenWatch
        from kazoo.recipe.watchers import DataWatch

        self.Barrier = partial(Barrier, self)
        self.Counter = partial(Counter, self)
        self.DoubleBarrier = partial(DoubleBarrier, self)
        self.ChildrenWatch = partial(ChildrenWatch, self)
        self.DataWatch = partial(DataWatch, self)
        self.Election = partial(Election, self)
        self.Lock = partial(Lock, self)
        self.Party = partial(Party, self)
        self.Queue = partial(Queue, self)
        self.LockingQueue = partial(LockingQueue, self)
        self.SetPartitioner = partial(SetPartitioner, self)
        self.Semaphore = partial(Semaphore, self)
        self.ShallowParty = partial(ShallowParty, self)
Example #16
0
 def test_normpath_relative(self):
     with pytest.raises(ValueError):
         paths.normpath('./a/b')
     with pytest.raises(ValueError):
         paths.normpath('/a/../b')
Example #17
0
 def test_normpath_trailing(self):
     self.assertEqual(paths.normpath('/', trailing=True), '/')
Example #18
0
 def test_normpath(self):
     self.assertEqual(paths.normpath('/a/b'), '/a/b')
Example #19
0
 def test_normpath_slash(self):
     self.assertEqual(paths.normpath('/'), '/')
Example #20
0
 def test_normpath_unicode(self):
     self.assertEqual(paths.normpath(u'/\xe4/b'), u'/\xe4/b')
Example #21
0
 def test_normpath_empty(self):
     assert paths.normpath('') == ''
Example #22
0
    def __init__(self, hosts='127.0.0.1:2181', watcher=None,
                 timeout=10.0, client_id=None, max_retries=None, retry_delay=0.1,
                 retry_backoff=2, retry_jitter=0.8, handler=None,
                 default_acl=None, auth_data=None, read_only=None):
        """Create a :class:`KazooClient` instance. All time arguments
        are in seconds.

        :param hosts: Comma-separated list of hosts to connect to
                      (e.g. 127.0.0.1:2181,127.0.0.1:2182).
        :param watcher:
            Set a default watcher. This will be called by the actual
            default watcher that :class:`KazooClient` establishes.
        :param timeout: The longest to wait for a Zookeeper connection.
        :param client_id: A Zookeeper client id, used when
                          re-establishing a prior session connection.
        :param max_retries: Maximum retries when using the
                            :meth:`KazooClient.retry` method.
        :param retry_delay: Initial delay when retrying a call.
        :param retry_backoff:
            Backoff multiplier between retry attempts. Defaults to 2
            for exponential back-off.
        :param retry_jitter:
            How much jitter delay to introduce per call. An amount of
            time up to this will be added per retry call to avoid
            hammering the server.
        :param handler: An instance of a class implementing the
                        :class:`~kazoo.interfaces.IHandler` interface
                        for callback handling.
        :param default_acl: A default ACL used on node creation.
        :param auth_data:
            A list of authentication credentials to use for the
            connection. Should be a list of (scheme, credential)
            tuples as :meth:`add_auth` takes.

        Retry parameters will be used for connection establishment
        attempts and reconnects.


        Basic Example:

        .. code-block:: python

            zk = KazooClient()
            zk.start()
            children = zk.get_children('/')
            zk.stop()

        As a convenience all recipe classes are available as attributes
        and get automatically bound to the client. For example::

            zk = KazooClient()
            zk.start()
            lock = zk.Lock('/lock_path')

        """
        self.log_debug = logging.DEBUG >= log.getEffectiveLevel()

        # Record the handler strategy used
        self.handler = handler if handler else SequentialThreadingHandler()
        if inspect.isclass(self.handler):
            raise ConfigurationError("Handler must be an instance of a class, "
                                     "not the class: %s" % self.handler)

        self.auth_data = auth_data if auth_data else set([])
        self.default_acl = default_acl
        self.hosts, chroot = collect_hosts(hosts)
        if chroot:
            self.chroot = normpath(chroot)
        else:
            self.chroot = ''

        # Curator like simplified state tracking, and listeners for
        # state transitions
        self._state_lock = self.handler.rlock_object()
        self._state = KeeperState.CLOSED
        self.state = KazooState.LOST
        self.state_listeners = set()

        self._reset()
        self.read_only = read_only

        if client_id:
            self._session_id = client_id[0]
            self._session_passwd = client_id[1]
        else:
            self._session_id = None
            self._session_passwd = str(bytearray([0] * 16))

        # ZK uses milliseconds
        self._session_timeout = int(timeout * 1000)

        # We use events like twitter's client to track current and
        # desired state (connected, and whether to shutdown)
        self._live = self.handler.event_object()
        self._writer_stopped = self.handler.event_object()
        self._stopped = self.handler.event_object()
        self._stopped.set()
        self._writer_stopped.set()

        self.retry = KazooRetry(
            max_tries=max_retries,
            delay=retry_delay,
            backoff=retry_backoff,
            max_jitter=retry_jitter,
            sleep_func=self.handler.sleep_func
        )
        self.retry_sleeper = self.retry.retry_sleeper.copy()

        self._connection = ConnectionHandler(
            self, self.retry.retry_sleeper.copy(), log_debug=self.log_debug)

        # convenience API
        from kazoo.recipe.barrier import Barrier
        from kazoo.recipe.barrier import DoubleBarrier
        from kazoo.recipe.election import Election
        from kazoo.recipe.lock import Lock
        from kazoo.recipe.partitioner import SetPartitioner
        from kazoo.recipe.party import Party
        from kazoo.recipe.party import ShallowParty
        from kazoo.recipe.watchers import ChildrenWatch
        from kazoo.recipe.watchers import DataWatch

        self.Barrier = partial(Barrier, self)
        self.DoubleBarrier = partial(DoubleBarrier, self)
        self.ChildrenWatch = partial(ChildrenWatch, self)
        self.DataWatch = partial(DataWatch, self)
        self.Election = partial(Election, self)
        self.Lock = partial(Lock, self)
        self.Party = partial(Party, self)
        self.SetPartitioner = partial(SetPartitioner, self)
        self.ShallowParty = partial(ShallowParty, self)
Example #23
0
 def test_normpath_trailing(self):
     self.assertEqual(paths.normpath('/', trailing=True), '/')
Example #24
0
 def test_normpath_unicode(self):
     assert paths.normpath(u('/\xe4/b')) == u('/\xe4/b')
Example #25
0
 def test_normpath_dots(self):
     assert paths.normpath('/a./b../c') == '/a./b../c'
Example #26
0
 def test_normpath_slash(self):
     assert paths.normpath('/') == '/'
Example #27
0
 def test_normpath_multiple_slashes(self):
     assert paths.normpath('//') == '/'
     assert paths.normpath('//a/b') == '/a/b'
     assert paths.normpath('/a//b//') == '/a/b'
     assert paths.normpath('//a////b///c/') == '/a/b/c'
Example #28
0
    def __init__(self, hosts='127.0.0.1:2181',
                 timeout=10.0, client_id=None, handler=None,
                 default_acl=None, auth_data=None, read_only=None,
                 randomize_hosts=True, connection_retry=None,
                 command_retry=None, logger=None, **kwargs):
        """Create a :class:`KazooClient` instance. All time arguments
        are in seconds.

        :param hosts: Comma-separated list of hosts to connect to
                      (e.g. 127.0.0.1:2181,127.0.0.1:2182,[::1]:2183).
        :param timeout: The longest to wait for a Zookeeper connection.
        :param client_id: A Zookeeper client id, used when
                          re-establishing a prior session connection.
        :param handler: An instance of a class implementing the
                        :class:`~kazoo.interfaces.IHandler` interface
                        for callback handling.
        :param default_acl: A default ACL used on node creation.
        :param auth_data:
            A list of authentication credentials to use for the
            connection. Should be a list of (scheme, credential)
            tuples as :meth:`add_auth` takes.
        :param read_only: Allow connections to read only servers.
        :param randomize_hosts: By default randomize host selection.
        :param connection_retry:
            A :class:`kazoo.retry.KazooRetry` object to use for
            retrying the connection to Zookeeper. Also can be a dict of
            options which will be used for creating one.
        :param command_retry:
            A :class:`kazoo.retry.KazooRetry` object to use for
            the :meth:`KazooClient.retry` method. Also can be a dict of
            options which will be used for creating one.
        :param logger: A custom logger to use instead of the module
            global `log` instance.

        Basic Example:

        .. code-block:: python

            zk = KazooClient()
            zk.start()
            children = zk.get_children('/')
            zk.stop()

        As a convenience all recipe classes are available as attributes
        and get automatically bound to the client. For example::

            zk = KazooClient()
            zk.start()
            lock = zk.Lock('/lock_path')

        .. versionadded:: 0.6
            The read_only option. Requires Zookeeper 3.4+

        .. versionadded:: 0.6
            The retry_max_delay option.

        .. versionadded:: 0.6
            The randomize_hosts option.

        .. versionchanged:: 0.8
            Removed the unused watcher argument (was second argument).

        .. versionadded:: 1.2
            The connection_retry, command_retry and logger options.

        """
        self.logger = logger or log

        # Record the handler strategy used
        self.handler = handler if handler else SequentialThreadingHandler()
        if inspect.isclass(self.handler):
            raise ConfigurationError("Handler must be an instance of a class, "
                                     "not the class: %s" % self.handler)

        self.auth_data = auth_data if auth_data else set([])
        self.default_acl = default_acl
        self.randomize_hosts = randomize_hosts
        self.hosts, chroot = collect_hosts(hosts, randomize_hosts)
        if chroot:
            self.chroot = normpath(chroot)
        else:
            self.chroot = ''

        # Curator like simplified state tracking, and listeners for
        # state transitions
        self._state = KeeperState.CLOSED
        self.state = KazooState.LOST
        self.state_listeners = set()

        self._reset()
        self.read_only = read_only

        if client_id:
            self._session_id = client_id[0]
            self._session_passwd = client_id[1]
        else:
            self._reset_session()

        # ZK uses milliseconds
        self._session_timeout = int(timeout * 1000)

        # We use events like twitter's client to track current and
        # desired state (connected, and whether to shutdown)
        self._live = self.handler.event_object()
        self._writer_stopped = self.handler.event_object()
        self._stopped = self.handler.event_object()
        self._stopped.set()
        self._writer_stopped.set()

        self.retry = self._conn_retry = None

        if connection_retry is not None:
            self._conn_retry = connection_retry
            if self.handler.sleep_func != self._conn_retry.sleep_func:
                raise ConfigurationError("Retry handler and event handler "
                                         " must use the same sleep func")

        if command_retry is not None:
            self.retry = command_retry
            if self.handler.sleep_func != self.comand_retry.sleep_func:
                raise ConfigurationError("Command retry handler and event handler "
                                         " must use the same sleep func")

        if self.retry is None or self._conn_retry is None:
            old_retry_keys = dict(_RETRY_COMPAT_DEFAULTS)
            for key in old_retry_keys:
                try:
                    old_retry_keys[key] = kwargs.pop(key)
                    warnings.warn('Passing retry configuration param %s to the'
                            ' client directly is deprecated, please pass a'
                            ' configured retry object (using param %s)' % (
                                key, _RETRY_COMPAT_MAPPING[key]),
                            DeprecationWarning, stacklevel=2)
                except KeyError:
                    pass

            retry_keys = {}
            for oldname, value in old_retry_keys.items():
                retry_keys[_RETRY_COMPAT_MAPPING[oldname]] = value

            if self._conn_retry is None:
                self._conn_retry = KazooRetry(
                    sleep_func=self.handler.sleep_func,
                    **retry_keys)
            if self.retry is None:
                self.retry = KazooRetry(
                    sleep_func=self.handler.sleep_func,
                    **retry_keys)

        self._conn_retry.interrupt = lambda: self._stopped.is_set()
        self._connection = ConnectionHandler(self, self._conn_retry.copy(),
            logger=self.logger)

        self.Barrier = partial(Barrier, self)
        self.Counter = partial(Counter, self)
        self.DoubleBarrier = partial(DoubleBarrier, self)
        self.ChildrenWatch = partial(ChildrenWatch, self)
        self.DataWatch = partial(DataWatch, self)
        self.Election = partial(Election, self)
        self.Lock = partial(Lock, self)
        self.Party = partial(Party, self)
        self.Queue = partial(Queue, self)
        self.LockingQueue = partial(LockingQueue, self)
        self.SetPartitioner = partial(SetPartitioner, self)
        self.Semaphore = partial(Semaphore, self)
        self.ShallowParty = partial(ShallowParty, self)

         # If we got any unhandled keywords, complain like python would
        if kwargs:
            raise TypeError('__init__() got unexpected keyword arguments: %s'
                            % (kwargs.keys(),))
Example #29
0
 def test_normpath_empty(self):
     self.assertEqual(paths.normpath(''), '')
Example #30
0
 def test_normpath_empty(self):
     self.assertEqual(paths.normpath(''), '')
Example #31
0
 def test_normpath_dots(self):
     self.assertEqual(paths.normpath('/a./b../c'), '/a./b../c')
Example #32
0
 def test_normpath_unicode(self):
     self.assertEqual(paths.normpath(u('/\xe4/b')), u('/\xe4/b'))
Example #33
0
 def test_normpath_multiple_slashes(self):
     self.assertEqual(paths.normpath('//'), '/')
     self.assertEqual(paths.normpath('//a/b'), '/a/b')
     self.assertEqual(paths.normpath('/a//b//'), '/a/b')
     self.assertEqual(paths.normpath('//a////b///c/'), '/a/b/c')
Example #34
0
 def test_normpath_dots(self):
     self.assertEqual(paths.normpath('/a./b../c'), '/a./b../c')
Example #35
0
 def test_normpath(self):
     assert paths.normpath('/a/b') == '/a/b'
Example #36
0
 def test_normpath_slash(self):
     self.assertEqual(paths.normpath('/'), '/')
Example #37
0
 def test_normpath_multiple_slashes(self):
     self.assertEqual(paths.normpath('//'), '/')
     self.assertEqual(paths.normpath('//a/b'), '/a/b')
     self.assertEqual(paths.normpath('/a//b//'), '/a/b')
     self.assertEqual(paths.normpath('//a////b///c/'), '/a/b/c')
Example #38
0
 def test_normpath_trailing(self):
     assert paths.normpath('/', trailing=True) == '/'
Example #39
0
 def test_normpath(self):
     self.assertEqual(paths.normpath('/a/b'), '/a/b')
Example #40
0
def normpath(path):
    """Really normalize the path by adding a missing leading slash."""
    path = k_paths.normpath(path)
    if not path.startswith('/'):
        return '/' + path
    return path