Esempio n. 1
0
    def __init__(
        self,
        hosts,
        session_id=None,
        session_passwd=None,
        session_timeout=30.0,
        auth_data=None,
        watcher=None,
        allow_reconnect=True,
    ):
        self.hosts, chroot = collect_hosts(hosts)
        if chroot:
            self.chroot = zkpath.normpath(chroot)
            if not zkpath.isabs(self.chroot):
                raise ValueError("chroot not absolute")
        else:
            self.chroot = ""

        self.session_id = session_id
        self.session_passwd = session_passwd if session_passwd else str(bytearray([0] * 16))
        self.session_timeout = session_timeout
        self.connect_timeout = session_timeout / len(self.hosts)
        self.read_timeout = session_timeout * 2.0 / 3.0
        self.auth_data = auth_data if auth_data else set([])
        self.read_only = False
        LOGGER.debug("session_id: %s", self.session_id)
        LOGGER.debug("session_passwd: 0x%s", self.session_passwd.encode("hex"))
        LOGGER.debug("session_timeout: %s", self.session_timeout)
        LOGGER.debug("connect_timeout: %s", self.connect_timeout)
        LOGGER.debug("   len(hosts): %s", len(self.hosts))
        LOGGER.debug("read_timeout: %s", self.read_timeout)
        LOGGER.debug("auth_data: %s", self.auth_data)

        self.allow_reconnect = allow_reconnect
        LOGGER.debug("allow_reconnect: %s", self.allow_reconnect)

        self.last_zxid = 0

        self._queue = PeekableQueue()
        self._pending = Queue()

        self._events = Queue()
        self._child_watchers = defaultdict(set)
        self._data_watchers = defaultdict(set)
        self._exists_watchers = defaultdict(set)
        self._default_watcher = watcher or Watcher()

        self.state = CONNECTING
        self._state_lock = threading.RLock()

        self._event_thread_completed = threading.Event()

        def event_worker():
            try:
                while True:
                    notification = self._events.get()

                    if notification == self:
                        break

                    try:
                        notification()
                    except Exception:
                        LOGGER.exception("Unforeseen error during notification")
            finally:
                LOGGER.debug("Event loop completed")
                self._event_thread_completed.set()

        self._event_thread = threading.Thread(target=event_worker, name="event-%s" % self.id)
        self._event_thread.daemon = True
        self._event_thread.start()

        self._writer_thread = WriterThread(self)
        self._writer_thread.setDaemon(True)
        self._writer_thread.start()

        self._check_state()
Esempio n. 2
0
class Client33(object):
    @log_wrapper()
    def __init__(
        self,
        hosts,
        session_id=None,
        session_passwd=None,
        session_timeout=30.0,
        auth_data=None,
        watcher=None,
        allow_reconnect=True,
    ):
        self.hosts, chroot = collect_hosts(hosts)
        if chroot:
            self.chroot = zkpath.normpath(chroot)
            if not zkpath.isabs(self.chroot):
                raise ValueError("chroot not absolute")
        else:
            self.chroot = ""

        self.session_id = session_id
        self.session_passwd = session_passwd if session_passwd else str(bytearray([0] * 16))
        self.session_timeout = session_timeout
        self.connect_timeout = session_timeout / len(self.hosts)
        self.read_timeout = session_timeout * 2.0 / 3.0
        self.auth_data = auth_data if auth_data else set([])
        self.read_only = False
        LOGGER.debug("session_id: %s", self.session_id)
        LOGGER.debug("session_passwd: 0x%s", self.session_passwd.encode("hex"))
        LOGGER.debug("session_timeout: %s", self.session_timeout)
        LOGGER.debug("connect_timeout: %s", self.connect_timeout)
        LOGGER.debug("   len(hosts): %s", len(self.hosts))
        LOGGER.debug("read_timeout: %s", self.read_timeout)
        LOGGER.debug("auth_data: %s", self.auth_data)

        self.allow_reconnect = allow_reconnect
        LOGGER.debug("allow_reconnect: %s", self.allow_reconnect)

        self.last_zxid = 0

        self._queue = PeekableQueue()
        self._pending = Queue()

        self._events = Queue()
        self._child_watchers = defaultdict(set)
        self._data_watchers = defaultdict(set)
        self._exists_watchers = defaultdict(set)
        self._default_watcher = watcher or Watcher()

        self.state = CONNECTING
        self._state_lock = threading.RLock()

        self._event_thread_completed = threading.Event()

        def event_worker():
            try:
                while True:
                    notification = self._events.get()

                    if notification == self:
                        break

                    try:
                        notification()
                    except Exception:
                        LOGGER.exception("Unforeseen error during notification")
            finally:
                LOGGER.debug("Event loop completed")
                self._event_thread_completed.set()

        self._event_thread = threading.Thread(target=event_worker, name="event-%s" % self.id)
        self._event_thread.daemon = True
        self._event_thread.start()

        self._writer_thread = WriterThread(self)
        self._writer_thread.setDaemon(True)
        self._writer_thread.start()

        self._check_state()

    @log_wrapper()
    def close(self):
        """ Close this client object

        Once the client is closed, its session becomes invalid. All the
        ephemeral nodes in the ZooKeeper server associated with the session
        will be removed. The watches left on those nodes (and on their parents)
        will be triggered.
        """

        LOGGER.debug("close()")

        with self._state_lock:
            if self.state == AUTH_FAILED:
                return
            if self.state == CLOSED:
                return

            call_exception = None
            event = threading.Event()

            def close(exception):
                global call_exception
                call_exception = exception
                event.set()
                LOGGER.debug("Closing handler called")

            self._queue.put((CloseRequest(), CloseResponse(), close))

        event.wait()

        self._event_thread_completed.wait()

        self.session_id = None
        self.session_passwd = str(bytearray([0] * 16))

        if call_exception:
            raise call_exception

    @log_wrapper()
    def create(self, path, acls, code, data=None):
        """ Create a node with the given path

        The node data will be the given data, and node acl will be the given
        acl.

        The code argument specifies whether the created node will be ephemeral
        or not.

        An ephemeral node will be removed by the ZooKeeper automatically when the
        session associated with the creation of the node expires.

        The code argument can also specify to create a sequential node. The
        actual path name of a sequential node will be the given path plus a
        suffix "i" where i is the current sequential number of the node. The sequence
        number is always fixed length of 10 digits, 0 padded. Once
        such a node is created, the sequential number will be incremented by one.

        If a node with the same actual path already exists in the ZooKeeper, a
        NodeExistsError will be raised. Note that since a different actual path
        is used for each invocation of creating sequential node with the same
        path argument, the call will never raise NodeExistsError.

        If the parent node does not exist in the ZooKeeper, a NoNodeError will
        be raised.

        An ephemeral node cannot have children. If the parent node of the given
        path is ephemeral, a NoChildrenForEphemeralsError will be raised.

        This operation, if successful, will trigger all the watches left on the
        node of the given path by exists and get_data() API calls, and the watches
        left on the parent node by get_children() API calls.

        If a node is created successfully, the ZooKeeper server will trigger the
        watches on the path left by exists calls, and the watches on the parent
        of the node by getChildren calls.

        The maximum allowable size of the data array is 1 MB (1,048,576 bytes).
        Arrays larger than this will cause a ZookeeperError to be raised.

        Args:
            path: the path for the node
            acl: the acl for the node
            code: specifying whether the node to be created is ephemeral
                and/or sequential
            data: optional initial data for the node

        Returns:
            the actual path of the created node

        Raises:
            ZookeeperError: if the server returns a non-zero error code
            InvalidACLError: if the ACL is invalid, null, or empty
            ValueError: if an invalid path is specified

        """

        LOGGER.debug("create(%r, %r, %r, %r)", path, acls, code, data)

        if not acls:
            raise InvalidACLError("ACLs cannot be None or empty")
        if not code:
            raise ValueError("Creation code cannot be None")

        request = CreateRequest(_prefix_root(self.chroot, path), data, acls, code.flags)
        response = CreateResponse(None)

        self._call(request, response)

        return response.path[len(self.chroot) :]

    @log_wrapper()
    def delete(self, path, version=-1):
        """ Delete the node with the given path

        The call will succeed if such a node exists, and the given version
        matches the node's version (if the given version is -1, the default,
        it matches any node's versions).

        A NodeExistsError will be raised if the nodes does not exist.

        A BadVersionError will be raised if the given version does not match
        the node's version.

        A NotEmptyError will be raised if the node has children.

        This operation, if successful, will trigger all the watches on the node
        of the given path left by exists API calls, and the watches on the parent
        node left by get_children() API calls.

        Args:
            path: the path of the node to be deleted.
            version: the expected node version

        Raises:
            ZookeeperError: if the server returns a non-zero error code

        """

        LOGGER.debug("delete(%r, %r)", path, version)

        request = DeleteRequest(_prefix_root(self.chroot, path), version)

        self._call(request, None)

    @log_wrapper()
    def exists(self, path, watch=False, watcher=None):
        """ Return the stat of the node of the given path

        Return null if no such a node exists.

        If the watcher is non-null and the call is successful (no error is raised),
        a watcher will be left on the node with the given path. The watcher will be
        triggered by a successful operation that creates/delete the node or sets
        the data on the node.

        Args:
            path: the node path
            watch: designate the default watcher associated with this connection
                to be the watcher
            watcher: explicit watcher

        Returns:
            The stat of the node of the given path; return null if no such a
            node exists.

        Raises:
            ZookeeperError: if the server returns a non-zero error code

        """

        LOGGER.debug("exists(%r, %r, %r)", path, watch, watcher)

        if watch and watcher:
            LOGGER.warn("Both watch and watcher were specified, registering watcher")

        request = ExistsRequest(_prefix_root(self.chroot, path), watch or watcher is not None)
        response = ExistsResponse(None)

        def register_watcher(exception):
            if not exception:
                with self._state_lock:
                    self._data_watchers[_prefix_root(self.chroot, path)].add(watcher or self._default_watcher)
            elif exception == NoNodeError:
                with self._state_lock:
                    self._exists_watchers[_prefix_root(self.chroot, path)].add(watcher or self._default_watcher)

        try:
            self._call(request, response, register_watcher if (watch or watcher) else lambda e: True)

            return response.stat if response.stat.czxid != -1 else None
        except NoNodeError:
            register_watcher(NoNodeError)
            return None

    @log_wrapper()
    def get_data(self, path, watch=False, watcher=None):
        """ Return the data and the stat of the node of the given path

        If the watch is non-null and the call is successful (no error is
        raised), a watch will be left on the node with the given path. The watch
        will be triggered by a successful operation that sets data on the node, or
        deletes the node.

        NoNodeError will be raised if no node with the given path exists.

        Args:
            path: the given path
            watch: designate the default watcher associated with this connection
                to be the watcher
            watcher: explicit watcher

        Returns:
            The data of the node

        Raises:
            ZookeeperError: if the server returns a non-zero error code


        """

        LOGGER.debug("get_data(%r, %r, %r)", path, watch, watcher)

        if watch and watcher:
            LOGGER.warn("Both watch and watcher were specified, registering watcher")

        request = GetDataRequest(_prefix_root(self.chroot, path), watch or watcher is not None)
        response = GetDataResponse(None, None)

        def register_watcher(exception):
            if not exception:
                with self._state_lock:
                    self._data_watchers[_prefix_root(self.chroot, path)].add(watcher or self._default_watcher)

        self._call(request, response, register_watcher if (watch or watcher) else lambda e: True)

        return response.data, response.stat

    @log_wrapper()
    def set_data(self, path, data, version=-1):
        """ Set the data for the node of the given path

        Set the data for the node of the given path if such a node exists and the
        given version matches the version of the node (if the given version is
        -1, the default, it matches any node's versions). Return the stat of the node.

        This operation, if successful, will trigger all the watches on the node
        of the given path left by get_data() calls.

        NoNodeError will be raised if no node with the given path exists.

        BadVersionError will be raised if the given version does not match the node's version.

        The maximum allowable size of the data array is 1 MB (1,048,576 bytes).
        Arrays larger than this will cause a ZookeeperError to be thrown.

        Args:
            path: the path of the node
            data: the data to set
            version: the expected matching version

        Returns:
            The state of the node

        Raises:
            ZookeeperError: if the server returns a non-zero error code

        """

        LOGGER.debug("set_data(%r, %r, %r)", path, data, version)

        request = SetDataRequest(_prefix_root(self.chroot, path), data, version)
        response = SetDataResponse(None)

        self._call(request, response)

        return response.stat

    @log_wrapper()
    def get_acls(self, path):
        """ Return the ACL and stat of the node of the given path

        NoNodeError will be raised if no node with the given path exists.

        Args:
            path: the given path for the node

        Returns:
            The ACL array of the given node

        Raises:
            ZookeeperError: if the server returns a non-zero error code

        """

        LOGGER.debug("get_acls(%r)", path)

        request = GetACLRequest(_prefix_root(self.chroot, path))
        response = GetACLResponse(None, None)

        self._call(request, response)

        return response.acl, response.stat

    @log_wrapper()
    def set_acls(self, path, acls, version=-1):
        """ Set the ACL for the node of the given path

        Set the ACL for the node of the given path if such a node exists and the
        given version matches the version of the node. Return the stat of the
        node.

        NoNodeError will be raised if no node with the given path exists.

        BadVersionError will be raised if the given version does not match the node's version.

        Args:
            path: the given path for the node
            acl: the ACLs to set
            version: the expected matching version

        Returns:
            The stat of the node.

        Raises:
            ZookeeperError: if the server returns a non-zero error code
            InvalidACLError: if the acl is invalid

        """

        LOGGER.debug("set_acls(%r, %r, %r)", path, acls, version)

        request = SetACLRequest(_prefix_root(self.chroot, path), acls, version)
        response = SetACLResponse(None)

        self._call(request, response)

        return response.stat

    @log_wrapper()
    def sync(self, path):
        """ Asynchronous sync

        Flushes channel between process and leader.

        Args:
            path: the given path for the node

        Raises:
            ZookeeperError: if the server returns a non-zero error code

        """

        LOGGER.debug("sync(%r)", path)

        request = SyncRequest(_prefix_root(self.chroot, path))
        response = SyncResponse(None)

        self._call(request, response)

    @log_wrapper()
    def get_children(self, path, watch=False, watcher=None):
        """ Return the list of the children of the node of the given path

        If the watch is non-null and the call is successful (no error is raised),
        a watch will be left on the node with the given path. The watch will be
        triggered by a successful operation that deletes the node of the given
        path or creates/delete a child under the node.

        The list of children returned is not sorted and no guarantee is provided
        as to its natural or lexical order.

        NoNodeError will be raised if no node with the given path exists.

        Args:
            path: the given path
            watch: designate the default watcher associated with this connection
                to be the watcher
            watcher: explicit watcher

        Returns:
            An unordered array of children of the node with the given path

        Raises:
            ZookeeperError: if the server returns a non-zero error code

        """

        LOGGER.debug("get_children(%r, %r, %r)", path, watch, watcher)

        if watch and watcher:
            LOGGER.warn("Both watch and watcher were specified, registering watcher")

        request = GetChildren2Request(_prefix_root(self.chroot, path), watch or watcher is not None)
        response = GetChildren2Response(None, None)

        def register_watcher(exception):
            if not exception:
                with self._state_lock:
                    self._child_watchers[_prefix_root(self.chroot, path)].add(watcher or self._default_watcher)

        self._call(request, response, register_watcher if (watch or watcher) else lambda e: True)

        return response.children, response.stat

    def _call(self, request, response, register_watcher=None):
        with self._state_lock:
            self._check_state()

            call_exception = [None]
            event = threading.Event()

            def callback(exception):
                if exception:
                    call_exception[0] = exception
                if register_watcher:
                    register_watcher(exception)

                event.set()

            self._queue.put((request, response, callback))

        event.wait()
        if call_exception[0]:
            raise call_exception[0]

    def _allocate_socket(self):
        """ Used to allow the replacement of a socket with a mock socket
        """
        return socket.socket()

    def _check_state(self):
        with self._state_lock:
            if self.state == AUTH_FAILED:
                raise AuthFailedError()
            if self.state == CLOSED:
                raise SessionExpiredError()
            if self.state == CONNECTION_DROPPED_FOR_TEST:
                raise ConnectionDroppedForTest()

    def _connected(self, session_id, session_passwd, read_only):
        with self._state_lock:
            LOGGER.debug("Connected %s", "read-only mode" if read_only else "")

            self.state = CONNECTED_RO if read_only else CONNECTED
            self._events.put(lambda: self._default_watcher.session_connected(session_id, session_passwd, read_only))

    def _disconnected(self):
        assert self.state in set([CONNECTING, CONNECTED, CONNECTED_RO, CONNECTION_DROPPED_FOR_TEST])
        with self._state_lock:
            if self.state in set([CONNECTING, CONNECTION_DROPPED_FOR_TEST]):
                return

            LOGGER.debug("Disconnected %s %s pending calls", self.state, self._pending.qsize())
            LOGGER.debug("        %s %s queued calls", " " * len(str(self.state)), self._queue.qsize())

            self.state = CONNECTING

            self._events.put(lambda: self._default_watcher.connection_dropped())

            # drain queues
            self._drain(ConnectionLoss())

    def _closed(self, state, session_expired=False):
        """ The party is over.  Time to clean up
        """
        assert state in set([CLOSED, AUTH_FAILED, CONNECTION_DROPPED_FOR_TEST])
        with self._state_lock:
            self.state = state

            LOGGER.debug("CLOSING %s %s pending calls", state, self._pending.qsize())
            LOGGER.debug("        %s %s queued calls", " " * len(str(state)), self._queue.qsize())
            if session_expired:
                LOGGER.debug("        session expired")

            # notify watchers
            if state == AUTH_FAILED:
                self._events.put(lambda: self._default_watcher.auth_failed())
            elif session_expired:
                self._events.put(lambda: self._default_watcher.session_expired(self.session_id))
            else:
                self._events.put(lambda: self._default_watcher.connection_closed())

            # drain queues
            if state == CLOSED:
                self._drain(SessionExpiredError() if session_expired else ConnectionLoss())
            elif state == AUTH_FAILED:
                self._drain(AuthFailedError())

            # when the event thread encounters the connection on the queue, it
            # will kill itself
            self._events.put(self)

    def _drain(self, error):
        assert self._state_lock._is_owned()

        while not self._pending.empty():
            _, _, callback, _ = self._pending.get()
            try:
                callback(error)
            except Exception:
                LOGGER.exception("Error while draining")

        while not self._queue.empty():
            _, _, callback = self._queue.get()
            try:
                callback(error)
            except Exception:
                LOGGER.exception("Error while draining")