Example #1
0
    def _create_scope_container(self, scope_path, unit_name):
        # this is a container scoped relation and we must add
        # node_data as was done in _add_relation_state.
        # before we can do that however we need to extract the
        # proper relation information from the topology. This
        # includes the relation name and the relation_role
        from .service import parse_service_name
        topology = yield self._read_topology()

        service_id = topology.find_service_with_name(
            parse_service_name(unit_name))
        interface, relation_data = topology.get_relation_service(
            self._relation_id, service_id)

        try:
            yield self._client.create(scope_path)
        except zookeeper.NodeExistsException:
            pass

        node_data = yaml.safe_dump(relation_data)
        role_path = "%s/%s" % (scope_path, relation_data["role"])

        yield retry_change(
            self._client, role_path, lambda c, s: node_data)
        yield retry_change(
            self._client, "%s/settings" % scope_path, lambda c, s: "")
Example #2
0
    def set_config_state(self, config, environment_name):
        serialized_env = config.serialize(environment_name)

        def change_environment(old_content, stat):
            return serialized_env
        yield retry_change(self._client, "/environment",
                           change_environment)
Example #3
0
    def test_concurrent_update_bad_version(self):
        """
        If the node is updated after the retry function has read
        the node but before the content is set, the retry function
        will perform another read/change_func/set cycle.
        """
        yield self.client.create("/animals")
        content, stat = yield self.client.get("/animals")
        yield self.client.set("/animals", "5")

        real_get = self.client.get
        p_client = self.mocker.proxy(self.client)
        p_client.get("/animals")
        self.mocker.result(succeed((content, stat)))

        p_client.get("/animals")
        self.mocker.call(real_get)

        self.mocker.replay()

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "6")
        self.assertEqual(stat["version"], 2)
Example #4
0
    def set_config_state(self, config, environment_name):
        serialized_env = config.serialize(environment_name)

        def change_environment(old_content, stat):
            return serialized_env

        yield retry_change(self._client, "/environment", change_environment)
Example #5
0
    def test_set(self):
        yield self.proxied_client.connect()

        # Setup tree
        cpath = "/test-tree"
        yield self.direct_client.create(cpath, json.dumps({"a": 1, "c": 2}))

        def update_node(content, stat):
            data = json.loads(content)
            data["a"] += 1
            data["b"] = 0
            return json.dumps(data)

        # Block the request (drops all packets.)
        self.proxy.set_blocked(True)
        mod_d = retry_change(self.proxied_client, cpath, update_node)

        # Unblock and disconnect
        self.proxy.set_blocked(False)
        self.proxy.lose_connection()

        # Call goes through, contents verified.
        yield mod_d
        content, stat = yield self.direct_client.get(cpath)
        self.assertEqual(json.loads(content),
                         {"a": 2, "b": 0, "c": 2})
Example #6
0
    def _retry_topology_change(self, change_topology_function):
        """Change the current /topology node in a reliable way.

        @param change_topology_function: A function/method which accepts a
            InternalTopology instance as an argument.  This function can read
            and modify the topology instance, and after it returns (or after
            the returned deferred fires) the modified topology will be
            persisted into the /topology node.  Note that this function must
            have no side-effects, since it may be called multiple times
            depending on conflict situations.

        Note that this method name is underlined to mean "protected", not
        "private", since the only purpose of this method is to be used by
        subclasses.
        """

        @inlineCallbacks
        def change_content_function(content, stat):
            topology = InternalTopology()
            if content:
                topology.parse(content)
            yield change_topology_function(topology)
            returnValue(topology.dump())
        return retry_change(self._client, "/topology",
                            change_content_function)
Example #7
0
    def _set_value(self, key, value):
        def set_value(old_content, stat):
            if not old_content:
                data = {}
            else:
                data = yaml.load(old_content)
            data[key] = value
            return yaml.safe_dump(data)

        return retry_change(self._client, SETTINGS_PATH, set_value)
Example #8
0
    def test_error_in_change_function_propogates(self):
        """
        an error in the change function propogates to the caller.
        """

        def error_function(content, stat):
            raise SyntaxError()

        d = retry_change(self.client, "/magic-beans", error_function)
        self.failUnlessFailure(d, SyntaxError)
        return d
Example #9
0
    def _set_value(self, key, value):

        def set_value(old_content, stat):
            if not old_content:
                data = {}
            else:
                data = yaml.load(old_content)
            data[key] = value
            return yaml.safe_dump(data)

        return retry_change(self._client, SETTINGS_PATH, set_value)
Example #10
0
    def set_data(self, data):
        """Set the relation local configuration data for a unit.

        This call overwrites any data currently in the node with the
        dictionary supplied as `data`.
        """
        path = yield self.get_settings_path()
        # encode as a YAML string
        data = yaml.safe_dump(data)

        yield retry_change(
            self._client, path, lambda content, stat: data)
Example #11
0
    def set_instance_id(self, instance_id):
        """Set the provider-specific machine id in this machine state."""
        def set_id(old_content, stat):
            if not stat:
                raise StateChanged("Machine got removed!")
            if old_content:
                data = yaml.load(old_content)
            else:
                data = {}
            data["provider-machine-id"] = instance_id
            return yaml.safe_dump(data)

        return retry_change(self._client, self._zk_path, set_id)
Example #12
0
    def set_instance_id(self, instance_id):
        """Set the provider-specific machine id in this machine state."""

        def set_id(old_content, stat):
            if not stat:
                raise StateChanged("Machine got removed!")
            if old_content:
                data = yaml.load(old_content)
            else:
                data = {}
            data["provider-machine-id"] = instance_id
            return yaml.safe_dump(data)
        return retry_change(self._client, self._zk_path, set_id)
Example #13
0
    def _add_service_relation_state(
        self, relation_id, service_id, endpoint):
        """Add a service relation state.
        """
        # Add service container in relation.
        node_data = yaml.safe_dump(
            {"name": endpoint.relation_name, "role": endpoint.relation_role})
        path = "/relations/%s/%s" % (relation_id, endpoint.relation_role)

        try:
            yield self._client.create(path, node_data)
        except zookeeper.NodeExistsException:
            # If its not, then update the node, and continue.
            yield retry_change(
                self._client, path, lambda content, stat: node_data)
Example #14
0
    def set_data(self, data):
        """Set the relation local configuration data for a unit.

        This call overwrites any data currently in the node with the
        dictionary supplied as `data`.
        """
        unit_settings_path = "/relations/%s/settings/%s" % (
            self._relation_id, self._unit_id)

        # encode as a YAML string
        data = yaml.safe_dump(data)

        yield retry_change(
            self._client, unit_settings_path,
            lambda content, stat: data)
Example #15
0
    def test_identical_content_noop(self):
        """
        If the change function generates identical content to
        the existing node, the retry change function exits without
        modifying the node.
        """
        self.client.create("/animals", "hello")

        def update(content, stat):
            return content

        yield retry_change(self.client, "/animals", update)
        content, stat = self.client.get("/animals")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 0)
Example #16
0
    def _store(self, state_dict):
        """Store the workflow state dictionary in zookeeper."""
        state_serialized = yaml.safe_dump(state_dict)

        def update_state(content, stat):
            unit_data = yaml.load(content)
            if not unit_data:
                unit_data = {}

            persistent_workflow = unit_data.setdefault("workflow_state", {})
            persistent_workflow[self.zk_state_id] = state_serialized
            return yaml.dump(unit_data)

        yield retry_change(self._client, self.zk_state_path, update_state)
        yield super(ZookeeperWorkflowState, self)._store(
            state_dict)
Example #17
0
    def test_node_create(self):
        """
        retry_change will create a node if one does not exist.
        """
        #use a mock to ensure the change function is only invoked once
        func = self.mocker.mock()
        func(None, None)
        self.mocker.result("hello")
        self.mocker.replay()

        yield retry_change(
            self.client, "/magic-beans", func)

        content, stat = yield self.client.get("/magic-beans")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 0)
Example #18
0
    def test_node_update(self):
        """
        retry_change will update an existing node.
        """
        #use a mock to ensure the change function is only invoked once
        func = self.mocker.mock()
        func("", MATCH_STAT)
        self.mocker.result("hello")
        self.mocker.replay()

        yield self.client.create("/magic-beans")
        yield retry_change(
            self.client, "/magic-beans", func)

        content, stat = yield self.client.get("/magic-beans")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 1)
Example #19
0
    def test_set_node_does_not_exist(self):
        """
        if the retry function goes to update a node which has been
        deleted since it was read, it will cycle through to another
        read/change_func set cycle.
        """
        real_get = self.client.get
        p_client = self.mocker.patch(self.client)
        p_client.get("/animals")
        self.mocker.result("5", {"version": 1})

        p_client.get("/animals")
        self.mocker.call(real_get)

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "0")
        self.assertEqual(stat["version"], 0)
Example #20
0
File: utils.py Project: malthe/pop
    def write(self):
        """Write object state to Zookeeper.

        This will write the current state of the object to Zookeeper,
        taking the final merged state as the new one, and resetting
        any write buffers.
        """
        self._check()
        cache = self._cache
        pristine_cache = self._pristine_cache
        self._pristine_cache = cache.copy()

        # Used by `apply_changes` function to return the changes to
        # this scope.
        changes = []

        def apply_changes(content, stat):
            """Apply the local state to the Zookeeper node state."""
            del changes[:]
            current = yaml.load(content) if content else {}
            missing = object()
            for key in set(pristine_cache).union(cache):
                old_value = pristine_cache.get(key, missing)
                new_value = cache.get(key, missing)
                if old_value != new_value:
                    if new_value != missing:
                        current[key] = new_value
                        if old_value != missing:
                            changes.append(
                                ModifiedItem(key, old_value, new_value))
                        else:
                            changes.append(AddedItem(key, new_value))
                    elif key in current:
                        del current[key]
                        changes.append(DeletedItem(key, old_value))
            return yaml.safe_dump(current)

        # Apply the change till it takes.
        yield retry_change(self._client, self._path, apply_changes)
        returnValue(changes)
Example #21
0
    def write(self):
        """Write object state to Zookeeper.

        This will write the current state of the object to Zookeeper,
        taking the final merged state as the new one, and resetting
        any write buffers.
        """
        self._check()
        cache = self._cache
        pristine_cache = self._pristine_cache
        self._pristine_cache = cache.copy()

        # Used by `apply_changes` function to return the changes to
        # this scope.
        changes = []

        def apply_changes(content, stat):
            """Apply the local state to the Zookeeper node state."""
            del changes[:]
            current = yaml.load(content) if content else {}
            missing = object()
            for key in set(pristine_cache).union(cache):
                old_value = pristine_cache.get(key, missing)
                new_value = cache.get(key, missing)
                if old_value != new_value:
                    if new_value != missing:
                        current[key] = new_value
                        if old_value != missing:
                            changes.append(
                                ModifiedItem(key, old_value, new_value))
                        else:
                            changes.append(AddedItem(key, new_value))
                    elif key in current:
                        del current[key]
                        changes.append(DeletedItem(key, old_value))
            return yaml.safe_dump(current)

        # Apply the change till it takes.
        yield retry_change(self._client, self._path, apply_changes)
        returnValue(changes)
Example #22
0
    def test_create_node_exists(self):
        """
        If the node is created after the retry function has determined
        the node doesn't exist but before the node is created by the
        retry function. the retry function will perform another
        read/change_func/set cycle.
        """
        yield self.client.create("/animals", "5")

        real_get = self.client.get
        p_client = self.mocker.patch(self.client)
        p_client.get("/animals")
        self.mocker.result(fail(zookeeper.NoNodeException()))

        p_client.get("/animals")
        self.mocker.call(real_get)
        self.mocker.replay()

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "6")
        self.assertEqual(stat["version"], 1)
Example #23
0
    def _retry_topology_change(self, change_topology_function):
        """Change the current /topology node in a reliable way.

        @param change_topology_function: A function/method which accepts a
            InternalTopology instance as an argument.  This function can read
            and modify the topology instance, and after it returns (or after
            the returned deferred fires) the modified topology will be
            persisted into the /topology node.  Note that this function must
            have no side-effects, since it may be called multiple times
            depending on conflict situations.

        Note that this method name is underlined to mean "protected", not
        "private", since the only purpose of this method is to be used by
        subclasses.
        """

        def change_content_function(content, stat):
            topology = InternalTopology()
            if content:
                topology.parse(content)
            change_topology_function(topology)
            return topology.dump()
        return retry_change(self._client, "/topology",
                            change_content_function)
Example #24
0
    def add_unit_state(self, unit_state):
        """Add a unit to the service relation.

        This api is intended for use by the unit agent, as it
        also creates an ephemeral presence node, denoting the
        active existence of the unit in the relation.

        returns a unit relation state.
        """
        container = yield unit_state.get_container()
        scope_path = self._get_scope_path(unit_state, container)
        settings_path = "%s/settings/%s" % (
            scope_path, unit_state.internal_id)

        # Pre-populate the relation node with the node's private address.
        if self._relation_scope == "container":
            # Create the service relation node data in the proper scope
            yield self._create_scope_container(
                scope_path, unit_state.unit_name)
        if container:
            private_address = yield container.get_private_address()
        else:
            private_address = yield unit_state.get_private_address()

        def update_address(content, stat):
            unit_map = None
            if content:
                unit_map = yaml.load(content)
            if not unit_map:
                unit_map = {}
            unit_map["private-address"] = private_address
            return yaml.safe_dump(unit_map)

        yield retry_change(self._client, settings_path, update_address)

        # Update the unit name -> id mapping on the relation node
        def update_unit_mapping(content, stat):
            if content:
                unit_map = yaml.load(content)
            else:
                unit_map = {}
            # If it's already present, we're done, just return the
            # existing content, to avoid unstable yaml dict
            # serialization.
            if unit_state.internal_id in unit_map:
                return content
            unit_map[unit_state.internal_id] = unit_state.unit_name
            return yaml.dump(unit_map)

        yield retry_change(self._client,
                           "/relations/%s" % self._relation_id,
                           update_unit_mapping)

        # Create the presence node.
        role_path = scope_path + "/" + self._role
        alive_path = role_path + "/" + unit_state.internal_id

        try:
            # create the role node
            yield self._client.create(role_path)
        except zookeeper.NodeExistsException:
            pass

        try:
            yield self._client.create(alive_path, flags=zookeeper.EPHEMERAL)
        except zookeeper.NodeExistsException:
            # Concurrent creation is okay, end state is the same.
            pass

        returnValue(
            UnitRelationState(
                self._client,
                self._service_id,
                unit_state.internal_id,
                self._relation_id,
                self._relation_scope))
Example #25
0
 def _force(self, path, content):
     return retry_change(self._client, path, lambda *_: content)
Example #26
0
    def add_unit_state(self, unit_state):
        """Add a unit to the service relation.

        This api is intended for use by the unit agent, as it
        also creates an ephemeral presence node, denoting the
        active existance of the unit in the relation.

        returns a unit relation state.
        """
        settings_path = "/relations/%s/settings/%s" % (
            self._relation_id, unit_state.internal_id)

        # We create settings node first, so that presence node events
        # have a chance to inspect state.

        # Prepopulate the relation node with the node's private address.
        private_address = yield unit_state.get_private_address()
        try:
            yield self._client.create(
                settings_path,
                yaml.safe_dump({"private-address": private_address}))
        except zookeeper.NodeExistsException:
            # previous persistent settings are not an error, but
            # update the unit address
            def update_address(content, stat):
                unit_map = yaml.load(content)
                if not unit_map:
                    unit_map = {}
                unit_map["private-address"] = private_address
                return yaml.safe_dump(unit_map)
            yield retry_change(self._client, settings_path, update_address)

        # Update the unit name -> id mapping on the relation node
        def update_unit_mapping(content, stat):
            if content:
                unit_map = yaml.load(content)
            else:
                unit_map = {}
            # If its already present, we're done, just return the
            # existing content, to avoid unstable yaml dict
            # serialization.
            if unit_state.internal_id in unit_map:
                return content
            unit_map[unit_state.internal_id] = unit_state.unit_name
            return yaml.dump(unit_map)

        yield retry_change(self._client,
                           "/relations/%s" % self._relation_id,
                           update_unit_mapping)

        # Create the presence node.
        alive_path = "/relations/%s/%s/%s" % (
            self._relation_id, self._role, unit_state.internal_id)

        try:
            yield self._client.create(alive_path, flags=zookeeper.EPHEMERAL)
        except zookeeper.NodeExistsException:
            # Concurrent creation is okay, end state is the same.
            pass

        returnValue(
            UnitRelationState(
                self._client,
                self._service_id,
                unit_state.internal_id,
                self._relation_id))