Beispiel #1
0
        def mk_instructions(fun, nodelist):
            """
            Creates a list of independent instructions based on a single
            topological level. The type of instructions will be determined
            by the logical core function: ``fun``.

            :param list nodelist: List of nodes.
            :param fun: Core function that is called for each node in
                ``nodelist``.
            :type fun: ``(node x [node] x int) -> [command]``.

            ``fun`` returns a *set* (generator) of instructions *for each*
            node. E.g.: when multiple instances of a single node must be
            created at once. If nothing is to be done with a node, an empty
            list is returned by ``fun``.

            These individual sets are then unioned, as they all pertain to a
            single topological level (hence ``flatten``)
            """
            return util.flatten(  # Union
                fun(node,
                    existing=dynamic_state.get(node['name'], dict()),
                    target=self.calc_target(
                        node, dynamic_state.get(node['name'], dict())))
                for node in nodelist)
Beispiel #2
0
 def _extract_nodes(self, infra_id, name):
     infrastate = self.get_infrastructure_state(infra_id)
     if name:
         return iter(list(infrastate[name].values())) \
             if name in infrastate \
             else []
     else:
         return flatten(
             iter(list(i.values())) for i in list(infrastate.values()))
Beispiel #3
0
 def _extract_nodes(self, infra_id, name):
     infrastate = self.get_infrastructure_state(infra_id)
     if name:
         return infrastate[name].itervalues() \
             if name in infrastate \
             else []
     else:
         return flatten(i.itervalues()
                        for i in infrastate.itervalues())
Beispiel #4
0
    def _filtered_infra(self, infra_id, name):
        def cut_id(s):
            parts = s.split(':')
            return parts[1]

        if infra_id:
            infra_ids = [infra_id]
        else:
            warnings.warn('Filtering nodes without infra_id specified. '
                          'As there are no DB indexes, this is an *extremely* '
                          'inefficient operation (full DB sweep). Consider '
                          'specifying an infra_id too.',
                          UserWarning)
            infra_ids = self.kvstore.enumerate('infra:*:state', cut_id)

        return flatten(self._extract_nodes(i, name) for i in infra_ids)
Beispiel #5
0
    def _filtered_infra(self, infra_id, name):
        def cut_id(s):
            parts = s.split(':')
            return parts[1].split('@', 1)[1]

        if infra_id:
            infra_ids = [infra_id]
        else:
            warnings.warn(
                'Filtering nodes without infra_id specified. '
                'As there are no DB indexes, this is an *extremely* '
                'inefficient operation (full DB sweep). Consider '
                'specifying an infra_id too.', UserWarning)
            infra_filter_str = 'infra:{0!s}@*:state'.format(getpass.getuser())
            infra_ids = self.kvstore.enumerate(infra_filter_str, cut_id)

        return flatten(self._extract_nodes(i, name) for i in infra_ids)
Beispiel #6
0
def teardown(infra_id, ip):
    import logging
    from ruamel import yaml
    log = logging.getLogger('occo.occoapp')
    datalog = logging.getLogger('occo.data.occoapp')

    from occo.infobroker import main_info_broker

    state = main_info_broker.get('infrastructure.node_instances', infra_id)
    from occo.util import flatten
    nodes = list(flatten(iter(i.values()) for i in state.values()))
    drop_node_commands = [ip.cri_drop_node(n) for n in nodes]
    log.debug('Dropping nodes: %s', [n['node_id'] for n in nodes])
    datalog.debug('DropNode:\n%s',
                  yaml.dump(drop_node_commands, default_flow_style=False))

    ip.push_instructions(infra_id, drop_node_commands)

    ip.push_instructions(infra_id, ip.cri_drop_infrastructure(infra_id))
Beispiel #7
0
def teardown(infra_id, ip):
    import logging
    log = logging.getLogger('occo.occoapp')
    datalog = logging.getLogger('occo.data.occoapp')

    log.info('Tearing down infrastructure %r', infra_id)

    from occo.infobroker import main_info_broker
    dynamic_state = main_info_broker.get('infrastructure.state', infra_id)

    from occo.util import flatten
    nodes = list(flatten(i.itervalues() for i in dynamic_state.itervalues()))

    import yaml
    drop_node_commands = [ip.cri_drop_node(n) for n in nodes]
    log.debug('Dropping nodes: %r', [n['node_id'] for n in nodes])
    datalog.debug('DropNode:\n%s',
                  yaml.dump(drop_node_commands, default_flow_style=False))

    ip.push_instructions(drop_node_commands)

    ip.push_instructions(ip.cri_drop_infrastructure(infra_id))
Beispiel #8
0
        def mk_instructions(fun, nodelist):
            """
            Creates a list of independent instructions based on a single
            topological level. The type of instructions will be determined
            by the logical core function: ``fun``.

            :param list nodelist: List of nodes.
            :param fun: Core function that is called for each node in
                ``nodelist``.
            :type fun: ``(node x [node] x int) -> [command]``.

            ``fun`` returns a *set* (generator) of instructions *for each*
            node. E.g.: when multiple instances of a single node must be
            created at once. If nothing is to be done with a node, an empty
            list is returned by ``fun``.
            
            These individual sets are then unioned, as they all pertain to a
            single topological level (hence ``flatten``)
            """
            return util.flatten(  # Union
                fun(node, existing=dynamic_state.get(node["name"], dict()), target=self.calc_target(node))
                for node in nodelist
            )
Beispiel #9
0
 def test_flatten(self):
     l1, l2, l3 = [0, 1, 2, 3], [], [4, 5, 6]
     self.assertEqual(list(util.flatten([l1, l2, l3])), range(7))
Beispiel #10
0
    def calculate_delta(self, static_description, dynamic_state, failed_nodes):
        """
        Calculates a list of instructions to be executed to bring the
        infrastructure in its desired state.

        :param static_description: Description of the desired state of the
            infrastructure.
        :type static_description:
            :class:`~occo.compiler.compiler.StaticDescription`

        :param dynamic_state: The actual state of the infrastructure.
        :type dynamic_state: See
            :meth:`occo.infobroker.dynamic_state_provider.DynamicStateProvider.infra_state`

        The result is a list of lists (generator of generators).
        The main result list is called the *delta*. Each item of the delta
        is a list of instructions that can be executed asynchronously and
        independently of each other. Each such a list pertains to a level of
        the topological ordering of the infrastructure.

        :rtype:
            .. code::

                delta <generator> = (
                    instructions <generator> = (instr1 <Command>, instr2 <Command>, ...),
                    instructions <generator> = (instr21 <Command>, instr22 <Command>, ...),
                    ...
                )
        """

        # Possibly this is the most complex method in the OCCO, utilizing
        # generators upon generators for efficiency.
        #
        # Actually, it's just five lines constituting only three logical parts.
        # The parts are broken down into smaller parts, all being nested
        # methods inside this one. It may be possible to simplify this method
        # (and it would be desirable if possible), but I couldn't. The biggest
        # problem is with type-matching, as Python cannot do a static type
        # checking, so it is easy to make a mistake. Nevertheless, it currently
        # works and is efficient.
        #
        # Iterators, although efficient, must be used carefully, as they cannot
        # be traversed twice (unlike lists). Typical error: one of them is
        # printed into the log (traversed), and then the actual code will see
        # an empty list (as the generator is finished). But they're extremely
        # memory-efficient, so we have to just deal with it.

        def mk_instructions(fun, nodelist):
            """
            Creates a list of independent instructions based on a single
            topological level. The type of instructions will be determined
            by the logical core function: ``fun``.

            :param list nodelist: List of nodes.
            :param fun: Core function that is called for each node in
                ``nodelist``.
            :type fun: ``(node x [node] x int) -> [command]``.

            ``fun`` returns a *set* (generator) of instructions *for each*
            node. E.g.: when multiple instances of a single node must be
            created at once. If nothing is to be done with a node, an empty
            list is returned by ``fun``.

            These individual sets are then unioned, as they all pertain to a
            single topological level (hence ``flatten``)
            """
            return util.flatten(  # Union
                fun(node,
                    existing=dynamic_state.get(node['name'], dict()),
                    target=self.calc_target(
                        node, dynamic_state.get(node['name'], dict())))
                for node in nodelist)

        def mkdelinst(node, existing, target):
            """
            MaKe DELete INSTructions

            Used as a core to ``mk_instructions``; it creates a list of
            DropNode instructions, for a single node type, as necessary.

            :param node: The node to be acted upon.
            :param existing: Nodes that already exists.
            :param int target: The target number of nodes.
            """
            exst_count = len(existing)
            if target < exst_count:
                return (self.ip.cri_drop_node(instance_data=instance_data)
                        for instance_data in self.select_nodes_to_drop(
                            existing, exst_count - target))
            return []

        def mkcrinst(node, existing, target):
            """
            MaKe CReate INSTructions

            Used as a core to ``mk_instructions``; it creates a list of
            CreateNode instructions, for a single node type, as necessary.

            :param node: The node to be acted upon.
            :param existing: Nodes that already exists.
            :param int target: The target number of nodes.
            """
            exst_count = len(existing)
            if target > exst_count:
                return (self.ip.cri_create_node(node)
                        for i in range(target - exst_count))
            return []

        def mkdelinstforfailednode(failed_node):
            """
            MaKe DELete INSTructions

            Used as a core to ``mk_instructions``; it creates a list of
            DropNode instructions, for a single node type, as necessary.

            :param node: The node to be acted upon.
            :param existing: Nodes that already exists.
            :param int target: The target number of nodes.
            """
            return [self.ip.cri_drop_node(failed_node)]

        def mkdrinst(node):
            """
            MaKe DRop INSTructions

            Used as a core to ``mk_instructions``; it creates a list of DropNode
            instructions, for nodes that are removed from the infrastructure by
            an updated infra_desc
            """
            return [self.ip.cri_drop_node(node)]

        # ShorthandGG
        infra_id = static_description.infra_id

        # Each `yield' returns an element of the delta
        # The bootstrap elements of the delta, iff needed.
        # This is a single list.
        yield self.gen_bootstrap_instructions(infra_id)

        # Node deletions.
        # Drop instructions are generated for each node, and then they are
        # merged in a single list (as they have no dependencies among them).
        yield util.flatten(
            mk_instructions(mkdelinst, nodelist)
            for nodelist in static_description.topological_order)

        # Node deletion for node types which were removed from the
        # infrastructure by an updated infra_desc
        static_list = []
        for node in static_description.nodes:
            static_list.append(node.get('name'))

        nodelist = []
        for node in dynamic_state:
            if node not in static_list:
                for key in dynamic_state.get(node):
                    nodelist.append(dynamic_state.get(node).get(key))

        yield util.flatten(mkdrinst(node) for node in nodelist)

        # Failed node deletions.
        # Drop instructions are generated for each node, and then they are
        # merged in a single list (as they have no dependencies among them).
        yield util.flatten(
            mkdelinstforfailednode(node) for node in failed_nodes)

        # Node creations.
        # Create-instructions are generated for each node.
        # Each of these lists pertains to a topological level of the dependency
        # graph, so each of these lists is returned individually.
        for nodelist in static_description.topological_order:
            yield mk_instructions(mkcrinst, nodelist)
Beispiel #11
0
    def calculate_delta(self, static_description, dynamic_state, failed_nodes):
        """
        Calculates a list of instructions to be executed to bring the
        infrastructure in its desired state.

        :param static_description: Description of the desired state of the
            infrastructure.
        :type static_description:
            :class:`~occo.compiler.compiler.StaticDescription`

        :param dynamic_state: The actual state of the infrastructure.
        :type dynamic_state: See
            :meth:`occo.infobroker.dynamic_state_provider.DynamicStateProvider.infra_state`

        The result is a list of lists (generator of generators).
        The main result list is called the *delta*. Each item of the delta
        is a list of instructions that can be executed asynchronously and
        independently of each other. Each such a list pertains to a level of
        the topological ordering of the infrastructure.
        
        :rtype:
            .. code::

                delta <generator> = (
                    instructions <generator> = (instr1 <Command>, instr2 <Command>, ...),
                    instructions <generator> = (instr21 <Command>, instr22 <Command>, ...),
                    ...
                )
        """
        # Possibly this is the most complex method in the OCCO, utilizing
        # generators upon generators for efficiency.
        #
        # Actually, it's just five lines constituting only three logical parts.
        # The parts are broken down into smaller parts, all being nested
        # methods inside this one. It may be possible to simplify this method
        # (and it would be desirable if possible), but I couldn't. The biggest
        # problem is with type-matching, as Python cannot do a static type
        # checking, so it is easy to make a mistake. Nevertheless, it currently
        # works and is efficient.
        #
        # Iterators, although efficient, must be used carefully, as they cannot
        # be traversed twice (unlike lists). Typical error: one of them is
        # printed into the log (traversed), and then the actual code will see
        # an empty list (as the generator is finished). But they're extremely
        # memory-efficient, so we have to just deal with it.

        def mk_instructions(fun, nodelist):
            """
            Creates a list of independent instructions based on a single
            topological level. The type of instructions will be determined
            by the logical core function: ``fun``.

            :param list nodelist: List of nodes.
            :param fun: Core function that is called for each node in
                ``nodelist``.
            :type fun: ``(node x [node] x int) -> [command]``.

            ``fun`` returns a *set* (generator) of instructions *for each*
            node. E.g.: when multiple instances of a single node must be
            created at once. If nothing is to be done with a node, an empty
            list is returned by ``fun``.
            
            These individual sets are then unioned, as they all pertain to a
            single topological level (hence ``flatten``)
            """
            return util.flatten(  # Union
                fun(node, existing=dynamic_state.get(node["name"], dict()), target=self.calc_target(node))
                for node in nodelist
            )

        def mkdelinst(node, existing, target):
            """
            MaKe DELete INSTructions

            Used as a core to ``mk_instructions``; it creates a list of
            DropNode instructions, for a single node type, as necessary.

            :param node: The node to be acted upon.
            :param existing: Nodes that already exists.
            :param int target: The target number of nodes.
            """
            exst_count = len(existing)
            if target < exst_count:
                return (
                    self.ip.cri_drop_node(instance_data=instance_data)
                    for instance_data in self.select_nodes_to_drop(existing, exst_count - target)
                )
            return []

        def mkcrinst(node, existing, target):
            """
            MaKe CReate INSTructions

            Used as a core to ``mk_instructions``; it creates a list of
            CreateNode instructions, for a single node type, as necessary.

            :param node: The node to be acted upon.
            :param existing: Nodes that already exists.
            :param int target: The target number of nodes.
            """
            exst_count = len(existing)
            if target > exst_count:
                return (self.ip.cri_create_node(node) for i in xrange(target - exst_count))
            return []

        def mkdelinstforfailednode(failed_node):
            """
            MaKe DELete INSTructions

            Used as a core to ``mk_instructions``; it creates a list of
            DropNode instructions, for a single node type, as necessary.

            :param node: The node to be acted upon.
            :param existing: Nodes that already exists.
            :param int target: The target number of nodes.
            """
            return [self.ip.cri_drop_node(failed_node)]

        def mkdrinst(node):
            """
            MaKe DRop INSTructions

            Used as a core to ``mk_instructions``; it creates a list of DropNode
            instructions, for nodes that are removed from the infrastructure by
            an updated infra_desc
            """
            return [self.ip.cri_drop_node(node)]

        # ShorthandGG
        infra_id = static_description.infra_id

        # Each `yield' returns an element of the delta
        # The bootstrap elements of the delta, iff needed.
        # This is a single list.
        yield self.gen_bootstrap_instructions(infra_id)

        # Node deletions.
        # Drop instructions are generated for each node, and then they are
        # merged in a single list (as they have no dependencies among them).
        yield util.flatten(mk_instructions(mkdelinst, nodelist) for nodelist in static_description.topological_order)

        # Node deletion for node types which were removed from the
        # infrastructure by an updated infra_desc
        static_list = []
        for node in static_description.nodes:
            static_list.append(node.get("name"))

        nodelist = []
        for node in dynamic_state:
            if node not in static_list:
                for key in dynamic_state.get(node):
                    nodelist.append(dynamic_state.get(node).get(key))

        yield util.flatten(mkdrinst(node) for node in nodelist)

        # Failed node deletions.
        # Drop instructions are generated for each node, and then they are
        # merged in a single list (as they have no dependencies among them).
        yield util.flatten(mkdelinstforfailednode(node) for node in failed_nodes)

        # Node creations.
        # Create-instructions are generated for each node.
        # Each of these lists pertains to a topological level of the dependency
        # graph, so each of these lists is returned individually.
        for nodelist in static_description.topological_order:
            yield mk_instructions(mkcrinst, nodelist)
Beispiel #12
0
 def test_flatten(self):
     l1, l2, l3 = [0, 1, 2, 3], [], [4, 5, 6]
     self.assertEqual(list(util.flatten([l1, l2, l3])), list(range(7)))
Beispiel #13
0
 def iterkeys(self):
     """ Overrides :meth:`InfoRouter.iterkeys` """
     mykeys = super(InfoRouter, self).iterkeys
     sub_keys = (i.iterkeys for i in self.sub_providers)
     return flatten(it.chain([mykeys], sub_keys))