def mk_instructions(fun, nodelist):
"""
Creates a list of independent instructions based on a single
topological level. The type of instructions will be determined
by the logical core function: ``fun``.
:param list nodelist: List of nodes.
:param fun: Core function that is called for each node in
``nodelist``.
:type fun: ``(node x [node] x int) -> [command]``.
``fun`` returns a *set* (generator) of instructions *for each*
node. E.g.: when multiple instances of a single node must be
created at once. If nothing is to be done with a node, an empty
list is returned by ``fun``.
These individual sets are then unioned, as they all pertain to a
single topological level (hence ``flatten``)
"""
return util.flatten( # Union
fun(node,
existing=dynamic_state.get(node['name'], dict()),
target=self.calc_target(
node, dynamic_state.get(node['name'], dict())))
for node in nodelist)
def _extract_nodes(self, infra_id, name):
infrastate = self.get_infrastructure_state(infra_id)
if name:
return iter(list(infrastate[name].values())) \
if name in infrastate \
else []
else:
return flatten(
iter(list(i.values())) for i in list(infrastate.values()))
def _extract_nodes(self, infra_id, name):
infrastate = self.get_infrastructure_state(infra_id)
if name:
return infrastate[name].itervalues() \
if name in infrastate \
else []
else:
return flatten(i.itervalues()
for i in infrastate.itervalues())
def _filtered_infra(self, infra_id, name):
def cut_id(s):
parts = s.split(':')
return parts[1]
if infra_id:
infra_ids = [infra_id]
else:
warnings.warn('Filtering nodes without infra_id specified. '
'As there are no DB indexes, this is an *extremely* '
'inefficient operation (full DB sweep). Consider '
'specifying an infra_id too.',
UserWarning)
infra_ids = self.kvstore.enumerate('infra:*:state', cut_id)
return flatten(self._extract_nodes(i, name) for i in infra_ids)
def _filtered_infra(self, infra_id, name):
def cut_id(s):
parts = s.split(':')
return parts[1].split('@', 1)[1]
if infra_id:
infra_ids = [infra_id]
else:
warnings.warn(
'Filtering nodes without infra_id specified. '
'As there are no DB indexes, this is an *extremely* '
'inefficient operation (full DB sweep). Consider '
'specifying an infra_id too.', UserWarning)
infra_filter_str = 'infra:{0!s}@*:state'.format(getpass.getuser())
infra_ids = self.kvstore.enumerate(infra_filter_str, cut_id)
return flatten(self._extract_nodes(i, name) for i in infra_ids)
def teardown(infra_id, ip):
import logging
from ruamel import yaml
log = logging.getLogger('occo.occoapp')
datalog = logging.getLogger('occo.data.occoapp')
from occo.infobroker import main_info_broker
state = main_info_broker.get('infrastructure.node_instances', infra_id)
from occo.util import flatten
nodes = list(flatten(iter(i.values()) for i in state.values()))
drop_node_commands = [ip.cri_drop_node(n) for n in nodes]
log.debug('Dropping nodes: %s', [n['node_id'] for n in nodes])
datalog.debug('DropNode:\n%s',
yaml.dump(drop_node_commands, default_flow_style=False))
ip.push_instructions(infra_id, drop_node_commands)
ip.push_instructions(infra_id, ip.cri_drop_infrastructure(infra_id))
def teardown(infra_id, ip):
import logging
log = logging.getLogger('occo.occoapp')
datalog = logging.getLogger('occo.data.occoapp')
log.info('Tearing down infrastructure %r', infra_id)
from occo.infobroker import main_info_broker
dynamic_state = main_info_broker.get('infrastructure.state', infra_id)
from occo.util import flatten
nodes = list(flatten(i.itervalues() for i in dynamic_state.itervalues()))
import yaml
drop_node_commands = [ip.cri_drop_node(n) for n in nodes]
log.debug('Dropping nodes: %r', [n['node_id'] for n in nodes])
datalog.debug('DropNode:\n%s',
yaml.dump(drop_node_commands, default_flow_style=False))
ip.push_instructions(drop_node_commands)
ip.push_instructions(ip.cri_drop_infrastructure(infra_id))
def mk_instructions(fun, nodelist):
"""
Creates a list of independent instructions based on a single
topological level. The type of instructions will be determined
by the logical core function: ``fun``.
:param list nodelist: List of nodes.
:param fun: Core function that is called for each node in
``nodelist``.
:type fun: ``(node x [node] x int) -> [command]``.
``fun`` returns a *set* (generator) of instructions *for each*
node. E.g.: when multiple instances of a single node must be
created at once. If nothing is to be done with a node, an empty
list is returned by ``fun``.
These individual sets are then unioned, as they all pertain to a
single topological level (hence ``flatten``)
"""
return util.flatten( # Union
fun(node, existing=dynamic_state.get(node["name"], dict()), target=self.calc_target(node))
for node in nodelist
)
def test_flatten(self):
l1, l2, l3 = [0, 1, 2, 3], [], [4, 5, 6]
self.assertEqual(list(util.flatten([l1, l2, l3])), range(7))
def calculate_delta(self, static_description, dynamic_state, failed_nodes):
"""
Calculates a list of instructions to be executed to bring the
infrastructure in its desired state.
:param static_description: Description of the desired state of the
infrastructure.
:type static_description:
:class:`~occo.compiler.compiler.StaticDescription`
:param dynamic_state: The actual state of the infrastructure.
:type dynamic_state: See
:meth:`occo.infobroker.dynamic_state_provider.DynamicStateProvider.infra_state`
The result is a list of lists (generator of generators).
The main result list is called the *delta*. Each item of the delta
is a list of instructions that can be executed asynchronously and
independently of each other. Each such a list pertains to a level of
the topological ordering of the infrastructure.
:rtype:
.. code::
delta <generator> = (
instructions <generator> = (instr1 <Command>, instr2 <Command>, ...),
instructions <generator> = (instr21 <Command>, instr22 <Command>, ...),
...
)
"""
# Possibly this is the most complex method in the OCCO, utilizing
# generators upon generators for efficiency.
#
# Actually, it's just five lines constituting only three logical parts.
# The parts are broken down into smaller parts, all being nested
# methods inside this one. It may be possible to simplify this method
# (and it would be desirable if possible), but I couldn't. The biggest
# problem is with type-matching, as Python cannot do a static type
# checking, so it is easy to make a mistake. Nevertheless, it currently
# works and is efficient.
#
# Iterators, although efficient, must be used carefully, as they cannot
# be traversed twice (unlike lists). Typical error: one of them is
# printed into the log (traversed), and then the actual code will see
# an empty list (as the generator is finished). But they're extremely
# memory-efficient, so we have to just deal with it.
def mk_instructions(fun, nodelist):
"""
Creates a list of independent instructions based on a single
topological level. The type of instructions will be determined
by the logical core function: ``fun``.
:param list nodelist: List of nodes.
:param fun: Core function that is called for each node in
``nodelist``.
:type fun: ``(node x [node] x int) -> [command]``.
``fun`` returns a *set* (generator) of instructions *for each*
node. E.g.: when multiple instances of a single node must be
created at once. If nothing is to be done with a node, an empty
list is returned by ``fun``.
These individual sets are then unioned, as they all pertain to a
single topological level (hence ``flatten``)
"""
return util.flatten( # Union
fun(node,
existing=dynamic_state.get(node['name'], dict()),
target=self.calc_target(
node, dynamic_state.get(node['name'], dict())))
for node in nodelist)
def mkdelinst(node, existing, target):
"""
MaKe DELete INSTructions
Used as a core to ``mk_instructions``; it creates a list of
DropNode instructions, for a single node type, as necessary.
:param node: The node to be acted upon.
:param existing: Nodes that already exists.
:param int target: The target number of nodes.
"""
exst_count = len(existing)
if target < exst_count:
return (self.ip.cri_drop_node(instance_data=instance_data)
for instance_data in self.select_nodes_to_drop(
existing, exst_count - target))
return []
def mkcrinst(node, existing, target):
"""
MaKe CReate INSTructions
Used as a core to ``mk_instructions``; it creates a list of
CreateNode instructions, for a single node type, as necessary.
:param node: The node to be acted upon.
:param existing: Nodes that already exists.
:param int target: The target number of nodes.
"""
exst_count = len(existing)
if target > exst_count:
return (self.ip.cri_create_node(node)
for i in range(target - exst_count))
return []
def mkdelinstforfailednode(failed_node):
"""
MaKe DELete INSTructions
Used as a core to ``mk_instructions``; it creates a list of
DropNode instructions, for a single node type, as necessary.
:param node: The node to be acted upon.
:param existing: Nodes that already exists.
:param int target: The target number of nodes.
"""
return [self.ip.cri_drop_node(failed_node)]
def mkdrinst(node):
"""
MaKe DRop INSTructions
Used as a core to ``mk_instructions``; it creates a list of DropNode
instructions, for nodes that are removed from the infrastructure by
an updated infra_desc
"""
return [self.ip.cri_drop_node(node)]
# ShorthandGG
infra_id = static_description.infra_id
# Each `yield' returns an element of the delta
# The bootstrap elements of the delta, iff needed.
# This is a single list.
yield self.gen_bootstrap_instructions(infra_id)
# Node deletions.
# Drop instructions are generated for each node, and then they are
# merged in a single list (as they have no dependencies among them).
yield util.flatten(
mk_instructions(mkdelinst, nodelist)
for nodelist in static_description.topological_order)
# Node deletion for node types which were removed from the
# infrastructure by an updated infra_desc
static_list = []
for node in static_description.nodes:
static_list.append(node.get('name'))
nodelist = []
for node in dynamic_state:
if node not in static_list:
for key in dynamic_state.get(node):
nodelist.append(dynamic_state.get(node).get(key))
yield util.flatten(mkdrinst(node) for node in nodelist)
# Failed node deletions.
# Drop instructions are generated for each node, and then they are
# merged in a single list (as they have no dependencies among them).
yield util.flatten(
mkdelinstforfailednode(node) for node in failed_nodes)
# Node creations.
# Create-instructions are generated for each node.
# Each of these lists pertains to a topological level of the dependency
# graph, so each of these lists is returned individually.
for nodelist in static_description.topological_order:
yield mk_instructions(mkcrinst, nodelist)
def calculate_delta(self, static_description, dynamic_state, failed_nodes):
"""
Calculates a list of instructions to be executed to bring the
infrastructure in its desired state.
:param static_description: Description of the desired state of the
infrastructure.
:type static_description:
:class:`~occo.compiler.compiler.StaticDescription`
:param dynamic_state: The actual state of the infrastructure.
:type dynamic_state: See
:meth:`occo.infobroker.dynamic_state_provider.DynamicStateProvider.infra_state`
The result is a list of lists (generator of generators).
The main result list is called the *delta*. Each item of the delta
is a list of instructions that can be executed asynchronously and
independently of each other. Each such a list pertains to a level of
the topological ordering of the infrastructure.
:rtype:
.. code::
delta <generator> = (
instructions <generator> = (instr1 <Command>, instr2 <Command>, ...),
instructions <generator> = (instr21 <Command>, instr22 <Command>, ...),
...
)
"""
# Possibly this is the most complex method in the OCCO, utilizing
# generators upon generators for efficiency.
#
# Actually, it's just five lines constituting only three logical parts.
# The parts are broken down into smaller parts, all being nested
# methods inside this one. It may be possible to simplify this method
# (and it would be desirable if possible), but I couldn't. The biggest
# problem is with type-matching, as Python cannot do a static type
# checking, so it is easy to make a mistake. Nevertheless, it currently
# works and is efficient.
#
# Iterators, although efficient, must be used carefully, as they cannot
# be traversed twice (unlike lists). Typical error: one of them is
# printed into the log (traversed), and then the actual code will see
# an empty list (as the generator is finished). But they're extremely
# memory-efficient, so we have to just deal with it.
def mk_instructions(fun, nodelist):
"""
Creates a list of independent instructions based on a single
topological level. The type of instructions will be determined
by the logical core function: ``fun``.
:param list nodelist: List of nodes.
:param fun: Core function that is called for each node in
``nodelist``.
:type fun: ``(node x [node] x int) -> [command]``.
``fun`` returns a *set* (generator) of instructions *for each*
node. E.g.: when multiple instances of a single node must be
created at once. If nothing is to be done with a node, an empty
list is returned by ``fun``.
These individual sets are then unioned, as they all pertain to a
single topological level (hence ``flatten``)
"""
return util.flatten( # Union
fun(node, existing=dynamic_state.get(node["name"], dict()), target=self.calc_target(node))
for node in nodelist
)
def mkdelinst(node, existing, target):
"""
MaKe DELete INSTructions
Used as a core to ``mk_instructions``; it creates a list of
DropNode instructions, for a single node type, as necessary.
:param node: The node to be acted upon.
:param existing: Nodes that already exists.
:param int target: The target number of nodes.
"""
exst_count = len(existing)
if target < exst_count:
return (
self.ip.cri_drop_node(instance_data=instance_data)
for instance_data in self.select_nodes_to_drop(existing, exst_count - target)
)
return []
def mkcrinst(node, existing, target):
"""
MaKe CReate INSTructions
Used as a core to ``mk_instructions``; it creates a list of
CreateNode instructions, for a single node type, as necessary.
:param node: The node to be acted upon.
:param existing: Nodes that already exists.
:param int target: The target number of nodes.
"""
exst_count = len(existing)
if target > exst_count:
return (self.ip.cri_create_node(node) for i in xrange(target - exst_count))
return []
def mkdelinstforfailednode(failed_node):
"""
MaKe DELete INSTructions
Used as a core to ``mk_instructions``; it creates a list of
DropNode instructions, for a single node type, as necessary.
:param node: The node to be acted upon.
:param existing: Nodes that already exists.
:param int target: The target number of nodes.
"""
return [self.ip.cri_drop_node(failed_node)]
def mkdrinst(node):
"""
MaKe DRop INSTructions
Used as a core to ``mk_instructions``; it creates a list of DropNode
instructions, for nodes that are removed from the infrastructure by
an updated infra_desc
"""
return [self.ip.cri_drop_node(node)]
# ShorthandGG
infra_id = static_description.infra_id
# Each `yield' returns an element of the delta
# The bootstrap elements of the delta, iff needed.
# This is a single list.
yield self.gen_bootstrap_instructions(infra_id)
# Node deletions.
# Drop instructions are generated for each node, and then they are
# merged in a single list (as they have no dependencies among them).
yield util.flatten(mk_instructions(mkdelinst, nodelist) for nodelist in static_description.topological_order)
# Node deletion for node types which were removed from the
# infrastructure by an updated infra_desc
static_list = []
for node in static_description.nodes:
static_list.append(node.get("name"))
nodelist = []
for node in dynamic_state:
if node not in static_list:
for key in dynamic_state.get(node):
nodelist.append(dynamic_state.get(node).get(key))
yield util.flatten(mkdrinst(node) for node in nodelist)
# Failed node deletions.
# Drop instructions are generated for each node, and then they are
# merged in a single list (as they have no dependencies among them).
yield util.flatten(mkdelinstforfailednode(node) for node in failed_nodes)
# Node creations.
# Create-instructions are generated for each node.
# Each of these lists pertains to a topological level of the dependency
# graph, so each of these lists is returned individually.
for nodelist in static_description.topological_order:
yield mk_instructions(mkcrinst, nodelist)
def test_flatten(self):
l1, l2, l3 = [0, 1, 2, 3], [], [4, 5, 6]
self.assertEqual(list(util.flatten([l1, l2, l3])), list(range(7)))
def iterkeys(self):
""" Overrides :meth:`InfoRouter.iterkeys` """
mykeys = super(InfoRouter, self).iterkeys
sub_keys = (i.iterkeys for i in self.sub_providers)
return flatten(it.chain([mykeys], sub_keys))