def start_munge(args):
"""Start munge service on all nodes.
Args:
args (Namespace): Commandline arguments
"""
sudo = "sudo" if args.sudo else ""
all_nodes = NodeSet("{},{}".format(str(args.control), str(args.nodes)))
# exclude the control node
nodes = NodeSet(str(args.nodes))
nodes.difference_update(str(args.control))
# copy key to all nodes FROM slurmctl node;
# change the protections/ownership on the munge dir on all nodes
cmd_list = [
"{0} chmod -R 777 /etc/munge; {0} chown {1}. /etc/munge".format(
sudo, args.user)
]
if execute_cluster_cmds(all_nodes, cmd_list) > 0:
return 1
# Check if file exists on slurm control node
# change the protections/ownership on the munge key before copying
cmd_list = [
"set -Eeu", "rc=0", "if [ ! -f /etc/munge/munge.key ]",
"then {} create-munge-key".format(sudo), "fi",
"{} chmod 777 /etc/munge/munge.key".format(sudo),
"{} chown {}. /etc/munge/munge.key".format(sudo, args.user)
]
if execute_cluster_cmds(args.control, ["; ".join(cmd_list)]) > 0:
return 1
# remove any existing key from other nodes
cmd_list = [
"{} rm -f /etc/munge/munge.key".format(sudo),
"scp -p {}:/etc/munge/munge.key /etc/munge/munge.key".format(
args.control)
]
if execute_cluster_cmds(nodes, ["; ".join(cmd_list)]) > 0:
return 1
# set the protection back to defaults
cmd_list = [
"{} chmod 400 /etc/munge/munge.key".format(sudo),
"{} chown munge. /etc/munge/munge.key".format(sudo),
"{} chmod 700 /etc/munge".format(sudo),
"{} chown munge. /etc/munge".format(sudo)
]
if execute_cluster_cmds(all_nodes, ["; ".join(cmd_list)]) > 0:
return 1
# Start Munge service on all nodes
all_nodes = NodeSet("{},{}".format(str(args.control), str(args.nodes)))
return execute_cluster_cmds(all_nodes, MUNGE_STARTUP, args.sudo)
def testBadTopologies(self):
"""test detecting invalid topologies"""
g = TopologyGraph()
admin = NodeSet('admin')
# Add the same nodeset twice
ns0 = NodeSet('nodes[0-9]')
ns1 = NodeSet('nodes[10-19]')
ns2 = NodeSet('nodes[20-29]')
g.add_route(admin, ns0)
g.add_route(ns0, ns1)
g.add_route(ns0, ns2)
# add a superset of a known destination as source
ns2_sup = NodeSet('somenode[0-10]')
ns2_sup.add(ns2)
self.assertRaises(TopologyError, g.add_route, ns2_sup, NodeSet('foo1'))
# Add a known dst nodeset as a src nodeset
ns3 = NodeSet('nodes[30-39]')
g.add_route(ns1, ns3)
# Add a subset of a known src nodeset as src
ns0_sub = NodeSet(','.join(ns0[:3:]))
ns4 = NodeSet('nodes[40-49]')
g.add_route(ns0_sub, ns4)
# Add a subset of a known dst nodeset as src
ns1_sub = NodeSet(','.join(ns1[:3:]))
self.assertRaises(TopologyError, g.add_route, ns4, ns1_sub)
# Add a subset of a known src nodeset as dst
self.assertRaises(TopologyError, g.add_route, ns4, ns0_sub)
# Add a subset of a known dst nodeset as dst
self.assertRaises(TopologyError, g.add_route, ns4, ns1_sub)
# src <- subset of -> dst
ns5 = NodeSet('nodes[50-59]')
ns5_sub = NodeSet(','.join(ns5[:3:]))
self.assertRaises(TopologyError, g.add_route, ns5, ns5_sub)
self.assertRaises(TopologyError, g.add_route, ns5_sub, ns5)
self.assertEqual(g.dest(ns0), (ns1 | ns2))
self.assertEqual(g.dest(ns1), ns3)
self.assertEqual(g.dest(ns2), None)
self.assertEqual(g.dest(ns3), None)
self.assertEqual(g.dest(ns4), None)
self.assertEqual(g.dest(ns5), None)
self.assertEqual(g.dest(ns0_sub), (ns1 | ns2 | ns4))
g = TopologyGraph()
root = NodeSet('root')
ns01 = NodeSet('nodes[0-1]')
ns23 = NodeSet('nodes[2-3]')
ns45 = NodeSet('nodes[4-5]')
ns67 = NodeSet('nodes[6-7]')
ns89 = NodeSet('nodes[8-9]')
g.add_route(root, ns01)
g.add_route(root, ns23 | ns45)
self.assertRaises(TopologyError, g.add_route, ns23, ns23)
self.assertRaises(TopologyError, g.add_route, ns45, root)
g.add_route(ns23, ns67)
g.add_route(ns67, ns89)
self.assertRaises(TopologyError, g.add_route, ns89, ns67)
self.assertRaises(TopologyError, g.add_route, ns89, ns89)
self.assertRaises(TopologyError, g.add_route, ns89, ns23)
ns_all = NodeSet('root,nodes[0-9]')
for nodegroup in g.to_tree('root'):
ns_all.difference_update(nodegroup.nodeset)
self.assertEqual(len(ns_all), 0)
def testBadTopologies(self):
"""test detecting invalid topologies"""
g = TopologyGraph()
admin = NodeSet('admin')
# Add the same nodeset twice
ns0 = NodeSet('nodes[0-9]')
ns1 = NodeSet('nodes[10-19]')
ns2 = NodeSet('nodes[20-29]')
g.add_route(admin, ns0)
g.add_route(ns0, ns1)
g.add_route(ns0, ns2)
# add a superset of a known destination as source
ns2_sup = NodeSet('somenode[0-10]')
ns2_sup.add(ns2)
self.assertRaises(TopologyError, g.add_route, ns2_sup, NodeSet('foo1'))
# Add a known dst nodeset as a src nodeset
ns3 = NodeSet('nodes[30-39]')
g.add_route(ns1, ns3)
# Add a subset of a known src nodeset as src
ns0_sub = NodeSet(','.join(ns0[:3:]))
ns4 = NodeSet('nodes[40-49]')
g.add_route(ns0_sub, ns4)
# Add a subset of a known dst nodeset as src
ns1_sub = NodeSet(','.join(ns1[:3:]))
self.assertRaises(TopologyError, g.add_route, ns4, ns1_sub)
# Add a subset of a known src nodeset as dst
self.assertRaises(TopologyError, g.add_route, ns4, ns0_sub)
# Add a subset of a known dst nodeset as dst
self.assertRaises(TopologyError, g.add_route, ns4, ns1_sub)
# src <- subset of -> dst
ns5 = NodeSet('nodes[50-59]')
ns5_sub = NodeSet(','.join(ns5[:3:]))
self.assertRaises(TopologyError, g.add_route, ns5, ns5_sub)
self.assertRaises(TopologyError, g.add_route, ns5_sub, ns5)
self.assertEqual(g.dest(ns0), (ns1 | ns2))
self.assertEqual(g.dest(ns1), ns3)
self.assertEqual(g.dest(ns2), None)
self.assertEqual(g.dest(ns3), None)
self.assertEqual(g.dest(ns4), None)
self.assertEqual(g.dest(ns5), None)
self.assertEqual(g.dest(ns0_sub), (ns1 | ns2 | ns4))
g = TopologyGraph()
root = NodeSet('root')
ns01 = NodeSet('nodes[0-1]')
ns23 = NodeSet('nodes[2-3]')
ns45 = NodeSet('nodes[4-5]')
ns67 = NodeSet('nodes[6-7]')
ns89 = NodeSet('nodes[8-9]')
g.add_route(root, ns01)
g.add_route(root, ns23 | ns45)
self.assertRaises(TopologyError, g.add_route, ns23, ns23)
self.assertRaises(TopologyError, g.add_route, ns45, root)
g.add_route(ns23, ns67)
g.add_route(ns67, ns89)
self.assertRaises(TopologyError, g.add_route, ns89, ns67)
self.assertRaises(TopologyError, g.add_route, ns89, ns89)
self.assertRaises(TopologyError, g.add_route, ns89, ns23)
ns_all = NodeSet('root,nodes[0-9]')
for nodegroup in g.to_tree('root'):
ns_all.difference_update(nodegroup.nodeset)
self.assertEqual(len(ns_all), 0)
class TopologyNodeGroup(object):
"""Base element for in-memory representation of the propagation tree.
Contains a nodeset, with parent-children relationships with other
instances.
"""
def __init__(self, nodeset=None):
"""initialize a new TopologyNodeGroup instance."""
# Base nodeset
self.nodeset = nodeset
# Parent TopologyNodeGroup (TNG) instance
self.parent = None
# List of children TNG instances
self._children = []
self._children_len = 0
# provided for convenience
self._children_ns = None
def printable_subtree(self, prefix=''):
"""recursive method that returns a printable version the subtree from
the current node with a nice presentation
"""
res = ''
# For now, it is ok to use a recursive method here as we consider that
# tree depth is relatively small.
if self.parent is None:
# root
res = '%s\n' % str(self.nodeset)
elif self.parent.parent is None:
# first level
if not self._is_last():
res = '|- %s\n' % str(self.nodeset)
else:
res = '`- %s\n' % str(self.nodeset)
else:
# deepest levels...
if not self.parent._is_last():
prefix += '| '
else:
# fix last line
prefix += ' '
if not self._is_last():
res = '%s|- %s\n' % (prefix, str(self.nodeset))
else:
res = '%s`- %s\n' % (prefix, str(self.nodeset))
# perform recursive calls to print out every node
for child in self._children:
res += child.printable_subtree(prefix)
return res
def add_child(self, child):
"""add a child to the children list and define the current instance as
its parent
"""
assert isinstance(child, TopologyNodeGroup)
if child in self._children:
return
child.parent = self
self._children.append(child)
if self._children_ns is None:
self._children_ns = NodeSet()
self._children_ns.add(child.nodeset)
def clear_child(self, child, strict=False):
"""remove a child"""
try:
self._children.remove(child)
self._children_ns.difference_update(child.nodeset)
if len(self._children_ns) == 0:
self._children_ns = None
except ValueError:
if strict:
raise
def clear_children(self):
"""delete all children"""
self._children = []
self._children_ns = None
def children(self):
"""get the children list"""
return self._children
def children_ns(self):
"""return the children as a nodeset"""
return self._children_ns
def children_len(self):
"""returns the number of children as the sum of the size of the
children's nodeset
"""
if self._children_ns is None:
return 0
else:
return len(self._children_ns)
def _is_last(self):
"""used to display the subtree: we won't prefix the line the same way if
the current instance is the last child of the children list of its
parent.
"""
return self.parent._children[-1::][0] == self
def __str__(self):
"""printable representation of the nodegroup"""
return '<TopologyNodeGroup (%s)>' % str(self.nodeset)
class BaseEntity(object):
'''
This class is abstract and shall not be instanciated.
A BaseEntity object basically represents a node of graph with reference
on parents and children.
'''
LOCAL_VARIABLES = {
'NAME': 'name',
'FANOUT': 'fanout',
'TIMEOUT': 'timeout',
'TARGET': 'target',
'DESC': 'desc',
}
def __init__(self, name, target=None, delay=0):
# Entity name
self.name = name
# Each entity has a status which it state
self.status = NO_STATUS
# Description of an entity
self.desc = None
# Maximum window for parallelism. A None fanout means
# that the task will be limited by the default value of
# ClusterShell 64
self.fanout = -1
# Nodes on which the entity is launched
self._target = None
self.target = target
self._target_backup = self.target
# Special mode which change entity behaviour
# 'delegate' means manage targets but run localy.
self.mode = None
# Maximum error authorized for the entity.
self.errors = 0
# Error threshold before reaching the warning status
# (should be <= self.errors)
self.warnings = 0
# Max time allowed to compute an entity, -1 means no timeout
self.timeout = -1
# Delay to wait before launching an action
self.delay = delay
self.maxretry = 0
# Parent of the current object. Must be a subclass of BaseEntity
self.parent = None
# Parents dependencies (e.g A->B so B is the parent of A)
self.parents = {}
# Children dependencies (e.g A<-B) so A is a child of B)
self.children = {}
self.simulate = False
# Agorithm's direction used
# False : go in parent's direction
# True : go in children direction
self._algo_reversed = False
# Tag the entity. By this way we know if the entity have to be
# call by her dependencies
self._tagged = False
# Variables
self.variables = {}
def add_var(self, varname, value):
'''Add a new variable within the entity context'''
if varname not in self.variables:
self.variables[varname] = value
else:
raise VariableAlreadyExistError
def remove_var(self, varname):
'''Remove an existing var from the entity'''
if varname in self.variables:
del self.variables[varname]
def update_target(self, nodeset, mode=None):
'''Update the attribute target of an entity'''
assert nodeset is not None
if not mode:
self.target = NodeSet(nodeset)
elif mode is 'DIF' and self.target:
self.target.difference_update(nodeset)
elif mode is 'INT' and self.target:
self.target.intersection_update(nodeset)
def _get_target(self):
'''Return self._target'''
return self._target
def _set_target(self, value):
'''Assign nodeset to _target'''
self._target = None
if value is not None:
self._target = NodeSet(self._resolve(value))
target = property(fset=_set_target, fget=_get_target)
def reset(self):
'''Reset values of attributes in order to perform multiple exec.'''
self._tagged = False
self.target = self._target_backup
self.status = NO_STATUS
self.algo_reversed = False
def search(self, name, reverse=False):
'''
Search an entity through the overall graph. This recursive algorithm
stops as soon as the node searched is reached.
'''
target = None
deps = self.parents
if reverse:
deps = self.children
if name in deps:
return deps[name].target
else:
for dep in deps.values():
target = dep.target.search(name, reverse)
if target:
return target
return target
def add_dep(self, target, sgth=REQUIRE, parent=True):
'''
Add a dependency in both direction. This method allow the user to
specify the dependency type. It is also possible to specify that
the target is the parent or the child of the current entity.
'''
assert target, "target must not be None"
if sgth in (CHECK, REQUIRE, REQUIRE_WEAK):
if parent:
if target.name in self.parents:
raise DependencyAlreadyReferenced()
else:
# This dependency is considered as a parent
self.parents[target.name] = Dependency(target, sgth, False)
target.children[self.name] = Dependency(self, sgth, False)
else:
if target.name in self.children:
raise DependencyAlreadyReferenced()
else:
# This dependency is considered as a child
self.children[target.name] = Dependency(target, sgth, False)
target.parents[self.name] = Dependency(self, sgth, False)
else:
raise IllegalDependencyTypeError()
def remove_dep(self, dep_name, parent=True):
'''
Remove a dependency on both side, in the current object and in the
target object concerned by the dependency.
'''
assert dep_name, "Dependency specified must not be None"
if parent and dep_name in self.parents:
dep = self.parents[dep_name]
del self.parents[dep_name]
del dep.target.children[self.name]
elif dep_name in self.children:
dep = self.children[dep_name]
del self.children[dep_name]
del dep.target.parents[self.name]
def clear_parent_deps(self):
'''Remove all parent dependencies of an entity'''
for dpname in self.parents.keys():
self.remove_dep(dpname)
def clear_child_deps(self):
'''Remove all child dependencies of an entity'''
for dpname in self.children.keys():
self.remove_dep(dep_name=dpname, parent=False)
def has_child_dep(self, dep_name=None):
'''
Determine whether the current object has a child dependency called
dep_name.
'''
return dep_name in self.children
def has_parent_dep(self, dep_name=None):
'''
Determine whether the current object has a parent dependency called
dep_name
'''
return dep_name in self.parents
def clear_deps(self):
'''Clear parent/child dependencies.'''
self.parents.clear()
self.children.clear()
def deps(self):
"""
Return parent dependency list.
Return children deps as parent if algo is reversed.
"""
if self._algo_reversed:
return self.children
else:
return self.parents
def is_ready(self):
'''
Determine if the current services has to wait before to
start due to unterminated dependencies.
'''
for dep in self.deps().values():
if dep.target.status in (NO_STATUS, WAITING_STATUS):
return False
return True
def search_deps(self, symbols=None):
'''
Look for parent/child dependencies matching to the symbols. The
search direction depends on the direction specified for the entiy.
'''
# No selection criteria, return everything
if not symbols:
return self.deps().values()
# Else, only keep matching deps
else:
dep_list = self.deps().values()
return [dep for dep in dep_list if dep.target.status in symbols]
def graph_info(self):
""" Return a tuple to manage dependencies output """
return (self.fullname(), None)
def graph(self, excluded=None):
""" Generate a graph of dependencies"""
grph = ""
# If the entity has a no dependency we just return the entity fullname
if not self.deps().values():
grph += '"%s";\n' % self.fullname()
else:
for dep in self.deps().values():
if not dep.target.excluded(excluded):
if not dep.target.simulate:
grph += dep.graph(self)
else:
grph += '"%s";\n' % self.fullname()
return grph
def excluded(self, excluded=None):
"""Is the entity ecluded recusively"""
if not excluded:
return False
if not self.deps().values():
return self.fullname() in excluded
# FIXME: Better loop detection
if self.search(self.name):
return True
for dep in self.deps().values():
if dep.target.excluded(excluded):
return True
return self.fullname() in excluded
def eval_deps_status(self):
'''
Evaluate the result of the dependencies in order to check to establish
a status.
'''
if len(self.deps()):
order = lambda dep: DEP_ORDER[dep.status()]
sorted_deps = sorted(self.deps().values(), key=order)
return sorted_deps[-1].status()
else:
return MISSING
def set_algo_reversed(self, flag):
'''Assign the right values for the property algo_reversed'''
self._algo_reversed = flag
algo_reversed = property(fset=set_algo_reversed)
def longname(self):
'''Return entity fullname and descrition if available '''
label = self.fullname()
if self.desc:
label += " - %s" % self.desc
return label
def fullname(self):
'''Return the fullname of the current entity'''
names = []
if self.parent:
names.append(self.parent.fullname())
names.append(self.name)
return '.'.join(names)
def _lookup_variable(self, varname):
'''
Return the value of the specified variable name.
If is not found in current object, it searches recursively in the
parent object.
If it cannot solve the variable name, it raises UndefinedVariableError.
'''
from MilkCheck.ServiceManager import service_manager_self
if varname in self.variables:
return self.variables[varname]
elif varname.upper() in self.LOCAL_VARIABLES:
value = self.LOCAL_VARIABLES[varname.upper()]
return self.resolve_property(value)
elif self.parent:
return self.parent._lookup_variable(varname)
elif varname in service_manager_self().variables:
return service_manager_self().variables[varname]
else:
raise UndefinedVariableError(varname)
def _substitute(self, template):
"""Substitute %xxx patterns from the provided template."""
delimiter = '%'
pattern = r"""
%(delim)s(?:
(?P<escaped>%(delim)s) | # Escape sequence of two delimiters
(?P<named>%(id)s) | # delimiter and a Python identifier
{(?P<braced>%(id)s)} | # delimiter and a braced identifier
\((?P<parenth>.+?)\) | # delimiter and parenthesis
(?P<invalid>) # Other ill-formed delimiter exprs
)""" % {
'delim' : delimiter,
'id' : r'[_a-z][_a-z0-9]*',
}
pattern = re.compile(pattern, re.IGNORECASE | re.VERBOSE)
# Command substitution
def _cmd_repl(raw):
'''Replace a command execution pattern by its result.'''
logger = logging.getLogger('milkcheck')
cmd = Popen(raw, stdout=PIPE, stderr=PIPE, shell=True)
stdout = cmd.communicate()[0]
logger.debug("External command exited with %d: '%s'" %
(cmd.returncode, stdout))
if cmd.returncode >= 126:
raise InvalidVariableError(raw)
return stdout.rstrip('\n')
def _invalid(mobj, template):
'''Helper to raise a detail error message'''
i = mobj.start('invalid')
lines = template[:i].splitlines(True)
# With the current regexp, it is impossible that lines is empty.
assert lines, "invalid pattern as the begining of template"
colno = i - len(''.join(lines[:-1]))
lineno = len(lines)
raise ValueError('Invalid placeholder in string: line %d, col %d' %
(lineno, colno))
def _convert(mobj):
"""Helper function for .sub()"""
# Check the mobjst commobjn path first.
named = mobj.group('named') or mobj.group('braced')
if named is not None:
val = str(self._lookup_variable(named))
return self._resolve(val)
if mobj.group('escaped') is not None:
return delimiter
if mobj.group('parenth') is not None:
val = self._resolve(mobj.group('parenth'))
return _cmd_repl(val)
if mobj.group('invalid') is not None:
_invalid(mobj, template)
raise ValueError('Unrecognized named group in pattern', pattern)
return pattern.sub(_convert, template)
def _resolve(self, value):
'''
This method takes a string containing symbols. Those strings may
look like to :
+ %(nodeset -f epsilon[5-8] -x epsilon7)
+ %CMD echo %(nodeset -f epsilon[5-8])
+ ps -e | grep myprogram
After computation this method return a string with all the symbols
resolved.
The '%' character could be inserted using '%%'.
'''
# For compat: if provided value is not a str, we should not convert
# it to a str if nothing matches.
if type(value) is not str:
return value
# Replace all %xxx patterns
origvalue = value
value = self._substitute(value)
# Debugging
if origvalue != value:
logger = logging.getLogger('milkcheck')
logger.info("Variable content '%s' replaced by '%s'",
origvalue, value)
return value
def resolve_property(self, prop):
'''
Resolve the variables contained within the property. It proceeds by
looking for the values required to replace the symbols. This method
returns None whether the property does not exist.
'''
pvalue = None
if hasattr(self, prop):
pvalue = self._resolve(getattr(self, prop))
return pvalue
def inherits_from(self, entity):
'''Inheritance of properties between entities'''
# Beware to check the default value of all of theses properties.
# Some of theses have a two possible 'false' value (None or '').
# * The init value should always be None
# * '' is set by the user
if self.fanout <= -1 and entity.fanout:
self.fanout = entity.fanout
self.errors = self.errors or entity.errors
self.warnings = self.warnings or entity.warnings
if self.timeout is not None and self.timeout <= -1 and \
entity.timeout >= 0:
self.timeout = entity.timeout
if self.target is None:
self.target = entity.target
self.mode = self.mode or entity.mode
if self.desc is None:
self.desc = entity.desc
self.delay = self.delay or entity.delay
self.maxretry = self.maxretry or entity.maxretry
def fromdict(self, entdict):
"""Populate entity attributes from dict."""
for item, prop in entdict.items():
if item == 'target':
self.target = prop
self._target_backup = prop
elif item == 'mode':
self.mode = prop
elif item == 'fanout':
self.fanout = prop
elif item == 'timeout':
self.timeout = prop
elif item == 'delay':
self.delay = prop
elif item == 'retry':
self.maxretry = prop
elif item == 'errors':
self.errors = prop
elif item == 'warnings':
self.warnings = prop
elif item == 'desc':
self.desc = prop
elif item == 'variables':
for varname, value in prop.items():
self.add_var(varname, value)
def ttyloop(task, nodeset, timeout, display, remote):
"""Manage the interactive prompt to run command"""
readline_avail = False
interactive = task.default("USER_interactive")
if interactive:
try:
import readline
readline_setup()
readline_avail = True
except ImportError:
pass
display.vprint(VERB_STD, \
"Enter 'quit' to leave this interactive mode")
rc = 0
ns = NodeSet(nodeset)
ns_info = True
cmd = ""
while task.default("USER_running") or \
(interactive and cmd.lower() != 'quit'):
try:
# Set SIGUSR1 handler if needed
if task.default("USER_handle_SIGUSR1"):
signal.signal(signal.SIGUSR1, signal_handler)
if task.default("USER_interactive") and \
not task.default("USER_running"):
if ns_info:
display.vprint(VERB_QUIET, \
"Working with nodes: %s" % ns)
ns_info = False
prompt = "clush> "
else:
prompt = ""
try:
cmd = raw_input(prompt)
assert cmd is not None, "Result of raw_input() is None!"
finally:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
except EOFError:
print()
return
except UpdatePromptException:
if task.default("USER_interactive"):
continue
return
except KeyboardInterrupt as kbe:
# Caught SIGINT here (main thread) but the signal will also reach
# subprocesses (that will most likely kill them)
if display.gather:
# Suspend task, so we can safely access its data from here
task.suspend()
# If USER_running is not set, the task had time to finish,
# that could mean all subprocesses have been killed and all
# handlers have been processed.
if not task.default("USER_running"):
# let's clush_excepthook handle the rest
raise kbe
# If USER_running is set, the task didn't have time to finish
# its work, so we must print something for the user...
print_warn = False
# Display command output, but cannot order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
for buf, nodeset in sorted(map(nodesetify,
task.iter_buffers()),
key=bufnodeset_cmpkey):
if not print_warn:
print_warn = True
display.vprint_err(VERB_STD, \
"Warning: Caught keyboard interrupt!")
display.print_gather(nodeset, buf)
# Return code handling
verbexit = VERB_QUIET
if display.maxrc:
verbexit = VERB_STD
ns_ok = NodeSet()
for rc, nodelist in task.iter_retcodes():
ns_ok.add(NodeSet._fromlist1(nodelist))
if rc != 0:
# Display return code if not ok ( != 0)
nsdisp = ns = NodeSet._fromlist1(nodelist)
if display.verbosity >= VERB_QUIET and len(ns) > 1:
nsdisp = "%s (%d)" % (ns, len(ns))
msgrc = "clush: %s: exited with exit code %d" % (
nsdisp, rc)
display.vprint_err(verbexit, msgrc)
# Add uncompleted nodeset to exception object
kbe.uncompleted_nodes = ns - ns_ok
# Display nodes that didn't answer within command timeout delay
if task.num_timeout() > 0:
display.vprint_err(verbexit, \
"clush: %s: command timeout" % \
NodeSet._fromlist1(task.iter_keys_timeout()))
raise kbe
if task.default("USER_running"):
ns_reg, ns_unreg = NodeSet(), NodeSet()
for client in task._engine.clients():
if client.registered:
ns_reg.add(client.key)
else:
ns_unreg.add(client.key)
if ns_unreg:
pending = "\nclush: pending(%d): %s" % (len(ns_unreg),
ns_unreg)
else:
pending = ""
display.vprint_err(VERB_QUIET,
"clush: interrupt (^C to abort task)")
gws = list(task.gateways)
if not gws:
display.vprint_err(
VERB_QUIET, "clush: in progress(%d): %s%s" %
(len(ns_reg), ns_reg, pending))
else:
display.vprint_err(
VERB_QUIET, "clush: in progress(%d): %s%s\n"
"clush: [tree] open gateways(%d): %s" %
(len(ns_reg), ns_reg, pending, len(gws),
NodeSet._fromlist1(gws)))
for gw, (chan, metaworkers) in task.gateways.items():
act_targets = NodeSet.fromlist(mw.gwtargets[gw]
for mw in metaworkers)
if act_targets:
display.vprint_err(
VERB_QUIET, "clush: [tree] in progress(%d) on %s: %s" %
(len(act_targets), gw, act_targets))
else:
cmdl = cmd.lower()
try:
ns_info = True
if cmdl.startswith('+'):
ns.update(cmdl[1:])
elif cmdl.startswith('-'):
ns.difference_update(cmdl[1:])
elif cmdl.startswith('@'):
ns = NodeSet(cmdl[1:])
elif cmdl == '=':
display.gather = not display.gather
if display.gather:
display.vprint(VERB_STD, \
"Switching to gathered output format")
else:
display.vprint(VERB_STD, \
"Switching to standard output format")
task.set_default("stdout_msgtree", \
display.gather or display.line_mode)
ns_info = False
continue
elif not cmdl.startswith('?'): # if ?, just print ns_info
ns_info = False
except NodeSetParseError:
display.vprint_err(VERB_QUIET, \
"clush: nodeset parse error (ignoring)")
if ns_info:
continue
if cmdl.startswith('!') and len(cmd.strip()) > 0:
run_command(task, cmd[1:], None, timeout, display, remote)
elif cmdl != "quit":
if not cmd:
continue
if readline_avail:
readline.write_history_file(get_history_file())
run_command(task, cmd, ns, timeout, display, remote)
return rc
def ttyloop(task, nodeset, timeout, display, remote):
"""Manage the interactive prompt to run command"""
readline_avail = False
interactive = task.default("USER_interactive")
if interactive:
try:
import readline
readline_setup()
readline_avail = True
except ImportError:
pass
display.vprint(VERB_STD, \
"Enter 'quit' to leave this interactive mode")
rc = 0
ns = NodeSet(nodeset)
ns_info = True
cmd = ""
while task.default("USER_running") or \
(interactive and cmd.lower() != 'quit'):
try:
# Set SIGUSR1 handler if needed
if task.default("USER_handle_SIGUSR1"):
signal.signal(signal.SIGUSR1, signal_handler)
if task.default("USER_interactive") and \
not task.default("USER_running"):
if ns_info:
display.vprint(VERB_QUIET, \
"Working with nodes: %s" % ns)
ns_info = False
prompt = "clush> "
else:
prompt = ""
try:
cmd = raw_input(prompt)
assert cmd is not None, "Result of raw_input() is None!"
finally:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
except EOFError:
print()
return
except UpdatePromptException:
if task.default("USER_interactive"):
continue
return
except KeyboardInterrupt as kbe:
# Caught SIGINT here (main thread) but the signal will also reach
# subprocesses (that will most likely kill them)
if display.gather:
# Suspend task, so we can safely access its data from here
task.suspend()
# If USER_running is not set, the task had time to finish,
# that could mean all subprocesses have been killed and all
# handlers have been processed.
if not task.default("USER_running"):
# let's clush_excepthook handle the rest
raise kbe
# If USER_running is set, the task didn't have time to finish
# its work, so we must print something for the user...
print_warn = False
# Display command output, but cannot order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
for buf, nodeset in sorted(map(nodesetify, task.iter_buffers()),
key=bufnodeset_cmpkey):
if not print_warn:
print_warn = True
display.vprint_err(VERB_STD, \
"Warning: Caught keyboard interrupt!")
display.print_gather(nodeset, buf)
# Return code handling
verbexit = VERB_QUIET
if display.maxrc:
verbexit = VERB_STD
ns_ok = NodeSet()
for rc, nodelist in task.iter_retcodes():
ns_ok.add(NodeSet._fromlist1(nodelist))
if rc != 0:
# Display return code if not ok ( != 0)
nsdisp = ns = NodeSet._fromlist1(nodelist)
if display.verbosity >= VERB_QUIET and len(ns) > 1:
nsdisp = "%s (%d)" % (ns, len(ns))
msgrc = "clush: %s: exited with exit code %d" % (nsdisp,
rc)
display.vprint_err(verbexit, msgrc)
# Add uncompleted nodeset to exception object
kbe.uncompleted_nodes = ns - ns_ok
# Display nodes that didn't answer within command timeout delay
if task.num_timeout() > 0:
display.vprint_err(verbexit, \
"clush: %s: command timeout" % \
NodeSet._fromlist1(task.iter_keys_timeout()))
raise kbe
if task.default("USER_running"):
ns_reg, ns_unreg = NodeSet(), NodeSet()
for client in task._engine.clients():
if client.registered:
ns_reg.add(client.key)
else:
ns_unreg.add(client.key)
if ns_unreg:
pending = "\nclush: pending(%d): %s" % (len(ns_unreg), ns_unreg)
else:
pending = ""
display.vprint_err(VERB_QUIET,
"clush: interrupt (^C to abort task)")
gws = list(task.gateways)
if not gws:
display.vprint_err(VERB_QUIET,
"clush: in progress(%d): %s%s"
% (len(ns_reg), ns_reg, pending))
else:
display.vprint_err(VERB_QUIET,
"clush: in progress(%d): %s%s\n"
"clush: [tree] open gateways(%d): %s"
% (len(ns_reg), ns_reg, pending,
len(gws), NodeSet._fromlist1(gws)))
for gw, (chan, metaworkers) in task.gateways.items():
act_targets = NodeSet.fromlist(mw.gwtargets[gw]
for mw in metaworkers)
if act_targets:
display.vprint_err(VERB_QUIET,
"clush: [tree] in progress(%d) on %s: %s"
% (len(act_targets), gw, act_targets))
else:
cmdl = cmd.lower()
try:
ns_info = True
if cmdl.startswith('+'):
ns.update(cmdl[1:])
elif cmdl.startswith('-'):
ns.difference_update(cmdl[1:])
elif cmdl.startswith('@'):
ns = NodeSet(cmdl[1:])
elif cmdl == '=':
display.gather = not display.gather
if display.gather:
display.vprint(VERB_STD, \
"Switching to gathered output format")
else:
display.vprint(VERB_STD, \
"Switching to standard output format")
task.set_default("stdout_msgtree", \
display.gather or display.line_mode)
ns_info = False
continue
elif not cmdl.startswith('?'): # if ?, just print ns_info
ns_info = False
except NodeSetParseError:
display.vprint_err(VERB_QUIET, \
"clush: nodeset parse error (ignoring)")
if ns_info:
continue
if cmdl.startswith('!') and len(cmd.strip()) > 0:
run_command(task, cmd[1:], None, timeout, display, remote)
elif cmdl != "quit":
if not cmd:
continue
if readline_avail:
readline.write_history_file(get_history_file())
run_command(task, cmd, ns, timeout, display, remote)
return rc
class TopologyNodeGroup(object):
"""Base element for in-memory representation of the propagation tree.
Contains a nodeset, with parent-children relationships with other
instances.
"""
def __init__(self, nodeset=None):
"""initialize a new TopologyNodeGroup instance."""
# Base nodeset
self.nodeset = nodeset
# Parent TopologyNodeGroup (TNG) instance
self.parent = None
# List of children TNG instances
self._children = []
self._children_len = 0
# provided for convenience
self._children_ns = None
def printable_subtree(self, prefix=''):
"""recursive method that returns a printable version the subtree from
the current node with a nice presentation
"""
res = ''
# For now, it is ok to use a recursive method here as we consider that
# tree depth is relatively small.
if self.parent is None:
# root
res = '%s\n' % str(self.nodeset)
elif self.parent.parent is None:
# first level
if not self._is_last():
res = '|- %s\n' % str(self.nodeset)
else:
res = '`- %s\n' % str(self.nodeset)
else:
# deepest levels...
if not self.parent._is_last():
prefix += '| '
else:
# fix last line
prefix += ' '
if not self._is_last():
res = '%s|- %s\n' % (prefix, str(self.nodeset))
else:
res = '%s`- %s\n' % (prefix, str(self.nodeset))
# perform recursive calls to print out every node
for child in self._children:
res += child.printable_subtree(prefix)
return res
def add_child(self, child):
"""add a child to the children list and define the current instance as
its parent
"""
assert isinstance(child, TopologyNodeGroup)
if child in self._children:
return
child.parent = self
self._children.append(child)
if self._children_ns is None:
self._children_ns = NodeSet()
self._children_ns.add(child.nodeset)
def clear_child(self, child, strict=False):
"""remove a child"""
try:
self._children.remove(child)
self._children_ns.difference_update(child.nodeset)
if len(self._children_ns) == 0:
self._children_ns = None
except ValueError:
if strict:
raise
def clear_children(self):
"""delete all children"""
self._children = []
self._children_ns = None
def children(self):
"""get the children list"""
return self._children
def children_ns(self):
"""return the children as a nodeset"""
return self._children_ns
def children_len(self):
"""returns the number of children as the sum of the size of the
children's nodeset
"""
if self._children_ns is None:
return 0
else:
return len(self._children_ns)
def _is_last(self):
"""used to display the subtree: we won't prefix the line the same way if
the current instance is the last child of the children list of its
parent.
"""
return self.parent._children[-1::][0] == self
def __str__(self):
"""printable representation of the nodegroup"""
return '<TopologyNodeGroup (%s)>' % str(self.nodeset)
