def display(tree, disp, gather, trace_mode, enable_nodeset_key):
"""nicely display MsgTree instance `tree' content according to
`disp' Display object and `gather' boolean flag"""
out = sys.stdout
try:
if trace_mode:
display_tree(tree, disp, out)
else:
if gather:
if enable_nodeset_key:
# lambda to create a NodeSet from keys returned by walk()
ns_getter = lambda x: NodeSet.fromlist(x[1])
for nodeset in sorted(imap(ns_getter, tree.walk()),
cmp=nodeset_cmp):
disp.print_gather(nodeset, tree[nodeset[0]])
else:
for msg, key in tree.walk():
disp.print_gather_keys(key, msg)
else:
if enable_nodeset_key:
# nodes are automagically sorted by NodeSet
for node in NodeSet.fromlist(tree.keys()).nsiter():
disp.print_gather(node, tree[str(node)])
else:
for key in tree.keys():
disp.print_gather_keys([key], tree[key])
finally:
out.flush()
def __init__(self, display, nodes):
assert nodes is not None, "cannot gather local command"
GatherOutputHandler.__init__(self, display)
self._nodes = NodeSet(nodes)
self._nodecnt = dict.fromkeys(self._nodes, 0)
self._mtreeq = []
self._offload = 0
def ev_close(self, worker):
# Worker is closing -- it's time to gather results...
self._runtimer_finalize(worker)
# Display command output, try to order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
cleaned = False
for _rc, nodelist in sorted(worker.iter_retcodes()):
ns_remain = NodeSet._fromlist1(nodelist)
# Then order by node/nodeset (see bufnodeset_cmp)
for buf, nodeset in sorted(map(nodesetify,
worker.iter_buffers(nodelist)),
cmp=bufnodeset_cmp):
if not cleaned:
# clean runtimer line before printing first result
self._runtimer_clean()
cleaned = True
self._display.print_gather(nodeset, buf)
ns_remain.difference_update(nodeset)
if ns_remain:
self._display.print_gather_finalize(ns_remain)
self._display.flush()
self._close_common(worker)
# Notify main thread to update its prompt
self.update_prompt(worker)
def display(tree, disp, gather, trace_mode, enable_nodeset_key):
"""nicely display MsgTree instance `tree' content according to
`disp' Display object and `gather' boolean flag"""
out = sys.stdout
try:
if trace_mode:
display_tree(tree, disp, out)
else:
if gather:
if enable_nodeset_key:
# lambda to create a NodeSet from keys returned by walk()
ns_getter = lambda x: NodeSet.fromlist(x[1])
for nodeset in sorted(imap(ns_getter, tree.walk()),
cmp=nodeset_cmp):
disp.print_gather(nodeset, tree[nodeset[0]])
else:
for msg, key in tree.walk():
disp.print_gather_keys(key, msg)
else:
if enable_nodeset_key:
# nodes are automagically sorted by NodeSet
for node in NodeSet.fromlist(tree.keys()).nsiter():
disp.print_gather(node, tree[str(node)])
else:
for key in tree.keys():
disp.print_gather_keys([ key ], tree[key])
finally:
out.flush()
def update(self):
"""Update runtime progress info"""
wrbwinfo = ''
if self.bytes_written > 0:
bandwidth = self.bytes_written / (time.time() - self.start_time)
wrbwinfo = " write: %s/s" % human_bi_bytes_unit(bandwidth)
gws = self.task.gateways.keys()
if gws:
# tree mode
act_targets = NodeSet()
for gw, (chan, metaworkers) in self.task.gateways.iteritems():
act_targets.updaten(mw.gwtargets[gw] for mw in metaworkers)
cnt = len(act_targets) + len(
self.task._engine.clients()) - len(gws)
gwinfo = ' gw %d' % len(gws)
else:
cnt = len(self.task._engine.clients())
gwinfo = ''
if self.bytes_written > 0 or cnt != self.cnt_last:
self.cnt_last = cnt
# display completed/total clients
towrite = 'clush: %*d/%*d%s%s\r' % (self.tslen, self.total - cnt,
self.tslen, self.total, gwinfo,
wrbwinfo)
self.wholelen = len(towrite)
sys.stderr.write(towrite)
self.started = True
def update(self):
"""Update runtime progress info"""
wrbwinfo = ''
if self.bytes_written > 0:
bandwidth = self.bytes_written/(time.time() - self.start_time)
wrbwinfo = " write: %s/s" % human_bi_bytes_unit(bandwidth)
gws = self.task.gateways.keys()
if gws:
# tree mode
act_targets = NodeSet()
for gw, (chan, metaworkers) in self.task.gateways.iteritems():
act_targets.updaten(mw.gwtargets[gw] for mw in metaworkers)
cnt = len(act_targets) + len(self.task._engine.clients()) - len(gws)
gwinfo = ' gw %d' % len(gws)
else:
cnt = len(self.task._engine.clients())
gwinfo = ''
if self.bytes_written > 0 or cnt != self.cnt_last:
self.cnt_last = cnt
# display completed/total clients
towrite = 'clush: %*d/%*d%s%s\r' % (self.tslen, self.total - cnt,
self.tslen, self.total, gwinfo,
wrbwinfo)
self.wholelen = len(towrite)
sys.stderr.write(towrite)
self.started = True
def ev_close(self, worker):
# Worker is closing -- it's time to gather results...
self._runtimer_finalize(worker)
# Display command output, try to order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
cleaned = False
for _rc, nodelist in sorted(worker.iter_retcodes()):
ns_remain = NodeSet._fromlist1(nodelist)
# Then order by node/nodeset (see bufnodeset_cmp)
for buf, nodeset in sorted(map(nodesetify,
worker.iter_buffers(nodelist)),
cmp=bufnodeset_cmp):
if not cleaned:
# clean runtimer line before printing first result
self._runtimer_clean()
cleaned = True
self._display.print_gather(nodeset, buf)
ns_remain.difference_update(nodeset)
if ns_remain:
self._display.print_gather_finalize(ns_remain)
self._display.flush()
self._close_common(worker)
# Notify main thread to update its prompt
self.update_prompt(worker)
class LiveGatherOutputHandler(GatherOutputHandler):
"""Live line-gathered output event handler class."""
def __init__(self, display, nodes):
assert nodes is not None, "cannot gather local command"
GatherOutputHandler.__init__(self, display)
self._nodes = NodeSet(nodes)
self._nodecnt = dict.fromkeys(self._nodes, 0)
self._mtreeq = []
self._offload = 0
def ev_read(self, worker):
# Read new line from node
node = worker.current_node
self._nodecnt[node] += 1
cnt = self._nodecnt[node]
if len(self._mtreeq) < cnt:
self._mtreeq.append(MsgTree())
self._mtreeq[cnt - self._offload - 1].add(node, worker.current_msg)
self._live_line(worker)
def ev_hup(self, worker):
if self._mtreeq and worker.current_node not in self._mtreeq[0]:
# forget a node that doesn't answer to continue live line
# gathering anyway
self._nodes.remove(worker.current_node)
self._live_line(worker)
def _live_line(self, worker):
# if all nodes have replied, display gathered line
while self._mtreeq and len(self._mtreeq[0]) == len(self._nodes):
mtree = self._mtreeq.pop(0)
self._offload += 1
self._runtimer_clean()
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()),
cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._runtimer_set_dirty()
def ev_close(self, worker):
# Worker is closing -- it's time to gather results...
self._runtimer_finalize(worker)
for mtree in self._mtreeq:
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()),
cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._close_common(worker)
# Notify main thread to update its prompt
self.update_prompt(worker)
class LiveGatherOutputHandler(GatherOutputHandler):
"""Live line-gathered output event handler class."""
def __init__(self, display, nodes):
assert nodes is not None, "cannot gather local command"
GatherOutputHandler.__init__(self, display)
self._nodes = NodeSet(nodes)
self._nodecnt = dict.fromkeys(self._nodes, 0)
self._mtreeq = []
self._offload = 0
def ev_read(self, worker):
# Read new line from node
node = worker.current_node
self._nodecnt[node] += 1
cnt = self._nodecnt[node]
if len(self._mtreeq) < cnt:
self._mtreeq.append(MsgTree())
self._mtreeq[cnt - self._offload - 1].add(node, worker.current_msg)
self._live_line(worker)
def ev_hup(self, worker):
if self._mtreeq and worker.current_node not in self._mtreeq[0]:
# forget a node that doesn't answer to continue live line
# gathering anyway
self._nodes.remove(worker.current_node)
self._live_line(worker)
def _live_line(self, worker):
# if all nodes have replied, display gathered line
while self._mtreeq and len(self._mtreeq[0]) == len(self._nodes):
mtree = self._mtreeq.pop(0)
self._offload += 1
self._runtimer_clean()
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()),
cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._runtimer_set_dirty()
def ev_close(self, worker):
# Worker is closing -- it's time to gather results...
self._runtimer_finalize(worker)
for mtree in self._mtreeq:
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()),
cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._close_common(worker)
# Notify main thread to update its prompt
self.update_prompt(worker)
def _close_common(self, worker):
verbexit = VERB_QUIET
if self._display.maxrc:
verbexit = VERB_STD
# Display return code if not ok ( != 0)
for rc, nodelist in worker.iter_retcodes():
if rc != 0:
ns = NodeSet._fromlist1(nodelist)
self._display.vprint_err(verbexit, \
"clush: %s: exited with exit code %d" % (ns, rc))
# Display nodes that didn't answer within command timeout delay
if worker.num_timeout() > 0:
self._display.vprint_err(verbexit, "clush: %s: command timeout" % \
NodeSet._fromlist1(worker.iter_keys_timeout()))
def _close_common(self, worker):
verbexit = VERB_QUIET
if self._display.maxrc:
verbexit = VERB_STD
# Display return code if not ok ( != 0)
for rc, nodelist in worker.iter_retcodes():
if rc != 0:
ns = NodeSet._fromlist1(nodelist)
self._display.vprint_err(verbexit, \
"clush: %s: exited with exit code %d" % (ns, rc))
# Display nodes that didn't answer within command timeout delay
if worker.num_timeout() > 0:
self._display.vprint_err(verbexit, "clush: %s: command timeout" % \
NodeSet._fromlist1(worker.iter_keys_timeout()))
def command_list(options, xset):
"""List (-l/-ll/-lll) command handler."""
list_level = options.list
# list groups of some specified nodes?
if options.all or xset or \
options.and_nodes or options.sub_nodes or options.xor_nodes:
# When some node sets are provided as argument, the list command
# retrieves node groups these nodes belong to, thanks to the
# groups() method (new in 1.6). Note: stdin support is enabled
# when the '-' special character is encountered.
groups = xset.groups(options.groupsource, options.groupbase)
for group, (gnodes, inodes) in groups.iteritems():
if list_level == 1: # -l
print group
elif list_level == 2: # -ll
print "%s %s" % (group, inodes)
else: # -lll
print "%s %s %d/%d" % (group, inodes, len(inodes), \
len(gnodes))
return
# "raw" group list when no argument at all
for group in grouplist(options.groupsource):
if options.groupsource and not options.groupbase:
nsgroup = "@%s:%s" % (options.groupsource, group)
else:
nsgroup = "@%s" % group
if list_level == 1: # -l
print nsgroup
else:
nodes = NodeSet(nsgroup)
if list_level == 2: # -ll
print "%s %s" % (nsgroup, nodes)
else: # -lll
print "%s %s %d" % (nsgroup, nodes, len(nodes))
def __init__(self, display, nodes):
assert nodes is not None, "cannot gather local command"
GatherOutputHandler.__init__(self, display)
self._nodes = NodeSet(nodes)
self._nodecnt = dict.fromkeys(self._nodes, 0)
self._mtreeq = []
self._offload = 0
def print_source_groups(source, level, xset, opts):
"""
Print groups from a source, a level of verbosity and an optional
nodeset acting as a filter.
"""
# list groups of some specified nodes?
if opts.all or xset or opts.and_nodes or opts.sub_nodes or opts.xor_nodes:
# When some node sets are provided as argument, the list command
# retrieves node groups these nodes belong to, thanks to the
# groups() method.
# Note: stdin support is enabled when '-' is found.
groups = xset.groups(source, opts.groupbase)
for group, (gnodes, inodes) in groups.iteritems():
if level == 1:
print group
elif level == 2:
print "%s %s" % (group, inodes)
else:
print "%s %s %d/%d" % (group, inodes, len(inodes), len(gnodes))
else:
# "raw" group list when no argument at all
for group in grouplist(source):
if source and not opts.groupbase:
nsgroup = "@%s:%s" % (source, group)
else:
nsgroup = "@%s" % group
if level == 1:
print nsgroup
else:
nodes = NodeSet(nsgroup)
if level == 2:
print "%s %s" % (nsgroup, nodes)
else:
print "%s %s %d" % (nsgroup, nodes, len(nodes))
def clubak():
"""script subroutine"""
# Argument management
parser = OptionParser("%prog [options]")
parser.install_display_options(verbose_options=True,
separator_option=True,
dshbak_compat=True,
msgtree_mode=True)
options = parser.parse_args()[0]
if options.interpret_keys == THREE_CHOICES[-1]: # auto?
enable_nodeset_key = None # AUTO
else:
enable_nodeset_key = (options.interpret_keys == THREE_CHOICES[1])
# Create new message tree
if options.trace_mode:
tree_mode = MODE_TRACE
else:
tree_mode = MODE_DEFER
tree = MsgTree(mode=tree_mode)
fast_mode = options.fast_mode
if fast_mode:
if tree_mode != MODE_DEFER or options.line_mode:
parser.error("incompatible tree options")
preload_msgs = {}
# Feed the tree from standard input lines
for line in sys.stdin:
try:
linestripped = line.rstrip('\r\n')
if options.verbose or options.debug:
print "INPUT %s" % linestripped
key, content = linestripped.split(options.separator, 1)
key = key.strip()
if not key:
raise ValueError("no node found")
if enable_nodeset_key is False: # interpret-keys=never?
keyset = [key]
else:
try:
keyset = NodeSet(key)
except NodeSetParseError:
if enable_nodeset_key: # interpret-keys=always?
raise
enable_nodeset_key = False # auto => switch off
keyset = [key]
if fast_mode:
for node in keyset:
preload_msgs.setdefault(node, []).append(content)
else:
for node in keyset:
tree.add(node, content)
except ValueError, ex:
raise ValueError("%s (\"%s\")" % (ex, linestripped))
def update(self):
gws = self.task.gateways.keys()
if gws:
# tree mode
act_targets = NodeSet()
for gw, (chan, metaworkers) in self.task.gateways.iteritems():
act_targets.updaten(mw.gwtargets[gw] for mw in metaworkers)
cnt = len(act_targets) + len(self.task._engine.clients()) - len(gws)
gwinfo = " gw %d" % len(gws)
else:
cnt = len(self.task._engine.clients())
gwinfo = ""
if cnt != self.cnt_last:
self.cnt_last = cnt
# display completed/total clients
towrite = "clush: %*d/%*d%s\r" % (self.tslen, self.total - cnt, self.tslen, self.total, gwinfo)
self.wholelen = len(towrite)
sys.stderr.write(towrite)
self.started = True
def main():
"""clush script entry point"""
sys.excepthook = clush_excepthook
# Default values
nodeset_base, nodeset_exclude = NodeSet(), NodeSet()
#
# Argument management
#
usage = "%prog [options] command"
parser = OptionParser(usage)
parser.add_option("--nostdin",
action="store_true",
dest="nostdin",
help="don't watch for possible input from stdin")
parser.install_nodes_options()
parser.install_display_options(verbose_options=True)
parser.install_filecopy_options()
parser.install_ssh_options()
(options, args) = parser.parse_args()
#
# Load config file and apply overrides
#
config = ClushConfig(options)
# Should we use ANSI colors for nodes?
if config.color == "auto":
color = sys.stdout.isatty() and (options.gatherall or \
sys.stderr.isatty())
else:
color = config.color == "always"
try:
# Create and configure display object.
display = Display(options, config, color)
except ValueError, exc:
parser.error("option mismatch (%s)" % exc)
def _live_line(self, worker):
# if all nodes have replied, display gathered line
while self._mtreeq and len(self._mtreeq[0]) == len(self._nodes):
mtree = self._mtreeq.pop(0)
self._offload += 1
self._runtimer_clean()
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()), cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._runtimer_set_dirty()
def display_tree(tree, disp, out):
"""display sub-routine for clubak -T (msgtree trace mode)"""
togh = True
offset = 2
reldepth = -offset
reldepths = {}
line_mode = disp.line_mode
for msgline, keys, depth, nchildren in tree.walk_trace():
if togh:
if depth in reldepths:
reldepth = reldepths[depth]
else:
reldepth = reldepths[depth] = reldepth + offset
if line_mode:
out.write("%s:\n" % NodeSet.fromlist(keys))
else:
out.write("%s\n" % \
(disp.format_header(NodeSet.fromlist(keys), reldepth)))
out.write("%s%s\n" % (" " * reldepth, msgline))
togh = nchildren != 1
def display_tree(tree, disp, out):
"""display sub-routine for clubak -T (msgtree trace mode)"""
togh = True
offset = 2
reldepth = -offset
reldepths = {}
line_mode = disp.line_mode
for msgline, keys, depth, nchildren in tree.walk_trace():
if togh:
if depth in reldepths:
reldepth = reldepths[depth]
else:
reldepth = reldepths[depth] = reldepth + offset
if line_mode:
out.write("%s:\n" % NodeSet.fromlist(keys))
else:
out.write("%s\n" % \
(disp.format_header(NodeSet.fromlist(keys), reldepth)))
out.write("%s%s\n" % (" " * reldepth, msgline))
togh = nchildren != 1
def _live_line(self, worker):
# if all nodes have replied, display gathered line
while self._mtreeq and len(self._mtreeq[0]) == len(self._nodes):
mtree = self._mtreeq.pop(0)
self._offload += 1
self._runtimer_clean()
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()),
cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._runtimer_set_dirty()
def update(self):
gws = self.task.gateways.keys()
if gws:
# tree mode
act_targets = NodeSet()
for gw, (chan, metaworkers) in self.task.gateways.iteritems():
act_targets.updaten(mw.gwtargets[gw] for mw in metaworkers)
cnt = len(act_targets) + len(
self.task._engine.clients()) - len(gws)
gwinfo = ' gw %d' % len(gws)
else:
cnt = len(self.task._engine.clients())
gwinfo = ''
if cnt != self.cnt_last:
self.cnt_last = cnt
# display completed/total clients
towrite = 'clush: %*d/%*d%s\r' % (self.tslen, self.total - cnt,
self.tslen, self.total, gwinfo)
self.wholelen = len(towrite)
sys.stderr.write(towrite)
self.started = True
def ev_close(self, worker):
# Worker is closing -- it's time to gather results...
self._runtimer_finalize(worker)
for mtree in self._mtreeq:
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()), cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._close_common(worker)
# Notify main thread to update its prompt
self.update_prompt(worker)
def ev_close(self, worker):
# Worker is closing -- it's time to gather results...
self._runtimer_finalize(worker)
for mtree in self._mtreeq:
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
for buf, nodeset in sorted(map(nodesetify, mtree.walk()),
cmp=bufnodeset_cmp):
self._display.print_gather(nodeset, buf)
self._close_common(worker)
# Notify main thread to update its prompt
self.update_prompt(worker)
def display_tree(tree, disp, out):
"""display sub-routine for clubak -T (msgtree trace mode)"""
togh = True
offset = 2
reldepth = -offset
reldepths = {}
line_mode = disp.line_mode
for msgline, keys, depth, nchildren in tree.walk_trace():
if togh:
if depth in reldepths:
reldepth = reldepths[depth]
else:
reldepth = reldepths[depth] = reldepth + offset
nodeset = NodeSet.fromlist(keys)
if line_mode:
out.write(str(nodeset).encode() + b':\n')
else:
out.write(disp.format_header(nodeset, reldepth) + b'\n')
out.write(b' ' * reldepth + msgline + b'\n')
togh = nchildren != 1
def ev_close(self, worker):
# Worker is closing -- it's time to gather results...
self._runtimer_finalize(worker)
assert worker.current_node is not None, "cannot gather local command"
# Display command output, try to order buffers by rc
nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1]))
cleaned = False
for rc, nodelist in worker.iter_retcodes():
# Then order by node/nodeset (see bufnodeset_cmp)
for buf, nodeset in sorted(map(nodesetify,
worker.iter_buffers(nodelist)),
cmp=bufnodeset_cmp):
if not cleaned:
# clean runtimer line before printing first result
self._runtimer_clean()
cleaned = True
self._display.print_gather(nodeset, buf)
self._close_common(worker)
# Notify main thread to update its prompt
self.update_prompt(worker)
# Compute the nodeset and warn for possible use of shell pathname
# expansion (#225)
wnodelist = xnodelist = []
if options.nodes:
wnodelist = [NodeSet(nodes) for nodes in options.nodes]
if options.exclude:
xnodelist = [NodeSet(nodes) for nodes in options.exclude]
for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)):
for nodes in nodelist:
if len(nodes) == 1 and exists(str(nodes)):
display.vprint_err(VERB_STD, "Warning: using '-%s %s' and "
"local path '%s' exists, was it expanded "
"by the shell?" % (opt, nodes, nodes))
nodeset_base = NodeSet.fromlist(wnodelist)
nodeset_exclude = NodeSet.fromlist(xnodelist)
# FIXME: add public API to enforce engine
Task._std_default['engine'] = options.engine
# Do we have nodes group?
task = task_self()
task.set_info("debug", config.verbosity >= VERB_DEBUG)
if config.verbosity == VERB_DEBUG:
std_group_resolver().set_verbosity(1)
if options.nodes_all:
all_nodeset = NodeSet.fromall()
display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \
all_nodeset)
nodeset_base.add(all_nodeset)
wnodelist = [NodeSet(nodes) for nodes in options.nodes]
if options.exclude:
xnodelist = [NodeSet(nodes) for nodes in options.exclude]
for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)):
for nodes in nodelist:
if len(nodes) == 1 and exists(str(nodes)):
display.vprint_err(VERB_STD, "Warning: using '-%s %s' and "
"local path '%s' exists, was it expanded "
"by the shell?" % (opt, nodes, nodes))
# --hostfile support (#235)
for opt_hostfile in options.hostfile:
try:
fnodeset = NodeSet()
hostfile = open(opt_hostfile)
for line in hostfile.read().splitlines():
fnodeset.updaten(nodes for nodes in line.split())
hostfile.close()
display.vprint_err(VERB_DEBUG,
"Using nodeset %s from hostfile %s"
% (fnodeset, opt_hostfile))
wnodelist.append(fnodeset)
except IOError, exc:
# re-raise as OSError to be properly handled
errno, strerror = exc.args
raise OSError(errno, strerror, exc.filename)
# Instantiate target nodeset from command line and hostfile
nodeset_base = NodeSet.fromlist(wnodelist)
try:
# Create and configure display object.
display = Display(options, config, color)
except ValueError, exc:
parser.error("option mismatch (%s)" % exc)
if options.groupsource:
# Be sure -a/g -s source work as espected.
std_group_resolver().default_source_name = options.groupsource
# Compute the nodeset and warn for possible use of shell pathname
# expansion (#225)
wnodelist = []
xnodelist = []
if options.nodes:
wnodelist = [NodeSet(nodes) for nodes in options.nodes]
if options.exclude:
xnodelist = [NodeSet(nodes) for nodes in options.exclude]
for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)):
for nodes in nodelist:
if len(nodes) == 1 and exists(str(nodes)):
display.vprint_err(
VERB_STD, "Warning: using '-%s %s' and "
"local path '%s' exists, was it expanded "
"by the shell?" % (opt, nodes, nodes))
# --hostfile support (#235)
for opt_hostfile in options.hostfile:
try:
def ttyloop(task, nodeset, timeout, display):
"""Manage the interactive prompt to run command"""
readline_avail = False
if task.default("USER_interactive"):
try:
import readline
readline_setup()
readline_avail = True
except ImportError:
pass
display.vprint(VERB_STD, \
"Enter 'quit' to leave this interactive mode")
rc = 0
ns = NodeSet(nodeset)
ns_info = True
cmd = ""
while task.default("USER_running") or cmd.lower() != 'quit':
try:
if task.default("USER_interactive") and \
not task.default("USER_running"):
if ns_info:
display.vprint(VERB_QUIET, \
"Working with nodes: %s" % ns)
ns_info = False
prompt = "clush> "
else:
prompt = ""
# Set SIGUSR1 handler if needed
if task.default("USER_handle_SIGUSR1"):
signal.signal(signal.SIGUSR1, signal_handler)
try:
cmd = raw_input(prompt)
finally:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
except EOFError:
print
return
except UpdatePromptException:
if task.default("USER_interactive"):
continue
return
except KeyboardInterrupt, kbe:
if display.gather:
# Suspend task, so we can safely access its data from
# the main thread
task.suspend()
print_warn = False
# Display command output, but cannot order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
for buf, nodeset in sorted(map(nodesetify, task.iter_buffers()),
cmp=bufnodeset_cmp):
if not print_warn:
print_warn = True
display.vprint_err(VERB_STD, \
"Warning: Caught keyboard interrupt!")
display.print_gather(nodeset, buf)
# Return code handling
verbexit = VERB_QUIET
if display.maxrc:
verbexit = VERB_STD
ns_ok = NodeSet()
for rc, nodelist in task.iter_retcodes():
ns_ok.add(NodeSet._fromlist1(nodelist))
if rc != 0:
# Display return code if not ok ( != 0)
ns = NodeSet._fromlist1(nodelist)
display.vprint_err(verbexit, \
"clush: %s: exited with exit code %s" % (ns, rc))
# Add uncompleted nodeset to exception object
kbe.uncompleted_nodes = ns - ns_ok
# Display nodes that didn't answer within command timeout delay
if task.num_timeout() > 0:
display.vprint_err(verbexit, \
"clush: %s: command timeout" % \
NodeSet._fromlist1(task.iter_keys_timeout()))
raise kbe
if task.default("USER_running"):
ns_reg, ns_unreg = NodeSet(), NodeSet()
for c in task._engine.clients():
if c.registered:
ns_reg.add(c.key)
else:
ns_unreg.add(c.key)
if ns_unreg:
pending = "\nclush: pending(%d): %s" % (len(ns_unreg), ns_unreg)
else:
pending = ""
display.vprint_err(VERB_QUIET, "clush: interrupt (^C to " \
"abort task)\nclush: in progress(%d): %s%s" % (len(ns_reg), \
ns_reg, pending))
else:
cmdl = cmd.lower()
try:
ns_info = True
if cmdl.startswith('+'):
ns.update(cmdl[1:])
elif cmdl.startswith('-'):
ns.difference_update(cmdl[1:])
elif cmdl.startswith('@'):
ns = NodeSet(cmdl[1:])
elif cmdl == '=':
display.gather = not display.gather
if display.gather:
display.vprint(VERB_STD, \
"Switching to gathered output format")
else:
display.vprint(VERB_STD, \
"Switching to standard output format")
task.set_default("stdout_msgtree", \
display.gather or display.line_mode)
ns_info = False
continue
elif not cmdl.startswith('?'): # if ?, just print ns_info
ns_info = False
except NodeSetParseError:
display.vprint_err(VERB_QUIET, \
"clush: nodeset parse error (ignoring)")
if ns_info:
continue
if cmdl.startswith('!') and len(cmd.strip()) > 0:
run_command(task, cmd[1:], None, timeout, display)
elif cmdl != "quit":
if not cmd:
continue
if readline_avail:
readline.write_history_file(get_history_file())
run_command(task, cmd, ns, timeout, display)
def ttyloop(task, nodeset, timeout, display):
"""Manage the interactive prompt to run command"""
readline_avail = False
if task.default("USER_interactive"):
try:
import readline
readline_setup()
readline_avail = True
except ImportError:
pass
display.vprint(VERB_STD, \
"Enter 'quit' to leave this interactive mode")
rc = 0
ns = NodeSet(nodeset)
ns_info = True
cmd = ""
while task.default("USER_running") or cmd.lower() != 'quit':
try:
if task.default("USER_interactive") and \
not task.default("USER_running"):
if ns_info:
display.vprint(VERB_QUIET, \
"Working with nodes: %s" % ns)
ns_info = False
prompt = "clush> "
else:
prompt = ""
# Set SIGUSR1 handler if needed
if task.default("USER_handle_SIGUSR1"):
signal.signal(signal.SIGUSR1, signal_handler)
try:
cmd = raw_input(prompt)
finally:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
except EOFError:
print
return
except UpdatePromptException:
if task.default("USER_interactive"):
continue
return
except KeyboardInterrupt, kbe:
if display.gather:
# Suspend task, so we can safely access its data from
# the main thread
task.suspend()
print_warn = False
# Display command output, but cannot order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
for buf, nodeset in sorted(map(nodesetify, task.iter_buffers()),
cmp=bufnodeset_cmp):
if not print_warn:
print_warn = True
display.vprint_err(VERB_STD, \
"Warning: Caught keyboard interrupt!")
display.print_gather(nodeset, buf)
# Return code handling
verbexit = VERB_QUIET
if display.maxrc:
verbexit = VERB_STD
ns_ok = NodeSet()
for rc, nodelist in task.iter_retcodes():
ns_ok.add(NodeSet._fromlist1(nodelist))
if rc != 0:
# Display return code if not ok ( != 0)
ns = NodeSet._fromlist1(nodelist)
display.vprint_err(verbexit, \
"clush: %s: exited with exit code %s" % (ns, rc))
# Add uncompleted nodeset to exception object
kbe.uncompleted_nodes = ns - ns_ok
# Display nodes that didn't answer within command timeout delay
if task.num_timeout() > 0:
display.vprint_err(verbexit, \
"clush: %s: command timeout" % \
NodeSet._fromlist1(task.iter_keys_timeout()))
raise kbe
if task.default("USER_running"):
ns_reg, ns_unreg = NodeSet(), NodeSet()
for c in task._engine.clients():
if c.registered:
ns_reg.add(c.key)
else:
ns_unreg.add(c.key)
if ns_unreg:
pending = "\nclush: pending(%d): %s" % (len(ns_unreg), ns_unreg)
else:
pending = ""
display.vprint_err(VERB_QUIET, "clush: interrupt (^C to " \
"abort task)\nclush: in progress(%d): %s%s" % (len(ns_reg), \
ns_reg, pending))
else:
cmdl = cmd.lower()
try:
ns_info = True
if cmdl.startswith('+'):
ns.update(cmdl[1:])
elif cmdl.startswith('-'):
ns.difference_update(cmdl[1:])
elif cmdl.startswith('@'):
ns = NodeSet(cmdl[1:])
elif cmdl == '=':
display.gather = not display.gather
if display.gather:
display.vprint(VERB_STD, \
"Switching to gathered output format")
else:
display.vprint(VERB_STD, \
"Switching to standard output format")
task.set_default("stdout_msgtree", \
display.gather or display.line_mode)
ns_info = False
continue
elif not cmdl.startswith('?'): # if ?, just print ns_info
ns_info = False
except NodeSetParseError:
display.vprint_err(VERB_QUIET, \
"clush: nodeset parse error (ignoring)")
if ns_info:
continue
if cmdl.startswith('!') and len(cmd.strip()) > 0:
run_command(task, cmd[1:], None, timeout, display)
elif cmdl != "quit":
if not cmd:
continue
if readline_avail:
readline.write_history_file(get_history_file())
run_command(task, cmd, ns, timeout, display)
def nodeset():
"""script subroutine"""
class_set = NodeSet
usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
parser = OptionParser(usage)
parser.install_nodeset_commands()
parser.install_nodeset_operations()
parser.install_nodeset_options()
(options, args) = parser.parse_args()
group_resolver = std_group_resolver()
if options.debug:
group_resolver.set_verbosity(1)
# Check for command presence
cmdcount = int(options.count) + int(options.expand) + \
int(options.fold) + int(bool(options.list)) + \
int(options.regroup) + int(options.groupsources)
if not cmdcount:
parser.error("No command specified.")
elif cmdcount > 1:
parser.error("Multiple commands not allowed.")
if options.rangeset:
class_set = RangeSet
if options.all or options.regroup:
if class_set != NodeSet:
parser.error("-a/-r only supported in NodeSet mode")
if options.maxsplit is not None and options.contiguous:
parser.error("incompatible splitting options (split, contiguous)")
if options.maxsplit is None:
options.maxsplit = 1
if options.groupsource and not options.quiet and class_set == RangeSet:
print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \
% options.groupsource
# We want -s <groupsource> to act as a substition of default groupsource
# (ie. it's not necessary to prefix group names by this group source).
if options.groupsource:
group_resolver.default_source_name = options.groupsource
# The groupsources command simply lists group sources.
if options.groupsources:
if options.quiet:
dispdefault = "" # don't show (default) if quiet is set
else:
dispdefault = " (default)"
for src in group_resolver.sources():
print "%s%s" % (src, dispdefault)
dispdefault = ""
return
# Instantiate RangeSet or NodeSet object
xset = class_set(autostep=options.autostep)
if options.all:
# Include all nodes from external node groups support.
xset.update(NodeSet.fromall()) # uses default_source when set
if not args and not options.all and not options.list:
# No need to specify '-' to read stdin in these cases
process_stdin(xset.update, xset.__class__, options.autostep)
# Apply first operations (before first non-option)
for nodes in options.and_nodes:
if nodes == '-':
process_stdin(xset.intersection_update, xset.__class__,
options.autostep)
else:
xset.intersection_update(
class_set(nodes, autostep=options.autostep))
for nodes in options.sub_nodes:
if nodes == '-':
process_stdin(xset.difference_update, xset.__class__,
options.autostep)
else:
xset.difference_update(class_set(nodes, autostep=options.autostep))
for nodes in options.xor_nodes:
if nodes == '-':
process_stdin(xset.symmetric_difference_update, xset.__class__,
options.autostep)
else:
xset.symmetric_difference_update(class_set(nodes, \
autostep=options.autostep))
# Finish xset computing from args
compute_nodeset(xset, args, options.autostep)
# The list command has a special handling
if options.list > 0:
return command_list(options, xset)
# Interprete special characters (may raise SyntaxError)
separator = eval('\'%s\'' % options.separator, {"__builtins__": None}, {})
if options.slice_rangeset:
_xset = class_set()
for sli in RangeSet(options.slice_rangeset).slices():
_xset.update(xset[sli])
xset = _xset
format = options.output_format # default to '%s'
# Display result according to command choice
if options.expand:
xsubres = lambda x: separator.join((format % s for s in x.striter()))
elif options.fold:
xsubres = lambda x: format % x
elif options.regroup:
xsubres = lambda x: format % x.regroup(options.groupsource,
noprefix=options.groupbase)
else:
xsubres = lambda x: format % len(x)
if not xset or options.maxsplit <= 1 and not options.contiguous:
print xsubres(xset)
else:
if options.contiguous:
xiterator = xset.contiguous()
else:
xiterator = xset.split(options.maxsplit)
for xsubset in xiterator:
print xsubres(xsubset)
def nodeset():
"""script subroutine"""
class_set = NodeSet
usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
parser = OptionParser(usage)
parser.install_nodeset_commands()
parser.install_nodeset_operations()
parser.install_nodeset_options()
(options, args) = parser.parse_args()
group_resolver = std_group_resolver()
if options.debug:
group_resolver.set_verbosity(1)
# Check for command presence
cmdcount = int(options.count) + int(options.expand) + \
int(options.fold) + int(bool(options.list)) + \
int(bool(options.listall)) + int(options.regroup) + \
int(options.groupsources)
if not cmdcount:
parser.error("No command specified.")
elif cmdcount > 1:
parser.error("Multiple commands not allowed.")
if options.rangeset:
class_set = RangeSet
if options.all or options.regroup:
if class_set != NodeSet:
parser.error("-a/-r only supported in NodeSet mode")
if options.maxsplit is not None and options.contiguous:
parser.error("incompatible splitting options (split, contiguous)")
if options.maxsplit is None:
options.maxsplit = 1
if options.axis and (not options.fold or options.rangeset):
parser.error("--axis option is only supported when folding nodeset")
if options.groupsource and not options.quiet and class_set == RangeSet:
print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \
% options.groupsource
# We want -s <groupsource> to act as a substition of default groupsource
# (ie. it's not necessary to prefix group names by this group source).
if options.groupsource:
group_resolver.default_source_name = options.groupsource
# The groupsources command simply lists group sources.
if options.groupsources:
if options.quiet:
dispdefault = "" # don't show (default) if quiet is set
else:
dispdefault = " (default)"
for src in group_resolver.sources():
print "%s%s" % (src, dispdefault)
dispdefault = ""
return
autostep = options.autostep
# Do not use autostep for computation when a percentage or the special
# value 'auto' is specified. Real autostep value is set post-process.
if type(autostep) is float or autostep == 'auto':
autostep = None
# Instantiate RangeSet or NodeSet object
xset = class_set(autostep=autostep)
if options.all:
# Include all nodes from external node groups support.
xset.update(NodeSet.fromall()) # uses default_source when set
if not args and not options.all and not (options.list or options.listall):
# No need to specify '-' to read stdin in these cases
process_stdin(xset.update, xset.__class__, autostep)
# Apply first operations (before first non-option)
for nodes in options.and_nodes:
if nodes == '-':
process_stdin(xset.intersection_update, xset.__class__, autostep)
else:
xset.intersection_update(class_set(nodes, autostep=autostep))
for nodes in options.sub_nodes:
if nodes == '-':
process_stdin(xset.difference_update, xset.__class__, autostep)
else:
xset.difference_update(class_set(nodes, autostep=autostep))
for nodes in options.xor_nodes:
if nodes == '-':
process_stdin(xset.symmetric_difference_update, xset.__class__,
autostep)
else:
xset.symmetric_difference_update(
class_set(nodes, autostep=autostep))
# Finish xset computing from args
compute_nodeset(xset, args, autostep)
# The list command has a special handling
if options.list > 0 or options.listall > 0:
return command_list(options, xset, group_resolver)
# Interprete special characters (may raise SyntaxError)
separator = eval('\'%s\'' % options.separator, {"__builtins__": None}, {})
if options.slice_rangeset:
_xset = class_set()
for sli in RangeSet(options.slice_rangeset).slices():
_xset.update(xset[sli])
xset = _xset
if options.autostep == 'auto':
# Simple implementation of --autostep=auto
# if we have at least 3 nodes, all index should be foldable as a-b/n
xset.autostep = max(3, len(xset))
elif type(options.autostep) is float:
# at least % of nodes should be foldable as a-b/n
autofactor = float(options.autostep)
xset.autostep = int(math.ceil(float(len(xset)) * autofactor))
# user-specified nD-nodeset fold axis
if options.axis:
if not options.axis.startswith('-'):
# axis are 1-indexed in nodeset CLI (0 ignored)
xset.fold_axis = tuple(x - 1 for x in RangeSet(options.axis)
if x > 0)
else:
# negative axis index (only single number supported)
xset.fold_axis = [int(options.axis)]
fmt = options.output_format # default to '%s'
# Display result according to command choice
if options.expand:
xsubres = lambda x: separator.join((fmt % s for s in x.striter()))
elif options.fold:
# Special case when folding using NodeSet and format is set (#277)
if class_set is NodeSet and fmt != '%s':
# Create a new set after format has been applied to each node
xset = class_set._fromlist1((fmt % xnodestr for xnodestr in xset),
autostep=xset.autostep)
xsubres = lambda x: x
else:
xsubres = lambda x: fmt % x
elif options.regroup:
xsubres = lambda x: fmt % x.regroup(options.groupsource,
noprefix=options.groupbase)
else:
xsubres = lambda x: fmt % len(x)
if not xset or options.maxsplit <= 1 and not options.contiguous:
print xsubres(xset)
else:
if options.contiguous:
xiterator = xset.contiguous()
else:
xiterator = xset.split(options.maxsplit)
for xsubset in xiterator:
print xsubres(xsubset)
def nodeset():
"""script subroutine"""
class_set = NodeSet
usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
parser = OptionParser(usage)
parser.install_nodeset_commands()
parser.install_nodeset_operations()
parser.install_nodeset_options()
(options, args) = parser.parse_args()
group_resolver = std_group_resolver()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
# Check for command presence
cmdcount = int(options.count) + int(options.expand) + \
int(options.fold) + int(bool(options.list)) + \
int(bool(options.listall)) + int(options.regroup) + \
int(options.groupsources)
if not cmdcount:
parser.error("No command specified.")
elif cmdcount > 1:
parser.error("Multiple commands not allowed.")
if options.rangeset:
class_set = RangeSet
if options.all or options.regroup:
if class_set != NodeSet:
parser.error("-a/-r only supported in NodeSet mode")
if options.maxsplit is not None and options.contiguous:
parser.error("incompatible splitting options (split, contiguous)")
if options.maxsplit is None:
options.maxsplit = 1
if options.axis and (not options.fold or options.rangeset):
parser.error("--axis option is only supported when folding nodeset")
if options.groupsource and not options.quiet and class_set == RangeSet:
print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \
% options.groupsource
# We want -s <groupsource> to act as a substition of default groupsource
# (ie. it's not necessary to prefix group names by this group source).
if options.groupsource:
group_resolver.default_source_name = options.groupsource
# The groupsources command simply lists group sources.
if options.groupsources:
if options.quiet:
dispdefault = "" # don't show (default) if quiet is set
else:
dispdefault = " (default)"
for src in group_resolver.sources():
print "%s%s" % (src, dispdefault)
dispdefault = ""
return
autostep = options.autostep
# Do not use autostep for computation when a percentage or the special
# value 'auto' is specified. Real autostep value is set post-process.
if type(autostep) is float or autostep == 'auto':
autostep = None
# Instantiate RangeSet or NodeSet object
xset = class_set(autostep=autostep)
if options.all:
# Include all nodes from external node groups support.
xset.update(NodeSet.fromall()) # uses default_source when set
if not args and not options.all and not (options.list or options.listall):
# No need to specify '-' to read stdin in these cases
process_stdin(xset.update, xset.__class__, autostep)
# Apply first operations (before first non-option)
for nodes in options.and_nodes:
if nodes == '-':
process_stdin(xset.intersection_update, xset.__class__, autostep)
else:
xset.intersection_update(class_set(nodes, autostep=autostep))
for nodes in options.sub_nodes:
if nodes == '-':
process_stdin(xset.difference_update, xset.__class__, autostep)
else:
xset.difference_update(class_set(nodes, autostep=autostep))
for nodes in options.xor_nodes:
if nodes == '-':
process_stdin(xset.symmetric_difference_update, xset.__class__,
autostep)
else:
xset.symmetric_difference_update(class_set(nodes,
autostep=autostep))
# Finish xset computing from args
compute_nodeset(xset, args, autostep)
# The list command has a special handling
if options.list > 0 or options.listall > 0:
return command_list(options, xset, group_resolver)
# Interprete special characters (may raise SyntaxError)
separator = eval('\'%s\'' % options.separator, {"__builtins__":None}, {})
if options.slice_rangeset:
_xset = class_set()
for sli in RangeSet(options.slice_rangeset).slices():
_xset.update(xset[sli])
xset = _xset
if options.autostep == 'auto':
# Simple implementation of --autostep=auto
# if we have at least 3 nodes, all index should be foldable as a-b/n
xset.autostep = max(3, len(xset))
elif type(options.autostep) is float:
# at least % of nodes should be foldable as a-b/n
autofactor = float(options.autostep)
xset.autostep = int(math.ceil(float(len(xset)) * autofactor))
# user-specified nD-nodeset fold axis
if options.axis:
if not options.axis.startswith('-'):
# axis are 1-indexed in nodeset CLI (0 ignored)
xset.fold_axis = tuple(x-1 for x in RangeSet(options.axis) if x > 0)
else:
# negative axis index (only single number supported)
xset.fold_axis = [int(options.axis)]
fmt = options.output_format # default to '%s'
# Display result according to command choice
if options.expand:
xsubres = lambda x: separator.join((fmt % s for s in x.striter()))
elif options.fold:
# Special case when folding using NodeSet and format is set (#277)
if class_set is NodeSet and fmt != '%s':
# Create a new set after format has been applied to each node
xset = class_set._fromlist1((fmt % xnodestr for xnodestr in xset),
autostep=xset.autostep)
xsubres = lambda x: x
else:
xsubres = lambda x: fmt % x
elif options.regroup:
xsubres = lambda x: fmt % x.regroup(options.groupsource,
noprefix=options.groupbase)
else:
xsubres = lambda x: fmt % len(x)
if not xset or options.maxsplit <= 1 and not options.contiguous:
print xsubres(xset)
else:
if options.contiguous:
xiterator = xset.contiguous()
else:
xiterator = xset.split(options.maxsplit)
for xsubset in xiterator:
print xsubres(xsubset)
def main(args=sys.argv):
"""clush script entry point"""
sys.excepthook = clush_excepthook
# Default values
nodeset_base, nodeset_exclude = NodeSet(), NodeSet()
#
# Argument management
#
usage = "%prog [options] command"
parser = OptionParser(usage)
parser.add_option("--nostdin", action="store_true", dest="nostdin",
help="don't watch for possible input from stdin")
parser.install_nodes_options()
parser.install_display_options(verbose_options=True)
parser.install_filecopy_options()
parser.install_ssh_options()
(options, args) = parser.parse_args(args[1:])
#
# Load config file and apply overrides
#
config = ClushConfig(options)
# Should we use ANSI colors for nodes?
if config.color == "auto":
color = sys.stdout.isatty() and (options.gatherall or \
sys.stderr.isatty())
else:
color = config.color == "always"
# Create and configure display object.
display = Display(options, config, color)
#
# Compute the nodeset
#
if options.nodes:
nodeset_base = NodeSet.fromlist(options.nodes)
if options.exclude:
nodeset_exclude = NodeSet.fromlist(options.exclude)
if options.groupsource:
# Be sure -a/g -s source work as espected.
STD_GROUP_RESOLVER.default_sourcename = options.groupsource
# FIXME: add public API to enforce engine
Task._std_default['engine'] = options.engine
# Do we have nodes group?
task = task_self()
task.set_info("debug", config.verbosity > 1)
if config.verbosity == VERB_DEBUG:
STD_GROUP_RESOLVER.set_verbosity(1)
if options.nodes_all:
all_nodeset = NodeSet.fromall()
display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \
all_nodeset)
nodeset_base.add(all_nodeset)
if options.group:
grp_nodeset = NodeSet.fromlist(options.group,
resolver=NOGROUP_RESOLVER)
for grp in grp_nodeset:
addingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Adding nodes from option -g %s: %s" % (grp, addingrp))
nodeset_base.update(addingrp)
if options.exgroup:
grp_nodeset = NodeSet.fromlist(options.exgroup,
resolver=NOGROUP_RESOLVER)
for grp in grp_nodeset:
removingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Excluding nodes from option -X %s: %s" % (grp, removingrp))
nodeset_exclude.update(removingrp)
# Do we have an exclude list? (-x ...)
nodeset_base.difference_update(nodeset_exclude)
if len(nodeset_base) < 1:
parser.error('No node to run on.')
# Set open files limit.
set_fdlimit(config.fd_max, display)
#
# Task management
#
# check for clush interactive mode
interactive = not len(args) and \
not (options.copy or options.rcopy)
# check for foreground ttys presence (input)
stdin_isafgtty = sys.stdin.isatty() and \
os.tcgetpgrp(sys.stdin.fileno()) == os.getpgrp()
# check for special condition (empty command and stdin not a tty)
if interactive and not stdin_isafgtty:
# looks like interactive but stdin is not a tty:
# switch to non-interactive + disable ssh pseudo-tty
interactive = False
# SSH: disable pseudo-tty allocation (-T)
ssh_options = config.ssh_options or ''
ssh_options += ' -T'
config._set_main("ssh_options", ssh_options)
if options.nostdin and interactive:
parser.error("illegal option `--nostdin' in that case")
user_interaction = False
if not options.nostdin:
# Try user interaction: check for foreground ttys presence (ouput)
stdout_isafgtty = sys.stdout.isatty() and \
os.tcgetpgrp(sys.stdout.fileno()) == os.getpgrp()
user_interaction = stdin_isafgtty and stdout_isafgtty
display.vprint(VERB_DEBUG, "User interaction: %s" % user_interaction)
if user_interaction:
# Standard input is a terminal and we want to perform some user
# interactions in the main thread (using blocking calls), so
# we run cluster commands in a new ClusterShell Task (a new
# thread is created).
task = Task()
# else: perform everything in the main thread
# Handle special signal only when user_interaction is set
task.set_default("USER_handle_SIGUSR1", user_interaction)
task.excepthook = sys.excepthook
task.set_default("USER_stdin_worker", not (sys.stdin.isatty() or \
options.nostdin))
display.vprint(VERB_DEBUG, "Create STDIN worker: %s" % \
task.default("USER_stdin_worker"))
task.set_info("debug", config.verbosity >= VERB_DEBUG)
task.set_info("fanout", config.fanout)
if config.ssh_user:
task.set_info("ssh_user", config.ssh_user)
if config.ssh_path:
task.set_info("ssh_path", config.ssh_path)
if config.ssh_options:
task.set_info("ssh_options", config.ssh_options)
# Set detailed timeout values
task.set_info("connect_timeout", config.connect_timeout)
command_timeout = config.command_timeout
task.set_info("command_timeout", command_timeout)
# Enable stdout/stderr separation
task.set_default("stderr", not options.gatherall)
# Disable MsgTree buffering if not gathering outputs
task.set_default("stdout_msgtree", display.gather or display.line_mode)
# Always disable stderr MsgTree buffering
task.set_default("stderr_msgtree", False)
# Set timeout at worker level when command_timeout is defined.
if command_timeout > 0:
timeout = command_timeout
else:
timeout = -1
# Configure task custom status
task.set_default("USER_interactive", interactive)
task.set_default("USER_running", False)
if (options.copy or options.rcopy) and not args:
parser.error("--[r]copy option requires at least one argument")
if options.copy:
if not options.dest_path:
options.dest_path = os.path.dirname(args[0])
op = "copy sources=%s dest=%s" % (args, options.dest_path)
elif options.rcopy:
if not options.dest_path:
options.dest_path = os.path.dirname(args[0])
op = "rcopy sources=%s dest=%s" % (args, options.dest_path)
else:
op = "command=\"%s\"" % ' '.join(args)
# print debug values (fanout value is get from the config object
# and not task itself as set_info() is an asynchronous call.
display.vprint(VERB_DEBUG, "clush: nodeset=%s fanout=%d [timeout " \
"conn=%.1f cmd=%.1f] %s" % (nodeset_base, config.fanout,
task.info("connect_timeout"),
task.info("command_timeout"),
op))
if not task.default("USER_interactive"):
if options.copy:
run_copy(task, args, options.dest_path, nodeset_base, 0,
options.preserve_flag, display)
elif options.rcopy:
run_rcopy(task, args, options.dest_path, nodeset_base, 0,
options.preserve_flag, display)
else:
run_command(task, ' '.join(args), nodeset_base, timeout, display)
if user_interaction:
ttyloop(task, nodeset_base, timeout, display)
elif task.default("USER_interactive"):
display.vprint_err(VERB_QUIET, \
"ERROR: interactive mode requires a tty")
clush_exit(1)
rc = 0
if options.maxrc:
# Instead of clush return code, return commands retcode
rc = task.max_retcode()
if task.num_timeout() > 0:
rc = 255
clush_exit(rc)
def ev_timeout(self, worker):
self._display.vprint_err(VERB_QUIET, "clush: %s: command timeout" % \
NodeSet._fromlist1(worker.iter_keys_timeout()))
def nodeset():
"""script subroutine"""
class_set = NodeSet
usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
parser = OptionParser(usage)
parser.install_nodeset_commands()
parser.install_nodeset_operations()
parser.install_nodeset_options()
(options, args) = parser.parse_args()
group_resolver = std_group_resolver()
if options.debug:
group_resolver.set_verbosity(1)
# Check for command presence
cmdcount = int(options.count) + int(options.expand) + \
int(options.fold) + int(bool(options.list)) + \
int(options.regroup) + int(options.groupsources)
if not cmdcount:
parser.error("No command specified.")
elif cmdcount > 1:
parser.error("Multiple commands not allowed.")
if options.rangeset:
class_set = RangeSet
if options.all or options.regroup:
if class_set != NodeSet:
parser.error("-a/-r only supported in NodeSet mode")
if options.maxsplit is not None and options.contiguous:
parser.error("incompatible splitting options (split, contiguous)")
if options.maxsplit is None:
options.maxsplit = 1
if options.groupsource and not options.quiet and class_set == RangeSet:
print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \
% options.groupsource
# We want -s <groupsource> to act as a substition of default groupsource
# (ie. it's not necessary to prefix group names by this group source).
if options.groupsource:
group_resolver.default_source_name = options.groupsource
# The groupsources command simply lists group sources.
if options.groupsources:
if options.quiet:
dispdefault = "" # don't show (default) if quiet is set
else:
dispdefault = " (default)"
for src in group_resolver.sources():
print "%s%s" % (src, dispdefault)
dispdefault = ""
return
# Instantiate RangeSet or NodeSet object
xset = class_set(autostep=options.autostep)
if options.all:
# Include all nodes from external node groups support.
xset.update(NodeSet.fromall()) # uses default_source when set
if not args and not options.all and not options.list:
# No need to specify '-' to read stdin in these cases
process_stdin(xset.update, xset.__class__, options.autostep)
# Apply first operations (before first non-option)
for nodes in options.and_nodes:
if nodes == '-':
process_stdin(xset.intersection_update, xset.__class__,
options.autostep)
else:
xset.intersection_update(class_set(nodes,
autostep=options.autostep))
for nodes in options.sub_nodes:
if nodes == '-':
process_stdin(xset.difference_update, xset.__class__,
options.autostep)
else:
xset.difference_update(class_set(nodes, autostep=options.autostep))
for nodes in options.xor_nodes:
if nodes == '-':
process_stdin(xset.symmetric_difference_update, xset.__class__,
options.autostep)
else:
xset.symmetric_difference_update(class_set(nodes, \
autostep=options.autostep))
# Finish xset computing from args
compute_nodeset(xset, args, options.autostep)
# The list command has a special handling
if options.list > 0:
return command_list(options, xset)
# Interprete special characters (may raise SyntaxError)
separator = eval('\'%s\'' % options.separator, {"__builtins__":None}, {})
if options.slice_rangeset:
_xset = class_set()
for sli in RangeSet(options.slice_rangeset).slices():
_xset.update(xset[sli])
xset = _xset
format = options.output_format # default to '%s'
# Display result according to command choice
if options.expand:
xsubres = lambda x: separator.join((format % s for s in x.striter()))
elif options.fold:
xsubres = lambda x: format % x
elif options.regroup:
xsubres = lambda x: format % x.regroup(options.groupsource,
noprefix=options.groupbase)
else:
xsubres = lambda x: format % len(x)
if not xset or options.maxsplit <= 1 and not options.contiguous:
print xsubres(xset)
else:
if options.contiguous:
xiterator = xset.contiguous()
else:
xiterator = xset.split(options.maxsplit)
for xsubset in xiterator:
print xsubres(xsubset)
color = sys.stdout.isatty() and (options.gatherall or \
sys.stderr.isatty())
else:
color = config.color == "always"
try:
# Create and configure display object.
display = Display(options, config, color)
except ValueError, exc:
parser.error("option mismatch (%s)" % exc)
#
# Compute the nodeset
#
if options.nodes:
nodeset_base = NodeSet.fromlist(options.nodes)
if options.exclude:
nodeset_exclude = NodeSet.fromlist(options.exclude)
if options.groupsource:
# Be sure -a/g -s source work as espected.
std_group_resolver().default_source_name = options.groupsource
# FIXME: add public API to enforce engine
Task._std_default['engine'] = options.engine
# Do we have nodes group?
task = task_self()
task.set_info("debug", config.verbosity >= VERB_DEBUG)
if config.verbosity == VERB_DEBUG:
std_group_resolver().set_verbosity(1)
def ttyloop(task, nodeset, timeout, display, remote):
"""Manage the interactive prompt to run command"""
readline_avail = False
interactive = task.default("USER_interactive")
if interactive:
try:
import readline
readline_setup()
readline_avail = True
except ImportError:
pass
display.vprint(VERB_STD, \
"Enter 'quit' to leave this interactive mode")
rc = 0
ns = NodeSet(nodeset)
ns_info = True
cmd = ""
while task.default("USER_running") or \
(interactive and cmd.lower() != 'quit'):
try:
# Set SIGUSR1 handler if needed
if task.default("USER_handle_SIGUSR1"):
signal.signal(signal.SIGUSR1, signal_handler)
if task.default("USER_interactive") and \
not task.default("USER_running"):
if ns_info:
display.vprint(VERB_QUIET, \
"Working with nodes: %s" % ns)
ns_info = False
prompt = "clush> "
else:
prompt = ""
try:
cmd = raw_input(prompt)
assert cmd is not None, "Result of raw_input() is None!"
finally:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
except EOFError:
print
return
except UpdatePromptException:
if task.default("USER_interactive"):
continue
return
except KeyboardInterrupt as kbe:
# Caught SIGINT here (main thread) but the signal will also reach
# subprocesses (that will most likely kill them)
if display.gather:
# Suspend task, so we can safely access its data from here
task.suspend()
# If USER_running is not set, the task had time to finish,
# that could mean all subprocesses have been killed and all
# handlers have been processed.
if not task.default("USER_running"):
# let's clush_excepthook handle the rest
raise kbe
# If USER_running is set, the task didn't have time to finish
# its work, so we must print something for the user...
print_warn = False
# Display command output, but cannot order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
for buf, nodeset in sorted(map(nodesetify,
task.iter_buffers()),
cmp=bufnodeset_cmp):
if not print_warn:
print_warn = True
display.vprint_err(VERB_STD, \
"Warning: Caught keyboard interrupt!")
display.print_gather(nodeset, buf)
# Return code handling
verbexit = VERB_QUIET
if display.maxrc:
verbexit = VERB_STD
ns_ok = NodeSet()
for rc, nodelist in task.iter_retcodes():
ns_ok.add(NodeSet._fromlist1(nodelist))
if rc != 0:
# Display return code if not ok ( != 0)
nsdisp = ns = NodeSet._fromlist1(nodelist)
if display.verbosity >= VERB_QUIET and len(ns) > 1:
nsdisp = "%s (%d)" % (ns, len(ns))
msgrc = "clush: %s: exited with exit code %d" % (
nsdisp, rc)
display.vprint_err(verbexit, msgrc)
# Add uncompleted nodeset to exception object
kbe.uncompleted_nodes = ns - ns_ok
# Display nodes that didn't answer within command timeout delay
if task.num_timeout() > 0:
display.vprint_err(verbexit, \
"clush: %s: command timeout" % \
NodeSet._fromlist1(task.iter_keys_timeout()))
raise kbe
if task.default("USER_running"):
ns_reg, ns_unreg = NodeSet(), NodeSet()
for client in task._engine.clients():
if client.registered:
ns_reg.add(client.key)
else:
ns_unreg.add(client.key)
if ns_unreg:
pending = "\nclush: pending(%d): %s" % (len(ns_unreg),
ns_unreg)
else:
pending = ""
display.vprint_err(VERB_QUIET,
"clush: interrupt (^C to abort task)")
gws = task.gateways.keys()
if not gws:
display.vprint_err(
VERB_QUIET, "clush: in progress(%d): %s%s" %
(len(ns_reg), ns_reg, pending))
else:
display.vprint_err(
VERB_QUIET, "clush: in progress(%d): %s%s\n"
"clush: [tree] open gateways(%d): %s" %
(len(ns_reg), ns_reg, pending, len(gws),
NodeSet._fromlist1(gws)))
for gw, (chan, metaworkers) in task.gateways.iteritems():
act_targets = NodeSet.fromlist(mw.gwtargets[gw]
for mw in metaworkers)
if act_targets:
display.vprint_err(
VERB_QUIET, "clush: [tree] in progress(%d) on %s: %s" %
(len(act_targets), gw, act_targets))
else:
cmdl = cmd.lower()
try:
ns_info = True
if cmdl.startswith('+'):
ns.update(cmdl[1:])
elif cmdl.startswith('-'):
ns.difference_update(cmdl[1:])
elif cmdl.startswith('@'):
ns = NodeSet(cmdl[1:])
elif cmdl == '=':
display.gather = not display.gather
if display.gather:
display.vprint(VERB_STD, \
"Switching to gathered output format")
else:
display.vprint(VERB_STD, \
"Switching to standard output format")
task.set_default("stdout_msgtree", \
display.gather or display.line_mode)
ns_info = False
continue
elif not cmdl.startswith('?'): # if ?, just print ns_info
ns_info = False
except NodeSetParseError:
display.vprint_err(VERB_QUIET, \
"clush: nodeset parse error (ignoring)")
if ns_info:
continue
if cmdl.startswith('!') and len(cmd.strip()) > 0:
run_command(task, cmd[1:], None, timeout, display, remote)
elif cmdl != "quit":
if not cmd:
continue
if readline_avail:
readline.write_history_file(get_history_file())
run_command(task, cmd, ns, timeout, display, remote)
return rc
def main():
"""clush script entry point"""
sys.excepthook = clush_excepthook
#
# Argument management
#
usage = "%prog [options] command"
parser = OptionParser(usage)
parser.add_option("--nostdin",
action="store_true",
dest="nostdin",
help="don't watch for possible input from stdin")
parser.install_config_options('clush.conf(5)')
parser.install_nodes_options()
parser.install_display_options(verbose_options=True)
parser.install_filecopy_options()
parser.install_connector_options()
(options, args) = parser.parse_args()
#
# Load config file and apply overrides
#
config = ClushConfig(options)
# Should we use ANSI colors for nodes?
if config.color == "auto":
color = sys.stdout.isatty() and (options.gatherall or \
sys.stderr.isatty())
else:
color = config.color == "always"
try:
# Create and configure display object.
display = Display(options, config, color)
except ValueError as exc:
parser.error("option mismatch (%s)" % exc)
if options.groupsource:
# Be sure -a/g -s source work as espected.
std_group_resolver().default_source_name = options.groupsource
# Compute the nodeset and warn for possible use of shell pathname
# expansion (#225)
wnodelist = []
xnodelist = []
if options.nodes:
wnodelist = [NodeSet(nodes) for nodes in options.nodes]
if options.exclude:
xnodelist = [NodeSet(nodes) for nodes in options.exclude]
for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)):
for nodes in nodelist:
if len(nodes) == 1 and exists(str(nodes)):
display.vprint_err(
VERB_STD, "Warning: using '-%s %s' and "
"local path '%s' exists, was it expanded "
"by the shell?" % (opt, nodes, nodes))
# --hostfile support (#235)
for opt_hostfile in options.hostfile:
try:
fnodeset = NodeSet()
hostfile = open(opt_hostfile)
for line in hostfile.read().splitlines():
fnodeset.updaten(nodes for nodes in line.split())
hostfile.close()
display.vprint_err(
VERB_DEBUG,
"Using nodeset %s from hostfile %s" % (fnodeset, opt_hostfile))
wnodelist.append(fnodeset)
except IOError as exc:
# re-raise as OSError to be properly handled
errno, strerror = exc.args
raise OSError(errno, strerror, exc.filename)
# Instantiate target nodeset from command line and hostfile
nodeset_base = NodeSet.fromlist(wnodelist)
# Instantiate filter nodeset (command line only)
nodeset_exclude = NodeSet.fromlist(xnodelist)
# Specified engine prevails over default engine
DEFAULTS.engine = options.engine
# Do we have nodes group?
task = task_self()
task.set_info("debug", config.verbosity >= VERB_DEBUG)
if config.verbosity == VERB_DEBUG:
std_group_resolver().set_verbosity(1)
if options.nodes_all:
all_nodeset = NodeSet.fromall()
display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \
all_nodeset)
nodeset_base.add(all_nodeset)
if options.group:
grp_nodeset = NodeSet.fromlist(options.group,
resolver=RESOLVER_NOGROUP)
for grp in grp_nodeset:
addingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Adding nodes from option -g %s: %s" % (grp, addingrp))
nodeset_base.update(addingrp)
if options.exgroup:
grp_nodeset = NodeSet.fromlist(options.exgroup,
resolver=RESOLVER_NOGROUP)
for grp in grp_nodeset:
removingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Excluding nodes from option -X %s: %s" % (grp, removingrp))
nodeset_exclude.update(removingrp)
# Do we have an exclude list? (-x ...)
nodeset_base.difference_update(nodeset_exclude)
if len(nodeset_base) < 1:
parser.error('No node to run on.')
if options.pick and options.pick < len(nodeset_base):
# convert to string for sample as nsiter() is slower for big
# nodesets; and we assume options.pick will remain small-ish
keep = random.sample(nodeset_base, options.pick)
nodeset_base.intersection_update(','.join(keep))
if config.verbosity >= VERB_VERB:
msg = "Picked random nodes: %s" % nodeset_base
print Display.COLOR_RESULT_FMT % msg
# Set open files limit.
set_fdlimit(config.fd_max, display)
#
# Task management
#
# check for clush interactive mode
interactive = not len(args) and \
not (options.copy or options.rcopy)
# check for foreground ttys presence (input)
stdin_isafgtty = sys.stdin.isatty() and \
os.tcgetpgrp(sys.stdin.fileno()) == os.getpgrp()
# check for special condition (empty command and stdin not a tty)
if interactive and not stdin_isafgtty:
# looks like interactive but stdin is not a tty:
# switch to non-interactive + disable ssh pseudo-tty
interactive = False
# SSH: disable pseudo-tty allocation (-T)
ssh_options = config.ssh_options or ''
ssh_options += ' -T'
config._set_main("ssh_options", ssh_options)
if options.nostdin and interactive:
parser.error("illegal option `--nostdin' in that case")
# Force user_interaction if Clush._f_user_interaction for test purposes
user_interaction = hasattr(sys.modules[__name__], '_f_user_interaction')
if not options.nostdin:
# Try user interaction: check for foreground ttys presence (ouput)
stdout_isafgtty = sys.stdout.isatty() and \
os.tcgetpgrp(sys.stdout.fileno()) == os.getpgrp()
user_interaction |= stdin_isafgtty and stdout_isafgtty
display.vprint(VERB_DEBUG, "User interaction: %s" % user_interaction)
if user_interaction:
# Standard input is a terminal and we want to perform some user
# interactions in the main thread (using blocking calls), so
# we run cluster commands in a new ClusterShell Task (a new
# thread is created).
task = Task()
# else: perform everything in the main thread
# Handle special signal only when user_interaction is set
task.set_default("USER_handle_SIGUSR1", user_interaction)
task.excepthook = sys.excepthook
task.set_default("USER_stdin_worker", not (sys.stdin.isatty() or \
options.nostdin or \
user_interaction))
display.vprint(VERB_DEBUG, "Create STDIN worker: %s" % \
task.default("USER_stdin_worker"))
if config.verbosity >= VERB_DEBUG:
task.set_info("debug", True)
logging.basicConfig(level=logging.DEBUG)
logging.debug("clush: STARTING DEBUG")
else:
logging.basicConfig(level=logging.CRITICAL)
task.set_info("fanout", config.fanout)
if options.worker:
try:
if options.remote == 'no':
task.set_default('local_worker',
_load_workerclass(options.worker))
else:
task.set_default('distant_worker',
_load_workerclass(options.worker))
except (ImportError, AttributeError):
msg = "ERROR: Could not load worker '%s'" % options.worker
display.vprint_err(VERB_QUIET, msg)
clush_exit(1, task)
if options.topofile or task._default_tree_is_enabled():
if options.topofile:
task.load_topology(options.topofile)
if config.verbosity >= VERB_VERB:
roots = len(task.topology.root.nodeset)
gws = task.topology.inner_node_count() - roots
msg = "enabling tree topology (%d gateways)" % gws
print >> sys.stderr, "clush: %s" % msg
if options.grooming_delay:
if config.verbosity >= VERB_VERB:
msg = Display.COLOR_RESULT_FMT % ("Grooming delay: %f" %
options.grooming_delay)
print >> sys.stderr, msg
task.set_info("grooming_delay", options.grooming_delay)
elif options.rcopy:
# By default, --rcopy should inhibit grooming
task.set_info("grooming_delay", 0)
if config.ssh_user:
task.set_info("ssh_user", config.ssh_user)
if config.ssh_path:
task.set_info("ssh_path", config.ssh_path)
if config.ssh_options:
task.set_info("ssh_options", config.ssh_options)
if config.scp_path:
task.set_info("scp_path", config.scp_path)
if config.scp_options:
task.set_info("scp_options", config.scp_options)
if config.rsh_path:
task.set_info("rsh_path", config.rsh_path)
if config.rcp_path:
task.set_info("rcp_path", config.rcp_path)
if config.rsh_options:
task.set_info("rsh_options", config.rsh_options)
# Set detailed timeout values
task.set_info("connect_timeout", config.connect_timeout)
task.set_info("command_timeout", config.command_timeout)
# Enable stdout/stderr separation
task.set_default("stderr", not options.gatherall)
# Disable MsgTree buffering if not gathering outputs
task.set_default("stdout_msgtree", display.gather or display.line_mode)
# Always disable stderr MsgTree buffering
task.set_default("stderr_msgtree", False)
# Set timeout at worker level when command_timeout is defined.
if config.command_timeout > 0:
timeout = config.command_timeout
else:
timeout = -1
# Configure task custom status
task.set_default("USER_interactive", interactive)
task.set_default("USER_running", False)
if (options.copy or options.rcopy) and not args:
parser.error("--[r]copy option requires at least one argument")
if options.copy:
if not options.dest_path:
# append '/' to clearly indicate a directory for tree mode
options.dest_path = join(dirname(abspath(args[0])), '')
op = "copy sources=%s dest=%s" % (args, options.dest_path)
elif options.rcopy:
if not options.dest_path:
options.dest_path = dirname(abspath(args[0]))
op = "rcopy sources=%s dest=%s" % (args, options.dest_path)
else:
op = "command=\"%s\"" % ' '.join(args)
# print debug values (fanout value is get from the config object
# and not task itself as set_info() is an asynchronous call.
display.vprint(VERB_DEBUG, "clush: nodeset=%s fanout=%d [timeout " \
"conn=%.1f cmd=%.1f] %s" % (nodeset_base, config.fanout,
config.connect_timeout,
config.command_timeout,
op))
if not task.default("USER_interactive"):
if display.verbosity >= VERB_DEBUG and task.topology:
print Display.COLOR_RESULT_FMT % '-' * 15
print Display.COLOR_RESULT_FMT % task.topology,
print Display.COLOR_RESULT_FMT % '-' * 15
if options.copy:
run_copy(task, args, options.dest_path, nodeset_base, timeout,
options.preserve_flag, display)
elif options.rcopy:
run_rcopy(task, args, options.dest_path, nodeset_base, timeout,
options.preserve_flag, display)
else:
run_command(task, ' '.join(args), nodeset_base, timeout, display,
options.remote != 'no')
if user_interaction:
ttyloop(task, nodeset_base, timeout, display, options.remote != 'no')
elif task.default("USER_interactive"):
display.vprint_err(VERB_QUIET, \
"ERROR: interactive mode requires a tty")
clush_exit(1, task)
rc = 0
if options.maxrc:
# Instead of clush return code, return commands retcode
rc = task.max_retcode()
if task.num_timeout() > 0:
rc = 255
clush_exit(rc, task)
def ttyloop(task, nodeset, timeout, display, remote):
"""Manage the interactive prompt to run command"""
readline_avail = False
interactive = task.default("USER_interactive")
if interactive:
try:
import readline
readline_setup()
readline_avail = True
except ImportError:
pass
display.vprint(VERB_STD, \
"Enter 'quit' to leave this interactive mode")
rc = 0
ns = NodeSet(nodeset)
ns_info = True
cmd = ""
while task.default("USER_running") or \
(interactive and cmd.lower() != 'quit'):
try:
# Set SIGUSR1 handler if needed
if task.default("USER_handle_SIGUSR1"):
signal.signal(signal.SIGUSR1, signal_handler)
if task.default("USER_interactive") and \
not task.default("USER_running"):
if ns_info:
display.vprint(VERB_QUIET, \
"Working with nodes: %s" % ns)
ns_info = False
prompt = "clush> "
else:
prompt = ""
try:
cmd = raw_input(prompt)
assert cmd is not None, "Result of raw_input() is None!"
finally:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
except EOFError:
print
return
except UpdatePromptException:
if task.default("USER_interactive"):
continue
return
except KeyboardInterrupt, kbe:
# Caught SIGINT here (main thread) but the signal will also reach
# subprocesses (that will most likely kill them)
if display.gather:
# Suspend task, so we can safely access its data from here
task.suspend()
# If USER_running is not set, the task had time to finish,
# that could mean all subprocesses have been killed and all
# handlers have been processed.
if not task.default("USER_running"):
# let's clush_excepthook handle the rest
raise kbe
# If USER_running is set, the task didn't have time to finish
# its work, so we must print something for the user...
print_warn = False
# Display command output, but cannot order buffers by rc
nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1]))
for buf, nodeset in sorted(map(nodesetify, task.iter_buffers()),
cmp=bufnodeset_cmp):
if not print_warn:
print_warn = True
display.vprint_err(VERB_STD, \
"Warning: Caught keyboard interrupt!")
display.print_gather(nodeset, buf)
# Return code handling
verbexit = VERB_QUIET
if display.maxrc:
verbexit = VERB_STD
ns_ok = NodeSet()
for rc, nodelist in task.iter_retcodes():
ns_ok.add(NodeSet._fromlist1(nodelist))
if rc != 0:
# Display return code if not ok ( != 0)
ns = NodeSet._fromlist1(nodelist)
display.vprint_err(verbexit, \
"clush: %s: exited with exit code %s" % (ns, rc))
# Add uncompleted nodeset to exception object
kbe.uncompleted_nodes = ns - ns_ok
# Display nodes that didn't answer within command timeout delay
if task.num_timeout() > 0:
display.vprint_err(verbexit, \
"clush: %s: command timeout" % \
NodeSet._fromlist1(task.iter_keys_timeout()))
raise kbe
if task.default("USER_running"):
ns_reg, ns_unreg = NodeSet(), NodeSet()
for client in task._engine.clients():
if client.registered:
ns_reg.add(client.key)
else:
ns_unreg.add(client.key)
if ns_unreg:
pending = "\nclush: pending(%d): %s" % (len(ns_unreg), ns_unreg)
else:
pending = ""
display.vprint_err(VERB_QUIET,
"clush: interrupt (^C to abort task)")
gws = task.gateways.keys()
if not gws:
display.vprint_err(VERB_QUIET,
"clush: in progress(%d): %s%s"
% (len(ns_reg), ns_reg, pending))
else:
display.vprint_err(VERB_QUIET,
"clush: in progress(%d): %s%s\n"
"clush: [tree] open gateways(%d): %s"
% (len(ns_reg), ns_reg, pending,
len(gws), NodeSet._fromlist1(gws)))
for gw, (chan, metaworkers) in task.gateways.iteritems():
act_targets = NodeSet.fromlist(mw.gwtargets[gw]
for mw in metaworkers)
if act_targets:
display.vprint_err(VERB_QUIET,
"clush: [tree] in progress(%d) on %s: %s"
% (len(act_targets), gw, act_targets))
else:
cmdl = cmd.lower()
try:
ns_info = True
if cmdl.startswith('+'):
ns.update(cmdl[1:])
elif cmdl.startswith('-'):
ns.difference_update(cmdl[1:])
elif cmdl.startswith('@'):
ns = NodeSet(cmdl[1:])
elif cmdl == '=':
display.gather = not display.gather
if display.gather:
display.vprint(VERB_STD, \
"Switching to gathered output format")
else:
display.vprint(VERB_STD, \
"Switching to standard output format")
task.set_default("stdout_msgtree", \
display.gather or display.line_mode)
ns_info = False
continue
elif not cmdl.startswith('?'): # if ?, just print ns_info
ns_info = False
except NodeSetParseError:
display.vprint_err(VERB_QUIET, \
"clush: nodeset parse error (ignoring)")
if ns_info:
continue
if cmdl.startswith('!') and len(cmd.strip()) > 0:
run_command(task, cmd[1:], None, timeout, display, remote)
elif cmdl != "quit":
if not cmd:
continue
if readline_avail:
readline.write_history_file(get_history_file())
run_command(task, cmd, ns, timeout, display, remote)
def clubak():
"""script subroutine"""
# Argument management
parser = OptionParser("%prog [options]")
parser.install_display_options(verbose_options=True,
separator_option=True,
dshbak_compat=True,
msgtree_mode=True)
options = parser.parse_args()[0]
if options.interpret_keys == THREE_CHOICES[-1]: # auto?
enable_nodeset_key = None # AUTO
else:
enable_nodeset_key = (options.interpret_keys == THREE_CHOICES[1])
# Create new message tree
if options.trace_mode:
tree_mode = MODE_TRACE
else:
tree_mode = MODE_DEFER
tree = MsgTree(mode=tree_mode)
fast_mode = options.fast_mode
if fast_mode:
if tree_mode != MODE_DEFER or options.line_mode:
parser.error("incompatible tree options")
preload_msgs = {}
# Feed the tree from standard input lines
for line in sys.stdin:
try:
linestripped = line.rstrip('\r\n')
if options.verbose or options.debug:
print "INPUT %s" % linestripped
key, content = linestripped.split(options.separator, 1)
key = key.strip()
if not key:
raise ValueError("no node found")
if enable_nodeset_key is False: # interpret-keys=never?
keyset = [key]
else:
try:
keyset = NodeSet(key)
except NodeSetParseError:
if enable_nodeset_key: # interpret-keys=always?
raise
enable_nodeset_key = False # auto => switch off
keyset = [key]
if fast_mode:
for node in keyset:
preload_msgs.setdefault(node, []).append(content)
else:
for node in keyset:
tree.add(node, content)
except ValueError as ex:
raise ValueError("%s (\"%s\")" % (ex, linestripped))
if fast_mode:
# Messages per node have been aggregated, now add to tree one
# full msg per node
for key, wholemsg in preload_msgs.iteritems():
tree.add(key, '\n'.join(wholemsg))
# Display results
try:
disp = Display(options)
if options.debug:
std_group_resolver().set_verbosity(1)
print >> sys.stderr, \
"clubak: line_mode=%s gather=%s tree_depth=%d" % \
(bool(options.line_mode), bool(disp.gather), tree._depth())
display(tree, disp, disp.gather or disp.regroup, \
options.trace_mode, enable_nodeset_key is not False)
except ValueError as exc:
parser.error("option mismatch (%s)" % exc)
def nodeset():
"""script subroutine"""
class_set = NodeSet
usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
parser = OptionParser(usage)
parser.install_nodeset_commands()
parser.install_nodeset_operations()
parser.install_nodeset_options()
(options, args) = parser.parse_args()
if options.debug:
STD_GROUP_RESOLVER.set_verbosity(1)
# Check for command presence
cmdcount = int(options.count) + int(options.expand) + \
int(options.fold) + int(bool(options.list)) + \
int(options.regroup) + int(options.groupsources)
if not cmdcount:
parser.error("No command specified.")
elif cmdcount > 1:
parser.error("Multiple commands not allowed.")
if options.rangeset:
class_set = RangeSet
if options.all or options.regroup:
assert class_set == NodeSet, "-a/-r only supported in NodeSet mode"
if options.groupsource and not options.quiet and \
(class_set == RangeSet or options.groupsources):
print >> sys.stderr, "WARNING: option group source \"%s\" ignored" \
% options.groupsource
# The list command doesn't need any NodeSet, check for it first.
if options.list > 0:
list_level = options.list
for group in grouplist(options.groupsource):
if options.groupsource and not options.groupbase:
nsgroup = "@%s:%s" % (options.groupsource, group)
else:
nsgroup = "@%s" % group
if list_level == 1:
print nsgroup
else:
nodes = NodeSet(nsgroup)
if list_level == 2: # -ll ?
print "%s %s" % (nsgroup, nodes)
else: # -lll ?
print "%s %s %d" % (nsgroup, nodes, len(nodes))
return
# Also, the groupsources command simply lists group sources.
elif options.groupsources:
if options.quiet:
dispdefault = "" # don't show (default) if quiet is set
else:
dispdefault = " (default)"
for src in STD_GROUP_RESOLVER.sources():
print "%s%s" % (src, dispdefault)
dispdefault = ""
return
# We want -s <groupsource> to act as a substition of default groupsource
# (ie. it's not necessary to prefix group names by this group source).
if options.groupsource:
STD_GROUP_RESOLVER.default_sourcename = options.groupsource
# Instantiate RangeSet or NodeSet object
xset = class_set(autostep=options.autostep)
if options.all:
# Include all nodes from external node groups support.
xset.update(NodeSet.fromall()) # uses default_sourcename
elif not args:
# No need to specify '-' to read stdin if no argument at all.
process_stdin(xset.update, xset.__class__, options.autostep)
# Apply first operations (before first non-option)
for nodes in options.and_nodes:
if nodes == '-':
process_stdin(xset.intersection_update, xset.__class__,
options.autostep)
else:
xset.intersection_update(class_set(nodes,
autostep=options.autostep))
for nodes in options.sub_nodes:
if nodes == '-':
process_stdin(xset.difference_update, xset.__class__,
options.autostep)
else:
xset.difference_update(class_set(nodes, autostep=options.autostep))
for nodes in options.xor_nodes:
if nodes == '-':
process_stdin(xset.symmetric_difference_update, xset.__class__,
options.autostep)
else:
xset.symmetric_difference_update(class_set(nodes, \
autostep=options.autostep))
# Finish xset computing from args
compute_nodeset(xset, args, options.autostep)
# Interprate special characters (may raise SyntaxError)
separator = eval('\'%s\'' % options.separator, {"__builtins__":None}, {})
if options.slice_rangeset:
_xset = class_set()
for sli in RangeSet(options.slice_rangeset).slices(False):
_xset.update(xset[sli])
xset = _xset
# Display result according to command choice
if options.expand:
xsubres = separator.join
elif options.fold:
xsubres = lambda x: x
elif options.regroup:
xsubres = lambda x: x.regroup(options.groupsource, \
noprefix=options.groupbase)
else:
xsubres = len
if not xset or options.maxsplit <= 1:
print xsubres(xset)
else:
for xsubset in xset.split(options.maxsplit):
print xsubres(xsubset)
def ev_timeout(self, worker):
self._display.vprint_err(VERB_QUIET, "clush: %s: command timeout" % \
NodeSet._fromlist1(worker.iter_keys_timeout()))
