def display(tree, disp, gather, trace_mode, enable_nodeset_key): """nicely display MsgTree instance `tree' content according to `disp' Display object and `gather' boolean flag""" out = sys.stdout try: if trace_mode: display_tree(tree, disp, out) else: if gather: if enable_nodeset_key: # lambda to create a NodeSet from keys returned by walk() ns_getter = lambda x: NodeSet.fromlist(x[1]) for nodeset in sorted(imap(ns_getter, tree.walk()), cmp=nodeset_cmp): disp.print_gather(nodeset, tree[nodeset[0]]) else: for msg, key in tree.walk(): disp.print_gather_keys(key, msg) else: if enable_nodeset_key: # nodes are automagically sorted by NodeSet for node in NodeSet.fromlist(tree.keys()).nsiter(): disp.print_gather(node, tree[str(node)]) else: for key in tree.keys(): disp.print_gather_keys([ key ], tree[key]) finally: out.flush()
def display(tree, disp, gather, trace_mode, enable_nodeset_key): """nicely display MsgTree instance `tree' content according to `disp' Display object and `gather' boolean flag""" out = sys.stdout try: if trace_mode: display_tree(tree, disp, out) else: if gather: if enable_nodeset_key: # lambda to create a NodeSet from keys returned by walk() ns_getter = lambda x: NodeSet.fromlist(x[1]) for nodeset in sorted(imap(ns_getter, tree.walk()), cmp=nodeset_cmp): disp.print_gather(nodeset, tree[nodeset[0]]) else: for msg, key in tree.walk(): disp.print_gather_keys(key, msg) else: if enable_nodeset_key: # nodes are automagically sorted by NodeSet for node in NodeSet.fromlist(tree.keys()).nsiter(): disp.print_gather(node, tree[str(node)]) else: for key in tree.keys(): disp.print_gather_keys([key], tree[key]) finally: out.flush()
def _close_common(self, worker): verbexit = VERB_QUIET if self._display.maxrc: verbexit = VERB_STD # Display return code if not ok ( != 0) for rc, nodelist in worker.iter_retcodes(): if rc != 0: ns = NodeSet.fromlist(nodelist) self._display.vprint_err(verbexit, \ "clush: %s: exited with exit code %d" % (ns, rc)) # Display nodes that didn't answer within command timeout delay if worker.num_timeout() > 0: self._display.vprint_err(verbexit, "clush: %s: command timeout" % \ NodeSet.fromlist(worker.iter_keys_timeout()))
def _live_line(self, worker): # if all nodes have replied, display gathered line while self._mtreeq and len(self._mtreeq[0]) == len(self._nodes): mtree = self._mtreeq.pop(0) self._offload += 1 self._runtimer_clean() nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1])) for buf, nodeset in sorted(map(nodesetify, mtree.walk()), cmp=bufnodeset_cmp): self._display.print_gather(nodeset, buf) self._runtimer_set_dirty()
def display_tree(tree, disp, out): """display sub-routine for clubak -T (msgtree trace mode)""" togh = True offset = 2 reldepth = -offset reldepths = {} line_mode = disp.line_mode for msgline, keys, depth, nchildren in tree.walk_trace(): if togh: if depth in reldepths: reldepth = reldepths[depth] else: reldepth = reldepths[depth] = reldepth + offset if line_mode: out.write("%s:\n" % NodeSet.fromlist(keys)) else: out.write("%s\n" % \ (disp.format_header(NodeSet.fromlist(keys), reldepth))) out.write("%s%s\n" % (" " * reldepth, msgline)) togh = nchildren != 1
def _live_line(self, worker): # if all nodes have replied, display gathered line while self._mtreeq and len(self._mtreeq[0]) == len(self._nodes): mtree = self._mtreeq.pop(0) self._offload += 1 self._runtimer_clean() nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1])) for buf, nodeset in sorted(map(nodesetify, mtree.walk()), cmp=bufnodeset_cmp): self._display.print_gather(nodeset, buf) self._runtimer_set_dirty()
def display_tree(tree, disp, out): """display sub-routine for clubak -T (msgtree trace mode)""" togh = True offset = 2 reldepth = -offset reldepths = {} line_mode = disp.line_mode for msgline, keys, depth, nchildren in tree.walk_trace(): if togh: if depth in reldepths: reldepth = reldepths[depth] else: reldepth = reldepths[depth] = reldepth + offset if line_mode: out.write("%s:\n" % NodeSet.fromlist(keys)) else: out.write("%s\n" % \ (disp.format_header(NodeSet.fromlist(keys), reldepth))) out.write("%s%s\n" % (" " * reldepth, msgline)) togh = nchildren != 1
def ev_close(self, worker): # Worker is closing -- it's time to gather results... self._runtimer_finalize(worker) for mtree in self._mtreeq: nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1])) for buf, nodeset in sorted(map(nodesetify, mtree.walk()), cmp=bufnodeset_cmp): self._display.print_gather(nodeset, buf) self._close_common(worker) # Notify main thread to update its prompt self.update_prompt(worker)
def ev_close(self, worker): # Worker is closing -- it's time to gather results... self._runtimer_finalize(worker) for mtree in self._mtreeq: nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1])) for buf, nodeset in sorted(map(nodesetify, mtree.walk()), cmp=bufnodeset_cmp): self._display.print_gather(nodeset, buf) self._close_common(worker) # Notify main thread to update its prompt self.update_prompt(worker)
def display_tree(tree, disp, out): """display sub-routine for clubak -T (msgtree trace mode)""" togh = True offset = 2 reldepth = -offset reldepths = {} line_mode = disp.line_mode for msgline, keys, depth, nchildren in tree.walk_trace(): if togh: if depth in reldepths: reldepth = reldepths[depth] else: reldepth = reldepths[depth] = reldepth + offset nodeset = NodeSet.fromlist(keys) if line_mode: out.write(str(nodeset).encode() + b':\n') else: out.write(disp.format_header(nodeset, reldepth) + b'\n') out.write(b' ' * reldepth + msgline + b'\n') togh = nchildren != 1
def ev_close(self, worker): # Worker is closing -- it's time to gather results... self._runtimer_finalize(worker) assert worker.current_node is not None, "cannot gather local command" # Display command output, try to order buffers by rc nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1])) cleaned = False for rc, nodelist in worker.iter_retcodes(): # Then order by node/nodeset (see bufnodeset_cmp) for buf, nodeset in sorted(map(nodesetify, worker.iter_buffers(nodelist)), cmp=bufnodeset_cmp): if not cleaned: # clean runtimer line before printing first result self._runtimer_clean() cleaned = True self._display.print_gather(nodeset, buf) self._close_common(worker) # Notify main thread to update its prompt self.update_prompt(worker)
def ttyloop(task, nodeset, timeout, display, remote): """Manage the interactive prompt to run command""" readline_avail = False interactive = task.default("USER_interactive") if interactive: try: import readline readline_setup() readline_avail = True except ImportError: pass display.vprint(VERB_STD, \ "Enter 'quit' to leave this interactive mode") rc = 0 ns = NodeSet(nodeset) ns_info = True cmd = "" while task.default("USER_running") or \ (interactive and cmd.lower() != 'quit'): try: # Set SIGUSR1 handler if needed if task.default("USER_handle_SIGUSR1"): signal.signal(signal.SIGUSR1, signal_handler) if task.default("USER_interactive") and \ not task.default("USER_running"): if ns_info: display.vprint(VERB_QUIET, \ "Working with nodes: %s" % ns) ns_info = False prompt = "clush> " else: prompt = "" try: cmd = raw_input(prompt) assert cmd is not None, "Result of raw_input() is None!" finally: signal.signal(signal.SIGUSR1, signal.SIG_IGN) except EOFError: print return except UpdatePromptException: if task.default("USER_interactive"): continue return except KeyboardInterrupt, kbe: # Caught SIGINT here (main thread) but the signal will also reach # subprocesses (that will most likely kill them) if display.gather: # Suspend task, so we can safely access its data from here task.suspend() # If USER_running is not set, the task had time to finish, # that could mean all subprocesses have been killed and all # handlers have been processed. if not task.default("USER_running"): # let's clush_excepthook handle the rest raise kbe # If USER_running is set, the task didn't have time to finish # its work, so we must print something for the user... print_warn = False # Display command output, but cannot order buffers by rc nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1])) for buf, nodeset in sorted(map(nodesetify, task.iter_buffers()), cmp=bufnodeset_cmp): if not print_warn: print_warn = True display.vprint_err(VERB_STD, \ "Warning: Caught keyboard interrupt!") display.print_gather(nodeset, buf) # Return code handling verbexit = VERB_QUIET if display.maxrc: verbexit = VERB_STD ns_ok = NodeSet() for rc, nodelist in task.iter_retcodes(): ns_ok.add(NodeSet._fromlist1(nodelist)) if rc != 0: # Display return code if not ok ( != 0) ns = NodeSet._fromlist1(nodelist) display.vprint_err(verbexit, \ "clush: %s: exited with exit code %s" % (ns, rc)) # Add uncompleted nodeset to exception object kbe.uncompleted_nodes = ns - ns_ok # Display nodes that didn't answer within command timeout delay if task.num_timeout() > 0: display.vprint_err(verbexit, \ "clush: %s: command timeout" % \ NodeSet._fromlist1(task.iter_keys_timeout())) raise kbe if task.default("USER_running"): ns_reg, ns_unreg = NodeSet(), NodeSet() for client in task._engine.clients(): if client.registered: ns_reg.add(client.key) else: ns_unreg.add(client.key) if ns_unreg: pending = "\nclush: pending(%d): %s" % (len(ns_unreg), ns_unreg) else: pending = "" display.vprint_err(VERB_QUIET, "clush: interrupt (^C to abort task)") gws = task.gateways.keys() if not gws: display.vprint_err(VERB_QUIET, "clush: in progress(%d): %s%s" % (len(ns_reg), ns_reg, pending)) else: display.vprint_err(VERB_QUIET, "clush: in progress(%d): %s%s\n" "clush: [tree] open gateways(%d): %s" % (len(ns_reg), ns_reg, pending, len(gws), NodeSet._fromlist1(gws))) for gw, (chan, metaworkers) in task.gateways.iteritems(): act_targets = NodeSet.fromlist(mw.gwtargets[gw] for mw in metaworkers) if act_targets: display.vprint_err(VERB_QUIET, "clush: [tree] in progress(%d) on %s: %s" % (len(act_targets), gw, act_targets)) else: cmdl = cmd.lower() try: ns_info = True if cmdl.startswith('+'): ns.update(cmdl[1:]) elif cmdl.startswith('-'): ns.difference_update(cmdl[1:]) elif cmdl.startswith('@'): ns = NodeSet(cmdl[1:]) elif cmdl == '=': display.gather = not display.gather if display.gather: display.vprint(VERB_STD, \ "Switching to gathered output format") else: display.vprint(VERB_STD, \ "Switching to standard output format") task.set_default("stdout_msgtree", \ display.gather or display.line_mode) ns_info = False continue elif not cmdl.startswith('?'): # if ?, just print ns_info ns_info = False except NodeSetParseError: display.vprint_err(VERB_QUIET, \ "clush: nodeset parse error (ignoring)") if ns_info: continue if cmdl.startswith('!') and len(cmd.strip()) > 0: run_command(task, cmd[1:], None, timeout, display, remote) elif cmdl != "quit": if not cmd: continue if readline_avail: readline.write_history_file(get_history_file()) run_command(task, cmd, ns, timeout, display, remote)
# Compute the nodeset and warn for possible use of shell pathname # expansion (#225) wnodelist = xnodelist = [] if options.nodes: wnodelist = [NodeSet(nodes) for nodes in options.nodes] if options.exclude: xnodelist = [NodeSet(nodes) for nodes in options.exclude] for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)): for nodes in nodelist: if len(nodes) == 1 and exists(str(nodes)): display.vprint_err(VERB_STD, "Warning: using '-%s %s' and " "local path '%s' exists, was it expanded " "by the shell?" % (opt, nodes, nodes)) nodeset_base = NodeSet.fromlist(wnodelist) nodeset_exclude = NodeSet.fromlist(xnodelist) # FIXME: add public API to enforce engine Task._std_default['engine'] = options.engine # Do we have nodes group? task = task_self() task.set_info("debug", config.verbosity >= VERB_DEBUG) if config.verbosity == VERB_DEBUG: std_group_resolver().set_verbosity(1) if options.nodes_all: all_nodeset = NodeSet.fromall() display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \ all_nodeset) nodeset_base.add(all_nodeset)
def ev_timeout(self, worker): self._display.vprint_err(VERB_QUIET, "clush: %s: command timeout" % \ NodeSet.fromlist(worker.iter_keys_timeout()))
def main(): """clush script entry point""" sys.excepthook = clush_excepthook # # Argument management # usage = "%prog [options] command" parser = OptionParser(usage) parser.add_option("--nostdin", action="store_true", dest="nostdin", help="don't watch for possible input from stdin") parser.install_config_options('clush.conf(5)') parser.install_nodes_options() parser.install_display_options(verbose_options=True) parser.install_filecopy_options() parser.install_connector_options() (options, args) = parser.parse_args() # # Load config file and apply overrides # config = ClushConfig(options) # Should we use ANSI colors for nodes? if config.color == "auto": color = sys.stdout.isatty() and (options.gatherall or \ sys.stderr.isatty()) else: color = config.color == "always" try: # Create and configure display object. display = Display(options, config, color) except ValueError as exc: parser.error("option mismatch (%s)" % exc) if options.groupsource: # Be sure -a/g -s source work as espected. std_group_resolver().default_source_name = options.groupsource # Compute the nodeset and warn for possible use of shell pathname # expansion (#225) wnodelist = [] xnodelist = [] if options.nodes: wnodelist = [NodeSet(nodes) for nodes in options.nodes] if options.exclude: xnodelist = [NodeSet(nodes) for nodes in options.exclude] for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)): for nodes in nodelist: if len(nodes) == 1 and exists(str(nodes)): display.vprint_err( VERB_STD, "Warning: using '-%s %s' and " "local path '%s' exists, was it expanded " "by the shell?" % (opt, nodes, nodes)) # --hostfile support (#235) for opt_hostfile in options.hostfile: try: fnodeset = NodeSet() hostfile = open(opt_hostfile) for line in hostfile.read().splitlines(): fnodeset.updaten(nodes for nodes in line.split()) hostfile.close() display.vprint_err( VERB_DEBUG, "Using nodeset %s from hostfile %s" % (fnodeset, opt_hostfile)) wnodelist.append(fnodeset) except IOError as exc: # re-raise as OSError to be properly handled errno, strerror = exc.args raise OSError(errno, strerror, exc.filename) # Instantiate target nodeset from command line and hostfile nodeset_base = NodeSet.fromlist(wnodelist) # Instantiate filter nodeset (command line only) nodeset_exclude = NodeSet.fromlist(xnodelist) # Specified engine prevails over default engine DEFAULTS.engine = options.engine # Do we have nodes group? task = task_self() task.set_info("debug", config.verbosity >= VERB_DEBUG) if config.verbosity == VERB_DEBUG: std_group_resolver().set_verbosity(1) if options.nodes_all: all_nodeset = NodeSet.fromall() display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \ all_nodeset) nodeset_base.add(all_nodeset) if options.group: grp_nodeset = NodeSet.fromlist(options.group, resolver=RESOLVER_NOGROUP) for grp in grp_nodeset: addingrp = NodeSet("@" + grp) display.vprint(VERB_DEBUG, \ "Adding nodes from option -g %s: %s" % (grp, addingrp)) nodeset_base.update(addingrp) if options.exgroup: grp_nodeset = NodeSet.fromlist(options.exgroup, resolver=RESOLVER_NOGROUP) for grp in grp_nodeset: removingrp = NodeSet("@" + grp) display.vprint(VERB_DEBUG, \ "Excluding nodes from option -X %s: %s" % (grp, removingrp)) nodeset_exclude.update(removingrp) # Do we have an exclude list? (-x ...) nodeset_base.difference_update(nodeset_exclude) if len(nodeset_base) < 1: parser.error('No node to run on.') if options.pick and options.pick < len(nodeset_base): # convert to string for sample as nsiter() is slower for big # nodesets; and we assume options.pick will remain small-ish keep = random.sample(nodeset_base, options.pick) nodeset_base.intersection_update(','.join(keep)) if config.verbosity >= VERB_VERB: msg = "Picked random nodes: %s" % nodeset_base print Display.COLOR_RESULT_FMT % msg # Set open files limit. set_fdlimit(config.fd_max, display) # # Task management # # check for clush interactive mode interactive = not len(args) and \ not (options.copy or options.rcopy) # check for foreground ttys presence (input) stdin_isafgtty = sys.stdin.isatty() and \ os.tcgetpgrp(sys.stdin.fileno()) == os.getpgrp() # check for special condition (empty command and stdin not a tty) if interactive and not stdin_isafgtty: # looks like interactive but stdin is not a tty: # switch to non-interactive + disable ssh pseudo-tty interactive = False # SSH: disable pseudo-tty allocation (-T) ssh_options = config.ssh_options or '' ssh_options += ' -T' config._set_main("ssh_options", ssh_options) if options.nostdin and interactive: parser.error("illegal option `--nostdin' in that case") # Force user_interaction if Clush._f_user_interaction for test purposes user_interaction = hasattr(sys.modules[__name__], '_f_user_interaction') if not options.nostdin: # Try user interaction: check for foreground ttys presence (ouput) stdout_isafgtty = sys.stdout.isatty() and \ os.tcgetpgrp(sys.stdout.fileno()) == os.getpgrp() user_interaction |= stdin_isafgtty and stdout_isafgtty display.vprint(VERB_DEBUG, "User interaction: %s" % user_interaction) if user_interaction: # Standard input is a terminal and we want to perform some user # interactions in the main thread (using blocking calls), so # we run cluster commands in a new ClusterShell Task (a new # thread is created). task = Task() # else: perform everything in the main thread # Handle special signal only when user_interaction is set task.set_default("USER_handle_SIGUSR1", user_interaction) task.excepthook = sys.excepthook task.set_default("USER_stdin_worker", not (sys.stdin.isatty() or \ options.nostdin or \ user_interaction)) display.vprint(VERB_DEBUG, "Create STDIN worker: %s" % \ task.default("USER_stdin_worker")) if config.verbosity >= VERB_DEBUG: task.set_info("debug", True) logging.basicConfig(level=logging.DEBUG) logging.debug("clush: STARTING DEBUG") else: logging.basicConfig(level=logging.CRITICAL) task.set_info("fanout", config.fanout) if options.worker: try: if options.remote == 'no': task.set_default('local_worker', _load_workerclass(options.worker)) else: task.set_default('distant_worker', _load_workerclass(options.worker)) except (ImportError, AttributeError): msg = "ERROR: Could not load worker '%s'" % options.worker display.vprint_err(VERB_QUIET, msg) clush_exit(1, task) if options.topofile or task._default_tree_is_enabled(): if options.topofile: task.load_topology(options.topofile) if config.verbosity >= VERB_VERB: roots = len(task.topology.root.nodeset) gws = task.topology.inner_node_count() - roots msg = "enabling tree topology (%d gateways)" % gws print >> sys.stderr, "clush: %s" % msg if options.grooming_delay: if config.verbosity >= VERB_VERB: msg = Display.COLOR_RESULT_FMT % ("Grooming delay: %f" % options.grooming_delay) print >> sys.stderr, msg task.set_info("grooming_delay", options.grooming_delay) elif options.rcopy: # By default, --rcopy should inhibit grooming task.set_info("grooming_delay", 0) if config.ssh_user: task.set_info("ssh_user", config.ssh_user) if config.ssh_path: task.set_info("ssh_path", config.ssh_path) if config.ssh_options: task.set_info("ssh_options", config.ssh_options) if config.scp_path: task.set_info("scp_path", config.scp_path) if config.scp_options: task.set_info("scp_options", config.scp_options) if config.rsh_path: task.set_info("rsh_path", config.rsh_path) if config.rcp_path: task.set_info("rcp_path", config.rcp_path) if config.rsh_options: task.set_info("rsh_options", config.rsh_options) # Set detailed timeout values task.set_info("connect_timeout", config.connect_timeout) task.set_info("command_timeout", config.command_timeout) # Enable stdout/stderr separation task.set_default("stderr", not options.gatherall) # Disable MsgTree buffering if not gathering outputs task.set_default("stdout_msgtree", display.gather or display.line_mode) # Always disable stderr MsgTree buffering task.set_default("stderr_msgtree", False) # Set timeout at worker level when command_timeout is defined. if config.command_timeout > 0: timeout = config.command_timeout else: timeout = -1 # Configure task custom status task.set_default("USER_interactive", interactive) task.set_default("USER_running", False) if (options.copy or options.rcopy) and not args: parser.error("--[r]copy option requires at least one argument") if options.copy: if not options.dest_path: # append '/' to clearly indicate a directory for tree mode options.dest_path = join(dirname(abspath(args[0])), '') op = "copy sources=%s dest=%s" % (args, options.dest_path) elif options.rcopy: if not options.dest_path: options.dest_path = dirname(abspath(args[0])) op = "rcopy sources=%s dest=%s" % (args, options.dest_path) else: op = "command=\"%s\"" % ' '.join(args) # print debug values (fanout value is get from the config object # and not task itself as set_info() is an asynchronous call. display.vprint(VERB_DEBUG, "clush: nodeset=%s fanout=%d [timeout " \ "conn=%.1f cmd=%.1f] %s" % (nodeset_base, config.fanout, config.connect_timeout, config.command_timeout, op)) if not task.default("USER_interactive"): if display.verbosity >= VERB_DEBUG and task.topology: print Display.COLOR_RESULT_FMT % '-' * 15 print Display.COLOR_RESULT_FMT % task.topology, print Display.COLOR_RESULT_FMT % '-' * 15 if options.copy: run_copy(task, args, options.dest_path, nodeset_base, timeout, options.preserve_flag, display) elif options.rcopy: run_rcopy(task, args, options.dest_path, nodeset_base, timeout, options.preserve_flag, display) else: run_command(task, ' '.join(args), nodeset_base, timeout, display, options.remote != 'no') if user_interaction: ttyloop(task, nodeset_base, timeout, display, options.remote != 'no') elif task.default("USER_interactive"): display.vprint_err(VERB_QUIET, \ "ERROR: interactive mode requires a tty") clush_exit(1, task) rc = 0 if options.maxrc: # Instead of clush return code, return commands retcode rc = task.max_retcode() if task.num_timeout() > 0: rc = 255 clush_exit(rc, task)
def ttyloop(task, nodeset, timeout, display, remote): """Manage the interactive prompt to run command""" readline_avail = False interactive = task.default("USER_interactive") if interactive: try: import readline readline_setup() readline_avail = True except ImportError: pass display.vprint(VERB_STD, \ "Enter 'quit' to leave this interactive mode") rc = 0 ns = NodeSet(nodeset) ns_info = True cmd = "" while task.default("USER_running") or \ (interactive and cmd.lower() != 'quit'): try: # Set SIGUSR1 handler if needed if task.default("USER_handle_SIGUSR1"): signal.signal(signal.SIGUSR1, signal_handler) if task.default("USER_interactive") and \ not task.default("USER_running"): if ns_info: display.vprint(VERB_QUIET, \ "Working with nodes: %s" % ns) ns_info = False prompt = "clush> " else: prompt = "" try: cmd = raw_input(prompt) assert cmd is not None, "Result of raw_input() is None!" finally: signal.signal(signal.SIGUSR1, signal.SIG_IGN) except EOFError: print return except UpdatePromptException: if task.default("USER_interactive"): continue return except KeyboardInterrupt as kbe: # Caught SIGINT here (main thread) but the signal will also reach # subprocesses (that will most likely kill them) if display.gather: # Suspend task, so we can safely access its data from here task.suspend() # If USER_running is not set, the task had time to finish, # that could mean all subprocesses have been killed and all # handlers have been processed. if not task.default("USER_running"): # let's clush_excepthook handle the rest raise kbe # If USER_running is set, the task didn't have time to finish # its work, so we must print something for the user... print_warn = False # Display command output, but cannot order buffers by rc nodesetify = lambda v: (v[0], NodeSet._fromlist1(v[1])) for buf, nodeset in sorted(map(nodesetify, task.iter_buffers()), cmp=bufnodeset_cmp): if not print_warn: print_warn = True display.vprint_err(VERB_STD, \ "Warning: Caught keyboard interrupt!") display.print_gather(nodeset, buf) # Return code handling verbexit = VERB_QUIET if display.maxrc: verbexit = VERB_STD ns_ok = NodeSet() for rc, nodelist in task.iter_retcodes(): ns_ok.add(NodeSet._fromlist1(nodelist)) if rc != 0: # Display return code if not ok ( != 0) nsdisp = ns = NodeSet._fromlist1(nodelist) if display.verbosity >= VERB_QUIET and len(ns) > 1: nsdisp = "%s (%d)" % (ns, len(ns)) msgrc = "clush: %s: exited with exit code %d" % ( nsdisp, rc) display.vprint_err(verbexit, msgrc) # Add uncompleted nodeset to exception object kbe.uncompleted_nodes = ns - ns_ok # Display nodes that didn't answer within command timeout delay if task.num_timeout() > 0: display.vprint_err(verbexit, \ "clush: %s: command timeout" % \ NodeSet._fromlist1(task.iter_keys_timeout())) raise kbe if task.default("USER_running"): ns_reg, ns_unreg = NodeSet(), NodeSet() for client in task._engine.clients(): if client.registered: ns_reg.add(client.key) else: ns_unreg.add(client.key) if ns_unreg: pending = "\nclush: pending(%d): %s" % (len(ns_unreg), ns_unreg) else: pending = "" display.vprint_err(VERB_QUIET, "clush: interrupt (^C to abort task)") gws = task.gateways.keys() if not gws: display.vprint_err( VERB_QUIET, "clush: in progress(%d): %s%s" % (len(ns_reg), ns_reg, pending)) else: display.vprint_err( VERB_QUIET, "clush: in progress(%d): %s%s\n" "clush: [tree] open gateways(%d): %s" % (len(ns_reg), ns_reg, pending, len(gws), NodeSet._fromlist1(gws))) for gw, (chan, metaworkers) in task.gateways.iteritems(): act_targets = NodeSet.fromlist(mw.gwtargets[gw] for mw in metaworkers) if act_targets: display.vprint_err( VERB_QUIET, "clush: [tree] in progress(%d) on %s: %s" % (len(act_targets), gw, act_targets)) else: cmdl = cmd.lower() try: ns_info = True if cmdl.startswith('+'): ns.update(cmdl[1:]) elif cmdl.startswith('-'): ns.difference_update(cmdl[1:]) elif cmdl.startswith('@'): ns = NodeSet(cmdl[1:]) elif cmdl == '=': display.gather = not display.gather if display.gather: display.vprint(VERB_STD, \ "Switching to gathered output format") else: display.vprint(VERB_STD, \ "Switching to standard output format") task.set_default("stdout_msgtree", \ display.gather or display.line_mode) ns_info = False continue elif not cmdl.startswith('?'): # if ?, just print ns_info ns_info = False except NodeSetParseError: display.vprint_err(VERB_QUIET, \ "clush: nodeset parse error (ignoring)") if ns_info: continue if cmdl.startswith('!') and len(cmd.strip()) > 0: run_command(task, cmd[1:], None, timeout, display, remote) elif cmdl != "quit": if not cmd: continue if readline_avail: readline.write_history_file(get_history_file()) run_command(task, cmd, ns, timeout, display, remote) return rc
fnodeset = NodeSet() hostfile = open(opt_hostfile) for line in hostfile.read().splitlines(): fnodeset.updaten(nodes for nodes in line.split()) hostfile.close() display.vprint_err( VERB_DEBUG, "Using nodeset %s from hostfile %s" % (fnodeset, opt_hostfile)) wnodelist.append(fnodeset) except IOError, exc: # re-raise as OSError to be properly handled errno, strerror = exc.args raise OSError(errno, strerror, exc.filename) # Instantiate target nodeset from command line and hostfile nodeset_base = NodeSet.fromlist(wnodelist) # Instantiate filter nodeset (command line only) nodeset_exclude = NodeSet.fromlist(xnodelist) # Specified engine prevails over default engine DEFAULTS.engine = options.engine # Do we have nodes group? task = task_self() task.set_info("debug", config.verbosity >= VERB_DEBUG) if config.verbosity == VERB_DEBUG: std_group_resolver().set_verbosity(1) if options.nodes_all: all_nodeset = NodeSet.fromall() display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \ all_nodeset)
def main(args=sys.argv): """clush script entry point""" sys.excepthook = clush_excepthook # Default values nodeset_base, nodeset_exclude = NodeSet(), NodeSet() # # Argument management # usage = "%prog [options] command" parser = OptionParser(usage) parser.add_option("--nostdin", action="store_true", dest="nostdin", help="don't watch for possible input from stdin") parser.install_nodes_options() parser.install_display_options(verbose_options=True) parser.install_filecopy_options() parser.install_ssh_options() (options, args) = parser.parse_args(args[1:]) # # Load config file and apply overrides # config = ClushConfig(options) # Should we use ANSI colors for nodes? if config.color == "auto": color = sys.stdout.isatty() and (options.gatherall or \ sys.stderr.isatty()) else: color = config.color == "always" # Create and configure display object. display = Display(options, config, color) # # Compute the nodeset # if options.nodes: nodeset_base = NodeSet.fromlist(options.nodes) if options.exclude: nodeset_exclude = NodeSet.fromlist(options.exclude) if options.groupsource: # Be sure -a/g -s source work as espected. STD_GROUP_RESOLVER.default_sourcename = options.groupsource # FIXME: add public API to enforce engine Task._std_default['engine'] = options.engine # Do we have nodes group? task = task_self() task.set_info("debug", config.verbosity > 1) if config.verbosity == VERB_DEBUG: STD_GROUP_RESOLVER.set_verbosity(1) if options.nodes_all: all_nodeset = NodeSet.fromall() display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \ all_nodeset) nodeset_base.add(all_nodeset) if options.group: grp_nodeset = NodeSet.fromlist(options.group, resolver=NOGROUP_RESOLVER) for grp in grp_nodeset: addingrp = NodeSet("@" + grp) display.vprint(VERB_DEBUG, \ "Adding nodes from option -g %s: %s" % (grp, addingrp)) nodeset_base.update(addingrp) if options.exgroup: grp_nodeset = NodeSet.fromlist(options.exgroup, resolver=NOGROUP_RESOLVER) for grp in grp_nodeset: removingrp = NodeSet("@" + grp) display.vprint(VERB_DEBUG, \ "Excluding nodes from option -X %s: %s" % (grp, removingrp)) nodeset_exclude.update(removingrp) # Do we have an exclude list? (-x ...) nodeset_base.difference_update(nodeset_exclude) if len(nodeset_base) < 1: parser.error('No node to run on.') # Set open files limit. set_fdlimit(config.fd_max, display) # # Task management # # check for clush interactive mode interactive = not len(args) and \ not (options.copy or options.rcopy) # check for foreground ttys presence (input) stdin_isafgtty = sys.stdin.isatty() and \ os.tcgetpgrp(sys.stdin.fileno()) == os.getpgrp() # check for special condition (empty command and stdin not a tty) if interactive and not stdin_isafgtty: # looks like interactive but stdin is not a tty: # switch to non-interactive + disable ssh pseudo-tty interactive = False # SSH: disable pseudo-tty allocation (-T) ssh_options = config.ssh_options or '' ssh_options += ' -T' config._set_main("ssh_options", ssh_options) if options.nostdin and interactive: parser.error("illegal option `--nostdin' in that case") user_interaction = False if not options.nostdin: # Try user interaction: check for foreground ttys presence (ouput) stdout_isafgtty = sys.stdout.isatty() and \ os.tcgetpgrp(sys.stdout.fileno()) == os.getpgrp() user_interaction = stdin_isafgtty and stdout_isafgtty display.vprint(VERB_DEBUG, "User interaction: %s" % user_interaction) if user_interaction: # Standard input is a terminal and we want to perform some user # interactions in the main thread (using blocking calls), so # we run cluster commands in a new ClusterShell Task (a new # thread is created). task = Task() # else: perform everything in the main thread # Handle special signal only when user_interaction is set task.set_default("USER_handle_SIGUSR1", user_interaction) task.excepthook = sys.excepthook task.set_default("USER_stdin_worker", not (sys.stdin.isatty() or \ options.nostdin)) display.vprint(VERB_DEBUG, "Create STDIN worker: %s" % \ task.default("USER_stdin_worker")) task.set_info("debug", config.verbosity >= VERB_DEBUG) task.set_info("fanout", config.fanout) if config.ssh_user: task.set_info("ssh_user", config.ssh_user) if config.ssh_path: task.set_info("ssh_path", config.ssh_path) if config.ssh_options: task.set_info("ssh_options", config.ssh_options) # Set detailed timeout values task.set_info("connect_timeout", config.connect_timeout) command_timeout = config.command_timeout task.set_info("command_timeout", command_timeout) # Enable stdout/stderr separation task.set_default("stderr", not options.gatherall) # Disable MsgTree buffering if not gathering outputs task.set_default("stdout_msgtree", display.gather or display.line_mode) # Always disable stderr MsgTree buffering task.set_default("stderr_msgtree", False) # Set timeout at worker level when command_timeout is defined. if command_timeout > 0: timeout = command_timeout else: timeout = -1 # Configure task custom status task.set_default("USER_interactive", interactive) task.set_default("USER_running", False) if (options.copy or options.rcopy) and not args: parser.error("--[r]copy option requires at least one argument") if options.copy: if not options.dest_path: options.dest_path = os.path.dirname(args[0]) op = "copy sources=%s dest=%s" % (args, options.dest_path) elif options.rcopy: if not options.dest_path: options.dest_path = os.path.dirname(args[0]) op = "rcopy sources=%s dest=%s" % (args, options.dest_path) else: op = "command=\"%s\"" % ' '.join(args) # print debug values (fanout value is get from the config object # and not task itself as set_info() is an asynchronous call. display.vprint(VERB_DEBUG, "clush: nodeset=%s fanout=%d [timeout " \ "conn=%.1f cmd=%.1f] %s" % (nodeset_base, config.fanout, task.info("connect_timeout"), task.info("command_timeout"), op)) if not task.default("USER_interactive"): if options.copy: run_copy(task, args, options.dest_path, nodeset_base, 0, options.preserve_flag, display) elif options.rcopy: run_rcopy(task, args, options.dest_path, nodeset_base, 0, options.preserve_flag, display) else: run_command(task, ' '.join(args), nodeset_base, timeout, display) if user_interaction: ttyloop(task, nodeset_base, timeout, display) elif task.default("USER_interactive"): display.vprint_err(VERB_QUIET, \ "ERROR: interactive mode requires a tty") clush_exit(1) rc = 0 if options.maxrc: # Instead of clush return code, return commands retcode rc = task.max_retcode() if task.num_timeout() > 0: rc = 255 clush_exit(rc)
def ttyloop(task, nodeset, timeout, display): """Manage the interactive prompt to run command""" readline_avail = False if task.default("USER_interactive"): assert sys.stdin.isatty() try: import readline readline_setup() readline_avail = True except ImportError: pass display.vprint(VERB_STD, \ "Enter 'quit' to leave this interactive mode") rc = 0 ns = NodeSet(nodeset) ns_info = True cmd = "" while task.default("USER_running") or cmd.lower() != 'quit': try: if task.default("USER_interactive") and \ not task.default("USER_running"): if ns_info: display.vprint(VERB_QUIET, \ "Working with nodes: %s" % ns) ns_info = False prompt = "clush> " else: prompt = "" # Set SIGUSR1 handler if needed if task.default("USER_handle_SIGUSR1"): signal.signal(signal.SIGUSR1, signal_handler) try: cmd = raw_input(prompt) finally: signal.signal(signal.SIGUSR1, signal.SIG_IGN) except EOFError: print return except UpdatePromptException: if task.default("USER_interactive"): continue return except KeyboardInterrupt, kbe: if display.gather: # Suspend task, so we can safely access its data from # the main thread task.suspend() print_warn = False # Display command output, but cannot order buffers by rc nodesetify = lambda v: (v[0], NodeSet.fromlist(v[1])) for buf, nodeset in sorted(map(nodesetify, task.iter_buffers()), cmp=bufnodeset_cmp): if not print_warn: print_warn = True display.vprint_err(VERB_STD, \ "Warning: Caught keyboard interrupt!") display.print_gather(nodeset, buf) # Return code handling verbexit = VERB_QUIET if display.maxrc: verbexit = VERB_STD ns_ok = NodeSet() for rc, nodelist in task.iter_retcodes(): ns_ok.add(NodeSet.fromlist(nodelist)) if rc != 0: # Display return code if not ok ( != 0) ns = NodeSet.fromlist(nodelist) display.vprint_err(verbexit, \ "clush: %s: exited with exit code %s" % (ns, rc)) # Add uncompleted nodeset to exception object kbe.uncompleted_nodes = ns - ns_ok # Display nodes that didn't answer within command timeout delay if task.num_timeout() > 0: display.vprint_err(verbexit, \ "clush: %s: command timeout" % \ NodeSet.fromlist(task.iter_keys_timeout())) raise kbe if task.default("USER_running"): ns_reg = NodeSet.fromlist([c.key for c in task._engine.clients()]) ns_unreg = NodeSet.fromlist([c.key for c in \ task._engine.pending_clients()]) ns_reg -= ns_unreg if ns_unreg: pending = "\nclush: pending(%d): %s" % (len(ns_unreg), ns_unreg) else: pending = "" display.vprint_err(VERB_QUIET, "clush: interrupt (^C to " \ "abort task)\nclush: in progress(%d): %s%s" % (len(ns_reg), \ ns_reg, pending)) else: cmdl = cmd.lower() try: ns_info = True if cmdl.startswith('+'): ns.update(cmdl[1:]) elif cmdl.startswith('-'): ns.difference_update(cmdl[1:]) elif cmdl.startswith('@'): ns = NodeSet(cmdl[1:]) elif cmdl == '=': display.gather = not display.gather if display.gather: display.vprint(VERB_STD, \ "Switching to gathered output format") else: display.vprint(VERB_STD, \ "Switching to standard output format") ns_info = False continue elif not cmdl.startswith('?'): # if ?, just print ns_info ns_info = False except NodeSetParseError: display.vprint_err(VERB_QUIET, \ "clush: nodeset parse error (ignoring)") if ns_info: continue if cmdl.startswith('!') and len(cmd.strip()) > 0: run_command(task, cmd[1:], None, timeout, display) elif cmdl != "quit": if not cmd: continue if readline_avail: readline.write_history_file(get_history_file()) run_command(task, cmd, ns, timeout, display)