def testClushConfigDefaultWithOptions(self): """test CLI.Config.ClushConfig (default with options)""" f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write( dedent(""" [Main] fanout: 42 connect_timeout: 14 command_timeout: 0 history_size: 100 color: auto verbosity: 1""").encode()) f.flush() parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([ "-f", "36", "-u", "3", "-t", "7", "--user", "foobar", "--color", "always", "-d", "-v", "-q", "-o", "-oSomething" ]) config = ClushConfig(options, filename=f.name) display = Display(options, config) display.vprint(VERB_STD, "test") display.vprint(VERB_DEBUG, "test") self.assertEqual(config.color, WHENCOLOR_CHOICES[1]) self.assertEqual(config.verbosity, VERB_DEBUG) # takes biggest self.assertEqual(config.fanout, 36) self.assertEqual(config.connect_timeout, 7) self.assertEqual(config.command_timeout, 3) self.assertEqual(config.ssh_user, "foobar") self.assertEqual(config.ssh_path, None) self.assertEqual(config.ssh_options, "-oSomething") f.close()
def testClushConfigError(self): """test CLI.Config.ClushConfig (error)""" f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write( dedent(""" [Main] fanout: 3.2 connect_timeout: foo command_timeout: bar history_size: 100 color: maybe node_count: 3 verbosity: bar ssh_user: root ssh_path: /usr/bin/ssh ssh_options: -oStrictHostKeyChecking=no """).encode()) f.flush() parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options, filename=f.name) try: c = config.color self.fail("Exception ClushConfigError not raised (color)") except ClushConfigError: pass self.assertEqual(config.verbosity, 0) # probably for compatibility try: f = config.fanout self.fail("Exception ClushConfigError not raised (fanout)") except ClushConfigError: pass try: f = config.node_count self.fail("Exception ClushConfigError not raised (node_count)") except ClushConfigError: pass try: f = config.fanout except ClushConfigError as e: self.assertEqual(str(e)[0:20], "(Config Main.fanout)") try: t = config.connect_timeout self.fail( "Exception ClushConfigError not raised (connect_timeout)") except ClushConfigError: pass try: m = config.command_timeout self.fail( "Exception ClushConfigError not raised (command_timeout)") except ClushConfigError: pass f.close()
def testClushConfigSetRlimit(self): """test CLI.Config.ClushConfig (setrlimit)""" soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) hard2 = min(32768, hard) f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write( dedent(""" [Main] fanout: 42 connect_timeout: 14 command_timeout: 0 history_size: 100 color: auto fd_max: %d verbosity: 1 """ % hard2).encode()) f.flush() parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options, filename=f.name) display = Display(options, config) # force a lower soft limit resource.setrlimit(resource.RLIMIT_NOFILE, (hard2 / 2, hard)) # max_fdlimit should increase soft limit again set_fdlimit(config.fd_max, display) # verify soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) self.assertEqual(soft, hard2) f.close()
def testClushConfigSetRlimitValueError(self): """test CLI.Config.ClushConfig (setrlimit ValueError)""" soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write(dedent(""" [Main] fanout: 42 connect_timeout: 14 command_timeout: 0 history_size: 100 color: auto # Use wrong fd_max value to generate ValueError fd_max: -1 verbosity: 1""")) f.flush() parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options, filename=f.name) f.close() display = Display(options, config) class TestException(Exception): pass def mock_vprint_err(level, message): if message.startswith('Warning: Failed to set max open files'): raise TestException() display.vprint_err = mock_vprint_err self.assertRaises(TestException, set_fdlimit, config.fd_max, display) soft2, _ = resource.getrlimit(resource.RLIMIT_NOFILE) self.assertEqual(soft, soft2)
def testClushConfigFull(self): """test CLI.Config.ClushConfig (full)""" f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write(""" [Main] fanout: 42 connect_timeout: 14 command_timeout: 0 history_size: 100 color: auto node_count: yes verbosity: 1 ssh_user: root ssh_path: /usr/bin/ssh ssh_options: -oStrictHostKeyChecking=no """) f.flush() parser = OptionParser("dummy") parser.install_display_options(verbose_options=True) parser.install_ssh_options() options, _ = parser.parse_args([]) config = ClushConfig(options, filename=f.name) self.assert_(config != None) self.assertEqual(config.color, WHENCOLOR_CHOICES[2]) self.assertEqual(config.verbosity, VERB_STD) self.assertEqual(config.node_count, True) self.assertEqual(config.fanout, 42) self.assertEqual(config.connect_timeout, 14) self.assertEqual(config.command_timeout, 0) self.assertEqual(config.ssh_user, "root") self.assertEqual(config.ssh_path, "/usr/bin/ssh") self.assertEqual(config.ssh_options, "-oStrictHostKeyChecking=no") f.close()
def testClushConfigUserOverride(self): """test CLI.Config.ClushConfig (XDG_CONFIG_HOME user config)""" # XXX Test should be improved when CLUSTERSHELL_CONFIG is available # Improvement: override CLUSTERSHELL_CONFIG and set a sys clush config # then verify that user config overrides CLUSTERSHELL_CONFIG as # expected... # For now, it has been tested manually. This test only really only # ensures that user config is taken into account. xdg_config_home_save = os.environ.get('XDG_CONFIG_HOME') # Create fake XDG_CONFIG_HOME dname = make_temp_dir() try: os.environ['XDG_CONFIG_HOME'] = dname # create $XDG_CONFIG_HOME/clustershell/clush.conf usercfgdir = os.path.join(dname, 'clustershell') os.mkdir(usercfgdir) cfgfile = open(os.path.join(usercfgdir, 'clush.conf'), 'w') cfgfile.write( dedent(""" [Main] fanout: 42 connect_timeout: 14 command_timeout: 0 history_size: 100 color: never verbosity: 2 ssh_user: trump ssh_path: ~/bin/ssh ssh_options: -oSomeDummyUserOption=yes """)) cfgfile.flush() parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options) # filename=None to use defaults! self.assertEqual(config.color, WHENCOLOR_CHOICES[0]) self.assertEqual(config.verbosity, VERB_VERB) # takes biggest self.assertEqual(config.fanout, 42) self.assertEqual(config.connect_timeout, 14) self.assertEqual(config.command_timeout, 0) self.assertEqual(config.ssh_user, 'trump') self.assertEqual(config.ssh_path, '~/bin/ssh') self.assertEqual(config.ssh_options, '-oSomeDummyUserOption=yes') cfgfile.close() finally: if xdg_config_home_save: os.environ['XDG_CONFIG_HOME'] = xdg_config_home_save else: del os.environ['XDG_CONFIG_HOME'] shutil.rmtree(dname, ignore_errors=True)
def testClushConfigWithInstalledConfig(self): """test CLI.Config.ClushConfig (installed config required)""" # This test needs installed configuration files (needed for # maximum coverage). parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options)
def main(): """clush script entry point""" sys.excepthook = clush_excepthook # Default values nodeset_base, nodeset_exclude = NodeSet(), NodeSet() # # Argument management # usage = "%prog [options] command" parser = OptionParser(usage) parser.add_option("--nostdin", action="store_true", dest="nostdin", help="don't watch for possible input from stdin") parser.install_nodes_options() parser.install_display_options(verbose_options=True) parser.install_filecopy_options() parser.install_ssh_options() (options, args) = parser.parse_args() # # Load config file and apply overrides # config = ClushConfig(options) # Should we use ANSI colors for nodes? if config.color == "auto": color = sys.stdout.isatty() and (options.gatherall or \ sys.stderr.isatty()) else: color = config.color == "always" try: # Create and configure display object. display = Display(options, config, color) except ValueError, exc: parser.error("option mismatch (%s)" % exc)
def testClushConfigAlmostEmpty(self): """test CLI.Config.ClushConfig (almost empty)""" f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write("[Main]\n".encode()) parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options, filename=f.name) self.assertEqual(config.color, WHENCOLOR_CHOICES[-1]) self.assertEqual(config.verbosity, VERB_STD) self.assertEqual(config.node_count, True) self.assertEqual(config.fanout, 64) self.assertEqual(config.connect_timeout, 10) self.assertEqual(config.command_timeout, 0) self.assertEqual(config.ssh_user, None) self.assertEqual(config.ssh_path, None) self.assertEqual(config.ssh_options, None) f.close()
def testClushConfigDefault(self): """test CLI.Config.ClushConfig (default)""" f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write(""" [Main] fanout: 42 connect_timeout: 14 command_timeout: 0 history_size: 100 color: auto verbosity: 1 #ssh_user: root #ssh_path: /usr/bin/ssh #ssh_options: -oStrictHostKeyChecking=no """) f.flush() parser = OptionParser("dummy") parser.install_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options, filename=f.name) self.assert_(config != None) display = Display(options, config) self.assert_(display != None) display.vprint(VERB_STD, "test") display.vprint(VERB_DEBUG, "shouldn't see this") self.assertEqual(config.color, WHENCOLOR_CHOICES[2]) self.assertEqual(config.verbosity, VERB_STD) self.assertEqual(config.node_count, True) self.assertEqual(config.fanout, 42) self.assertEqual(config.connect_timeout, 14) self.assertEqual(config.command_timeout, 0) self.assertEqual(config.ssh_user, None) self.assertEqual(config.ssh_path, None) self.assertEqual(config.ssh_options, None) f.close()
def main(): """clush script entry point""" sys.excepthook = clush_excepthook # # Argument management # usage = "%prog [options] command" parser = OptionParser(usage) parser.add_option("--nostdin", action="store_true", dest="nostdin", help="don't watch for possible input from stdin") parser.install_config_options('clush.conf(5)') parser.install_nodes_options() parser.install_display_options(verbose_options=True) parser.install_filecopy_options() parser.install_connector_options() (options, args) = parser.parse_args() # # Load config file and apply overrides # config = ClushConfig(options) # Should we use ANSI colors for nodes? if config.color == "auto": color = sys.stdout.isatty() and (options.gatherall or \ sys.stderr.isatty()) else: color = config.color == "always" try: # Create and configure display object. display = Display(options, config, color) except ValueError as exc: parser.error("option mismatch (%s)" % exc) if options.groupsource: # Be sure -a/g -s source work as espected. std_group_resolver().default_source_name = options.groupsource # Compute the nodeset and warn for possible use of shell pathname # expansion (#225) wnodelist = [] xnodelist = [] if options.nodes: wnodelist = [NodeSet(nodes) for nodes in options.nodes] if options.exclude: xnodelist = [NodeSet(nodes) for nodes in options.exclude] for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)): for nodes in nodelist: if len(nodes) == 1 and exists(str(nodes)): display.vprint_err( VERB_STD, "Warning: using '-%s %s' and " "local path '%s' exists, was it expanded " "by the shell?" % (opt, nodes, nodes)) # --hostfile support (#235) for opt_hostfile in options.hostfile: try: fnodeset = NodeSet() hostfile = open(opt_hostfile) for line in hostfile.read().splitlines(): fnodeset.updaten(nodes for nodes in line.split()) hostfile.close() display.vprint_err( VERB_DEBUG, "Using nodeset %s from hostfile %s" % (fnodeset, opt_hostfile)) wnodelist.append(fnodeset) except IOError as exc: # re-raise as OSError to be properly handled errno, strerror = exc.args raise OSError(errno, strerror, exc.filename) # Instantiate target nodeset from command line and hostfile nodeset_base = NodeSet.fromlist(wnodelist) # Instantiate filter nodeset (command line only) nodeset_exclude = NodeSet.fromlist(xnodelist) # Specified engine prevails over default engine DEFAULTS.engine = options.engine # Do we have nodes group? task = task_self() task.set_info("debug", config.verbosity >= VERB_DEBUG) if config.verbosity == VERB_DEBUG: std_group_resolver().set_verbosity(1) if options.nodes_all: all_nodeset = NodeSet.fromall() display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \ all_nodeset) nodeset_base.add(all_nodeset) if options.group: grp_nodeset = NodeSet.fromlist(options.group, resolver=RESOLVER_NOGROUP) for grp in grp_nodeset: addingrp = NodeSet("@" + grp) display.vprint(VERB_DEBUG, \ "Adding nodes from option -g %s: %s" % (grp, addingrp)) nodeset_base.update(addingrp) if options.exgroup: grp_nodeset = NodeSet.fromlist(options.exgroup, resolver=RESOLVER_NOGROUP) for grp in grp_nodeset: removingrp = NodeSet("@" + grp) display.vprint(VERB_DEBUG, \ "Excluding nodes from option -X %s: %s" % (grp, removingrp)) nodeset_exclude.update(removingrp) # Do we have an exclude list? (-x ...) nodeset_base.difference_update(nodeset_exclude) if len(nodeset_base) < 1: parser.error('No node to run on.') if options.pick and options.pick < len(nodeset_base): # convert to string for sample as nsiter() is slower for big # nodesets; and we assume options.pick will remain small-ish keep = random.sample(nodeset_base, options.pick) nodeset_base.intersection_update(','.join(keep)) if config.verbosity >= VERB_VERB: msg = "Picked random nodes: %s" % nodeset_base print Display.COLOR_RESULT_FMT % msg # Set open files limit. set_fdlimit(config.fd_max, display) # # Task management # # check for clush interactive mode interactive = not len(args) and \ not (options.copy or options.rcopy) # check for foreground ttys presence (input) stdin_isafgtty = sys.stdin.isatty() and \ os.tcgetpgrp(sys.stdin.fileno()) == os.getpgrp() # check for special condition (empty command and stdin not a tty) if interactive and not stdin_isafgtty: # looks like interactive but stdin is not a tty: # switch to non-interactive + disable ssh pseudo-tty interactive = False # SSH: disable pseudo-tty allocation (-T) ssh_options = config.ssh_options or '' ssh_options += ' -T' config._set_main("ssh_options", ssh_options) if options.nostdin and interactive: parser.error("illegal option `--nostdin' in that case") # Force user_interaction if Clush._f_user_interaction for test purposes user_interaction = hasattr(sys.modules[__name__], '_f_user_interaction') if not options.nostdin: # Try user interaction: check for foreground ttys presence (ouput) stdout_isafgtty = sys.stdout.isatty() and \ os.tcgetpgrp(sys.stdout.fileno()) == os.getpgrp() user_interaction |= stdin_isafgtty and stdout_isafgtty display.vprint(VERB_DEBUG, "User interaction: %s" % user_interaction) if user_interaction: # Standard input is a terminal and we want to perform some user # interactions in the main thread (using blocking calls), so # we run cluster commands in a new ClusterShell Task (a new # thread is created). task = Task() # else: perform everything in the main thread # Handle special signal only when user_interaction is set task.set_default("USER_handle_SIGUSR1", user_interaction) task.excepthook = sys.excepthook task.set_default("USER_stdin_worker", not (sys.stdin.isatty() or \ options.nostdin or \ user_interaction)) display.vprint(VERB_DEBUG, "Create STDIN worker: %s" % \ task.default("USER_stdin_worker")) if config.verbosity >= VERB_DEBUG: task.set_info("debug", True) logging.basicConfig(level=logging.DEBUG) logging.debug("clush: STARTING DEBUG") else: logging.basicConfig(level=logging.CRITICAL) task.set_info("fanout", config.fanout) if options.worker: try: if options.remote == 'no': task.set_default('local_worker', _load_workerclass(options.worker)) else: task.set_default('distant_worker', _load_workerclass(options.worker)) except (ImportError, AttributeError): msg = "ERROR: Could not load worker '%s'" % options.worker display.vprint_err(VERB_QUIET, msg) clush_exit(1, task) if options.topofile or task._default_tree_is_enabled(): if options.topofile: task.load_topology(options.topofile) if config.verbosity >= VERB_VERB: roots = len(task.topology.root.nodeset) gws = task.topology.inner_node_count() - roots msg = "enabling tree topology (%d gateways)" % gws print >> sys.stderr, "clush: %s" % msg if options.grooming_delay: if config.verbosity >= VERB_VERB: msg = Display.COLOR_RESULT_FMT % ("Grooming delay: %f" % options.grooming_delay) print >> sys.stderr, msg task.set_info("grooming_delay", options.grooming_delay) elif options.rcopy: # By default, --rcopy should inhibit grooming task.set_info("grooming_delay", 0) if config.ssh_user: task.set_info("ssh_user", config.ssh_user) if config.ssh_path: task.set_info("ssh_path", config.ssh_path) if config.ssh_options: task.set_info("ssh_options", config.ssh_options) if config.scp_path: task.set_info("scp_path", config.scp_path) if config.scp_options: task.set_info("scp_options", config.scp_options) if config.rsh_path: task.set_info("rsh_path", config.rsh_path) if config.rcp_path: task.set_info("rcp_path", config.rcp_path) if config.rsh_options: task.set_info("rsh_options", config.rsh_options) # Set detailed timeout values task.set_info("connect_timeout", config.connect_timeout) task.set_info("command_timeout", config.command_timeout) # Enable stdout/stderr separation task.set_default("stderr", not options.gatherall) # Disable MsgTree buffering if not gathering outputs task.set_default("stdout_msgtree", display.gather or display.line_mode) # Always disable stderr MsgTree buffering task.set_default("stderr_msgtree", False) # Set timeout at worker level when command_timeout is defined. if config.command_timeout > 0: timeout = config.command_timeout else: timeout = -1 # Configure task custom status task.set_default("USER_interactive", interactive) task.set_default("USER_running", False) if (options.copy or options.rcopy) and not args: parser.error("--[r]copy option requires at least one argument") if options.copy: if not options.dest_path: # append '/' to clearly indicate a directory for tree mode options.dest_path = join(dirname(abspath(args[0])), '') op = "copy sources=%s dest=%s" % (args, options.dest_path) elif options.rcopy: if not options.dest_path: options.dest_path = dirname(abspath(args[0])) op = "rcopy sources=%s dest=%s" % (args, options.dest_path) else: op = "command=\"%s\"" % ' '.join(args) # print debug values (fanout value is get from the config object # and not task itself as set_info() is an asynchronous call. display.vprint(VERB_DEBUG, "clush: nodeset=%s fanout=%d [timeout " \ "conn=%.1f cmd=%.1f] %s" % (nodeset_base, config.fanout, config.connect_timeout, config.command_timeout, op)) if not task.default("USER_interactive"): if display.verbosity >= VERB_DEBUG and task.topology: print Display.COLOR_RESULT_FMT % '-' * 15 print Display.COLOR_RESULT_FMT % task.topology, print Display.COLOR_RESULT_FMT % '-' * 15 if options.copy: run_copy(task, args, options.dest_path, nodeset_base, timeout, options.preserve_flag, display) elif options.rcopy: run_rcopy(task, args, options.dest_path, nodeset_base, timeout, options.preserve_flag, display) else: run_command(task, ' '.join(args), nodeset_base, timeout, display, options.remote != 'no') if user_interaction: ttyloop(task, nodeset_base, timeout, display, options.remote != 'no') elif task.default("USER_interactive"): display.vprint_err(VERB_QUIET, \ "ERROR: interactive mode requires a tty") clush_exit(1, task) rc = 0 if options.maxrc: # Instead of clush return code, return commands retcode rc = task.max_retcode() if task.num_timeout() > 0: rc = 255 clush_exit(rc, task)