def testClushConfigSetRlimitValueError(self):
"""test CLI.Config.ClushConfig (setrlimit ValueError)"""
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
# Use wrong fd_max value to generate ValueError
fd_max: -1
verbosity: 1""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
f.close()
display = Display(options, config)
class TestException(Exception): pass
def mock_vprint_err(level, message):
if message.startswith('Warning: Failed to set max open files'):
raise TestException()
display.vprint_err = mock_vprint_err
self.assertRaises(TestException, set_fdlimit, config.fd_max, display)
soft2, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEqual(soft, soft2)
def testClushConfigDefaultWithOptions(self):
"""test CLI.Config.ClushConfig (default with options)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
verbosity: 1""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args(["-f", "36", "-u", "3", "-t", "7",
"--user", "foobar", "--color",
"always", "-d", "-v", "-q", "-o",
"-oSomething"])
config = ClushConfig(options, filename=f.name)
display = Display(options, config)
display.vprint(VERB_STD, "test")
display.vprint(VERB_DEBUG, "test")
self.assertEqual(config.color, WHENCOLOR_CHOICES[1])
self.assertEqual(config.verbosity, VERB_DEBUG) # takes biggest
self.assertEqual(config.fanout, 36)
self.assertEqual(config.connect_timeout, 7)
self.assertEqual(config.command_timeout, 3)
self.assertEqual(config.ssh_user, "foobar")
self.assertEqual(config.ssh_path, None)
self.assertEqual(config.ssh_options, "-oSomething")
f.close()
def testClushConfigSetRlimit(self):
"""test CLI.Config.ClushConfig (setrlimit)"""
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
hard2 = min(32768, hard)
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
fd_max: %d
verbosity: 1
""" % hard2).encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
display = Display(options, config)
# force a lower soft limit
resource.setrlimit(resource.RLIMIT_NOFILE, (hard2/2, hard))
# max_fdlimit should increase soft limit again
set_fdlimit(config.fd_max, display)
# verify
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEqual(soft, hard2)
f.close()
def testClushConfigDefault(self):
"""test CLI.Config.ClushConfig (default)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
verbosity: 1
#ssh_user: root
#ssh_path: /usr/bin/ssh
#ssh_options: -oStrictHostKeyChecking=no""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
display = Display(options, config)
display.vprint(VERB_STD, "test")
display.vprint(VERB_DEBUG, "shouldn't see this")
self.assertEqual(config.color, WHENCOLOR_CHOICES[2])
self.assertEqual(config.verbosity, VERB_STD)
self.assertEqual(config.node_count, True)
self.assertEqual(config.fanout, 42)
self.assertEqual(config.connect_timeout, 14)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, None)
self.assertEqual(config.ssh_path, None)
self.assertEqual(config.ssh_options, None)
f.close()
def testClushConfigFull(self):
"""test CLI.Config.ClushConfig (full)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
node_count: yes
verbosity: 1
ssh_user: root
ssh_path: /usr/bin/ssh
ssh_options: -oStrictHostKeyChecking=no
""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
self.assertEqual(config.color, WHENCOLOR_CHOICES[2])
self.assertEqual(config.verbosity, VERB_STD)
self.assertEqual(config.node_count, True)
self.assertEqual(config.fanout, 42)
self.assertEqual(config.connect_timeout, 14)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, "root")
self.assertEqual(config.ssh_path, "/usr/bin/ssh")
self.assertEqual(config.ssh_options, "-oStrictHostKeyChecking=no")
f.close()
def testClushConfigError(self):
"""test CLI.Config.ClushConfig (error)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(
dedent("""
[Main]
fanout: 3.2
connect_timeout: foo
command_timeout: bar
history_size: 100
color: maybe
node_count: 3
verbosity: bar
ssh_user: root
ssh_path: /usr/bin/ssh
ssh_options: -oStrictHostKeyChecking=no
""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
try:
c = config.color
self.fail("Exception ClushConfigError not raised (color)")
except ClushConfigError:
pass
self.assertEqual(config.verbosity, 0) # probably for compatibility
try:
f = config.fanout
self.fail("Exception ClushConfigError not raised (fanout)")
except ClushConfigError:
pass
try:
f = config.node_count
self.fail("Exception ClushConfigError not raised (node_count)")
except ClushConfigError:
pass
try:
f = config.fanout
except ClushConfigError as e:
self.assertEqual(str(e)[0:20], "(Config Main.fanout)")
try:
t = config.connect_timeout
self.fail(
"Exception ClushConfigError not raised (connect_timeout)")
except ClushConfigError:
pass
try:
m = config.command_timeout
self.fail(
"Exception ClushConfigError not raised (command_timeout)")
except ClushConfigError:
pass
f.close()
def testClushConfigDefaultWithOptions(self):
"""test CLI.Config.ClushConfig (default with options)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(
dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
verbosity: 1""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([
"-f", "36", "-u", "3", "-t", "7", "--user", "foobar", "--color",
"always", "-d", "-v", "-q", "-o", "-oSomething"
])
config = ClushConfig(options, filename=f.name)
display = Display(options, config)
display.vprint(VERB_STD, "test")
display.vprint(VERB_DEBUG, "test")
self.assertEqual(config.color, THREE_CHOICES[2])
self.assertEqual(config.verbosity, VERB_DEBUG) # takes biggest
self.assertEqual(config.fanout, 36)
self.assertEqual(config.connect_timeout, 7)
self.assertEqual(config.command_timeout, 3)
self.assertEqual(config.ssh_user, "foobar")
self.assertEqual(config.ssh_path, None)
self.assertEqual(config.ssh_options, "-oSomething")
f.close()
def testClushConfigSetRlimit(self):
"""test CLI.Config.ClushConfig (setrlimit)"""
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
hard2 = min(32768, hard)
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(
dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
fd_max: %d
verbosity: 1
""" % hard2).encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
display = Display(options, config)
# force a lower soft limit
resource.setrlimit(resource.RLIMIT_NOFILE, (hard2 / 2, hard))
# max_fdlimit should increase soft limit again
set_fdlimit(config.fd_max, display)
# verify
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEqual(soft, hard2)
f.close()
def testClushConfigUserOverride(self):
"""test CLI.Config.ClushConfig (XDG_CONFIG_HOME user config)"""
# XXX Test should be improved when CLUSTERSHELL_CONFIG is available
# Improvement: override CLUSTERSHELL_CONFIG and set a sys clush config
# then verify that user config overrides CLUSTERSHELL_CONFIG as
# expected...
# For now, it has been tested manually. This test only really only
# ensures that user config is taken into account.
xdg_config_home_save = os.environ.get('XDG_CONFIG_HOME')
# Create fake XDG_CONFIG_HOME
dname = make_temp_dir()
try:
os.environ['XDG_CONFIG_HOME'] = dname
# create $XDG_CONFIG_HOME/clustershell/clush.conf
usercfgdir = os.path.join(dname, 'clustershell')
os.mkdir(usercfgdir)
cfgfile = open(os.path.join(usercfgdir, 'clush.conf'), 'w')
cfgfile.write(
dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: never
verbosity: 2
ssh_user: trump
ssh_path: ~/bin/ssh
ssh_options: -oSomeDummyUserOption=yes
"""))
cfgfile.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options) # filename=None to use defaults!
self.assertEqual(config.color, THREE_CHOICES[1])
self.assertEqual(config.verbosity, VERB_VERB) # takes biggest
self.assertEqual(config.fanout, 42)
self.assertEqual(config.connect_timeout, 14)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, 'trump')
self.assertEqual(config.ssh_path, '~/bin/ssh')
self.assertEqual(config.ssh_options, '-oSomeDummyUserOption=yes')
cfgfile.close()
finally:
if xdg_config_home_save:
os.environ['XDG_CONFIG_HOME'] = xdg_config_home_save
else:
del os.environ['XDG_CONFIG_HOME']
shutil.rmtree(dname, ignore_errors=True)
def testClushConfigError(self):
"""test CLI.Config.ClushConfig (error)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(dedent("""
[Main]
fanout: 3.2
connect_timeout: foo
command_timeout: bar
history_size: 100
color: maybe
node_count: 3
verbosity: bar
ssh_user: root
ssh_path: /usr/bin/ssh
ssh_options: -oStrictHostKeyChecking=no
""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
try:
c = config.color
self.fail("Exception ClushConfigError not raised (color)")
except ClushConfigError:
pass
self.assertEqual(config.verbosity, 0) # probably for compatibility
try:
f = config.fanout
self.fail("Exception ClushConfigError not raised (fanout)")
except ClushConfigError:
pass
try:
f = config.node_count
self.fail("Exception ClushConfigError not raised (node_count)")
except ClushConfigError:
pass
try:
f = config.fanout
except ClushConfigError as e:
self.assertEqual(str(e)[0:20], "(Config Main.fanout)")
try:
t = config.connect_timeout
self.fail("Exception ClushConfigError not raised (connect_timeout)")
except ClushConfigError:
pass
try:
m = config.command_timeout
self.fail("Exception ClushConfigError not raised (command_timeout)")
except ClushConfigError:
pass
f.close()
def testClushConfigUserOverride(self):
"""test CLI.Config.ClushConfig (XDG_CONFIG_HOME user config)"""
# XXX Test should be improved when CLUSTERSHELL_CONFIG is available
# Improvement: override CLUSTERSHELL_CONFIG and set a sys clush config
# then verify that user config overrides CLUSTERSHELL_CONFIG as
# expected...
# For now, it has been tested manually. This test only really only
# ensures that user config is taken into account.
xdg_config_home_save = os.environ.get('XDG_CONFIG_HOME')
# Create fake XDG_CONFIG_HOME
dname = make_temp_dir()
try:
os.environ['XDG_CONFIG_HOME'] = dname
# create $XDG_CONFIG_HOME/clustershell/clush.conf
usercfgdir = os.path.join(dname, 'clustershell')
os.mkdir(usercfgdir)
cfgfile = open(os.path.join(usercfgdir, 'clush.conf'), 'w')
cfgfile.write(dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: never
verbosity: 2
ssh_user: trump
ssh_path: ~/bin/ssh
ssh_options: -oSomeDummyUserOption=yes
"""))
cfgfile.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options) # filename=None to use defaults!
self.assertEqual(config.color, WHENCOLOR_CHOICES[0])
self.assertEqual(config.verbosity, VERB_VERB) # takes biggest
self.assertEqual(config.fanout, 42)
self.assertEqual(config.connect_timeout, 14)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, 'trump')
self.assertEqual(config.ssh_path, '~/bin/ssh')
self.assertEqual(config.ssh_options, '-oSomeDummyUserOption=yes')
cfgfile.close()
finally:
if xdg_config_home_save:
os.environ['XDG_CONFIG_HOME'] = xdg_config_home_save
else:
del os.environ['XDG_CONFIG_HOME']
shutil.rmtree(dname, ignore_errors=True)
def testClushConfigWithInstalledConfig(self):
"""test CLI.Config.ClushConfig (installed config required)"""
# This test needs installed configuration files (needed for
# maximum coverage).
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options)
def testClushConfigWithInstalledConfig(self):
"""test CLI.Config.ClushConfig (installed config required)"""
# This test needs installed configuration files (needed for
# maximum coverage).
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options)
def testClushConfigAlmostEmpty(self):
"""test CLI.Config.ClushConfig (almost empty)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write("[Main]\n".encode())
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
self.assertEqual(config.color, THREE_CHOICES[0])
self.assertEqual(config.verbosity, VERB_STD)
self.assertEqual(config.node_count, True)
self.assertEqual(config.fanout, 64)
self.assertEqual(config.connect_timeout, 10)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, None)
self.assertEqual(config.ssh_path, None)
self.assertEqual(config.ssh_options, None)
f.close()
def testClushConfigAlmostEmpty(self):
"""test CLI.Config.ClushConfig (almost empty)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write("[Main]\n".encode())
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
self.assertEqual(config.color, WHENCOLOR_CHOICES[-1])
self.assertEqual(config.verbosity, VERB_STD)
self.assertEqual(config.node_count, True)
self.assertEqual(config.fanout, 64)
self.assertEqual(config.connect_timeout, 10)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, None)
self.assertEqual(config.ssh_path, None)
self.assertEqual(config.ssh_options, None)
f.close()
def testClushConfigSetRlimitValueError(self):
"""test CLI.Config.ClushConfig (setrlimit ValueError)"""
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(
dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
# Use wrong fd_max value to generate ValueError
fd_max: -1
verbosity: 1""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
f.close()
display = Display(options, config)
class TestException(Exception):
pass
def mock_vprint_err(level, message):
if message.startswith('Warning: Failed to set max open files'):
raise TestException()
display.vprint_err = mock_vprint_err
self.assertRaises(TestException, set_fdlimit, config.fd_max, display)
soft2, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEqual(soft, soft2)
def testClushConfigFull(self):
"""test CLI.Config.ClushConfig (full)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(
dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
node_count: yes
verbosity: 1
ssh_user: root
ssh_path: /usr/bin/ssh
ssh_options: -oStrictHostKeyChecking=no
""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
self.assertEqual(config.color, THREE_CHOICES[-1])
self.assertEqual(config.verbosity, VERB_STD)
self.assertEqual(config.node_count, True)
self.assertEqual(config.fanout, 42)
self.assertEqual(config.connect_timeout, 14)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, "root")
self.assertEqual(config.ssh_path, "/usr/bin/ssh")
self.assertEqual(config.ssh_options, "-oStrictHostKeyChecking=no")
f.close()
def testClushConfigDefault(self):
"""test CLI.Config.ClushConfig (default)"""
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(
dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
verbosity: 1
#ssh_user: root
#ssh_path: /usr/bin/ssh
#ssh_options: -oStrictHostKeyChecking=no""").encode())
f.flush()
parser = OptionParser("dummy")
parser.install_clush_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
display = Display(options, config)
display.vprint(VERB_STD, "test")
display.vprint(VERB_DEBUG, "shouldn't see this")
self.assertEqual(config.color, THREE_CHOICES[-1])
self.assertEqual(config.verbosity, VERB_STD)
self.assertEqual(config.node_count, True)
self.assertEqual(config.fanout, 42)
self.assertEqual(config.connect_timeout, 14)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, None)
self.assertEqual(config.ssh_path, None)
self.assertEqual(config.ssh_options, None)
f.close()
def main():
"""clush script entry point"""
sys.excepthook = clush_excepthook
#
# Argument management
#
usage = "%prog [options] command"
parser = OptionParser(usage)
parser.add_option("-n",
"--nostdin",
action="store_true",
dest="nostdin",
help="don't watch for possible input from stdin")
parser.install_groupsconf_option()
parser.install_clush_config_options()
parser.install_nodes_options()
parser.install_display_options(verbose_options=True)
parser.install_filecopy_options()
parser.install_connector_options()
(options, args) = parser.parse_args()
set_std_group_resolver_config(options.groupsconf)
#
# Load config file and apply overrides
#
config = ClushConfig(options, options.conf)
# Initialize logging
if config.verbosity >= VERB_DEBUG:
logging.basicConfig(level=logging.DEBUG)
logging.debug("clush: STARTING DEBUG")
else:
logging.basicConfig(level=logging.CRITICAL)
# Should we use ANSI colors for nodes?
if config.color == "auto":
color = sys.stdout.isatty() and (options.gatherall or \
sys.stderr.isatty())
else:
color = config.color == "always"
try:
# Create and configure display object.
display = Display(options, config, color)
except ValueError as exc:
parser.error("option mismatch (%s)" % exc)
if options.groupsource:
# Be sure -a/g -s source work as espected.
std_group_resolver().default_source_name = options.groupsource
# Compute the nodeset and warn for possible use of shell pathname
# expansion (#225)
wnodelist = []
xnodelist = []
if options.nodes:
wnodelist = [NodeSet(nodes) for nodes in options.nodes]
if options.exclude:
xnodelist = [NodeSet(nodes) for nodes in options.exclude]
for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)):
for nodes in nodelist:
if len(nodes) == 1 and exists(str(nodes)):
display.vprint_err(
VERB_STD, "Warning: using '-%s %s' and "
"local path '%s' exists, was it expanded "
"by the shell?" % (opt, nodes, nodes))
# --hostfile support (#235)
for opt_hostfile in options.hostfile:
try:
fnodeset = NodeSet()
with open(opt_hostfile) as hostfile:
for line in hostfile.read().splitlines():
fnodeset.updaten(nodes for nodes in line.split())
display.vprint_err(
VERB_DEBUG,
"Using nodeset %s from hostfile %s" % (fnodeset, opt_hostfile))
wnodelist.append(fnodeset)
except IOError as exc:
# re-raise as OSError to be properly handled
errno, strerror = exc.args
raise OSError(errno, strerror, exc.filename)
# Instantiate target nodeset from command line and hostfile
nodeset_base = NodeSet.fromlist(wnodelist)
# Instantiate filter nodeset (command line only)
nodeset_exclude = NodeSet.fromlist(xnodelist)
# Specified engine prevails over default engine
DEFAULTS.engine = options.engine
# Do we have nodes group?
task = task_self()
task.set_info("debug", config.verbosity >= VERB_DEBUG)
if config.verbosity == VERB_DEBUG:
std_group_resolver().set_verbosity(1)
if options.nodes_all:
all_nodeset = NodeSet.fromall()
display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \
all_nodeset)
nodeset_base.add(all_nodeset)
if options.group:
grp_nodeset = NodeSet.fromlist(options.group,
resolver=RESOLVER_NOGROUP)
for grp in grp_nodeset:
addingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Adding nodes from option -g %s: %s" % (grp, addingrp))
nodeset_base.update(addingrp)
if options.exgroup:
grp_nodeset = NodeSet.fromlist(options.exgroup,
resolver=RESOLVER_NOGROUP)
for grp in grp_nodeset:
removingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Excluding nodes from option -X %s: %s" % (grp, removingrp))
nodeset_exclude.update(removingrp)
# Do we have an exclude list? (-x ...)
nodeset_base.difference_update(nodeset_exclude)
if len(nodeset_base) < 1:
parser.error('No node to run on.')
if options.pick and options.pick < len(nodeset_base):
# convert to string for sample as nsiter() is slower for big
# nodesets; and we assume options.pick will remain small-ish
keep = random.sample(list(nodeset_base), options.pick)
nodeset_base.intersection_update(','.join(keep))
if config.verbosity >= VERB_VERB:
msg = "Picked random nodes: %s" % nodeset_base
print(Display.COLOR_RESULT_FMT % msg)
# Set open files limit.
set_fdlimit(config.fd_max, display)
#
# Task management
#
# check for clush interactive mode
interactive = not len(args) and \
not (options.copy or options.rcopy)
# check for foreground ttys presence (input)
stdin_isafgtty = sys.stdin.isatty() and \
os.tcgetpgrp(sys.stdin.fileno()) == os.getpgrp()
# check for special condition (empty command and stdin not a tty)
if interactive and not stdin_isafgtty:
# looks like interactive but stdin is not a tty:
# switch to non-interactive + disable ssh pseudo-tty
interactive = False
# SSH: disable pseudo-tty allocation (-T)
ssh_options = config.ssh_options or ''
ssh_options += ' -T'
config._set_main("ssh_options", ssh_options)
if options.nostdin and interactive:
parser.error("illegal option `--nostdin' in that case")
# Force user_interaction if Clush._f_user_interaction for test purposes
user_interaction = hasattr(sys.modules[__name__], '_f_user_interaction')
if not options.nostdin:
# Try user interaction: check for foreground ttys presence (ouput)
stdout_isafgtty = sys.stdout.isatty() and \
os.tcgetpgrp(sys.stdout.fileno()) == os.getpgrp()
user_interaction |= stdin_isafgtty and stdout_isafgtty
display.vprint(VERB_DEBUG, "User interaction: %s" % user_interaction)
if user_interaction:
# Standard input is a terminal and we want to perform some user
# interactions in the main thread (using blocking calls), so
# we run cluster commands in a new ClusterShell Task (a new
# thread is created).
task = Task()
# else: perform everything in the main thread
# Handle special signal only when user_interaction is set
task.set_default("USER_handle_SIGUSR1", user_interaction)
task.excepthook = sys.excepthook
task.set_default("USER_stdin_worker", not (sys.stdin.isatty() or \
options.nostdin or \
user_interaction))
display.vprint(VERB_DEBUG, "Create STDIN worker: %s" % \
task.default("USER_stdin_worker"))
task.set_info("debug", config.verbosity >= VERB_DEBUG)
task.set_info("fanout", config.fanout)
if options.worker:
try:
if options.remote == 'no':
task.set_default('local_worker',
_load_workerclass(options.worker))
else:
task.set_default('distant_worker',
_load_workerclass(options.worker))
except (ImportError, AttributeError):
msg = "ERROR: Could not load worker '%s'" % options.worker
display.vprint_err(VERB_QUIET, msg)
clush_exit(1, task)
if options.topofile or task._default_tree_is_enabled():
if options.topofile:
task.load_topology(options.topofile)
if config.verbosity >= VERB_VERB:
roots = len(task.topology.root.nodeset)
gws = task.topology.inner_node_count() - roots
msg = "enabling tree topology (%d gateways)" % gws
print("clush: %s" % msg, file=sys.stderr)
if options.grooming_delay:
if config.verbosity >= VERB_VERB:
msg = Display.COLOR_RESULT_FMT % ("Grooming delay: %f" %
options.grooming_delay)
print(msg, file=sys.stderr)
task.set_info("grooming_delay", options.grooming_delay)
elif options.rcopy:
# By default, --rcopy should inhibit grooming
task.set_info("grooming_delay", 0)
if config.ssh_user:
task.set_info("ssh_user", config.ssh_user)
if config.ssh_path:
task.set_info("ssh_path", config.ssh_path)
if config.ssh_options:
task.set_info("ssh_options", config.ssh_options)
if config.scp_path:
task.set_info("scp_path", config.scp_path)
if config.scp_options:
task.set_info("scp_options", config.scp_options)
if config.rsh_path:
task.set_info("rsh_path", config.rsh_path)
if config.rcp_path:
task.set_info("rcp_path", config.rcp_path)
if config.rsh_options:
task.set_info("rsh_options", config.rsh_options)
# Set detailed timeout values
task.set_info("connect_timeout", config.connect_timeout)
task.set_info("command_timeout", config.command_timeout)
# Enable stdout/stderr separation
task.set_default("stderr", not options.gatherall)
# Prevent reading from stdin?
task.set_default("stdin", not options.nostdin)
# Disable MsgTree buffering if not gathering outputs
task.set_default("stdout_msgtree", display.gather or display.line_mode)
# Always disable stderr MsgTree buffering
task.set_default("stderr_msgtree", False)
# Set timeout at worker level when command_timeout is defined.
if config.command_timeout > 0:
timeout = config.command_timeout
else:
timeout = -1
# Configure task custom status
task.set_default("USER_interactive", interactive)
task.set_default("USER_running", False)
if (options.copy or options.rcopy) and not args:
parser.error("--[r]copy option requires at least one argument")
if options.copy:
if not options.dest_path:
# append '/' to clearly indicate a directory for tree mode
options.dest_path = join(dirname(abspath(args[0])), '')
op = "copy sources=%s dest=%s" % (args, options.dest_path)
elif options.rcopy:
if not options.dest_path:
options.dest_path = dirname(abspath(args[0]))
op = "rcopy sources=%s dest=%s" % (args, options.dest_path)
else:
op = "command=\"%s\"" % ' '.join(args)
# print debug values (fanout value is get from the config object
# and not task itself as set_info() is an asynchronous call.
display.vprint(VERB_DEBUG, "clush: nodeset=%s fanout=%d [timeout " \
"conn=%.1f cmd=%.1f] %s" % (nodeset_base, config.fanout,
config.connect_timeout,
config.command_timeout,
op))
if not task.default("USER_interactive"):
if display.verbosity >= VERB_DEBUG and task.topology:
print(Display.COLOR_RESULT_FMT % '-' * 15)
print(Display.COLOR_RESULT_FMT % task.topology, end='')
print(Display.COLOR_RESULT_FMT % '-' * 15)
if options.copy:
run_copy(task, args, options.dest_path, nodeset_base, timeout,
options.preserve_flag, display)
elif options.rcopy:
run_rcopy(task, args, options.dest_path, nodeset_base, timeout,
options.preserve_flag, display)
else:
run_command(task, ' '.join(args), nodeset_base, timeout, display,
options.remote != 'no')
if user_interaction:
ttyloop(task, nodeset_base, timeout, display, options.remote != 'no')
elif task.default("USER_interactive"):
display.vprint_err(VERB_QUIET, \
"ERROR: interactive mode requires a tty")
clush_exit(1, task)
rc = 0
if options.maxrc:
# Instead of clush return code, return commands retcode
rc = task.max_retcode()
if task.num_timeout() > 0:
rc = 255
clush_exit(rc, task)
def main():
"""clush script entry point"""
sys.excepthook = clush_excepthook
#
# Argument management
#
usage = "%prog [options] command"
parser = OptionParser(usage)
parser.add_option("-n", "--nostdin", action="store_true", dest="nostdin",
help="don't watch for possible input from stdin")
parser.install_groupsconf_option()
parser.install_clush_config_options()
parser.install_nodes_options()
parser.install_display_options(verbose_options=True)
parser.install_filecopy_options()
parser.install_connector_options()
(options, args) = parser.parse_args()
set_std_group_resolver_config(options.groupsconf)
#
# Load config file and apply overrides
#
config = ClushConfig(options, options.conf)
# Initialize logging
if config.verbosity >= VERB_DEBUG:
logging.basicConfig(level=logging.DEBUG)
logging.debug("clush: STARTING DEBUG")
else:
logging.basicConfig(level=logging.CRITICAL)
# Should we use ANSI colors for nodes?
if config.color == "auto":
color = sys.stdout.isatty() and (options.gatherall or \
sys.stderr.isatty())
else:
color = config.color == "always"
try:
# Create and configure display object.
display = Display(options, config, color)
except ValueError as exc:
parser.error("option mismatch (%s)" % exc)
if options.groupsource:
# Be sure -a/g -s source work as espected.
std_group_resolver().default_source_name = options.groupsource
# Compute the nodeset and warn for possible use of shell pathname
# expansion (#225)
wnodelist = []
xnodelist = []
if options.nodes:
wnodelist = [NodeSet(nodes) for nodes in options.nodes]
if options.exclude:
xnodelist = [NodeSet(nodes) for nodes in options.exclude]
for (opt, nodelist) in (('w', wnodelist), ('x', xnodelist)):
for nodes in nodelist:
if len(nodes) == 1 and exists(str(nodes)):
display.vprint_err(VERB_STD, "Warning: using '-%s %s' and "
"local path '%s' exists, was it expanded "
"by the shell?" % (opt, nodes, nodes))
# --hostfile support (#235)
for opt_hostfile in options.hostfile:
try:
fnodeset = NodeSet()
with open(opt_hostfile) as hostfile:
for line in hostfile.read().splitlines():
fnodeset.updaten(nodes for nodes in line.split())
display.vprint_err(VERB_DEBUG,
"Using nodeset %s from hostfile %s"
% (fnodeset, opt_hostfile))
wnodelist.append(fnodeset)
except IOError as exc:
# re-raise as OSError to be properly handled
errno, strerror = exc.args
raise OSError(errno, strerror, exc.filename)
# Instantiate target nodeset from command line and hostfile
nodeset_base = NodeSet.fromlist(wnodelist)
# Instantiate filter nodeset (command line only)
nodeset_exclude = NodeSet.fromlist(xnodelist)
# Specified engine prevails over default engine
DEFAULTS.engine = options.engine
# Do we have nodes group?
task = task_self()
task.set_info("debug", config.verbosity >= VERB_DEBUG)
if config.verbosity == VERB_DEBUG:
std_group_resolver().set_verbosity(1)
if options.nodes_all:
all_nodeset = NodeSet.fromall()
display.vprint(VERB_DEBUG, "Adding nodes from option -a: %s" % \
all_nodeset)
nodeset_base.add(all_nodeset)
if options.group:
grp_nodeset = NodeSet.fromlist(options.group,
resolver=RESOLVER_NOGROUP)
for grp in grp_nodeset:
addingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Adding nodes from option -g %s: %s" % (grp, addingrp))
nodeset_base.update(addingrp)
if options.exgroup:
grp_nodeset = NodeSet.fromlist(options.exgroup,
resolver=RESOLVER_NOGROUP)
for grp in grp_nodeset:
removingrp = NodeSet("@" + grp)
display.vprint(VERB_DEBUG, \
"Excluding nodes from option -X %s: %s" % (grp, removingrp))
nodeset_exclude.update(removingrp)
# Do we have an exclude list? (-x ...)
nodeset_base.difference_update(nodeset_exclude)
if len(nodeset_base) < 1:
parser.error('No node to run on.')
if options.pick and options.pick < len(nodeset_base):
# convert to string for sample as nsiter() is slower for big
# nodesets; and we assume options.pick will remain small-ish
keep = random.sample(list(nodeset_base), options.pick)
nodeset_base.intersection_update(','.join(keep))
if config.verbosity >= VERB_VERB:
msg = "Picked random nodes: %s" % nodeset_base
print(Display.COLOR_RESULT_FMT % msg)
# Set open files limit.
set_fdlimit(config.fd_max, display)
#
# Task management
#
# check for clush interactive mode
interactive = not len(args) and \
not (options.copy or options.rcopy)
# check for foreground ttys presence (input)
stdin_isafgtty = sys.stdin.isatty() and \
os.tcgetpgrp(sys.stdin.fileno()) == os.getpgrp()
# check for special condition (empty command and stdin not a tty)
if interactive and not stdin_isafgtty:
# looks like interactive but stdin is not a tty:
# switch to non-interactive + disable ssh pseudo-tty
interactive = False
# SSH: disable pseudo-tty allocation (-T)
ssh_options = config.ssh_options or ''
ssh_options += ' -T'
config._set_main("ssh_options", ssh_options)
if options.nostdin and interactive:
parser.error("illegal option `--nostdin' in that case")
# Force user_interaction if Clush._f_user_interaction for test purposes
user_interaction = hasattr(sys.modules[__name__], '_f_user_interaction')
if not options.nostdin:
# Try user interaction: check for foreground ttys presence (ouput)
stdout_isafgtty = sys.stdout.isatty() and \
os.tcgetpgrp(sys.stdout.fileno()) == os.getpgrp()
user_interaction |= stdin_isafgtty and stdout_isafgtty
display.vprint(VERB_DEBUG, "User interaction: %s" % user_interaction)
if user_interaction:
# Standard input is a terminal and we want to perform some user
# interactions in the main thread (using blocking calls), so
# we run cluster commands in a new ClusterShell Task (a new
# thread is created).
task = Task()
# else: perform everything in the main thread
# Handle special signal only when user_interaction is set
task.set_default("USER_handle_SIGUSR1", user_interaction)
task.excepthook = sys.excepthook
task.set_default("USER_stdin_worker", not (sys.stdin.isatty() or \
options.nostdin or \
user_interaction))
display.vprint(VERB_DEBUG, "Create STDIN worker: %s" % \
task.default("USER_stdin_worker"))
task.set_info("debug", config.verbosity >= VERB_DEBUG)
task.set_info("fanout", config.fanout)
if options.worker:
try:
if options.remote == 'no':
task.set_default('local_worker',
_load_workerclass(options.worker))
else:
task.set_default('distant_worker',
_load_workerclass(options.worker))
except (ImportError, AttributeError):
msg = "ERROR: Could not load worker '%s'" % options.worker
display.vprint_err(VERB_QUIET, msg)
clush_exit(1, task)
if options.topofile or task._default_tree_is_enabled():
if options.topofile:
task.load_topology(options.topofile)
if config.verbosity >= VERB_VERB:
roots = len(task.topology.root.nodeset)
gws = task.topology.inner_node_count() - roots
msg = "enabling tree topology (%d gateways)" % gws
print("clush: %s" % msg, file=sys.stderr)
if options.grooming_delay:
if config.verbosity >= VERB_VERB:
msg = Display.COLOR_RESULT_FMT % ("Grooming delay: %f" %
options.grooming_delay)
print(msg, file=sys.stderr)
task.set_info("grooming_delay", options.grooming_delay)
elif options.rcopy:
# By default, --rcopy should inhibit grooming
task.set_info("grooming_delay", 0)
if config.ssh_user:
task.set_info("ssh_user", config.ssh_user)
if config.ssh_path:
task.set_info("ssh_path", config.ssh_path)
if config.ssh_options:
task.set_info("ssh_options", config.ssh_options)
if config.scp_path:
task.set_info("scp_path", config.scp_path)
if config.scp_options:
task.set_info("scp_options", config.scp_options)
if config.rsh_path:
task.set_info("rsh_path", config.rsh_path)
if config.rcp_path:
task.set_info("rcp_path", config.rcp_path)
if config.rsh_options:
task.set_info("rsh_options", config.rsh_options)
# Set detailed timeout values
task.set_info("connect_timeout", config.connect_timeout)
task.set_info("command_timeout", config.command_timeout)
# Enable stdout/stderr separation
task.set_default("stderr", not options.gatherall)
# Prevent reading from stdin?
task.set_default("stdin", not options.nostdin)
# Disable MsgTree buffering if not gathering outputs
task.set_default("stdout_msgtree", display.gather or display.line_mode)
# Always disable stderr MsgTree buffering
task.set_default("stderr_msgtree", False)
# Set timeout at worker level when command_timeout is defined.
if config.command_timeout > 0:
timeout = config.command_timeout
else:
timeout = -1
# Configure task custom status
task.set_default("USER_interactive", interactive)
task.set_default("USER_running", False)
if (options.copy or options.rcopy) and not args:
parser.error("--[r]copy option requires at least one argument")
if options.copy:
if not options.dest_path:
# append '/' to clearly indicate a directory for tree mode
options.dest_path = join(dirname(abspath(args[0])), '')
op = "copy sources=%s dest=%s" % (args, options.dest_path)
elif options.rcopy:
if not options.dest_path:
options.dest_path = dirname(abspath(args[0]))
op = "rcopy sources=%s dest=%s" % (args, options.dest_path)
else:
op = "command=\"%s\"" % ' '.join(args)
# print debug values (fanout value is get from the config object
# and not task itself as set_info() is an asynchronous call.
display.vprint(VERB_DEBUG, "clush: nodeset=%s fanout=%d [timeout " \
"conn=%.1f cmd=%.1f] %s" % (nodeset_base, config.fanout,
config.connect_timeout,
config.command_timeout,
op))
if not task.default("USER_interactive"):
if display.verbosity >= VERB_DEBUG and task.topology:
print(Display.COLOR_RESULT_FMT % '-' * 15)
print(Display.COLOR_RESULT_FMT % task.topology, end='')
print(Display.COLOR_RESULT_FMT % '-' * 15)
if options.copy:
run_copy(task, args, options.dest_path, nodeset_base, timeout,
options.preserve_flag, display)
elif options.rcopy:
run_rcopy(task, args, options.dest_path, nodeset_base, timeout,
options.preserve_flag, display)
else:
run_command(task, ' '.join(args), nodeset_base, timeout, display,
options.remote != 'no')
if user_interaction:
ttyloop(task, nodeset_base, timeout, display, options.remote != 'no')
elif task.default("USER_interactive"):
display.vprint_err(VERB_QUIET, \
"ERROR: interactive mode requires a tty")
clush_exit(1, task)
rc = 0
if options.maxrc:
# Instead of clush return code, return commands retcode
rc = task.max_retcode()
if task.num_timeout() > 0:
rc = 255
clush_exit(rc, task)
