def test_option_universe(self):
"""Test --universe command line option"""
# test with Intel MPI versions where mpd & hydra as the default process managers
for impi_ver in ['4.0.3', '4.2', '5.1.2']:
mpirun = os.path.join(self.tmpdir, 'mpirun')
if os.path.exists(mpirun):
os.remove(mpirun)
install_fake_mpirun('mpirun', self.tmpdir, 'impi', impi_ver)
self.change_env(5)
ec, out = run([sys.executable, self.mympiscript, '--universe', '4', 'hostname'])
self.assertEqual(ec, 0)
regex = re.compile('-np 4')
self.assertTrue(regex.search(out))
self.change_env(1)
# intel mpi without hydra
os.environ['I_MPI_PROCESS_MANAGER'] = 'mpd'
ec, out = run([sys.executable, self.mympiscript, '--universe', '1', 'hostname'])
self.assertEqual(ec, 0)
np_regex = re.compile('-np 1')
ncpus_regex = re.compile('--ncpus=1')
self.assertTrue(np_regex.search(out))
self.assertTrue(ncpus_regex.search(out))
del os.environ['I_MPI_PROCESS_MANAGER']
def test_launcher_opt_impi_hydra(self):
"""Test --launcher command line option with impi >= 5.0.3 (supports pbsdsh)"""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.0.3')
# default behavior
ec, out = run([sys.executable, self.mympiscript, 'hostname'])
regex = r'-bootstrap pbsdsh'
self.assertTrue(re.search(regex, out), "-bootstrap option is pbsdsh (default for impi/5.1): " + out)
# if --launcher ssh is used, behaviour depends on whether or not pbsdsh are available
ec, out = run([sys.executable, self.mympiscript, '--launcher', 'ssh', 'hostname'])
regexes = [
(r'-bootstrap ssh', "bootstrap option is ssh (possibly with -bootstrap-exec option)"),
]
pbsdsh = which('pbsdsh')
if pbsdsh:
# '-bootstrap-exec pbsssh' is used only when pbsdsh is available
regexes.append((r'-bootstrap-exec pbsssh', "pbsssh is used when launcher is ssh and pbsdsh is there"))
else:
# withtout pbsdsh, no -bootstrap-exec
fail_msg = "No -bootstrap-exec option when launcher is ssh and no pbsdsh: %s" % out
self.assertFalse(re.search(r'-bootstrap-exec', out), fail_msg)
for regex in regexes:
self.assertTrue(re.search(regex[0], out), regex[1] + ": " + out)
# unknown launcher being specified only results in a warning (we allow specifying launchers that are not listed)
ec, out = run([sys.executable, self.mympiscript, '--launcher', 'doesnotexist', 'hostname'])
regex = r'WARNING .* Specified launcher doesnotexist does not exist'
self.assertTrue(re.search(regex, out), "mympirun should warn for non-existing launcher")
ec, out = run([sys.executable, self.mympiscript, '--sched', 'local', 'hostname'])
self.assertFalse("-bootstrap" in out, "using local scheduler, no bootstrap launcher should be specified: " + out)
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertEqual(ec, 127)
self.assertTrue('test/sandbox/testpkg/*: No such file or directory' in output)
ec, output = run_simple(['ls','test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def test_convert_to_unix_timestamp(self):
ec, out = run(['date', '+%s'])
nowts = convert_to_unix_timestamp()
self.assertTrue(abs(nowts - int(out.strip())) < 2)
date = 0
self.assertEqual(convert_to_unix_timestamp("19700101"), date)
self.assertEqual(convert_to_unix_timestamp("0000000000"), date)
self.assertEqual(convert_to_unix_timestamp(0), date)
self.assertEqual(convert_to_unix_timestamp("197001010000"), date)
self.assertEqual(convert_to_unix_timestamp("19700101000000Z"), date)
self.assertEqual(convert_to_unix_timestamp(datetime(1970, 1, 1)), date)
date = 1521936000
self.assertEqual(convert_to_unix_timestamp("20180325"), date)
date = 1521939900
self.assertEqual(convert_to_unix_timestamp("1521939900"), date)
self.assertEqual(convert_to_unix_timestamp(1521939900), date)
self.assertEqual(convert_to_unix_timestamp("201803250105"), date)
self.assertEqual(convert_to_unix_timestamp("201803250105"), date)
self.assertEqual(convert_to_unix_timestamp("20180325010500Z"), date)
self.assertEqual(convert_to_unix_timestamp(datetime(2018, 3, 25, 1, 5)), date)
date = 1509239100
self.assertEqual(convert_to_unix_timestamp("1509239100"), date)
self.assertEqual(convert_to_unix_timestamp(1509239100), date)
self.assertEqual(convert_to_unix_timestamp("201710290105"), date)
self.assertEqual(convert_to_unix_timestamp("20171029010500Z"), date)
self.assertEqual(convert_to_unix_timestamp(date), date)
def test_launcher_opt_ompi(self):
"""Test ompi v 2.0 bug (mympirun should produce error and stop)"""
install_fake_mpirun('mpirun', self.tmpdir, 'openmpi', '2.0')
ec, out = run([sys.executable, self.mympiscript, 'hostname'])
self.assertEqual(ec, 1)
regex = r"OpenMPI 2\.0\.x uses a different naming protocol for nodes"
self.assertTrue(re.search(regex, out), "mympirun should produce an error with ompi 2.0: %s" % out)
def test_option_multi(self):
"""Test --multi command line option"""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=FAKE_MPIRUN_MACHINEFILE)
ec, out = run([sys.executable, self.mympiscript, '--multi', '3', '--', 'hostname'])
# set_pbs_env() sets 2 cores, so *3 = 6
self.assertEqual(out, ('\n'.join(['localhost'] * 6)))
self.assertEqual(len(out.split('\n')), 6)
def run_cache_create():
"""Run the script to create the Lmod cache"""
lmod_dir = os.environ.get("LMOD_DIR", None)
if not lmod_dir:
LOGGER.raiseException("Cannot find $LMOD_DIR in the environment.", RuntimeError)
return run([os.path.join(lmod_dir, 'update_lmod_system_cache_files'), MODULEROOT])
def test_convert_to_unix_timestamp(self):
ec, out = run(['date', '+%s'])
nowts = convert_to_unix_timestamp()
self.assertTrue(abs(nowts - int(out.strip())) < 2)
date = 0
self.assertEqual(convert_to_unix_timestamp("19700101"), date)
self.assertEqual(convert_to_unix_timestamp("0000000000"), date)
self.assertEqual(convert_to_unix_timestamp(0), date)
self.assertEqual(convert_to_unix_timestamp("197001010000"), date)
self.assertEqual(convert_to_unix_timestamp("19700101000000Z"), date)
self.assertEqual(convert_to_unix_timestamp(datetime(1970, 1, 1)), date)
date = 1521936000
self.assertEqual(convert_to_unix_timestamp("20180325"), date)
date = 1521939900
self.assertEqual(convert_to_unix_timestamp("1521939900"), date)
self.assertEqual(convert_to_unix_timestamp(1521939900), date)
self.assertEqual(convert_to_unix_timestamp("201803250105"), date)
self.assertEqual(convert_to_unix_timestamp("201803250105"), date)
self.assertEqual(convert_to_unix_timestamp("20180325010500Z"), date)
self.assertEqual(
convert_to_unix_timestamp(datetime(2018, 3, 25, 1, 5)), date)
date = 1509239100
self.assertEqual(convert_to_unix_timestamp("1509239100"), date)
self.assertEqual(convert_to_unix_timestamp(1509239100), date)
self.assertEqual(convert_to_unix_timestamp("201710290105"), date)
self.assertEqual(convert_to_unix_timestamp("20171029010500Z"), date)
self.assertEqual(convert_to_unix_timestamp(date), date)
def get_localhosts(self):
"""
Get the localhost interfaces, based on the hostnames from the nodes in self.nodes_uniq.
Raises Exception if no localhost interface was found.
@return: the list of interfaces that correspond to the list of unique nodes
"""
iface_prefix = ['eth', 'em', 'ib', 'wlan']
reg_iface = re.compile(r'((?:%s)\d+(?:\.\d+)?(?::\d+)?|lo)' % '|'.join(iface_prefix))
# iterate over unique nodes and get their interfaces
# add the found interface to res if it matches reg_iface
res = []
for idx, nodename in enumerate(self.nodes_uniq):
ip = socket.gethostbyname(nodename)
cmd = "/sbin/ip -4 -o addr show to %s/32" % ip
exitcode, out = run(cmd)
if exitcode == 0:
regex = reg_iface.search(out)
if regex:
iface = regex.group(1)
self.log.debug("get_localhost idx %s: localhost interface %s found for %s (ip: %s)",
idx, iface, nodename, ip)
res.append((nodename, iface))
else:
self.log.debug("get_localhost idx %s: no interface match for prefixes %s out %s",
idx, iface_prefix, out)
else:
self.log.error("get_localhost idx %s: cmd %s failed with output %s", idx, cmd, out)
if not res:
self.log.raiseException("get_localhost: can't find localhost from nodes %s" % self.nodes_uniq)
return res
def test_env_variables(self):
""" Test the passing of (extra) variables """
fake_mpirun_env = """#!/bin/bash
echo 'fake mpirun called with args:' $@
env
"""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=fake_mpirun_env)
os.environ['PYTHONPATH'] = '/just/an/example:%s' % os.getenv('PYTHONPATH', '')
command = [
sys.executable,
self.mympiscript,
"--variablesprefix=USER",
"hostname",
]
ec, out = run(command)
for key in nub(filter(os.environ.has_key, MPI.OPTS_FROM_ENV_BASE)):
self.assertTrue(key in out, "%s is not in out" % key)
regex = r'.*-envlist [^ ]*USER.*'
self.assertTrue(re.search(regex, out), "Variablesprefix USER isn't passed to mympirun script env")
regex = r'PYTHONPATH=/just/an/example:.*'
self.assertTrue(re.search(regex, out), "PYTHONPATH isn't passed to mympirun script env correctly: %s" % out)
def test_openmpi_slurm(self):
"""Test running mympirun with OpenMPI in a SLURM environment."""
set_SLURM_env(self.tmpdir)
os.environ['SLURM_TASKS_PER_NODE'] = '2'
# patch scontrol to spit out "localhost" hostnames
scontrol = os.path.join(self.tmpdir, 'scontrol')
os.chmod(scontrol, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
fh = open(scontrol, 'w')
fh.write("#!/bin/bash\necho localhost")
fh.close()
install_fake_mpirun('mpirun', self.tmpdir, 'openmpi', '2.1', txt=FAKE_MPIRUN + "\nenv | sort")
ec, out = run([sys.executable, self.mympiscript, 'hostname'])
self.assertEqual(ec, 0, "Command exited normally: exit code %s; output: %s" % (ec, out))
# make sure output includes defined environment variables
# and "--mca orte_keep_fqdn_hostnames 1"
mca_keep_fqdn = "^fake mpirun called with args:.*--mca orte_keep_fqdn_hostnames 1 .*hostname$"
for pattern in ['^HOME=', '^USER='******'^SLURM_JOBID=', mca_keep_fqdn]:
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out))
# $SLURM_EXPORT_ENV should no longer be defined in environment
regex = re.compile('SLURM_EXPORT_ENV', re.M)
self.assertFalse(regex.search(out), "Pattern '%s' *not* found in: %s" % (regex.pattern, out))
def setUp(self):
"""Prepare to run test."""
self.orig_environ = copy.deepcopy(os.environ)
# add /bin to $PATH, /lib to $PYTHONPATH
self.topdir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
self.mympiscript = os.path.join(os.path.join(self.topdir, 'bin'), 'mympirun.py')
lib = os.path.join(self.topdir, 'lib')
# make sure subshell finds .egg files by adding them to the pythonpath
eggs = ':'.join(glob.glob(os.path.join(self.topdir, '.eggs', '*.egg')))
os.environ['PYTHONPATH'] = '%s:%s:%s' % (eggs, lib, os.getenv('PYTHONPATH', ''))
self.tmpdir = tempfile.mkdtemp()
os.environ['HOME'] = self.tmpdir
mpdconf = open(os.path.join(self.tmpdir, '.mpd.conf'), 'w')
mpdconf.write("password=topsecretpassword")
mpdconf.close()
# add location of symlink 'pbsssh' to actual pbsssh.sh script to $PATH too
os.mkdir(os.path.join(self.tmpdir, 'bin'))
os.symlink(os.path.join(self.topdir, 'bin', 'pbsssh.sh'), os.path.join(self.tmpdir, 'bin', 'pbsssh'))
os.environ['PATH'] = '%s:%s' % (os.path.join(self.tmpdir, 'bin'), os.getenv('PATH', ''))
# make sure we're using the right mympirun installation...
ec, out = run([sys.executable, '-c', "import vsc.mympirun; print vsc.mympirun.__file__"])
out = out.strip()
expected_path = os.path.join(self.topdir, 'lib', 'vsc', 'mympirun')
self.assertTrue(os.path.samefile(os.path.dirname(out), expected_path), "%s not in %s" % (out, expected_path))
# set variables that exist within jobs, but not locally, for testing
self.tmpdir = tempfile.mkdtemp()
pbs_tmpdir = os.path.join(self.tmpdir, 'pbs')
os.makedirs(pbs_tmpdir)
set_PBS_env(pbs_tmpdir)
def report(self, cluster):
"""Make a report for the given cluster"""
logging.debug("Start date: %s", self.convert_date(self.options.start))
logging.debug("End date: %s", self.convert_date(self.options.end))
sreport_command = SREPORT_TEMPLATE.format(
cluster=cluster,
startdate=self.convert_date(self.options.start),
enddate=self.convert_date(self.options.end))
ec, output = run(sreport_command)
logging.info("Report command ran, ec = %d", ec)
info = self.process(output)
relevant_info = {}
for (company, users) in self.users.items():
relevant_info[company] = dict(
filter(lambda kv: kv[0] in users, info.items()))
body = []
for (company, users) in sorted(relevant_info.items()):
for (user, usage) in sorted(users.items()):
body += [
"%s:%s:%s:%s" %
(cluster, company, user, usage_info_to_string(usage))
]
return body
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertTrue(ec > 0)
regex = re.compile(r"'?test/sandbox/testpkg/\*'?: No such file or directory")
self.assertTrue(regex.search(output), "Pattern '%s' found in: %s" % (regex.pattern, output))
ec, output = run_simple(TEST_GLOB)
self.glob_output(output, ec=ec)
def test_noshell_executable(self):
ec, output = run("echo '(foo bar)'")
self.assertEqual(ec, 0)
self.assertTrue('(foo bar)' in output)
ec, output = run(['echo', "(foo bar)"])
self.assertEqual(ec, 0)
self.assertTrue('(foo bar)' in output)
# to run Python command, it's required to use the right executable (Python shell rather than default)
ec, output = run("""%s -c 'print ("foo")'""" % sys.executable)
self.assertEqual(ec, 0)
self.assertTrue('foo' in output)
ec, output = run([sys.executable, '-c', 'print ("foo")'])
self.assertEqual(ec, 0)
self.assertTrue('foo' in output)
def test_noshell_executable(self):
ec, output = run("echo '(foo bar)'")
self.assertEqual(ec, 0)
self.assertTrue('(foo bar)' in output)
ec, output = run(['echo', "(foo bar)"])
self.assertEqual(ec, 0)
self.assertTrue('(foo bar)' in output)
# to run Python command, it's required to use the right executable (Python shell rather than default)
ec, output = run("""%s -c 'print ("foo")'""" % sys.executable)
self.assertEqual(ec, 0)
self.assertTrue('foo' in output)
ec, output = run([sys.executable, '-c', 'print ("foo")'])
self.assertEqual(ec, 0)
self.assertTrue('foo' in output)
def test_dry_run(self):
"""Test use of --dry-run/-D option."""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2')
for dry_run_opt in ['--dry-run', '-D']:
ec, out = run([sys.executable, self.mympiscript, dry_run_opt, 'hostname'])
self.assertEqual(ec, 0)
regex = re.compile('^mpirun .* hostname$')
self.assertTrue(regex.search(out.strip()), "Pattern '%s' found in: %s" % (regex.pattern, out))
extra_opts = ['--hybrid', '9']
ec, out = run([sys.executable, self.mympiscript, dry_run_opt] + extra_opts + ['hostname'])
self.assertEqual(ec, 0)
regex = re.compile('^mpirun .* -np 9 .* hostname$')
self.assertTrue(regex.search(out.strip()), "Pattern '%s' found in: %s" % (regex.pattern, out))
def test_launcher_opt_old_impi(self):
"""Test --launcher command line option with impi < 5.0.3"""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '4.2')
ec, out = run([sys.executable, self.mympiscript, 'hostname'])
regex = r'-bootstrap ssh'
self.assertTrue(re.search(regex, out), "-bootstrap option is not ssh (default for impi/4.2)" + out)
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertTrue(ec > 0)
regex = re.compile(r"'?test/sandbox/testpkg/\*'?: No such file or directory")
self.assertTrue(regex.search(output), "Pattern '%s' found in: %s" % (regex.pattern, output))
ec, output = run_simple(['ls','test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def run_test(mpi_name, mpi_ver, mpirun, pattern):
"""Utilitiy function to run test for a specific case."""
tmpdir = os.path.join(self.tmpdir, '%s-%s' % (mpi_name, mpi_ver))
install_fake_mpirun('mpirun', tmpdir, mpi_name, mpi_ver)
ec, out = run([sys.executable, self.mympiscript, '--setmpi', mpirun, '--sched', 'local', 'hostname'])
self.assertEqual(ec, 0, "Command exited normally: exit code %s; output: %s" % (ec, out))
regex = re.compile(regex_tmpl % pattern)
self.assertTrue(regex.match(out.strip()), "Pattern '%s' found in: %s" % (regex.pattern, out))
def test_serial(self):
"""Test running of a serial command via mympirun."""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2')
ec, out = run([sys.executable, self.mympiscript, 'hostname'])
self.assertEqual(ec, 0, "Command exited normally: exit code %s; output: %s" % (ec, out))
regex = re.compile("^fake mpirun called with args: .*hostname$")
self.assertTrue(regex.match(out.strip()), "Pattern '%s' found in: %s" % (regex.pattern, out))
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertEqual(ec, 127)
self.assertTrue(
'test/sandbox/testpkg/*: No such file or directory' in output)
ec, output = run_simple(['ls', 'test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(
all(x in output.lower()
for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def test_option_universe_hydra(self):
"""Test --universe command line option with hydra impi"""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.124.67')
cmd = "%s %s --universe 1 hostname"
ec, out = run([sys.executable, self.mympiscript, '--universe', '1', 'hostname'])
np_regex = re.compile('-np 1')
ncpus_regex = re.compile('--ncpus=1')
self.assertTrue(np_regex.search(out), "Pattern %s found in %s" % (np_regex, out))
self.assertFalse(ncpus_regex.search(out), "Pattern %s found in %s" % (ncpus_regex, out))
def test_get_localhosts(self):
"""test if localhost returns a list containing that are sourced correctly"""
mpi_instance = getinstance(mpim.MPI, Local, MympirunOption())
res = mpi_instance.get_localhosts()
_, out = run("/sbin/ip -4 -o addr show")
print("localhosts: %s" % res)
for (nodename, interface) in res:
self.assertTrue(nodename in mpi_instance.nodes,
msg="%s is not a node from the nodes list" % nodename)
self.assertTrue(interface in out,
msg="%s can not be found in the output of `/sbin/ip -4 -o addr show`, output: %s" %
(interface, out))
def test_which(self):
"""test if which returns a path that corresponds to unix which"""
testnames = ["python", "head", "tail", "cat"]
for scriptname in testnames:
mpiwhich = mpim.which(scriptname)
exitcode, unixwhich = run("which " + scriptname)
if exitcode > 0:
raise Exception("Something went wrong while trying to run `which`: %s" % unixwhich)
self.assertTrue(mpiwhich, msg="mpi which did not return anything, (unix which: %s" % unixwhich)
self.assertEqual(mpiwhich, string.strip(unixwhich),
msg="the return values of unix which and which() aren't"" the same: %s != %s" %
(mpiwhich, string.strip(unixwhich)))
def test_output(self):
""" Test --output option """
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=FAKE_MPIRUN + "\necho 'fake mpirun error' >&2")
f_out = os.path.join(self.tmpdir, "temp.out")
ec, out = run([sys.executable, self.mympiscript, '--output', f_out, 'hostname'])
self.assertEqual(ec, 0, "Command exited normally: exit code %s; output: %s" % (ec, out))
self.assertFalse(out, "Found output in stdout/stderr: %s" % out)
self.assertTrue(os.path.isfile(f_out))
regex = re.compile("^fake mpirun called with args: .* hostname\nfake mpirun error$")
with open(f_out, 'r') as output:
text = output.read()
self.assertTrue(regex.match(text), "Pattern '%s' found in: %s" % (regex.pattern, text))
def test_getgrouplist(self):
"""Test getgrouplist"""
for user in pwd.getpwall():
gidgroups = sorted(getgrouplist(user.pw_uid))
namegroups = sorted(getgrouplist(user.pw_name))
# get named groups from id
# grp.getgrall is wrong, e.g. for root user on F30,
# gr_mem for root group is empty (as is entry in /etc/group),
# while id returns 1 group
#groups = [g.gr_name for g in grp.getgrall() if user.pw_name in g.gr_mem]
ec, groups_txt = run(['id', '-Gn', user.pw_name])
groups = sorted(groups_txt.strip().split())
logging.debug("User %s gidgroups %s namegroups %s groups %s",
user, gidgroups, namegroups, groups)
self.assertEqual(gidgroups, namegroups)
self.assertEqual(gidgroups, groups)
def test_getgrouplist(self):
"""Test getgrouplist"""
for user in pwd.getpwall():
gidgroups = sorted(getgrouplist(user.pw_uid))
namegroups = sorted(getgrouplist(user.pw_name))
# get named groups from id
# grp.getgrall is wrong, e.g. for root user on F30,
# gr_mem for root group is empty (as is entry in /etc/group),
# while id returns 1 group
#groups = [g.gr_name for g in grp.getgrall() if user.pw_name in g.gr_mem]
ec, groups_txt = run(['id', '-Gn', user.pw_name])
groups = sorted(groups_txt.strip().split())
logging.debug("User %s gidgroups %s namegroups %s groups %s", user,
gidgroups, namegroups, groups)
self.assertEqual(gidgroups, namegroups)
self.assertEqual(gidgroups, groups)
def get_hydra_info(self):
"""Get a dict with hydra info."""
reg_hydra_info = re.compile(r"^\s+(?P<key>\S[^:\n]*)\s*:(?P<value>.*?)\s*$", re.M)
cmd = "mpirun -info"
exitcode, out = run(cmd)
if exitcode > 0:
self.log.raiseException("get_hydra_info: failed to run cmd %s: %s" % (cmd, out))
hydra_info = {}
for regex in reg_hydra_info.finditer(out):
key = regex.groupdict()['key']
if key is None:
self.log.raiseException("get_hydra_info: failed to get hydra info: missing key in %s (out: %s)" %
(regex.groupdict(), out))
key = key.strip().lower()
value = regex.groupdict()['value']
if value is None:
self.log.debug("get_hydra_info: failed to get hydra info: missing value in %s (out: %s)" %
(regex.groupdict(), out))
value = ''
values = [x.strip().strip('"').strip("'") for x in value.split() if x.strip()]
hydra_info[key] = values
self.log.debug("get_hydra_info: found info %s", hydra_info)
keymap = {
"rmk": r'^resource\s+management\s+kernel.*available',
"launcher": r'^%s.*available' % self.HYDRA_LAUNCHER_NAME,
"chkpt": r'^checkpointing.*available',
}
self.hydra_info = {}
for newkey, regtxt in keymap.items():
reg = re.compile(regtxt, re.I)
matches = [v for k, v in hydra_info.items() if reg.search(k)]
if len(matches) == 0:
continue
else:
if len(matches) > 1:
self.log.warning("get_hydra_info: more than one match %s found: newkey %s regtxt %s hydrainfo %s",
matches, newkey, regtxt, hydra_info)
self.hydra_info[newkey] = matches[0]
self.log.debug("get_hydra_info: filtered info %s", self.hydra_info)
def _get_uniq_nodes(self):
"""Get list of unique nodes, via $SLURM_NODELIST."""
res = None
nodelist = os.environ.get(self.SCHED_ENVIRON_NODE_INFO)
if nodelist is None:
self.log.raiseException("set_nodes: failed to get $%s from environment" % self.SCHED_ENVIRON_NODE_INFO)
else:
self.log.debug("set_nodes: obtained $%s value: %s", self.SCHED_ENVIRON_NODE_INFO, nodelist)
cmd = "scontrol show hostname %s" % nodelist
ec, out = run(cmd)
if ec:
self.log.raiseException("set_nodes: failed to get list of unique hostnames using '%s': %s" % (cmd, out))
else:
res = out.strip().split('\n')
self.log.debug("_get_uniq_nodes: %s" % res)
return res
def test_hanging_fatal(self):
"""Test fatal hanging check when program has no output"""
no_output_mpirun = '\n'.join([
"#!/bin/bash",
"sleep 4",
])
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=no_output_mpirun)
cmd = [
sys.executable,
self.mympiscript,
'--output-check-fatal',
'--output-check-timeout', '2',
'hostname',
]
ec, out = run(cmd)
self.assertEqual(ec, 124)
regex = re.compile("mympirun has been running for .* seconds without seeing any output.")
self.assertTrue(len(regex.findall(out)) == 1, "Pattern '%s' found in: %s" % (regex.pattern, out))
def set_netmask(self):
"""
Set self.netmask to a list containing (ip address/netmask).
Based on the hosts IP address (from ip addr show) and the selected netmasktype from select_device.
"""
if self.netmasktype is None:
self.select_device()
device_ip_reg_map = {
'eth': r"ether.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)",
'ib': r"infiniband.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)",
}
if self.netmasktype not in device_ip_reg_map:
self.log.raiseException("set_netmask: can't get netmask for %s: unknown mode (device_ip_reg_map %s)" %
(self.netmasktype, device_ip_reg_map))
cmd = "/sbin/ip addr show"
exitcode, out = run(cmd)
if exitcode > 0:
self.log.raiseException("set_netmask: failed to run cmd '%s', ec: %s, out: %s" % (cmd, exitcode, out))
reg = re.compile(device_ip_reg_map[self.netmasktype])
if not reg.search(out):
self.log.raiseException("set_netmask: can't get netmask for %s: no matches found (reg %s out %s)" %
(self.netmasktype, device_ip_reg_map[self.netmasktype], out))
res = []
for ipaddr_mask in reg.finditer(out):
ip_info = IP(ipaddr_mask.group(1), make_net=True)
network_netmask = "%s/%s" % (ip_info.net(), ip_info.netmask())
res.append(network_netmask)
self.log.debug("set_netmask: convert ipaddr_mask %s into network_netmask %s",
ipaddr_mask.group(1), network_netmask)
self.log.debug("set_netmask: return complete netmask %s", res)
if res:
self.netmask = os.pathsep.join(res)
def test_hanging(self):
""" Test --output-check-timeout option when program has no output"""
no_output_mpirun = '\n'.join([
"#!/bin/bash",
"sleep 4",
"echo 'some output'",
"sleep 3"
])
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=no_output_mpirun)
cmd = [
sys.executable,
self.mympiscript,
'--output-check-timeout', '2',
'hostname',
]
ec, out = run(cmd)
self.assertEqual(ec, 0, "Command exited normally: exit code %s; output: %s" % (ec, out))
regex = re.compile(("mympirun has been running for .* seconds without seeing any output.\n"
"This may mean that your program is hanging, please check and make sure that is not the case!"))
self.assertTrue(len(regex.findall(out)) == 1, "Pattern '%s' found in: %s" % (regex.pattern, out))
def test_hanging_file(self):
""" Test --output-check-timeout option when program has no output writing to file"""
no_output_mpirun = '\n'.join([
"#!/bin/bash",
"sleep 4",
"echo 'some output'",
"sleep 3"
])
f_out = os.path.join(self.tmpdir, "temp.out")
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=no_output_mpirun)
cmd = [
sys.executable,
self.mympiscript,
'--output', f_out,
'--output-check-timeout', '2',
'hostname',
]
ec, out = run(cmd)
self.assertEqual(ec, 0, "Command exited normally: exit code %s; output: %s" % (ec, out))
regex = re.compile("mympirun has been running for .* seconds without seeing any output.")
self.assertTrue(len(regex.findall(out)) == 1, "Pattern '%s' found in: %s" % (regex.pattern, out))
def test_env(self):
ec, output = run(cmd="/usr/bin/env", env={"MYENVVAR": "something"})
self.assertEqual(ec, 0)
self.assertTrue('myenvvar=something' in output.lower())
def test_option_hybrid(self):
"""Test --hybrid command line option"""
install_fake_mpirun('mpirun', self.tmpdir, 'impi', '5.1.2', txt=FAKE_MPIRUN_MACHINEFILE)
ec, out = run([sys.executable, self.mympiscript, '--hybrid', '5', 'hostname'])
self.assertEqual(out, ('\n'.join(['localhost'] * 5)))
self.assertEqual(len(out.split('\n')), 5)
