def test_optcomplete(self):
"""Test optcomplete support"""
reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$')
script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'runtests', 'simple_option.py')
partial = '-'
cmd_list = [script, partial]
ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0)
compl_opts = reg_reply.search(out).group(1).split()
basic_opts = ['--debug', '--enable-debug', '--disable-debug', '-d',
'--help', '-h', '-H', '--shorthelp',
'--configfiles', '--info',
]
for opt in basic_opts:
self.assertTrue(opt in compl_opts)
# test --deb autocompletion
partial = '--deb'
cmd_list = [script, partial]
ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0)
compl_opts = reg_reply.search(out).group(1).split()
self.assertEqual(compl_opts, ['--debug'])
def test_optcomplete(self):
"""Test optcomplete support"""
reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$')
script_name = 'simple_option.py'
script_simple = os.path.join(os.path.dirname(__file__), 'runtests', script_name)
partial = '-'
cmd_list = [script_simple, partial]
ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0)
compl_opts = reg_reply.search(out).group(1).split()
basic_opts = ['--debug', '--enable-debug', '--disable-debug', '-d',
'--help', '-h', '-H', '--shorthelp',
'--configfiles', '--info',
]
for opt in basic_opts:
self.assertTrue(opt in compl_opts)
# test --deb autocompletion
partial = '--deb'
cmd_list = [script_simple, partial]
ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0)
compl_opts = reg_reply.search(out).group(1).split()
self.assertEqual(compl_opts, ['--debug'])
def test_simple_glob(self):
ec, output = run_simple('ls test/sandbox/testpkg/*')
self.assertEqual(ec, 0)
self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
ec, output = run_simple(['ls','test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def test_optcomplete(self):
"""Test optcomplete support"""
reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$')
script = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'runtests', 'simple_option.py')
partial = '-'
cmd_list = [script, partial]
os.environ['SHELL'] = "bash"
pythonpath = 'PYTHONPATH="%s"' % os.pathsep.join(
[p for p in sys.path if p.startswith(self.setup.REPO_BASE_DIR)])
ec, out = run_simple(
'%s %s; test $? == 1' %
(pythonpath, gen_cmdline(cmd_list, partial, shebang=False)))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0, msg="simple_option.py test script ran success")
print(out)
reply_match = reg_reply.search(out)
self.assertTrue(reply_match,
msg="COMPREPLY %s in output %s" %
(reg_reply.pattern, out))
compl_opts = reply_match.group(1).split()
basic_opts = [
'--debug',
'--enable-debug',
'--disable-debug',
'-d',
'--help',
'-h',
'-H',
'--shorthelp',
'--configfiles',
'--info',
]
for opt in basic_opts:
self.assertTrue(opt in compl_opts)
# test --deb autocompletion
partial = '--deb'
cmd_list = [script, partial]
ec, out = run_simple(
'%s %s; test $? == 1' %
(pythonpath, gen_cmdline(cmd_list, partial, shebang=False)))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0)
compl_opts = reg_reply.search(out).group(1).split()
self.assertEqual(compl_opts, ['--debug'])
def test_simple_glob(self):
ec, output = run_simple('ls test/sandbox/testpkg/*')
self.assertEqual(ec, 0)
self.assertTrue(
all(x in output.lower()
for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
ec, output = run_simple(['ls', 'test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(
all(x in output.lower()
for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def get_localhosts(self):
"""
Get the localhost interfaces, based on the hostnames from the nodes in self.uniquenodes.
Raises Exception if no localhost interface was found.
@return: the list of interfaces that correspond to the list of uniquenodes
"""
iface_prefix = ['eth', 'em', 'ib', 'wlan']
reg_iface = re.compile(r'((?:%s)\d+(?:\.\d+)?(?::\d+)?|lo)' % '|'.join(iface_prefix))
# iterate over uniquenodes and get their interfaces
# add the found interface to res if it matches reg_iface
res = []
for idx, nodename in enumerate(self.uniquenodes):
ip = socket.gethostbyname(nodename)
cmd = "/sbin/ip -4 -o addr show to %s/32" % ip
exitcode, out = run_simple(cmd)
if exitcode == 0:
regex = reg_iface.search(out)
if regex:
iface = regex.group(1)
self.log.debug("get_localhost idx %s: localhost interface %s found for %s (ip: %s)",
idx, iface, nodename, ip)
res.append((nodename, iface))
else:
self.log.debug("get_localhost idx %s: no interface match for prefixes %s out %s",
idx, iface_prefix, out)
else:
self.log.error("get_localhost idx %s: cmd %s failed with output %s", idx, cmd, out)
if not res:
self.log.raiseException("get_localhost: can't find localhost from uniq nodes %s" % self.uniquenodes)
return res
def set_netmask(self):
if self.netmasktype is None:
self.set_device()
device_ip_reg_map = {'eth': "ether.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)",
'ib': "infiniband.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)"
}
if not self.netmasktype in device_ip_reg_map:
self.log.raiseException("set_netmask: can't get netmask for %s: unknown mode (device_ip_reg_map %s)" %
(self.netmasktype, device_ip_reg_map))
cmd = "/sbin/ip addr show"
ec, out = run_simple(cmd)
if ec > 0:
self.log.raiseException("set_netmask: failed to run cmd %s: %s" % (cmd, out))
reg = re.compile(r"" + device_ip_reg_map[self.netmasktype])
if not reg.search(out):
self.log.raiseException("set_netmask: can't get netmask for %s: no matches found (reg %s out %s)" %
(self.netmasktype, device_ip_reg_map[self.netmasktype], out))
res = []
for ipaddr_mask in reg.finditer(out):
ip = IP(ipaddr_mask.group(1), make_net=True)
network_netmask = "%s/%s" % (ip.net(), ip.netmask())
res.append(network_netmask)
self.log.debug("set_netmask: convert ipaddr_mask %s into network_netmask %s" %
(ipaddr_mask.group(1), network_netmask))
self.log.debug("set_netmask: return complete netmask %s" % res)
if len(res) > 0:
self.netmask = ":".join(res)
def get_localhosts(self):
"""
Get the localhost interfaces from the uniquenodes list
-- if hostname is different from the name in the nodelist
"""
iface_prefix = ['eth', 'em', 'ib', 'wlan']
reg_iface = re.compile(r'((?:%s)\d+(?:\.\d+)?(?::\d+)?|lo)' % '|'.join(iface_prefix))
res = []
for idx, hn in enumerate(self.uniquenodes):
ip = socket.gethostbyname(hn)
cmd = "/sbin/ip -4 -o addr show to %s/32" % ip # TODO ipv6
ec, out = run_simple(cmd)
if ec == 0:
r = reg_iface.search(out)
if r:
iface = r.group(1)
self.log.debug("get_localhost idx %s: localhost interface %s found for %s (ip: %s)" %
(idx, iface, hn, ip))
res.append((hn, iface))
else:
# not a big issue, probably not
self.log.debug(("get_localhost idx %s: no interface match for "
"prefixes %s out %s") % (idx, iface_prefix, out))
else:
self.log.error("get_localhost idx %s: cmd %s failed with output %s" % (idx, cmd, out))
if len(res) == 0:
self.log.raiseException("get_localhost: can't find localhost from uniq nodes %s" %
(self.uniquenodes))
return res
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertEqual(ec, 127)
self.assertTrue('test/sandbox/testpkg/*: No such file or directory' in output)
ec, output = run_simple(['ls','test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def run_cache_create(archs=None):
"""Run the script to create the Lmod cache"""
lmod_dir = os.environ.get("LMOD_DIR", None)
if not lmod_dir:
raise RuntimeError("Cannot find $LMOD_DIR in the environment.")
if not archs:
archs = os.listdir(MODULES_BASEDIR)
for arch in archs:
modpath = os.path.join(MODULES_BASEDIR, arch, "modules")
if not os.path.isdir(modpath):
continue
logger.debug("Creating cache for %s", arch)
modsubpathglob = os.path.join(modpath, "20[0-9][0-9][ab]", "all")
modsubpaths = os.pathsep.join(sorted(glob.glob(modsubpathglob)))
cachedir = os.path.join(MODULES_BASEDIR, arch, "cacheDir")
systemfile = os.path.join(cachedir, "system.txt")
cmd = "%s/update_lmod_system_cache_files -d %s -t %s %s" % (
lmod_dir, cachedir, systemfile, modsubpaths)
exitcode, msg = run_simple(cmd)
if exitcode != 0:
return exitcode, msg
return 0, ''
def set_netmask(self):
if self.netmasktype is None:
self.set_device()
device_ip_reg_map = {'eth':"ether.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)",
'ib':"infiniband.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)"
}
if not self.netmasktype in device_ip_reg_map:
self.log.raiseException("set_netmask: can't get netmask for %s: unknown mode (device_ip_reg_map %s)" %
(self.netmasktype, device_ip_reg_map))
cmd = "/sbin/ip addr show"
ec, out = run_simple(cmd)
if ec > 0:
self.log.raiseException("set_netmask: failed to run cmd %s: %s" % (cmd, out))
reg = re.compile(r"" + device_ip_reg_map[self.netmasktype])
if not reg.search(out):
self.log.raiseException("set_netmask: can't get netmask for %s: no matches found (reg %s out %s)" %
(self.netmasktype, device_ip_reg_map[self.netmasktype], out))
res = []
for ipaddr_mask in reg.finditer(out):
ip = IP(ipaddr_mask.group(1), make_net=True)
network_netmask = "%s/%s" % (ip.net(), ip.netmask())
res.append(network_netmask)
self.log.debug("set_netmask: convert ipaddr_mask %s into network_netmask %s" %
(ipaddr_mask.group(1), network_netmask))
self.log.debug("set_netmask: return complete netmask %s" % res)
if len(res) > 0:
self.netmask = ":".join(res)
def get_localhosts(self):
"""
Get the localhost interfaces from the uniquenodes list
-- if hostname is different from the name in the nodelist
"""
iface_prefix = ['eth', 'em', 'ib', 'wlan']
reg_iface = re.compile(r'((?:%s)\d+(?:\.\d+)?(?::\d+)?|lo)' % '|'.join(iface_prefix))
res = []
for idx, hn in enumerate(self.uniquenodes):
ip = socket.gethostbyname(hn)
cmd = "/sbin/ip -4 -o addr show to %s/32" % ip # TODO ipv6
ec, out = run_simple(cmd)
if ec == 0:
r = reg_iface.search(out)
if r:
iface = r.group(1)
self.log.debug("get_localhost idx %s: localhost interface %s found for %s (ip: %s)" %
(idx, iface, hn, ip))
res.append((hn, iface))
else:
# not a big issue, probably not
self.log.debug(("get_localhost idx %s: no interface match for "
"prefixes %s out %s") % (idx, iface_prefix, out))
else:
self.log.error("get_localhost idx %s: cmd %s failed with output %s" % (idx, cmd, out))
if len(res) == 0:
self.log.raiseException("get_localhost: can't find localhost from uniq nodes %s" %
(self.uniquenodes))
return res
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertTrue(ec > 0)
regex = re.compile(r"'?test/sandbox/testpkg/\*'?: No such file or directory")
self.assertTrue(regex.search(output), "Pattern '%s' found in: %s" % (regex.pattern, output))
ec, output = run_simple(TEST_GLOB)
self.glob_output(output, ec=ec)
def run_cache_create(modules_root):
"""Run the script to create the Lmod cache"""
lmod_dir = os.environ.get("LMOD_DIR", None)
if not lmod_dir:
raise RuntimeError("Cannot find $LMOD_DIR in the environment.")
cmd = "%s/update_lmod_system_cache_files %s" % (lmod_dir, modules_root)
return run_simple(cmd)
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertTrue(ec > 0)
regex = re.compile(r"'?test/sandbox/testpkg/\*'?: No such file or directory")
self.assertTrue(regex.search(output), "Pattern '%s' found in: %s" % (regex.pattern, output))
ec, output = run_simple(['ls','test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def worker_run_simple(counter):
"""Execute the cmd
to be called with
"""
cmd_sanity = ["%s" % x for x in parse_worker_args()] ## ready to join
set_scoop_env('counter', counter)
ec, out = run_simple(' '.join(cmd_sanity), disable_log=True)
return ec, out ## return 1 item
def worker_run_simple(counter):
"""Execute the cmd
to be called with
"""
cmd_sanity = ["%s" % x for x in parse_worker_args()] ## ready to join
set_scoop_env('counter', counter)
ec, out = run_simple(' '.join(cmd_sanity), disable_log=True)
return ec, out ## return 1 item
def test_noshell_glob(self):
ec, output = run('ls test/sandbox/testpkg/*')
self.assertEqual(ec, 127)
self.assertTrue(
'test/sandbox/testpkg/*: No such file or directory' in output)
ec, output = run_simple(['ls', 'test/sandbox/testpkg/*'])
self.assertEqual(ec, 0)
self.assertTrue(
all(x in output.lower()
for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def test_optcomplete(self):
"""Test optcomplete support"""
reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$')
script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'runtests', 'simple_option.py')
partial = '-'
cmd_list = [script, partial]
os.environ['SHELL'] = "bash"
pythonpath = 'PYTHONPATH="%s"' % os.pathsep.join([p for p in sys.path if p.startswith(self.setup.REPO_BASE_DIR)])
ec, out = run_simple('%s %s; test $? == 1' % (pythonpath, gen_cmdline(cmd_list, partial, shebang=False)))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0, msg="simple_option.py test script ran success")
print out
reply_match = reg_reply.search(out)
self.assertTrue(reply_match, msg="COMPREPLY %s in output %s" % (reg_reply.pattern, out))
compl_opts = reply_match.group(1).split()
basic_opts = ['--debug', '--enable-debug', '--disable-debug', '-d',
'--help', '-h', '-H', '--shorthelp',
'--configfiles', '--info',
]
for opt in basic_opts:
self.assertTrue(opt in compl_opts)
# test --deb autocompletion
partial = '--deb'
cmd_list = [script, partial]
ec, out = run_simple('%s %s; test $? == 1' % (pythonpath, gen_cmdline(cmd_list, partial, shebang=False)))
# tabcompletion ends with exit 1!; test returns this to 0
# avoids run.log.error message
self.assertEqual(ec, 0)
compl_opts = reg_reply.search(out).group(1).split()
self.assertEqual(compl_opts, ['--debug'])
def get_hydra_info(self):
"""Get a dict with hydra info"""
reg_hydra_info = re.compile(
r"^\s+(?P<key>\S[^:\n]*)\s*:(?P<value>.*?)\s*$", re.M)
cmd = "mpirun -info"
ec, out = run_simple(cmd)
if ec > 0:
self.log.raiseException(
"get_hydra_info: failed to run cmd %s: %s" % (cmd, out))
hydra_info = {}
for r in reg_hydra_info.finditer(out):
key = r.groupdict()['key']
if key is None:
self.log.raiseException(
"get_hydra_info: failed to get hydra info: missing key in %s (out: %s)"
% (r.groupdict(), out))
key = key.strip().lower()
value = r.groupdict()['value']
if value is None:
self.log.debug(
"get_hydra_info: failed to get hydra info: missing value in %s (out: %s)"
% (r.groupdict(), out))
value = ''
values = [
x.strip().strip('"').strip("'") for x in value.split()
if len(x.strip()) > 0
]
hydra_info[key] = values
self.log.debug("get_hydra_info: found info %s" % hydra_info)
keymap = {
"rmk": r'^resource\s+management\s+kernel.*available',
"launcher": r'^%s.*available' % self.HYDRA_LAUNCHER_NAME,
"chkpt": r'^checkpointing.*available',
}
self.hydra_info = {}
for newkey, regtxt in keymap.items():
reg = re.compile(regtxt, re.I)
matches = [v for k, v in hydra_info.items() if reg.search(k)]
if len(matches) == 0:
continue
else:
if len(matches) > 1:
self.log.warning((
"get_hydra_info: more then one match %s found for newkey %s "
"regtxt %s hydrainfo %s") %
(matches, newkey, regtxt, hydra_info))
self.hydra_info[newkey] = matches[0]
self.log.debug("get_hydra_info: filtered info %s" % self.hydra_info)
def hwlocmap(self):
"""parse and return output from hwloc-ls
Returns a dict that maps the absolute Processor Unit ID to its socket-id and its core-id
"""
res = {}
xmlout = "/tmp/test.xml.%s" % os.getpid()
exe = "/usr/bin/hwloc-ls"
if not os.path.exists(exe):
self.log.error("hwlocmap: Can't find exe %s", exe)
cmd = "%s --output-format xml %s" % (exe, xmlout)
ec, txt = run_simple(cmd)
# parse xmloutput
base = etree.parse(xmlout)
sks_xpath = './/object[@type="Socket"]'
# list of socket ids
sks = map(int, [i.attrib['os_index'] for i in base.findall(sks_xpath)])
self.log.debug("sockets: %s", sks)
aPU = 0
for x, sk in enumerate(sks):
cr_xpath = '%s[@os_index="%s"]//object[@type="Core"]' % (sks_xpath,
x)
# list of core ids in socket x
crs = map(int,
[j.attrib['os_index'] for j in base.findall(cr_xpath)])
self.log.debug("cores: %s", crs)
for y, cr in enumerate(crs):
pu_xpath = '%s[@os_index="%s"]//object[@type="PU"]' % (
cr_xpath, y)
# list of PU ids in core y from socket x
pus = map(
int,
[k.attrib['os_index'] for k in base.findall(pu_xpath)])
self.log.debug("PU's: %s", pus)
# absolute PU id = (socket id * cores per socket * PU's in core) + PU id
# in case of errors, revert back to this
# aPU = sks[x] * len(crs) * len(pus) + pus[z]
for pu in pus:
t = "socket %s core %s abscore %s" % (sk, cr, aPU)
res[aPU] = t
aPU += 1
self.log.debug("result map: %s", res)
return res
def test_run_subshell(self):
"""Read data from testjobs_submitfilter and feed it through submitfilter script"""
testdir = os.path.join(os.path.dirname(__file__), 'testjobs_submitfilter')
for scriptfn in glob.glob("%s/*.script" % testdir):
scriptname = os.path.basename(scriptfn)
name = '.'.join(scriptname.split('.')[:-1])
script = os.path.join(testdir, scriptname)
out = os.path.join(testdir, "%s.out" % name)
err = os.path.join(testdir, "%s.err" % name)
log = os.path.join(testdir, "%s.log" % name)
cmdline = os.path.join(testdir, "%s.cmdline" % name)
# avoid pyc files in e.g. bin
cmd = 'PYTHONPATH=%s:$PYTHONPATH ' % os.pathsep.join([p for p in sys.path if p.startswith(REPO_BASE_DIR)])
cmd += "python -B %s" % submitfilter.__file__
if os.path.exists(cmdline):
cmd += " " + open(cmdline).readline().strip()
# make command
# get output, and exitcode
ec, output = run_simple(cmd, input=open(script).read())
self.assertEqual(ec, 0, msg="submitfiler ended with ec 0 for script %s and cmdline %s" % (name, cmd))
res = ''
if os.path.exists(log):
# multiline pattern match, line per line
for pattern in open(log).readlines():
if not pattern or pattern.startswith('#'):
continue
reg = re.compile(r''+pattern, re.M)
if reg.search(output):
output = reg.sub('', output)
else:
self.assertTrue(False, "Expected a log pattern match %s for script %s" % (pattern, name))
if os.path.exists(out):
res += open(out).read()
else:
self.assertTrue(False, msg='no output file %s for script %s' % (out, name))
if os.path.exists(err):
res += open(err).read()
self.assertEqual(output, res, msg="expected output for script %s and cmdline %s" % (name, cmd))
def test_run_subshell(self):
"""Read data from testjobs_submitfilter and feed it through submitfilter script"""
testdir = os.path.join(os.path.dirname(__file__), "testjobs_submitfilter")
for scriptfn in glob.glob("%s/*.script" % testdir):
scriptname = os.path.basename(scriptfn)
name = ".".join(scriptname.split(".")[:-1])
script = os.path.join(testdir, scriptname)
out = os.path.join(testdir, "%s.out" % name)
err = os.path.join(testdir, "%s.err" % name)
log = os.path.join(testdir, "%s.log" % name)
cmdline = os.path.join(testdir, "%s.cmdline" % name)
# avoid pyc files in e.g. bin
cmd = "PYTHONPATH=%s:$PYTHONPATH " % os.pathsep.join([p for p in sys.path if p.startswith(REPO_BASE_DIR)])
cmd += "python -B %s" % submitfilter.__file__
if os.path.exists(cmdline):
cmd += " " + open(cmdline).readline().strip()
# make command
# get output, and exitcode
ec, output = run_simple(cmd, input=open(script).read())
self.assertEqual(ec, 0, msg="submitfiler ended with ec 0 for script %s and cmdline %s" % (name, cmd))
res = ""
if os.path.exists(log):
# multiline pattern match, line per line
for pattern in open(log).readlines():
if not pattern or pattern.startswith("#"):
continue
reg = re.compile(r"" + pattern, re.M)
if reg.search(output):
output = reg.sub("", output)
else:
self.assertTrue(False, "Expected a log pattern match %s for script %s" % (pattern, name))
if os.path.exists(out):
res += open(out).read()
else:
self.assertTrue(False, msg="no output file %s for script %s" % (out, name))
if os.path.exists(err):
res += open(err).read()
self.assertEqual(output, res, msg="expected output for script %s and cmdline %s" % (name, cmd))
def hwlocmap(self):
"""parse and return output from hwloc-ls
Returns a dict that maps the absolute Processor Unit ID to its socket-id and its core-id
"""
res = {}
xmlout = "/tmp/test.xml.%s" % os.getpid()
exe = "/usr/bin/hwloc-ls"
if not os.path.exists(exe):
self.log.error("hwlocmap: Can't find exe %s", exe)
cmd = "%s --output-format xml %s" % (exe, xmlout)
ec, txt = run_simple(cmd)
# parse xmloutput
base = etree.parse(xmlout)
sks_xpath = './/object[@type="Socket"]'
# list of socket ids
sks = map(int, [i.attrib['os_index'] for i in base.findall(sks_xpath)])
self.log.debug("sockets: %s", sks)
aPU = 0
for x, sk in enumerate(sks):
cr_xpath = '%s[@os_index="%s"]//object[@type="Core"]' % (sks_xpath, x)
# list of core ids in socket x
crs = map(int, [j.attrib['os_index'] for j in base.findall(cr_xpath)])
self.log.debug("cores: %s", crs)
for y, cr in enumerate(crs):
pu_xpath = '%s[@os_index="%s"]//object[@type="PU"]' % (cr_xpath, y)
# list of PU ids in core y from socket x
pus = map(int, [k.attrib['os_index'] for k in base.findall(pu_xpath)])
self.log.debug("PU's: %s", pus)
# absolute PU id = (socket id * cores per socket * PU's in core) + PU id
# in case of errors, revert back to this
# aPU = sks[x] * len(crs) * len(pus) + pus[z]
for pu in pus:
t = "socket %s core %s abscore %s" % (sk, cr, aPU)
res[aPU] = t
aPU += 1
self.log.debug("result map: %s", res)
return res
def get_hydra_info(self):
"""Get a dict with hydra info."""
reg_hydra_info = re.compile(r"^\s+(?P<key>\S[^:\n]*)\s*:(?P<value>.*?)\s*$", re.M)
cmd = "mpirun -info"
exitcode, out = run_simple(cmd)
if exitcode > 0:
self.log.raiseException("get_hydra_info: failed to run cmd %s: %s" % (cmd, out))
hydra_info = {}
for regex in reg_hydra_info.finditer(out):
key = regex.groupdict()['key']
if key is None:
self.log.raiseException("get_hydra_info: failed to get hydra info: missing key in %s (out: %s)" %
(regex.groupdict(), out))
key = key.strip().lower()
value = regex.groupdict()['value']
if value is None:
self.log.debug("get_hydra_info: failed to get hydra info: missing value in %s (out: %s)" %
(regex.groupdict(), out))
value = ''
values = [x.strip().strip('"').strip("'") for x in value.split() if x.strip()]
hydra_info[key] = values
self.log.debug("get_hydra_info: found info %s", hydra_info)
keymap = {
"rmk": r'^resource\s+management\s+kernel.*available',
"launcher": r'^%s.*available' % self.HYDRA_LAUNCHER_NAME,
"chkpt": r'^checkpointing.*available',
}
self.hydra_info = {}
for newkey, regtxt in keymap.items():
reg = re.compile(regtxt, re.I)
matches = [v for k, v in hydra_info.items() if reg.search(k)]
if len(matches) == 0:
continue
else:
if len(matches) > 1:
self.log.warning("get_hydra_info: more than one match %s found: newkey %s regtxt %s hydrainfo %s",
matches, newkey, regtxt, hydra_info)
self.hydra_info[newkey] = matches[0]
self.log.debug("get_hydra_info: filtered info %s", self.hydra_info)
def test_startpath(self):
cwd = os.getcwd()
cmd = "echo foo > bar"
self.assertErrorRegex(ValueError, '.*', run_simple, cmd, startpath='/no/such/directory')
ec, out = run_simple(cmd, startpath=self.tempdir)
# successfull command, no output
self.assertEqual(ec, 0)
self.assertEqual(out, '')
# command was actually executed, in specified directory
self.assertTrue(os.path.exists(os.path.join(self.tempdir, 'bar')))
txt = open(os.path.join(self.tempdir, 'bar')).read()
self.assertEqual(txt, 'foo\n')
# we should still be in directory we were in originally
self.assertEqual(cwd, os.getcwd())
def test_startpath(self):
cwd = os.getcwd()
cmd = "echo foo > bar"
self.assertErrorRegex(Exception, '.*', run_simple, cmd, startpath='/no/such/directory')
ec, out = run_simple(cmd, startpath=self.tempdir)
# successfull command, no output
self.assertEqual(ec, 0)
self.assertEqual(out, '')
# command was actually executed, in specified directory
self.assertTrue(os.path.exists(os.path.join(self.tempdir, 'bar')))
txt = open(os.path.join(self.tempdir, 'bar')).read()
self.assertEqual(txt, 'foo\n')
# we should still be in directory we were in originally
self.assertEqual(cwd, os.getcwd())
def set_netmask(self):
"""
Set self.netmask to a list containing (ip address/netmask).
Based on the hosts IP address (from ip addr show) and the selected netmasktype from select_device.
"""
if self.netmasktype is None:
self.select_device()
device_ip_reg_map = {
'eth': r"ether.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)",
'ib': r"infiniband.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)",
}
if self.netmasktype not in device_ip_reg_map:
self.log.raiseException("set_netmask: can't get netmask for %s: unknown mode (device_ip_reg_map %s)" %
(self.netmasktype, device_ip_reg_map))
cmd = "/sbin/ip addr show"
exitcode, out = run_simple(cmd)
if exitcode > 0:
self.log.raiseException("set_netmask: failed to run cmd %s, ec: %s" % (cmd, exitcode))
reg = re.compile(device_ip_reg_map[self.netmasktype])
if not reg.search(out):
self.log.raiseException("set_netmask: can't get netmask for %s: no matches found (reg %s out %s)" %
(self.netmasktype, device_ip_reg_map[self.netmasktype], out))
res = []
for ipaddr_mask in reg.finditer(out):
ip_info = IP(ipaddr_mask.group(1), make_net=True)
network_netmask = "%s/%s" % (ip_info.net(), ip_info.netmask())
res.append(network_netmask)
self.log.debug("set_netmask: convert ipaddr_mask %s into network_netmask %s",
ipaddr_mask.group(1), network_netmask)
self.log.debug("set_netmask: return complete netmask %s", res)
if res:
self.netmask = os.pathsep.join(res)
def get_lmod_config():
"""Get the modules root and cache path from the Lmod config"""
lmod_cmd = os.environ.get("LMOD_CMD", None)
if not lmod_cmd:
raise RuntimeError("Cannot find $LMOD_CMD in the environment.")
ec, out = run_simple("%s bash --config-json" % lmod_cmd)
if ec != 0:
raise RuntimeError("Failed to get Lmod configuration: %s", out)
try:
lmodconfig = json.loads(out)
config = {
'modules_root': lmodconfig['configT']['mpath_root'],
'cache_dir': lmodconfig['cache'][0][0],
'cache_timestamp': lmodconfig['cache'][0][1],
}
logger.debug("Found Lmod config: %s", config)
except (ValueError, KeyError, IndexError, TypeError) as err:
raise RuntimeError("Failed to parse the Lmod configuration: %s", err)
return config
def test_simple(self):
ec, output = run_simple([sys.executable, SCRIPT_SIMPLE, 'shortsleep'])
self.assertEqual(ec, 0)
self.assertTrue('shortsleep' in output.lower())
def test_simple(self):
ec, output = run_simple([SCRIPT_SIMPLE, "shortsleep"])
self.assertEqual(ec, 0)
self.assertTrue("shortsleep" in output.lower())
def test_simple(self):
ec, output = run_simple([sys.executable, SCRIPT_SIMPLE, 'shortsleep'])
self.assertEqual(ec, 0)
self.assertTrue('shortsleep' in output.lower())
def test_simple_glob(self):
ec, output = run_simple(" ".join(TEST_GLOB))
self.glob_output(output, ec=ec)
ec, output = run_simple(TEST_GLOB)
self.glob_output(output, ec=ec)
