def test_optcomplete(self): """Test optcomplete support""" reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$') script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'runtests', 'simple_option.py') partial = '-' cmd_list = [script, partial] ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial)) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0) compl_opts = reg_reply.search(out).group(1).split() basic_opts = ['--debug', '--enable-debug', '--disable-debug', '-d', '--help', '-h', '-H', '--shorthelp', '--configfiles', '--info', ] for opt in basic_opts: self.assertTrue(opt in compl_opts) # test --deb autocompletion partial = '--deb' cmd_list = [script, partial] ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial)) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0) compl_opts = reg_reply.search(out).group(1).split() self.assertEqual(compl_opts, ['--debug'])
def test_optcomplete(self): """Test optcomplete support""" reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$') script_name = 'simple_option.py' script_simple = os.path.join(os.path.dirname(__file__), 'runtests', script_name) partial = '-' cmd_list = [script_simple, partial] ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial)) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0) compl_opts = reg_reply.search(out).group(1).split() basic_opts = ['--debug', '--enable-debug', '--disable-debug', '-d', '--help', '-h', '-H', '--shorthelp', '--configfiles', '--info', ] for opt in basic_opts: self.assertTrue(opt in compl_opts) # test --deb autocompletion partial = '--deb' cmd_list = [script_simple, partial] ec, out = run_simple('%s; test $? == 1' % gen_cmdline(cmd_list, partial)) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0) compl_opts = reg_reply.search(out).group(1).split() self.assertEqual(compl_opts, ['--debug'])
def test_simple_glob(self): ec, output = run_simple('ls test/sandbox/testpkg/*') self.assertEqual(ec, 0) self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py'])) ec, output = run_simple(['ls','test/sandbox/testpkg/*']) self.assertEqual(ec, 0) self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def test_optcomplete(self): """Test optcomplete support""" reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$') script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'runtests', 'simple_option.py') partial = '-' cmd_list = [script, partial] os.environ['SHELL'] = "bash" pythonpath = 'PYTHONPATH="%s"' % os.pathsep.join( [p for p in sys.path if p.startswith(self.setup.REPO_BASE_DIR)]) ec, out = run_simple( '%s %s; test $? == 1' % (pythonpath, gen_cmdline(cmd_list, partial, shebang=False))) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0, msg="simple_option.py test script ran success") print(out) reply_match = reg_reply.search(out) self.assertTrue(reply_match, msg="COMPREPLY %s in output %s" % (reg_reply.pattern, out)) compl_opts = reply_match.group(1).split() basic_opts = [ '--debug', '--enable-debug', '--disable-debug', '-d', '--help', '-h', '-H', '--shorthelp', '--configfiles', '--info', ] for opt in basic_opts: self.assertTrue(opt in compl_opts) # test --deb autocompletion partial = '--deb' cmd_list = [script, partial] ec, out = run_simple( '%s %s; test $? == 1' % (pythonpath, gen_cmdline(cmd_list, partial, shebang=False))) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0) compl_opts = reg_reply.search(out).group(1).split() self.assertEqual(compl_opts, ['--debug'])
def test_simple_glob(self): ec, output = run_simple('ls test/sandbox/testpkg/*') self.assertEqual(ec, 0) self.assertTrue( all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py'])) ec, output = run_simple(['ls', 'test/sandbox/testpkg/*']) self.assertEqual(ec, 0) self.assertTrue( all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def get_localhosts(self): """ Get the localhost interfaces, based on the hostnames from the nodes in self.uniquenodes. Raises Exception if no localhost interface was found. @return: the list of interfaces that correspond to the list of uniquenodes """ iface_prefix = ['eth', 'em', 'ib', 'wlan'] reg_iface = re.compile(r'((?:%s)\d+(?:\.\d+)?(?::\d+)?|lo)' % '|'.join(iface_prefix)) # iterate over uniquenodes and get their interfaces # add the found interface to res if it matches reg_iface res = [] for idx, nodename in enumerate(self.uniquenodes): ip = socket.gethostbyname(nodename) cmd = "/sbin/ip -4 -o addr show to %s/32" % ip exitcode, out = run_simple(cmd) if exitcode == 0: regex = reg_iface.search(out) if regex: iface = regex.group(1) self.log.debug("get_localhost idx %s: localhost interface %s found for %s (ip: %s)", idx, iface, nodename, ip) res.append((nodename, iface)) else: self.log.debug("get_localhost idx %s: no interface match for prefixes %s out %s", idx, iface_prefix, out) else: self.log.error("get_localhost idx %s: cmd %s failed with output %s", idx, cmd, out) if not res: self.log.raiseException("get_localhost: can't find localhost from uniq nodes %s" % self.uniquenodes) return res
def set_netmask(self): if self.netmasktype is None: self.set_device() device_ip_reg_map = {'eth': "ether.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)", 'ib': "infiniband.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)" } if not self.netmasktype in device_ip_reg_map: self.log.raiseException("set_netmask: can't get netmask for %s: unknown mode (device_ip_reg_map %s)" % (self.netmasktype, device_ip_reg_map)) cmd = "/sbin/ip addr show" ec, out = run_simple(cmd) if ec > 0: self.log.raiseException("set_netmask: failed to run cmd %s: %s" % (cmd, out)) reg = re.compile(r"" + device_ip_reg_map[self.netmasktype]) if not reg.search(out): self.log.raiseException("set_netmask: can't get netmask for %s: no matches found (reg %s out %s)" % (self.netmasktype, device_ip_reg_map[self.netmasktype], out)) res = [] for ipaddr_mask in reg.finditer(out): ip = IP(ipaddr_mask.group(1), make_net=True) network_netmask = "%s/%s" % (ip.net(), ip.netmask()) res.append(network_netmask) self.log.debug("set_netmask: convert ipaddr_mask %s into network_netmask %s" % (ipaddr_mask.group(1), network_netmask)) self.log.debug("set_netmask: return complete netmask %s" % res) if len(res) > 0: self.netmask = ":".join(res)
def get_localhosts(self): """ Get the localhost interfaces from the uniquenodes list -- if hostname is different from the name in the nodelist """ iface_prefix = ['eth', 'em', 'ib', 'wlan'] reg_iface = re.compile(r'((?:%s)\d+(?:\.\d+)?(?::\d+)?|lo)' % '|'.join(iface_prefix)) res = [] for idx, hn in enumerate(self.uniquenodes): ip = socket.gethostbyname(hn) cmd = "/sbin/ip -4 -o addr show to %s/32" % ip # TODO ipv6 ec, out = run_simple(cmd) if ec == 0: r = reg_iface.search(out) if r: iface = r.group(1) self.log.debug("get_localhost idx %s: localhost interface %s found for %s (ip: %s)" % (idx, iface, hn, ip)) res.append((hn, iface)) else: # not a big issue, probably not self.log.debug(("get_localhost idx %s: no interface match for " "prefixes %s out %s") % (idx, iface_prefix, out)) else: self.log.error("get_localhost idx %s: cmd %s failed with output %s" % (idx, cmd, out)) if len(res) == 0: self.log.raiseException("get_localhost: can't find localhost from uniq nodes %s" % (self.uniquenodes)) return res
def test_noshell_glob(self): ec, output = run('ls test/sandbox/testpkg/*') self.assertEqual(ec, 127) self.assertTrue('test/sandbox/testpkg/*: No such file or directory' in output) ec, output = run_simple(['ls','test/sandbox/testpkg/*']) self.assertEqual(ec, 0) self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def run_cache_create(archs=None): """Run the script to create the Lmod cache""" lmod_dir = os.environ.get("LMOD_DIR", None) if not lmod_dir: raise RuntimeError("Cannot find $LMOD_DIR in the environment.") if not archs: archs = os.listdir(MODULES_BASEDIR) for arch in archs: modpath = os.path.join(MODULES_BASEDIR, arch, "modules") if not os.path.isdir(modpath): continue logger.debug("Creating cache for %s", arch) modsubpathglob = os.path.join(modpath, "20[0-9][0-9][ab]", "all") modsubpaths = os.pathsep.join(sorted(glob.glob(modsubpathglob))) cachedir = os.path.join(MODULES_BASEDIR, arch, "cacheDir") systemfile = os.path.join(cachedir, "system.txt") cmd = "%s/update_lmod_system_cache_files -d %s -t %s %s" % ( lmod_dir, cachedir, systemfile, modsubpaths) exitcode, msg = run_simple(cmd) if exitcode != 0: return exitcode, msg return 0, ''
def set_netmask(self): if self.netmasktype is None: self.set_device() device_ip_reg_map = {'eth':"ether.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)", 'ib':"infiniband.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)" } if not self.netmasktype in device_ip_reg_map: self.log.raiseException("set_netmask: can't get netmask for %s: unknown mode (device_ip_reg_map %s)" % (self.netmasktype, device_ip_reg_map)) cmd = "/sbin/ip addr show" ec, out = run_simple(cmd) if ec > 0: self.log.raiseException("set_netmask: failed to run cmd %s: %s" % (cmd, out)) reg = re.compile(r"" + device_ip_reg_map[self.netmasktype]) if not reg.search(out): self.log.raiseException("set_netmask: can't get netmask for %s: no matches found (reg %s out %s)" % (self.netmasktype, device_ip_reg_map[self.netmasktype], out)) res = [] for ipaddr_mask in reg.finditer(out): ip = IP(ipaddr_mask.group(1), make_net=True) network_netmask = "%s/%s" % (ip.net(), ip.netmask()) res.append(network_netmask) self.log.debug("set_netmask: convert ipaddr_mask %s into network_netmask %s" % (ipaddr_mask.group(1), network_netmask)) self.log.debug("set_netmask: return complete netmask %s" % res) if len(res) > 0: self.netmask = ":".join(res)
def test_noshell_glob(self): ec, output = run('ls test/sandbox/testpkg/*') self.assertTrue(ec > 0) regex = re.compile(r"'?test/sandbox/testpkg/\*'?: No such file or directory") self.assertTrue(regex.search(output), "Pattern '%s' found in: %s" % (regex.pattern, output)) ec, output = run_simple(TEST_GLOB) self.glob_output(output, ec=ec)
def run_cache_create(modules_root): """Run the script to create the Lmod cache""" lmod_dir = os.environ.get("LMOD_DIR", None) if not lmod_dir: raise RuntimeError("Cannot find $LMOD_DIR in the environment.") cmd = "%s/update_lmod_system_cache_files %s" % (lmod_dir, modules_root) return run_simple(cmd)
def test_noshell_glob(self): ec, output = run('ls test/sandbox/testpkg/*') self.assertTrue(ec > 0) regex = re.compile(r"'?test/sandbox/testpkg/\*'?: No such file or directory") self.assertTrue(regex.search(output), "Pattern '%s' found in: %s" % (regex.pattern, output)) ec, output = run_simple(['ls','test/sandbox/testpkg/*']) self.assertEqual(ec, 0) self.assertTrue(all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def worker_run_simple(counter): """Execute the cmd to be called with """ cmd_sanity = ["%s" % x for x in parse_worker_args()] ## ready to join set_scoop_env('counter', counter) ec, out = run_simple(' '.join(cmd_sanity), disable_log=True) return ec, out ## return 1 item
def test_noshell_glob(self): ec, output = run('ls test/sandbox/testpkg/*') self.assertEqual(ec, 127) self.assertTrue( 'test/sandbox/testpkg/*: No such file or directory' in output) ec, output = run_simple(['ls', 'test/sandbox/testpkg/*']) self.assertEqual(ec, 0) self.assertTrue( all(x in output.lower() for x in ['__init__.py', 'testmodule.py', 'testmodulebis.py']))
def test_optcomplete(self): """Test optcomplete support""" reg_reply = re.compile(r'^COMPREPLY=\((.*)\)$') script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'runtests', 'simple_option.py') partial = '-' cmd_list = [script, partial] os.environ['SHELL'] = "bash" pythonpath = 'PYTHONPATH="%s"' % os.pathsep.join([p for p in sys.path if p.startswith(self.setup.REPO_BASE_DIR)]) ec, out = run_simple('%s %s; test $? == 1' % (pythonpath, gen_cmdline(cmd_list, partial, shebang=False))) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0, msg="simple_option.py test script ran success") print out reply_match = reg_reply.search(out) self.assertTrue(reply_match, msg="COMPREPLY %s in output %s" % (reg_reply.pattern, out)) compl_opts = reply_match.group(1).split() basic_opts = ['--debug', '--enable-debug', '--disable-debug', '-d', '--help', '-h', '-H', '--shorthelp', '--configfiles', '--info', ] for opt in basic_opts: self.assertTrue(opt in compl_opts) # test --deb autocompletion partial = '--deb' cmd_list = [script, partial] ec, out = run_simple('%s %s; test $? == 1' % (pythonpath, gen_cmdline(cmd_list, partial, shebang=False))) # tabcompletion ends with exit 1!; test returns this to 0 # avoids run.log.error message self.assertEqual(ec, 0) compl_opts = reg_reply.search(out).group(1).split() self.assertEqual(compl_opts, ['--debug'])
def get_hydra_info(self): """Get a dict with hydra info""" reg_hydra_info = re.compile( r"^\s+(?P<key>\S[^:\n]*)\s*:(?P<value>.*?)\s*$", re.M) cmd = "mpirun -info" ec, out = run_simple(cmd) if ec > 0: self.log.raiseException( "get_hydra_info: failed to run cmd %s: %s" % (cmd, out)) hydra_info = {} for r in reg_hydra_info.finditer(out): key = r.groupdict()['key'] if key is None: self.log.raiseException( "get_hydra_info: failed to get hydra info: missing key in %s (out: %s)" % (r.groupdict(), out)) key = key.strip().lower() value = r.groupdict()['value'] if value is None: self.log.debug( "get_hydra_info: failed to get hydra info: missing value in %s (out: %s)" % (r.groupdict(), out)) value = '' values = [ x.strip().strip('"').strip("'") for x in value.split() if len(x.strip()) > 0 ] hydra_info[key] = values self.log.debug("get_hydra_info: found info %s" % hydra_info) keymap = { "rmk": r'^resource\s+management\s+kernel.*available', "launcher": r'^%s.*available' % self.HYDRA_LAUNCHER_NAME, "chkpt": r'^checkpointing.*available', } self.hydra_info = {} for newkey, regtxt in keymap.items(): reg = re.compile(regtxt, re.I) matches = [v for k, v in hydra_info.items() if reg.search(k)] if len(matches) == 0: continue else: if len(matches) > 1: self.log.warning(( "get_hydra_info: more then one match %s found for newkey %s " "regtxt %s hydrainfo %s") % (matches, newkey, regtxt, hydra_info)) self.hydra_info[newkey] = matches[0] self.log.debug("get_hydra_info: filtered info %s" % self.hydra_info)
def hwlocmap(self): """parse and return output from hwloc-ls Returns a dict that maps the absolute Processor Unit ID to its socket-id and its core-id """ res = {} xmlout = "/tmp/test.xml.%s" % os.getpid() exe = "/usr/bin/hwloc-ls" if not os.path.exists(exe): self.log.error("hwlocmap: Can't find exe %s", exe) cmd = "%s --output-format xml %s" % (exe, xmlout) ec, txt = run_simple(cmd) # parse xmloutput base = etree.parse(xmlout) sks_xpath = './/object[@type="Socket"]' # list of socket ids sks = map(int, [i.attrib['os_index'] for i in base.findall(sks_xpath)]) self.log.debug("sockets: %s", sks) aPU = 0 for x, sk in enumerate(sks): cr_xpath = '%s[@os_index="%s"]//object[@type="Core"]' % (sks_xpath, x) # list of core ids in socket x crs = map(int, [j.attrib['os_index'] for j in base.findall(cr_xpath)]) self.log.debug("cores: %s", crs) for y, cr in enumerate(crs): pu_xpath = '%s[@os_index="%s"]//object[@type="PU"]' % ( cr_xpath, y) # list of PU ids in core y from socket x pus = map( int, [k.attrib['os_index'] for k in base.findall(pu_xpath)]) self.log.debug("PU's: %s", pus) # absolute PU id = (socket id * cores per socket * PU's in core) + PU id # in case of errors, revert back to this # aPU = sks[x] * len(crs) * len(pus) + pus[z] for pu in pus: t = "socket %s core %s abscore %s" % (sk, cr, aPU) res[aPU] = t aPU += 1 self.log.debug("result map: %s", res) return res
def test_run_subshell(self): """Read data from testjobs_submitfilter and feed it through submitfilter script""" testdir = os.path.join(os.path.dirname(__file__), 'testjobs_submitfilter') for scriptfn in glob.glob("%s/*.script" % testdir): scriptname = os.path.basename(scriptfn) name = '.'.join(scriptname.split('.')[:-1]) script = os.path.join(testdir, scriptname) out = os.path.join(testdir, "%s.out" % name) err = os.path.join(testdir, "%s.err" % name) log = os.path.join(testdir, "%s.log" % name) cmdline = os.path.join(testdir, "%s.cmdline" % name) # avoid pyc files in e.g. bin cmd = 'PYTHONPATH=%s:$PYTHONPATH ' % os.pathsep.join([p for p in sys.path if p.startswith(REPO_BASE_DIR)]) cmd += "python -B %s" % submitfilter.__file__ if os.path.exists(cmdline): cmd += " " + open(cmdline).readline().strip() # make command # get output, and exitcode ec, output = run_simple(cmd, input=open(script).read()) self.assertEqual(ec, 0, msg="submitfiler ended with ec 0 for script %s and cmdline %s" % (name, cmd)) res = '' if os.path.exists(log): # multiline pattern match, line per line for pattern in open(log).readlines(): if not pattern or pattern.startswith('#'): continue reg = re.compile(r''+pattern, re.M) if reg.search(output): output = reg.sub('', output) else: self.assertTrue(False, "Expected a log pattern match %s for script %s" % (pattern, name)) if os.path.exists(out): res += open(out).read() else: self.assertTrue(False, msg='no output file %s for script %s' % (out, name)) if os.path.exists(err): res += open(err).read() self.assertEqual(output, res, msg="expected output for script %s and cmdline %s" % (name, cmd))
def test_run_subshell(self): """Read data from testjobs_submitfilter and feed it through submitfilter script""" testdir = os.path.join(os.path.dirname(__file__), "testjobs_submitfilter") for scriptfn in glob.glob("%s/*.script" % testdir): scriptname = os.path.basename(scriptfn) name = ".".join(scriptname.split(".")[:-1]) script = os.path.join(testdir, scriptname) out = os.path.join(testdir, "%s.out" % name) err = os.path.join(testdir, "%s.err" % name) log = os.path.join(testdir, "%s.log" % name) cmdline = os.path.join(testdir, "%s.cmdline" % name) # avoid pyc files in e.g. bin cmd = "PYTHONPATH=%s:$PYTHONPATH " % os.pathsep.join([p for p in sys.path if p.startswith(REPO_BASE_DIR)]) cmd += "python -B %s" % submitfilter.__file__ if os.path.exists(cmdline): cmd += " " + open(cmdline).readline().strip() # make command # get output, and exitcode ec, output = run_simple(cmd, input=open(script).read()) self.assertEqual(ec, 0, msg="submitfiler ended with ec 0 for script %s and cmdline %s" % (name, cmd)) res = "" if os.path.exists(log): # multiline pattern match, line per line for pattern in open(log).readlines(): if not pattern or pattern.startswith("#"): continue reg = re.compile(r"" + pattern, re.M) if reg.search(output): output = reg.sub("", output) else: self.assertTrue(False, "Expected a log pattern match %s for script %s" % (pattern, name)) if os.path.exists(out): res += open(out).read() else: self.assertTrue(False, msg="no output file %s for script %s" % (out, name)) if os.path.exists(err): res += open(err).read() self.assertEqual(output, res, msg="expected output for script %s and cmdline %s" % (name, cmd))
def hwlocmap(self): """parse and return output from hwloc-ls Returns a dict that maps the absolute Processor Unit ID to its socket-id and its core-id """ res = {} xmlout = "/tmp/test.xml.%s" % os.getpid() exe = "/usr/bin/hwloc-ls" if not os.path.exists(exe): self.log.error("hwlocmap: Can't find exe %s", exe) cmd = "%s --output-format xml %s" % (exe, xmlout) ec, txt = run_simple(cmd) # parse xmloutput base = etree.parse(xmlout) sks_xpath = './/object[@type="Socket"]' # list of socket ids sks = map(int, [i.attrib['os_index'] for i in base.findall(sks_xpath)]) self.log.debug("sockets: %s", sks) aPU = 0 for x, sk in enumerate(sks): cr_xpath = '%s[@os_index="%s"]//object[@type="Core"]' % (sks_xpath, x) # list of core ids in socket x crs = map(int, [j.attrib['os_index'] for j in base.findall(cr_xpath)]) self.log.debug("cores: %s", crs) for y, cr in enumerate(crs): pu_xpath = '%s[@os_index="%s"]//object[@type="PU"]' % (cr_xpath, y) # list of PU ids in core y from socket x pus = map(int, [k.attrib['os_index'] for k in base.findall(pu_xpath)]) self.log.debug("PU's: %s", pus) # absolute PU id = (socket id * cores per socket * PU's in core) + PU id # in case of errors, revert back to this # aPU = sks[x] * len(crs) * len(pus) + pus[z] for pu in pus: t = "socket %s core %s abscore %s" % (sk, cr, aPU) res[aPU] = t aPU += 1 self.log.debug("result map: %s", res) return res
def get_hydra_info(self): """Get a dict with hydra info.""" reg_hydra_info = re.compile(r"^\s+(?P<key>\S[^:\n]*)\s*:(?P<value>.*?)\s*$", re.M) cmd = "mpirun -info" exitcode, out = run_simple(cmd) if exitcode > 0: self.log.raiseException("get_hydra_info: failed to run cmd %s: %s" % (cmd, out)) hydra_info = {} for regex in reg_hydra_info.finditer(out): key = regex.groupdict()['key'] if key is None: self.log.raiseException("get_hydra_info: failed to get hydra info: missing key in %s (out: %s)" % (regex.groupdict(), out)) key = key.strip().lower() value = regex.groupdict()['value'] if value is None: self.log.debug("get_hydra_info: failed to get hydra info: missing value in %s (out: %s)" % (regex.groupdict(), out)) value = '' values = [x.strip().strip('"').strip("'") for x in value.split() if x.strip()] hydra_info[key] = values self.log.debug("get_hydra_info: found info %s", hydra_info) keymap = { "rmk": r'^resource\s+management\s+kernel.*available', "launcher": r'^%s.*available' % self.HYDRA_LAUNCHER_NAME, "chkpt": r'^checkpointing.*available', } self.hydra_info = {} for newkey, regtxt in keymap.items(): reg = re.compile(regtxt, re.I) matches = [v for k, v in hydra_info.items() if reg.search(k)] if len(matches) == 0: continue else: if len(matches) > 1: self.log.warning("get_hydra_info: more than one match %s found: newkey %s regtxt %s hydrainfo %s", matches, newkey, regtxt, hydra_info) self.hydra_info[newkey] = matches[0] self.log.debug("get_hydra_info: filtered info %s", self.hydra_info)
def test_startpath(self): cwd = os.getcwd() cmd = "echo foo > bar" self.assertErrorRegex(ValueError, '.*', run_simple, cmd, startpath='/no/such/directory') ec, out = run_simple(cmd, startpath=self.tempdir) # successfull command, no output self.assertEqual(ec, 0) self.assertEqual(out, '') # command was actually executed, in specified directory self.assertTrue(os.path.exists(os.path.join(self.tempdir, 'bar'))) txt = open(os.path.join(self.tempdir, 'bar')).read() self.assertEqual(txt, 'foo\n') # we should still be in directory we were in originally self.assertEqual(cwd, os.getcwd())
def test_startpath(self): cwd = os.getcwd() cmd = "echo foo > bar" self.assertErrorRegex(Exception, '.*', run_simple, cmd, startpath='/no/such/directory') ec, out = run_simple(cmd, startpath=self.tempdir) # successfull command, no output self.assertEqual(ec, 0) self.assertEqual(out, '') # command was actually executed, in specified directory self.assertTrue(os.path.exists(os.path.join(self.tempdir, 'bar'))) txt = open(os.path.join(self.tempdir, 'bar')).read() self.assertEqual(txt, 'foo\n') # we should still be in directory we were in originally self.assertEqual(cwd, os.getcwd())
def set_netmask(self): """ Set self.netmask to a list containing (ip address/netmask). Based on the hosts IP address (from ip addr show) and the selected netmasktype from select_device. """ if self.netmasktype is None: self.select_device() device_ip_reg_map = { 'eth': r"ether.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)", 'ib': r"infiniband.*?\n.*?inet\s+(\d+\.\d+.\d+.\d+/\d+)", } if self.netmasktype not in device_ip_reg_map: self.log.raiseException("set_netmask: can't get netmask for %s: unknown mode (device_ip_reg_map %s)" % (self.netmasktype, device_ip_reg_map)) cmd = "/sbin/ip addr show" exitcode, out = run_simple(cmd) if exitcode > 0: self.log.raiseException("set_netmask: failed to run cmd %s, ec: %s" % (cmd, exitcode)) reg = re.compile(device_ip_reg_map[self.netmasktype]) if not reg.search(out): self.log.raiseException("set_netmask: can't get netmask for %s: no matches found (reg %s out %s)" % (self.netmasktype, device_ip_reg_map[self.netmasktype], out)) res = [] for ipaddr_mask in reg.finditer(out): ip_info = IP(ipaddr_mask.group(1), make_net=True) network_netmask = "%s/%s" % (ip_info.net(), ip_info.netmask()) res.append(network_netmask) self.log.debug("set_netmask: convert ipaddr_mask %s into network_netmask %s", ipaddr_mask.group(1), network_netmask) self.log.debug("set_netmask: return complete netmask %s", res) if res: self.netmask = os.pathsep.join(res)
def get_lmod_config(): """Get the modules root and cache path from the Lmod config""" lmod_cmd = os.environ.get("LMOD_CMD", None) if not lmod_cmd: raise RuntimeError("Cannot find $LMOD_CMD in the environment.") ec, out = run_simple("%s bash --config-json" % lmod_cmd) if ec != 0: raise RuntimeError("Failed to get Lmod configuration: %s", out) try: lmodconfig = json.loads(out) config = { 'modules_root': lmodconfig['configT']['mpath_root'], 'cache_dir': lmodconfig['cache'][0][0], 'cache_timestamp': lmodconfig['cache'][0][1], } logger.debug("Found Lmod config: %s", config) except (ValueError, KeyError, IndexError, TypeError) as err: raise RuntimeError("Failed to parse the Lmod configuration: %s", err) return config
def test_simple(self): ec, output = run_simple([sys.executable, SCRIPT_SIMPLE, 'shortsleep']) self.assertEqual(ec, 0) self.assertTrue('shortsleep' in output.lower())
def test_simple(self): ec, output = run_simple([SCRIPT_SIMPLE, "shortsleep"]) self.assertEqual(ec, 0) self.assertTrue("shortsleep" in output.lower())
def test_simple_glob(self): ec, output = run_simple(" ".join(TEST_GLOB)) self.glob_output(output, ec=ec) ec, output = run_simple(TEST_GLOB) self.glob_output(output, ec=ec)