def _which(cmd):
exe = distutils.spawn.find_executable(cmd)
if exe is None:
return None
exe = util.decode(os.path.realpath(exe))
return exe
def _parse_mpich_version(mpiexec):
out = util.decode(check_output([mpiexec, '--version']))
# Split the --version output into lines.
lines = out.split("\n")[1:]
lines = [ln.strip() for ln in lines]
d = {}
for ln in lines:
if len(ln.strip()) == 0:
continue
m = re.match(r'^([^:]*):\s*(\S.*)?$', ln)
if m is None:
print("Internal warning: m is None!!! ln='{}'".format(ln))
else:
d[m.group(1)] = m.group(2)
conf = re.findall(r'\'[^\']+\'', d['Configure options'])
conf = [re.sub(r'\'$', '', re.sub(r'^\'', '', c)) for c in conf]
d['Configure options'] = {}
for c in conf:
m = c.split('=')
if len(m) == 1:
m[1:] = [True]
d['Configure options'][m[0]] = m[1]
return d
def find_mpi_h(mpiexec, ver_str=None):
"""Find mpi.h file from MPICH mpiexec binary"""
if ver_str is None:
p = Popen([mpiexec, '--version'], stderr=PIPE, stdout=PIPE)
out, err = p.communicate()
ver_str = util.decode(out + err)
# Search prefix from configure options
line = next(ln for ln in ver_str.split("\n")
if re.search(r'Configure options', ln))
dir_cands = re.findall(r'--includedir=([^\' \n]+)', line)
inc_paths = re.findall(r'--includedir=([^\' \n]+)', line)
prefixes = [
d for d in re.findall(r'--prefix=([^\' \n]+)', line)
if os.path.isdir(d)
]
# Search prefix from the binary's path
m = re.match(r'^(.*)/bin/[^/]+$', mpiexec)
if m is not None:
incdir = m.group(1)
if os.path.isdir(incdir):
prefixes += [incdir]
dir_cands = set(inc_paths + [os.path.join(d, 'include') for d in prefixes])
try:
inc_dir = next(p for p in dir_cands
if os.path.exists(os.path.join(p, 'mpi.h')))
except StopIteration:
raise FileNotFoundError("mpi.h not found in {}".format(
",".join(dir_cands)))
return os.path.join(inc_dir, 'mpi.h')
def _call_ompi_info(bin):
if not os.path.exists(bin):
raise RuntimeError("ompi_info does not exist: {}".format(bin))
out = check_output([bin, '--all', '--parsable'], stderr=util.DEVNULL)
out = util.decode(out)
return parse_ompi_info(out)
def _parse_mpich_mpicc_show(mpicc):
"""Obtain inc_dir and lib_dir by parsing `mpicc -show` of MPICH"""
out = util.decode(check_output([mpicc, '-show']))
# returns inc_dir, lib_dir
m = re.search(r'-I(\S+)', out)
inc_dir = m.group(1)
m = re.search(r'-L(\S+)', out)
lib_dir = m.group(1)
return inc_dir, lib_dir
def get_mpi_class(mpienv, mpiexec):
"""Return the class of the MPI"""
if not os.path.exists(mpiexec):
# prefix directory does exist but prefix/bin/mpiexec
# does not. --> It seems that the MPI has been
# uninstalled after registered to mpienv?
sys.stderr.write("'{}' seems to be broken because "
"there's no such file or directory\n".format(mpiexec))
return BrokenMPI
if _is_broken_symlink(mpiexec):
return BrokenMPI
# Add LD_LIBRARY_PATH
bin_dir = os.path.dirname(mpiexec)
lib_dir = os.path.abspath(os.path.join(bin_dir, os.pardir, 'lib'))
env = os.environ.copy()
ld_lib_path = [lib_dir] + env.get('LD_LIBRARY_PATH', '').split(':')
env['LD_LIBRARY_PATH'] = ':'.join(ld_lib_path)
p = Popen([mpiexec, '--version'], stderr=PIPE, stdout=PIPE, env=env)
out, err = p.communicate()
ver_str = util.decode(out + err)
if re.search(r'OpenRTE', ver_str, re.MULTILINE):
# Open MPI
return openmpi.OpenMPI
if re.search(r'HYDRA', ver_str, re.MULTILINE):
# MPICH or MVAPICH
# if mpi.h is installed, check it to identiy
# the MPI type.
# This is because MVAPCIH uses MPICH's mpiexec,
# so we cannot distinguish them only from mpiexec.
mpi_h = mpich.find_mpi_h(mpiexec, ver_str)
ret = call(['grep', 'MVAPICH2_VERSION', '-q', mpi_h], stderr=DEVNULL)
if ret == 0:
# MVAPICH
return mvapich.Mvapich
else:
# MPICH
# on some platform, sometimes only runtime
# is installed and developemnt kit (i.e. compilers)
# are not installed.
# In this case, we assume it's mpich.
return mpich.Mpich
# Failed to detect MPI
sys.stderr.write("ver_str = {}\n".format(ver_str))
raise RuntimeError("Unknown MPI type '{}'".format(mpiexec))
def __init__(self, *args):
super(Mvapich, self).__init__(*args)
self._type = 'MVAPICH'
mpi_h = mpich.find_mpi_h(self.mpiexec)
if not os.path.exists(mpi_h):
raise RuntimeError("Error: Cannot find {}".format(mpi_h))
mv_ver = check_output(
['grep', '-E', 'define *MVAPICH2_VERSION', mpi_h],
stderr=util.DEVNULL)
mch_ver = check_output(['grep', '-E', 'define *MPICH_VERSION', mpi_h],
stderr=util.DEVNULL)
mv_ver = util.decode(mv_ver)
mch_ver = util.decode(mch_ver)
mv_ver = re.search(r'"([.0-9]+(a\d*|b\d*|rc\d*)?)"', mv_ver).group(1)
mch_ver = re.search(r'"([.0-9]+)"', mch_ver).group(1)
self._version = mv_ver
self._mpich_ver = mch_ver
self._default_name = "mvapich2-{}".format(mv_ver)
def _get_pip_ver():
global _pip_ver
p = Popen(['pip', '--version'], stdout=PIPE)
out, err = p.communicate()
m = re.match(r'pip (\S+)', util.decode(out))
ver = m.group(1)
m = re.match(r'(\d+)[.](\S+)', ver)
major_ver = int(m.group(1))
if major_ver >= 9:
_pip_ver = str(major_ver)
elif ver.startswith("1.5"):
_pip_ver = '1.5'
else:
raise RuntimeError("Error: Unsupported pip version")
def _call_ompi_info(bin):
out = check_output([bin, '--all', '--parsable'], stderr=util.DEVNULL)
out = util.decode(out)
return parse_ompi_info(out)