def install_step(self):
"""
Execute the all.bash script to build and install the Go compiler,
specifying the final installation prefix by setting $GOROOT_FINAL.
"""
srcdir = os.path.join(self.cfg['start_dir'], 'src')
try:
os.chdir(srcdir)
except OSError as err:
raise EasyBuildError("Failed to move to %s: %s", srcdir, err)
# $GOROOT_FINAL only specifies the location of the final installation, which gets baked into the binaries
# the installation itself is *not* done by the all.bash script, that needs to be done manually
# $GOROOT_BOOTSTRAP needs to specify a Go installation directory to build the go toolchain for versions
# 1.5 and later.
if LooseVersion(self.version) >= LooseVersion('1.5'):
go_root = get_software_root('Go')
if go_root:
cmd = "GOROOT_BOOTSTRAP=%s GOROOT_FINAL=%s ./all.bash" % (go_root, self.installdir)
else:
raise EasyBuildError("Go is required as a build dependency for installing Go since version 1.5")
else:
cmd = "GOROOT_FINAL=%s ./all.bash" % self.installdir
run_cmd(cmd, log_all=True, simple=False)
try:
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks'])
except OSError as err:
raise EasyBuildError("Failed to copy installation to %s: %s", self.installdir, err)
def extract_step(self):
"""
Prepare a combined MXNet source tree. Move all submodules
to their right place.
"""
# Extract everything into separate directories.
super(EB_MXNet, self).extract_step()
mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*'))
if len(mxnet_dirs) == 1:
self.mxnet_src_dir = mxnet_dirs[0]
self.log.debug("MXNet dir is: %s", self.mxnet_src_dir)
else:
raise EasyBuildError("Failed to find/isolate MXNet source directory: %s", mxnet_dirs)
for srcdir in [d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir)]:
submodule, _, _ = srcdir.rpartition('-')
newdir = os.path.join(self.mxnet_src_dir, submodule)
olddir = os.path.join(self.builddir, srcdir)
# first remove empty existing directory
rmtree2(newdir)
try:
shutil.move(olddir, newdir)
except IOError as err:
raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err)
# the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place.
newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core")
olddir = os.path.join(self.mxnet_src_dir, "dmlc-core")
rmtree2(newdir)
symlink(olddir, newdir)
def extract_step(self):
"""
Prepare a combined MXNet source tree. Move all submodules
to their right place.
"""
# Extract everything into separate directories.
super(EB_MXNet, self).extract_step()
mxnet_dirs = glob.glob(os.path.join(self.builddir, 'mxnet-*'))
if len(mxnet_dirs) == 1:
self.mxnet_src_dir = mxnet_dirs[0]
self.log.debug("MXNet dir is: %s", self.mxnet_src_dir)
else:
raise EasyBuildError("Failed to find/isolate MXNet source directory: %s", mxnet_dirs)
for srcdir in [d for d in os.listdir(self.builddir) if not d.startswith('mxnet-')]:
submodule, _, _ = srcdir.rpartition('-')
newdir = os.path.join(self.mxnet_src_dir, submodule)
olddir = os.path.join(self.builddir, srcdir)
# first remove empty existing directory
rmtree2(newdir)
try:
shutil.move(olddir, newdir)
except IOError, err:
raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err)
def install_step(self):
"""
Execute the all.bash script to build and install the Go compiler,
specifying the final installation prefix by setting $GOROOT_FINAL.
"""
srcdir = os.path.join(self.cfg['start_dir'], 'src')
try:
os.chdir(srcdir)
except OSError as err:
raise EasyBuildError("Failed to move to %s: %s", srcdir, err)
# $GOROOT_FINAL only specifies the location of the final installation, which gets baked into the binaries
# the installation itself is *not* done by the all.bash script, that needs to be done manually
# $GOROOT_BOOTSTRAP needs to specify a Go installation directory to build the go toolchain for versions
# 1.5 and later.
if LooseVersion(self.version) >= LooseVersion('1.5'):
go_root = get_software_root('Go')
if go_root:
cmd = "GOROOT_BOOTSTRAP=%s GOROOT_FINAL=%s ./all.bash" % (go_root, self.installdir)
else:
raise EasyBuildError("Go is required as a build dependency for installing Go since version 1.5")
else:
cmd = "GOROOT_FINAL=%s ./all.bash" % self.installdir
run_cmd(cmd, log_all=True, simple=False)
try:
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks'])
except OSError as err:
raise EasyBuildError("Failed to copy installation to %s: %s", self.installdir, err)
def test_step(self):
"""Test the built Python package."""
if isinstance(self.cfg['runtest'], basestring):
self.testcmd = self.cfg['runtest']
if self.cfg['runtest'] and not self.testcmd is None:
extrapath = ""
testinstalldir = None
if self.testinstall:
# install in test directory and export PYTHONPATH
try:
testinstalldir = tempfile.mkdtemp()
mkdir(os.path.join(testinstalldir, self.pylibdir), parents=True)
except OSError, err:
self.log.error("Failed to create test install dir: %s" % err)
tup = (self.cfg['preinstallopts'], testinstalldir, self.cfg['installopts'])
cmd = "%s python setup.py install --prefix=%s %s" % tup
run_cmd(cmd, log_all=True, simple=True)
run_cmd("python -c 'import sys; print(sys.path)'") # print Python search path (debug)
extrapath = "export PYTHONPATH=%s:$PYTHONPATH && " % os.path.join(testinstalldir, self.pylibdir)
if self.testcmd:
cmd = "%s%s" % (extrapath, self.testcmd)
run_cmd(cmd, log_all=True, simple=True)
if testinstalldir:
try:
rmtree2(testinstalldir)
except OSError, err:
self.log.exception("Removing testinstalldir %s failed: %s" % (testinstalldir, err))
def test_cases_step(self):
"""Run test cases, if specified."""
for test in self.cfg['tests']:
# check expected format
if not len(test) == 4:
raise EasyBuildError("WIEN2k test case not specified in expected format: "
"(testcase_name, init_lapw_args, run_lapw_args, [scf_regexp_pattern])")
test_name = test[0]
init_args = test[1]
run_args = test[2]
scf_regexp_patterns = test[3]
try:
cwd = os.getcwd()
# WIEN2k enforces that working dir has same name as test case
tmpdir = os.path.join(tempfile.mkdtemp(), test_name)
scratch = os.path.join(tmpdir, 'scratch')
mkdir(scratch, parents=True)
env.setvar('SCRATCH', scratch)
os.chdir(tmpdir)
self.log.info("Running test case %s in %s" % (test_name, tmpdir))
except OSError as err:
raise EasyBuildError("Failed to create temporary directory for test %s: %s", test_name, err)
# try and find struct file for test
test_fp = self.obtain_file("%s.struct" % test_name)
try:
shutil.copy2(test_fp, tmpdir)
except OSError as err:
raise EasyBuildError("Failed to copy %s: %s", test_fp, err)
# run test
cmd = "init_lapw %s" % init_args
run_cmd(cmd, log_all=True, simple=True)
cmd = "run_lapw %s" % run_args
run_cmd(cmd, log_all=True, simple=True)
# check output
scf_fn = "%s.scf" % test_name
self.log.debug("Checking output of test %s in %s" % (str(test), scf_fn))
scftxt = read_file(scf_fn)
for regexp_pat in scf_regexp_patterns:
regexp = re.compile(regexp_pat, re.M)
if not regexp.search(scftxt):
raise EasyBuildError("Failed to find pattern %s in %s", regexp.pattern, scf_fn)
else:
self.log.debug("Found pattern %s in %s" % (regexp.pattern, scf_fn))
# cleanup
try:
os.chdir(cwd)
rmtree2(tmpdir)
except OSError as err:
raise EasyBuildError("Failed to clean up temporary test dir: %s", err)
def extract_step(self):
"""
Prepare a combined MXNet source tree. Move all submodules
to their right place.
"""
# Extract everything into separate directories.
super(EB_MXNet, self).extract_step()
mxnet_dirs = glob.glob(os.path.join(self.builddir, 'mxnet-*'))
if len(mxnet_dirs) == 1:
self.mxnet_src_dir = mxnet_dirs[0]
self.log.debug("MXNet dir is: %s", self.mxnet_src_dir)
else:
raise EasyBuildError(
"Failed to find/isolate MXNet source directory: %s",
mxnet_dirs)
for srcdir in [
d for d in os.listdir(self.builddir)
if not d.startswith('mxnet-')
]:
submodule, _, _ = srcdir.rpartition('-')
newdir = os.path.join(self.mxnet_src_dir, submodule)
olddir = os.path.join(self.builddir, srcdir)
# first remove empty existing directory
rmtree2(newdir)
try:
shutil.move(olddir, newdir)
except IOError, err:
raise EasyBuildError("Failed to move %s to %s: %s", olddir,
newdir, err)
def install_step(self):
"""Install IMOD using install script."""
# -dir: Choose location of installation directory
# -skip: do not attempt to deploy resource files in /etc
# -yes: do not prompt for confirmation
script = '{0}_{1}{2}.csh'.format(self.name.lower(), self.version,
self.cfg['versionsuffix'])
cmd = "tcsh {0} -dir {1} -script {1} -skip -yes".format(
script, self.installdir)
run_cmd(cmd, log_all=True, simple=True)
# The assumption by the install script is that installdir will be something
# like /usr/local. So it creates, within the specified install location, a
# number of additional directories within which to install IMOD. We will,
# therefore, move the contents of these directories up and throw away the
# directories themselves. Doing so apparently is not a problem so long as
# IMOD_DIR is correctly set in the module.
link_to_remove = os.path.join(self.installdir, self.name)
dir_to_remove = os.path.join(
self.installdir, "{0}_{1}".format(self.name.lower(), self.version))
try:
for entry in os.listdir(dir_to_remove):
shutil.move(os.path.join(dir_to_remove, entry),
self.installdir)
if os.path.realpath(link_to_remove) != os.path.realpath(
dir_to_remove):
raise EasyBuildError(
"Something went wrong: %s doesn't point to %s",
link_to_remove, dir_to_remove)
rmtree2(dir_to_remove)
os.remove(link_to_remove)
except OSError as err:
raise EasyBuildError("Failed to clean up install dir: %s", err)
def install_step(self):
"""Install IMOD using install script."""
# -dir: Choose location of installation directory
# -skip: do not attempt to deploy resource files in /etc
# -yes: do not prompt for confirmation
script = '{0}_{1}{2}.csh'.format(self.name.lower(), self.version, self.cfg['versionsuffix'])
cmd = "tcsh {0} -dir {1} -script {1} -skip -yes".format(script, self.installdir)
run_cmd(cmd, log_all=True, simple=True)
# The assumption by the install script is that installdir will be something
# like /usr/local. So it creates, within the specified install location, a
# number of additional directories within which to install IMOD. We will,
# therefore, move the contents of these directories up and throw away the
# directories themselves. Doing so apparently is not a problem so long as
# IMOD_DIR is correctly set in the module.
link_to_remove = os.path.join(self.installdir, self.name)
dir_to_remove = os.path.join(self.installdir, "{0}_{1}".format(self.name.lower(), self.version))
try:
for entry in os.listdir(dir_to_remove):
shutil.move(os.path.join(dir_to_remove, entry), self.installdir)
if os.path.realpath(link_to_remove) != os.path.realpath(dir_to_remove):
raise EasyBuildError("Something went wrong: %s doesn't point to %s", link_to_remove, dir_to_remove)
rmtree2(dir_to_remove)
os.remove(link_to_remove)
except OSError as err:
raise EasyBuildError("Failed to clean up install dir: %s", err)
def test_step(self):
"""Test the compilation
- default: None
"""
extrapath = ""
testinstalldir = os.path.join(self.builddir, "mytemporarytestinstalldir")
if self.testinstall:
# Install in test directory and export PYTHONPATH
try:
os.makedirs(testinstalldir)
except OSError:
self.log.exception("Creating testinstalldir %s failed" % testinstalldir)
ppath = "%s/reallib" % testinstalldir
cmd = "python setup.py install --install-scripts=%s --install-purelib=%s %s" % \
(testinstalldir, ppath, self.installopts)
run_cmd(cmd, log_all=True, simple=True)
if os.environ.has_key('PYTHONPATH'):
extrapath = "export PYTHONPATH=%s:%s && " % (ppath, os.environ['PYTHONPATH'])
else:
extrapath = "export PYTHONPATH=%s && " % ppath
if self.runtest:
cmd = "%s%s" % (extrapath, self.runtest)
run_cmd(cmd, log_all=True, simple=True)
if self.testinstall:
try:
rmtree2(testinstalldir)
except OSError, err:
self.log.exception("Removing testinstalldir %s failed: %s" % (testinstalldir, err))
def test_step(self):
"""Test the built Python package."""
if isinstance(self.cfg['runtest'], basestring):
self.testcmd = self.cfg['runtest']
if self.cfg['runtest'] and not self.testcmd is None:
extrapath = ""
testinstalldir = None
if self.testinstall:
# install in test directory and export PYTHONPATH
try:
testinstalldir = tempfile.mkdtemp()
mkdir(os.path.join(testinstalldir, self.pylibdir), parents=True)
except OSError, err:
self.log.error("Failed to create test install dir: %s" % err)
tup = (self.cfg['preinstallopts'], testinstalldir, self.cfg['installopts'])
cmd = "%s python setup.py install --prefix=%s %s" % tup
run_cmd(cmd, log_all=True, simple=True)
run_cmd("python -c 'import sys; print(sys.path)'") # print Python search path (debug)
extrapath = "export PYTHONPATH=%s:$PYTHONPATH && " % os.path.join(testinstalldir, self.pylibdir)
if self.testcmd:
cmd = "%s%s" % (extrapath, self.testcmd)
run_cmd(cmd, log_all=True, simple=True)
if testinstalldir:
try:
rmtree2(testinstalldir)
except OSError, err:
self.log.exception("Removing testinstalldir %s failed: %s" % (testinstalldir, err))
def install_step(self):
"""Install python but only keep the bits we need"""
super(EB_Tkinter, self).install_step()
tmpdir = tempfile.mkdtemp(dir=self.builddir)
pylibdir = os.path.join(self.installdir, os.path.dirname(det_pylibdir()))
shlib_ext = get_shared_lib_ext()
tkinter_so = os.path.join(pylibdir, 'lib-dynload', '_tkinter*.' + shlib_ext)
tkinter_so_hits = glob.glob(tkinter_so)
if len(tkinter_so_hits) != 1:
raise EasyBuildError("Expected to find exactly one _tkinter*.so: %s", tkinter_so_hits)
self.tkinter_so_basename = os.path.basename(tkinter_so_hits[0])
if LooseVersion(self.version) >= LooseVersion('3'):
tkparts = ["tkinter", os.path.join("lib-dynload", self.tkinter_so_basename)]
else:
tkparts = ["lib-tk", os.path.join("lib-dynload", self.tkinter_so_basename)]
copy([os.path.join(pylibdir, x) for x in tkparts], tmpdir)
rmtree2(self.installdir)
mkdir(pylibdir, parents=True)
try:
shutil.move(os.path.join(tmpdir, tkparts[0]), pylibdir)
shutil.move(os.path.join(tmpdir, os.path.basename(tkparts[1])), pylibdir)
except (IOError, OSError) as err:
raise EasyBuildError("Failed to move Tkinter back to the install directory: %s", err)
def cleanup(self):
"""
Clean up SVN working copy.
"""
try:
rmtree2(self.wc)
except OSError, err:
raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def cleanup(self):
"""
Clean up SVN working copy.
"""
try:
rmtree2(self.wc)
except OSError, err:
self.log.exception("Can't remove working copy %s: %s" % (self.wc, err))
def cleanup(self):
"""
Clean up mercurial working copy.
"""
try:
rmtree2(self.wc)
except IOError, err:
raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def cleanup(self):
"""
Clean up SVN working copy.
"""
try:
rmtree2(self.wc)
except OSError as err:
raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def cleanup(self):
"""
Clean up SVN working copy.
"""
try:
rmtree2(self.wc)
except OSError, err:
self.log.exception("Can't remove working copy %s: %s" % (self.wc, err))
def clean_homedir(self):
"""Remove 'intel' directory from home directory, where stuff is cached."""
intelhome = os.path.join(os.getenv('HOME'), self.home_subdir)
if os.path.exists(intelhome):
try:
rmtree2(intelhome)
self.log.info("Cleaning up intel dir %s" % intelhome)
except OSError, err:
self.log.warning("Cleaning up intel dir %s failed: %s" % (intelhome, err))
def obtain_repo(self, url, force_download=False):
"""
Downloads the repo from the given url, then updates submodules
:param url: url of git repository
"""
srcpaths = source_paths()
if re.match(r"^(https?)://", url):
# URL detected, try and download it
reponame = url.split('/')[-1].split('.')[0]
# Try and find the repo in the sources directory first
foundfile = None
failedpaths = []
for path in srcpaths:
# create list of candidate filepaths
namepath = os.path.join(path, self.name)
letterpath = os.path.join(path, letter_dir_for(self.name),
self.name)
candidate_filepaths = [letterpath, namepath, path]
# look for repo in those locations
for cfp in candidate_filepaths:
fullpath = os.path.join(cfp, reponame)
if os.path.isdir(fullpath):
self.log.info("Found repo %s at %s", reponame,
fullpath)
foundfile = os.path.abspath(fullpath)
if force_download:
print_warning(
"Found file %s at %s, but re-downloading it anyway..."
% (reponame, foundfile))
foundfile = None
break
else:
failedpaths.append(fullpath)
if foundfile:
if self.dry_run:
self.dry_run_msg(" * %s found at %s", filename, foundfile)
return foundfile
else:
# Figure out where to clone the repo to
repopath = os.path.join(srcpaths[0], letter_dir_for(self.name),
self.name)
self.log.info("Creating path %s to clone repo to" % repopath)
mkdir(repopath, parents=True)
# Remove repo, if it has previously been downloaded (we must be forcing download)
fullpath = os.path.join(repopath, reponame)
if os.path.exists(fullpath):
rmtree2(fullpath)
try:
if self.download_repo(reponame, url, repopath):
return fullpath
except IOError, err:
raise EasyBuildError(
"Downloading file %s from url %s to %s failed: %s",
filename, url, fullpath, err)
def cleanup(self):
"""
Clean up git working copy.
"""
try:
self.wc = os.path.dirname(self.wc)
rmtree2(self.wc)
except IOError, err:
self.log.exception("Can't remove working copy %s: %s" % (self.wc, err))
def cleanup(self):
"""
Clean up git working copy.
"""
try:
self.wc = os.path.dirname(self.wc)
rmtree2(self.wc)
except IOError, err:
raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def cleanup(self):
"""
Clean up git working copy.
"""
try:
self.wc = os.path.dirname(self.wc)
rmtree2(self.wc)
except IOError, err:
raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def install_step(self):
"""Install numpy and remove numpy build dir, so scipy doesn't find it by accident."""
super(EB_numpy, self).install_step()
builddir = os.path.join(self.builddir, "numpy")
if os.path.isdir(builddir):
rmtree2(builddir)
else:
self.log.debug("build dir %s already clean" % builddir)
def cleanup(self):
"""
Clean up mercurial working copy.
"""
try:
rmtree2(self.wc)
except IOError as err:
raise EasyBuildError("Can't remove working copy %s: %s", self.wc,
err)
def install_step(self):
"""Copy all files in build directory to the install directory"""
if self.cfg['install_cmd'] is None:
try:
# shutil.copytree doesn't allow the target directory to exist already
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir)
except OSError, err:
self.log.error("Failed to copy %s to %s: %s" % (self.cfg['start_dir'], self.installdir, err))
def install_step(self):
"""Copy all files in build directory to the install directory"""
if self.cfg['install_cmd'] is None:
try:
# shutil.copytree doesn't allow the target directory to exist already
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks'])
except OSError, err:
raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err)
def install_step(self):
"""Install numpy and remove numpy build dir, so scipy doesn't find it by accident."""
super(EB_numpy, self).install_step()
builddir = os.path.join(self.builddir, "numpy")
if os.path.isdir(builddir):
rmtree2(builddir)
else:
self.log.debug("build dir %s already clean" % builddir)
def install_step(self):
"""Copy all files in build directory to the install directory"""
if self.cfg['install_cmd'] is None:
try:
# shutil.copytree doesn't allow the target directory to exist already
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks'])
except OSError, err:
raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err)
def cleanup(self):
"""
Clean up git working copy.
"""
try:
self.wc = os.path.dirname(self.wc)
rmtree2(self.wc)
except IOError, err:
self.log.exception("Can't remove working copy %s: %s" %
(self.wc, err))
def install_step(self):
"""Copy all files in build directory to the install directory"""
if self.cfg['install_cmd'] is None:
try:
# shutil.copytree doesn't allow the target directory to exist already
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir)
except OSError, err:
self.log.error("Failed to copy %s to %s: %s" %
(self.cfg['start_dir'], self.installdir, err))
def install_step(self):
"""Copy all files in build directory to the install directory"""
rmtree2(self.installdir)
install_script = self.src[0]['name']
adjust_permissions(os.path.join(self.builddir, install_script), stat.S_IRUSR|stat.S_IXUSR)
cmd = "%s ./%s -p %s -b -f" % (self.cfg['preinstallopts'], install_script, self.installdir)
self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)
def install_step(self):
src = self.cfg['start_dir']
# shutil.copytree cannot handle destination dirs that exist already.
# On the other hand, Python2.4 cannot create entire paths during copytree.
# Therefore, only the final directory is deleted.
rmtree2(self.installdir)
try:
# self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file
shutil.copytree(src,self.installdir, symlinks=self.cfg['keepsymlinks'])
except:
self.log.exception("Copying %s to installation dir %s failed" % (src,self.installdir))
def test_step(self):
"""Test the built Python package."""
if isinstance(self.cfg['runtest'], basestring):
self.testcmd = self.cfg['runtest']
if self.cfg['runtest'] and self.testcmd is not None:
extrapath = ""
testinstalldir = None
if self.testinstall:
# install in test directory and export PYTHONPATH
try:
testinstalldir = tempfile.mkdtemp()
for pylibdir in self.all_pylibdirs:
mkdir(os.path.join(testinstalldir, pylibdir),
parents=True)
except OSError, err:
raise EasyBuildError(
"Failed to create test install dir: %s", err)
# print Python search path (just debugging purposes)
run_cmd("%s -c 'import sys; print(sys.path)'" %
self.python_cmd,
verbose=False,
trace=False)
abs_pylibdirs = [
os.path.join(testinstalldir, pylibdir)
for pylibdir in self.all_pylibdirs
]
extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join(
abs_pylibdirs + ['$PYTHONPATH'])
cmd = self.compose_install_command(testinstalldir,
extrapath=extrapath)
run_cmd(cmd, log_all=True, simple=True, verbose=False)
if self.testcmd:
cmd = "%s%s" % (extrapath, self.testcmd % {
'python': self.python_cmd
})
run_cmd(cmd, log_all=True, simple=True)
if testinstalldir:
try:
rmtree2(testinstalldir)
except OSError, err:
raise EasyBuildError(
"Removing testinstalldir %s failed: %s",
testinstalldir, err)
def install_step(self):
"""Copy all files in build directory to the install directory"""
rmtree2(self.installdir)
install_script = self.src[0]['name']
adjust_permissions(os.path.join(self.builddir, install_script),
stat.S_IRUSR | stat.S_IXUSR)
cmd = "%s ./%s -p %s -b -f" % (self.cfg['preinstallopts'],
install_script, self.installdir)
self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)
def post_install_step(self):
"""Copy installation to actual installation directory in case of a staged installation."""
if self.cfg['staged_install']:
staged_installdir = self.installdir
self.installdir = self.actual_installdir
try:
# copytree expects target directory to not exist yet
if os.path.exists(self.installdir):
rmtree2(self.installdir)
shutil.copytree(staged_installdir, self.installdir)
except OSError, err:
raise EasyBuildError("Failed to move staged install from %s to %s: %s",
staged_installdir, self.installdir, err)
def post_install_step(self):
"""Copy installation to actual installation directory in case of a staged installation."""
if self.cfg.get('staged_install', False):
staged_installdir = self.installdir
self.installdir = self.actual_installdir
try:
# copytree expects target directory to not exist yet
if os.path.exists(self.installdir):
rmtree2(self.installdir)
shutil.copytree(staged_installdir, self.installdir)
except OSError, err:
raise EasyBuildError("Failed to move staged install from %s to %s: %s",
staged_installdir, self.installdir, err)
def install_step(self, src=None):
"""Install by copying from specified source directory (or 'start_dir' if not specified)."""
if src is None:
src = self.cfg['start_dir']
# shutil.copytree cannot handle destination dirs that exist already.
# On the other hand, Python2.4 cannot create entire paths during copytree.
# Therefore, only the final directory is deleted.
rmtree2(self.installdir)
try:
# self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file
shutil.copytree(src, self.installdir, symlinks=self.cfg['keepsymlinks'])
except OSError, err:
self.log.error("Copying %s to installation dir %s failed: %s" % (src, self.installdir, err))
def install_step(self):
"""Copy all files in build directory to the install directory"""
install_cmd = self.cfg.get('install_cmd', None)
if install_cmd is None:
try:
# shutil.copytree doesn't allow the target directory to exist already
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks'])
except OSError as err:
raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err)
else:
cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']])
self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)
def install_step(self):
"""Install numpy and remove numpy build dir, so scipy doesn't find it by accident."""
super(EB_numpy, self).install_step()
builddir = os.path.join(self.builddir, "numpy")
try:
if os.path.isdir(builddir):
os.chdir(self.builddir)
rmtree2(builddir)
else:
self.log.debug("build dir %s already clean" % builddir)
except OSError as err:
raise EasyBuildError("Failed to clean up numpy build dir %s: %s", builddir, err)
def build_image(self, dockerfile):
ec = self.easyconfigs[-1]['ec']
module_name = self.mns.det_full_module_name(ec)
tempdir = tempfile.mkdtemp(prefix='easybuild-docker')
container_name = self.img_name or "%s:latest" % module_name.replace('/', '-')
docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.'])
print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd)
run_cmd(docker_cmd, path=tempdir, stream_output=True)
print_msg("Docker image created at %s" % container_name, log=self.log)
rmtree2(tempdir)
def install_step(self):
"""Install numpy and remove numpy build dir, so scipy doesn't find it by accident."""
super(EB_numpy, self).install_step()
builddir = os.path.join(self.builddir, "numpy")
try:
if os.path.isdir(builddir):
os.chdir(self.builddir)
rmtree2(builddir)
else:
self.log.debug("build dir %s already clean" % builddir)
except OSError as err:
raise EasyBuildError("Failed to clean up numpy build dir %s: %s", builddir, err)
def install_step(self, src=None):
"""Install by copying from specified source directory (or 'start_dir' if not specified)."""
if src is None:
src = self.cfg["start_dir"]
# shutil.copytree cannot handle destination dirs that exist already.
# On the other hand, Python2.4 cannot create entire paths during copytree.
# Therefore, only the final directory is deleted.
rmtree2(self.installdir)
try:
# self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file
shutil.copytree(src, self.installdir, symlinks=self.cfg["keepsymlinks"])
except OSError, err:
raise EasyBuildError("Copying %s to installation dir %s failed: %s", src, self.installdir, err)
def install_step(self):
"""Copy all files in build directory to the install directory"""
pre_install_step(self.log, self.cfg['pre_install_cmd'])
rmtree2(self.installdir)
sources = self.cfg['sources']
install_script = sources[0]
cmd = "chmod 777 {} && ./{} -p {} -b -f".format(install_script, install_script, self.installdir )
self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)
post_install_step(self.log, self.installdir, self.cfg['post_install_cmd'])
def install_step(self):
src = self.cfg['start_dir']
# shutil.copytree cannot handle destination dirs that exist already.
# On the other hand, Python2.4 cannot create entire paths during copytree.
# Therefore, only the final directory is deleted.
rmtree2(self.installdir)
try:
# self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file
shutil.copytree(src,
self.installdir,
symlinks=self.cfg['keepsymlinks'])
except:
self.log.exception("Copying %s to installation dir %s failed" %
(src, self.installdir))
class EB_Go(ConfigureMake):
"""
Build Go compiler
"""
def configure_step(self):
"""No dedicated configure step."""
pass
def build_step(self):
"""No dedicated build step, building & installing is done in one go."""
pass
def install_step(self):
"""
Execute the all.bash script to build and install the Go compiler,
specifying the final installation prefix by setting $GOROOT_FINAL.
"""
srcdir = os.path.join(self.cfg['start_dir'], 'src')
try:
os.chdir(srcdir)
except OSError, err:
raise EasyBuildError("Failed to move to %s: %s", srcdir, err)
# $GOROOT_FINAL only specifies the location of the final installation, which gets baked into the binaries
# the installation itself is *not* done by the all.bash script, that needs to be done manually
cmd = "GOROOT_FINAL=%s ./all.bash" % self.installdir
run_cmd(cmd, log_all=True, simple=False)
try:
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks'])
except OSError, err:
raise EasyBuildError("Failed to copy installation to %s: %s", self.installdir, err)
def install_step(self):
"""
Install by unpacking tarball in dist directory,
and copying site-packages dir to installdir.
"""
# locate tarball
tarball = None
shortver = '.'.join(self.version.split('.')[0:2])
fn_pattern = os.path.join(self.cfg['start_dir'], 'dist',
"%s-%s.*.tar.gz" % (self.name, shortver))
matches = glob.glob(fn_pattern)
if not matches:
raise EasyBuildError("No tarball found at %s", fn_pattern)
elif len(matches) > 1:
raise EasyBuildError("Multiple matches found for tarball: %s",
matches)
else:
tarball = matches[0]
self.log.info("Tarball found at %s" % tarball)
# unpack tarball to temporary directory
tmpdir = tempfile.mkdtemp()
srcdir = extract_file(tarball, tmpdir)
if not srcdir:
raise EasyBuildError("Unpacking tarball %s failed?", tarball)
# locate site-packages dir to copy by diving into unpacked tarball
src = srcdir
while len(os.listdir(src)) == 1:
src = os.path.join(src, os.listdir(src)[0])
if not os.path.basename(src) == 'site-packages':
raise EasyBuildError(
"Expected to find a site-packages path, but found something else: %s",
src)
# copy contents of site-packages dir
dest = os.path.join(self.installdir, 'site-packages')
try:
shutil.copytree(src, dest)
rmtree2(tmpdir)
os.chdir(self.installdir)
except OSError, err:
raise EasyBuildError("Failed to copy directory %s to %s: %s", src,
dest, err)
def install_step(self):
"""Custom install step for netpbm."""
# Preinstallation to a tmp directory. Can't install directly to installdir because the make command fails if the
# directory already exists
cmd = "make package pkgdir=%s/pkg" % self.builddir
(out, _) = run_cmd(cmd, log_all=True, simple=False)
# Move things to installdir
copy(["%s/pkg/%s" % (self.builddir, x) for x in os.listdir("%s/pkg/" % self.builddir)], self.installdir)
# Need to do manually the last bits of the installation
configs = [
("%s/config_template" % self.installdir, "%s/bin/netpbm-config" % self.installdir),
("%s/pkgconfig_template" % self.installdir, "%s/lib/pkgconfig/netpbm.pc" % self.installdir)
]
mkdir("%s/lib/pkgconfig" % self.installdir)
for template, config_file in configs:
move_file(template, config_file)
for line in fileinput.input(config_file, inplace=1, backup='.orig'):
if re.match(r"^@", line):
continue
else:
line = re.sub(r'@VERSION@', 'Netpbm %s' % self.version, line)
line = re.sub(r'@DATADIR@', '%s/lib' % self.installdir, line)
line = re.sub(r'@LINKDIR@', '%s/lib' % self.installdir, line)
line = re.sub(r'@INCLUDEDIR@', '%s/include' % self.installdir, line)
line = re.sub(r'@BINDIR@', '%s/bin' % self.installdir, line)
sys.stdout.write(line)
adjust_permissions("%s/bin/netpbm-config" % self.installdir, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
move_file("%s/link/libnetpbm.a" % self.installdir, "%s/lib/libnetpbm.a" % self.installdir)
symlink("%s/lib/libnetpbm.so.11" % self.installdir, "%s/lib/libnetpbm.so" % self.installdir)
for f in os.listdir("%s/misc/" % self.installdir):
move_file("%s/misc/%s" % (self.installdir, f), "%s/lib/%s" % (self.installdir, f))
rmtree2("%s/misc/" % self.installdir)
rmtree2("%s/link/" % self.installdir)
headers = os.listdir("%s/include/netpbm" % self.installdir)
for header in headers:
symlink("%s/include/netpbm/%s" % (self.installdir, header), "%s/include/%s" % (self.installdir, header))
return out
def install_step(self):
"""
Install by unpacking tarball in dist directory,
and copying site-packages dir to installdir.
"""
# locate tarball
tarball = None
shortver = '.'.join(self.version.split('.')[0:2])
fn_pattern = os.path.join(self.cfg['start_dir'],
'dist',
"%s-%s.*.tar.gz" % (self.name, shortver))
matches = glob.glob(fn_pattern)
if not matches:
raise EasyBuildError("No tarball found at %s", fn_pattern)
elif len(matches) > 1:
raise EasyBuildError("Multiple matches found for tarball: %s", matches)
else:
tarball = matches[0]
self.log.info("Tarball found at %s" % tarball)
# unpack tarball to temporary directory
tmpdir = tempfile.mkdtemp()
srcdir = extract_file(tarball, tmpdir)
if not srcdir:
raise EasyBuildError("Unpacking tarball %s failed?", tarball)
# locate site-packages dir to copy by diving into unpacked tarball
src = srcdir
while len(os.listdir(src)) == 1:
src = os.path.join(src, os.listdir(src)[0])
if not os.path.basename(src) =='site-packages':
raise EasyBuildError("Expected to find a site-packages path, but found something else: %s", src)
# copy contents of site-packages dir
dest = os.path.join(self.installdir, 'site-packages')
try:
shutil.copytree(src, dest)
rmtree2(tmpdir)
os.chdir(self.installdir)
except OSError, err:
raise EasyBuildError("Failed to copy directory %s to %s: %s", src, dest, err)
def build_image(self, dockerfile):
ec = self.easyconfigs[-1]['ec']
module_name = self.mns.det_full_module_name(ec)
tempdir = tempfile.mkdtemp(prefix='easybuild-docker')
container_name = self.img_name or "%s:latest" % module_name.replace(
'/', '-')
docker_cmd = ' '.join([
'sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name,
'.'
])
print_msg(
"Running '%s', you may need to enter your 'sudo' password..." %
docker_cmd)
run_cmd(docker_cmd, path=tempdir, stream_output=True)
print_msg("Docker image created at %s" % container_name, log=self.log)
rmtree2(tempdir)
def install_step(self):
"""Install python but only keep the bits we need"""
super(EB_Tkinter, self).install_step()
tmpdir = tempfile.mkdtemp(dir=self.builddir)
pylibdir = os.path.join(self.installdir,
os.path.dirname(det_pylibdir()))
shlib_ext = get_shared_lib_ext()
tkinter_so = os.path.join(pylibdir, 'lib-dynload',
'_tkinter*.' + shlib_ext)
tkinter_so_hits = glob.glob(tkinter_so)
if len(tkinter_so_hits) != 1:
raise EasyBuildError(
"Expected to find exactly one _tkinter*.so: %s",
tkinter_so_hits)
self.tkinter_so_basename = os.path.basename(tkinter_so_hits[0])
if LooseVersion(self.version) >= LooseVersion('3'):
tkparts = [
"tkinter",
os.path.join("lib-dynload", self.tkinter_so_basename)
]
else:
tkparts = [
"lib-tk",
os.path.join("lib-dynload", self.tkinter_so_basename)
]
copy([os.path.join(pylibdir, x) for x in tkparts], tmpdir)
rmtree2(self.installdir)
mkdir(pylibdir, parents=True)
try:
shutil.move(os.path.join(tmpdir, tkparts[0]), pylibdir)
shutil.move(os.path.join(tmpdir, os.path.basename(tkparts[1])),
pylibdir)
except (IOError, OSError) as err:
raise EasyBuildError(
"Failed to move Tkinter back to the install directory: %s",
err)
def install_step(self):
"""Copy all files in build directory to the install directory"""
if self.cfg['install_cmd'] is None:
try:
cmd = self.cfg['preinstallopts']
if cmd:
self.log.info(
"Running preinstallopts of %s using command '%s'..." %
(self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)
self.log.info("Copying %s into '%s'..." %
(self.cfg['start_dir'], self.installdir))
# shutil.copytree doesn't allow the target directory to exist already
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'],
self.installdir,
symlinks=self.cfg['keepsymlinks'])
except OSError, err:
raise EasyBuildError("Failed to copy %s to %s: %s",
self.cfg['start_dir'], self.installdir,
err)
class EB_MXNet(MakeCp):
"""Easyblock to build and install MXNet"""
@staticmethod
def extra_options(extra_vars=None):
"""Change default values of options"""
extra = MakeCp.extra_options()
# files_to_copy is not mandatory here
extra['files_to_copy'][2] = CUSTOM
return extra
def __init__(self, *args, **kwargs):
"""Initialize custom class variables."""
super(EB_MXNet, self).__init__(*args, **kwargs)
self.mxnet_src_dir = None
self.py_ext = PythonPackage(self, {'name': self.name, 'version': self.version})
self.py_ext.module_generator = self.module_generator
self.r_ext = RPackage(self, {'name': self.name, 'version': self.version})
self.r_ext.module_generator = self.module_generator
def extract_step(self):
"""
Prepare a combined MXNet source tree. Move all submodules
to their right place.
"""
# Extract everything into separate directories.
super(EB_MXNet, self).extract_step()
mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*'))
if len(mxnet_dirs) == 1:
self.mxnet_src_dir = mxnet_dirs[0]
self.log.debug("MXNet dir is: %s", self.mxnet_src_dir)
else:
raise EasyBuildError("Failed to find/isolate MXNet source directory: %s", mxnet_dirs)
for srcdir in [d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir)]:
submodule, _, _ = srcdir.rpartition('-')
newdir = os.path.join(self.mxnet_src_dir, submodule)
olddir = os.path.join(self.builddir, srcdir)
# first remove empty existing directory
rmtree2(newdir)
try:
shutil.move(olddir, newdir)
except IOError, err:
raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err)
# the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place.
newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core")
olddir = os.path.join(self.mxnet_src_dir, "dmlc-core")
rmtree2(newdir)
symlink(olddir, newdir)
def install_step(self):
"""Copy all files in build directory to the install directory"""
install_cmd = self.cfg.get('install_cmd', None)
if install_cmd is None:
try:
# shutil.copytree doesn't allow the target directory to exist already
rmtree2(self.installdir)
shutil.copytree(self.cfg['start_dir'],
self.installdir,
symlinks=self.cfg['keepsymlinks'])
except OSError as err:
raise EasyBuildError("Failed to copy %s to %s: %s",
self.cfg['start_dir'], self.installdir,
err)
else:
cmd = ' '.join([
self.cfg['preinstallopts'], install_cmd,
self.cfg['installopts']
])
self.log.info("Installing %s using command '%s'..." %
(self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)
def test_step(self):
"""Test the built Python package."""
if isinstance(self.cfg['runtest'], basestring):
self.testcmd = self.cfg['runtest']
if self.cfg['runtest'] and self.testcmd is not None:
extrapath = ""
testinstalldir = None
if self.testinstall:
# install in test directory and export PYTHONPATH
try:
testinstalldir = tempfile.mkdtemp()
for pylibdir in self.all_pylibdirs:
mkdir(os.path.join(testinstalldir, pylibdir), parents=True)
except OSError, err:
raise EasyBuildError("Failed to create test install dir: %s", err)
# print Python search path (just debugging purposes)
run_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, verbose=False)
abs_pylibdirs = [os.path.join(testinstalldir, pylibdir) for pylibdir in self.all_pylibdirs]
extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join(abs_pylibdirs + ['$PYTHONPATH'])
cmd = self.compose_install_command(testinstalldir, extrapath=extrapath)
run_cmd(cmd, log_all=True, simple=True, verbose=False)
if self.testcmd:
cmd = "%s%s" % (extrapath, self.testcmd % {'python': self.python_cmd})
run_cmd(cmd, log_all=True, simple=True)
if testinstalldir:
try:
rmtree2(testinstalldir)
except OSError, err:
raise EasyBuildError("Removing testinstalldir %s failed: %s", testinstalldir, err)
def install_step(self):
"""Install python but only keep the bits we need"""
super(EB_Tkinter, self).install_step()
tmpdir = tempfile.mkdtemp(dir=self.builddir)
pylibdir = os.path.join(self.installdir,
os.path.dirname(det_pylibdir()))
tkparts = ["lib-tk", "lib-dynload/_tkinter.so"]
copy([os.path.join(pylibdir, x) for x in tkparts], tmpdir)
rmtree2(self.installdir)
mkdir(pylibdir, parents=True)
try:
shutil.move(os.path.join(tmpdir, tkparts[0]), pylibdir)
shutil.move(os.path.join(tmpdir, os.path.basename(tkparts[1])),
pylibdir)
except (IOError, OSError) as err:
raise EasyBuildError(
"Failed to move Tkinter back to the install directory: %s",
err)
class EB_numpy(FortranPythonPackage):
"""Support for installing the numpy Python package as part of a Python installation."""
def __init__(self, *args, **kwargs):
"""Initialize numpy-specific class variables."""
super(EB_numpy, self).__init__(*args, **kwargs)
self.sitecfg = None
self.sitecfgfn = 'site.cfg'
self.installopts = ''
self.testinstall = True
self.testcmd = "cd .. && python -c 'import numpy; numpy.test(verbose=2)'"
def configure_step(self):
"""Configure numpy build by composing site.cfg contents."""
# see e.g. https://github.com/numpy/numpy/pull/2809/files
self.sitecfg = '\n'.join([
"[DEFAULT]",
"library_dirs = %(libs)s",
"include_dirs= %(includes)s",
"search_static_first=True",
])
if get_software_root("imkl"):
extrasiteconfig = '\n'.join([
"[mkl]",
"lapack_libs = %(lapack)s",
"mkl_libs = %(blas)s",
])
else:
# [atlas] the only real alternative, even for non-ATLAS BLAS libs (e.g., OpenBLAS, ACML, ...)
# using only the [blas] and [lapack] sections results in sub-optimal builds that don't provide _dotblas.so;
# it does require a CBLAS interface to be available for the BLAS library being used
# e.g. for ACML, the CBLAS module providing a C interface needs to be used
extrasiteconfig = '\n'.join([
"[atlas]",
"atlas_libs = %(lapack)s",
"[lapack]",
"lapack_libs = %(lapack)s", # required by scipy, that uses numpy's site.cfg
])
blas = None
lapack = None
fft = None
if get_software_root("imkl"):
# with IMKL, no spaces and use '-Wl:'
# redefine 'Wl,' to 'Wl:' so that the patch file can do its job
def get_libs_for_mkl(varname):
"""Get list of libraries as required for MKL patch file."""
libs = self.toolchain.variables['LIB%s' % varname].copy()
libs.try_remove(['pthread'])
tweaks = {
'prefix': '',
'prefix_begin_end': '-Wl:',
'separator': ',',
'separator_begin_end': ',',
}
libs.try_function_on_element('change', kwargs=tweaks)
libs.SEPARATOR = ','
return str(
libs
) # str causes list concatenation and adding prefixes & separators
blas = get_libs_for_mkl('BLAS_MT')
lapack = get_libs_for_mkl('LAPACK_MT')
fft = get_libs_for_mkl('FFT')
# make sure the patch file is there
# we check for a typical characteristic of a patch file that cooperates with the above
# not fool-proof, but better than enforcing a particular patch filename
patch_found = False
patch_wl_regex = re.compile(r"replace\(':',\s*','\)")
for patch in self.patches:
# patches are either strings (extension) or dicts (easyblock)
if isinstance(patch, dict):
patch = patch['path']
if patch_wl_regex.search(open(patch, 'r').read()):
patch_found = True
break
if not patch_found:
self.log.error(
"Building numpy on top of Intel MKL requires a patch to "
"handle -Wl linker flags correctly, which doesn't seem to be there."
)
else:
# unless Intel MKL is used, $ATLAS should be set to take full control,
# and to make sure a fully optimized version is built, including _dotblas.so
# which is critical for decent performance of the numpy.dot (matrix dot product) function!
env.setvar('ATLAS', '1')
lapack = ', '.join([
x
for x in self.toolchain.get_variable('LIBLAPACK_MT', typ=list)
if x != "pthread"
])
fft = ', '.join(self.toolchain.get_variable('LIBFFT', typ=list))
libs = ':'.join(self.toolchain.get_variable('LDFLAGS', typ=list))
includes = ':'.join(self.toolchain.get_variable('CPPFLAGS', typ=list))
# CBLAS is required for ACML, because it doesn't offer a C interface to BLAS
if get_software_root('ACML'):
cblasroot = get_software_root('CBLAS')
if cblasroot:
lapack = ', '.join([lapack, "cblas"])
cblaslib = os.path.join(cblasroot, 'lib')
# with numpy as extension, CBLAS might not be included in LDFLAGS because it's not part of a toolchain
if not cblaslib in libs:
libs = ':'.join([libs, cblaslib])
else:
self.log.error(
"CBLAS is required next to ACML to provide a C interface to BLAS, but it's not loaded."
)
if fft:
extrasiteconfig += "\n[fftw]\nlibraries = %s" % fft
self.sitecfg = '\n'.join([self.sitecfg, extrasiteconfig])
self.sitecfg = self.sitecfg % {
'blas': blas,
'lapack': lapack,
'libs': libs,
'includes': includes,
}
super(EB_numpy, self).configure_step()
# check configuration (for debugging purposes)
cmd = "python setup.py config"
run_cmd(cmd, log_all=True, simple=True)
def test_step(self):
"""Run available numpy unit tests, and more."""
super(EB_numpy, self).test_step()
# temporarily install numpy, it doesn't alow to be used straight from the source dir
tmpdir = tempfile.mkdtemp()
cmd = "python setup.py install --prefix=%s %s" % (tmpdir,
self.installopts)
run_cmd(cmd, log_all=True, simple=True)
try:
pwd = os.getcwd()
os.chdir(tmpdir)
except OSError, err:
self.log.error("Faild to change to %s: %s" % (tmpdir, err))
# evaluate performance of numpy.dot (3 runs, 3 loops each)
size = 1000
max_time_msec = 1000 # 1 second should really do it for a 1000x1000 matrix dot product
cmd = ' '.join([
'export PYTHONPATH=%s:$PYTHONPATH &&' %
os.path.join(tmpdir, self.pylibdir),
'python -m timeit -n 3 -r 3',
'-s "import numpy; x = numpy.random.random((%(size)d, %(size)d))"'
% {
'size': size
},
'"numpy.dot(x, x.T)"',
])
(out, ec) = run_cmd(cmd, simple=False)
self.log.debug("Test output: %s" % out)
# fetch result
time_msec = None
msec_re = re.compile(
"\d+ loops, best of \d+: (?P<time>[0-9.]+) msec per loop")
res = msec_re.search(out)
if res:
time_msec = float(res.group('time'))
else:
sec_re = re.compile(
"\d+ loops, best of \d+: (?P<time>[0-9.]+) sec per loop")
res = sec_re.search(out)
if res:
time_msec = 1000 * float(res.group('time'))
else:
self.log.error(
"Failed to determine time for numpy.dot test run.")
# make sure we observe decent performance
if time_msec < max_time_msec:
self.log.info(
"Time for %(size)dx%(size)d matrix dot product: %(time)d msec < %(maxtime)d msec => OK"
% {
'size': size,
'time': time_msec,
'maxtime': max_time_msec
})
else:
self.log.error(
"Time for %(size)dx%(size)d matrix dot product: %(time)d msec >= %(maxtime)d msec => ERROR"
% {
'size': size,
'time': time_msec,
'maxtime': max_time_msec
})
try:
os.chdir(pwd)
rmtree2(tmpdir)
except OSError, err:
self.log.error("Failed to change back to %s: %s" % (pwd, err))
def test_step(self):
"""Run WPS test (requires large dataset to be downloaded). """
wpsdir = None
def run_wps_cmd(cmdname, mpi_cmd=True):
"""Run a WPS command, and check for success."""
cmd = os.path.join(wpsdir, "%s.exe" % cmdname)
if mpi_cmd:
cmd = self.toolchain.mpi_cmd_for(cmd, 1)
(out, _) = run_cmd(cmd, log_all=True, simple=False)
re_success = re.compile("Successful completion of %s" % cmdname)
if not re_success.search(out):
self.log.error("%s.exe failed (pattern '%s' not found)?" % (cmdname, re_success.pattern))
if self.cfg['runtest']:
if not self.cfg['testdata']:
self.log.error("List of URLs for testdata not provided.")
wpsdir = os.path.join(self.builddir, "WPS")
try:
# create temporary directory
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
# download data
testdata_paths = []
for testdata in self.cfg['testdata']:
path = self.obtain_file(testdata)
if not path:
self.log.error("Downloading file from %s failed?" % testdata)
testdata_paths.append(path)
# unpack data
for path in testdata_paths:
extract_file(path, tmpdir)
# copy namelist.wps file
fn = "namelist.wps"
shutil.copy2(os.path.join(wpsdir, fn), tmpdir)
namelist_file = os.path.join(tmpdir, fn)
# GEOGRID
# setup directories and files
for d in os.listdir(os.path.join(tmpdir, "geog")):
os.symlink(os.path.join(tmpdir, "geog", d), os.path.join(tmpdir, d))
# patch namelist.wps file for geogrib
for line in fileinput.input(namelist_file, inplace=1, backup='.orig.geogrid'):
line = re.sub(r"^(\s*geog_data_path\s*=\s*).*$", r"\1 '%s'" % tmpdir, line)
sys.stdout.write(line)
# GEOGRID.TBL
geogrid_dir = os.path.join(tmpdir, "geogrid")
os.mkdir(geogrid_dir)
os.symlink(os.path.join(wpsdir, "geogrid", "GEOGRID.TBL.ARW"),
os.path.join(geogrid_dir, "GEOGRID.TBL"))
# run geogrid.exe
run_wps_cmd("geogrid")
# UNGRIB
# determine start and end time stamps of grib files
grib_file_prefix = "fnl_"
k = len(grib_file_prefix)
fs = [f for f in sorted(os.listdir('.')) if f.startswith(grib_file_prefix)]
start = "%s:00:00" % fs[0][k:]
end = "%s:00:00" % fs[-1][k:]
# patch namelist.wps file for ungrib
shutil.copy2(os.path.join(wpsdir, "namelist.wps"), tmpdir)
for line in fileinput.input(namelist_file, inplace=1, backup='.orig.ungrib'):
line = re.sub(r"^(\s*start_date\s*=\s*).*$", r"\1 '%s','%s'," % (start, start), line)
line = re.sub(r"^(\s*end_date\s*=\s*).*$", r"\1 '%s','%s'," % (end, end), line)
sys.stdout.write(line)
# copy correct Vtable
shutil.copy2(os.path.join(wpsdir, "ungrib", "Variable_Tables", "Vtable.ARW"),
os.path.join(tmpdir, "Vtable"))
# run link_grib.csh script
cmd = "%s %s*" % (os.path.join(wpsdir, "link_grib.csh"), grib_file_prefix)
run_cmd(cmd, log_all=True, simple=True)
# run ungrib.exe
run_wps_cmd("ungrib", mpi_cmd=False)
# METGRID.TBL
metgrid_dir = os.path.join(tmpdir, "metgrid")
os.mkdir(metgrid_dir)
os.symlink(os.path.join(wpsdir, "metgrid", "METGRID.TBL.ARW"),
os.path.join(metgrid_dir, "METGRID.TBL"))
# run metgrid.exe
run_wps_cmd('metgrid')
# clean up
rmtree2(tmpdir)
os.chdir(self.builddir)
except OSError, err:
self.log.error("Failed to run WPS test: %s" % err)
def test_step(self):
"""Run WIEN2k test benchmarks. """
def run_wien2k_test(cmd_arg):
"""Run a WPS command, and check for success."""
cmd = "x_lapw lapw1 %s" % cmd_arg
(out, _) = run_cmd(cmd, log_all=True, simple=False)
re_success = re.compile("LAPW1\s+END")
if not re_success.search(out):
raise EasyBuildError("Test '%s' in %s failed (pattern '%s' not found)?",
cmd, os.getcwd(), re_success.pattern)
else:
self.log.info("Test '%s' seems to have run successfully: %s" % (cmd, out))
if self.cfg['runtest']:
if not self.cfg['testdata']:
raise EasyBuildError("List of URLs for testdata not provided.")
# prepend $PATH with install directory, define $SCRATCH which is used by the tests
env.setvar('PATH', "%s:%s" % (self.installdir, os.environ['PATH']))
try:
cwd = os.getcwd()
# create temporary directory
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
self.log.info("Running tests in %s" % tmpdir)
scratch = os.path.join(tmpdir, 'scratch')
mkdir(scratch)
env.setvar('SCRATCH', scratch)
# download data
testdata_paths = {}
for testdata in self.cfg['testdata']:
td_path = self.obtain_file(testdata)
if not td_path:
raise EasyBuildError("Downloading file from %s failed?", testdata)
testdata_paths.update({os.path.basename(testdata): td_path})
self.log.debug('testdata_paths: %s' % testdata_paths)
# unpack serial benchmark
serial_test_name = "test_case"
extract_file(testdata_paths['%s.tar.gz' % serial_test_name], tmpdir)
# run serial benchmark
os.chdir(os.path.join(tmpdir, serial_test_name))
run_wien2k_test("-c")
# unpack parallel benchmark (in serial benchmark dir)
parallel_test_name = "mpi-benchmark"
extract_file(testdata_paths['%s.tar.gz' % parallel_test_name], tmpdir)
# run parallel benchmark
os.chdir(os.path.join(tmpdir, serial_test_name))
run_wien2k_test("-p")
os.chdir(cwd)
rmtree2(tmpdir)
except OSError, err:
raise EasyBuildError("Failed to run WIEN2k benchmark tests: %s", err)
self.log.debug("Current dir: %s" % os.getcwd())
def test_step(self):
"""Run DOLFIN demos by means of test."""
if self.cfg['runtest']:
# set cache/error dirs for Instant
tmpdir = tempfile.mkdtemp()
instant_cache_dir = os.path.join(tmpdir, '.instant', 'cache')
instant_error_dir = os.path.join(tmpdir, '.instant', 'error')
try:
os.makedirs(instant_cache_dir)
os.makedirs(instant_error_dir)
except OSError, err:
raise EasyBuildError("Failed to create Instant cache/error dirs: %s", err)
env_vars = [
('INSTANT_CACHE_DIR', instant_cache_dir),
('INSTANT_ERROR_DIR', instant_error_dir),
]
env_var_cmds = ' && '.join(['export %s="%s"' % (var, val) for (var, val) in env_vars])
# test command templates
cmd_template_python = " && ".join([
env_var_cmds,
"cd %(dir)s",
"python demo_%(name)s.py",
"cd -",
])
cpp_cmds = [
env_var_cmds,
"cd %(dir)s",
]
if LooseVersion(self.version) < LooseVersion('1.1'):
cpp_cmds.append("cmake . %s" % self.saved_configopts)
cpp_cmds.extend([
"make VERBOSE=1",
"./demo_%(name)s",
"cd -",
])
cmd_template_cpp = " && ".join(cpp_cmds)
# list based on demos available for DOLFIN v1.0.0
pde_demos = ['biharmonic', 'cahn-hilliard', 'hyperelasticity', 'mixed-poisson',
'navier-stokes', 'poisson', 'stokes-iterative']
if LooseVersion(self.version) < LooseVersion('1.1'):
demos = [os.path.join('demo', 'la', 'eigenvalue')] + [os.path.join('demo', 'pde', x) for x in pde_demos]
else:
# verified with v1.6.0
demos = [os.path.join('demo', 'documented', x) for x in pde_demos]
# construct commands
cmds = [tmpl % {'dir': os.path.join(d, subdir), 'name': os.path.basename(d)}
for d in demos for (tmpl, subdir) in [(cmd_template_cpp, 'cpp')]]
# exclude Python tests for now, because they 'hang' sometimes (unclear why)
# they can be reinstated once run_cmd (or its equivalent) has support for timeouts
# see https://github.com/hpcugent/easybuild-framework/issues/581
#for (tmpl, subdir) in [(cmd_template_python, 'python'), (cmd_template_cpp, 'cpp')]]
# subdomains-poisson has no C++ get_version, only Python
# Python tests excluded, see above
#name = 'subdomains-poisson'
#path = os.path.join('demo', 'pde', name, 'python')
#cmds += [cmd_template_python % {'dir': path, 'name': name}]
# supply empty argument to each command
for cmd in cmds:
run_cmd(cmd, log_all=True)
# clean up temporary dir
try:
rmtree2(tmpdir)
except OSError, err:
raise EasyBuildError("Failed to remove Instant cache/error dirs: %s", err)
