def install_step(self):
"""Custom install procedure for FLUENT."""
cmd = "./INSTALL -noroot -silent -install_dir %s" % self.installdir
run_cmd(cmd, log_all=True, simple=True)
adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self):
"""Install by copying unzipped binaries to 'bin' subdir of installation dir, and fixing permissions."""
bindir = os.path.join(self.installdir, 'bin')
try:
os.makedirs(bindir)
for item in os.listdir(self.cfg['start_dir']):
if os.path.isfile(item):
shutil.copy2(os.path.join(self.cfg['start_dir'], item),
bindir)
# make sure binary has executable permissions
adjust_permissions(os.path.join(bindir, item),
stat.S_IXUSR | stat.S_IXGRP
| stat.S_IXOTH,
add=True)
self.log.debug("Copied %s to %s and fixed permissions" %
(item, bindir))
else:
self.log.warning(
"Skipping non-file %s in %s, not copying it." %
(item, self.cfg['start_dir']))
except OSError, err:
raise EasyBuildError(
"Copying binaries in %s to install dir 'bin' failed: %s",
self.cfg['start_dir'], err)
def install_step(self):
"""MATLAB install procedure using 'install' command."""
src = os.path.join(self.cfg['start_dir'], 'install')
# make sure install script is executable
adjust_permissions(src, stat.S_IXUSR)
if LooseVersion(self.version) >= LooseVersion('2016b'):
jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre',
'glnxa64', 'jre', 'bin')
for perm_dir in [
os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir
]:
adjust_permissions(perm_dir, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
if 'DISPLAY' in os.environ:
os.environ.pop('DISPLAY')
if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']:
self.cfg['preinstallopts'] = (
'export _JAVA_OPTIONS="%s" && ' %
self.cfg['java_options']) + self.cfg['preinstallopts']
if LooseVersion(self.version) >= LooseVersion('2016b'):
change_dir(self.builddir)
cmd = "%s %s -v -inputFile %s %s" % (self.cfg['preinstallopts'], src,
self.configfile,
self.cfg['installopts'])
run_cmd(cmd, log_all=True, simple=True)
def install_step(self):
"""
Install by copying files to install dir
"""
install_files = [
('include/bam', self.include_files),
('lib', self.lib_files),
]
# v1.3 and more recent supports 'make install', but this only installs (some of) the binaries...
if LooseVersion(self.version) >= LooseVersion('1.3'):
super(EB_SAMtools, self).install_step()
# figure out which bin files are missing, and try copying them
missing_bin_files = []
for binfile in self.bin_files:
if not os.path.exists(os.path.join(self.installdir, 'bin', os.path.basename(binfile))):
missing_bin_files.append(binfile)
install_files.append(('bin', missing_bin_files))
else:
# copy binaries manually for older versions
install_files.append(('bin', self.bin_files))
self.log.debug("Installing files by copying them 'manually': %s", install_files)
for (destdir, files) in install_files:
for fn in files:
dest = os.path.join(self.installdir, destdir, os.path.basename(fn))
copy_file(os.path.join(self.cfg['start_dir'], fn), dest)
# enable r-x permissions for group/others
perms = stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH
adjust_permissions(self.installdir, perms, add=True, recursive=True)
def test_toolchain_prepare_rpath(self):
"""Test toolchain.prepare under --rpath"""
# put fake 'gcc' command in place that just echos its arguments
fake_gcc = os.path.join(self.test_prefix, 'fake', 'gcc')
write_file(fake_gcc, '#!/bin/bash\necho "$@"')
adjust_permissions(fake_gcc, stat.S_IXUSR)
os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', ''))
# enable --rpath and prepare toolchain
init_config(build_options={'rpath': True, 'rpath_filter': ['/ba.*']})
tc = self.get_toolchain('gompi', version='1.3.12')
# preparing RPATH wrappers requires --experimental, need to bypass that here
tc.log.experimental = lambda x: x
tc.prepare()
# check whether fake gcc was wrapped and that arguments are what they should be
# no -rpath for /bar because of rpath filter
out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""')
expected = ' '.join([
'-Wl,-rpath=$ORIGIN/../lib',
'-Wl,-rpath=$ORIGIN/../lib64',
'-Wl,--disable-new-dtags',
'-Wl,-rpath=/foo',
'%(user)s.c',
'-L/foo',
'-L/bar',
'$FOO',
'-DX=""',
])
self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
def test_run_cmd_qa_buffering(self):
"""Test whether run_cmd_qa uses unbuffered output."""
# command that generates a lot of output before waiting for input
# note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000!
cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; '
cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"'
(out, ec) = run_cmd_qa(cmd, {'Pick a number: ': '42'},
log_all=True,
maxhits=5)
self.assertEqual(ec, 0)
regex = re.compile("Picked number: 42$")
self.assertTrue(regex.search(out),
"Pattern '%s' found in: %s" % (regex.pattern, out))
# also test with script run as interactive command that quickly exits with non-zero exit code;
# see https://github.com/easybuilders/easybuild-framework/issues/3593
script_txt = '\n'.join([
"#/bin/bash",
"echo 'Hello, I am about to exit'",
"echo 'ERROR: I failed' >&2",
"exit 1",
])
script = os.path.join(self.test_prefix, 'test.sh')
write_file(script, script_txt)
adjust_permissions(script, stat.S_IXUSR)
out, ec = run_cmd_qa(script, {}, log_ok=False)
self.assertEqual(ec, 1)
self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n")
def install_step(self):
"""Install by copying files to install dir and binary to 'bin' install subdir, and fixing permissions."""
bindir = os.path.join(self.installdir, 'bin')
binaries = [
'foldx_%s.linux' % self.version, # FoldX v2.x
'FoldX.linux64', # FoldX 3.x
'foldx64Linux', # FoldX 3.0-beta6
'foldx3b6', # FoldX 3.0 beta 6.1 >_<
]
try:
os.makedirs(bindir)
for item in os.listdir(self.cfg['start_dir']):
if os.path.isfile(item):
if item in binaries:
shutil.copy2(os.path.join(self.cfg['start_dir'], item), bindir)
# make sure binary has executable permissions
adjust_permissions(os.path.join(bindir, item), stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH, add=True)
self.log.debug("Copied %s to %s and fixed permissions" % (item, bindir))
else:
# copy everything else straight into install directory
shutil.copy2(os.path.join(self.cfg['start_dir'], item), self.installdir)
self.log.debug("Copied %s to install dir %s" % (item, self.installdir))
else:
self.log.warning("Skipping non-file %s in %s, not copying it." % (item, self.cfg['start_dir']))
except OSError, err:
raise EasyBuildError("Copying binaries in %s to install dir 'bin' failed: %s", self.cfg['start_dir'], err)
def install_step(self):
"""
Install by copying files to install dir
"""
install_files = [
('include/bam', self.include_files),
('lib', self.lib_files),
]
# v1.3 and more recent supports 'make install', but this only installs (some of) the binaries...
if LooseVersion(self.version) >= LooseVersion('1.3'):
super(EB_SAMtools, self).install_step()
# figure out which bin files are missing, and try copying them
missing_bin_files = []
for binfile in self.bin_files:
if not os.path.exists(os.path.join(self.installdir, 'bin', os.path.basename(binfile))):
missing_bin_files.append(binfile)
install_files.append(('bin', missing_bin_files))
else:
# copy binaries manually for older versions
install_files.append(('bin', self.bin_files))
self.log.debug("Installing files by copying them 'manually': %s", install_files)
for (destdir, files) in install_files:
for fn in files:
dest = os.path.join(self.installdir, destdir, os.path.basename(fn))
copy_file(os.path.join(self.cfg['start_dir'], fn), dest)
# enable r-x permissions for group/others
perms = stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH
adjust_permissions(self.installdir, perms, add=True, recursive=True)
def install_step(self):
"""Custom install procedure for Samcef."""
# e.g.: 17.0-03 => 17, 17.0, 170
maj_ver = self.version.split('.')[0]
main_ver = self.version.split('-')[0]
flat_ver = ''.join(main_ver.split('.'))
qa = {
"Default language ? (E=English, F=French, default E): ": 'E',
"Default Postscript printer ? (default: none): ": '',
"Type RETURN to continue": '',
}
no_qa = []
std_qa = {
r"Installation for:.*[\s\n]*Continue \? \(y/n, default y\):":
'y',
r" 1 Install Samcef %s \(%s\)[\s\n]*.*[\s\n]*Install type .*" % (maj_ver, main_ver):
'1',
r"Install SHORT INTEGERS VERSION \(i4\)[\s\na-zA-Z0-9().-]*Continue \? \(y/n, default y\):":
'n',
r"Install LONG INTEGERS VERSION \(i8\)[\s\na-zA-Z0-9().-]*Continue \? \(y/n, default y\):":
'y',
r"Pathname of install directory \(.*\):":
self.installdir,
r"Directory .* exists, use it \? \(y/n, default y\):":
'y',
r"Installation directory\s*:.*[\s\n]*.*[\s\n]*Continue \? \(y/n, default:y\):":
'y',
# option '2' is SAMCEF + object libraries
r"Selection size: 0[\s\n]*Your selection:":
'2',
# 'i' for install after making a selection
r"Selection size: [1-9][0-9]*[\s\n]*Your selection:":
'i',
r"Prerequisite OK[\s\n]*Continue \? \(y/n, default y\):":
'y',
r"Samcef users can modify these values.*[\s\n]*Continue \? \(y/n, default y\):":
'y',
r"What is the hostname of your license server \( without at sign @ \) \? ":
'localhost',
r"Confirm .* as the LMS Samtech Licence server \? \(y/n, default n\): ":
'y',
r"2 Floating RLM license[\s\n]*Type a value \(1/2, default 1 \)":
'2',
}
# e.g., v170_inst.csh
install_script = './v%s_inst.csh' % flat_ver
# make sure script is executable (unpack with 7z doesn't preserve permissions)
adjust_permissions(install_script, stat.S_IXUSR, add=True)
run_cmd_qa(install_script,
qa,
no_qa=no_qa,
std_qa=std_qa,
log_all=True,
simple=True)
def install_step(self):
"""MCR install procedure using 'install' command."""
src = os.path.join(self.cfg['start_dir'], 'install')
# make sure install script is executable
adjust_permissions(src, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
if 'DISPLAY' in os.environ:
os.environ.pop('DISPLAY')
if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']:
java_options = 'export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options']
self.cfg['preinstallopts'] = java_options + self.cfg['preinstallopts']
configfile = "%s/%s" % (self.builddir, self.configfilename)
cmd = "%s ./install -v -inputFile %s %s" % (self.cfg['preinstallopts'], configfile, self.cfg['installopts'])
run_cmd(cmd, log_all=True, simple=True)
# determine subdirectory (e.g. v84 (2014a, 2014b), v85 (2015a), ...)
subdirs = os.listdir(self.installdir)
if len(subdirs) == 1:
self.subdir = subdirs[0]
else:
raise EasyBuildError("Found multiple subdirectories, don't know which one to pick: %s", subdirs)
def install_step(self):
"""Install by copying the contents of the builddir to the installdir (preserving permissions)"""
for x in os.listdir(self.cfg['start_dir']):
src = os.path.join(self.cfg['start_dir'], x)
dst = os.path.join(self.installdir, x)
try:
if os.path.isdir(src):
shutil.copytree(src, dst)
# symlink
# - dst/Lib to dst/lib
# - dst/Include to dst/include
for c in ['Lib', 'Include']:
nsrc = os.path.join(dst, c)
ndst = os.path.join(dst, c.lower())
if os.path.exists(nsrc):
os.symlink(nsrc, ndst)
# enable r-x permissions for group/others
perms = stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
adjust_permissions(dst,
perms,
add=True,
recursive=True,
onlydirs=True)
else:
shutil.copy2(src, dst)
except OSError, err:
raise EasyBuildError("Copying src %s to dst %s failed: %s",
src, dst, err)
def build_step(self):
"""No build step for SNPhylo."""
# check for required dependencies
for dep in ['MUSCLE', 'PHYLIP', 'Python', 'R']:
if not get_software_root(dep):
raise EasyBuildError("Required dependency '%s' not loaded",
dep)
# check for required R libraries
rver = get_software_version('R')
r_libs, _ = run_cmd(
"R --vanilla --no-save --slave -e 'print(installed.packages())'",
simple=False)
for rpkg in ['gdsfmt', 'getopt', 'SNPRelate', 'phangorn']:
if not re.search(r'^%s\s.*%s' % (rpkg, rver), r_libs, re.M):
raise EasyBuildError("Required R package '%s' not installed",
rpkg)
# run setup.sh, and send a bunch of newlines as stdin to 'answer' the Q&A;
# all questions can be answered with the default answer (if the dependencies are specified correctly);
# use run_cmd_qa doesn not work because of buffering issues (questions are not coming through)
adjust_permissions('setup.sh', stat.S_IXUSR, add=True)
(out, _) = run_cmd('bash ./setup.sh', inp='\n' * 10, simple=False)
success_msg = "SNPHYLO is successfully installed!!!"
if success_msg not in out:
raise EasyBuildError(
"Success message '%s' not found in setup.sh output: %s",
success_msg, out)
def install_step(self):
"""Install by copying files to install dir and binary to 'bin' install subdir, and fixing permissions."""
bindir = os.path.join(self.installdir, 'bin')
binaries = [
'foldx_%s.linux' % self.version, # FoldX v2.x
'FoldX.linux64', # FoldX 3.x
'foldx64Linux', # FoldX 3.0-beta6
'foldx3b6', # FoldX 3.0 beta 6.1 >_<
]
try:
os.makedirs(bindir)
for item in os.listdir(self.cfg['start_dir']):
if os.path.isfile(item):
if item in binaries:
shutil.copy2(os.path.join(self.cfg['start_dir'], item), bindir)
# make sure binary has executable permissions
adjust_permissions(os.path.join(bindir, item), stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH, add=True)
self.log.debug("Copied %s to %s and fixed permissions" % (item, bindir))
else:
# copy everything else straight into install directory
shutil.copy2(os.path.join(self.cfg['start_dir'], item), self.installdir)
self.log.debug("Copied %s to install dir %s" % (item, self.installdir))
else:
self.log.warning("Skipping non-file %s in %s, not copying it." % (item, self.cfg['start_dir']))
except OSError, err:
raise EasyBuildError("Copying binaries in %s to install dir 'bin' failed: %s", self.cfg['start_dir'], err)
def test_remove_file(self):
"""Test remove_file"""
testfile = os.path.join(self.test_prefix, 'foo')
ft.write_file(testfile, 'bar')
self.assertTrue(os.path.exists(testfile))
ft.remove_file(testfile)
ft.write_file(testfile, 'bar')
ft.adjust_permissions(self.test_prefix,
stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False)
self.assertErrorRegex(EasyBuildError, "Failed to remove",
ft.remove_file, testfile)
# also test behaviour of remove_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
self.mock_stdout(True)
ft.remove_file(testfile)
txt = self.get_stdout()
self.mock_stdout(False)
regex = re.compile("^file [^ ]* removed$")
self.assertTrue(regex.match(txt),
"Pattern '%s' found in: %s" % (regex.pattern, txt))
def install_step(self):
"""MCR install procedure using 'install' command."""
src = os.path.join(self.cfg['start_dir'], 'install')
# make sure install script is executable
adjust_permissions(src, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
if 'DISPLAY' in os.environ:
os.environ.pop('DISPLAY')
if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']:
java_options = 'export _JAVA_OPTIONS="%s" && ' % self.cfg[
'java_options']
self.cfg[
'preinstallopts'] = java_options + self.cfg['preinstallopts']
configfile = "%s/%s" % (self.builddir, self.configfilename)
cmd = "%s ./install -v -inputFile %s %s" % (
self.cfg['preinstallopts'], configfile, self.cfg['installopts'])
run_cmd(cmd, log_all=True, simple=True)
# determine subdirectory (e.g. v84 (2014a, 2014b), v85 (2015a), ...)
subdirs = os.listdir(self.installdir)
if len(subdirs) == 1:
self.subdir = subdirs[0]
else:
raise EasyBuildError(
"Found multiple subdirectories, don't know which one to pick: %s",
subdirs)
def install_step(self):
"""Install by running install command."""
pgi_env_vars = {
'PGI_ACCEPT_EULA': 'accept',
'PGI_INSTALL_AMD': str(self.cfg['install_amd']).lower(),
'PGI_INSTALL_DIR': self.installdir,
'PGI_INSTALL_JAVA': str(self.cfg['install_java']).lower(),
'PGI_INSTALL_MANAGED': str(self.cfg['install_managed']).lower(),
'PGI_INSTALL_NVIDIA': str(self.cfg['install_nvidia']).lower(),
'PGI_SILENT': 'true',
}
cmd = "%s ./install" % ' '.join(
['%s=%s' % x for x in sorted(pgi_env_vars.items())])
run_cmd(cmd, log_all=True, simple=True)
# make sure localrc uses GCC in PATH, not always the system GCC, and does not use a system g77 but gfortran
for subdir in self.pgi_install_subdirs:
install_abs_subdir = os.path.join(self.installdir, subdir)
filename = os.path.join(install_abs_subdir, "bin", "makelocalrc")
for line in fileinput.input(filename, inplace='1', backup='.orig'):
line = re.sub(r"^PATH=/", r"#PATH=/", line)
sys.stdout.write(line)
cmd = "%s -x %s -g77 /" % (filename, install_abs_subdir)
run_cmd(cmd, log_all=True, simple=True)
# If an OS libnuma is NOT found, makelocalrc creates symbolic links to libpgnuma.so
# If we use the EB libnuma, delete those symbolic links to ensure they are not used
if get_software_root("numactl"):
for filename in ["libnuma.so", "libnuma.so.1"]:
path = os.path.join(install_abs_subdir, "lib", filename)
if os.path.islink(path):
os.remove(path)
# install (or update) siterc file to make PGI consider $LIBRARY_PATH and accept -pthread
siterc_path = os.path.join(self.installdir, subdir, 'bin',
'siterc')
# adding LIBRARY_PATH has the side effect of adding llvm to nollvm siterc files and viceversa
# this is done dynamically, so we can't account for it in all cases. Simply do it for the default
#isdefault = False
#if ((LooseVersion(self.version) > LooseVersion('18') and LooseVersion(self.version) < LooseVersion('19') and
# subdir == 'linux86-64-nollvm') or
# (LooseVersion(self.version) > LooseVersion('19') and subdir == 'linux86-64-llvm')):
# isdefault = True
if subdir == os.path.join('linux86-64', self.version):
write_file(siterc_path, SITERC_LIBRARY_PATH, append=True)
self.log.info(
"Appended instructions to pick up $LIBRARY_PATH to siterc file at %s: %s",
siterc_path, SITERC_LIBRARY_PATH)
write_file(siterc_path, SITERC_PTHREAD_SWITCH, append=True)
self.log.info(
"Append instructions to replace -pthread with -lpthread to siterc file at %s: %s",
siterc_path, SITERC_PTHREAD_SWITCH)
# The cuda nvvp tar file has broken permissions
adjust_permissions(self.installdir,
stat.S_IWUSR,
add=True,
onlydirs=True)
def test_remove_file(self):
"""Test remove_file"""
testfile = os.path.join(self.test_prefix, 'foo')
ft.write_file(testfile, 'bar')
self.assertTrue(os.path.exists(testfile))
ft.remove_file(testfile)
ft.write_file(testfile, 'bar')
ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False)
self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile)
# also test behaviour of remove_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
self.mock_stdout(True)
ft.remove_file(testfile)
txt = self.get_stdout()
self.mock_stdout(False)
regex = re.compile("^file [^ ]* removed$")
self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def install_step(self):
"""
Install binary and a wrapper that loads correct CUDA version.
"""
src_gctf_bin = os.path.join(self.builddir, 'bin', self.gctf_bin)
if not os.path.exists(src_gctf_bin):
raise EasyBuildError(
"Specified CUDA version has no corresponding Gctf binary")
bindir = os.path.join(self.installdir, 'bin')
mkdir(bindir)
dst_gctf_bin = os.path.join(bindir, self.gctf_bin)
copy_file(src_gctf_bin, dst_gctf_bin)
exe_perms = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
read_perms = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
perms = read_perms | exe_perms
adjust_permissions(dst_gctf_bin, perms, add=True)
# Install a wrapper that loads CUDA before starting the binary
wrapper = os.path.join(bindir, 'Gctf')
txt = '\n'.join([
'#!/bin/bash', '',
'# Wrapper for Gctf binary that loads the required',
'# version of CUDA',
'module unload %s' % self.cuda_name,
'module add %s' % self.cuda_mod_name,
'exec %s "$@"' % dst_gctf_bin
])
write_file(wrapper, txt)
adjust_permissions(wrapper, exe_perms, add=True)
symlink('Gctf', os.path.join(bindir, 'gctf'), use_abspath_source=False)
def create_wrapper(wrapper_name, wrapper_comp):
"""Create for a particular compiler, with a particular name"""
wrapper_f = os.path.join(self.installdir, 'bin', wrapper_name)
write_file(wrapper_f, WRAPPER_TEMPLATE % wrapper_comp)
adjust_permissions(
wrapper_f, stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP
| stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH)
def install_step(self):
"""Custom install procedure for FLUENT."""
cmd = "./INSTALL -noroot -silent -install_dir %s" % self.installdir
run_cmd(cmd, log_all=True, simple=True)
adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self):
"""MATLAB install procedure using 'install' command."""
src = os.path.join(self.cfg['start_dir'], 'install')
# make sure install script is executable
adjust_permissions(src, stat.S_IXUSR)
if LooseVersion(self.version) >= LooseVersion('2016b'):
jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre', 'glnxa64', 'jre', 'bin')
for perm_dir in [os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir]:
adjust_permissions(perm_dir, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
if 'DISPLAY' in os.environ:
os.environ.pop('DISPLAY')
if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']:
self.cfg['preinstallopts'] = ('export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options']) + self.cfg['preinstallopts']
if LooseVersion(self.version) >= LooseVersion('2016b'):
change_dir(self.builddir)
cmd = "%s %s -v -inputFile %s %s" % (self.cfg['preinstallopts'], src, self.configfile, self.cfg['installopts'])
run_cmd(cmd, log_all=True, simple=True)
def test_build_easyconfigs_in_parallel_slurm(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job."""
# install mocked versions of 'sbatch' and 'scontrol' commands
sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch')
write_file(sbatch, MOCKED_SBATCH)
adjust_permissions(sbatch, stat.S_IXUSR, add=True)
scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol')
write_file(scontrol, MOCKED_SCONTROL)
adjust_permissions(scontrol, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join(
[os.path.join(self.test_prefix, 'bin'),
os.getenv('PATH')])
topdir = os.path.dirname(os.path.abspath(__file__))
test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip',
'gzip-1.5-foss-2018a.eb')
foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss',
'foss-2018a.eb')
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
'job_max_walltime': 5,
'force': True,
}
init_config(args=['--job-backend=Slurm'], build_options=build_options)
easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
self.mock_stdout(True)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'",
ordered_ecs,
prepare_first=False)
self.mock_stdout(False)
# jobs are submitted for foss & gzip (listed easyconfigs)
self.assertEqual(len(jobs), 2)
# last job (gzip) has a dependency on second-to-last job (foss)
self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a')
expected = {
'dependency': 'afterok:%s' % jobs[0].jobid,
'hold': True,
'job-name': 'gzip-1.5-foss-2018a',
'nodes': 1,
'ntasks': 3,
'ntasks-per-node': 3,
'output': '%x-%j.out',
'time': 300, # 60*5 (unit is minutes)
'wrap': "echo '%s'" % test_ec,
}
self.assertEqual(jobs[1].job_specs, expected)
def configure_step(self):
"""Configure COMSOL installation: create license file."""
# The tar file comes from the DVD and has 0444 as permission at the top dir.
adjust_permissions(self.start_dir, stat.S_IWUSR)
default_lic_env_var = 'LMCOMSOL_LICENSE_FILE'
lic_specs, self.license_env_var = find_flexlm_license(
custom_env_vars=[default_lic_env_var],
lic_specs=[self.cfg['license_file']])
if lic_specs:
if self.license_env_var is None:
self.log.info(
"Using COMSOL license specifications from 'license_file': %s",
lic_specs)
self.license_env_var = default_lic_env_var
else:
self.log.info(
"Using COMSOL license specifications from $%s: %s",
self.license_env_var, lic_specs)
self.license_file = os.pathsep.join(lic_specs)
env.setvar(self.license_env_var, self.license_file)
else:
msg = "No viable license specifications found; "
msg += "specify 'license_file', or define $%s" % default_lic_env_var
raise EasyBuildError(msg)
copy_file(os.path.join(self.start_dir, 'setupconfig.ini'),
self.configfile)
config = read_file(self.configfile)
config_vars = {
'agree': '1',
'desktopshortcuts': '0',
'fileassoc': '0',
'firewall': '0',
'installdir': self.installdir,
'license': self.license_file,
'licmanager': '0',
'linuxlauncher': '0',
'showgui': '0',
'startmenushortcuts': '0',
'symlinks': '0',
}
matlab_root = get_software_root("MATLAB")
if matlab_root:
config_vars.update({'matlabdir': matlab_root})
for key, val in config_vars.items():
regex = re.compile(r"^%s\s*=.*" % key, re.M)
config = regex.sub("%s=%s" % (key, val), config)
write_file(self.configfile, config)
self.log.debug('configuration file written to %s:\n %s',
self.configfile, config)
def test_end2end_docker_image(self):
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
containerpath = os.path.join(self.test_prefix, 'containers')
os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
# --containerpath must be an existing directory (this is done to avoid misconfiguration)
mkdir(containerpath)
args = [
toy_ec,
'-C', # equivalent with --containerize
'--experimental',
'--container-type=docker',
'--container-config=ubuntu:16.04',
'--container-build-image',
]
if not which('docker'):
error_pattern = "docker not found on your system."
self.assertErrorRegex(EasyBuildError,
error_pattern,
self.run_main,
args,
raise_error=True)
# install mocked versions of 'sudo' and 'docker' commands
docker = os.path.join(self.test_prefix, 'bin', 'docker')
write_file(docker, MOCKED_DOCKER)
adjust_permissions(docker, stat.S_IXUSR, add=True)
sudo = os.path.join(self.test_prefix, 'bin', 'sudo')
write_file(
sudo,
'#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n'
)
adjust_permissions(sudo, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join(
[os.path.join(self.test_prefix, 'bin'),
os.getenv('PATH')])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs = [
"^== docker tool found at %s/bin/docker" % self.test_prefix,
"^== Dockerfile definition file created at %s/containers/Dockerfile\.toy-0.0"
% self.test_prefix,
"^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...",
"^== Docker image created at toy-0.0:latest",
]
self.check_regexs(regexs, stdout)
args.extend(['--force', '--extended-dry-run'])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
self.check_regexs(regexs, stdout)
def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
"""Build and install software for all provided parsed easyconfig files."""
# obtain a copy of the starting environment so each build can start afresh
# we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
# e.g. via easyconfig.handle_allowed_system_deps
init_env = copy.deepcopy(os.environ)
res = []
for ec in ecs:
ec_res = {}
try:
(ec_res['success'], app_log,
err) = build_and_install_one(ec, init_env)
ec_res['log_file'] = app_log
if not ec_res['success']:
ec_res['err'] = EasyBuildError(err)
except Exception, err:
# purposely catch all exceptions
ec_res['success'] = False
ec_res['err'] = err
ec_res['traceback'] = traceback.format_exc()
# keep track of success/total count
if ec_res['success']:
test_msg = "Successfully built %s" % ec['spec']
else:
test_msg = "Build of %s failed" % ec['spec']
if 'err' in ec_res:
test_msg += " (err: %s)" % ec_res['err']
# dump test report next to log file
test_report_txt = create_test_report(test_msg, [(ec, ec_res)],
init_session_state)
if 'log_file' in ec_res and ec_res['log_file']:
test_report_fp = "%s_test_report.md" % '.'.join(
ec_res['log_file'].split('.')[:-1])
parent_dir = os.path.dirname(test_report_fp)
# parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
if os.stat(parent_dir).st_mode & 0200:
write_file(test_report_fp, test_report_txt)
else:
adjust_permissions(parent_dir,
stat.S_IWUSR,
add=True,
recursive=False)
write_file(test_report_fp, test_report_txt)
adjust_permissions(parent_dir,
stat.S_IWUSR,
add=False,
recursive=False)
if not ec_res['success'] and exit_on_failure:
if 'traceback' in ec_res:
raise EasyBuildError(ec_res['traceback'])
else:
raise EasyBuildError(test_msg)
res.append((ec, ec_res))
def build_and_install_software(ecs, init_session_state, exit_on_failure=True, hooks=None):
"""
Build and install software for all provided parsed easyconfig files.
:param ecs: easyconfig files to install software with
:param init_session_state: initial session state, to use in test reports
:param exit_on_failure: whether or not to exit on installation failure
:param hooks: list of defined pre- and post-step hooks
"""
# obtain a copy of the starting environment so each build can start afresh
# we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
# e.g. via easyconfig.handle_allowed_system_deps
init_env = copy.deepcopy(os.environ)
run_hook(START, hooks)
res = []
for ec in ecs:
ec_res = {}
try:
(ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, hooks=hooks)
ec_res['log_file'] = app_log
if not ec_res['success']:
ec_res['err'] = EasyBuildError(err)
except Exception, err:
# purposely catch all exceptions
ec_res['success'] = False
ec_res['err'] = err
ec_res['traceback'] = traceback.format_exc()
# keep track of success/total count
if ec_res['success']:
test_msg = "Successfully built %s" % ec['spec']
else:
test_msg = "Build of %s failed" % ec['spec']
if 'err' in ec_res:
test_msg += " (err: %s)" % ec_res['err']
# dump test report next to log file
test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state)
if 'log_file' in ec_res and ec_res['log_file']:
test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1])
parent_dir = os.path.dirname(test_report_fp)
# parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
if os.stat(parent_dir).st_mode & 0200:
write_file(test_report_fp, test_report_txt)
else:
adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False)
write_file(test_report_fp, test_report_txt)
adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
if not ec_res['success'] and exit_on_failure:
if 'traceback' in ec_res:
raise EasyBuildError(ec_res['traceback'])
else:
raise EasyBuildError(test_msg)
res.append((ec, ec_res))
def post_install_step(self):
"""Add a shebang to the .py files and make them executable."""
for pyfile in self.pyfiles:
pf_path = os.path.join(self.installdir, pyfile)
pf_contents = read_file(pf_path)
write_file(pf_path, "#!/usr/bin/env python\n" + pf_contents)
adjust_permissions(pf_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
super(EB_fastStructure, self).post_install_step()
def post_install_step(self):
"""Add a shebang to the .py files and make them executable."""
for pyfile in self.pyfiles:
pf_path = os.path.join(self.installdir, pyfile)
pf_contents = read_file(pf_path)
write_file(pf_path, "#!/usr/bin/env python\n" + pf_contents)
adjust_permissions(pf_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
super(EB_fastStructure, self).post_install_step()
def install_step(self):
"""MATLAB install procedure using 'install' command."""
src = os.path.join(self.cfg['start_dir'], 'install')
# make sure install script is executable
adjust_permissions(src, stat.S_IXUSR)
if LooseVersion(self.version) >= LooseVersion('2016b'):
jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre', 'glnxa64', 'jre', 'bin')
for perm_dir in [os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir]:
adjust_permissions(perm_dir, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
if 'DISPLAY' in os.environ:
os.environ.pop('DISPLAY')
if '_JAVA_OPTIONS' not in self.cfg['preinstallopts']:
java_opts = 'export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options']
self.cfg['preinstallopts'] = java_opts + self.cfg['preinstallopts']
if LooseVersion(self.version) >= LooseVersion('2016b'):
change_dir(self.builddir)
# MATLAB installer ignores TMPDIR (always uses /tmp) and might need a large tmpdir
tmpdir = "-tmpdir %s" % tempfile.mkdtemp()
keys = self.cfg['key']
if keys is None:
keys = os.getenv('EB_MATLAB_KEY', '00000-00000-00000-00000-00000-00000-00000-00000-00000-00000')
if isinstance(keys, basestring):
keys = keys.split(',')
# Make one install for each key
for key in keys:
cmd = ' '.join([
self.cfg['preinstallopts'],
src,
'-v',
tmpdir,
'-inputFile',
self.configfile,
'-fileInstallationKey',
key,
self.cfg['installopts'],
])
(out, _) = run_cmd(cmd, log_all=True, simple=False)
# check installer output for known signs of trouble
patterns = [
"Error: You have entered an invalid File Installation Key",
]
for pattern in patterns:
regex = re.compile(pattern, re.I)
if regex.search(out):
raise EasyBuildError("Found error pattern '%s' in output of installation command '%s': %s",
regex.pattern, cmd, out)
def install_step(self):
"""Custom install procedure for ANSYS."""
licserv = self.cfg["license_server"]
licport = self.cfg["license_server_port"]
cmd = "./INSTALL -silent -install_dir %s -licserverinfo %s:%s" % (self.installdir, licport, licserv)
run_cmd(cmd, log_all=True, simple=True)
adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self):
"""MATLAB install procedure using 'install' command."""
src = os.path.join(self.cfg['start_dir'], 'install')
# make sure install script is executable
adjust_permissions(src, stat.S_IXUSR)
if LooseVersion(self.version) >= LooseVersion('2016b'):
jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre', 'glnxa64', 'jre', 'bin')
for perm_dir in [os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir]:
adjust_permissions(perm_dir, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
if 'DISPLAY' in os.environ:
os.environ.pop('DISPLAY')
if '_JAVA_OPTIONS' not in self.cfg['preinstallopts']:
java_opts = 'export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options']
self.cfg['preinstallopts'] = java_opts + self.cfg['preinstallopts']
if LooseVersion(self.version) >= LooseVersion('2016b'):
change_dir(self.builddir)
# MATLAB installer ignores TMPDIR (always uses /tmp) and might need a large tmpdir
tmpdir = "-tmpdir %s" % tempfile.mkdtemp()
keys = self.cfg['key']
if keys is None:
keys = os.getenv('EB_MATLAB_KEY', '00000-00000-00000-00000-00000-00000-00000-00000-00000-00000')
if isinstance(keys, basestring):
keys = keys.split(',')
# Make one install for each key
for key in keys:
cmd = ' '.join([
self.cfg['preinstallopts'],
src,
'-v',
tmpdir,
'-inputFile',
self.configfile,
'-fileInstallationKey',
key,
self.cfg['installopts'],
])
(out, _) = run_cmd(cmd, log_all=True, simple=False)
# check installer output for known signs of trouble
patterns = [
"Error: You have entered an invalid File Installation Key",
]
for pattern in patterns:
regex = re.compile(pattern, re.I)
if regex.search(out):
raise EasyBuildError("Found error pattern '%s' in output of installation command '%s': %s",
regex.pattern, cmd, out)
def install_step(self):
"""Custom install procedure for ANSYS."""
licserv = self.cfg['license_server']
licport = self.cfg['license_server_port']
cmd = "./INSTALL -silent -install_dir %s -licserverinfo %s:%s" % (self.installdir, licport, licserv)
run_cmd(cmd, log_all=True, simple=True)
adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def configure_step(self):
"""Configure MATLAB installation: create license file."""
licfile = self.cfg['license_file']
if licfile is None:
licserv = self.cfg['license_server']
if licserv is None:
licserv = os.getenv('EB_MATLAB_LICENSE_SERVER',
'license.example.com')
licport = self.cfg['license_server_port']
if licport is None:
licport = os.getenv('EB_MATLAB_LICENSE_SERVER_PORT', '00000')
# create license file
lictxt = '\n'.join([
"SERVER %s 000000000000 %s" % (licserv, licport),
"USE_SERVER",
])
licfile = os.path.join(self.builddir, 'matlab.lic')
write_file(licfile, lictxt)
try:
copy_file(
os.path.join(self.cfg['start_dir'], 'installer_input.txt'),
self.configfile)
adjust_permissions(self.configfile, stat.S_IWUSR)
# read file in binary mode to avoid UTF-8 encoding issues when using Python 3,
# due to non-UTF-8 characters...
config = read_file(self.configfile, mode='rb')
# use raw byte strings (must be 'br', not 'rb'),
# required when using Python 3 because file was read in binary mode
regdest = re.compile(br"^# destinationFolder=.*", re.M)
regagree = re.compile(br"^# agreeToLicense=.*", re.M)
regmode = re.compile(br"^# mode=.*", re.M)
reglicpath = re.compile(br"^# licensePath=.*", re.M)
# must use byte-strings here when using Python 3, see above
config = regdest.sub(
b"destinationFolder=%s" % self.installdir.encode('utf-8'),
config)
config = regagree.sub(b"agreeToLicense=Yes", config)
config = regmode.sub(b"mode=silent", config)
config = reglicpath.sub(
b"licensePath=%s" % licfile.encode('utf-8'), config)
write_file(self.configfile, config)
except IOError as err:
raise EasyBuildError(
"Failed to create installation config file %s: %s",
self.configfile, err)
self.log.debug('configuration file written to %s:\n %s',
self.configfile, config)
def install_step(self):
"""Install by running install command."""
pgi_env_vars = {
'PGI_ACCEPT_EULA': 'accept',
'PGI_INSTALL_AMD': str(self.cfg['install_amd']).lower(),
'PGI_INSTALL_DIR': self.installdir,
'PGI_INSTALL_JAVA': str(self.cfg['install_java']).lower(),
'PGI_INSTALL_MANAGED': str(self.cfg['install_managed']).lower(),
'PGI_INSTALL_NVIDIA': str(self.cfg['install_nvidia']).lower(),
'PGI_SILENT': 'true',
'PGI_INSTALL_MPI': str(self.cfg['install_mpi']).lower(),
'PGI_MPI_GPU_SUPPORT': str(self.cfg['mpi_gpu_support']).lower(),
}
cmd = "%s ./install" % ' '.join(
['%s=%s' % x for x in sorted(pgi_env_vars.items())])
run_cmd(cmd, log_all=True, simple=True)
# make sure localrc uses GCC in PATH, not always the system GCC, and does not use a system g77 but gfortran
install_abs_subdir = os.path.join(self.installdir,
self.pgi_install_subdir)
filename = os.path.join(install_abs_subdir, "bin", "makelocalrc")
for line in fileinput.input(filename, inplace='1', backup='.orig'):
line = re.sub(r"^PATH=/", r"#PATH=/", line)
sys.stdout.write(line)
cmd = "%s -x %s -g77 /" % (filename, install_abs_subdir)
run_cmd(cmd, log_all=True, simple=True)
# If an OS libnuma is NOT found, makelocalrc creates symbolic links to libpgnuma.so
# If we use the EB libnuma, delete those symbolic links to ensure they are not used
if get_software_root("numactl"):
for subdir in self.pgi_install_subdirs:
install_abs_subdir = os.path.join(self.installdir, subdir)
for filename in ["libnuma.so", "libnuma.so.1"]:
path = os.path.join(install_abs_subdir, "lib", filename)
if os.path.islink(path):
os.remove(path)
# install (or update) siterc file to make PGI consider $LIBRARY_PATH and accept -pthread
siterc_path = os.path.join(self.installdir, self.pgi_install_subdir,
'bin', 'siterc')
write_file(siterc_path, SITERC_LIBRARY_PATH, append=True)
self.log.info(
"Appended instructions to pick up $LIBRARY_PATH to siterc file at %s: %s",
siterc_path, SITERC_LIBRARY_PATH)
write_file(siterc_path, SITERC_PTHREAD_SWITCH, append=True)
self.log.info(
"Append instructions to replace -pthread with -lpthread to siterc file at %s: %s",
siterc_path, SITERC_PTHREAD_SWITCH)
# The cuda nvvp tar file has broken permissions
adjust_permissions(self.installdir,
stat.S_IWUSR,
add=True,
onlydirs=True)
def install_step(self):
super(EB_BWISE, self).install_step()
# BWISE expects BCALM to be at exactly this location...
bcalmwrapper = """#!/bin/sh
$EBROOTBCALM/bin/bcalm "$@"
"""
write_file(os.path.join(self.installdir, "bin", "bcalm"), bcalmwrapper)
adjust_permissions(os.path.join(self.installdir, "bin"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def extract_step(self):
"""Unpack the source"""
if LooseVersion(self.version) < LooseVersion('1.7'):
copy_file(self.src[0]['path'], self.builddir)
adjust_permissions(os.path.join(self.builddir, self.src[0]['name']), stat.S_IXUSR, add=True)
change_dir(self.builddir)
run_cmd(os.path.join(self.builddir, self.src[0]['name']), log_all=True, simple=True, inp='')
else:
PackedBinary.extract_step(self)
def install_step(self):
"""Custom install procedure for FLUENT."""
extra_args =''
# only include -noroot flag for older versions
if LooseVersion(self.version) < LooseVersion('15.0'):
extra_args += '-noroot'
cmd = "./INSTALL %s -debug -silent -install_dir %s %s" % (extra_args, self.installdir, self.cfg['installopts'])
run_cmd(cmd, log_all=True, simple=True)
adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self):
"""Copy all files in build directory to the install directory"""
rmtree2(self.installdir)
install_script = self.src[0]['name']
adjust_permissions(os.path.join(self.builddir, install_script), stat.S_IRUSR|stat.S_IXUSR)
cmd = "%s ./%s -p %s -b -f" % (self.cfg['preinstallopts'], install_script, self.installdir)
self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
run_cmd(cmd, log_all=True, simple=True)
def install_step(self):
"""Custom install procedure for FLUENT."""
extra_args =''
# only include -noroot flag for older versions
if LooseVersion(self.version) < LooseVersion('15.0'):
extra_args += '-noroot'
cmd = "./INSTALL %s -debug -silent -install_dir %s %s" % (extra_args, self.installdir, self.cfg['installopts'])
run_cmd(cmd, log_all=True, simple=True)
adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self):
"""Custom install procedure for NWChem."""
try:
# binary
bindir = os.path.join(self.installdir, 'bin')
mkdir(bindir)
shutil.copy(
os.path.join(self.cfg['start_dir'], 'bin', self.cfg['target'],
'nwchem'), bindir)
# data
shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'data'),
os.path.join(self.installdir, 'data'))
shutil.copytree(
os.path.join(self.cfg['start_dir'], 'src', 'basis',
'libraries'),
os.path.join(self.installdir, 'data', 'libraries'))
shutil.copytree(
os.path.join(self.cfg['start_dir'], 'src', 'nwpw',
'libraryps'),
os.path.join(self.installdir, 'data', 'libraryps'))
except OSError as err:
raise EasyBuildError("Failed to install NWChem: %s", err)
# create NWChem settings file
default_nwchemrc = os.path.join(self.installdir, 'data',
'default.nwchemrc')
txt = '\n'.join([
"nwchem_basis_library %(path)s/data/libraries/",
"nwchem_nwpw_library %(path)s/data/libraryps/",
"ffield amber",
"amber_1 %(path)s/data/amber_s/",
"amber_2 %(path)s/data/amber_q/",
"amber_3 %(path)s/data/amber_x/",
"amber_4 %(path)s/data/amber_u/",
"spce %(path)s/data/solvents/spce.rst",
"charmm_s %(path)s/data/charmm_s/",
"charmm_x %(path)s/data/charmm_x/",
]) % {
'path': self.installdir
}
write_file(default_nwchemrc, txt)
# fix permissions in data directory
datadir = os.path.join(self.installdir, 'data')
adjust_permissions(datadir, stat.S_IROTH, add=True, recursive=True)
adjust_permissions(datadir,
stat.S_IXOTH,
add=True,
recursive=True,
onlydirs=True)
def install_step(self):
super(EB_BWISE, self).install_step()
# BWISE expects BCALM to be at exactly this location...
bcalmwrapper = """#!/bin/sh
$EBROOTBCALM/bin/bcalm "$@"
"""
write_file(os.path.join(self.installdir, "bin", "bcalm"), bcalmwrapper)
adjust_permissions(os.path.join(self.installdir, "bin"),
stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def obtain_config_guess(download_source_path=None, search_source_paths=None):
"""
Locate or download an up-to-date config.guess
:param download_source_path: Path to download config.guess to
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
log = fancylogger.getLogger('obtain_config_guess')
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
else:
log.deprecated("Specifying custom source path to download config.guess via 'download_source_path'", '5.0')
if search_source_paths is None:
search_source_paths = eb_source_paths
else:
log.deprecated("Specifying custom location to search for updated config.guess via 'search_source_paths'", '5.0')
config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')
config_guess_path = None
# check if config.guess has already been downloaded to source path
for path in search_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path) and check_config_guess(cand_config_guess_path):
force_download = build_option('force_download')
if force_download:
print_warning("Found file %s at %s, but re-downloading it anyway..."
% (config_guess, cand_config_guess_path))
else:
config_guess_path = cand_config_guess_path
log.info("Found %s at %s", config_guess, config_guess_path)
break
if not config_guess_path:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
if not download_file(config_guess, config_guess_url, cand_config_guess_path):
print_warning("Failed to download recent %s to %s", config_guess, cand_config_guess_path, log=log)
elif not check_config_guess(cand_config_guess_path):
print_warning("Verification failed for file %s, not using it!", cand_config_guess_path, log=log)
remove_file(cand_config_guess_path)
else:
config_guess_path = cand_config_guess_path
adjust_permissions(config_guess_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
log.info("Verified %s at %s, using it if required", config_guess, config_guess_path)
return config_guess_path
def test_build_easyconfigs_in_parallel_slurm(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job."""
# install mocked versions of 'sbatch' and 'scontrol' commands
sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch')
write_file(sbatch, MOCKED_SBATCH)
adjust_permissions(sbatch, stat.S_IXUSR, add=True)
scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol')
write_file(scontrol, MOCKED_SCONTROL)
adjust_permissions(scontrol, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')])
topdir = os.path.dirname(os.path.abspath(__file__))
test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb')
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
'job_max_walltime': 5,
'force': True,
}
init_config(args=['--job-backend=Slurm'], build_options=build_options)
easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
self.mock_stdout(True)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
self.mock_stdout(False)
# jobs are submitted for foss & gzip (listed easyconfigs)
self.assertEqual(len(jobs), 2)
# last job (gzip) has a dependency on second-to-last job (foss)
self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a')
expected = {
'dependency': 'afterok:%s' % jobs[0].jobid,
'hold': True,
'job-name': 'gzip-1.5-foss-2018a',
'nodes': 1,
'ntasks': 3,
'ntasks-per-node': 3,
'output': '%x-%j.out',
'time': 300, # 60*5 (unit is minutes)
'wrap': "echo '%s'" % test_ec,
}
self.assertEqual(jobs[1].job_specs, expected)
def extract_step(self):
"""Unpack the source"""
if LooseVersion(self.version) < LooseVersion('1.7'):
try:
shutil.copy2(self.src[0]['path'], self.builddir)
adjust_permissions(os.path.join(self.builddir, self.src[0]['name']), stat.S_IXUSR, add=True)
except OSError, err:
raise EasyBuildError("Failed copying installer to builddir or adjunting permissions: %s", err)
try:
os.chdir(self.builddir)
except OSError, err:
raise EasyBuildError("Failed to move to build dir: %s", err)
def extract_step(self):
"""Unpack the source"""
if LooseVersion(self.version) < LooseVersion('1.7'):
try:
shutil.copy2(self.src[0]['path'], self.builddir)
adjust_permissions(os.path.join(self.builddir, self.src[0]['name']), stat.S_IXUSR, add=True)
except OSError, err:
raise EasyBuildError("Failed copying installer to builddir or adjunting permissions: %s", err)
try:
os.chdir(self.builddir)
except OSError, err:
raise EasyBuildError("Failed to move to build dir: %s", err)
def install_step(self):
"""CPLEX has an interactive installer, so use Q&A"""
tmpdir = os.path.join(self.builddir, 'tmp')
stagedir = os.path.join(self.builddir, 'staged')
change_dir(self.builddir)
mkdir(tmpdir)
mkdir(stagedir)
env.setvar('IATEMPDIR', tmpdir)
dst = os.path.join(self.builddir, self.src[0]['name'])
cmd = "%s -i console" % dst
qanda = {
"PRESS <ENTER> TO CONTINUE:":
'',
'Press Enter to continue viewing the license agreement, or enter'
' "1" to accept the agreement, "2" to decline it, "3" to print it,'
' or "99" to go back to the previous screen.:':
'1',
'ENTER AN ABSOLUTE PATH, OR PRESS <ENTER> TO ACCEPT THE DEFAULT :':
self.installdir,
'IS THIS CORRECT? (Y/N):':
'y',
'PRESS <ENTER> TO INSTALL:':
'',
"PRESS <ENTER> TO EXIT THE INSTALLER:":
'',
"CHOOSE LOCALE BY NUMBER:":
'',
"Choose Instance Management Option:":
'',
"No model content or proprietary data will be sent.\n1- Yes\n2- No\n"
"ENTER THE NUMBER OF THE DESIRED CHOICE:":
'2',
}
noqanda = [r'Installing\.\.\..*\n.*------.*\n\n.*============.*\n.*$']
run_cmd_qa(cmd, qanda, no_qa=noqanda, log_all=True, simple=True)
# fix permissions on install dir
perms = stat.S_IRWXU | stat.S_IXOTH | stat.S_IXGRP | stat.S_IROTH | stat.S_IRGRP
adjust_permissions(self.installdir,
perms,
recursive=False,
relative=False)
# also install Python bindings if Python is included as a dependency
if self.with_python:
cwd = change_dir(os.path.join(self.installdir, 'python'))
run_cmd("python setup.py install --prefix=%s" % self.installdir)
change_dir(cwd)
def configure_step(self):
"""Configure COMSOL installation: create license file."""
# The tar file comes from the DVD and has 0444 as permission at the top dir.
adjust_permissions(self.start_dir, stat.S_IWUSR)
default_lic_env_var = 'LMCOMSOL_LICENSE_FILE'
lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var],
lic_specs=[self.cfg['license_file']])
if lic_specs:
if self.license_env_var is None:
self.log.info("Using COMSOL license specifications from 'license_file': %s", lic_specs)
self.license_env_var = default_lic_env_var
else:
self.log.info("Using COMSOL license specifications from $%s: %s", self.license_env_var, lic_specs)
self.license_file = os.pathsep.join(lic_specs)
env.setvar(self.license_env_var, self.license_file)
else:
msg = "No viable license specifications found; "
msg += "specify 'license_file', or define $%s" % default_lic_env_var
raise EasyBuildError(msg)
copy_file(os.path.join(self.start_dir, 'setupconfig.ini'), self.configfile)
config = read_file(self.configfile)
config_vars = {
'agree': '1',
'desktopshortcuts': '0',
'fileassoc': '0',
'firewall': '0',
'installdir': self.installdir,
'license': self.license_file,
'licmanager': '0',
'linuxlauncher': '0',
'showgui': '0',
'startmenushortcuts': '0',
'symlinks': '0',
}
matlab_root = get_software_root("MATLAB")
if matlab_root:
config_vars.update({'matlabdir': matlab_root})
for key, val in config_vars.items():
regex = re.compile(r"^%s\s*=.*" % key, re.M)
config = regex.sub("%s=%s" % (key, val), config)
write_file(self.configfile, config)
self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def mock_fpm(tmpdir):
"""Put mocked version of fpm command in place in specified tmpdir."""
# put mocked 'fpm' command in place, just for testing purposes
fpm = os.path.join(tmpdir, 'fpm')
write_file(fpm, MOCKED_FPM)
adjust_permissions(fpm, stat.S_IXUSR, add=True)
# also put mocked rpmbuild in place
rpmbuild = os.path.join(tmpdir, 'rpmbuild')
write_file(rpmbuild, '#!/bin/bash') # only needs to be there, doesn't need to actually do something...
adjust_permissions(rpmbuild, stat.S_IXUSR, add=True)
os.environ['PATH'] = '%s:%s' % (tmpdir, os.environ['PATH'])
def install_step(self):
"""Custom install procedure for ANSYS."""
licserv = self.cfg['license_server']
if licserv is None:
licserv = os.getenv('EB_ANSYS_LICENSE_SERVER', 'license.example.com')
licport = self.cfg['license_server_port']
if licport is None:
licport = os.getenv('EB_ANSYS_LICENSE_SERVER_PORT', '2325:1055')
cmd = "./INSTALL -silent -install_dir %s -licserverinfo %s:%s" % (self.installdir, licport, licserv)
run_cmd(cmd, log_all=True, simple=True)
adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs # noqa (ignore unused import)
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False, recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0'))
self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def install_step(self):
"""COMSOL install procedure using 'install' command."""
setup_script = os.path.join(self.start_dir, 'setup')
# make sure setup script is executable
adjust_permissions(setup_script, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
env.unset_env_vars(['DISPLAY'])
cmd = ' '.join([self.cfg['preinstallopts'], setup_script, '-s', self.configfile, self.cfg['installopts']])
run_cmd(cmd, log_all=True, simple=True)
def extract_step(self):
"""Copy all source files to the build directory"""
if self.cfg.get('extract_sources', False):
super(Binary, self).extract_step()
else:
# required for correctly guessing start directory
self.src[0]['finalpath'] = self.builddir
# copy source to build dir
for source in self.src:
dst = os.path.join(self.builddir, source['name'])
copy_file(source['path'], dst)
adjust_permissions(dst, stat.S_IRWXU, add=True)
def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
"""Build and install software for all provided parsed easyconfig files."""
# obtain a copy of the starting environment so each build can start afresh
# we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
# e.g. via easyconfig.handle_allowed_system_deps
init_env = copy.deepcopy(os.environ)
res = []
for ec in ecs:
ec_res = {}
try:
(ec_res["success"], app_log, err) = build_and_install_one(ec, init_env)
ec_res["log_file"] = app_log
if not ec_res["success"]:
ec_res["err"] = EasyBuildError(err)
except Exception, err:
# purposely catch all exceptions
ec_res["success"] = False
ec_res["err"] = err
ec_res["traceback"] = traceback.format_exc()
# keep track of success/total count
if ec_res["success"]:
test_msg = "Successfully built %s" % ec["spec"]
else:
test_msg = "Build of %s failed" % ec["spec"]
if "err" in ec_res:
test_msg += " (err: %s)" % ec_res["err"]
# dump test report next to log file
test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state)
if "log_file" in ec_res and ec_res["log_file"]:
test_report_fp = "%s_test_report.md" % ".".join(ec_res["log_file"].split(".")[:-1])
parent_dir = os.path.dirname(test_report_fp)
# parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
if os.stat(parent_dir).st_mode & 0200:
write_file(test_report_fp, test_report_txt)
else:
adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False)
write_file(test_report_fp, test_report_txt)
adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
if not ec_res["success"] and exit_on_failure:
if "traceback" in ec_res:
raise EasyBuildError(ec_res["traceback"])
else:
raise EasyBuildError(test_msg)
res.append((ec, ec_res))
def test_end2end_docker_image(self):
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
containerpath = os.path.join(self.test_prefix, 'containers')
os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
# --containerpath must be an existing directory (this is done to avoid misconfiguration)
mkdir(containerpath)
args = [
toy_ec,
'-C', # equivalent with --containerize
'--experimental',
'--container-type=docker',
'--container-base=ubuntu:16.04',
'--container-build-image',
]
if not which('docker'):
error_pattern = "docker not found on your system."
self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
# install mocked versions of 'sudo' and 'docker' commands
docker = os.path.join(self.test_prefix, 'bin', 'docker')
write_file(docker, MOCKED_DOCKER)
adjust_permissions(docker, stat.S_IXUSR, add=True)
sudo = os.path.join(self.test_prefix, 'bin', 'sudo')
write_file(sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n')
adjust_permissions(sudo, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs = [
"^== docker tool found at %s/bin/docker" % self.test_prefix,
"^== Dockerfile definition file created at %s/containers/Dockerfile\.toy-0.0" % self.test_prefix,
"^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...",
"^== Docker image created at toy-0.0:latest",
]
self.check_regexs(regexs, stdout)
args.extend(['--force', '--extended-dry-run'])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
self.check_regexs(regexs, stdout)
def test_check_pkg_support(self):
"""Test check_pkg_support()."""
# clear $PATH to make sure fpm/rpmbuild can not be found
os.environ['PATH'] = ''
self.assertErrorRegex(EasyBuildError, "Selected packaging tool 'fpm' not found", check_pkg_support)
for binary in ['fpm', 'rpmbuild']:
binpath = os.path.join(self.test_prefix, binary)
write_file(binpath, '#!/bin/bash')
adjust_permissions(binpath, stat.S_IXUSR, add=True)
os.environ['PATH'] = self.test_prefix
# no errors => support check passes
check_pkg_support()
def test_which(self):
"""Test which function for locating commands."""
python = ft.which('python')
self.assertTrue(python and os.path.exists(python) and os.path.isabs(python))
path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available')
self.assertTrue(path is None)
os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH'])
foo, bar = os.path.join(self.test_prefix, 'foo'), os.path.join(self.test_prefix, 'bar')
ft.mkdir(foo)
ft.adjust_permissions(foo, stat.S_IRUSR|stat.S_IXUSR)
ft.write_file(bar, '#!/bin/bash')
ft.adjust_permissions(bar, stat.S_IRUSR|stat.S_IXUSR)
self.assertEqual(ft.which('foo'), None)
self.assertTrue(os.path.samefile(ft.which('bar'), bar))
def install_step(self):
"""Install by copying unzipped binaries to 'bin' subdir of installation dir, and fixing permissions."""
bindir = os.path.join(self.installdir, 'bin')
try:
os.makedirs(bindir)
for item in os.listdir(self.cfg['start_dir']):
if os.path.isfile(item):
shutil.copy2(os.path.join(self.cfg['start_dir'], item), bindir)
# make sure binary has executable permissions
adjust_permissions(os.path.join(bindir, item), stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH, add=True)
self.log.debug("Copied %s to %s and fixed permissions" % (item, bindir))
else:
self.log.warning("Skipping non-file %s in %s, not copying it." % (item, self.cfg['start_dir']))
except OSError, err:
raise EasyBuildError("Copying binaries in %s to install dir 'bin' failed: %s", self.cfg['start_dir'], err)
def test_run_cmd_script(self):
"""Testing use of run_cmd with shell=False to call external scripts"""
py_test_script = os.path.join(self.test_prefix, 'test.py')
write_file(py_test_script, '\n'.join([
'#!/usr/bin/python',
'print("hello")',
]))
adjust_permissions(py_test_script, stat.S_IXUSR)
(out, ec) = run_cmd(py_test_script)
self.assertEqual(ec, 0)
self.assertEqual(out, "hello\n")
(out, ec) = run_cmd([py_test_script], shell=False)
self.assertEqual(ec, 0)
self.assertEqual(out, "hello\n")
def write_wrapper(self, wrapper_dir, compiler, i_mpi_root):
"""Helper function to write a compiler wrapper."""
wrapper_txt = INTEL_COMPILER_WRAPPER % {
'compiler_path': which(compiler),
'intel_mpi_root': i_mpi_root,
'cpath': os.getenv('CPATH'),
'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')),
'wrapper_dir': wrapper_dir,
}
wrapper = os.path.join(wrapper_dir, compiler)
write_file(wrapper, wrapper_txt)
if self.dry_run:
self.dry_run_msg("Wrapper for '%s' was put in place: %s", compiler, wrapper)
else:
adjust_permissions(wrapper, stat.S_IXUSR)
self.log.info("Using wrapper script for '%s': %s", compiler, which(compiler))