def test_apply_regex_substitutions(self):
"""Test apply_regex_substitutions function."""
testfile = os.path.join(self.test_prefix, 'test.txt')
testtxt = '\n'.join([
"CC = gcc",
"CFLAGS = -O3 -g",
"FC = gfortran",
"FFLAGS = -O3 -g -ffixed-form",
])
ft.write_file(testfile, testtxt)
regex_subs = [
(r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"),
(r"^(FC\s*=\s*).*$", r"\1${FC}"),
(r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"),
]
ft.apply_regex_substitutions(testfile, regex_subs)
expected_testtxt = '\n'.join([
"CC = ${CC}",
"CFLAGS = -O2",
"FC = ${FC}",
"FFLAGS = -O2 -ffixed-form",
])
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, expected_testtxt)
def configure_step(self):
"""Configure SCOTCH build: locate the template makefile, copy it to a general Makefile.inc and patch it."""
# pick template makefile
comp_fam = self.toolchain.comp_family()
if comp_fam == toolchain.INTELCOMP: # @UndefinedVariable
makefilename = 'Makefile.inc.x86-64_pc_linux2.icc'
elif comp_fam == toolchain.GCC: # @UndefinedVariable
makefilename = 'Makefile.inc.x86-64_pc_linux2'
else:
raise EasyBuildError("Unknown compiler family used: %s", comp_fam)
srcdir = os.path.join(self.cfg['start_dir'], 'src')
# create Makefile.inc
makefile_inc = os.path.join(srcdir, 'Makefile.inc')
copy_file(os.path.join(srcdir, 'Make.inc', makefilename), makefile_inc)
self.log.debug("Successfully copied Makefile.inc to src dir: %s", makefile_inc)
# the default behaviour of these makefiles is still wrong
# e.g., compiler settings, and we need -lpthread
regex_subs = [
(r"^CCS\s*=.*$", "CCS\t= $(CC)"),
(r"^CCP\s*=.*$", "CCP\t= $(MPICC)"),
(r"^CCD\s*=.*$", "CCD\t= $(MPICC)"),
# append -lpthread to LDFLAGS
(r"^LDFLAGS\s*=(?P<ldflags>.*$)", "LDFLAGS\t=\g<ldflags> -lpthread"),
]
apply_regex_substitutions(makefile_inc, regex_subs)
# change to src dir for building
change_dir(srcdir)
def find_build_subdir(pattern):
"""Changes to the sub directory that matches the given pattern"""
subdir = glob.glob(os.path.join(self.builddir, pattern))
if subdir:
change_dir(subdir[0])
apply_regex_substitutions('makefile', makefiles_fixes)
super(EB_BWISE, self).build_step()
return subdir[0]
else:
raise EasyBuildError("Could not find a subdirectory matching the pattern %s", pattern)
def configure_step(self):
"""Custom configuration procedure for binutils: statically link to zlib, configure options."""
# determine list of 'lib' directories to use rpath for;
# this should 'harden' the resulting binutils to bootstrap GCC (no trouble when other libstdc++ is build etc)
libdirs = []
for libdir in ['/usr/lib', '/usr/lib64', '/usr/lib/x86_64-linux-gnu/']:
# also consider /lib, /lib64
alt_libdir = libdir.replace('usr/', '')
if os.path.exists(libdir):
libdirs.append(libdir)
if os.path.exists(alt_libdir) and not os.path.samefile(libdir, alt_libdir):
libdirs.append(alt_libdir)
elif os.path.exists(alt_libdir):
libdirs.append(alt_libdir)
libs = ' '.join('-Wl,-rpath=%s' % libdir for libdir in libdirs)
# statically link to zlib if it is a (build) dependency
zlibroot = get_software_root('zlib')
if zlibroot:
self.cfg.update('configopts', '--with-system-zlib')
libz_path = os.path.join(zlibroot, get_software_libdir('zlib'), 'libz.a')
# for recent binutils versions, we need to override ZLIB in Makefile.in of components
if LooseVersion(self.version) >= LooseVersion('2.26'):
regex_subs = [
(r"^(ZLIB\s*=\s*).*$", r"\1%s" % libz_path),
(r"^(ZLIBINC\s*=\s*).*$", r"\1-I%s" % os.path.join(zlibroot, 'include')),
]
for makefile in glob.glob(os.path.join(self.cfg['start_dir'], '*', 'Makefile.in')):
apply_regex_substitutions(makefile, regex_subs)
# for older versions, injecting the path to the static libz library into $LIBS works
else:
libs += ' ' + libz_path
self.cfg.update('preconfigopts', "env LIBS='%s'" % libs)
self.cfg.update('prebuildopts', "env LIBS='%s'" % libs)
# use correct sysroot, to make sure 'ld' also considers system libraries
self.cfg.update('configopts', '--with-sysroot=/')
# build both static and shared libraries for recent binutils versions (default is only static)
if LooseVersion(self.version) > LooseVersion('2.24'):
self.cfg.update('configopts', "--enable-shared --enable-static")
# enable gold linker with plugin support, use ld as default linker (for recent versions of binutils)
if LooseVersion(self.version) > LooseVersion('2.24'):
self.cfg.update('configopts', "--enable-gold --enable-plugins --enable-ld=default")
# complete configuration with configure_method of parent
super(EB_binutils, self).configure_step()
def install_step(self):
"""
Custom install procedure for Molpro.
For source install:
* put license token in place in $installdir/.token
* run 'make tuning'
* install with 'make install'
For binary install:
* run interactive installer
"""
if self.cfg['precompiled_binaries']:
"""Build by running the command with the inputfiles"""
try:
os.chdir(self.cfg['start_dir'])
except OSError as err:
raise EasyBuildError("Failed to move (back) to %s: %s", self.cfg['start_dir'], err)
for src in self.src:
if LooseVersion(self.version) >= LooseVersion('2015'):
# install dir must be non-existent
shutil.rmtree(self.installdir)
cmd = "./{0} -batch -prefix {1}".format(src['name'], self.installdir)
else:
cmd = "./{0} -batch -instbin {1}/bin -instlib {1}/lib".format(src['name'], self.installdir)
# questions whose text must match exactly as asked
qa = {
"Please give your username for accessing molpro\n": '',
"Please give your password for accessing molpro\n": '',
}
# questions whose text may be matched as a regular expression
stdqa = {
r"Enter installation directory for executable files \[.*\]\n": os.path.join(self.installdir, 'bin'),
r"Enter installation directory for library files \[.*\]\n": os.path.join(self.installdir, 'lib'),
r"directory .* does not exist, try to create [Y]/n\n": '',
}
run_cmd_qa(cmd, qa=qa, std_qa=stdqa, log_all=True, simple=True)
else:
if os.path.isfile(self.license_token):
run_cmd("make tuning")
super(EB_Molpro, self).install_step()
# put original LAUNCHER definition back in place in bin/molpro that got installed,
# since the value used during installation point to temporary files
molpro_path = os.path.join(self.full_prefix, 'bin', 'molpro')
apply_regex_substitutions(molpro_path, [(r"^(LAUNCHER\s*=\s*).*$", r"\1 %s" % self.orig_launcher)])
if self.cleanup_token_symlink:
try:
os.remove(self.license_token)
self.log.debug("Symlink to license token %s removed", self.license_token)
except OSError as err:
raise EasyBuildError("Failed to remove %s: %s", self.license_token, err)
def configure_step(self):
"""Custom configuration procedure for Bazel."""
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
gcc_ver = get_software_version('GCCcore') or get_software_version('GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if binutils_root and gcc_root:
res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError("Failed to pinpoint location of GCC include files: %s", res)
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed')
if not os.path.exists(gcc_lib_inc_fixed):
raise EasyBuildError("Derived directory %s does not exist", gcc_lib_inc_fixed)
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc)
# replace hardcoded paths in CROSSTOOL
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc),
(r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_fixed),
(r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
apply_regex_substitutions(os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
# replace hardcoded paths in (unix_)cc_configure.bzl
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
]
for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
filepath = os.path.join('tools', 'cpp', conf_bzl)
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
else:
self.log.info("Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
# enable building in parallel
env.setvar('EXTRA_BAZEL_ARGS', '--jobs=%d' % self.cfg['parallel'])
def configure_step(self):
"""Configure Qt using interactive `configure` script."""
self.cfg.update('configopts', '-release')
platform = None
comp_fam = self.toolchain.comp_family()
if self.cfg['platform']:
platform = self.cfg['platform']
# if no platform is specified, try to derive it based on compiler in toolchain
elif comp_fam in [toolchain.GCC]: #@UndefinedVariable
platform = 'linux-g++-64'
elif comp_fam in [toolchain.INTELCOMP]: #@UndefinedVariable
if LooseVersion(self.version) >= LooseVersion('4'):
platform = 'linux-icc-64'
else:
platform = 'linux-icc'
# fix -fPIC flag (-KPIC is not correct for recent Intel compilers)
qmake_conf = os.path.join('mkspecs', platform, 'qmake.conf')
apply_regex_substitutions(qmake_conf, [('-KPIC', '-fPIC')])
if platform:
self.cfg.update('configopts', "-platform %s" % platform)
else:
raise EasyBuildError("Don't know which platform to set based on compiler family.")
# configure Qt such that xmlpatterns is also installed
# -xmlpatterns is not a known configure option for Qt 5.x, but there xmlpatterns support is enabled by default
if LooseVersion(self.version) >= LooseVersion('4') and LooseVersion(self.version) < LooseVersion('5'):
self.cfg.update('configopts', '-xmlpatterns')
cmd = "%s ./configure -prefix %s %s" % (self.cfg['preconfigopts'], self.installdir, self.cfg['configopts'])
qa = {
"Type 'o' if you want to use the Open Source Edition.": 'o',
"Do you accept the terms of either license?": 'yes',
"Which edition of Qt do you want to use?": 'o',
}
no_qa = [
"for .*pro",
r"%s.*" % os.getenv('CXX', '').replace('+', '\\+'), # need to escape + in 'g++'
"Reading .*",
"WARNING .*",
"Project MESSAGE:.*",
"rm -f .*",
'Creating qmake...',
'Checking for .*...',
]
run_cmd_qa(cmd, qa, no_qa=no_qa, log_all=True, simple=True, maxhits=120)
def configure_step(self, *args, **kwargs):
"""Configure build using 'make config' (only for recent versions (>= v5))."""
if LooseVersion(self.version) >= LooseVersion("5"):
if build_option('rpath'):
# patch Makefile to tell CMake not to wipe the RPATHs we inject...
apply_regex_substitutions('Makefile', [(r'^(CONFIG_FLAGS\s*=\s*)', r'\1 -DCMAKE_SKIP_RPATH=ON ')])
cmd = "make %s config prefix=%s" % (self.cfg['configopts'], self.installdir)
run_cmd(cmd, log_all=True, simple=True)
if 'shared=1' in self.cfg['configopts']:
self.lib_exts.append('so')
else:
self.lib_exts.append('a')
def post_install_step(self):
"""Custom post install step for IMPI, fix broken env scripts after moving installed files."""
super(EB_impi, self).post_install_step()
impiver = LooseVersion(self.version)
if impiver == LooseVersion('4.1.1.036') or impiver >= LooseVersion('5.0.1.035'):
if impiver >= LooseVersion('2018.0.128'):
script_paths = [os.path.join('intel64', 'bin')]
else:
script_paths = [os.path.join('intel64', 'bin'), os.path.join('mic', 'bin')]
# fix broken env scripts after the move
regex_subs = [(r"^setenv I_MPI_ROOT.*", r"setenv I_MPI_ROOT %s" % self.installdir)]
for script in [os.path.join(script_path,'mpivars.csh') for script_path in script_paths]:
apply_regex_substitutions(os.path.join(self.installdir, script), regex_subs)
regex_subs = [(r"^I_MPI_ROOT=.*", r"I_MPI_ROOT=%s; export I_MPI_ROOT" % self.installdir)]
for script in [os.path.join(script_path,'mpivars.sh') for script_path in script_paths]:
apply_regex_substitutions(os.path.join(self.installdir, script), regex_subs)
def build_step(self):
"""Run multiple times for different sources"""
# BCALM is a git submodule of BWISE but we use it as a dependency because
# it also has submodules and it's from a different developer
bcalm = get_software_root('BCALM')
if not bcalm:
raise EasyBuildError("BWISE needs BCALM to work")
makefiles_fixes = [
(r'^CC=.*$', 'CC=$(CXX)'),
(r'^CFLAGS=.*$', 'CFLAGS:=$(CFLAGS)'),
(r'^LDFLAGS=.*$', 'LDFLAGS:=$(LDFLAGS) -fopenmp')
]
def find_build_subdir(pattern):
"""Changes to the sub directory that matches the given pattern"""
subdir = glob.glob(os.path.join(self.builddir, pattern))
if subdir:
change_dir(subdir[0])
apply_regex_substitutions('makefile', makefiles_fixes)
super(EB_BWISE, self).build_step()
return subdir[0]
else:
raise EasyBuildError("Could not find a subdirectory matching the pattern %s", pattern)
# BWISE has 3 independant parts, we build them one by one
# first BWISE itself
subdir = find_build_subdir(os.path.join('BWISE-*', 'src'))
apply_regex_substitutions(os.path.join(subdir, '..', 'Bwise.py'),
[(r'^BWISE_MAIN = .*$', 'BWISE_MAIN = os.environ[\'EBROOTBWISE\']')])
# Onwards to BGREAT
subdir = find_build_subdir('BGREAT2-*')
copy_file(os.path.join(subdir, 'bgreat'), self.cfg['start_dir'])
# Finally, BTRIM
subdir = find_build_subdir('BTRIM-*')
copy_file(os.path.join(subdir, 'btrim'), self.cfg['start_dir'])
binaries = ['sequencesToNumbers', 'numbersFilter', 'path_counter', 'maximal_sr', 'simulator',
'path_to_kmer', 'K2000/*.py', 'K2000/*.sh']
self.cfg['files_to_copy'] = [(['bgreat', 'btrim', 'Bwise.py'] + ['src/%s' % x for x in binaries], 'bin'),
'data']
def configure_step(self):
"""Custom configuration procedure for QScintilla."""
srcdir = os.path.join(self.cfg['start_dir'], 'Qt4Qt5')
try:
os.chdir(srcdir)
except OSError as err:
raise EasyBuildError("Failed to change to %s: %s", srcdir, err)
# replace template values for install locations in qscintilla.pro configuration file
regex_subs = [
(r'\$\$\[QT_HOST_DATA\]', os.path.join(self.installdir, 'data')),
(r'\$\$\[QT_INSTALL_DATA\]', os.path.join(self.installdir, 'data')),
(r'\$\$\[QT_INSTALL_HEADERS\]', os.path.join(self.installdir, 'include')),
(r'\$\$\[QT_INSTALL_LIBS\]', os.path.join(self.installdir, 'lib')),
(r'\$\$\[QT_INSTALL_TRANSLATIONS\]', os.path.join(self.installdir, 'trans')),
]
apply_regex_substitutions('qscintilla.pro', regex_subs)
run_cmd("qmake qscintilla.pro")
def disable_sanitizer_tests(self):
"""Disable the tests of all the sanitizers by removing the test directories from the build system"""
if LooseVersion(self.version) < LooseVersion('3.6'):
# for Clang 3.5 and lower, the tests are scattered over several CMakeLists.
# We loop over them, and patch out the rule that adds the sanitizers tests to the testsuite
patchfiles = ['lib/asan', 'lib/dfsan', 'lib/lsan', 'lib/msan', 'lib/tsan', 'lib/ubsan']
for patchfile in patchfiles:
cmakelists = os.path.join(self.llvm_src_dir, 'projects/compiler-rt', patchfile, 'CMakeLists.txt')
if os.path.exists(cmakelists):
regex_subs = [('.*add_subdirectory\(lit_tests\).*', '')]
apply_regex_substitutions(cmakelists, regex_subs)
# There is a common part seperate for the specific saniters, we disable all the common tests
cmakelists = os.path.join('projects', 'compiler-rt', 'lib', 'sanitizer_common', 'CMakeLists.txt')
regex_subs = [('.*add_subdirectory\(tests\).*', '')]
apply_regex_substitutions(cmakelists, regex_subs)
else:
# In Clang 3.6, the sanitizer tests are grouped together in one CMakeLists
# We patch out adding the subdirectories with the sanitizer tests
cmakelists_tests = os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt', 'test', 'CMakeLists.txt')
regex_subs = []
if LooseVersion(self.version) >= LooseVersion('5.0'):
regex_subs.append((r'compiler_rt_test_runtime.*san.*', ''))
else:
regex_subs.append((r'add_subdirectory\((.*san|sanitizer_common)\)', ''))
apply_regex_substitutions(cmakelists_tests, regex_subs)
def disable_lto_mpfr_old_gcc(self, objdir):
"""
# if GCC version used to build stage 1 is too old, build MPFR without LTO in stage 1
# required for e.g. CentOS 6, cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/6374
"""
self.log.info("Checking whether we are trying to build a recent MPFR with an old GCC...")
# try to figure out MPFR version being built
mpfr_ver = '0.0'
mpfr_dirs = glob.glob(os.path.join(self.builddir, 'mpfr-*'))
if len(mpfr_dirs) == 1:
mpfr_dir = mpfr_dirs[0]
res = re.search('(?P<mpfr_ver>[0-9.]+)$', mpfr_dir)
if res:
mpfr_ver = res.group('mpfr_ver')
self.log.info("Found MPFR version %s (based name of MPFR source dir: %s)", mpfr_ver, mpfr_dir)
else:
self.log.warning("Failed to determine MPFR version from '%s', assuming v%s", mpfr_dir, mpfr_ver)
else:
self.log.warning("Failed to isolate MPFR source dir to determine MPFR version, assuming v%s", mpfr_ver)
# for MPFR v4.x & newer, we need a recent GCC that supports -flto
if LooseVersion(mpfr_ver) >= LooseVersion('4.0'):
# check GCC version being used
# GCC 4.5 is required for -flto (cfr. https://gcc.gnu.org/gcc-4.5/changes.html)
gcc_ver = get_gcc_version()
min_gcc_ver_lto = '4.5'
if gcc_ver is None:
self.log.warning("Failed to determine GCC version, assuming it's recent enough...")
elif LooseVersion(gcc_ver) < LooseVersion(min_gcc_ver_lto):
self.log.info("Configuring MPFR to build without LTO in stage 1 (GCC %s is too old: < %s)!",
gcc_ver, min_gcc_ver_lto)
# patch GCC's Makefile to inject --disable-lto when building MPFR
stage1_makefile = os.path.join(objdir, 'Makefile')
regex_subs = [(r'(--with-gmp-lib=\$\$r/\$\(HOST_SUBDIR\)/gmp/.libs) \\', r'\1 --disable-lto \\')]
apply_regex_substitutions(stage1_makefile, regex_subs)
else:
self.log.info("GCC %s (>= %s) is OK for building MPFR in stage 1 with LTO enabled",
gcc_ver, min_gcc_ver_lto)
def configure_step(self):
"""Custom configuration procedure for TINKER."""
# make sure FFTW is available
if get_software_root('FFTW') is None:
raise EasyBuildError("FFTW dependency is not available.")
os_dirs = {
LINUX: 'linux',
DARWIN: 'macosx',
}
os_type = get_os_type()
os_dir = os_dirs.get(os_type)
if os_dir is None:
raise EasyBuildError("Failed to determine OS directory for %s (known: %s)", os_type, os_dirs)
comp_dirs = {
toolchain.INTELCOMP: 'intel',
toolchain.GCC: 'gfortran',
}
comp_fam = self.toolchain.comp_family()
comp_dir = comp_dirs.get(comp_fam)
if comp_dir is None:
raise EasyBuildError("Failed to determine compiler directory for %s (known: %s)", comp_fam, comp_dirs)
self.build_subdir = os.path.join(os_dir, comp_dir)
self.log.info("Using build scripts from %s subdirectory" % self.build_subdir)
# patch 'link.make' script to use FFTW provided via EasyBuild
link_make_fp = os.path.join(self.cfg['start_dir'], self.build_subdir, 'link.make')
regex_subs = [(r"libfftw3_threads.a libfftw3.a", r"-L$EBROOTFFTW/lib -lfftw3_threads -lfftw3")]
apply_regex_substitutions(link_make_fp, regex_subs)
# patch *.make files to get rid of hardcoded -openmp flag,
# which doesn't work anymore with recent Intel compilers
if comp_fam == toolchain.INTELCOMP:
make_fps = glob.glob(os.path.join(self.cfg['start_dir'], self.build_subdir, '*.make'))
regex_subs = [(r'-openmp', r'-fopenmp')]
for make_fp in make_fps:
apply_regex_substitutions(make_fps, regex_subs)
def test_apply_regex_substitutions(self):
"""Test apply_regex_substitutions function."""
testfile = os.path.join(self.test_prefix, 'test.txt')
testtxt = '\n'.join([
"CC = gcc",
"CFLAGS = -O3 -g",
"FC = gfortran",
"FFLAGS = -O3 -g -ffixed-form",
])
ft.write_file(testfile, testtxt)
regex_subs = [
(r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"),
(r"^(FC\s*=\s*).*$", r"\1${FC}"),
(r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"),
]
ft.apply_regex_substitutions(testfile, regex_subs)
expected_testtxt = '\n'.join([
"CC = ${CC}",
"CFLAGS = -O2",
"FC = ${FC}",
"FFLAGS = -O2 -ffixed-form",
])
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, expected_testtxt)
# passing empty list of substitions is a no-op
ft.write_file(testfile, testtxt)
ft.apply_regex_substitutions(testfile, [])
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, testtxt)
# clean error on non-existing file
error_pat = "Failed to patch .*/nosuchfile.txt: .*No such file or directory"
path = os.path.join(self.test_prefix, 'nosuchfile.txt')
self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, path, regex_subs)
def install_step(self):
"""Custom install procedure for VEP."""
# patch INSTALL.pl script to use https:// rather than ftp://
apply_regex_substitutions('INSTALL.pl', [('ftp://', 'https://')])
# update PERL5LIB so tests can run (done automatically by INSTALL.pl unless --NO_TEST is used)
perl_majver = get_major_perl_version()
perllib_envvar = 'PERL%sLIB' % perl_majver
perllib = os.getenv(perllib_envvar, '')
api_mods_dir = os.path.join(self.installdir, self.api_mods_subdir)
self.log.info("Adding %s to $%s (%s)", api_mods_dir, perllib_envvar, perllib)
env.setvar(perllib_envvar, '%s:%s' % (api_mods_dir, perllib))
# see https://www.ensembl.org/info/docs/tools/vep/script/vep_download.html#installer
cmd = ' '.join([
self.cfg['preinstallopts'],
'perl',
'INSTALL.pl',
# don't try to install optional Bio::DB::HTS (can be provided as an extension instead)
'--NO_HTSLIB',
# a: API, f: FASTA
# not included:
# c: cache, should be downloaded by user
# l: Bio::DB::HTS, should be provided via EasyBuild
# p: plugins
'--AUTO af',
# install all species
'--SPECIES all',
# don't update VEP during installation
'--NO_UPDATE',
# location to install Perl API modules into
'--DESTDIR ' + api_mods_dir,
self.cfg['installopts'],
])
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def configure_step(self):
"""Custom configuration procedure for Molpro: use 'configure -batch'."""
if not os.path.isfile(self.license_token):
if self.cfg['license_file'] is not None and os.path.isfile(self.cfg['license_file']):
# put symlink in place to specified license file in $HOME/.molpro/token
# other approaches (like defining $MOLPRO_KEY) don't seem to work
self.cleanup_token_symlink = True
mkdir(os.path.dirname(self.license_token))
symlink(self.cfg['license_file'], self.license_token)
self.log.debug("Symlinked %s to %s", self.cfg['license_file'], self.license_token)
else:
self.log.warning("No licence token found at either {0} or via 'license_file'".format(self.license_token))
# Only do the rest of the configuration if we're building from source
if not self.cfg['precompiled_binaries']:
# installation prefix
self.cfg.update('configopts', "-prefix %s" % self.installdir)
# compilers
# compilers & MPI
if self.toolchain.options.get('usempi', None):
self.cfg.update('configopts', "-%s -%s" % (os.environ['CC_SEQ'], os.environ['F90_SEQ']))
if 'MPI_INC_DIR' in os.environ:
self.cfg.update('configopts', "-mpp -mppbase %s" % os.environ['MPI_INC_DIR'])
else:
raise EasyBuildError("$MPI_INC_DIR not defined")
else:
self.cfg.update('configopts', "-%s -%s" % (os.environ['CC'], os.environ['F90']))
# BLAS/LAPACK
if 'BLAS_LIB_DIR' in os.environ:
self.cfg.update('configopts', "-blas -blaspath %s" % os.environ['BLAS_LIB_DIR'])
else:
raise EasyBuildError("$BLAS_LIB_DIR not defined")
if 'LAPACK_LIB_DIR' in os.environ:
self.cfg.update('configopts', "-lapack -lapackpath %s" % os.environ['LAPACK_LIB_DIR'])
else:
raise EasyBuildError("$LAPACK_LIB_DIR not defined")
# 32 vs 64 bit
if self.toolchain.options.get('32bit', None):
self.cfg.update('configopts', '-i4')
else:
self.cfg.update('configopts', '-i8')
run_cmd("./configure -batch %s" % self.cfg['configopts'])
cfgfile = os.path.join(self.cfg['start_dir'], 'CONFIG')
cfgtxt = read_file(cfgfile)
# determine original LAUNCHER value
launcher_regex = re.compile('^LAUNCHER=(.*)$', re.M)
res = launcher_regex.search(cfgtxt)
if res:
self.orig_launcher = res.group(1)
self.log.debug("Found original value for LAUNCHER: %s", self.orig_launcher)
else:
raise EasyBuildError("Failed to determine LAUNCHER value")
# determine full installation prefix
prefix_regex = re.compile('^PREFIX=(.*)$', re.M)
res = prefix_regex.search(cfgtxt)
if res:
self.full_prefix = res.group(1)
self.log.debug("Found full installation prefix: %s", self.full_prefix)
else:
raise EasyBuildError("Failed to determine full installation prefix")
# determine MPI launcher command that can be used during build/test
# obtain command with specific number of cores (required by mpi_cmd_for), then replace that number with '%n'
launcher = self.toolchain.mpi_cmd_for('%x', self.cfg['parallel'])
launcher = launcher.replace(' %s' % self.cfg['parallel'], ' %n')
# patch CONFIG file to change LAUNCHER definition, in order to avoid having to start mpd
apply_regex_substitutions(cfgfile, [(r"^(LAUNCHER\s*=\s*).*$", r"\1 %s" % launcher)])
# reread CONFIG and log contents
cfgtxt = read_file(cfgfile)
self.log.info("Contents of CONFIG file:\n%s", cfgtxt)
def install_step(self):
"""Custom install procedure for Trinity."""
version = LooseVersion(self.version)
if version > LooseVersion('2012') and version < LooseVersion('2012-10-05'):
self.inchworm()
self.chrysalis()
self.kmer()
self.butterfly()
bwapluginver = self.cfg['bwapluginver']
if bwapluginver:
self.trinityplugin('bwa-%s-patched_multi_map' % bwapluginver)
if self.cfg['RSEMmod']:
self.trinityplugin('RSEM-mod', cc=os.getenv('CXX'))
else:
self.jellyfish()
inchworm_flags = self.inchworm(run=False)
chrysalis_flags = self.chrysalis(run=False)
cc = os.getenv('CC')
cxx = os.getenv('CXX')
lib_flags = ""
for lib in ['ncurses', 'zlib']:
libroot = get_software_root(lib)
if libroot:
lib_flags += " -L%s/lib" % libroot
if version >= LooseVersion('2.0') and version < LooseVersion('3.0'):
regex_subs = [
(r'^( INCHWORM_CONFIGURE_FLAGS\s*=\s*).*$', r'\1%s' % inchworm_flags),
(r'^( CHRYSALIS_MAKE_FLAGS\s*=\s*).*$', r'\1%s' % chrysalis_flags),
]
else:
regex_subs = [
(r'^(INCHWORM_CONFIGURE_FLAGS\s*=\s*).*$', r'\1%s' % inchworm_flags),
(r'^(CHRYSALIS_MAKE_FLAGS\s*=\s*).*$', r'\1%s' % chrysalis_flags),
(r'(/rsem && \$\(MAKE\))\s*$',
r'\1 CC=%s CXX="%s %s" CFLAGS_EXTRA="%s"\n' % (cc, cxx, lib_flags, lib_flags)),
(r'(/fastool && \$\(MAKE\))\s*$',
r'\1 CC="%s -std=c99" CFLAGS="%s ${CFLAGS}"\n' % (cc, lib_flags)),
]
apply_regex_substitutions('Makefile', regex_subs)
trinity_compiler = None
comp_fam = self.toolchain.comp_family()
if comp_fam in [toolchain.INTELCOMP]:
trinity_compiler = "intel"
elif comp_fam in [toolchain.GCC]:
trinity_compiler = "gcc"
else:
raise EasyBuildError("Don't know how to set TRINITY_COMPILER for %s compiler", comp_fam)
explicit_make_args = ''
if version >= LooseVersion('2.0') and version < LooseVersion('3.0'):
explicit_make_args = 'all plugins'
cmd = "make TRINITY_COMPILER=%s %s" % (trinity_compiler, explicit_make_args)
run_cmd(cmd)
# butterfly is not included in standard build
self.butterfly()
# remove sample data if desired
if not self.cfg['withsampledata']:
try:
shutil.rmtree(os.path.join(self.cfg['start_dir'], 'sample_data'))
except OSError, err:
raise EasyBuildError("Failed to remove sample data: %s", err)
stdqa = {
r"Enter installation directory for executable files \[.*\]\n": os.path.join(self.installdir, 'bin'),
r"Enter installation directory for library files \[.*\]\n": os.path.join(self.installdir, 'lib'),
r"directory .* does not exist, try to create [Y]/n\n": '',
}
run_cmd_qa(cmd, qa=qa, std_qa=stdqa, log_all=True, simple=True)
else:
if os.path.isfile(self.license_token):
run_cmd("make tuning")
super(EB_Molpro, self).install_step()
# put original LAUNCHER definition back in place in bin/molpro that got installed,
# since the value used during installation point to temporary files
molpro_path = os.path.join(self.full_prefix, 'bin', 'molpro')
apply_regex_substitutions(molpro_path, [(r"^(LAUNCHER\s*=\s*).*$", r"\1 %s" % self.orig_launcher)])
if self.cleanup_token_symlink:
try:
os.remove(self.license_token)
self.log.debug("Symlink to license token %s removed", self.license_token)
except OSError, err:
raise EasyBuildError("Failed to remove %s: %s", self.license_token, err)
def make_module_req_guess(self):
"""Customize $PATH guesses for Molpro module."""
guesses = super(EB_Molpro, self).make_module_req_guess()
guesses.update({
'PATH': [os.path.join(os.path.basename(self.full_prefix), x) for x in ['bin', 'utilities']],
})
return guesses
def test_step(self):
"""Run WPS test (requires large dataset to be downloaded). """
wpsdir = None
def run_wps_cmd(cmdname, mpi_cmd=True):
"""Run a WPS command, and check for success."""
cmd = os.path.join(wpsdir, "%s.exe" % cmdname)
if mpi_cmd:
if build_option('mpi_tests'):
cmd = self.toolchain.mpi_cmd_for(cmd, 1)
else:
self.log.info("Skipping MPI test for %s, since MPI tests are disabled", cmd)
return
(out, _) = run_cmd(cmd, log_all=True, simple=False)
re_success = re.compile("Successful completion of %s" % cmdname)
if not re_success.search(out):
raise EasyBuildError("%s.exe failed (pattern '%s' not found)?", cmdname, re_success.pattern)
if self.cfg['runtest']:
if not self.cfg['testdata']:
raise EasyBuildError("List of URLs for testdata not provided.")
wpsdir = os.path.join(self.builddir, self.wps_subdir)
try:
# create temporary directory
tmpdir = tempfile.mkdtemp()
change_dir(tmpdir)
# download data
testdata_paths = []
for testdata in self.cfg['testdata']:
path = self.obtain_file(testdata)
if not path:
raise EasyBuildError("Downloading file from %s failed?", testdata)
testdata_paths.append(path)
# unpack data
for path in testdata_paths:
extract_file(path, tmpdir)
namelist_file = os.path.join(tmpdir, 'namelist.wps')
# GEOGRID
# setup directories and files
if LooseVersion(self.version) < LooseVersion("4.0"):
geog_data_dir = "geog"
else:
geog_data_dir = "WPS_GEOG"
for dir_name in os.listdir(os.path.join(tmpdir, geog_data_dir)):
symlink(os.path.join(tmpdir, geog_data_dir, dir_name), os.path.join(tmpdir, dir_name))
# copy namelist.wps file and patch it for geogrid
copy_file(os.path.join(wpsdir, 'namelist.wps'), namelist_file)
regex_subs = [(r"^(\s*geog_data_path\s*=\s*).*$", r"\1 '%s'" % tmpdir)]
apply_regex_substitutions(namelist_file, regex_subs)
# GEOGRID.TBL
geogrid_dir = os.path.join(tmpdir, 'geogrid')
mkdir(geogrid_dir)
symlink(os.path.join(wpsdir, 'geogrid', 'GEOGRID.TBL.ARW'),
os.path.join(geogrid_dir, 'GEOGRID.TBL'))
# run geogrid.exe
run_wps_cmd("geogrid")
# UNGRIB
# determine start and end time stamps of grib files
grib_file_prefix = "fnl_"
k = len(grib_file_prefix)
fs = [f for f in sorted(os.listdir('.')) if f.startswith(grib_file_prefix)]
start = "%s:00:00" % fs[0][k:]
end = "%s:00:00" % fs[-1][k:]
# copy namelist.wps file and patch it for ungrib
copy_file(os.path.join(wpsdir, 'namelist.wps'), namelist_file)
regex_subs = [
(r"^(\s*start_date\s*=\s*).*$", r"\1 '%s','%s'," % (start, start)),
(r"^(\s*end_date\s*=\s*).*$", r"\1 '%s','%s'," % (end, end)),
]
apply_regex_substitutions(namelist_file, regex_subs)
# copy correct Vtable
vtable_dir = os.path.join(wpsdir, 'ungrib', 'Variable_Tables')
if os.path.exists(os.path.join(vtable_dir, 'Vtable.ARW')):
copy_file(os.path.join(vtable_dir, 'Vtable.ARW'), os.path.join(tmpdir, 'Vtable'))
elif os.path.exists(os.path.join(vtable_dir, 'Vtable.ARW.UPP')):
copy_file(os.path.join(vtable_dir, 'Vtable.ARW.UPP'), os.path.join(tmpdir, 'Vtable'))
else:
raise EasyBuildError("Could not find Vtable file to use for testing ungrib")
# run link_grib.csh script
cmd = "%s %s*" % (os.path.join(wpsdir, "link_grib.csh"), grib_file_prefix)
run_cmd(cmd, log_all=True, simple=True)
# run ungrib.exe
run_wps_cmd("ungrib", mpi_cmd=False)
# METGRID.TBL
metgrid_dir = os.path.join(tmpdir, 'metgrid')
mkdir(metgrid_dir)
symlink(os.path.join(wpsdir, 'metgrid', 'METGRID.TBL.ARW'),
os.path.join(metgrid_dir, 'METGRID.TBL'))
# run metgrid.exe
run_wps_cmd('metgrid')
# clean up
change_dir(self.builddir)
remove_dir(tmpdir)
except OSError as err:
raise EasyBuildError("Failed to run WPS test: %s", err)
def configure_step(self):
"""
Configure for GCC build:
- prepare extra source dirs (GMP, MPFR, MPC, ...)
- create obj dir to build in (GCC doesn't like to be built in source dir)
- add configure and make options, according to .eb spec file
- decide whether or not to do a staged build (which is required to enable PPL/CLooG support)
- set platform_lib based on config.guess output
"""
sysroot = build_option('sysroot')
if sysroot:
# based on changes made to GCC in Gentoo Prefix
# https://gitweb.gentoo.org/repo/gentoo.git/tree/profiles/features/prefix/standalone/profile.bashrc
# add --with-sysroot configure option, to instruct GCC to consider
# value set for EasyBuild's --sysroot configuration option as the root filesystem of the operating system
# (see https://gcc.gnu.org/install/configure.html)
self.cfg.update('configopts', '--with-sysroot=%s' % sysroot)
# avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c
apply_regex_substitutions(os.path.join('gcc', 'gcc.c'),
[('--sysroot=%R', '')])
# prefix dynamic linkers with sysroot
# this patches lines like:
# #define GLIBC_DYNAMIC_LINKER64 "/lib64/ld-linux-x86-64.so.2"
gcc_config_headers = glob.glob(
os.path.join('gcc', 'config', '*', '*linux*.h'))
regex_subs = [('(_DYNAMIC_LINKER.*[":])/lib',
r'\1%s/lib' % sysroot)]
for gcc_config_header in gcc_config_headers:
apply_regex_substitutions(gcc_config_header, regex_subs)
# self.configopts will be reused in a 3-staged build,
# configopts is only used in first configure
self.configopts = self.cfg['configopts']
# I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them
stage1_info = self.prep_extra_src_dirs("stage1")
configopts = stage1_info['configopts']
# II) update config options
# enable specified language support
if self.cfg['languages']:
self.configopts += " --enable-languages=%s" % ','.join(
self.cfg['languages'])
if self.cfg['withnvptx']:
if self.iter_idx == 0:
self.configopts += " --without-cuda-driver"
self.configopts += " --enable-offload-targets=nvptx-none"
else:
# register installed GCC as compiler to use nvptx
path = "%s/bin:%s" % (self.installdir, os.getenv('PATH'))
env.setvar('PATH', path)
ld_lib_path = "%(dir)s/lib64:%(dir)s/lib:%(val)s" % {
'dir': self.installdir,
'val': os.getenv('LD_LIBRARY_PATH')
}
env.setvar('LD_LIBRARY_PATH', ld_lib_path)
extra_source = {1: "nvptx-tools", 2: "newlib"}[self.iter_idx]
extra_source_dirs = glob.glob(
os.path.join(self.builddir, '%s-*' % extra_source))
if len(extra_source_dirs) != 1:
raise EasyBuildError("Failed to isolate %s source dir" %
extra_source)
if self.iter_idx == 1:
# compile nvptx-tools
change_dir(extra_source_dirs[0])
else: # self.iter_idx == 2
# compile nvptx target compiler
symlink(os.path.join(extra_source_dirs[0], 'newlib'),
'newlib')
self.create_dir("build-nvptx-gcc")
self.cfg.update('configopts', self.configopts)
self.cfg.update(
'configopts',
"--with-build-time-tools=%s/nvptx-none/bin" %
self.installdir)
self.cfg.update('configopts', "--target=nvptx-none")
host_type = self.determine_build_and_host_type()[1]
self.cfg.update(
'configopts',
"--enable-as-accelerator-for=%s" % host_type)
self.cfg.update('configopts', "--disable-sjlj-exceptions")
self.cfg.update('configopts',
"--enable-newlib-io-long-long")
self.cfg['configure_cmd_prefix'] = '../'
return super(EB_GCC, self).configure_step()
# enable building of libiberty, if desired
if self.cfg['withlibiberty']:
self.configopts += " --enable-install-libiberty"
# enable link-time-optimization (LTO) support, if desired
if self.cfg['withlto']:
self.configopts += " --enable-lto"
else:
self.configopts += " --disable-lto"
# configure for a release build
self.configopts += " --enable-checking=release "
# enable multilib: allow both 32 and 64 bit
if self.cfg['multilib']:
glibc_32bit = [
"glibc.i686", # Fedora, RedHat-based
"glibc.ppc", # "" on Power
"libc6-dev-i386", # Debian-based
"gcc-c++-32bit", # OpenSuSE, SLES
]
if not any([check_os_dependency(dep) for dep in glibc_32bit]):
raise EasyBuildError(
"Using multilib requires 32-bit glibc (install one of %s, depending on your OS)",
', '.join(glibc_32bit))
self.configopts += " --enable-multilib --with-multilib-list=m32,m64"
else:
self.configopts += " --disable-multilib"
# build both static and dynamic libraries (???)
self.configopts += " --enable-shared=yes --enable-static=yes "
# use POSIX threads
self.configopts += " --enable-threads=posix "
# enable plugin support
self.configopts += " --enable-plugins "
# use GOLD as default linker
if self.cfg['use_gold_linker']:
self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold"
else:
self.configopts += " --enable-gold --enable-ld=default"
# enable bootstrap build for self-containment (unless for staged build)
if not self.stagedbuild:
configopts += " --enable-bootstrap"
else:
configopts += " --disable-bootstrap"
if self.stagedbuild:
#
# STAGE 1: configure GCC build that will be used to build PPL/CLooG
#
self.log.info(
"Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support..."
)
self.stage1installdir = os.path.join(self.builddir,
'GCC_stage1_eb')
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {
'p': self.stage1installdir
}
else:
# unstaged build, so just run standard configure/make/make install
# set prefixes
self.log.info("Performing regular GCC build...")
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {
'p': self.installdir
}
# prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config
# only do this when multilib is not enabled
if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']:
cfgfile = 'gcc/config/i386/t-linux64'
multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32"
self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'",
cfgfile, multilib_osdirnames)
write_file(cfgfile, multilib_osdirnames, append=True)
elif self.cfg['multilib']:
self.log.info(
"Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled"
)
# III) create obj dir to build in, and change to it
# GCC doesn't like to be built in the source dir
if self.stagedbuild:
objdir = self.create_dir("stage1_obj")
self.stage1prefix = objdir
else:
objdir = self.create_dir("obj")
# IV) actual configure, but not on default path
cmd = "../configure %s %s" % (self.configopts, configopts)
self.run_configure_cmd(cmd)
self.disable_lto_mpfr_old_gcc(objdir)
def configure_step(self):
"""Configure build:
- set some magic environment variables
- run configure script
- adjust configure.wrf file if needed
"""
# define $NETCDF* for netCDF dependency (used when creating WRF module file)
set_netcdf_env_vars(self.log)
# HDF5 (optional) dependency
hdf5 = get_software_root('HDF5')
if hdf5:
# check if this is parallel HDF5
phdf5_bins = ['h5pcc', 'ph5diff']
parallel_hdf5 = True
for f in phdf5_bins:
if not os.path.exists(os.path.join(hdf5, 'bin', f)):
parallel_hdf5 = False
break
if not (hdf5 or parallel_hdf5):
raise EasyBuildError("Parallel HDF5 module not loaded?")
else:
env.setvar('PHDF5', hdf5)
else:
self.log.info("HDF5 module not loaded, assuming that's OK...")
# JasPer dependency check + setting env vars
jasper = get_software_root('JasPer')
if jasper:
jasperlibdir = os.path.join(jasper, "lib")
env.setvar('JASPERINC', os.path.join(jasper, "include"))
env.setvar('JASPERLIB', jasperlibdir)
else:
if os.getenv('JASPERINC') or os.getenv('JASPERLIB'):
raise EasyBuildError(
"JasPer module not loaded, but JASPERINC and/or JASPERLIB still set?"
)
else:
self.log.info(
"JasPer module not loaded, assuming that's OK...")
# enable support for large file support in netCDF
env.setvar('WRFIO_NCD_LARGE_FILE_SUPPORT', '1')
# patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join("arch", "Config_new.pl"))
# determine build type option to look for
build_type_option = None
self.comp_fam = self.toolchain.comp_family()
if self.comp_fam == toolchain.INTELCOMP: #@UndefinedVariable
build_type_option = "Linux x86_64 i486 i586 i686, ifort compiler with icc"
elif self.comp_fam == toolchain.GCC: #@UndefinedVariable
build_type_option = "x86_64 Linux, gfortran compiler with gcc"
else:
raise EasyBuildError(
"Don't know how to figure out build type to select.")
# fetch selected build type (and make sure it makes sense)
known_build_types = ['serial', 'smpar', 'dmpar', 'dm+sm']
self.parallel_build_types = ["dmpar", "smpar", "dm+sm"]
bt = self.cfg['buildtype']
if not bt in known_build_types:
raise EasyBuildError(
"Unknown build type: '%s'. Supported build types: %s", bt,
known_build_types)
# fetch option number based on build type option and selected build type
build_type_question = "\s*(?P<nr>[0-9]+).\s*%s\s*\(%s\)" % (
build_type_option, bt)
# run configure script
cmd = "./configure"
qa = {
# named group in match will be used to construct answer
"Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]:":
"1",
"Compile for nesting? (0=no nesting, 1=basic, 2=preset moves, 3=vortex following) [default 0]:":
"0"
}
no_qa = [
"testing for fseeko and fseeko64",
r"If you wish to change the default options, edit the file:[\s\n]*arch/configure_new.defaults"
]
std_qa = {
# named group in match will be used to construct answer
r"%s.*\n(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question:
"%(nr)s",
}
run_cmd_qa(cmd,
qa,
no_qa=no_qa,
std_qa=std_qa,
log_all=True,
simple=True)
cfgfile = 'configure.wrf'
# make sure correct compilers are being used
comps = {
'SCC': os.getenv('CC'),
'SFC': os.getenv('F90'),
'CCOMP': os.getenv('CC'),
'DM_FC': os.getenv('MPIF90'),
'DM_CC': "%s -DMPI2_SUPPORT" % os.getenv('MPICC'),
}
regex_subs = [(r"^(%s\s*=\s*).*$" % k, r"\1 %s" % v)
for (k, v) in comps.items()]
apply_regex_substitutions(cfgfile, regex_subs)
# rewrite optimization options if desired
if self.cfg['rewriteopts']:
# replace default -O3 option in configure.wrf with CFLAGS/FFLAGS from environment
self.log.info("Rewriting optimization options in %s" % cfgfile)
# set extra flags for Intel compilers
# see http://software.intel.com/en-us/forums/showthread.php?t=72109&p=1#146748
if self.comp_fam == toolchain.INTELCOMP: #@UndefinedVariable
# -O3 -heap-arrays is required to resolve compilation error
for envvar in ['CFLAGS', 'FFLAGS']:
val = os.getenv(envvar)
if '-O3' in val:
env.setvar(envvar, '%s -heap-arrays' % val)
self.log.info("Updated %s to '%s'" %
(envvar, os.getenv(envvar)))
# replace -O3 with desired optimization options
regex_subs = [
(r"^(FCOPTIM.*)(\s-O3)(\s.*)$",
r"\1 %s \3" % os.getenv('FFLAGS')),
(r"^(CFLAGS_LOCAL.*)(\s-O3)(\s.*)$",
r"\1 %s \3" % os.getenv('CFLAGS')),
]
apply_regex_substitutions(cfgfile, regex_subs)
def configure_step(self):
"""Custom configure procedure for PyTorch."""
super(EB_PyTorch, self).configure_step()
# Gather default options. Will be checked against (and can be overwritten by) custom_opts
options = [
'PYTORCH_BUILD_VERSION=' + self.version, 'PYTORCH_BUILD_NUMBER=1'
]
# enable verbose mode when --debug is used (to show compiler commands)
if build_option('debug'):
options.append('VERBOSE=1')
# Restrict parallelism
options.append('MAX_JOBS=%s' % self.cfg['parallel'])
# BLAS Interface
if get_software_root('imkl'):
options.append('BLAS=MKL')
options.append('INTEL_MKL_DIR=$MKLROOT')
elif LooseVersion(self.version) >= LooseVersion(
'1.9.0') and get_software_root('BLIS'):
options.append('BLAS=BLIS')
options.append('BLIS_HOME=' + get_software_root('BLIS'))
options.append('USE_MKLDNN_CBLAS=ON')
elif get_software_root('OpenBLAS'):
# This is what PyTorch defaults to if no MKL is found.
# Make this explicit here to avoid it finding MKL from the system
options.append('BLAS=Eigen')
# Still need to set a BLAS lib to use.
# Valid choices: mkl/open/goto/acml/atlas/accelerate/veclib/generic (+blis for 1.9+)
options.append('WITH_BLAS=open')
# Make sure this option is actually passed to CMake
apply_regex_substitutions(
os.path.join('tools', 'setup_helpers', 'cmake.py'),
[("'BLAS',", "'BLAS', 'WITH_BLAS',")])
else:
raise EasyBuildError(
"Did not find a supported BLAS in dependencies. Don't know which BLAS lib to use"
)
available_dependency_options = EB_PyTorch.get_dependency_options_for_version(
self.version)
dependency_names = set(dep['name'] for dep in self.cfg.dependencies())
not_used_dep_names = []
for enable_opt, dep_name in available_dependency_options:
if dep_name is None:
continue
if dep_name in dependency_names:
options.append(enable_opt)
else:
not_used_dep_names.append(dep_name)
self.log.info(
'Did not enable options for the following dependencies as they are not used in the EC: %s',
not_used_dep_names)
# Use Infiniband by default
# you can disable this by including 'USE_IBVERBS=0' in 'custom_opts' in the easyconfig file
options.append('USE_IBVERBS=1')
if get_software_root('CUDA'):
options.append('USE_CUDA=1')
cudnn_root = get_software_root('cuDNN')
if cudnn_root:
options.append('CUDNN_LIB_DIR=' +
os.path.join(cudnn_root, 'lib64'))
options.append('CUDNN_INCLUDE_DIR=' +
os.path.join(cudnn_root, 'include'))
nccl_root = get_software_root('NCCL')
if nccl_root:
options.append('USE_SYSTEM_NCCL=1')
options.append('NCCL_INCLUDE_DIR=' +
os.path.join(nccl_root, 'include'))
# list of CUDA compute capabilities to use can be specifed in two ways (where (2) overrules (1)):
# (1) in the easyconfig file, via the custom cuda_compute_capabilities;
# (2) in the EasyBuild configuration, via --cuda-compute-capabilities configuration option;
cuda_cc = build_option('cuda_compute_capabilities'
) or self.cfg['cuda_compute_capabilities']
if not cuda_cc:
raise EasyBuildError(
'List of CUDA compute capabilities must be specified, either via '
'cuda_compute_capabilities easyconfig parameter or via '
'--cuda-compute-capabilities')
self.log.info(
'Compiling with specified list of CUDA compute capabilities: %s',
', '.join(cuda_cc))
# This variable is also used at runtime (e.g. for tests) and if it is not set PyTorch will automatically
# determine the compute capability of a GPU in the system and use that which may fail tests if
# it is to new for the used nvcc
env.setvar('TORCH_CUDA_ARCH_LIST', ';'.join(cuda_cc))
else:
# Disable CUDA
options.append('USE_CUDA=0')
if get_cpu_architecture() == POWER:
# *NNPACK is not supported on Power, disable to avoid warnings
options.extend([
'USE_NNPACK=0', 'USE_QNNPACK=0', 'USE_PYTORCH_QNNPACK=0',
'USE_XNNPACK=0'
])
# Metal only supported on IOS which likely doesn't work with EB, so disabled
options.append('USE_METAL=0')
unique_options = self.cfg['custom_opts']
for option in options:
name = option.split('=')[
0] + '=' # Include the equals sign to avoid partial matches
if not any(opt.startswith(name) for opt in unique_options):
unique_options.append(option)
self.cfg.update('prebuildopts', ' '.join(unique_options) + ' ')
self.cfg.update('preinstallopts', ' '.join(unique_options) + ' ')
def configure_step(self):
"""Custom configuration procedure for TensorFlow."""
tmpdir = tempfile.mkdtemp(suffix='-bazel-configure')
# filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that
# might conflict with dependencies on the system and/or installed with EB. For example: protobuf
path_filter = self.cfg['path_filter']
if path_filter:
self.log.info(
"Filtering $CPATH and $LIBRARY_PATH with path filter %s",
path_filter)
for var in ['CPATH', 'LIBRARY_PATH']:
path = os.getenv(var).split(os.pathsep)
self.log.info("$%s old value was %s" % (var, path))
filtered_path = os.pathsep.join(
[p for fil in path_filter for p in path if fil not in p])
env.setvar(var, filtered_path)
wrapper_dir = os.path.join(tmpdir, 'bin')
use_wrapper = False
if self.toolchain.comp_family() == toolchain.INTELCOMP:
# put wrappers for Intel C/C++ compilers in place (required to make sure license server is found)
# cfr. https://github.com/bazelbuild/bazel/issues/663
for compiler in ('icc', 'icpc'):
self.write_wrapper(wrapper_dir, compiler, 'NOT-USED-WITH-ICC')
use_wrapper = True
use_mpi = self.toolchain.options.get('usempi', False)
impi_root = get_software_root('impi')
mpi_home = ''
if use_mpi and impi_root:
# put wrappers for Intel MPI compiler wrappers in place
# (required to make sure license server and I_MPI_ROOT are found)
for compiler in (os.getenv('MPICC'), os.getenv('MPICXX')):
self.write_wrapper(wrapper_dir, compiler,
os.getenv('I_MPI_ROOT'))
use_wrapper = True
# set correct value for MPI_HOME
mpi_home = os.path.join(impi_root, 'intel64')
self.log.debug("Derived value for MPI_HOME: %s", mpi_home)
if use_wrapper:
env.setvar('PATH',
os.pathsep.join([wrapper_dir,
os.getenv('PATH')]))
self.prepare_python()
self.handle_jemalloc()
cuda_root = get_software_root('CUDA')
cudnn_root = get_software_root('cuDNN')
opencl_root = get_software_root('OpenCL')
tensorrt_root = get_software_root('TensorRT')
nccl_root = get_software_root('NCCL')
config_env_vars = {
'CC_OPT_FLAGS': os.getenv('CXXFLAGS'),
'MPI_HOME': mpi_home,
'PYTHON_BIN_PATH': self.python_cmd,
'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir),
'TF_CUDA_CLANG': '0',
'TF_ENABLE_XLA': '0', # XLA JIT support
'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)],
'TF_NEED_GCP': '0', # Google Cloud Platform
'TF_NEED_GDR': '0',
'TF_NEED_HDFS': '0', # Hadoop File System
'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']],
'TF_NEED_MPI': ('0', '1')[bool(use_mpi)],
'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)],
'TF_NEED_OPENCL_SYCL': '0',
'TF_NEED_S3': '0', # Amazon S3 File System
'TF_NEED_VERBS': '0',
'TF_NEED_TENSORRT': ('0', '1')[bool(tensorrt_root)],
'TF_NEED_AWS': '0', # Amazon AWS Platform
'TF_NEED_KAFKA': '0', # Amazon Kafka Platform
}
if cuda_root:
config_env_vars.update({
'CUDA_TOOLKIT_PATH':
cuda_root,
'GCC_HOST_COMPILER_PATH':
which(os.getenv('CC')),
'TF_CUDA_COMPUTE_CAPABILITIES':
','.join(self.cfg['cuda_compute_capabilities']),
'TF_CUDA_VERSION':
get_software_version('CUDA'),
})
if cudnn_root:
config_env_vars.update({
'CUDNN_INSTALL_PATH':
cudnn_root,
'TF_CUDNN_VERSION':
get_software_version('cuDNN'),
})
else:
raise EasyBuildError(
"TensorFlow has a strict dependency on cuDNN if CUDA is enabled"
)
if nccl_root:
nccl_version = get_software_version('NCCL')
config_env_vars.update({
'NCCL_INSTALL_PATH': nccl_root,
})
else:
nccl_version = '1.3' # Use simple downloadable version
config_env_vars.update({
'TF_NCCL_VERSION': nccl_version,
})
for (key, val) in sorted(config_env_vars.items()):
env.setvar(key, val)
# patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel
regex_subs = [(r"(run_shell\(\['bazel')",
r"\1, '--output_base=%s', '--install_base=%s'" %
(tmpdir, os.path.join(tmpdir, 'inst_base')))]
apply_regex_substitutions('configure.py', regex_subs)
cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg[
'configopts']
run_cmd(cmd, log_all=True, simple=True)
def configure_step(self):
"""Configure OpenFOAM build by setting appropriate environment variables."""
# compiler & compiler flags
comp_fam = self.toolchain.comp_family()
extra_flags = ''
if comp_fam == toolchain.GCC: # @UndefinedVariable
if get_software_version('GCC') >= LooseVersion('4.8'):
# make sure non-gold version of ld is used, since OpenFOAM requires it
# see http://www.openfoam.org/mantisbt/view.php?id=685
extra_flags = '-fuse-ld=bfd'
# older versions of OpenFOAM-Extend require -fpermissive
if 'extend' in self.name.lower() and LooseVersion(self.version) < LooseVersion('2.0'):
extra_flags += ' -fpermissive'
if LooseVersion(self.version) < LooseVersion('3.0'):
extra_flags += ' -fno-delete-null-pointer-checks'
elif comp_fam == toolchain.INTELCOMP: # @UndefinedVariable
# make sure -no-prec-div is used with Intel compilers
extra_flags = '-no-prec-div'
for env_var in ['CFLAGS', 'CXXFLAGS']:
env.setvar(env_var, "%s %s" % (os.environ.get(env_var, ''), extra_flags))
# patch out hardcoding of WM_* environment variables
# for example, replace 'export WM_COMPILER=Gcc' with ': ${WM_COMPILER:=Gcc}; export WM_COMPILER'
for script in [os.path.join(self.builddir, self.openfoamdir, x) for x in ['etc/bashrc', 'etc/cshrc']]:
self.log.debug("Patching out hardcoded $WM_* env vars in %s", script)
# disable any third party stuff, we use EB controlled builds
regex_subs = [(r"^(setenv|export) WM_THIRD_PARTY_USE_.*[ =].*$", r"# \g<0>")]
# this does not work for OpenFOAM Extend lower than 2.0
if 'extend' not in self.name.lower() or LooseVersion(self.version) >= LooseVersion('2.0'):
key = "WM_PROJECT_VERSION"
regex_subs += [(r"^(setenv|export) %s=.*$" % key, r"export %s=%s #\g<0>" % (key, self.version))]
WM_env_var = ['WM_COMPILER', 'WM_MPLIB', 'WM_THIRD_PARTY_DIR']
# OpenFOAM >= 3.0.0 can use 64 bit integers
if 'extend' not in self.name.lower() and LooseVersion(self.version) >= LooseVersion('3.0'):
WM_env_var.append('WM_LABEL_SIZE')
for env_var in WM_env_var:
regex_subs.append((r"^(setenv|export) (?P<var>%s)[ =](?P<val>.*)$" % env_var,
r": ${\g<var>:=\g<val>}; export \g<var>"))
apply_regex_substitutions(script, regex_subs)
# inject compiler variables into wmake/rules files
ldirs = glob.glob(os.path.join(self.builddir, self.openfoamdir, 'wmake', 'rules', 'linux*'))
langs = ['c', 'c++']
# NOTE: we do not want to change the Debug rules files becuse
# that would change the cOPT/c++OPT values from their empty setting.
suffixes = ['', 'Opt']
wmake_rules_files = [os.path.join(ldir, lang + suff) for ldir in ldirs for lang in langs for suff in suffixes]
mpicc = os.environ['MPICC']
mpicxx = os.environ['MPICXX']
cc_seq = os.environ.get('CC_SEQ', os.environ['CC'])
cxx_seq = os.environ.get('CXX_SEQ', os.environ['CXX'])
if self.toolchain.mpi_family() == toolchain.OPENMPI:
# no -cc/-cxx flags supported in OpenMPI compiler wrappers
c_comp_cmd = 'OMPI_CC="%s" %s' % (cc_seq, mpicc)
cxx_comp_cmd = 'OMPI_CXX="%s" %s' % (cxx_seq, mpicxx)
else:
# -cc/-cxx should work for all MPICH-based MPIs (including Intel MPI)
c_comp_cmd = '%s -cc="%s"' % (mpicc, cc_seq)
cxx_comp_cmd = '%s -cxx="%s"' % (mpicxx, cxx_seq)
comp_vars = {
# specify MPI compiler wrappers and compiler commands + sequential compiler that should be used by them
'cc': c_comp_cmd,
'CC': cxx_comp_cmd,
'cOPT': os.environ['CFLAGS'],
'c++OPT': os.environ['CXXFLAGS'],
}
for wmake_rules_file in wmake_rules_files:
fullpath = os.path.join(self.builddir, self.openfoamdir, wmake_rules_file)
self.log.debug("Patching compiler variables in %s", fullpath)
regex_subs = []
for comp_var, newval in comp_vars.items():
regex_subs.append((r"^(%s\s*=\s*).*$" % re.escape(comp_var), r"\1%s" % newval))
apply_regex_substitutions(fullpath, regex_subs)
# enable verbose build for debug purposes
# starting with openfoam-extend 3.2, PS1 also needs to be set
env.setvar("FOAM_VERBOSE", '1')
# installation directory
env.setvar("FOAM_INST_DIR", self.installdir)
# third party directory
self.thrdpartydir = "ThirdParty-%s" % self.version
# only if third party stuff is actually installed
if os.path.exists(self.thrdpartydir):
os.symlink(os.path.join("..", self.thrdpartydir), self.thrdpartydir)
env.setvar("WM_THIRD_PARTY_DIR", os.path.join(self.installdir, self.thrdpartydir))
env.setvar("WM_COMPILER", self.wm_compiler)
env.setvar("WM_MPLIB", self.wm_mplib)
# Set Compile options according to build type
env.setvar("WM_COMPILE_OPTION", self.build_type)
# parallel build spec
env.setvar("WM_NCOMPPROCS", str(self.cfg['parallel']))
# OpenFOAM >= 3.0.0 can use 64 bit integers
if 'extend' not in self.name.lower() and LooseVersion(self.version) >= LooseVersion('3.0'):
if self.toolchain.options['i8']:
env.setvar("WM_LABEL_SIZE", '64')
else:
env.setvar("WM_LABEL_SIZE", '32')
# make sure lib/include dirs for dependencies are found
openfoam_extend_v3 = 'extend' in self.name.lower() and LooseVersion(self.version) >= LooseVersion('3.0')
if LooseVersion(self.version) < LooseVersion("2") or openfoam_extend_v3:
self.log.debug("List of deps: %s" % self.cfg.dependencies())
for dep in self.cfg.dependencies():
dep_name = dep['name'].upper(),
dep_root = get_software_root(dep['name'])
env.setvar("%s_SYSTEM" % dep_name, "1")
dep_vars = {
"%s_DIR": "%s",
"%s_BIN_DIR": "%s/bin",
"%s_LIB_DIR": "%s/lib",
"%s_INCLUDE_DIR": "%s/include",
}
for var, val in dep_vars.iteritems():
env.setvar(var % dep_name, val % dep_root)
else:
for depend in ['SCOTCH', 'METIS', 'CGAL', 'Paraview']:
dependloc = get_software_root(depend)
if dependloc:
if depend == 'CGAL' and get_software_root('Boost'):
env.setvar("CGAL_ROOT", dependloc)
env.setvar("BOOST_ROOT", get_software_root('Boost'))
else:
env.setvar("%s_ROOT" % depend.upper(), dependloc)
def configure_step(self):
"""Custom configuration procedure for TensorFlow."""
tmpdir = tempfile.mkdtemp(suffix='-bazel-configure')
# put wrapper for Intel C compiler in place (required to make sure license server is found)
# cfr. https://github.com/bazelbuild/bazel/issues/663
if self.toolchain.comp_family() == toolchain.INTELCOMP:
wrapper_dir = os.path.join(tmpdir, 'bin')
icc_wrapper_txt = INTEL_COMPILER_WRAPPER % {
'compiler_path':
which('icc'),
'cpath':
os.getenv('CPATH'),
'intel_license_file':
os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')),
'wrapper_dir':
wrapper_dir,
}
icc_wrapper = os.path.join(wrapper_dir, 'icc')
write_file(icc_wrapper, icc_wrapper_txt)
env.setvar(
'PATH',
':'.join([os.path.dirname(icc_wrapper),
os.getenv('PATH')]))
if self.dry_run:
self.dry_run_msg("Wrapper for 'icc' was put in place: %s",
icc_wrapper)
else:
adjust_permissions(icc_wrapper, stat.S_IXUSR)
self.log.info("Using wrapper script for 'icc': %s",
which('icc'))
self.prepare_python()
cuda_root = get_software_root('CUDA')
cudnn_root = get_software_root('cuDNN')
jemalloc_root = get_software_root('jemalloc')
opencl_root = get_software_root('OpenCL')
use_mpi = self.toolchain.options.get('usempi', False)
config_env_vars = {
'CC_OPT_FLAGS': os.getenv('CXXFLAGS'),
'MPI_HOME': '',
'PYTHON_BIN_PATH': self.python_cmd,
'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir),
'TF_CUDA_CLANG': '0',
'TF_ENABLE_XLA': '0', # XLA JIT support
'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)],
'TF_NEED_GCP': '0', # Google Cloud Platform
'TF_NEED_GDR': '0',
'TF_NEED_HDFS': '0', # Hadoop File System
'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']],
'TF_NEED_MPI': ('0', '1')[bool(use_mpi)],
'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)],
'TF_NEED_S3': '0', # Amazon S3 File System
'TF_NEED_VERBS': '0',
}
if cuda_root:
config_env_vars.update({
'CUDA_TOOLKIT_PATH':
cuda_root,
'GCC_HOST_COMPILER_PATH':
which(os.getenv('CC')),
'TF_CUDA_COMPUTE_CAPABILITIES':
','.join(self.cfg['cuda_compute_capabilities']),
'TF_CUDA_VERSION':
get_software_version('CUDA'),
})
if cudnn_root:
config_env_vars.update({
'CUDNN_INSTALL_PATH':
cudnn_root,
'TF_CUDNN_VERSION':
get_software_version('cuDNN'),
})
for (key, val) in sorted(config_env_vars.items()):
env.setvar(key, val)
# patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel
regex_subs = [(r"(run_shell\(\['bazel')",
r"\1, '--output_base=%s'" % tmpdir)]
apply_regex_substitutions('configure.py', regex_subs)
cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg[
'configopts']
run_cmd(cmd, log_all=True, simple=True)
def configure_step(self):
"""Custom configuration procedure for Quantum ESPRESSO."""
if self.toolchain.options.get('openmp', False) or self.cfg['hybrid']:
self.cfg.update('configopts', '--enable-openmp')
if not self.toolchain.options.get('usempi', None):
self.cfg.update('configopts', '--disable-parallel')
if not self.cfg['with_scalapack']:
self.cfg.update('configopts', '--without-scalapack')
repls = []
if self.toolchain.comp_family() in [toolchain.INTELCOMP]:
# set preprocessor command (-E to stop after preprocessing, -C to preserve comments)
cpp = "%s -E -C" % os.getenv('CC')
repls.append(('CPP', cpp, False))
env.setvar('CPP', cpp)
# also define $FCCPP, but do *not* include -C (comments should not be preserved when preprocessing Fortran)
env.setvar('FCCPP', "%s -E" % os.getenv('CC'))
super(EB_QuantumESPRESSO, self).configure_step()
# compose list of DFLAGS (flag, value, keep_stuff)
# for guidelines, see include/defs.h.README in sources
dflags = []
comp_fam_dflags = {
toolchain.INTELCOMP: '-D__INTEL',
toolchain.GCC: '-D__GFORTRAN -D__STD_F95',
}
dflags.append(comp_fam_dflags[self.toolchain.comp_family()])
if self.toolchain.options.get('openmp', False):
libfft = os.getenv('LIBFFT_MT')
else:
libfft = os.getenv('LIBFFT')
if libfft:
if "fftw3" in libfft:
dflags.append('-D__FFTW3')
else:
dflags.append('-D__FFTW')
env.setvar('FFTW_LIBS', libfft)
if get_software_root('ACML'):
dflags.append('-D__ACML')
if self.toolchain.options.get('usempi', None):
dflags.append('-D__MPI -D__PARA')
if self.toolchain.options.get('openmp', False) or self.cfg['hybrid']:
dflags.append(" -D__OPENMP")
if self.cfg['with_scalapack']:
dflags.append(" -D__SCALAPACK")
# always include -w to supress warnings
dflags.append('-w')
repls.append(('DFLAGS', ' '.join(dflags), False))
# complete C/Fortran compiler and LD flags
if self.toolchain.options.get('openmp', False) or self.cfg['hybrid']:
repls.append(('LDFLAGS', self.toolchain.get_flag('openmp'), True))
repls.append(
('(?:C|F90|F)FLAGS', self.toolchain.get_flag('openmp'), True))
# obtain library settings
libs = []
for lib in ['BLAS', 'LAPACK', 'FFT', 'SCALAPACK']:
if self.toolchain.options.get('openmp', False):
val = os.getenv('LIB%s_MT' % lib)
else:
val = os.getenv('LIB%s' % lib)
repls.append(('%s_LIBS' % lib, val, False))
libs.append(val)
libs = ' '.join(libs)
repls.append(('BLAS_LIBS_SWITCH', 'external', False))
repls.append(('LAPACK_LIBS_SWITCH', 'external', False))
repls.append(('LD_LIBS', os.getenv('LIBS'), False))
# check for external FoX
if get_software_root('FoX'):
self.log.debug(
"Found FoX external module, disabling libfox target in Makefile"
)
regex_subs = [
(r"(libfox: touch-dummy)\n.*",
r"\1\n\techo 'libfox: external module used' #"),
]
apply_regex_substitutions('Makefile', regex_subs)
self.log.debug("List of replacements to perform: %s" % repls)
if LooseVersion(self.version) >= LooseVersion("6"):
make_ext = '.inc'
else:
make_ext = '.sys'
# patch make.sys file
fn = os.path.join(self.cfg['start_dir'], 'make' + make_ext)
try:
for line in fileinput.input(fn, inplace=1, backup='.orig.eb'):
for (k, v, keep) in repls:
# need to use [ \t]* instead of \s*, because vars may be undefined as empty,
# and we don't want to include newlines
if keep:
line = re.sub(r"^(%s\s*=[ \t]*)(.*)$" % k,
r"\1\2 %s" % v, line)
else:
line = re.sub(r"^(%s\s*=[ \t]*).*$" % k, r"\1%s" % v,
line)
# fix preprocessing directives for .f90 files in make.sys if required
if self.toolchain.comp_family() in [toolchain.GCC]:
line = re.sub(
r"\$\(MPIF90\) \$\(F90FLAGS\) -c \$<",
"$(CPP) -C $(CPPFLAGS) $< -o $*.F90\n" +
"\t$(MPIF90) $(F90FLAGS) -c $*.F90 -o $*.o", line)
sys.stdout.write(line)
except IOError, err:
raise EasyBuildError("Failed to patch %s: %s", fn, err)
def configure_step(self):
"""Dedicated configure step for Mono: install Mono from RPM (if provided), then run configure."""
# install Mono from RPMs if provided (because we need Mono to build Mono)
if self.rpms:
# prepare path for installing RPMs in
monorpms_path = os.path.join(self.builddir, "monorpms")
try:
os.makedirs(os.path.join(monorpms_path, 'rpm'))
except OSError as err:
raise EasyBuildError(
"Failed to create directories for installing Mono RPMs in: %s",
err)
self.src = self.rpms
self.rebuildRPM = True
# rebuild RPMs to make them relocatable
Rpm.configure_step(self)
# prepare to install RPMs
self.log.debug(
"Initializing temporary RPM repository to install to...")
cmd = "rpm --initdb --dbpath /rpm --root %s" % monorpms_path
run_cmd(cmd, log_all=True, simple=True)
# install RPMs one by one
for rpm in self.src:
self.log.debug("Installing RPM %s ..." % rpm['name'])
if os.path.exists(rpm['path']):
cmd = ' '.join([
"rpm -i",
"--dbpath %(inst)s/rpm",
"--force",
"--relocate /=%(inst)s",
"--badreloc",
"--nodeps --nopost",
"%(rpm)s",
]) % {
'inst': monorpms_path,
'rpm': rpm['path'],
}
run_cmd(cmd, log_all=True, simple=True)
else:
raise EasyBuildError("RPM file %s not found", rpm['path'])
# create patched version of gmcs command
self.log.debug("Making our own copy of gmcs (one that works).")
mygmcs_path = os.path.join(monorpms_path, 'usr', 'bin', 'mygmcs')
try:
shutil.copy(os.path.join(monorpms_path, 'usr', 'bin', 'gmcs'),
mygmcs_path)
except OSError as err:
raise EasyBuildError("Failed to copy gmcs to %s: %s",
mygmcs_path, err)
rpls = [
("exec /usr/bin/mono", "exec %s/usr/bin/mono" % monorpms_path),
("`/usr/bin/monodir`", "%s/usr/lib64/mono" % monorpms_path),
]
apply_regex_substitutions(mygmcs_path, rpls)
self.log.debug("Patched version of gmcs (%s): %s" %
(mygmcs_path, read_file(mygmcs_path)))
# initiate bootstrap: build/install Mono with installed RPMs to temporary path
tmp_mono_path = os.path.join(self.builddir, "tmp_mono")
self.log.debug(
"Build/install temporary Mono version in %s using installed RPMs..."
% tmp_mono_path)
par = ''
if self.cfg['parallel']:
par = "-j %s" % self.cfg['parallel']
config_cmd = ' '.join([
self.cfg['preconfigopts'],
"./configure",
"--prefix=" + tmp_mono_path,
self.cfg['configopts'],
])
build_cmd = ' '.join([
"%(prebuildopts)s"
"make %(par)s",
"EXTERNAL_MCS=%(path)s/usr/bin/mygmcs",
"EXTERNAL_RUNTIME=%(path)s/usr/bin/mono",
"%(buildopts)s",
]) % {
'prebuildopts': self.cfg['prebuildopts'],
'par': par,
'path': monorpms_path,
'buildopts': self.cfg['buildopts'],
}
install_cmd = "make install"
for cmd in [config_cmd, build_cmd, install_cmd]:
run_cmd(cmd, log_all=True, simple=True)
more_buildopts = ' '.join([
"EXTERNAL_MCS=%(path)s/bin/gmcs",
"EXTERNAL_RUNTIME=%(path)s/bin/mono",
]) % {
'path': tmp_mono_path
}
self.cfg.update('buildopts', more_buildopts)
self.src = self.mono_srcs
# continue with normal configure, and subsequent make, make install
ConfigureMake.configure_step(self)
def configure_step(self):
"""Configure Boost build using custom tools"""
# boost_multi_thread is deprecated
if self.cfg['boost_multi_thread'] is not None:
self.log.deprecated(
"boost_multi_thread has been replaced by tagged_layout. "
"We build with tagged layout and both single and multi threading libraries "
"from version 1.69.0.", '5.0')
self.cfg['tagged_layout'] = True
# mpi sanity check
if self.cfg['boost_mpi'] and not self.toolchain.options.get(
'usempi', None):
raise EasyBuildError(
"When enabling building boost_mpi, also enable the 'usempi' toolchain option."
)
# generate config depending on compiler used
toolset = self.cfg['toolset']
if toolset is None:
if self.toolchain.comp_family() == toolchain.INTELCOMP:
toolset = 'intel-linux'
elif self.toolchain.comp_family() == toolchain.GCC:
toolset = 'gcc'
else:
raise EasyBuildError(
"Unknown compiler used, don't know what to specify to --with-toolset, aborting."
)
cmd = "%s ./bootstrap.sh --with-toolset=%s --prefix=%s %s"
tup = (self.cfg['preconfigopts'], toolset, self.installdir,
self.cfg['configopts'])
run_cmd(cmd % tup, log_all=True, simple=True)
# Use build_toolset if specified or the bootstrap toolset without the OS suffix
self.toolset = self.cfg['build_toolset'] or re.sub(
'-linux$', '', toolset)
user_config = []
# Explicitely set the compiler path to avoid B2 checking some standard paths like /opt
cxx = os.getenv('CXX')
if cxx:
cxx = which(cxx, on_error=ERROR)
# Remove default toolset config which may lead to duplicate toolsets (e.g. for intel-linux)
apply_regex_substitutions('project-config.jam',
[('using %s ;' % toolset, '')])
# Add our toolset config with no version and full path to compiler
user_config.append("using %s : : %s ;" % (self.toolset, cxx))
if self.cfg['boost_mpi']:
# configure the boost mpi module
# http://www.boost.org/doc/libs/1_47_0/doc/html/mpi/getting_started.html
# let Boost.Build know to look here for the config file
# Check if using a Cray toolchain and configure MPI accordingly
if self.toolchain.toolchain_family() == toolchain.CRAYPE:
if self.toolchain.PRGENV_MODULE_NAME_SUFFIX == 'gnu':
craympichdir = os.getenv('CRAY_MPICH2_DIR')
craygccversion = os.getenv('GCC_VERSION')
# We configure the gcc toolchain below, so make sure the EC doesn't use another toolset
if self.toolset != 'gcc':
raise EasyBuildError(
"For the cray toolchain the 'gcc' toolset must be used."
)
# Remove the previous "using gcc" line add above (via self.toolset) if present
user_config = [
x for x in user_config
if not x.startswith('using gcc :')
]
user_config.extend([
'local CRAY_MPICH2_DIR = %s ;' % craympichdir,
'using gcc ',
': %s' % craygccversion,
': CC ',
': <compileflags>-I$(CRAY_MPICH2_DIR)/include ',
r' <linkflags>-L$(CRAY_MPICH2_DIR)/lib \ ',
'; ',
'using mpi ',
': CC ',
': <find-shared-library>mpich ',
': %s' % self.cfg['mpi_launcher'],
';',
'',
])
else:
raise EasyBuildError(
"Bailing out: only PrgEnv-gnu supported for now")
else:
user_config.append("using mpi : %s ;" % os.getenv("MPICXX"))
write_file('user-config.jam', '\n'.join(user_config), append=True)
def configure_step(self):
"""Configure Ferret build."""
buildtype = "x86_64-linux"
if LooseVersion(self.version) < LooseVersion("7.3"):
change_dir('FERRET')
deps = ['HDF5', 'netCDF', 'Java']
for name in deps:
if not get_software_root(name):
raise EasyBuildError("%s module not loaded?", name)
if LooseVersion(self.version) >= LooseVersion("7.3"):
copy_file('external_functions/ef_utility/site_specific.mk.in',
'external_functions/ef_utility/site_specific.mk')
copy_file('site_specific.mk.in', 'site_specific.mk')
fns = [
"site_specific.mk",
"external_functions/ef_utility/site_specific.mk",
]
else:
fns = ["site_specific.mk"]
regex_subs = [
(r"^BUILDTYPE\s*=.*", "BUILDTYPE = %s" % buildtype),
(r"^INSTALL_FER_DIR =.*",
"INSTALL_FER_DIR = %s" % self.installdir),
]
for name in deps:
regex_subs.append((r"^(%s.*DIR\s*)=.*" % name.upper(),
r"\1 = %s" % get_software_root(name)))
if LooseVersion(self.version) >= LooseVersion("7.3"):
regex_subs.extend([
(r"^DIR_PREFIX =.*",
"DIR_PREFIX = %s" % self.cfg['start_dir']),
(r"^FER_LOCAL_EXTFCNS = $(FER_DIR)",
"FER_LOCAL_EXTFCNS = $(INSTALL_FER_DIR)/libs"),
])
for fn in fns:
apply_regex_substitutions(fn, regex_subs)
comp_vars = {
'CC': 'CC',
'CFLAGS': 'CFLAGS',
'CPPFLAGS': 'CPPFLAGS',
'FC': 'F77',
}
gfort2ifort = {
'-fno-second-underscore': ' ',
'-fno-backslash': ' ',
'-fdollar-ok': ' ',
'-ffast-math': ' ',
'-ffixed-line-length-132': '-132',
'-fno-automatic': ' ',
'-ffpe-trap=overflow': ' ',
'-fimplicit-none': '-implicitnone',
'-fdefault-real-8': '-r8',
'-fdefault-double-8': ' ',
'-Wl,-Bstatic -lgfortran -Wl,-Bdynamic': ' ',
'-v --verbose -m64': ' ',
'-export-dynamic': ' ',
'-DG77_SIGNAL': ' ',
}
fn = 'xgks/CUSTOMIZE.%s' % buildtype
regex_subs = [(r"^(FFLAGS\s*=').*-m64 (.*)",
r"\1%s \2" % os.getenv('FFLAGS'))]
for x, y in comp_vars.items():
regex_subs.append((r"^(%s\s*)=.*" % x, r"\1='%s'" % os.getenv(y)))
x11_root = get_software_root('X11')
if x11_root:
regex_subs.append(
(r"^(LD_X11\s*)=.*", r"\1='-L%s/lib -lX11'" % x11_root))
else:
regex_subs.append(
(r"^(LD_X11\s*)=.*", r"\1='-L/usr/lib64/X11 -lX11'"))
if LooseVersion(self.version) >= LooseVersion(
"7.3") and self.toolchain.comp_family() == toolchain.INTELCOMP:
regex_subs.extend(sorted(gfort2ifort.items()))
apply_regex_substitutions(fn, regex_subs)
comp_vars = {
'CC': 'CC',
'CXX': 'CXX',
'F77': 'F77',
'FC': 'F77',
}
if LooseVersion(self.version) >= LooseVersion("7.3"):
fns = [
'platform_specific.mk.%s' % buildtype,
'external_functions/ef_utility/platform_specific.mk.%s' %
buildtype,
]
else:
fns = [
'fer/platform_specific_flags.mk.%s' % buildtype,
'ppl/platform_specific_flags.mk.%s' % buildtype,
'external_functions/ef_utility/platform_specific_flags.mk.%s' %
buildtype,
]
regex_subs = []
for x, y in comp_vars.items():
regex_subs.append(
(r"^(\s*%s\s*)=.*" % x, r"\1 = %s" % os.getenv(y)))
if LooseVersion(self.version) >= LooseVersion("7.3"):
regex_subs.extend([
(r"^(\s*LDFLAGS\s*=).*",
r"\1 -fPIC %s -lnetcdff -lnetcdf -lhdf5_hl -lhdf5" %
os.getenv("LDFLAGS")),
(r"^(\s*)CDFLIB", r"\1NONEED"),
])
if self.toolchain.comp_family() == toolchain.INTELCOMP:
regex_subs.append(
(r"^(\s*LD\s*)=.*", r"\1 = %s -nofor-main" % os.getenv("F77")))
for x in ["CFLAGS", "FFLAGS"]:
regex_subs.append((r"^(\s*%s\s*=\s*\$\(CPP_FLAGS\)).*\\" % x,
r"\1 %s \\" % os.getenv(x)))
if LooseVersion(self.version) >= LooseVersion("7.3"):
for x in ["CFLAGS", "FFLAGS"]:
regex_subs.append((r"^(\s*%s\s*=).*-m64 (.*)" % x,
r"\1%s \2" % os.getenv(x)))
regex_subs.extend(sorted(gfort2ifort.items()))
regex_subs.append((r"^(\s*MYDEFINES\s*=.*)\\",
r"\1-DF90_SYSTEM_ERROR_CALLS \\"))
for fn in fns:
apply_regex_substitutions(fn, regex_subs)
if LooseVersion(self.version) >= LooseVersion("7.3"):
comp_vars = {
'CC': 'CC',
'LDFLAGS': 'LDFLAGS',
}
fn = 'gksm2ps/Makefile'
regex_subs = [(r"^(\s*CFLAGS=\")-m64 (.*)",
r"\1%s \2" % os.getenv('CFLAGS'))]
for x, y in comp_vars.items():
regex_subs.append(
(r"^(\s*%s)=.*" % x, r"\1='%s' \\" % os.getenv(y)))
apply_regex_substitutions(fn, regex_subs)
def configure_step(self):
"""Custom configuration procedure for Bazel."""
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if binutils_root and gcc_root:
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if not os.path.exists(gcc_lib_inc_fixed):
raise EasyBuildError("Derived directory %s does not exist",
gcc_lib_inc_fixed)
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
# replace hardcoded paths in CROSSTOOL
# CROSSTOOL script is no longer there in Bazel 0.24.0
if LooseVersion(self.version) < LooseVersion('0.24.0'):
regex_subs = [
(r'-B/usr/bin',
'-B%s' % os.path.join(binutils_root, 'bin')),
(r'(cxx_builtin_include_directory:.*)/usr/lib/gcc',
r'\1%s' % gcc_lib_inc),
(r'(cxx_builtin_include_directory:.*)/usr/local/include',
r'\1%s' % gcc_lib_inc_fixed),
(r'(cxx_builtin_include_directory:.*)/usr/include',
r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin',
tool), path))
else:
raise EasyBuildError(
"Failed to determine path to '%s'", tool)
apply_regex_substitutions(
os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
# replace hardcoded paths in (unix_)cc_configure.bzl
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
]
for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
filepath = os.path.join('tools', 'cpp', conf_bzl)
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
else:
self.log.info(
"Not patching Bazel build scripts, installation prefix for binutils/GCC not found"
)
# enable building in parallel
env.setvar('EXTRA_BAZEL_ARGS', '--jobs=%d' % self.cfg['parallel'])
def patch_crosstool_files(self):
"""Patches the CROSSTOOL files to include EasyBuild provided compiler paths"""
inc_paths, lib_paths = [], []
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
if gcc_root:
gcc_lib64 = os.path.join(gcc_root, 'lib64')
lib_paths.append(gcc_lib64)
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# figure out location of GCC include files
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
inc_paths.append(gcc_lib_inc)
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
# make sure include-fixed directory is where we expect it to be
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if os.path.exists(gcc_lib_inc_fixed):
inc_paths.append(gcc_lib_inc_fixed)
else:
self.log.info(
"Derived directory %s does not exist, so discarding it",
gcc_lib_inc_fixed)
# also check on location of include/c++/<gcc version> directory
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if os.path.exists(gcc_cplusplus_inc):
inc_paths.append(gcc_cplusplus_inc)
else:
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
else:
raise EasyBuildError(
"Failed to determine installation prefix for GCC")
cuda_root = get_software_root('CUDA')
if cuda_root:
inc_paths.append(os.path.join(cuda_root, 'include'))
lib_paths.append(os.path.join(cuda_root, 'lib64'))
# fix hardcoded locations of compilers & tools
cxx_inc_dirs = [
'cxx_builtin_include_directory: "%s"' % resolve_path(p)
for p in inc_paths
]
cxx_inc_dirs += [
'cxx_builtin_include_directory: "%s"' % p for p in inc_paths
]
regex_subs = [
(r'-B/usr/bin/', '-B%s %s' %
(self.binutils_bin_path, ' '.join('-L%s/' % p
for p in lib_paths))),
(r'(cxx_builtin_include_directory:).*', ''),
(r'^toolchain {', 'toolchain {\n' + '\n'.join(cxx_inc_dirs)),
]
for tool in [
'ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy',
'objdump', 'strip'
]:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
# -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used
if self.toolchain.options.get('pic', None):
regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')])
# patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries
for path, dirnames, filenames in os.walk(os.getcwd()):
for filename in filenames:
if filename.startswith('CROSSTOOL'):
full_path = os.path.join(path, filename)
self.log.info("Patching %s", full_path)
apply_regex_substitutions(full_path, regex_subs)
def configure_step(self):
"""Custom configuration procedure for WRF-Fire."""
comp_fam = self.toolchain.comp_family()
# define $NETCDF* for netCDF dependency
netcdf_fortran = get_software_root('netCDF-Fortran')
if netcdf_fortran:
env.setvar('NETCDF', netcdf_fortran)
else:
raise EasyBuildError("Required dependendy netCDF-Fortran is missing")
# define $PHDF5 for parallel HDF5 dependency
hdf5 = get_software_root('HDF5')
if hdf5 and os.path.exists(os.path.join(hdf5, 'bin', 'h5pcc')):
env.setvar('PHDF5', hdf5)
# first, configure WRF part
change_dir(os.path.join(self.cfg['start_dir'], 'WRFV3'))
# instruct WRF-Fire to create netCDF v4 output files
env.setvar('WRFIO_NETCDF4_FILE_SUPPORT', '1')
# patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join('arch', 'Config_new.pl'))
# determine build type option to look for
known_build_type_options = {
toolchain.INTELCOMP: "Linux x86_64 i486 i586 i686, ifort compiler with icc",
toolchain.GCC: "x86_64 Linux, gfortran compiler with gcc",
toolchain.PGI: "Linux x86_64, PGI compiler with pgcc",
}
build_type_option = known_build_type_options.get(comp_fam)
if build_type_option is None:
raise EasyBuildError("Don't know which WPS configure option to select for compiler family %s", comp_fam)
build_type_question = "\s*(?P<nr>[0-9]+).\s*%s\s*\(%s\)" % (build_type_option, self.cfg['buildtype'])
qa = {
"Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]:": '1',
}
std_qa = {
# named group in match will be used to construct answer
r"%s.*\n(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question: '%(nr)s',
}
run_cmd_qa('./configure', qa, std_qa=std_qa, log_all=True, simple=True)
cpp_flag = None
if comp_fam == toolchain.INTELCOMP:
cpp_flag = '-fpp'
elif comp_fam == toolchain.GCC:
cpp_flag = '-cpp'
else:
raise EasyBuildError("Don't know which flag to use to specify that Fortran files were preprocessed")
# patch configure.wrf to get things right
comps = {
'CFLAGS_LOCAL': os.getenv('CFLAGS'),
'DM_FC': os.getenv('MPIF90'),
'DM_CC': "%s -DMPI2_SUPPORT" % os.getenv('MPICC'),
'FCOPTIM': os.getenv('FFLAGS'),
# specify that Fortran files have been preprocessed with cpp,
# see http://forum.wrfforum.com/viewtopic.php?f=5&t=6086
'FORMAT_FIXED': "-FI %s" % cpp_flag,
'FORMAT_FREE': "-FR %s" % cpp_flag,
}
regex_subs = [(r"^(%s\s*=\s*).*$" % k, r"\1 %s" % v) for (k, v) in comps.items()]
apply_regex_substitutions('configure.wrf', regex_subs)
# also configure WPS part
change_dir(os.path.join(self.cfg['start_dir'], 'WPS'))
# patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join('arch', 'Config.pl'))
# determine build type option to look for
known_build_type_options = {
toolchain.INTELCOMP: "PC Linux x86_64, Intel compiler",
toolchain.GCC: "PC Linux x86_64, g95 compiler",
toolchain.PGI: "PC Linux x86_64 (IA64 and Opteron), PGI compiler 5.2 or higher",
}
build_type_option = known_build_type_options.get(comp_fam)
if build_type_option is None:
raise EasyBuildError("Don't know which WPS configure option to select for compiler family %s", comp_fam)
known_wps_build_types = {
'dmpar': 'DM parallel',
'smpar': 'serial',
}
wps_build_type = known_wps_build_types.get(self.cfg['buildtype'])
if wps_build_type is None:
raise EasyBuildError("Don't know which WPS build type to pick for '%s'", self.cfg['builddtype'])
build_type_question = "\s*(?P<nr>[0-9]+).\s*%s.*%s(?!NO GRIB2)" % (build_type_option, wps_build_type)
std_qa = {
# named group in match will be used to construct answer
r"%s.*\n(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question: '%(nr)s',
}
run_cmd_qa('./configure', {}, std_qa=std_qa, log_all=True, simple=True)
# patch configure.wps to get things right
comps = {
'CC': '%s %s' % (os.getenv('MPICC'), os.getenv('CFLAGS')),
'FC': '%s %s' % (os.getenv('MPIF90'), os.getenv('F90FLAGS'))
}
regex_subs = [(r"^(%s\s*=\s*).*$" % k, r"\1 %s" % v) for (k, v) in comps.items()]
# specify that Fortran90 files have been preprocessed with cpp
regex_subs.extend([
(r"^(F77FLAGS\s*=\s*)", r"\1 %s " % cpp_flag),
(r"^(FFLAGS\s*=\s*)", r"\1 %s " % cpp_flag),
])
apply_regex_substitutions('configure.wps', regex_subs)
def configure_step(self):
"""Custom configuration procedure for ALADIN."""
# unset $LIBRARY_PATH set by modules of dependencies, because it may screw up linking
if 'LIBRARY_PATH' in os.environ:
self.log.debug("Unsetting $LIBRARY_PATH (was: %s)" %
os.environ['LIBRARY_PATH'])
self.orig_library_path = os.environ.pop('LIBRARY_PATH')
# build auxiliary libraries
auxlibs_dir = None
my_gnu = None
if self.toolchain.comp_family() == toolchain.GCC:
my_gnu = 'y' # gfortran
for var in ['CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS']:
flags = os.getenv(var)
env.setvar(var,
"%s -fdefault-real-8 -fdefault-double-8" % flags)
self.log.info("Updated %s to '%s'" % (var, os.getenv(var)))
elif self.toolchain.comp_family() == toolchain.INTELCOMP:
my_gnu = 'i' # icc/ifort
else:
raise EasyBuildError(
"Don't know how to set 'my_gnu' variable in auxlibs build script."
)
self.log.info("my_gnu set to '%s'" % my_gnu)
tmp_installroot = tempfile.mkdtemp(prefix='aladin_auxlibs_')
try:
cwd = os.getcwd()
os.chdir(self.builddir)
builddirs = os.listdir(self.builddir)
auxlibs_dir = [
x for x in builddirs if x.startswith('auxlibs_installer')
][0]
os.chdir(auxlibs_dir)
auto_driver = 'driver_automatic'
for line in fileinput.input(auto_driver,
inplace=1,
backup='.orig.eb'):
line = re.sub(r"^(my_gnu\s*=\s*).*$", r"\1%s" % my_gnu, line)
line = re.sub(r"^(my_r32\s*=\s*).*$", r"\1n",
line) # always 64-bit real precision
line = re.sub(r"^(my_readonly\s*=\s*).*$", r"\1y",
line) # make libs read-only after build
line = re.sub(r"^(my_installroot\s*=\s*).*$",
r"\1%s" % tmp_installroot, line)
sys.stdout.write(line)
run_cmd("./%s" % auto_driver)
os.chdir(cwd)
except OSError as err:
raise EasyBuildError("Failed to build ALADIN: %s", err)
# build gmkpack, update PATH and set GMKROOT
# we build gmkpack here because a config file is generated in the gmkpack isntall path
try:
gmkpack_dir = [x for x in builddirs if x.startswith('gmkpack')][0]
os.chdir(os.path.join(self.builddir, gmkpack_dir))
qa = {
'Do you want to run the configuration file maker assistant now (y) or later [n] ?':
'n',
}
run_cmd_qa("./build_gmkpack", qa)
os.chdir(cwd)
paths = os.getenv('PATH').split(':')
paths.append(os.path.join(self.builddir, gmkpack_dir, 'util'))
env.setvar('PATH', ':'.join(paths))
env.setvar('GMKROOT', os.path.join(self.builddir, gmkpack_dir))
except OSError as err:
raise EasyBuildError("Failed to build gmkpack: %s", err)
# generate gmkpack configuration file
self.conf_file = 'ALADIN_%s' % self.version
self.conf_filepath = os.path.join(self.builddir, 'gmkpack_support',
'arch', '%s.x' % self.conf_file)
try:
if os.path.exists(self.conf_filepath):
os.remove(self.conf_filepath)
self.log.info("Removed existing gmpack config file %s" %
self.conf_filepath)
archdir = os.path.dirname(self.conf_filepath)
if not os.path.exists(archdir):
mkdir(archdir, parents=True)
except OSError as err:
raise EasyBuildError("Failed to remove existing file %s: %s",
self.conf_filepath, err)
mpich = 'n'
known_mpi_libs = [
toolchain.MPICH, toolchain.MPICH2, toolchain.INTELMPI
]
if self.toolchain.options.get(
'usempi',
None) and self.toolchain.mpi_family() in known_mpi_libs:
mpich = 'y'
qpref = 'Please type the ABSOLUTE name of '
qsuff = ', or ignore (environment variables allowed) :'
qsuff2 = ', or ignore : (environment variables allowed) :'
comp_fam = self.toolchain.comp_family()
if comp_fam == toolchain.GCC:
gribdir = 'GNU'
elif comp_fam == toolchain.INTELCOMP:
gribdir = 'INTEL'
else:
raise EasyBuildError(
"Don't know which grib lib dir to use for compiler %s",
comp_fam)
aux_lib_gribex = os.path.join(tmp_installroot, gribdir, 'lib',
'libgribex.a')
aux_lib_ibm = os.path.join(tmp_installroot, gribdir, 'lib',
'libibmdummy.a')
grib_api_lib = os.path.join(get_software_root('grib_api'), 'lib',
'libgrib_api.a')
grib_api_f90_lib = os.path.join(get_software_root('grib_api'), 'lib',
'libgrib_api_f90.a')
grib_api_inc = os.path.join(get_software_root('grib_api'), 'include')
jasperlib = os.path.join(get_software_root('JasPer'), 'lib',
'libjasper.a')
mpilib = os.path.join(os.getenv('MPI_LIB_DIR'),
os.getenv('MPI_LIB_SHARED'))
# netCDF
netcdf = get_software_root('netCDF')
netcdf_fortran = get_software_root('netCDF-Fortran')
if netcdf:
netcdfinc = os.path.join(netcdf, 'include')
if netcdf_fortran:
netcdflib = os.path.join(netcdf_fortran,
get_software_libdir('netCDF-Fortran'),
'libnetcdff.a')
else:
netcdflib = os.path.join(netcdf, get_software_libdir('netCDF'),
'libnetcdff.a')
if not os.path.exists(netcdflib):
raise EasyBuildError("%s does not exist", netcdflib)
else:
raise EasyBuildError("netCDF(-Fortran) not available")
ldpaths = [ldflag[2:] for ldflag in os.getenv('LDFLAGS').split(' ')
] # LDFLAGS have form '-L/path/to'
lapacklibs = []
for lib in os.getenv('LAPACK_STATIC_LIBS').split(','):
libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths]
lapacklibs.append([
libpath for libpath in libpaths if os.path.exists(libpath)
][0])
lapacklib = ' '.join(lapacklibs)
blaslibs = []
for lib in os.getenv('BLAS_STATIC_LIBS').split(','):
libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths]
blaslibs.append([
libpath for libpath in libpaths if os.path.exists(libpath)
][0])
blaslib = ' '.join(blaslibs)
qa = {
'Do you want to run the configuration file maker assistant now (y) or later [n] ?':
'y',
'Do you want to setup your configuration file for MPICH (y/n) [n] ?':
mpich,
'Please type the directory name where to find a dummy file mpif.h or ignore :':
os.getenv('MPI_INC_DIR'),
'%sthe library gribex or emos%s' % (qpref, qsuff2):
aux_lib_gribex,
'%sthe library ibm%s' % (qpref, qsuff):
aux_lib_ibm,
'%sthe library grib_api%s' % (qpref, qsuff):
grib_api_lib,
'%sthe library grib_api_f90%s' % (qpref, qsuff):
grib_api_f90_lib,
'%sthe JPEG auxilary library if enabled by Grib_api%s' % (qpref, qsuff2):
jasperlib,
'%sthe library netcdf%s' % (qpref, qsuff):
netcdflib,
'%sthe library lapack%s' % (qpref, qsuff):
lapacklib,
'%sthe library blas%s' % (qpref, qsuff):
blaslib,
'%sthe library mpi%s' % (qpref, qsuff):
mpilib,
'%sa MPI dummy library for serial executions, or ignore :' % qpref:
'',
'Please type the directory name where to find grib_api headers, or ignore :':
grib_api_inc,
'Please type the directory name where to find fortint.h or ignore :':
'',
'Please type the directory name where to find netcdf headers, or ignore :':
netcdfinc,
'Do you want to define CANARI (y/n) [y] ?':
'y',
'Please type the name of the script file used to generate a preprocessed blacklist file, or ignore :':
'',
'Please type the name of the script file used to recover local libraries (gget), or ignore :':
'',
'Please type the options to tune the gnu compilers, or ignore :':
os.getenv('F90FLAGS'),
}
f90_seq = os.getenv('F90_SEQ')
if not f90_seq:
# F90_SEQ is only defined when usempi is enabled
f90_seq = os.getenv('F90')
stdqa = OrderedDict([
(r'Confirm library .* is .*',
'y'), # this one needs to be tried first!
(r'.*fortran 90 compiler name .*\s*:\n\(suggestions\s*: .*\)',
f90_seq),
(r'.*fortran 90 compiler interfaced with .*\s*:\n\(suggestions\s*: .*\)',
f90_seq),
(r'Please type the ABSOLUTE name of .*library.*, or ignore\s*[:]*\s*[\n]*.*',
''),
(r'Please .* to save this draft configuration file :\n.*',
'%s.x' % self.conf_file),
])
no_qa = [
".*ignored.",
]
env.setvar('GMKTMP', self.builddir)
env.setvar('GMKFILE', self.conf_file)
run_cmd_qa("gmkfilemaker", qa, std_qa=stdqa, no_qa=no_qa)
# set environment variables for installation dirs
env.setvar('ROOTPACK', os.path.join(self.installdir, 'rootpack'))
env.setvar('ROOTBIN', os.path.join(self.installdir, 'rootpack'))
env.setvar('HOMEPACK', os.path.join(self.installdir, 'pack'))
env.setvar('HOMEBIN', os.path.join(self.installdir, 'pack'))
# patch config file to include right Fortran compiler flags
regex_subs = [(r"^(FRTFLAGS\s*=.*)$", r"\1 %s" % os.getenv('FFLAGS'))]
apply_regex_substitutions(self.conf_filepath, regex_subs)
def configure_step(self):
"""Configure Ferret build."""
buildtype = "x86_64-linux"
if LooseVersion(self.version) < LooseVersion("7.3"):
change_dir('FERRET')
deps = ['HDF5', 'netCDF', 'Java']
for name in deps:
if not get_software_root(name):
raise EasyBuildError("%s module not loaded?", name)
if LooseVersion(self.version) >= LooseVersion("7.3"):
copy_file('external_functions/ef_utility/site_specific.mk.in',
'external_functions/ef_utility/site_specific.mk')
copy_file('site_specific.mk.in', 'site_specific.mk')
fns = [
"site_specific.mk",
"external_functions/ef_utility/site_specific.mk",
]
else:
fns = ["site_specific.mk"]
regex_subs = [
(r"^BUILDTYPE\s*=.*", "BUILDTYPE = %s" % buildtype),
(r"^INSTALL_FER_DIR =.*", "INSTALL_FER_DIR = %s" % self.installdir),
]
for name in deps:
regex_subs.append((r"^(%s.*DIR\s*)=.*" % name.upper(), r"\1 = %s" % get_software_root(name)))
if LooseVersion(self.version) >= LooseVersion("7.3"):
regex_subs.extend([
(r"^DIR_PREFIX =.*", "DIR_PREFIX = %s" % self.cfg['start_dir']),
(r"^FER_LOCAL_EXTFCNS = $(FER_DIR)", "FER_LOCAL_EXTFCNS = $(INSTALL_FER_DIR)/libs"),
])
for fn in fns:
apply_regex_substitutions(fn, regex_subs)
comp_vars = {
'CC': 'CC',
'CFLAGS': 'CFLAGS',
'CPPFLAGS': 'CPPFLAGS',
'FC': 'F77',
}
gfort2ifort = {
'-fno-second-underscore': ' ',
'-fno-backslash': ' ',
'-fdollar-ok': ' ',
'-ffast-math': ' ',
'-ffixed-line-length-132': '-132',
'-fno-automatic': ' ',
'-ffpe-trap=overflow': ' ',
'-fimplicit-none': '-implicitnone',
'-fdefault-real-8': '-r8',
'-fdefault-double-8': ' ',
'-Wl,-Bstatic -lgfortran -Wl,-Bdynamic': ' ',
'-v --verbose -m64': ' ',
'-export-dynamic': ' ',
'-DG77_SIGNAL': ' ',
}
fn = 'xgks/CUSTOMIZE.%s' % buildtype
regex_subs = [(r"^(FFLAGS\s*=').*-m64 (.*)", r"\1%s \2" % os.getenv('FFLAGS'))]
for x, y in comp_vars.items():
regex_subs.append((r"^(%s\s*)=.*" % x, r"\1='%s'" % os.getenv(y)))
x11_root = get_software_root('X11')
if x11_root:
regex_subs.append((r"^(LD_X11\s*)=.*", r"\1='-L%s/lib -lX11'" % x11_root))
else:
regex_subs.append((r"^(LD_X11\s*)=.*", r"\1='-L/usr/lib64/X11 -lX11'"))
if LooseVersion(self.version) >= LooseVersion("7.3") and self.toolchain.comp_family() == toolchain.INTELCOMP:
regex_subs.extend(sorted(gfort2ifort.items()))
apply_regex_substitutions(fn, regex_subs)
comp_vars = {
'CC': 'CC',
'CXX': 'CXX',
'F77': 'F77',
'FC': 'F77',
}
if LooseVersion(self.version) >= LooseVersion("7.3"):
fns = [
'platform_specific.mk.%s' % buildtype,
'external_functions/ef_utility/platform_specific.mk.%s' % buildtype,
]
else:
fns = [
'fer/platform_specific_flags.mk.%s' % buildtype,
'ppl/platform_specific_flags.mk.%s' % buildtype,
'external_functions/ef_utility/platform_specific_flags.mk.%s' % buildtype,
]
regex_subs = []
for x, y in comp_vars.items():
regex_subs.append((r"^(\s*%s\s*)=.*" % x, r"\1 = %s" % os.getenv(y)))
if LooseVersion(self.version) >= LooseVersion("7.3"):
regex_subs.extend([
(r"^(\s*LDFLAGS\s*=).*", r"\1 -fPIC %s -lnetcdff -lnetcdf -lhdf5_hl -lhdf5" % os.getenv("LDFLAGS")),
(r"^(\s*)CDFLIB", r"\1NONEED"),
])
if self.toolchain.comp_family() == toolchain.INTELCOMP:
regex_subs.append((r"^(\s*LD\s*)=.*", r"\1 = %s -nofor-main" % os.getenv("F77")))
for x in ["CFLAGS", "FFLAGS"]:
regex_subs.append((r"^(\s*%s\s*=\s*\$\(CPP_FLAGS\)).*\\" % x, r"\1 %s \\" % os.getenv(x)))
if LooseVersion(self.version) >= LooseVersion("7.3"):
for x in ["CFLAGS", "FFLAGS"]:
regex_subs.append((r"^(\s*%s\s*=).*-m64 (.*)" % x, r"\1%s \2" % os.getenv(x)))
regex_subs.extend(sorted(gfort2ifort.items()))
regex_subs.append((r"^(\s*MYDEFINES\s*=.*)\\", r"\1-DF90_SYSTEM_ERROR_CALLS \\"))
for fn in fns:
apply_regex_substitutions(fn, regex_subs)
if LooseVersion(self.version) >= LooseVersion("7.3"):
comp_vars = {
'CC': 'CC',
'LDFLAGS': 'LDFLAGS',
}
fn = 'gksm2ps/Makefile'
regex_subs = [(r"^(\s*CFLAGS=\")-m64 (.*)", r"\1%s \2" % os.getenv('CFLAGS'))]
for x, y in comp_vars.items():
regex_subs.append((r"^(\s*%s)=.*" % x, r"\1='%s' \\" % os.getenv(y)))
apply_regex_substitutions(fn, regex_subs)
def configure_step(self):
"""Custom configuration procedure for Bazel."""
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
gcc_ver = get_software_version('GCCcore') or get_software_version('GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if binutils_root and gcc_root:
res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError("Failed to pinpoint location of GCC include files: %s", res)
gcc_lib_inc_bis = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed')
if not os.path.exists(gcc_lib_inc_bis):
self.log.info("Derived directory %s does not exist, falling back to %s", gcc_lib_inc_bis, gcc_lib_inc)
gcc_lib_inc_bis = gcc_lib_inc
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc)
# replace hardcoded paths in CROSSTOOL
# CROSSTOOL script is no longer there in Bazel 0.24.0
if LooseVersion(self.version) < LooseVersion('0.24.0'):
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc),
(r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_bis),
(r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
apply_regex_substitutions(os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
# replace hardcoded paths in (unix_)cc_configure.bzl
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
]
for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
filepath = os.path.join('tools', 'cpp', conf_bzl)
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
else:
self.log.info("Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
# enable building in parallel
bazel_args = '--jobs=%d' % self.cfg['parallel']
# Bazel provides a JDK by itself for some architectures
# We want to enforce it using the JDK we provided via modules
# This is required for Power where Bazel does not have a JDK, but requires it for building itself
# See https://github.com/bazelbuild/bazel/issues/10377
bazel_args += ' --host_javabase=@local_jdk//:jdk'
env.setvar('EXTRA_BAZEL_ARGS', bazel_args)
def build_step(self):
"""Custom build procedure for TensorFlow."""
# pre-create target installation directory
mkdir(os.path.join(self.installdir, self.pylibdir), parents=True)
binutils_root = get_software_root('binutils')
if binutils_root:
binutils_bin = os.path.join(binutils_root, 'bin')
else:
raise EasyBuildError(
"Failed to determine installation prefix for binutils")
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
if gcc_root:
gcc_lib64 = os.path.join(gcc_root, 'lib64')
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# figure out location of GCC include files
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
# make sure include-fixed directory is where we expect it to be
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if not os.path.exists(gcc_lib_inc_fixed):
raise EasyBuildError("Derived directory %s does not exist",
gcc_lib_inc_fixed)
# also check on location of include/c++/<gcc version> directory
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
else:
raise EasyBuildError(
"Failed to determine installation prefix for GCC")
inc_paths = [gcc_lib_inc, gcc_lib_inc_fixed, gcc_cplusplus_inc]
lib_paths = [gcc_lib64]
cuda_root = get_software_root('CUDA')
if cuda_root:
inc_paths.append(os.path.join(cuda_root, 'include'))
lib_paths.append(os.path.join(cuda_root, 'lib64'))
# fix hardcoded locations of compilers & tools
cxx_inc_dir_lines = '\n'.join(r'cxx_builtin_include_directory: "%s"' %
resolve_path(p) for p in inc_paths)
cxx_inc_dir_lines_no_resolv_path = '\n'.join(
r'cxx_builtin_include_directory: "%s"' % p for p in inc_paths)
regex_subs = [
(r'-B/usr/bin/',
'-B%s/ %s' % (binutils_bin, ' '.join('-L%s/' % p
for p in lib_paths))),
(r'(cxx_builtin_include_directory:).*', ''),
(r'^toolchain {', 'toolchain {\n' + cxx_inc_dir_lines + '\n' +
cxx_inc_dir_lines_no_resolv_path),
]
for tool in [
'ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy',
'objdump', 'strip'
]:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
# -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used
if self.toolchain.options.get('pic', None):
regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')])
# patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries
for path, dirnames, filenames in os.walk(self.start_dir):
for filename in filenames:
if filename.startswith('CROSSTOOL'):
full_path = os.path.join(path, filename)
self.log.info("Patching %s", full_path)
apply_regex_substitutions(full_path, regex_subs)
tmpdir = tempfile.mkdtemp(suffix='-bazel-build')
user_root_tmpdir = tempfile.mkdtemp(suffix='-user_root')
# compose "bazel build" command with all its options...
cmd = [
self.cfg['prebuildopts'], 'bazel',
'--output_base=%s' % tmpdir,
'--install_base=%s' % os.path.join(tmpdir, 'inst_base'),
'--output_user_root=%s' % user_root_tmpdir, 'build'
]
# build with optimization enabled
# cfr. https://docs.bazel.build/versions/master/user-manual.html#flag--compilation_mode
cmd.append('--compilation_mode=opt')
# select 'opt' config section (this is *not* the same as --compilation_mode=opt!)
# https://docs.bazel.build/versions/master/user-manual.html#flag--config
cmd.append('--config=opt')
# make Bazel print full command line + make it verbose on failures
# https://docs.bazel.build/versions/master/user-manual.html#flag--subcommands
# https://docs.bazel.build/versions/master/user-manual.html#flag--verbose_failures
cmd.extend(['--subcommands', '--verbose_failures'])
# limit the number of parallel jobs running simultaneously (useful on KNL)...
cmd.append('--jobs=%s' % self.cfg['parallel'])
if self.toolchain.options.get('pic', None):
cmd.append('--copt="-fPIC"')
cmd.append(self.cfg['buildopts'])
if cuda_root:
cmd.append('--config=cuda')
# if mkl-dnn is listed as a dependency it is used. Otherwise downloaded if with_mkl_dnn is true
mkl_root = get_software_root('mkl-dnn')
if mkl_root:
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=0 &&")
cmd.insert(0, "export TF_MKL_ROOT=%s &&" % mkl_root)
elif self.cfg['with_mkl_dnn']:
# this makes TensorFlow use mkl-dnn (cfr. https://github.com/01org/mkl-dnn)
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=1 && ")
# specify target of the build command as last argument
cmd.append('//tensorflow/tools/pip_package:build_pip_package')
run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True)
# run generated 'build_pip_package' script to build the .whl
cmd = "bazel-bin/tensorflow/tools/pip_package/build_pip_package %s" % self.builddir
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def configure_step(self):
"""Custom configure step for NAMD, we build charm++ first (if required)."""
# complete Charm ++ and NAMD architecture string with compiler family
comp_fam = self.toolchain.comp_family()
if self.toolchain.options.get('usempi', False):
charm_arch_comp = 'mpicxx'
else:
charm_arch_comps = {
toolchain.GCC: 'gcc',
toolchain.INTELCOMP: 'icc',
}
charm_arch_comp = charm_arch_comps.get(comp_fam, None)
namd_comps = {
toolchain.GCC: 'g++',
toolchain.INTELCOMP: 'icc',
}
namd_comp = namd_comps.get(comp_fam, None)
if charm_arch_comp is None or namd_comp is None:
raise EasyBuildError("Unknown compiler family, can't complete Charm++/NAMD target architecture.")
# NOTE: important to add smp BEFORE the compiler
# charm arch style is: mpi-linux-x86_64-smp-mpicxx
# otherwise the setting of name_charm_arch below will get things
# in the wrong order
if self.toolchain.options.get('openmp', False):
self.cfg.update('charm_arch', 'smp')
self.cfg.update('charm_arch', charm_arch_comp)
self.log.info("Updated 'charm_arch': %s", self.cfg['charm_arch'])
self.namd_arch = '%s-%s' % (self.cfg['namd_basearch'], namd_comp)
self.log.info("Completed NAMD target architecture: %s", self.namd_arch)
cmd = "./build charm++ %(arch)s %(opts)s --with-numa -j%(parallel)s '%(cxxflags)s'" % {
'arch': self.cfg['charm_arch'],
'cxxflags': os.environ['CXXFLAGS'] + ' -DMPICH_IGNORE_CXX_SEEK ' + self.cfg['charm_extra_cxxflags'],
'opts': self.cfg['charm_opts'],
'parallel': self.cfg['parallel'],
}
charm_subdir = '.'.join(os.path.basename(self.charm_tarballs[0]).split('.')[:-1])
self.log.debug("Building Charm++ using cmd '%s' in '%s'" % (cmd, charm_subdir))
run_cmd(cmd, path=charm_subdir)
# compiler (options)
self.cfg.update('namd_cfg_opts', '--cc "%s" --cc-opts "%s"' % (os.environ['CC'], os.environ['CFLAGS']))
cxxflags = os.environ['CXXFLAGS']
if LooseVersion(self.version) >= LooseVersion('2.12'):
cxxflags += ' --std=c++11'
self.cfg.update('namd_cfg_opts', '--cxx "%s" --cxx-opts "%s"' % (os.environ['CXX'], cxxflags))
# NAMD dependencies: CUDA, TCL, FFTW
cuda = get_software_root('CUDA')
if cuda and (self.cfg['cuda'] is None or self.cfg['cuda']):
self.cfg.update('namd_cfg_opts', "--with-cuda --cuda-prefix %s" % cuda)
elif not self.cfg['cuda']:
self.log.warning("CUDA is disabled")
elif not cuda and self.cfg['cuda']:
raise EasyBuildError("CUDA is not a dependency, but support for CUDA is enabled.")
tcl = get_software_root('Tcl')
if tcl:
self.cfg.update('namd_cfg_opts', '--with-tcl --tcl-prefix %s' % tcl)
tclversion = '.'.join(get_software_version('Tcl').split('.')[0:2])
tclv_subs = [(r'-ltcl[\d.]*\s', '-ltcl%s ' % tclversion)]
apply_regex_substitutions(os.path.join('arch', '%s.tcl' % self.cfg['namd_basearch']), tclv_subs)
fftw = get_software_root('FFTW')
if fftw:
if LooseVersion(get_software_version('FFTW')) >= LooseVersion('3.0'):
if LooseVersion(self.version) >= LooseVersion('2.9'):
self.cfg.update('namd_cfg_opts', "--with-fftw3")
else:
raise EasyBuildError("Using FFTW v3.x only supported in NAMD v2.9 and up.")
else:
self.cfg.update('namd_cfg_opts', "--with-fftw")
self.cfg.update('namd_cfg_opts', "--fftw-prefix %s" % fftw)
namd_charm_arch = "--charm-arch %s" % '-'.join(self.cfg['charm_arch'].strip().split())
cmd = "./config %s %s %s " % (self.namd_arch, namd_charm_arch, self.cfg["namd_cfg_opts"])
run_cmd(cmd)
def configure_step(self):
"""Dedicated configure step for Mono: install Mono from RPM (if provided), then run configure."""
# install Mono from RPMs if provided (because we need Mono to build Mono)
if self.rpms:
# prepare path for installing RPMs in
monorpms_path = os.path.join(self.builddir, "monorpms")
try:
os.makedirs(os.path.join(monorpms_path, 'rpm'))
except OSError as err:
raise EasyBuildError("Failed to create directories for installing Mono RPMs in: %s", err)
self.src = self.rpms
self.rebuildRPM = True
# rebuild RPMs to make them relocatable
Rpm.configure_step(self)
# prepare to install RPMs
self.log.debug("Initializing temporary RPM repository to install to...")
cmd = "rpm --initdb --dbpath /rpm --root %s" % monorpms_path
run_cmd(cmd, log_all=True, simple=True)
# install RPMs one by one
for rpm in self.src:
self.log.debug("Installing RPM %s ..." % rpm['name'])
if os.path.exists(rpm['path']):
cmd = ' '.join([
"rpm -i",
"--dbpath %(inst)s/rpm",
"--force",
"--relocate /=%(inst)s",
"--badreloc",
"--nodeps --nopost",
"%(rpm)s",
]) % {
'inst': monorpms_path,
'rpm': rpm['path'],
}
run_cmd(cmd,log_all=True,simple=True)
else:
raise EasyBuildError("RPM file %s not found", rpm['path'])
# create patched version of gmcs command
self.log.debug("Making our own copy of gmcs (one that works).")
mygmcs_path = os.path.join(monorpms_path, 'usr', 'bin', 'mygmcs')
try:
shutil.copy(os.path.join(monorpms_path, 'usr' ,'bin', 'gmcs'), mygmcs_path)
except OSError as err:
raise EasyBuildError("Failed to copy gmcs to %s: %s", mygmcs_path, err)
rpls = [
("exec /usr/bin/mono", "exec %s/usr/bin/mono" % monorpms_path),
("`/usr/bin/monodir`", "%s/usr/lib64/mono" % monorpms_path),
]
apply_regex_substitutions(mygmcs_path, rpls)
self.log.debug("Patched version of gmcs (%s): %s" % (mygmcs_path, read_file(mygmcs_path)))
# initiate bootstrap: build/install Mono with installed RPMs to temporary path
tmp_mono_path = os.path.join(self.builddir, "tmp_mono")
self.log.debug("Build/install temporary Mono version in %s using installed RPMs..." % tmp_mono_path)
par = ''
if self.cfg['parallel']:
par = "-j %s" % self.cfg['parallel']
config_cmd = "%s ./configure --prefix=%s %s" % (self.cfg['preconfigopts'], tmp_mono_path, self.cfg['configopts'])
build_cmd = ' '.join([
"%(prebuildopts)s"
"make %(par)s",
"EXTERNAL_MCS=%(path)s/usr/bin/mygmcs",
"EXTERNAL_RUNTIME=%(path)s/usr/bin/mono",
"%(buildopts)s",
]) %{
'prebuildopts': self.cfg['prebuildopts'],
'par': par,
'path': monorpms_path,
'buildopts': self.cfg['buildopts'],
}
install_cmd = "make install"
for cmd in [config_cmd, build_cmd, install_cmd]:
run_cmd(cmd, log_all=True, simple=True)
more_buildopts = ' '.join([
"EXTERNAL_MCS=%(path)s/bin/gmcs",
"EXTERNAL_RUNTIME=%(path)s/bin/mono",
]) % {'path': tmp_mono_path}
self.cfg.update('buildopts', more_buildopts)
self.src = self.mono_srcs
# continue with normal configure, and subsequent make, make install
ConfigureMake.configure_step(self)
def configure_step(self):
"""Configure build:
- set required environment variables (for netCDF, JasPer)
- patch compile script and ungrib Makefile for non-default install paths of WRF and JasPer
- run configure script and figure how to select desired build option
- patch configure.wps file afterwards to fix 'serial compiler' setting
"""
# netCDF dependency check + setting env vars (NETCDF, NETCDFF)
set_netcdf_env_vars(self.log)
# WRF dependency check
wrf = get_software_root('WRF')
if wrf:
wrfdir = os.path.join(wrf,
det_wrf_subdir(get_software_version('WRF')))
else:
raise EasyBuildError("WRF module not loaded?")
self.compile_script = 'compile'
if LooseVersion(self.version) >= LooseVersion('4.0.3'):
# specify install location of WRF via $WRF_DIR (supported since WPS 4.0.3)
# see https://github.com/wrf-model/WPS/pull/102
env.setvar('WRF_DIR', wrfdir)
else:
# patch compile script so that WRF is found
regex_subs = [(r"^(\s*set\s*WRF_DIR_PRE\s*=\s*)\${DEV_TOP}(.*)$",
r"\1%s\2" % wrfdir)]
apply_regex_substitutions(self.compile_script, regex_subs)
# libpng dependency check
libpng = get_software_root('libpng')
zlib = get_software_root('zlib')
if libpng:
paths = [libpng]
if zlib:
paths.insert(0, zlib)
libpnginc = ' '.join(
['-I%s' % os.path.join(path, 'include') for path in paths])
libpnglib = ' '.join(
['-L%s' % os.path.join(path, 'lib') for path in paths])
else:
# define these as empty, assume that libpng will be available via OS (e.g. due to --filter-deps=libpng)
libpnglib = ""
libpnginc = ""
# JasPer dependency check + setting env vars
jasper = get_software_root('JasPer')
if jasper:
env.setvar('JASPERINC', os.path.join(jasper, "include"))
jasperlibdir = os.path.join(jasper, "lib")
env.setvar('JASPERLIB', jasperlibdir)
jasperlib = "-L%s" % jasperlibdir
else:
raise EasyBuildError("JasPer module not loaded?")
# patch ungrib Makefile so that JasPer is found
jasperlibs = "%s -ljasper %s -lpng" % (jasperlib, libpnglib)
regex_subs = [
(r"^(\s*-L\.\s*-l\$\(LIBTARGET\))(\s*;.*)$",
r"\1 %s\2" % jasperlibs),
(r"^(\s*\$\(COMPRESSION_LIBS\))(\s*;.*)$",
r"\1 %s\2" % jasperlibs),
]
apply_regex_substitutions(os.path.join('ungrib', 'src', 'Makefile'),
regex_subs)
# patch arch/Config.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join("arch", "Config.pl"))
# configure
# determine build type option to look for
self.comp_fam = self.toolchain.comp_family()
build_type_option = None
if LooseVersion(self.version) >= LooseVersion("3.4"):
knownbuildtypes = {'smpar': 'serial', 'dmpar': 'dmpar'}
if self.comp_fam == toolchain.INTELCOMP: # @UndefinedVariable
build_type_option = " Linux x86_64, Intel compiler"
elif self.comp_fam == toolchain.GCC: # @UndefinedVariable
if LooseVersion(self.version) >= LooseVersion("3.6"):
build_type_option = "Linux x86_64, gfortran"
else:
build_type_option = "Linux x86_64 g95"
else:
raise EasyBuildError(
"Don't know how to figure out build type to select.")
else:
knownbuildtypes = {'smpar': 'serial', 'dmpar': 'DM parallel'}
if self.comp_fam == toolchain.INTELCOMP: # @UndefinedVariable
build_type_option = "PC Linux x86_64, Intel compiler"
elif self.comp_fam == toolchain.GCC: # @UndefinedVariable
build_type_option = "PC Linux x86_64, gfortran compiler,"
knownbuildtypes['dmpar'] = knownbuildtypes['dmpar'].upper()
else:
raise EasyBuildError(
"Don't know how to figure out build type to select.")
# check and fetch selected build type
bt = self.cfg['buildtype']
if bt not in knownbuildtypes.keys():
raise EasyBuildError(
"Unknown build type: '%s'. Supported build types: %s", bt,
knownbuildtypes.keys())
# fetch option number based on build type option and selected build type
build_type_question = r"\s*(?P<nr>[0-9]+).\s*%s\s*\(?%s\)?\s*\n" % (
build_type_option, knownbuildtypes[bt])
cmd = ' '.join(
[self.cfg['preconfigopts'], './configure', self.cfg['configopts']])
qa = {}
no_qa = [".*compiler is.*"]
std_qa = {
# named group in match will be used to construct answer
r"%s(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question:
"%(nr)s",
}
run_cmd_qa(cmd,
qa,
no_qa=no_qa,
std_qa=std_qa,
log_all=True,
simple=True)
# make sure correct compilers and compiler flags are being used
comps = {
'SCC': "%s -I$(JASPERINC) %s" % (os.getenv('CC'), libpnginc),
'SFC': os.getenv('F90'),
'DM_FC': os.getenv('MPIF90'),
'DM_CC': os.getenv('MPICC'),
'FC': os.getenv('MPIF90'),
'CC': os.getenv('MPICC'),
}
if self.toolchain.options.get('openmp', None):
comps.update({
'LDFLAGS':
'%s %s' %
(self.toolchain.get_flag('openmp'), os.environ['LDFLAGS'])
})
regex_subs = [(r"^(%s\s*=\s*).*$" % key, r"\1 %s" % val)
for (key, val) in comps.items()]
apply_regex_substitutions('configure.wps', regex_subs)
class EB_Molpro(ConfigureMake, Binary):
"""Support for building and installing Molpro."""
@staticmethod
def extra_options():
"""Define custom easyconfig parameters for Molpro."""
# Combine extra variables from Binary and ConfigureMake easyblocks as
# well as those needed for Molpro specifically
extra_vars = Binary.extra_options()
extra_vars = ConfigureMake.extra_options(extra_vars)
extra_vars.update({
'precompiled_binaries':
[False, "Are we installing precompiled binaries?", CUSTOM],
})
return EasyBlock.extra_options(extra_vars)
def __init__(self, *args, **kwargs):
"""Easyblock constructor, initialize class variables specific to Molpro and check on license token."""
super(EB_Molpro, self).__init__(*args, **kwargs)
self.full_prefix = '' # no None, to make easyblock compatible with --module-only
self.orig_launcher = None
self.cleanup_token_symlink = False
self.license_token = os.path.join(os.path.expanduser('~'), '.molpro',
'token')
def extract_step(self):
"""Extract Molpro source files, or just copy in case of binary install."""
if self.cfg['precompiled_binaries']:
Binary.extract_step(self)
else:
ConfigureMake.extract_step(self)
def configure_step(self):
"""Custom configuration procedure for Molpro: use 'configure -batch'."""
if not os.path.isfile(self.license_token):
if self.cfg['license_file'] is not None and os.path.isfile(
self.cfg['license_file']):
# put symlink in place to specified license file in $HOME/.molpro/token
# other approaches (like defining $MOLPRO_KEY) don't seem to work
self.cleanup_token_symlink = True
mkdir(os.path.dirname(self.license_token))
try:
os.symlink(self.cfg['license_file'], self.license_token)
self.log.debug("Symlinked %s to %s",
self.cfg['license_file'],
self.license_token)
except OSError, err:
raise EasyBuildError(
"Failed to create symlink for license token at %s",
self.license_token)
else:
self.log.warning(
"No licence token found at either {0} or via 'license_file'"
.format(self.license_token))
# Only do the rest of the configuration if we're building from source
if not self.cfg['precompiled_binaries']:
# installation prefix
self.cfg.update('configopts', "-prefix %s" % self.installdir)
# compilers
# compilers & MPI
if self.toolchain.options.get('usempi', None):
self.cfg.update(
'configopts',
"-%s -%s" % (os.environ['CC_SEQ'], os.environ['F90_SEQ']))
if 'MPI_INC_DIR' in os.environ:
self.cfg.update(
'configopts',
"-mpp -mppbase %s" % os.environ['MPI_INC_DIR'])
else:
raise EasyBuildError("$MPI_INC_DIR not defined")
else:
self.cfg.update(
'configopts',
"-%s -%s" % (os.environ['CC'], os.environ['F90']))
# BLAS/LAPACK
if 'BLAS_LIB_DIR' in os.environ:
self.cfg.update(
'configopts',
"-blas -blaspath %s" % os.environ['BLAS_LIB_DIR'])
else:
raise EasyBuildError("$BLAS_LIB_DIR not defined")
if 'LAPACK_LIB_DIR' in os.environ:
self.cfg.update(
'configopts',
"-lapack -lapackpath %s" % os.environ['LAPACK_LIB_DIR'])
else:
raise EasyBuildError("$LAPACK_LIB_DIR not defined")
# 32 vs 64 bit
if self.toolchain.options.get('32bit', None):
self.cfg.update('configopts', '-i4')
else:
self.cfg.update('configopts', '-i8')
run_cmd("./configure -batch %s" % self.cfg['configopts'])
cfgfile = os.path.join(self.cfg['start_dir'], 'CONFIG')
cfgtxt = read_file(cfgfile)
# determine original LAUNCHER value
launcher_regex = re.compile('^LAUNCHER=(.*)$', re.M)
res = launcher_regex.search(cfgtxt)
if res:
self.orig_launcher = res.group(1)
self.log.debug("Found original value for LAUNCHER: %s",
self.orig_launcher)
else:
raise EasyBuildError("Failed to determine LAUNCHER value")
# determine full installation prefix
prefix_regex = re.compile('^PREFIX=(.*)$', re.M)
res = prefix_regex.search(cfgtxt)
if res:
self.full_prefix = res.group(1)
self.log.debug("Found full installation prefix: %s",
self.full_prefix)
else:
raise EasyBuildError(
"Failed to determine full installation prefix")
# determine MPI launcher command that can be used during build/test
# obtain command with specific number of cores (required by mpi_cmd_for), then replace that number with '%n'
launcher = self.toolchain.mpi_cmd_for('%x', self.cfg['parallel'])
launcher = launcher.replace(' %s' % self.cfg['parallel'], ' %n')
# patch CONFIG file to change LAUNCHER definition, in order to avoid having to start mpd
apply_regex_substitutions(
cfgfile, [(r"^(LAUNCHER\s*=\s*).*$", r"\1 %s" % launcher)])
# reread CONFIG and log contents
cfgtxt = read_file(cfgfile)
self.log.info("Contents of CONFIG file:\n%s", cfgtxt)
def test_step(self):
"""Run WPS test (requires large dataset to be downloaded). """
wpsdir = None
def run_wps_cmd(cmdname, mpi_cmd=True):
"""Run a WPS command, and check for success."""
cmd = os.path.join(wpsdir, "%s.exe" % cmdname)
if mpi_cmd:
if build_option('mpi_tests'):
cmd = self.toolchain.mpi_cmd_for(cmd, 1)
else:
self.log.info(
"Skipping MPI test for %s, since MPI tests are disabled",
cmd)
return
(out, _) = run_cmd(cmd, log_all=True, simple=False)
re_success = re.compile("Successful completion of %s" % cmdname)
if not re_success.search(out):
raise EasyBuildError("%s.exe failed (pattern '%s' not found)?",
cmdname, re_success.pattern)
if self.cfg['runtest']:
if not self.cfg['testdata']:
raise EasyBuildError("List of URLs for testdata not provided.")
wpsdir = os.path.join(self.builddir, self.wps_subdir)
try:
# create temporary directory
tmpdir = tempfile.mkdtemp()
change_dir(tmpdir)
# download data
testdata_paths = []
for testdata in self.cfg['testdata']:
path = self.obtain_file(testdata)
if not path:
raise EasyBuildError(
"Downloading file from %s failed?", testdata)
testdata_paths.append(path)
# unpack data
for path in testdata_paths:
srcdir = extract_file(path, tmpdir, change_into_dir=False)
change_dir(srcdir)
namelist_file = os.path.join(tmpdir, 'namelist.wps')
# GEOGRID
# setup directories and files
if LooseVersion(self.version) < LooseVersion("4.0"):
geog_data_dir = "geog"
else:
geog_data_dir = "WPS_GEOG"
for dir_name in os.listdir(os.path.join(tmpdir,
geog_data_dir)):
symlink(os.path.join(tmpdir, geog_data_dir, dir_name),
os.path.join(tmpdir, dir_name))
# copy namelist.wps file and patch it for geogrid
copy_file(os.path.join(wpsdir, 'namelist.wps'), namelist_file)
regex_subs = [(r"^(\s*geog_data_path\s*=\s*).*$",
r"\1 '%s'" % tmpdir)]
apply_regex_substitutions(namelist_file, regex_subs)
# GEOGRID.TBL
geogrid_dir = os.path.join(tmpdir, 'geogrid')
mkdir(geogrid_dir)
symlink(os.path.join(wpsdir, 'geogrid', 'GEOGRID.TBL.ARW'),
os.path.join(geogrid_dir, 'GEOGRID.TBL'))
# run geogrid.exe
run_wps_cmd("geogrid")
# UNGRIB
# determine start and end time stamps of grib files
grib_file_prefix = "fnl_"
k = len(grib_file_prefix)
fs = [
f for f in sorted(os.listdir('.'))
if f.startswith(grib_file_prefix)
]
start = "%s:00:00" % fs[0][k:]
end = "%s:00:00" % fs[-1][k:]
# copy namelist.wps file and patch it for ungrib
copy_file(os.path.join(wpsdir, 'namelist.wps'), namelist_file)
regex_subs = [
(r"^(\s*start_date\s*=\s*).*$",
r"\1 '%s','%s'," % (start, start)),
(r"^(\s*end_date\s*=\s*).*$",
r"\1 '%s','%s'," % (end, end)),
]
apply_regex_substitutions(namelist_file, regex_subs)
# copy correct Vtable
vtable_dir = os.path.join(wpsdir, 'ungrib', 'Variable_Tables')
if os.path.exists(os.path.join(vtable_dir, 'Vtable.ARW')):
copy_file(os.path.join(vtable_dir, 'Vtable.ARW'),
os.path.join(tmpdir, 'Vtable'))
elif os.path.exists(os.path.join(vtable_dir,
'Vtable.ARW.UPP')):
copy_file(os.path.join(vtable_dir, 'Vtable.ARW.UPP'),
os.path.join(tmpdir, 'Vtable'))
else:
raise EasyBuildError(
"Could not find Vtable file to use for testing ungrib")
# run link_grib.csh script
cmd = "%s %s*" % (os.path.join(
wpsdir, "link_grib.csh"), grib_file_prefix)
run_cmd(cmd, log_all=True, simple=True)
# run ungrib.exe
run_wps_cmd("ungrib", mpi_cmd=False)
# METGRID.TBL
metgrid_dir = os.path.join(tmpdir, 'metgrid')
mkdir(metgrid_dir)
symlink(os.path.join(wpsdir, 'metgrid', 'METGRID.TBL.ARW'),
os.path.join(metgrid_dir, 'METGRID.TBL'))
# run metgrid.exe
run_wps_cmd('metgrid')
# clean up
change_dir(self.builddir)
remove_dir(tmpdir)
except OSError as err:
raise EasyBuildError("Failed to run WPS test: %s", err)
os.path.join(self.installdir, 'lib'),
r"directory .* does not exist, try to create [Y]/n\n":
'',
}
run_cmd_qa(cmd, qa=qa, std_qa=stdqa, log_all=True, simple=True)
else:
if os.path.isfile(self.license_token):
run_cmd("make tuning")
super(EB_Molpro, self).install_step()
# put original LAUNCHER definition back in place in bin/molpro that got installed,
# since the value used during installation point to temporary files
molpro_path = os.path.join(self.full_prefix, 'bin', 'molpro')
apply_regex_substitutions(
molpro_path,
[(r"^(LAUNCHER\s*=\s*).*$", r"\1 %s" % self.orig_launcher)])
if self.cleanup_token_symlink:
try:
os.remove(self.license_token)
self.log.debug("Symlink to license token %s removed",
self.license_token)
except OSError, err:
raise EasyBuildError("Failed to remove %s: %s",
self.license_token, err)
def make_module_req_guess(self):
"""Customize $PATH guesses for Molpro module."""
guesses = super(EB_Molpro, self).make_module_req_guess()
guesses.update({
def configure_step(self):
"""Custom configure step for NAMD, we build charm++ first (if required)."""
# complete Charm ++ and NAMD architecture string with compiler family
comp_fam = self.toolchain.comp_family()
if self.toolchain.options.get('usempi', False):
charm_arch_comp = 'mpicxx'
else:
charm_arch_comps = {
toolchain.GCC: 'gcc',
toolchain.INTELCOMP: 'icc',
}
charm_arch_comp = charm_arch_comps.get(comp_fam, None)
namd_comps = {
toolchain.GCC: 'g++',
toolchain.INTELCOMP: 'icc',
}
namd_comp = namd_comps.get(comp_fam, None)
if charm_arch_comp is None or namd_comp is None:
raise EasyBuildError("Unknown compiler family, can't complete Charm++/NAMD target architecture.")
# NOTE: important to add smp BEFORE the compiler
# charm arch style is: mpi-linux-x86_64-smp-mpicxx
# otherwise the setting of name_charm_arch below will get things
# in the wrong order
if self.toolchain.options.get('openmp', False):
self.cfg.update('charm_arch', 'smp')
self.cfg.update('charm_arch', charm_arch_comp)
self.log.info("Updated 'charm_arch': %s" % self.cfg['charm_arch'])
self.namd_arch = '%s-%s' % (self.cfg['namd_basearch'], namd_comp)
self.log.info("Completed NAMD target architecture: %s" % self.namd_arch)
tup = (self.cfg['charm_arch'], self.cfg['charm_opts'], self.cfg['parallel'], os.environ['CXXFLAGS'])
cmd = "./build charm++ %s %s --with-numa -j%s %s -DMPICH_IGNORE_CXX_SEEK" % tup
charm_subdir = '.'.join(os.path.basename(self.charm_tarballs[0]).split('.')[:-1])
self.log.debug("Building Charm++ using cmd '%s' in '%s'" % (cmd, charm_subdir))
run_cmd(cmd, path=charm_subdir)
# compiler (options)
self.cfg.update('namd_cfg_opts', '--cc "%s" --cc-opts "%s"' % (os.environ['CC'], os.environ['CFLAGS']))
cxxflags = os.environ['CXXFLAGS']
if LooseVersion(self.version) >= LooseVersion('2.12'):
cxxflags += ' --std=c++11'
self.cfg.update('namd_cfg_opts', '--cxx "%s" --cxx-opts "%s"' % (os.environ['CXX'], cxxflags))
# NAMD dependencies: CUDA, TCL, FFTW
cuda = get_software_root('CUDA')
if cuda:
self.cfg.update('namd_cfg_opts', "--with-cuda --cuda-prefix %s" % cuda)
tcl = get_software_root('Tcl')
if tcl:
self.cfg.update('namd_cfg_opts', '--with-tcl --tcl-prefix %s' % tcl)
tclversion = '.'.join(get_software_version('Tcl').split('.')[0:2])
tclv_subs = [(r'-ltcl[\d.]*\s', '-ltcl%s ' % tclversion)]
apply_regex_substitutions(os.path.join('arch', '%s.tcl' % self.cfg['namd_basearch']), tclv_subs)
fftw = get_software_root('FFTW')
if fftw:
if LooseVersion(get_software_version('FFTW')) >= LooseVersion('3.0'):
if LooseVersion(self.version) >= LooseVersion('2.9'):
self.cfg.update('namd_cfg_opts', "--with-fftw3")
else:
raise EasyBuildError("Using FFTW v3.x only supported in NAMD v2.9 and up.")
else:
self.cfg.update('namd_cfg_opts', "--with-fftw")
self.cfg.update('namd_cfg_opts', "--fftw-prefix %s" % fftw)
namd_charm_arch = "--charm-arch %s" % '-'.join(self.cfg['charm_arch'].strip().split())
cmd = "./config %s %s %s " % (self.namd_arch, namd_charm_arch, self.cfg["namd_cfg_opts"])
run_cmd(cmd)
def configure_step(self):
"""Configure build:
- set required environment variables (for netCDF, JasPer)
- patch compile script and ungrib Makefile for non-default install paths of WRF and JasPer
- run configure script and figure how to select desired build option
- patch configure.wps file afterwards to fix 'serial compiler' setting
"""
# netCDF dependency check + setting env vars (NETCDF, NETCDFF)
set_netcdf_env_vars(self.log)
# WRF dependency check
wrf = get_software_root('WRF')
if wrf:
wrfdir = os.path.join(wrf, det_wrf_subdir(get_software_version('WRF')))
else:
raise EasyBuildError("WRF module not loaded?")
# patch compile script so that WRF is found
self.compile_script = "compile"
regex_subs = [(r"^(\s*set\s*WRF_DIR_PRE\s*=\s*)\${DEV_TOP}(.*)$", r"\1%s\2" % wrfdir)]
apply_regex_substitutions(self.compile_script, regex_subs)
# libpng dependency check
libpng = get_software_root('libpng')
zlib = get_software_root('zlib')
if libpng:
paths = [libpng]
if zlib:
paths.insert(0, zlib)
libpnginc = ' '.join(['-I%s' % os.path.join(path, 'include') for path in paths])
libpnglib = ' '.join(['-L%s' % os.path.join(path, 'lib') for path in paths])
else:
# define these as empty, assume that libpng will be available via OS (e.g. due to --filter-deps=libpng)
libpnglib = ""
libpnginc = ""
# JasPer dependency check + setting env vars
jasper = get_software_root('JasPer')
if jasper:
env.setvar('JASPERINC', os.path.join(jasper, "include"))
jasperlibdir = os.path.join(jasper, "lib")
env.setvar('JASPERLIB', jasperlibdir)
jasperlib = "-L%s" % jasperlibdir
else:
raise EasyBuildError("JasPer module not loaded?")
# patch ungrib Makefile so that JasPer is found
jasperlibs = "%s -ljasper %s -lpng" % (jasperlib, libpnglib)
regex_subs = [
(r"^(\s*-L\.\s*-l\$\(LIBTARGET\))(\s*;.*)$", r"\1 %s\2" % jasperlibs),
(r"^(\s*\$\(COMPRESSION_LIBS\))(\s*;.*)$", r"\1 %s\2" % jasperlibs),
]
apply_regex_substitutions(os.path.join('ungrib', 'src', 'Makefile'), regex_subs)
# patch arch/Config.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join("arch", "Config.pl"))
# configure
# determine build type option to look for
self.comp_fam = self.toolchain.comp_family()
build_type_option = None
if LooseVersion(self.version) >= LooseVersion("3.4"):
knownbuildtypes = {
'smpar': 'serial',
'dmpar': 'dmpar'
}
if self.comp_fam == toolchain.INTELCOMP: # @UndefinedVariable
build_type_option = " Linux x86_64, Intel compiler"
elif self.comp_fam == toolchain.GCC: # @UndefinedVariable
build_type_option = "Linux x86_64 g95 compiler"
else:
raise EasyBuildError("Don't know how to figure out build type to select.")
else:
knownbuildtypes = {
'smpar': 'serial',
'dmpar': 'DM parallel'
}
if self.comp_fam == toolchain.INTELCOMP: # @UndefinedVariable
build_type_option = "PC Linux x86_64, Intel compiler"
elif self.comp_fam == toolchain.GCC: # @UndefinedVariable
build_type_option = "PC Linux x86_64, gfortran compiler,"
knownbuildtypes['dmpar'] = knownbuildtypes['dmpar'].upper()
else:
raise EasyBuildError("Don't know how to figure out build type to select.")
# check and fetch selected build type
bt = self.cfg['buildtype']
if bt not in knownbuildtypes.keys():
raise EasyBuildError("Unknown build type: '%s'. Supported build types: %s", bt, knownbuildtypes.keys())
# fetch option number based on build type option and selected build type
build_type_question = "\s*(?P<nr>[0-9]+).\s*%s\s*\(?%s\)?\s*\n" % (build_type_option, knownbuildtypes[bt])
cmd = "./configure"
qa = {}
no_qa = [".*compiler is.*"]
std_qa = {
# named group in match will be used to construct answer
r"%s(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question: "%(nr)s",
}
run_cmd_qa(cmd, qa, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True)
# make sure correct compilers and compiler flags are being used
comps = {
'SCC': "%s -I$(JASPERINC) %s" % (os.getenv('CC'), libpnginc),
'SFC': os.getenv('F90'),
'DM_FC': os.getenv('MPIF90'),
'DM_CC': os.getenv('MPICC'),
'FC': os.getenv('MPIF90'),
'CC': os.getenv('MPICC'),
}
regex_subs = [(r"^(%s\s*=\s*).*$" % key, r"\1 %s" % val) for (key, val) in comps.items()]
apply_regex_substitutions('configure.wps', regex_subs)
def configure_step(self):
"""Configure build:
- set some magic environment variables
- run configure script
- adjust configure.wrf file if needed
"""
# define $NETCDF* for netCDF dependency (used when creating WRF module file)
set_netcdf_env_vars(self.log)
# HDF5 (optional) dependency
hdf5 = get_software_root('HDF5')
if hdf5:
# check if this is parallel HDF5
phdf5_bins = ['h5pcc', 'ph5diff']
parallel_hdf5 = True
for f in phdf5_bins:
if not os.path.exists(os.path.join(hdf5, 'bin', f)):
parallel_hdf5 = False
break
if not (hdf5 or parallel_hdf5):
raise EasyBuildError("Parallel HDF5 module not loaded?")
else:
env.setvar('PHDF5', hdf5)
else:
self.log.info("HDF5 module not loaded, assuming that's OK...")
# JasPer dependency check + setting env vars
jasper = get_software_root('JasPer')
if jasper:
jasperlibdir = os.path.join(jasper, "lib")
env.setvar('JASPERINC', os.path.join(jasper, "include"))
env.setvar('JASPERLIB', jasperlibdir)
else:
if os.getenv('JASPERINC') or os.getenv('JASPERLIB'):
raise EasyBuildError("JasPer module not loaded, but JASPERINC and/or JASPERLIB still set?")
else:
self.log.info("JasPer module not loaded, assuming that's OK...")
# enable support for large file support in netCDF
env.setvar('WRFIO_NCD_LARGE_FILE_SUPPORT', '1')
# patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join("arch", "Config_new.pl"))
# determine build type option to look for
build_type_option = None
self.comp_fam = self.toolchain.comp_family()
if self.comp_fam == toolchain.INTELCOMP: #@UndefinedVariable
if LooseVersion(self.version) >= LooseVersion('3.7'):
build_type_option = "INTEL\ \(ifort\/icc\)"
else:
build_type_option = "Linux x86_64 i486 i586 i686, ifort compiler with icc"
elif self.comp_fam == toolchain.GCC: #@UndefinedVariable
if LooseVersion(self.version) >= LooseVersion('3.7'):
build_type_option = "GNU\ \(gfortran\/gcc\)"
else:
build_type_option = "x86_64 Linux, gfortran compiler with gcc"
else:
raise EasyBuildError("Don't know how to figure out build type to select.")
# fetch selected build type (and make sure it makes sense)
known_build_types = ['serial', 'smpar', 'dmpar', 'dm+sm']
self.parallel_build_types = ["dmpar", "smpar", "dm+sm"]
bt = self.cfg['buildtype']
if not bt in known_build_types:
raise EasyBuildError("Unknown build type: '%s'. Supported build types: %s", bt, known_build_types)
# fetch option number based on build type option and selected build type
if LooseVersion(self.version) >= LooseVersion('3.7'):
# the two relevant lines in the configure output for WRF 3.8 are:
# 13. (serial) 14. (smpar) 15. (dmpar) 16. (dm+sm) INTEL (ifort/icc)
# 32. (serial) 33. (smpar) 34. (dmpar) 35. (dm+sm) GNU (gfortran/gcc)
build_type_question = "\s*(?P<nr>[0-9]+)\.\ \(%s\).*%s" % (bt, build_type_option)
else:
# the relevant lines in the configure output for WRF 3.6 are:
# 13. Linux x86_64 i486 i586 i686, ifort compiler with icc (serial)
# 14. Linux x86_64 i486 i586 i686, ifort compiler with icc (smpar)
# 15. Linux x86_64 i486 i586 i686, ifort compiler with icc (dmpar)
# 16. Linux x86_64 i486 i586 i686, ifort compiler with icc (dm+sm)
# 32. x86_64 Linux, gfortran compiler with gcc (serial)
# 33. x86_64 Linux, gfortran compiler with gcc (smpar)
# 34. x86_64 Linux, gfortran compiler with gcc (dmpar)
# 35. x86_64 Linux, gfortran compiler with gcc (dm+sm)
build_type_question = "\s*(?P<nr>[0-9]+).\s*%s\s*\(%s\)" % (build_type_option, bt)
# run configure script
cmd = "./configure"
qa = {
# named group in match will be used to construct answer
"Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]:": "1",
"Compile for nesting? (0=no nesting, 1=basic, 2=preset moves, 3=vortex following) [default 0]:": "0"
}
no_qa = [
"testing for fseeko and fseeko64",
r"If you wish to change the default options, edit the file:[\s\n]*arch/configure_new.defaults"
]
std_qa = {
# named group in match will be used to construct answer
r"%s.*\n(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question: "%(nr)s",
}
run_cmd_qa(cmd, qa, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True)
cfgfile = 'configure.wrf'
# make sure correct compilers are being used
comps = {
'SCC': os.getenv('CC'),
'SFC': os.getenv('F90'),
'CCOMP': os.getenv('CC'),
'DM_FC': os.getenv('MPIF90'),
'DM_CC': "%s -DMPI2_SUPPORT" % os.getenv('MPICC'),
}
regex_subs = [(r"^(%s\s*=\s*).*$" % k, r"\1 %s" % v) for (k, v) in comps.items()]
apply_regex_substitutions(cfgfile, regex_subs)
# rewrite optimization options if desired
if self.cfg['rewriteopts']:
# replace default -O3 option in configure.wrf with CFLAGS/FFLAGS from environment
self.log.info("Rewriting optimization options in %s" % cfgfile)
# set extra flags for Intel compilers
# see http://software.intel.com/en-us/forums/showthread.php?t=72109&p=1#146748
if self.comp_fam == toolchain.INTELCOMP: #@UndefinedVariable
# -O3 -heap-arrays is required to resolve compilation error
for envvar in ['CFLAGS', 'FFLAGS']:
val = os.getenv(envvar)
if '-O3' in val:
env.setvar(envvar, '%s -heap-arrays' % val)
self.log.info("Updated %s to '%s'" % (envvar, os.getenv(envvar)))
# replace -O3 with desired optimization options
regex_subs = [
(r"^(FCOPTIM.*)(\s-O3)(\s.*)$", r"\1 %s \3" % os.getenv('FFLAGS')),
(r"^(CFLAGS_LOCAL.*)(\s-O3)(\s.*)$", r"\1 %s \3" % os.getenv('CFLAGS')),
]
apply_regex_substitutions(cfgfile, regex_subs)
]
env.setvar('GMKTMP', self.builddir)
env.setvar('GMKFILE', self.conf_file)
run_cmd_qa("gmkfilemaker", qa, std_qa=stdqa, no_qa=no_qa)
# set environment variables for installation dirs
env.setvar('ROOTPACK', os.path.join(self.installdir, 'rootpack'))
env.setvar('ROOTBIN', os.path.join(self.installdir, 'rootpack'))
env.setvar('HOMEPACK', os.path.join(self.installdir, 'pack'))
env.setvar('HOMEBIN', os.path.join(self.installdir, 'pack'))
# patch config file to include right Fortran compiler flags
regex_subs = [(r"^(FRTFLAGS\s*=.*)$", r"\1 %s" % os.getenv('FFLAGS'))]
apply_regex_substitutions(self.conf_filepath, regex_subs)
def build_step(self):
"""No separate build procedure for ALADIN (see install_step)."""
pass
def test_step(self):
"""Custom built-in test procedure for ALADIN."""
if self.cfg['runtest']:
cmd = "test-command"
run_cmd(cmd, simple=True, log_all=True, log_output=True)
def install_step(self):
"""Custom install procedure for ALADIN."""
def configure_step(self):
"""Custom configuration procedure for binutils: statically link to zlib, configure options."""
# take sysroot configuration option into account;
# make sure we don't use None going forward since the value is used in os.path.join expressions
sysroot = build_option('sysroot') or os.path.sep
libs = ''
if self.toolchain.is_system_toolchain():
# determine list of 'lib' directories to use rpath for;
# this should 'harden' the resulting binutils to bootstrap GCC
# (no trouble when other libstdc++ is build etc)
# The installed lib dir must come first though to avoid taking system libs over installed ones, see:
# https://github.com/easybuilders/easybuild-easyconfigs/issues/10056
# Escaping: Double $$ for Make, \$ for shell to get literal $ORIGIN in the file
libdirs = [r'\$\$ORIGIN/../lib']
for libdir in [
'lib', 'lib64',
os.path.join('lib', 'x86_64-linux-gnu')
]:
libdir = os.path.join(sysroot, 'usr', libdir)
# also consider /lib, /lib64 (without /usr/)
alt_libdir = os.path.join(sysroot, libdir)
if os.path.exists(libdir):
libdirs.append(libdir)
if os.path.exists(alt_libdir) and not os.path.samefile(
libdir, alt_libdir):
libdirs.append(alt_libdir)
elif os.path.exists(alt_libdir):
libdirs.append(alt_libdir)
# Mind the single quotes
libs += ' '.join("-Wl,-rpath='%s'" % libdir for libdir in libdirs)
# configure using `--with-system-zlib` if zlib is a (build) dependency
zlibroot = get_software_root('zlib')
if zlibroot:
self.cfg.update('configopts', '--with-system-zlib')
# statically link to zlib only if it is a build dependency
# see https://github.com/easybuilders/easybuild-easyblocks/issues/1350
build_deps = self.cfg.dependencies(build_only=True)
if any(dep['name'] == 'zlib' for dep in build_deps):
libz_path = os.path.join(zlibroot, get_software_libdir('zlib'),
'libz.a')
# for recent binutils versions, we need to override ZLIB in Makefile.in of components
if LooseVersion(self.version) >= LooseVersion('2.26'):
regex_subs = [
(r"^(ZLIB\s*=\s*).*$", r"\1%s" % libz_path),
(r"^(ZLIBINC\s*=\s*).*$",
r"\1-I%s" % os.path.join(zlibroot, 'include')),
]
for makefile in glob.glob(
os.path.join(self.cfg['start_dir'], '*',
'Makefile.in')):
apply_regex_substitutions(makefile, regex_subs)
# for older versions, injecting the path to the static libz library into $LIBS works
else:
libs += ' ' + libz_path
# Using double quotes for LIBS to allow single quotes in libs
self.cfg.update('preconfigopts', 'LIBS="%s"' % libs)
self.cfg.update('prebuildopts', 'LIBS="%s"' % libs)
# explicitly configure binutils to use / as sysroot
# this is required to ensure the binutils installation works correctly with a (system)
# GCC compiler that was explicitly configured with --with-sysroot=/;
# we should *not* use the value of the EasyBuild --sysroot configuration option here,
# since that leads to weird errors where the sysroot path is duplicated, like:
# /bin/ld.gold: error: cannot open /<sysroot>/<sysroot>/lib64/libc.so.6: No such file or directory
self.cfg.update('configopts', '--with-sysroot=/')
# build both static and shared libraries for recent binutils versions (default is only static)
if LooseVersion(self.version) > LooseVersion('2.24'):
self.cfg.update('configopts', "--enable-shared --enable-static")
# enable gold linker with plugin support, use ld as default linker (for recent versions of binutils)
if LooseVersion(self.version) > LooseVersion('2.24'):
self.cfg.update(
'configopts',
"--enable-gold --enable-plugins --enable-ld=default")
if LooseVersion(self.version) >= LooseVersion('2.34'):
if self.cfg['use_debuginfod']:
self.cfg.update('configopts', '--with-debuginfod')
else:
self.cfg.update('configopts', '--without-debuginfod')
# complete configuration with configure_method of parent
super(EB_binutils, self).configure_step()
if self.cfg['install_libiberty']:
cflags = os.getenv('CFLAGS')
if cflags:
self.cfg.update('buildopts', 'CFLAGS="$CFLAGS -fPIC"')
else:
# if $CFLAGS is not defined, make sure we retain "-g -O2",
# since not specifying any optimization level implies -O0...
self.cfg.update('buildopts', 'CFLAGS="-g -O2 -fPIC"')
class EB_imkl(IntelBase):
"""
Class that can be used to install mkl
- tested with 10.2.1.017
-- will fail for all older versions (due to newer silent installer)
"""
@staticmethod
def extra_options():
"""Add easyconfig parameters custom to imkl (e.g. interfaces)."""
extra_vars = {
'interfaces':
[True, "Indicates whether interfaces should be built", CUSTOM],
}
return IntelBase.extra_options(extra_vars)
def __init__(self, *args, **kwargs):
super(EB_imkl, self).__init__(*args, **kwargs)
# make sure $MKLROOT isn't set, it's known to cause problems with the installation
self.cfg.update('unwanted_env_vars', ['MKLROOT'])
def install_step(self):
"""
Actual installation
- create silent cfg file
- execute command
"""
silent_cfg_names_map = None
silent_cfg_extras = None
if LooseVersion(self.version) < LooseVersion('11.1'):
# since imkl v11.1, silent.cfg has been slightly changed to be 'more standard'
silent_cfg_names_map = {
'activation_name': ACTIVATION_NAME_2012,
'license_file_name': LICENSE_FILE_NAME_2012,
}
if LooseVersion(self.version) >= LooseVersion(
'11.1') and self.install_components is None:
silent_cfg_extras = {
'COMPONENTS': 'ALL',
}
super(EB_imkl,
self).install_step(silent_cfg_names_map=silent_cfg_names_map,
silent_cfg_extras=silent_cfg_extras)
def make_module_req_guess(self):
"""
A dictionary of possible directories to look for
"""
if LooseVersion(self.version) >= LooseVersion('10.3'):
if self.cfg['m32']:
raise EasyBuildError(
"32-bit not supported yet for IMKL v%s (>= 10.3)",
self.version)
else:
retdict = {
'PATH': [
'bin', 'mkl/bin', 'mkl/bin/intel64',
'composerxe-2011/bin'
],
'LD_LIBRARY_PATH': ['lib/intel64', 'mkl/lib/intel64'],
'LIBRARY_PATH': ['lib/intel64', 'mkl/lib/intel64'],
'MANPATH': ['man', 'man/en_US'],
'CPATH': ['mkl/include', 'mkl/include/fftw'],
'FPATH': ['mkl/include', 'mkl/include/fftw'],
}
if LooseVersion(self.version) >= LooseVersion('11.0'):
if LooseVersion(self.version) >= LooseVersion('11.3'):
retdict['MIC_LD_LIBRARY_PATH'] = [
'lib/intel64_lin_mic', 'mkl/lib/mic'
]
elif LooseVersion(self.version) >= LooseVersion('11.1'):
retdict['MIC_LD_LIBRARY_PATH'] = [
'lib/mic', 'mkl/lib/mic'
]
else:
retdict['MIC_LD_LIBRARY_PATH'] = [
'compiler/lib/mic', 'mkl/lib/mic'
]
return retdict
else:
if self.cfg['m32']:
return {
'PATH': ['bin', 'bin/ia32', 'tbb/bin/ia32'],
'LD_LIBRARY_PATH': ['lib', 'lib/32'],
'LIBRARY_PATH': ['lib', 'lib/32'],
'MANPATH': ['man', 'share/man', 'man/en_US'],
'CPATH': ['include'],
'FPATH': ['include']
}
else:
return {
'PATH': ['bin', 'bin/intel64', 'tbb/bin/em64t'],
'LD_LIBRARY_PATH': ['lib', 'lib/em64t'],
'LIBRARY_PATH': ['lib', 'lib/em64t'],
'MANPATH': ['man', 'share/man', 'man/en_US'],
'CPATH': ['include'],
'FPATH': ['include'],
}
def make_module_extra(self):
"""Overwritten from Application to add extra txt"""
txt = super(EB_imkl, self).make_module_extra()
txt += self.module_generator.set_environment(
'MKLROOT', os.path.join(self.installdir, 'mkl'))
return txt
def post_install_step(self):
"""
Install group libraries and interfaces (if desired).
"""
super(EB_imkl, self).post_install_step()
shlib_ext = get_shared_lib_ext()
# reload the dependencies
self.load_dependency_modules()
if self.cfg['m32']:
extra = {
'libmkl.%s' % shlib_ext:
'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a':
'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)',
'libmkl_lapack.a':
'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
else:
extra = {
'libmkl.%s' % shlib_ext:
'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a':
'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)',
'libmkl_lapack.a':
'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
if LooseVersion(self.version) >= LooseVersion('10.3'):
libsubdir = os.path.join('mkl', 'lib', 'intel64')
else:
if self.cfg['m32']:
libsubdir = os.path.join('lib', '32')
else:
libsubdir = os.path.join('lib', 'em64t')
for fil, txt in extra.items():
dest = os.path.join(self.installdir, libsubdir, fil)
if not os.path.exists(dest):
try:
f = open(dest, 'w')
f.write(txt)
f.close()
self.log.info("File %s written" % dest)
except IOError, err:
raise EasyBuildError("Can't write file %s: %s", dest, err)
# build the mkl interfaces, if desired
if self.cfg['interfaces']:
if LooseVersion(self.version) >= LooseVersion('10.3'):
intsubdir = os.path.join('mkl', 'interfaces')
inttarget = 'libintel64'
else:
intsubdir = 'interfaces'
if self.cfg['m32']:
inttarget = 'lib32'
else:
inttarget = 'libem64t'
cmd = "make -f makefile %s" % inttarget
# blas95 and lapack95 need more work, ignore for now
# blas95 and lapack also need include/.mod to be processed
fftw2libs = ['fftw2xc', 'fftw2xf']
fftw3libs = ['fftw3xc', 'fftw3xf']
cdftlibs = ['fftw2x_cdft']
if LooseVersion(self.version) >= LooseVersion('10.3'):
cdftlibs.append('fftw3x_cdft')
interfacedir = os.path.join(self.installdir, intsubdir)
try:
os.chdir(interfacedir)
self.log.info("Changed to interfaces directory %s" %
interfacedir)
except OSError, err:
raise EasyBuildError("Can't change to interfaces directory %s",
interfacedir)
compopt = None
# determine whether we're using a non-Intel GCC-based or PGI-based toolchain
# can't use toolchain.comp_family, because of dummy toolchain used when installing imkl
if get_software_root('icc') is None:
# check for PGI first, since there's a GCC underneath PGI too...
if get_software_root('PGI'):
compopt = 'compiler=pgi'
elif get_software_root('GCC'):
compopt = 'compiler=gnu'
else:
raise EasyBuildError(
"Not using Intel/GCC/PGI compilers, don't know how to build wrapper libs"
)
else:
compopt = 'compiler=intel'
# patch makefiles for cdft wrappers when PGI is used as compiler
if get_software_root('PGI'):
regex_subs = [
# pgi should be considered as a valid compiler
("intel gnu", "intel gnu pgi"),
# transform 'gnu' case to 'pgi' case
(r"ifeq \(\$\(compiler\),gnu\)", "ifeq ($(compiler),pgi)"),
('=gcc', '=pgcc'),
# correct flag to use C99 standard
('-std=c99', '-c99'),
# -Wall and -Werror are not valid options for pgcc, no close equivalent
('-Wall', ''),
('-Werror', ''),
]
for lib in cdftlibs:
apply_regex_substitutions(
os.path.join(interfacedir, lib, 'makefile'),
regex_subs)
for lib in fftw2libs + fftw3libs + cdftlibs:
buildopts = [compopt]
if lib in fftw3libs:
buildopts.append('install_to=$INSTALL_DIR')
elif lib in cdftlibs:
mpi_spec = None
# check whether MPI_FAMILY constant is defined, so mpi_family() can be used
if hasattr(self.toolchain, 'MPI_FAMILY'
) and self.toolchain.MPI_FAMILY is not None:
mpi_spec_by_fam = {
toolchain.MPICH:
'mpich2', # MPICH is MPICH v3.x, which is MPICH2 compatible
toolchain.MPICH2: 'mpich2',
toolchain.MVAPICH2: 'mpich2',
toolchain.OPENMPI: 'openmpi',
}
mpi_fam = self.toolchain.mpi_family()
mpi_spec = mpi_spec_by_fam.get(mpi_fam)
self.log.debug(
"Determined MPI specification based on MPI toolchain component: %s"
% mpi_spec)
else:
# can't use toolchain.mpi_family, because of dummy toolchain
if get_software_root('MPICH2') or get_software_root(
'MVAPICH2'):
mpi_spec = 'mpich2'
elif get_software_root('OpenMPI'):
mpi_spec = 'openmpi'
self.log.debug(
"Determined MPI specification based on loaded MPI module: %s"
% mpi_spec)
if mpi_spec is not None:
buildopts.append('mpi=%s' % mpi_spec)
precflags = ['']
if lib.startswith('fftw2x') and not self.cfg['m32']:
# build both single and double precision variants
precflags = [
'PRECISION=MKL_DOUBLE', 'PRECISION=MKL_SINGLE'
]
intflags = ['']
if lib in cdftlibs and not self.cfg['m32']:
# build both 32-bit and 64-bit interfaces
intflags = ['interface=lp64', 'interface=ilp64']
allopts = [
list(opts)
for opts in itertools.product(intflags, precflags)
]
for flags, extraopts in itertools.product(['', '-fPIC'],
allopts):
tup = (lib, flags, buildopts, extraopts)
self.log.debug(
"Building lib %s with: flags %s, buildopts %s, extraopts %s"
% tup)
tmpbuild = tempfile.mkdtemp(dir=self.builddir)
self.log.debug("Created temporary directory %s" % tmpbuild)
# always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS
# fftw2x(c|f): use $INSTALL_DIR, $CFLAGS and $COPTS
# fftw3x(c|f): use $CFLAGS
# fftw*cdft: use $INSTALL_DIR and $SPEC_OPT
env.setvar('INSTALL_DIR', tmpbuild)
env.setvar('SPEC_OPT', flags)
env.setvar('COPTS', flags)
env.setvar('CFLAGS', flags)
try:
intdir = os.path.join(interfacedir, lib)
os.chdir(intdir)
self.log.info("Changed to interface %s directory %s" %
(lib, intdir))
except OSError, err:
raise EasyBuildError(
"Can't change to interface %s directory %s: %s",
lib, intdir, err)
fullcmd = "%s %s" % (cmd, ' '.join(buildopts + extraopts))
res = run_cmd(fullcmd, log_all=True, simple=True)
if not res:
raise EasyBuildError(
"Building %s (flags: %s, fullcmd: %s) failed", lib,
flags, fullcmd)
for fn in os.listdir(tmpbuild):
src = os.path.join(tmpbuild, fn)
if flags == '-fPIC':
# add _pic to filename
ff = fn.split('.')
fn = '.'.join(ff[:-1]) + '_pic.' + ff[-1]
dest = os.path.join(self.installdir, libsubdir, fn)
try:
if os.path.isfile(src):
shutil.move(src, dest)
self.log.info("Moved %s to %s" % (src, dest))
except OSError, err:
raise EasyBuildError("Failed to move %s to %s: %s",
src, dest, err)
rmtree2(tmpbuild)
def install_step(self):
"""Custom install procedure for Trinity."""
version = LooseVersion(self.version)
if version > LooseVersion('2012') and version < LooseVersion('2012-10-05'):
self.inchworm()
self.chrysalis()
self.kmer()
self.butterfly()
bwapluginver = self.cfg['bwapluginver']
if bwapluginver:
self.trinityplugin('bwa-%s-patched_multi_map' % bwapluginver)
if self.cfg['RSEMmod']:
self.trinityplugin('RSEM-mod', cc=os.getenv('CXX'))
else:
self.jellyfish()
inchworm_flags = self.inchworm(run=False)
chrysalis_flags = self.chrysalis(run=False)
cc = os.getenv('CC')
cxx = os.getenv('CXX')
lib_flags = ""
for lib in ['ncurses', 'zlib']:
libroot = get_software_root(lib)
if libroot:
lib_flags += " -L%s/lib" % libroot
if version >= LooseVersion('2.0') and version < LooseVersion('3.0'):
regex_subs = [
(r'^( INCHWORM_CONFIGURE_FLAGS\s*=\s*).*$', r'\1%s' % inchworm_flags),
(r'^( CHRYSALIS_MAKE_FLAGS\s*=\s*).*$', r'\1%s' % chrysalis_flags),
]
else:
regex_subs = [
(r'^(INCHWORM_CONFIGURE_FLAGS\s*=\s*).*$', r'\1%s' % inchworm_flags),
(r'^(CHRYSALIS_MAKE_FLAGS\s*=\s*).*$', r'\1%s' % chrysalis_flags),
(r'(/rsem && \$\(MAKE\))\s*$',
r'\1 CC=%s CXX="%s %s" CFLAGS_EXTRA="%s"\n' % (cc, cxx, lib_flags, lib_flags)),
(r'(/fastool && \$\(MAKE\))\s*$',
r'\1 CC="%s -std=c99" CFLAGS="%s ${CFLAGS}"\n' % (cc, lib_flags)),
]
apply_regex_substitutions('Makefile', regex_subs)
trinity_compiler = None
comp_fam = self.toolchain.comp_family()
if comp_fam in [toolchain.INTELCOMP]:
trinity_compiler = "intel"
elif comp_fam in [toolchain.GCC]:
trinity_compiler = "gcc"
else:
raise EasyBuildError("Don't know how to set TRINITY_COMPILER for %s compiler", comp_fam)
explicit_make_args = ''
if version >= LooseVersion('2.0') and version < LooseVersion('3.0'):
explicit_make_args = 'all plugins'
cmd = "make TRINITY_COMPILER=%s %s" % (trinity_compiler, explicit_make_args)
run_cmd(cmd)
# butterfly is not included in standard build
self.butterfly()
# remove sample data if desired
if not self.cfg['withsampledata']:
try:
shutil.rmtree(os.path.join(self.cfg['start_dir'], 'sample_data'))
except OSError, err:
raise EasyBuildError("Failed to remove sample data: %s", err)
def fixup_hardcoded_paths(self):
"""Patch out hard coded paths to compiler and binutils tools"""
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if not binutils_root or not gcc_root:
self.log.info(
"Not patching Bazel build scripts, installation prefix for binutils/GCC not found"
)
return
# replace hardcoded paths in (unix_)cc_configure.bzl
# hard-coded paths in (unix_)cc_configure.bzl were removed in 0.19.0
if LooseVersion(self.version) < LooseVersion('0.19.0'):
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
]
for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
filepath = os.path.join('tools', 'cpp', conf_bzl)
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
# replace hardcoded paths in CROSSTOOL
# CROSSTOOL script is no longer there in Bazel 0.24.0
if LooseVersion(self.version) < LooseVersion('0.24.0'):
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
gcc_lib_inc_bis = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if not os.path.exists(gcc_lib_inc_bis):
self.log.info(
"Derived directory %s does not exist, falling back to %s",
gcc_lib_inc_bis, gcc_lib_inc)
gcc_lib_inc_bis = gcc_lib_inc
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'(cxx_builtin_include_directory:.*)/usr/lib/gcc',
r'\1%s' % gcc_lib_inc),
(r'(cxx_builtin_include_directory:.*)/usr/local/include',
r'\1%s' % gcc_lib_inc_bis),
(r'(cxx_builtin_include_directory:.*)/usr/include',
r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin',
tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'",
tool)
apply_regex_substitutions(
os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
def post_install_step(self):
"""
Install group libraries and interfaces (if desired).
"""
super(EB_imkl, self).post_install_step()
shlib_ext = get_shared_lib_ext()
# reload the dependencies
self.load_dependency_modules()
if self.cfg['m32']:
extra = {
'libmkl.%s' % shlib_ext : 'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)',
'libmkl_lapack.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
else:
extra = {
'libmkl.%s' % shlib_ext: 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)',
'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
if LooseVersion(self.version) >= LooseVersion('10.3'):
libsubdir = os.path.join('mkl', 'lib', 'intel64')
else:
if self.cfg['m32']:
libsubdir = os.path.join('lib', '32')
else:
libsubdir = os.path.join('lib', 'em64t')
for fil, txt in extra.items():
dest = os.path.join(self.installdir, libsubdir, fil)
if not os.path.exists(dest):
try:
f = open(dest, 'w')
f.write(txt)
f.close()
self.log.info("File %s written" % dest)
except IOError as err:
raise EasyBuildError("Can't write file %s: %s", dest, err)
# build the mkl interfaces, if desired
if self.cfg['interfaces']:
if LooseVersion(self.version) >= LooseVersion('10.3'):
intsubdir = os.path.join('mkl', 'interfaces')
inttarget = 'libintel64'
else:
intsubdir = 'interfaces'
if self.cfg['m32']:
inttarget = 'lib32'
else:
inttarget = 'libem64t'
cmd = "make -f makefile %s" % inttarget
# blas95 and lapack95 need more work, ignore for now
# blas95 and lapack also need include/.mod to be processed
fftw2libs = ['fftw2xc', 'fftw2xf']
fftw3libs = ['fftw3xc', 'fftw3xf']
interfacedir = os.path.join(self.installdir, intsubdir)
change_dir(interfacedir)
self.log.info("Changed to interfaces directory %s", interfacedir)
compopt = None
# determine whether we're using a non-Intel GCC-based or PGI-based toolchain
# can't use toolchain.comp_family, because of system toolchain used when installing imkl
if get_software_root('icc') is None:
# check for PGI first, since there's a GCC underneath PGI too...
if get_software_root('PGI'):
compopt = 'compiler=pgi'
elif get_software_root('GCC'):
compopt = 'compiler=gnu'
else:
raise EasyBuildError("Not using Intel/GCC/PGI compilers, don't know how to build wrapper libs")
else:
compopt = 'compiler=intel'
# patch makefiles for cdft wrappers when PGI is used as compiler
if get_software_root('PGI'):
regex_subs = [
# pgi should be considered as a valid compiler
("intel gnu", "intel gnu pgi"),
# transform 'gnu' case to 'pgi' case
(r"ifeq \(\$\(compiler\),gnu\)", "ifeq ($(compiler),pgi)"),
('=gcc', '=pgcc'),
# correct flag to use C99 standard
('-std=c99', '-c99'),
# -Wall and -Werror are not valid options for pgcc, no close equivalent
('-Wall', ''),
('-Werror', ''),
]
for lib in self.cdftlibs:
apply_regex_substitutions(os.path.join(interfacedir, lib, 'makefile'), regex_subs)
for lib in fftw2libs + fftw3libs + self.cdftlibs:
buildopts = [compopt]
if lib in fftw3libs:
buildopts.append('install_to=$INSTALL_DIR')
elif lib in self.cdftlibs:
if self.mpi_spec is not None:
buildopts.append('mpi=%s' % self.mpi_spec)
precflags = ['']
if lib.startswith('fftw2x') and not self.cfg['m32']:
# build both single and double precision variants
precflags = ['PRECISION=MKL_DOUBLE', 'PRECISION=MKL_SINGLE']
intflags = ['']
if lib in self.cdftlibs and not self.cfg['m32']:
# build both 32-bit and 64-bit interfaces
intflags = ['interface=lp64', 'interface=ilp64']
allopts = [list(opts) for opts in itertools.product(intflags, precflags)]
for flags, extraopts in itertools.product(['', '-fPIC'], allopts):
tup = (lib, flags, buildopts, extraopts)
self.log.debug("Building lib %s with: flags %s, buildopts %s, extraopts %s" % tup)
tmpbuild = tempfile.mkdtemp(dir=self.builddir)
self.log.debug("Created temporary directory %s" % tmpbuild)
# always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS
# fftw2x(c|f): use $INSTALL_DIR, $CFLAGS and $COPTS
# fftw3x(c|f): use $CFLAGS
# fftw*cdft: use $INSTALL_DIR and $SPEC_OPT
env.setvar('INSTALL_DIR', tmpbuild)
env.setvar('SPEC_OPT', flags)
env.setvar('COPTS', flags)
env.setvar('CFLAGS', flags)
try:
intdir = os.path.join(interfacedir, lib)
os.chdir(intdir)
self.log.info("Changed to interface %s directory %s" % (lib, intdir))
except OSError as err:
raise EasyBuildError("Can't change to interface %s directory %s: %s", lib, intdir, err)
fullcmd = "%s %s" % (cmd, ' '.join(buildopts + extraopts))
res = run_cmd(fullcmd, log_all=True, simple=True)
if not res:
raise EasyBuildError("Building %s (flags: %s, fullcmd: %s) failed", lib, flags, fullcmd)
for fn in os.listdir(tmpbuild):
src = os.path.join(tmpbuild, fn)
if flags == '-fPIC':
# add _pic to filename
ff = fn.split('.')
fn = '.'.join(ff[:-1]) + '_pic.' + ff[-1]
dest = os.path.join(self.installdir, libsubdir, fn)
try:
if os.path.isfile(src):
shutil.move(src, dest)
self.log.info("Moved %s to %s" % (src, dest))
except OSError as err:
raise EasyBuildError("Failed to move %s to %s: %s", src, dest, err)
remove_dir(tmpbuild)
def configure_step(self):
"""
Custom configure and build procedure for Siesta.
- There are two main builds to do, siesta and transiesta
- In addition there are multiple support tools to build
"""
start_dir = self.cfg['start_dir']
obj_dir = os.path.join(start_dir, 'Obj')
arch_make = os.path.join(obj_dir, 'arch.make')
bindir = os.path.join(start_dir, 'bin')
par = ''
if LooseVersion(self.version) >= LooseVersion('4.1'):
par = '-j %s' % self.cfg['parallel']
# enable OpenMP support if desired
env_var_suff = ''
if self.toolchain.options.get('openmp', None):
env_var_suff = '_MT'
scalapack = os.environ['LIBSCALAPACK' + env_var_suff]
blacs = os.environ['LIBSCALAPACK' + env_var_suff]
lapack = os.environ['LIBLAPACK' + env_var_suff]
blas = os.environ['LIBBLAS' + env_var_suff]
if get_software_root('imkl') or get_software_root('FFTW'):
fftw = os.environ['LIBFFT' + env_var_suff]
else:
fftw = None
regex_newlines = []
regex_subs = [
('dc_lapack.a', ''),
(r'^NETCDF_INTERFACE\s*=.*$', ''),
('libsiestaBLAS.a', ''),
('libsiestaLAPACK.a', ''),
# Needed here to allow 4.1-b1 to be built with openmp
(r"^(LDFLAGS\s*=).*$", r"\1 %s %s" % (os.environ['FCFLAGS'], os.environ['LDFLAGS'])),
]
netcdff_loc = get_software_root('netCDF-Fortran')
if netcdff_loc:
# Needed for gfortran at least
regex_newlines.append((r"^(ARFLAGS_EXTRA\s*=.*)$", r"\1\nNETCDF_INCFLAGS = -I%s/include" % netcdff_loc))
if fftw:
fft_inc, fft_lib = os.environ['FFT_INC_DIR'], os.environ['FFT_LIB_DIR']
fppflags = r"\1\nFFTW_INCFLAGS = -I%s\nFFTW_LIBS = -L%s %s" % (fft_inc, fft_lib, fftw)
regex_newlines.append((r'(FPPFLAGS\s*=.*)$', fppflags))
# Make a temp installdir during the build of the various parts
mkdir(bindir)
# change to actual build dir
change_dir(obj_dir)
# Populate start_dir with makefiles
run_cmd(os.path.join(start_dir, 'Src', 'obj_setup.sh'), log_all=True, simple=True, log_output=True)
if LooseVersion(self.version) < LooseVersion('4.1-b2'):
# MPI?
if self.toolchain.options.get('usempi', None):
self.cfg.update('configopts', '--enable-mpi')
# BLAS and LAPACK
self.cfg.update('configopts', '--with-blas="%s"' % blas)
self.cfg.update('configopts', '--with-lapack="%s"' % lapack)
# ScaLAPACK (and BLACS)
self.cfg.update('configopts', '--with-scalapack="%s"' % scalapack)
self.cfg.update('configopts', '--with-blacs="%s"' % blacs)
# NetCDF-Fortran
if netcdff_loc:
self.cfg.update('configopts', '--with-netcdf=-lnetcdff')
# Configure is run in obj_dir, configure script is in ../Src
super(EB_Siesta, self).configure_step(cmd_prefix='../Src/')
if LooseVersion(self.version) > LooseVersion('4.0'):
regex_subs_Makefile = [
(r'CFLAGS\)-c', r'CFLAGS) -c'),
]
apply_regex_substitutions('Makefile', regex_subs_Makefile)
else: # there's no configure on newer versions
if self.toolchain.comp_family() in [toolchain.INTELCOMP]:
copy_file(os.path.join(obj_dir, 'intel.make'), arch_make)
elif self.toolchain.comp_family() in [toolchain.GCC]:
copy_file(os.path.join(obj_dir, 'gfortran.make'), arch_make)
else:
raise EasyBuildError("There is currently no support for compiler: %s", self.toolchain.comp_family())
if self.toolchain.options.get('usempi', None):
regex_subs.extend([
(r"^(CC\s*=\s*).*$", r"\1%s" % os.environ['MPICC']),
(r"^(FC\s*=\s*).*$", r"\1%s" % os.environ['MPIF90']),
(r"^(FPPFLAGS\s*=.*)$", r"\1 -DMPI"),
])
regex_newlines.append((r"^(FPPFLAGS\s*=.*)$", r"\1\nMPI_INTERFACE = libmpi_f90.a\nMPI_INCLUDE = ."))
complibs = scalapack
else:
complibs = lapack
regex_subs.extend([
(r"^(LIBS\s*=\s).*$", r"\1 %s" % complibs),
# Needed for a couple of the utils
(r"^(FFLAGS\s*=\s*).*$", r"\1 -fPIC %s" % os.environ['FCFLAGS']),
])
regex_newlines.append((r"^(COMP_LIBS\s*=.*)$", r"\1\nWXML = libwxml.a"))
if netcdff_loc:
regex_subs.extend([
(r"^(LIBS\s*=.*)$", r"\1 $(NETCDF_LIBS)"),
(r"^(FPPFLAGS\s*=.*)$", r"\1 -DCDF"),
])
regex_newlines.append((r"^(COMP_LIBS\s*=.*)$", r"\1\nNETCDF_LIBS = -lnetcdff"))
apply_regex_substitutions(arch_make, regex_subs)
# individually apply substitutions that add lines
for regex_nl in regex_newlines:
apply_regex_substitutions(arch_make, [regex_nl])
run_cmd('make %s' % par, log_all=True, simple=True, log_output=True)
# Put binary in temporary install dir
copy_file(os.path.join(obj_dir, 'siesta'), bindir)
if self.cfg['with_utils']:
# Make the utils
change_dir(os.path.join(start_dir, 'Util'))
# clean_all.sh might be missing executable bit...
adjust_permissions('./clean_all.sh', stat.S_IXUSR, recursive=False, relative=True)
run_cmd('./clean_all.sh', log_all=True, simple=True, log_output=True)
if LooseVersion(self.version) >= LooseVersion('4.1'):
regex_subs_TS = [
(r"^default:.*$", r""),
(r"^EXE\s*=.*$", r""),
(r"^(include\s*..ARCH_MAKE.*)$", r"EXE=tshs2tshs\ndefault: $(EXE)\n\1"),
(r"^(INCFLAGS.*)$", r"\1 -I%s" % obj_dir),
]
makefile = os.path.join(start_dir, 'Util', 'TS', 'tshs2tshs', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_TS)
# SUFFIX rules in wrong place
regex_subs_suffix = [
(r'^(\.SUFFIXES:.*)$', r''),
(r'^(include\s*\$\(ARCH_MAKE\).*)$', r'\1\n.SUFFIXES:\n.SUFFIXES: .c .f .F .o .a .f90 .F90'),
]
makefile = os.path.join(start_dir, 'Util', 'Sockets', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_suffix)
makefile = os.path.join(start_dir, 'Util', 'SiestaSubroutine', 'SimpleTest', 'Src', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_suffix)
regex_subs_UtilLDFLAGS = [
(r'(\$\(FC\)\s*-o\s)', r'$(FC) %s %s -o ' % (os.environ['FCFLAGS'], os.environ['LDFLAGS'])),
]
makefile = os.path.join(start_dir, 'Util', 'Optimizer', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_UtilLDFLAGS)
makefile = os.path.join(start_dir, 'Util', 'JobList', 'Src', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_UtilLDFLAGS)
run_cmd('./build_all.sh', log_all=True, simple=True, log_output=True)
# Now move all the built utils to the temp installdir
expected_utils = [
'Bands/eigfat2plot',
'CMLComp/ccViz',
'Contrib/APostnikov/eig2bxsf', 'Contrib/APostnikov/rho2xsf',
'Contrib/APostnikov/vib2xsf', 'Contrib/APostnikov/fmpdos',
'Contrib/APostnikov/xv2xsf', 'Contrib/APostnikov/md2axsf',
'COOP/mprop', 'COOP/fat',
'Denchar/Src/denchar',
'DensityMatrix/dm2cdf', 'DensityMatrix/cdf2dm',
'Eig2DOS/Eig2DOS',
'Gen-basis/ioncat', 'Gen-basis/gen-basis',
'Grid/cdf2grid', 'Grid/cdf_laplacian', 'Grid/cdf2xsf',
'Grid/grid2cube',
'Grid/grid_rotate', 'Grid/g2c_ng', 'Grid/grid2cdf', 'Grid/grid2val',
'Helpers/get_chem_labels',
'HSX/hs2hsx', 'HSX/hsx2hs',
'JobList/Src/getResults', 'JobList/Src/countJobs',
'JobList/Src/runJobs', 'JobList/Src/horizontal',
'Macroave/Src/macroave',
'ON/lwf2cdf',
'Optimizer/simplex', 'Optimizer/swarm',
'pdosxml/pdosxml',
'Projections/orbmol_proj',
'SiestaSubroutine/FmixMD/Src/driver',
'SiestaSubroutine/FmixMD/Src/para',
'SiestaSubroutine/FmixMD/Src/simple',
'STM/simple-stm/plstm', 'STM/ol-stm/Src/stm',
'VCA/mixps', 'VCA/fractional',
'Vibra/Src/vibra', 'Vibra/Src/fcbuild',
'WFS/info_wfsx', 'WFS/wfsx2wfs',
'WFS/readwfx', 'WFS/wfsnc2wfsx', 'WFS/readwf', 'WFS/wfs2wfsx',
]
if LooseVersion(self.version) <= LooseVersion('4.0'):
expected_utils.extend([
'Bands/new.gnubands',
'TBTrans/tbtrans',
])
if LooseVersion(self.version) >= LooseVersion('4.0'):
expected_utils.extend([
'SiestaSubroutine/ProtoNEB/Src/protoNEB',
'SiestaSubroutine/SimpleTest/Src/simple_pipes_parallel',
'SiestaSubroutine/SimpleTest/Src/simple_pipes_serial',
'Sockets/f2fmaster', 'Sockets/f2fslave',
])
if LooseVersion(self.version) >= LooseVersion('4.1'):
expected_utils.extend([
'Bands/gnubands',
'Grimme/fdf2grimme',
'SpPivot/pvtsp',
'TS/ts2ts/ts2ts', 'TS/tshs2tshs/tshs2tshs', 'TS/TBtrans/tbtrans',
])
for util in expected_utils:
copy_file(os.path.join(start_dir, 'Util', util), bindir)
if self.cfg['with_transiesta']:
# Build transiesta
change_dir(obj_dir)
run_cmd('make clean', log_all=True, simple=True, log_output=True)
run_cmd('make %s transiesta' % par, log_all=True, simple=True, log_output=True)
copy_file(os.path.join(obj_dir, 'transiesta'), bindir)
def configure_step(self):
"""Custom configuration procedure for TensorFlow."""
binutils_root = get_software_root('binutils')
if not binutils_root:
raise EasyBuildError(
"Failed to determine installation prefix for binutils")
self.binutils_bin_path = os.path.join(binutils_root, 'bin')
tmpdir = tempfile.mkdtemp(suffix='-bazel-configure')
# filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that
# might conflict with dependencies on the system and/or installed with EB. For example: protobuf
path_filter = self.cfg['path_filter']
if path_filter:
self.log.info(
"Filtering $CPATH and $LIBRARY_PATH with path filter %s",
path_filter)
for var in ['CPATH', 'LIBRARY_PATH']:
path = os.getenv(var).split(os.pathsep)
self.log.info("$%s old value was %s" % (var, path))
filtered_path = os.pathsep.join(
[p for fil in path_filter for p in path if fil not in p])
env.setvar(var, filtered_path)
wrapper_dir = os.path.join(tmpdir, 'bin')
use_wrapper = False
if self.toolchain.comp_family() == toolchain.INTELCOMP:
# put wrappers for Intel C/C++ compilers in place (required to make sure license server is found)
# cfr. https://github.com/bazelbuild/bazel/issues/663
for compiler in ('icc', 'icpc'):
self.write_wrapper(wrapper_dir, compiler, 'NOT-USED-WITH-ICC')
use_wrapper = True
use_mpi = self.toolchain.options.get('usempi', False)
mpi_home = ''
if use_mpi:
impi_root = get_software_root('impi')
if impi_root:
# put wrappers for Intel MPI compiler wrappers in place
# (required to make sure license server and I_MPI_ROOT are found)
for compiler in (os.getenv('MPICC'), os.getenv('MPICXX')):
self.write_wrapper(wrapper_dir, compiler,
os.getenv('I_MPI_ROOT'))
use_wrapper = True
# set correct value for MPI_HOME
mpi_home = os.path.join(impi_root, 'intel64')
else:
self.log.debug("MPI module name: %s",
self.toolchain.MPI_MODULE_NAME[0])
mpi_home = get_software_root(self.toolchain.MPI_MODULE_NAME[0])
self.log.debug("Derived value for MPI_HOME: %s", mpi_home)
if use_wrapper:
env.setvar('PATH',
os.pathsep.join([wrapper_dir,
os.getenv('PATH')]))
self.prepare_python()
self.handle_jemalloc()
self.verify_system_libs_info()
self.system_libs_info = self.get_system_libs()
cuda_root = get_software_root('CUDA')
cudnn_root = get_software_root('cuDNN')
opencl_root = get_software_root('OpenCL')
tensorrt_root = get_software_root('TensorRT')
nccl_root = get_software_root('NCCL')
config_env_vars = {
'CC_OPT_FLAGS': os.getenv('CXXFLAGS'),
'MPI_HOME': mpi_home,
'PYTHON_BIN_PATH': self.python_cmd,
'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir),
'TF_CUDA_CLANG': '0',
'TF_ENABLE_XLA': '0', # XLA JIT support
'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)],
'TF_NEED_GCP': '0', # Google Cloud Platform
'TF_NEED_GDR': '0',
'TF_NEED_HDFS': '0', # Hadoop File System
'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']],
'TF_NEED_MPI': ('0', '1')[bool(use_mpi)],
'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)],
'TF_NEED_OPENCL_SYCL': '0',
'TF_NEED_ROCM': '0',
'TF_NEED_S3': '0', # Amazon S3 File System
'TF_NEED_TENSORRT': '0',
'TF_NEED_VERBS': '0',
'TF_NEED_AWS': '0', # Amazon AWS Platform
'TF_NEED_KAFKA': '0', # Amazon Kafka Platform
'TF_SET_ANDROID_WORKSPACE': '0',
'TF_DOWNLOAD_CLANG': '0', # Still experimental in TF 2.1.0
'TF_SYSTEM_LIBS': ','.join(self.system_libs_info[0]),
}
if cuda_root:
cuda_version = get_software_version('CUDA')
cuda_maj_min_ver = '.'.join(cuda_version.split('.')[:2])
# $GCC_HOST_COMPILER_PATH should be set to path of the actual compiler (not the MPI compiler wrapper)
if use_mpi:
compiler_path = which(os.getenv('CC_SEQ'))
else:
compiler_path = which(os.getenv('CC'))
# list of CUDA compute capabilities to use can be specifed in two ways (where (2) overrules (1)):
# (1) in the easyconfig file, via the custom cuda_compute_capabilities;
# (2) in the EasyBuild configuration, via --cuda-compute-capabilities configuration option;
ec_cuda_cc = self.cfg['cuda_compute_capabilities']
cfg_cuda_cc = build_option('cuda_compute_capabilities')
cuda_cc = cfg_cuda_cc or ec_cuda_cc or []
if cfg_cuda_cc and ec_cuda_cc:
warning_msg = "cuda_compute_capabilities specified in easyconfig (%s) are overruled by " % ec_cuda_cc
warning_msg += "--cuda-compute-capabilities configuration option (%s)" % cfg_cuda_cc
print_warning(warning_msg)
elif not cuda_cc:
warning_msg = "No CUDA compute capabilities specified, so using TensorFlow default "
warning_msg += "(which may not be optimal for your system).\nYou should use "
warning_msg += "the --cuda-compute-capabilities configuration option or the cuda_compute_capabilities "
warning_msg += "easyconfig parameter to specify a list of CUDA compute capabilities to compile with."
print_warning(warning_msg)
# TensorFlow 1.12.1 requires compute capability >= 3.5
# see https://github.com/tensorflow/tensorflow/pull/25767
if LooseVersion(self.version) >= LooseVersion('1.12.1'):
faulty_comp_caps = [
x for x in cuda_cc if LooseVersion(x) < LooseVersion('3.5')
]
if faulty_comp_caps:
error_msg = "TensorFlow >= 1.12.1 requires CUDA compute capabilities >= 3.5, "
error_msg += "found one or more older ones: %s"
raise EasyBuildError(error_msg,
', '.join(faulty_comp_caps))
if cuda_cc:
self.log.info(
"Compiling with specified list of CUDA compute capabilities: %s",
', '.join(cuda_cc))
config_env_vars.update({
'CUDA_TOOLKIT_PATH':
cuda_root,
'GCC_HOST_COMPILER_PATH':
compiler_path,
# This is the binutils bin folder: https://github.com/tensorflow/tensorflow/issues/39263
'GCC_HOST_COMPILER_PREFIX':
self.binutils_bin_path,
'TF_CUDA_COMPUTE_CAPABILITIES':
','.join(cuda_cc),
'TF_CUDA_VERSION':
cuda_maj_min_ver,
})
# for recent TensorFlow versions, $TF_CUDA_PATHS and $TF_CUBLAS_VERSION must also be set
if LooseVersion(self.version) >= LooseVersion('1.14'):
# figure out correct major/minor version for CUBLAS from cublas_api.h
cublas_api_header_glob_pattern = os.path.join(
cuda_root, 'targets', '*', 'include', 'cublas_api.h')
matches = glob.glob(cublas_api_header_glob_pattern)
if len(matches) == 1:
cublas_api_header_path = matches[0]
cublas_api_header_txt = read_file(cublas_api_header_path)
else:
raise EasyBuildError(
"Failed to isolate path to cublas_api.h: %s", matches)
cublas_ver_parts = []
for key in [
'CUBLAS_VER_MAJOR', 'CUBLAS_VER_MINOR',
'CUBLAS_VER_PATCH'
]:
regex = re.compile("^#define %s ([0-9]+)" % key, re.M)
res = regex.search(cublas_api_header_txt)
if res:
cublas_ver_parts.append(res.group(1))
else:
raise EasyBuildError(
"Failed to find pattern '%s' in %s", regex.pattern,
cublas_api_header_path)
config_env_vars.update({
'TF_CUDA_PATHS':
cuda_root,
'TF_CUBLAS_VERSION':
'.'.join(cublas_ver_parts),
})
if cudnn_root:
cudnn_version = get_software_version('cuDNN')
cudnn_maj_min_patch_ver = '.'.join(
cudnn_version.split('.')[:3])
config_env_vars.update({
'CUDNN_INSTALL_PATH':
cudnn_root,
'TF_CUDNN_VERSION':
cudnn_maj_min_patch_ver,
})
else:
raise EasyBuildError(
"TensorFlow has a strict dependency on cuDNN if CUDA is enabled"
)
if nccl_root:
nccl_version = get_software_version('NCCL')
# Ignore the PKG_REVISION identifier if it exists (i.e., report 2.4.6 for 2.4.6-1 or 2.4.6-2)
nccl_version = nccl_version.split('-')[0]
config_env_vars.update({
'NCCL_INSTALL_PATH': nccl_root,
})
else:
nccl_version = '1.3' # Use simple downloadable version
config_env_vars.update({
'TF_NCCL_VERSION': nccl_version,
})
if tensorrt_root:
tensorrt_version = get_software_version('TensorRT')
config_env_vars.update({
'TF_NEED_TENSORRT': '1',
'TENSORRT_INSTALL_PATH': tensorrt_root,
'TF_TENSORRT_VERSION': tensorrt_version,
})
for (key, val) in sorted(config_env_vars.items()):
env.setvar(key, val)
# patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel
regex_subs = [(r"(run_shell\(\['bazel')",
r"\1, '--output_base=%s', '--install_base=%s'" %
(tmpdir, os.path.join(tmpdir, 'inst_base')))]
apply_regex_substitutions('configure.py', regex_subs)
# Tell Bazel to not use $HOME/.cache/bazel at all
# See https://docs.bazel.build/versions/master/output_directories.html
env.setvar('TEST_TMPDIR', os.path.join(tmpdir, 'output_root'))
cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg[
'configopts']
run_cmd(cmd, log_all=True, simple=True)
def build_step(self):
"""Custom build procedure for TensorFlow."""
# pre-create target installation directory
mkdir(os.path.join(self.installdir, self.pylibdir), parents=True)
inc_paths, lib_paths = [], []
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
if gcc_root:
gcc_lib64 = os.path.join(gcc_root, 'lib64')
lib_paths.append(gcc_lib64)
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# figure out location of GCC include files
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
inc_paths.append(gcc_lib_inc)
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
# make sure include-fixed directory is where we expect it to be
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if os.path.exists(gcc_lib_inc_fixed):
inc_paths.append(gcc_lib_inc_fixed)
else:
self.log.info(
"Derived directory %s does not exist, so discarding it",
gcc_lib_inc_fixed)
# also check on location of include/c++/<gcc version> directory
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if os.path.exists(gcc_cplusplus_inc):
inc_paths.append(gcc_cplusplus_inc)
else:
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
else:
raise EasyBuildError(
"Failed to determine installation prefix for GCC")
cuda_root = get_software_root('CUDA')
if cuda_root:
inc_paths.append(os.path.join(cuda_root, 'include'))
lib_paths.append(os.path.join(cuda_root, 'lib64'))
# fix hardcoded locations of compilers & tools
cxx_inc_dirs = [
'cxx_builtin_include_directory: "%s"' % resolve_path(p)
for p in inc_paths
]
cxx_inc_dirs += [
'cxx_builtin_include_directory: "%s"' % p for p in inc_paths
]
regex_subs = [
(r'-B/usr/bin/', '-B%s %s' %
(self.binutils_bin_path, ' '.join('-L%s/' % p
for p in lib_paths))),
(r'(cxx_builtin_include_directory:).*', ''),
(r'^toolchain {', 'toolchain {\n' + '\n'.join(cxx_inc_dirs)),
]
for tool in [
'ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy',
'objdump', 'strip'
]:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
# -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used
if self.toolchain.options.get('pic', None):
regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')])
# patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries
for path, dirnames, filenames in os.walk(os.getcwd()):
for filename in filenames:
if filename.startswith('CROSSTOOL'):
full_path = os.path.join(path, filename)
self.log.info("Patching %s", full_path)
apply_regex_substitutions(full_path, regex_subs)
tmpdir = tempfile.mkdtemp(suffix='-bazel-build')
user_root_tmpdir = tempfile.mkdtemp(suffix='-user_root')
# compose "bazel build" command with all its options...
cmd = [
self.cfg['prebuildopts'], 'bazel',
'--output_base=%s' % tmpdir,
'--install_base=%s' % os.path.join(tmpdir, 'inst_base'),
'--output_user_root=%s' % user_root_tmpdir, 'build'
]
# build with optimization enabled
# cfr. https://docs.bazel.build/versions/master/user-manual.html#flag--compilation_mode
cmd.append('--compilation_mode=opt')
# select 'opt' config section (this is *not* the same as --compilation_mode=opt!)
# https://docs.bazel.build/versions/master/user-manual.html#flag--config
cmd.append('--config=opt')
# make Bazel print full command line + make it verbose on failures
# https://docs.bazel.build/versions/master/user-manual.html#flag--subcommands
# https://docs.bazel.build/versions/master/user-manual.html#flag--verbose_failures
cmd.extend(['--subcommands', '--verbose_failures'])
# Disable support of AWS platform via config switch introduced in 1.12.1
if LooseVersion(self.version) >= LooseVersion('1.12.1'):
cmd.append('--config=noaws')
# Bazel seems to not be able to handle a large amount of parallel jobs, e.g. 176 on some Power machines,
# and will hang forever building the TensorFlow package.
# So limit to something high but still reasonable while allowing ECs to overwrite it
parallel = self.cfg['parallel']
if self.cfg['maxparallel'] is None:
parallel = min(parallel, 64)
cmd.append('--jobs=%s' % parallel)
if self.toolchain.options.get('pic', None):
cmd.append('--copt="-fPIC"')
# include install location of Python packages in $PYTHONPATH,
# and specify that value of $PYTHONPATH should be passed down into Bazel build environment;
# this is required to make sure that Python packages included as extensions are found at build time;
# see also https://github.com/tensorflow/tensorflow/issues/22395
pythonpath = os.getenv('PYTHONPATH', '')
env.setvar(
'PYTHONPATH',
os.pathsep.join(
[os.path.join(self.installdir, self.pylibdir), pythonpath]))
# Make TF find our modules. LD_LIBRARY_PATH gets automatically added by configure.py
cpaths, libpaths = self.system_libs_info[1:]
if cpaths:
cmd.append("--action_env=CPATH='%s'" % ':'.join(cpaths))
if libpaths:
cmd.append("--action_env=LIBRARY_PATH='%s'" % ':'.join(libpaths))
cmd.append('--action_env=PYTHONPATH')
# Also export $EBPYTHONPREFIXES to handle the multi-deps python setup
# See https://github.com/easybuilders/easybuild-easyblocks/pull/1664
if 'EBPYTHONPREFIXES' in os.environ:
cmd.append('--action_env=EBPYTHONPREFIXES')
# Ignore user environment for Python
cmd.append('--action_env=PYTHONNOUSERSITE=1')
# use same configuration for both host and target programs, which can speed up the build
# only done when optarch is enabled, since this implicitely assumes that host and target platform are the same
# see https://docs.bazel.build/versions/master/guide.html#configurations
if self.toolchain.options.get('optarch'):
cmd.append('--distinct_host_configuration=false')
cmd.append(self.cfg['buildopts'])
# TF 2 (final) sets this in configure
if LooseVersion(self.version) < LooseVersion('2.0'):
if cuda_root:
cmd.append('--config=cuda')
# if mkl-dnn is listed as a dependency it is used. Otherwise downloaded if with_mkl_dnn is true
mkl_root = get_software_root('mkl-dnn')
if mkl_root:
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=0 &&")
cmd.insert(0, "export TF_MKL_ROOT=%s &&" % mkl_root)
elif self.cfg['with_mkl_dnn']:
# this makes TensorFlow use mkl-dnn (cfr. https://github.com/01org/mkl-dnn)
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=1 && ")
# specify target of the build command as last argument
cmd.append('//tensorflow/tools/pip_package:build_pip_package')
run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True)
# run generated 'build_pip_package' script to build the .whl
cmd = "bazel-bin/tensorflow/tools/pip_package/build_pip_package %s" % self.builddir
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def install_step(self):
"""Custom install procedure for TensorFlow."""
# avoid that pip (ab)uses $HOME/.cache/pip
# cfr. https://pip.pypa.io/en/stable/reference/pip_install/#caching
env.setvar('XDG_CACHE_HOME', tempfile.gettempdir())
self.log.info("Using %s as pip cache directory",
os.environ['XDG_CACHE_HOME'])
# find .whl file that was built, and install it using 'pip install'
if ("-rc" in self.version):
whl_version = self.version.replace("-rc", "rc")
else:
whl_version = self.version
whl_paths = glob.glob(
os.path.join(self.builddir, 'tensorflow-%s-*.whl' % whl_version))
if not whl_paths:
whl_paths = glob.glob(
os.path.join(self.builddir, 'tensorflow-*.whl'))
if len(whl_paths) == 1:
# --ignore-installed is required to ensure *this* wheel is installed
cmd = "pip install --ignore-installed --prefix=%s %s" % (
self.installdir, whl_paths[0])
# if extensions are listed, assume they will provide all required dependencies,
# so use --no-deps to prevent pip from downloading & installing them
if self.cfg['exts_list']:
cmd += ' --no-deps'
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
else:
raise EasyBuildError("Failed to isolate built .whl in %s: %s",
whl_paths, self.builddir)
# Fix for https://github.com/tensorflow/tensorflow/issues/6341 on Python < 3.3
# If the site-packages/google/__init__.py file is missing, make it an empty file.
# This fixes the "No module named google.protobuf" error that sometimes shows up during sanity_check
# For Python >= 3.3 the logic is reversed: The __init__.py must not exist.
# See e.g. http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html
google_protobuf_dir = os.path.join(self.installdir, self.pylibdir,
'google', 'protobuf')
google_init_file = os.path.join(self.installdir, self.pylibdir,
'google', '__init__.py')
if LooseVersion(det_python_version(
self.python_cmd)) < LooseVersion('3.3'):
if os.path.isdir(
google_protobuf_dir) and not is_readable(google_init_file):
self.log.debug("Creating (empty) missing %s", google_init_file)
write_file(google_init_file, '')
else:
if os.path.exists(google_init_file):
self.log.debug("Removing %s for Python >= 3.3",
google_init_file)
remove_file(google_init_file)
# Fix cuda header paths
# This is needed for building custom TensorFlow ops
if LooseVersion(self.version) < LooseVersion('1.14'):
pyshortver = '.'.join(
get_software_version('Python').split('.')[:2])
regex_subs = [(r'#include "cuda/include/', r'#include "')]
base_path = os.path.join(self.installdir, 'lib',
'python%s' % pyshortver, 'site-packages',
'tensorflow', 'include', 'tensorflow')
for header in glob.glob(
os.path.join(base_path, 'stream_executor', 'cuda',
'cuda*.h')) + glob.glob(
os.path.join(base_path, 'core', 'util',
'cuda*.h')):
apply_regex_substitutions(header, regex_subs)
def configure_step(self):
"""Configure OpenFOAM build by setting appropriate environment variables."""
# compiler & compiler flags
comp_fam = self.toolchain.comp_family()
extra_flags = ''
if comp_fam == toolchain.GCC: # @UndefinedVariable
self.wm_compiler = 'Gcc'
if get_software_version('GCC') >= LooseVersion('4.8'):
# make sure non-gold version of ld is used, since OpenFOAM requires it
# see http://www.openfoam.org/mantisbt/view.php?id=685
extra_flags = '-fuse-ld=bfd'
# older versions of OpenFOAM-Extend require -fpermissive
if 'extend' in self.name.lower() and LooseVersion(self.version) < LooseVersion('2.0'):
extra_flags += ' -fpermissive'
elif comp_fam == toolchain.INTELCOMP: # @UndefinedVariable
self.wm_compiler = 'Icc'
# make sure -no-prec-div is used with Intel compilers
extra_flags = '-no-prec-div'
else:
raise EasyBuildError("Unknown compiler family, don't know how to set WM_COMPILER")
for env_var in ['CFLAGS', 'CXXFLAGS']:
env.setvar(env_var, "%s %s" % (os.environ.get(env_var, ''), extra_flags))
# patch out hardcoding of WM_* environment variables
# for example, replace 'export WM_COMPILER=Gcc' with ': ${WM_COMPILER:=Gcc}; export WM_COMPILER'
for script in [os.path.join(self.builddir, self.openfoamdir, x) for x in ['etc/bashrc', 'etc/cshrc']]:
self.log.debug("Patching out hardcoded $WM_* env vars in %s", script)
# disable any third party stuff, we use EB controlled builds
regex_subs = [(r"^(setenv|export) WM_THIRD_PARTY_USE_.*[ =].*$", r"# \g<0>")]
WM_env_var = ['WM_COMPILER', 'WM_MPLIB', 'WM_THIRD_PARTY_DIR']
# OpenFOAM >= 3.0.0 can use 64 bit integers
if 'extend' not in self.name.lower() and LooseVersion(self.version) >= LooseVersion('3.0'):
WM_env_var.append('WM_LABEL_SIZE')
for env_var in WM_env_var:
regex_subs.append((r"^(setenv|export) (?P<var>%s)[ =](?P<val>.*)$" % env_var,
r": ${\g<var>:=\g<val>}; export \g<var>"))
apply_regex_substitutions(script, regex_subs)
# inject compiler variables into wmake/rules files
ldirs = glob.glob(os.path.join(self.builddir, self.openfoamdir, 'wmake', 'rules', 'linux*'))
langs = ['c', 'c++']
suffixes = ['', 'Opt']
wmake_rules_files = [os.path.join(ldir, lang + suff) for ldir in ldirs for lang in langs for suff in suffixes]
mpicc = os.environ['MPICC']
mpicxx = os.environ['MPICXX']
cc_seq = os.environ.get('CC_SEQ', os.environ['CC'])
cxx_seq = os.environ.get('CXX_SEQ', os.environ['CXX'])
if self.toolchain.mpi_family() == toolchain.OPENMPI:
# no -cc/-cxx flags supported in OpenMPI compiler wrappers
c_comp_cmd = 'OMPI_CC="%s" %s' % (cc_seq, mpicc)
cxx_comp_cmd = 'OMPI_CXX="%s" %s' % (cxx_seq, mpicxx)
else:
# -cc/-cxx should work for all MPICH-based MPIs (including Intel MPI)
c_comp_cmd = '%s -cc="%s"' % (mpicc, cc_seq)
cxx_comp_cmd = '%s -cxx="%s"' % (mpicxx, cxx_seq)
comp_vars = {
# specify MPI compiler wrappers and compiler commands + sequential compiler that should be used by them
'cc': c_comp_cmd,
'CC': cxx_comp_cmd,
'cOPT': os.environ['CFLAGS'],
'c++OPT': os.environ['CXXFLAGS'],
}
for wmake_rules_file in wmake_rules_files:
fullpath = os.path.join(self.builddir, self.openfoamdir, wmake_rules_file)
self.log.debug("Patching compiler variables in %s", fullpath)
regex_subs = []
for comp_var, newval in comp_vars.items():
regex_subs.append((r"^(%s\s*=\s*).*$" % re.escape(comp_var), r"\1%s" % newval))
apply_regex_substitutions(fullpath, regex_subs)
# enable verbose build for debug purposes
# starting with openfoam-extend 3.2, PS1 also needs to be set
env.setvar("FOAM_VERBOSE", '1')
# installation directory
env.setvar("FOAM_INST_DIR", self.installdir)
# third party directory
self.thrdpartydir = "ThirdParty-%s" % self.version
# only if third party stuff is actually installed
if os.path.exists(self.thrdpartydir):
os.symlink(os.path.join("..", self.thrdpartydir), self.thrdpartydir)
env.setvar("WM_THIRD_PARTY_DIR", os.path.join(self.installdir, self.thrdpartydir))
env.setvar("WM_COMPILER", self.wm_compiler)
# set to an MPI unknown by OpenFOAM, since we're handling the MPI settings ourselves (via mpicc, etc.)
# Note: this name must contain 'MPI' so the MPI version of the Pstream library is built (cf src/Pstream/Allwmake)
self.wm_mplib = "EASYBUILDMPI"
env.setvar("WM_MPLIB", self.wm_mplib)
# parallel build spec
env.setvar("WM_NCOMPPROCS", str(self.cfg['parallel']))
# OpenFOAM >= 3.0.0 can use 64 bit integers
if 'extend' not in self.name.lower() and LooseVersion(self.version) >= LooseVersion('3.0'):
if self.toolchain.options['i8']:
env.setvar("WM_LABEL_SIZE", '64')
else:
env.setvar("WM_LABEL_SIZE", '32')
# make sure lib/include dirs for dependencies are found
openfoam_extend_v3 = 'extend' in self.name.lower() and LooseVersion(self.version) >= LooseVersion('3.0')
if LooseVersion(self.version) < LooseVersion("2") or openfoam_extend_v3:
self.log.debug("List of deps: %s" % self.cfg.dependencies())
for dep in self.cfg.dependencies():
dep_name = dep['name'].upper(),
dep_root = get_software_root(dep['name'])
env.setvar("%s_SYSTEM" % dep_name, "1")
dep_vars = {
"%s_DIR": "%s",
"%s_BIN_DIR": "%s/bin",
"%s_LIB_DIR": "%s/lib",
"%s_INCLUDE_DIR": "%s/include",
}
for var, val in dep_vars.iteritems():
env.setvar(var % dep_name, val % dep_root)
else:
for depend in ['SCOTCH', 'METIS', 'CGAL', 'Paraview']:
dependloc = get_software_root(depend)
if dependloc:
if depend == 'CGAL' and get_software_root('Boost'):
env.setvar("CGAL_ROOT", dependloc)
env.setvar("BOOST_ROOT", get_software_root('Boost'))
else:
env.setvar("%s_ROOT" % depend.upper(), dependloc)
def configure_step(self):
"""Custom configuration procedure for WRF-Fire."""
comp_fam = self.toolchain.comp_family()
# define $NETCDF* for netCDF dependency
netcdf_fortran = get_software_root('netCDF-Fortran')
if netcdf_fortran:
env.setvar('NETCDF', netcdf_fortran)
else:
raise EasyBuildError("Required dependendy netCDF-Fortran is missing")
# define $PHDF5 for parallel HDF5 dependency
hdf5 = get_software_root('HDF5')
if hdf5 and os.path.exists(os.path.join(hdf5, 'bin', 'h5pcc')):
env.setvar('PHDF5', hdf5)
# first, configure WRF part
change_dir(os.path.join(self.cfg['start_dir'], 'WRFV3'))
# instruct WRF-Fire to create netCDF v4 output files
env.setvar('WRFIO_NETCDF4_FILE_SUPPORT', '1')
# patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join('arch', 'Config_new.pl'))
# determine build type option to look for
known_build_type_options = {
toolchain.INTELCOMP: "Linux x86_64 i486 i586 i686, ifort compiler with icc",
toolchain.GCC: "x86_64 Linux, gfortran compiler with gcc",
toolchain.PGI: "Linux x86_64, PGI compiler with pgcc",
}
build_type_option = known_build_type_options.get(comp_fam)
if build_type_option is None:
raise EasyBuildError("Don't know which WPS configure option to select for compiler family %s", comp_fam)
build_type_question = "\s*(?P<nr>[0-9]+).\s*%s\s*\(%s\)" % (build_type_option, self.cfg['buildtype'])
qa = {
"Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]:": '1',
}
std_qa = {
# named group in match will be used to construct answer
r"%s.*\n(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question: '%(nr)s',
}
run_cmd_qa('./configure', qa, std_qa=std_qa, log_all=True, simple=True)
cpp_flag = None
if comp_fam == toolchain.INTELCOMP:
cpp_flag = '-fpp'
elif comp_fam == toolchain.GCC:
cpp_flag = '-cpp'
else:
raise EasyBuildError("Don't know which flag to use to specify that Fortran files were preprocessed")
# patch configure.wrf to get things right
comps = {
'CFLAGS_LOCAL': os.getenv('CFLAGS'),
'DM_FC': os.getenv('MPIF90'),
'DM_CC': "%s -DMPI2_SUPPORT" % os.getenv('MPICC'),
'FCOPTIM': os.getenv('FFLAGS'),
# specify that Fortran files have been preprocessed with cpp,
# see http://forum.wrfforum.com/viewtopic.php?f=5&t=6086
'FORMAT_FIXED': "-FI %s" % cpp_flag,
'FORMAT_FREE': "-FR %s" % cpp_flag,
}
regex_subs = [(r"^(%s\s*=\s*).*$" % k, r"\1 %s" % v) for (k, v) in comps.items()]
apply_regex_substitutions('configure.wrf', regex_subs)
# also configure WPS part
change_dir(os.path.join(self.cfg['start_dir'], 'WPS'))
# patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions
patch_perl_script_autoflush(os.path.join('arch', 'Config.pl'))
# determine build type option to look for
known_build_type_options = {
toolchain.INTELCOMP: "PC Linux x86_64, Intel compiler",
toolchain.GCC: "PC Linux x86_64, g95 compiler",
toolchain.PGI: "PC Linux x86_64 (IA64 and Opteron), PGI compiler 5.2 or higher",
}
build_type_option = known_build_type_options.get(comp_fam)
if build_type_option is None:
raise EasyBuildError("Don't know which WPS configure option to select for compiler family %s", comp_fam)
known_wps_build_types = {
'dmpar': 'DM parallel',
'smpar': 'serial',
}
wps_build_type = known_wps_build_types.get(self.cfg['buildtype'])
if wps_build_type is None:
raise EasyBuildError("Don't know which WPS build type to pick for '%s'", self.cfg['builddtype'])
build_type_question = "\s*(?P<nr>[0-9]+).\s*%s.*%s(?!NO GRIB2)" % (build_type_option, wps_build_type)
std_qa = {
# named group in match will be used to construct answer
r"%s.*\n(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question: '%(nr)s',
}
run_cmd_qa('./configure', {}, std_qa=std_qa, log_all=True, simple=True)
# patch configure.wps to get things right
comps = {
'CC': '%s %s' % (os.getenv('MPICC'), os.getenv('CFLAGS')),
'FC': '%s %s' % (os.getenv('MPIF90'), os.getenv('F90FLAGS'))
}
regex_subs = [(r"^(%s\s*=\s*).*$" % k, r"\1 %s" % v) for (k, v) in comps.items()]
# specify that Fortran90 files have been preprocessed with cpp
regex_subs.extend([
(r"^(F77FLAGS\s*=\s*)", r"\1 %s " % cpp_flag),
(r"^(FFLAGS\s*=\s*)", r"\1 %s " % cpp_flag),
])
apply_regex_substitutions('configure.wps', regex_subs)
def configure_step(self):
"""Custom configuration procedure for Quantum ESPRESSO."""
if self.toolchain.options.get('openmp', False) or self.cfg['hybrid']:
self.cfg.update('configopts', '--enable-openmp')
if not self.toolchain.options.get('usempi', None):
self.cfg.update('configopts', '--disable-parallel')
if not self.cfg['with_scalapack']:
self.cfg.update('configopts', '--without-scalapack')
repls = []
if self.toolchain.comp_family() in [toolchain.INTELCOMP]:
# set preprocessor command (-E to stop after preprocessing, -C to preserve comments)
cpp = "%s -E -C" % os.getenv('CC')
repls.append(('CPP', cpp, False))
env.setvar('CPP', cpp)
# also define $FCCPP, but do *not* include -C (comments should not be preserved when preprocessing Fortran)
env.setvar('FCCPP', "%s -E" % os.getenv('CC'))
super(EB_QuantumESPRESSO, self).configure_step()
# compose list of DFLAGS (flag, value, keep_stuff)
# for guidelines, see include/defs.h.README in sources
dflags = []
comp_fam_dflags = {
toolchain.INTELCOMP: '-D__INTEL',
toolchain.GCC: '-D__GFORTRAN -D__STD_F95',
}
dflags.append(comp_fam_dflags[self.toolchain.comp_family()])
if self.toolchain.options.get('openmp', False):
libfft = os.getenv('LIBFFT_MT')
else:
libfft = os.getenv('LIBFFT')
if libfft:
if "fftw3" in libfft:
dflags.append('-D__FFTW3')
else:
dflags.append('-D__FFTW')
env.setvar('FFTW_LIBS', libfft)
if get_software_root('ACML'):
dflags.append('-D__ACML')
if self.toolchain.options.get('usempi', None):
dflags.append('-D__MPI -D__PARA')
if self.toolchain.options.get('openmp', False) or self.cfg['hybrid']:
dflags.append(" -D__OPENMP")
if self.cfg['with_scalapack']:
dflags.append(" -D__SCALAPACK")
# always include -w to supress warnings
dflags.append('-w')
repls.append(('DFLAGS', ' '.join(dflags), False))
# complete C/Fortran compiler and LD flags
if self.toolchain.options.get('openmp', False) or self.cfg['hybrid']:
repls.append(('LDFLAGS', self.toolchain.get_flag('openmp'), True))
repls.append(('(?:C|F90|F)FLAGS', self.toolchain.get_flag('openmp'), True))
# obtain library settings
libs = []
for lib in ['BLAS', 'LAPACK', 'FFT', 'SCALAPACK']:
if self.toolchain.options.get('openmp', False):
val = os.getenv('LIB%s_MT' % lib)
else:
val = os.getenv('LIB%s' % lib)
repls.append(('%s_LIBS' % lib, val, False))
libs.append(val)
libs = ' '.join(libs)
repls.append(('BLAS_LIBS_SWITCH', 'external', False))
repls.append(('LAPACK_LIBS_SWITCH', 'external', False))
repls.append(('LD_LIBS', os.getenv('LIBS'), False))
# check for external FoX
if get_software_root('FoX'):
self.log.debug("Found FoX external module, disabling libfox target in Makefile")
regex_subs = [(r"(libfox: touch-dummy)\n.*", r"\1\n\techo 'libfox: external module used' #")]
apply_regex_substitutions('Makefile', regex_subs)
# Make configure and make look in the correct directory
regex_subs = [(r"\(TOPDIR\)/FoX", r"(EBROOTFOX)")]
apply_regex_substitutions('install/configure', regex_subs)
apply_regex_substitutions('make.inc', regex_subs)
self.log.debug("List of replacements to perform: %s" % repls)
if LooseVersion(self.version) >= LooseVersion("6"):
make_ext = '.inc'
else:
make_ext = '.sys'
# patch make.sys file
fn = os.path.join(self.cfg['start_dir'], 'make' + make_ext)
try:
for line in fileinput.input(fn, inplace=1, backup='.orig.eb'):
for (k, v, keep) in repls:
# need to use [ \t]* instead of \s*, because vars may be undefined as empty,
# and we don't want to include newlines
if keep:
line = re.sub(r"^(%s\s*=[ \t]*)(.*)$" % k, r"\1\2 %s" % v, line)
else:
line = re.sub(r"^(%s\s*=[ \t]*).*$" % k, r"\1%s" % v, line)
# fix preprocessing directives for .f90 files in make.sys if required
if self.toolchain.comp_family() in [toolchain.GCC]:
line = re.sub(r"\$\(MPIF90\) \$\(F90FLAGS\) -c \$<",
"$(CPP) -C $(CPPFLAGS) $< -o $*.F90\n" +
"\t$(MPIF90) $(F90FLAGS) -c $*.F90 -o $*.o",
line)
sys.stdout.write(line)
except IOError, err:
raise EasyBuildError("Failed to patch %s: %s", fn, err)
def configure_step(self):
"""Custom configuration procedure for Xmipp."""
# Tell xmipp config that there is no Scipion.
env.setvar('XMIPP_NOSCIPION', 'True')
# Initialize the config file and then patch it with the correct values
cmd = ' '.join([
self.cfg['preconfigopts'],
self.xmipp_exe,
'config',
self.cfg['configopts'],
])
run_cmd(cmd, log_all=True, simple=True)
# Parameters to be set in the config file
params = {
'CC': os.environ['CC'],
'CXX': os.environ['CXX'],
'LINKERFORPROGRAMS': os.environ['CXX'],
'LIBDIRFLAGS': '',
# I don't think Xmipp can actually build without MPI.
'MPI_CC': os.environ.get('MPICC', 'UNKNOWN'),
'MPI_CXX': os.environ.get('MPICXX', 'UNKNOWN'),
'MPI_LINKERFORPROGRAMS': os.environ.get('MPICXX', 'UNKNOWN'),
}
if LooseVersion(self.version) >= LooseVersion('3.20.07'):
# Define include dirs with INCDIRFLAGS (Xmipp does not use CPPPATH directly)
incdirflags = os.getenv('INCDIRFLAGS', '')
incdirflags += '-I../ -I ' + ' -I'.join(
os.environ['CPATH'].split(':'))
opencv_root = get_software_root('OpenCV')
if opencv_root:
incdirflags += ' -I' + opencv_root + '/include/opencv4'
# env.setvar('INCDIRFLAGS', incdirflags)
params['INCDIRFLAGS'] = incdirflags
deps = [
# Name of dependency, name of env var to set, required or not
('HDF5', None, True),
('SQLite', None, True),
('LibTIFF', None, True),
('libjpeg-turbo', None, True),
('Java', 'JAVA_HOME', True),
('MATLAB', 'MATLAB_DIR', False),
]
cuda_root = get_software_root('CUDA')
if cuda_root:
params.update({'CUDA_BIN': os.path.join(cuda_root, 'bin')})
params.update({'CUDA_LIB': os.path.join(cuda_root, 'lib64')})
params.update({'NVCC': os.environ['CUDA_CXX']})
# Their default for NVCC is to use g++-5, fix that
nvcc_flags = '-v --x cu -D_FORCE_INLINES -Xcompiler -fPIC -Wno-deprecated-gpu-targets -std=c++11'
if LooseVersion(self.version) >= LooseVersion('3.20.07'):
nvcc_flags += ' --extended-lambda'
params.update({'NVCC_CXXFLAGS': nvcc_flags})
self.use_cuda = True
# Make sure cuFFTAdvisor is available even if unpacked under
# a different name
if not os.path.isdir('cuFFTAdvisor'):
matches = glob.glob(os.path.join(self.srcdir,
'cuFFTAdvisor-*'))
if len(matches) == 1:
cufft = os.path.basename(matches[0])
symlink(cufft,
os.path.join(self.srcdir, 'cuFFTAdvisor'),
use_abspath_source=False)
if LooseVersion(self.version) >= LooseVersion('3.20.07'):
symlink(
os.path.join(self.srcdir, cufft),
os.path.join(self.srcdir, 'xmipp', 'external',
'cuFFTAdvisor'))
else:
raise EasyBuildError(
"Failed to isolate path to cuFFTAdvisor-*: %s",
matches)
for dep in ['CUDA', 'MATLAB']:
use_dep = bool(get_software_root(dep))
params.update({dep: use_dep})
if get_software_root('OpenCV'):
params.update({'OPENCV': True})
if self.use_cuda:
params.update({'OPENCVSUPPORTSCUDA': True})
if LooseVersion(
get_software_version('OpenCV')) >= LooseVersion('3'):
params.update({'OPENCV3': True})
missing_deps = []
for dep, var, required in deps:
root = get_software_root(dep)
if root:
if var:
params.update({var: root})
elif required:
missing_deps.append(dep)
if missing_deps:
raise EasyBuildError(
"One or more required dependencies not available: %s",
', '.join(missing_deps))
# Here we know that Java exists since it is a required dependency
jnipath = os.path.join(get_software_root('Java'), 'include')
jnipath = os.pathsep.join([jnipath, os.path.join(jnipath, 'linux')])
params.update({'JNI_CPPPATH': jnipath})
regex_subs = []
# Set the variables in the config file to match the environment from EasyBuild
for (key, val) in params.items():
regex_subs.extend([(r'^%s\s*=.*$' % key, r'%s = %s' % (key, val))])
apply_regex_substitutions(self.cfgfile, regex_subs)
self.setup_xmipp_env()
py_ver = get_software_version('Python')
py_maj_ver = LooseVersion(py_ver).version[0]
py_min_ver = LooseVersion(py_ver).version[1]
pyver = 'python%s.%s' % (py_maj_ver, py_min_ver)
pyincpath = os.path.join(get_software_root('Python'), 'include', pyver)
# Temp workaround for missing include/pythonx.y in CPATH
env.setvar('CPATH', os.pathsep.join([os.environ['CPATH'], pyincpath]))
super(EB_Xmipp, self).configure_step()
