def get_software_libdir(name, only_one=True, fs=None):
"""
Find library subdirectories for the specified software package.
Returns the library subdirectory, relative to software root.
It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs.
@param: name of the software package
@param only_one: indicates whether only one lib path is expected to be found
@param fs: only retain library subdirs that contain one of the files in this list
"""
lib_subdirs = ['lib', 'lib64']
root = get_software_root(name)
res = []
if root:
for lib_subdir in lib_subdirs:
if os.path.exists(os.path.join(root, lib_subdir)):
if fs is None or any([os.path.exists(os.path.join(root, lib_subdir, f)) for f in fs]):
res.append(lib_subdir)
# if no library subdir was found, return None
if not res:
return None
if only_one:
if len(res) == 1:
res = res[0]
else:
_log.error("Multiple library subdirectories found for %s in %s: %s" % (name, root, ', '.join(res)))
return res
else:
# return None if software package root could not be determined
return None
def validate_os_deps(self):
"""
validate presence of OS dependencies
osdependencies should be a single list
"""
not_found = []
for dep in self['osdependencies']:
# make sure we have a tuple
if isinstance(dep, basestring):
dep = (dep, )
elif not isinstance(dep, tuple):
self.log.error(
"Non-tuple value type for OS dependency specification: %s (type %s)"
% (dep, type(dep)))
if not any([check_os_dependency(cand_dep) for cand_dep in dep]):
not_found.append(dep)
if not_found:
self.log.error("One or more OS dependencies were not found: %s" %
not_found)
else:
self.log.info("OS dependencies ok: %s" % self['osdependencies'])
return True
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any(
[
self.options.avail_easyconfig_params,
self.options.avail_easyconfig_templates,
self.options.list_easyblocks,
self.options.list_toolchains,
self.options.avail_easyconfig_constants,
self.options.avail_easyconfig_licenses,
self.options.avail_repositories,
self.options.show_default_moduleclasses,
self.options.avail_modules_tools,
self.options.avail_module_naming_schemes,
]
):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# prepare for --list/--avail
if any([
self.options.avail_easyconfig_params,
self.options.avail_easyconfig_templates,
self.options.list_easyblocks,
self.options.list_toolchains,
self.options.avail_easyconfig_constants,
self.options.avail_easyconfig_licenses,
self.options.avail_repositories,
self.options.show_default_moduleclasses,
self.options.avail_modules_tools,
self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict(
) # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
self.log.error(
"Required support for using GitHub API is not available (see warnings)."
)
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
self.log.error(
"Python 'keyring' module required for obtaining GitHub token is not available."
)
if self.options.github_user is None:
self.log.error(
"No GitHub user name provided, required for fetching GitHub token."
)
token = fetch_github_token(self.options.github_user)
if token is None:
self.log.error(
"Failed to obtain required GitHub token for user '%s'" %
self.options.github_user)
self._postprocess_config()
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def get_version_toolchain(self, version=None, tcname=None, tcversion=None):
"""Return tuple of version, toolchainname and toolchainversion (possibly using defaults)."""
# make sure that requested version/toolchain are supported by this easyconfig
versions = [x.get_version_str() for x in self.supported['versions']]
if version is None:
if 'version' in self.default:
version = self.default['version']
self.log.debug("No version specified, using default %s" % version)
else:
self.log.error("No version specified, no default found.")
elif version in versions:
self.log.debug("Version '%s' is supported in easyconfig." % version)
else:
self.log.error("Version '%s' not supported in easyconfig (only %s)" % (version, versions))
tcnames = [tc.tc_name for tc in self.supported['toolchains']]
if tcname is None:
if 'toolchain' in self.default and 'name' in self.default['toolchain']:
tcname = self.default['toolchain']['name']
self.log.debug("No toolchain name specified, using default %s" % tcname)
else:
self.log.error("No toolchain name specified, no default found.")
elif tcname in tcnames:
self.log.debug("Toolchain '%s' is supported in easyconfig." % tcname)
else:
self.log.error("Toolchain '%s' not supported in easyconfig (only %s)" % (tcname, tcnames))
tcs = [tc for tc in self.supported['toolchains'] if tc.tc_name == tcname]
if tcversion is None:
if 'toolchain' in self.default and 'version' in self.default['toolchain']:
tcversion = self.default['toolchain']['version']
self.log.debug("No toolchain version specified, using default %s" % tcversion)
else:
self.log.error("No toolchain version specified, no default found.")
elif any([tc.test(tcname, tcversion) for tc in tcs]):
self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion))
else:
tup = (tcname, tcversion, tcs)
self.log.error("Toolchain '%s' version '%s' not supported in easyconfig (only %s)" % tup)
tup = (version, tcname, tcversion)
self.log.debug('version %s, tcversion %s, tcname %s' % tup)
return tup
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# prepare for --list/--avail
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
self.log.error("Required support for using GitHub API is not available (see warnings).")
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.")
if self.options.github_user is None:
self.log.error("No GitHub user name provided, required for fetching GitHub token.")
token = fetch_github_token(self.options.github_user)
if token is None:
self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user)
self._postprocess_config()
def date_parser(txt):
"""Parse txt
@type txt: string
@param txt: date to be parsed. Usually in C{YYYY-MM-DD} format,
but also C{(BEGIN|END)(THIS|LAST|NEXT)MONTH}, or even
C{(BEGIN | END)(JANUARY | FEBRUARY | MARCH | APRIL | MAY | JUNE | JULY | AUGUST | SEPTEMBER | OCTOBER | NOVEMBER | DECEMBER)}
"""
reserveddate = ('TODAY', )
testsupportedmonths = [
txt.endswith(calendar.month_name[x].upper()) for x in range(1, 13)
]
if txt.endswith('MONTH'):
m = FancyMonth()
res = m.parser(txt)
elif any(testsupportedmonths):
# set day=1 or this will fail on day's with an index more then the count of days then the month you want to parse
# e.g. will fail on 31'st when trying to parse april
m = FancyMonth(month=testsupportedmonths.index(True) + 1, day=1)
res = m.parser(txt)
elif txt in reserveddate:
if txt in ('TODAY', ):
m = FancyMonth()
res = m.date
else:
msg = 'dateparser: unimplemented reservedword %s' % txt
raise (Exception(msg))
else:
try:
datetuple = [int(x) for x in txt.split("-")]
res = date(*datetuple)
except:
msg = (
"dateparser: failed on '%s' date txt expects a YYYY-MM-DD format or "
"reserved words %s") % (txt, ','.join(reserveddate))
raise (Exception(msg))
return res
def validate_os_deps(self):
"""
validate presence of OS dependencies
osdependencies should be a single list
"""
not_found = []
for dep in self['osdependencies']:
# make sure we have a tuple
if isinstance(dep, basestring):
dep = (dep,)
elif not isinstance(dep, tuple):
self.log.error("Non-tuple value type for OS dependency specification: %s (type %s)" % (dep, type(dep)))
if not any([check_os_dependency(cand_dep) for cand_dep in dep]):
not_found.append(dep)
if not_found:
self.log.error("One or more OS dependencies were not found: %s" % not_found)
else:
self.log.info("OS dependencies ok: %s" % self['osdependencies'])
return True
def date_parser(txt):
"""Parse txt
@type txt: string
@param txt: date to be parsed. Usually in C{YYYY-MM-DD} format,
but also C{(BEGIN|END)(THIS|LAST|NEXT)MONTH}, or even
C{(BEGIN | END)(JANUARY | FEBRUARY | MARCH | APRIL | MAY | JUNE | JULY | AUGUST | SEPTEMBER | OCTOBER | NOVEMBER | DECEMBER)}
"""
reserveddate = ('TODAY',)
testsupportedmonths = [txt.endswith(calendar.month_name[x].upper()) for x in range(1, 13)]
if txt.endswith('MONTH'):
m = FancyMonth()
res = m.parser(txt)
elif any(testsupportedmonths):
# set day=1 or this will fail on day's with an index more then the count of days then the month you want to parse
# e.g. will fail on 31'st when trying to parse april
m = FancyMonth(month=testsupportedmonths.index(True) + 1, day=1)
res = m.parser(txt)
elif txt in reserveddate:
if txt in ('TODAY',):
m = FancyMonth()
res = m.date
else:
msg = 'dateparser: unimplemented reservedword %s' % txt
raise(Exception(msg))
else:
try:
datetuple = [int(x) for x in txt.split("-")]
res = date(*datetuple)
except:
msg = ("dateparser: failed on '%s' date txt expects a YYYY-MM-DD format or "
"reserved words %s") % (txt, ','.join(reserveddate))
raise(Exception(msg))
return res
def get_software_libdir(name, only_one=True, fs=None):
"""
Find library subdirectories for the specified software package.
Returns the library subdirectory, relative to software root.
It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs.
@param: name of the software package
@param only_one: indicates whether only one lib path is expected to be found
@param fs: only retain library subdirs that contain one of the files in this list
"""
lib_subdirs = ['lib', 'lib64']
root = get_software_root(name)
res = []
if root:
for lib_subdir in lib_subdirs:
if os.path.exists(os.path.join(root, lib_subdir)):
if fs is None or any([
os.path.exists(os.path.join(root, lib_subdir, f))
for f in fs
]):
res.append(lib_subdir)
# if no library subdir was found, return None
if not res:
return None
if only_one:
if len(res) == 1:
res = res[0]
else:
_log.error(
"Multiple library subdirectories found for %s in %s: %s" %
(name, root, ', '.join(res)))
return res
else:
# return None if software package root could not be determined
return None
def is_dep_in_toolchain_module(self, name):
"""Check whether a specific software name is listed as a dependency in the module for this toolchain."""
return any(map(lambda m: self.mns.is_short_modname_for(m, name), self.toolchain_dep_mods))
def get_version_toolchain(self, version=None, tcname=None, tcversion=None):
"""Return tuple of version, toolchainname and toolchainversion (possibly using defaults)."""
# make sure that requested version/toolchain are supported by this easyconfig
versions = [x.get_version_str() for x in self.supported['versions']]
if version is None:
if 'version' in self.default:
version = self.default['version']
self.log.debug("No version specified, using default %s" %
version)
else:
self.log.error("No version specified, no default found.")
elif version in versions:
self.log.debug("Version '%s' is supported in easyconfig." %
version)
else:
self.log.error(
"Version '%s' not supported in easyconfig (only %s)" %
(version, versions))
tcnames = [tc.tc_name for tc in self.supported['toolchains']]
if tcname is None:
if 'toolchain' in self.default and 'name' in self.default[
'toolchain']:
tcname = self.default['toolchain']['name']
self.log.debug(
"No toolchain name specified, using default %s" % tcname)
else:
self.log.error(
"No toolchain name specified, no default found.")
elif tcname in tcnames:
self.log.debug("Toolchain '%s' is supported in easyconfig." %
tcname)
else:
self.log.error(
"Toolchain '%s' not supported in easyconfig (only %s)" %
(tcname, tcnames))
tcs = [
tc for tc in self.supported['toolchains'] if tc.tc_name == tcname
]
if tcversion is None:
if 'toolchain' in self.default and 'version' in self.default[
'toolchain']:
tcversion = self.default['toolchain']['version']
self.log.debug(
"No toolchain version specified, using default %s" %
tcversion)
else:
self.log.error(
"No toolchain version specified, no default found.")
elif any([tc.test(tcname, tcversion) for tc in tcs]):
self.log.debug(
"Toolchain '%s' version '%s' is supported in easyconfig" %
(tcname, tcversion))
else:
tup = (tcname, tcversion, tcs)
self.log.error(
"Toolchain '%s' version '%s' not supported in easyconfig (only %s)"
% tup)
tup = (version, tcname, tcversion)
self.log.debug('version %s, tcversion %s, tcname %s' % tup)
return tup
def configure_step(self):
"""Custom configuration procedure for GROMACS: set configure options for configure or cmake."""
if LooseVersion(self.version) < LooseVersion('4.6'):
self.log.info(
"Using configure script for configuring GROMACS build.")
# Use static libraries if possible
self.cfg.update('configopts', "--enable-static")
# Use external BLAS and LAPACK
self.cfg.update('configopts',
"--with-external-blas --with-external-lapack")
env.setvar('LIBS',
"%s %s" % (os.environ['LIBLAPACK'], os.environ['LIBS']))
# Don't use the X window system
self.cfg.update('configopts', "--without-x")
# OpenMP is not supported for versions older than 4.5.
if LooseVersion(self.version) >= LooseVersion('4.5'):
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "--enable-threads")
else:
self.cfg.update('configopts', "--disable-threads")
elif self.toolchain.options.get('openmp', None):
raise EasyBuildError(
"GROMACS version %s does not support OpenMP" %
self.version)
# GSL support
if get_software_root('GSL'):
self.cfg.update('configopts', "--with-gsl")
else:
self.cfg.update('configopts', "--without-gsl")
# actually run configure via ancestor (not direct parent)
ConfigureMake.configure_step(self)
else:
# build a release build
self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release")
# prefer static libraries, if available
if self.toolchain.options.get('dynamic', False):
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=OFF")
else:
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON")
if self.cfg['double_precision']:
self.cfg.update('configopts', "-DGMX_DOUBLE=ON")
# always specify to use external BLAS/LAPACK
self.cfg.update('configopts',
"-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON")
# disable GUI tools
self.cfg.update('configopts', "-DGMX_X11=OFF")
# set regression test path
prefix = 'regressiontests'
if any([src['name'].startswith(prefix) for src in self.src]):
major_minor_version = '.'.join(self.version.split('.')[:2])
self.cfg.update(
'configopts',
"-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " %
prefix)
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "-DGMX_OPENMP=ON")
else:
self.cfg.update('configopts', "-DGMX_OPENMP=OFF")
# disable MPI support for initial, serial/SMP build; actual MPI build is done later
self.cfg.update('configopts', "-DGMX_MPI=OFF")
# explicitly disable GPU support if CUDA is not available,
# to avoid that GROMACS find and uses a system-wide CUDA compiler
cuda = get_software_root('CUDA')
if cuda:
self.cfg.update(
'configopts',
"-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
else:
self.cfg.update('configopts', "-DGMX_GPU=OFF")
if get_software_root('imkl'):
# using MKL for FFT, so it will also be used for BLAS/LAPACK
self.cfg.update(
'configopts',
'-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" '
)
mkl_libs = [
os.path.join(os.getenv('LAPACK_LIB_DIR'), lib)
for lib in ['libmkl_lapack.a']
]
self.cfg.update('configopts',
'-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs))
else:
shlib_ext = get_shared_lib_ext()
for libname in ['BLAS', 'LAPACK']:
libdir = os.getenv('%s_LIB_DIR' % libname)
if self.toolchain.toolchain_family() == toolchain.CRAYPE:
libsci_mpi_mp_lib = glob.glob(
os.path.join(libdir, 'libsci_*_mpi_mp.a'))
if libsci_mpi_mp_lib:
self.cfg.update(
'configopts', '-DGMX_%s_USER=%s' %
(libname, libsci_mpi_mp_lib[0]))
else:
raise EasyBuildError(
"Failed to find libsci library to link with for %s",
libname)
else:
# -DGMX_BLAS_USER & -DGMX_LAPACK_USER require full path to library
libs = os.getenv('%s_STATIC_LIBS' % libname).split(',')
libpaths = [
os.path.join(libdir, lib) for lib in libs
if lib != 'libgfortran.a'
]
self.cfg.update(
'configopts', '-DGMX_%s_USER="******"' %
(libname, ';'.join(libpaths)))
# if libgfortran.a is listed, make sure it gets linked in too to avoiding linking issues
if 'libgfortran.a' in libs:
env.setvar(
'LDFLAGS', "%s -lgfortran -lm" %
os.environ.get('LDFLAGS', ''))
# no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472
if LooseVersion(self.version) < LooseVersion('5.0'):
# enable GSL when it's provided
if get_software_root('GSL'):
self.cfg.update('configopts', "-DGMX_GSL=ON")
else:
self.cfg.update('configopts', "-DGMX_GSL=OFF")
# include flags for linking to zlib/XZ in $LDFLAGS if they're listed as a dep;
# this is important for the tests, to correctly link against libxml2
for dep, link_flag in [('XZ', '-llzma'), ('zlib', '-lz')]:
root = get_software_root(dep)
if root:
libdir = get_software_libdir(dep)
ldflags = os.environ.get('LDFLAGS', '')
env.setvar(
'LDFLAGS', "%s -L%s %s" %
(ldflags, os.path.join(root, libdir), link_flag))
# complete configuration with configure_method of parent
self.cfg['separate_build_dir'] = True
out = super(EB_GROMACS, self).configure_step()
# for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used
if LooseVersion(self.version) >= LooseVersion('4.6.5'):
patterns = [
r"Using external FFT library - \S*",
r"Looking for dgemm_ - found",
r"Looking for cheev_ - found",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
if not regex.search(out):
raise EasyBuildError(
"Pattern '%s' not found in GROMACS configuration output.",
pattern)
def configure_step(self):
"""Custom configuration procedure for GROMACS: set configure options for configure or cmake."""
if LooseVersion(self.version) < LooseVersion('4.6'):
self.log.info("Using configure script for configuring GROMACS build.")
raise EasyBuildError("Configuration procedure for older GROMACS versions not implemented yet.")
else:
# build a release build
self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release")
# prefer static libraries, if available
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON")
# always specify to use external BLAS/LAPACK
self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON")
# disable GUI tools
self.cfg.update('configopts', "-DGMX_X11=OFF")
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "-DGMX_OPENMP=ON")
else:
self.cfg.update('configopts', "-DGMX_OPENMP=OFF")
# enable MPI support if desired
if self.toolchain.options.get('usempi', None):
self.cfg.update('configopts', "-DGMX_MPI=ON -DGMX_THREAD_MPI=OFF")
else:
self.cfg.update('configopts', "-DGMX_MPI=OFF")
# explicitely disable GPU support if CUDA is not available,
# to avoid that GROMACS find and uses a system-wide CUDA compiler
cuda = get_software_root('CUDA')
if cuda:
self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
else:
self.cfg.update('configopts', "-DGMX_GPU=OFF")
if get_software_root('imkl'):
# using MKL for FFT, so it will also be used for BLAS/LAPACK
self.cfg.update('configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" ')
mkl_libs = [os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in ['libmkl_lapack.a']]
self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs))
else:
for libname in ['BLAS', 'LAPACK']:
lib_dir = os.getenv('%s_LIB_DIR' % libname)
libs = os.getenv('LIB%s' % libname)
self.cfg.update('configopts', '-DGMX_%s_USER="******"' % (libname, lib_dir, libs))
# set regression test path
prefix = 'regressiontests'
if any([src['name'].startswith(prefix) for src in self.src]):
self.cfg.update('configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix)
# no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472
if LooseVersion(self.version) < LooseVersion('5.0'):
# enable GSL when it's provided
if get_software_root('GSL'):
self.cfg.update('configopts', "-DGMX_GSL=ON")
else:
self.cfg.update('configopts', "-DGMX_GSL=OFF")
# complete configuration with configure_method of parent
out = super(EB_GROMACS, self).configure_step()
# for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used
if LooseVersion(self.version) >= LooseVersion('4.6.5'):
patterns = [
r"Using external FFT library - \S*",
r"Looking for dgemm_ - found",
r"Looking for cheev_ - found",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
if not regex.search(out):
raise EasyBuildError("Pattern '%s' not found in GROMACS configuration output.", pattern)
def configure_step(self):
"""Custom configuration procedure for GROMACS: set configure options for configure or cmake."""
if LooseVersion(self.version) < LooseVersion('4.6'):
self.log.info("Using configure script for configuring GROMACS build.")
# Use static libraries if possible
self.cfg.update('configopts', "--enable-static")
# Use external BLAS and LAPACK
self.cfg.update('configopts', "--with-external-blas --with-external-lapack")
env.setvar('LIBS', "%s %s" % (os.environ['LIBLAPACK'], os.environ['LIBS']))
# Don't use the X window system
self.cfg.update('configopts', "--without-x")
# OpenMP is not supported for versions older than 4.5.
if LooseVersion(self.version) >= LooseVersion('4.5'):
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "--enable-threads")
else:
self.cfg.update('configopts', "--disable-threads")
elif self.toolchain.options.get('openmp', None):
raise EasyBuildError("GROMACS version %s does not support OpenMP" % self.version)
# GSL support
if get_software_root('GSL'):
self.cfg.update('configopts', "--with-gsl")
else:
self.cfg.update('configopts', "--without-gsl")
# actually run configure via ancestor (not direct parent)
ConfigureMake.configure_step(self)
else:
# build a release build
self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release")
# prefer static libraries, if available
if self.toolchain.options.get('dynamic', False):
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=OFF")
else:
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON")
if self.cfg['double_precision']:
self.cfg.update('configopts', "-DGMX_DOUBLE=ON")
# always specify to use external BLAS/LAPACK
self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON")
# disable GUI tools
self.cfg.update('configopts', "-DGMX_X11=OFF")
# set regression test path
prefix = 'regressiontests'
if any([src['name'].startswith(prefix) for src in self.src]):
major_minor_version = '.'.join(self.version.split('.')[:2])
self.cfg.update('configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix)
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "-DGMX_OPENMP=ON")
else:
self.cfg.update('configopts', "-DGMX_OPENMP=OFF")
# disable MPI support for initial, serial/SMP build; actual MPI build is done later
self.cfg.update('configopts', "-DGMX_MPI=OFF")
# explicitly disable GPU support if CUDA is not available,
# to avoid that GROMACS find and uses a system-wide CUDA compiler
cuda = get_software_root('CUDA')
if cuda:
self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
else:
self.cfg.update('configopts', "-DGMX_GPU=OFF")
if get_software_root('imkl'):
# using MKL for FFT, so it will also be used for BLAS/LAPACK
self.cfg.update('configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" ')
mkl_libs = [os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in ['libmkl_lapack.a']]
self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs))
else:
shlib_ext = get_shared_lib_ext()
for libname in ['BLAS', 'LAPACK']:
libdir = os.getenv('%s_LIB_DIR' % libname)
if self.toolchain.toolchain_family() == toolchain.CRAYPE:
libsci_mpi_mp_lib = glob.glob(os.path.join(libdir, 'libsci_*_mpi_mp.a'))
if libsci_mpi_mp_lib:
self.cfg.update('configopts', '-DGMX_%s_USER=%s' % (libname, libsci_mpi_mp_lib[0]))
else:
raise EasyBuildError("Failed to find libsci library to link with for %s", libname)
else:
# -DGMX_BLAS_USER & -DGMX_LAPACK_USER require full path to library
libs = os.getenv('%s_STATIC_LIBS' % libname).split(',')
libpaths = [os.path.join(libdir, lib) for lib in libs if lib != 'libgfortran.a']
self.cfg.update('configopts', '-DGMX_%s_USER="******"' % (libname, ';'.join(libpaths)))
# if libgfortran.a is listed, make sure it gets linked in too to avoiding linking issues
if 'libgfortran.a' in libs:
env.setvar('LDFLAGS', "%s -lgfortran -lm" % os.environ.get('LDFLAGS', ''))
# no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472
if LooseVersion(self.version) < LooseVersion('5.0'):
# enable GSL when it's provided
if get_software_root('GSL'):
self.cfg.update('configopts', "-DGMX_GSL=ON")
else:
self.cfg.update('configopts', "-DGMX_GSL=OFF")
# include flags for linking to zlib/XZ in $LDFLAGS if they're listed as a dep;
# this is important for the tests, to correctly link against libxml2
for dep, link_flag in [('XZ', '-llzma'), ('zlib', '-lz')]:
root = get_software_root(dep)
if root:
libdir = get_software_libdir(dep)
ldflags = os.environ.get('LDFLAGS', '')
env.setvar('LDFLAGS', "%s -L%s %s" % (ldflags, os.path.join(root, libdir), link_flag))
# complete configuration with configure_method of parent
self.cfg['separate_build_dir'] = True
out = super(EB_GROMACS, self).configure_step()
# for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used
if LooseVersion(self.version) >= LooseVersion('4.6.5'):
patterns = [
r"Using external FFT library - \S*",
r"Looking for dgemm_ - found",
r"Looking for cheev_ - found",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
if not regex.search(out):
raise EasyBuildError("Pattern '%s' not found in GROMACS configuration output.", pattern)
def configure_step(self):
"""
Configure for GCC build:
- prepare extra source dirs (GMP, MPFR, MPC, ...)
- create obj dir to build in (GCC doesn't like to be built in source dir)
- add configure and make options, according to .eb spec file
- decide whether or not to do a staged build (which is required to enable PPL/CLooG support)
- set platform_lib based on config.guess output
"""
# self.configopts will be reused in a 3-staged build,
# configopts is only used in first configure
self.configopts = self.cfg['configopts']
# I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them
stage1_info = self.prep_extra_src_dirs("stage1")
configopts = stage1_info['configopts']
# II) update config options
# enable specified language support
if self.cfg['languages']:
self.configopts += " --enable-languages=%s" % ','.join(
self.cfg['languages'])
# enable building of libiberty, if desired
if self.cfg['withlibiberty']:
self.configopts += " --enable-install-libiberty"
# enable link-time-optimization (LTO) support, if desired
if self.cfg['withlto']:
self.configopts += " --enable-lto"
else:
self.configopts += " --disable-lto"
# configure for a release build
self.configopts += " --enable-checking=release "
# enable multilib: allow both 32 and 64 bit
if self.cfg['multilib']:
glibc_32bit = [
"glibc.i686", # Fedora, RedHat-based
"glibc.ppc", # "" on Power
"libc6-dev-i386", # Debian-based
"gcc-c++-32bit", # OpenSuSE, SLES
]
if not any([check_os_dependency(dep) for dep in glibc_32bit]):
raise EasyBuildError(
"Using multilib requires 32-bit glibc (install one of %s, depending on your OS)",
', '.join(glibc_32bit))
self.configopts += " --enable-multilib --with-multilib-list=m32,m64"
else:
self.configopts += " --disable-multilib"
# build both static and dynamic libraries (???)
self.configopts += " --enable-shared=yes --enable-static=yes "
# use POSIX threads
self.configopts += " --enable-threads=posix "
# enable plugin support
self.configopts += " --enable-plugins "
# use GOLD as default linker
if self.cfg['use_gold_linker']:
self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold"
else:
self.configopts += " --enable-gold --enable-ld=default"
# enable bootstrap build for self-containment (unless for staged build)
if not self.stagedbuild:
configopts += " --enable-bootstrap"
else:
configopts += " --disable-bootstrap"
if self.stagedbuild:
#
# STAGE 1: configure GCC build that will be used to build PPL/CLooG
#
self.log.info(
"Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support..."
)
self.stage1installdir = os.path.join(self.builddir,
'GCC_stage1_eb')
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {
'p': self.stage1installdir
}
else:
# unstaged build, so just run standard configure/make/make install
# set prefixes
self.log.info("Performing regular GCC build...")
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {
'p': self.installdir
}
# prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config
# only do this when multilib is not enabled
if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']:
cfgfile = 'gcc/config/i386/t-linux64'
multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32"
self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'",
cfgfile, multilib_osdirnames)
write_file(cfgfile, multilib_osdirnames, append=True)
elif self.cfg['multilib']:
self.log.info(
"Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled"
)
# III) create obj dir to build in, and change to it
# GCC doesn't like to be built in the source dir
if self.stagedbuild:
objdir = self.create_dir("stage1_obj")
self.stage1prefix = objdir
else:
objdir = self.create_dir("obj")
# IV) actual configure, but not on default path
cmd = "../configure %s %s" % (self.configopts, configopts)
# instead of relying on uname, we run the same command GCC uses to
# determine the platform
out, ec = run_cmd("../config.guess", simple=False)
if ec == 0:
self.platform_lib = out.rstrip()
self.run_configure_cmd(cmd)
self.disable_lto_mpfr_old_gcc(objdir)
def is_dep_in_toolchain_module(self, name):
"""Check whether a specific software name is listed as a dependency in the module for this toolchain."""
return any(
map(lambda m: self.mns.is_short_modname_for(m, name),
self.toolchain_dep_mods))
def main(testing_data=(None, None, None)):
"""
Main function: parse command line options, and act accordingly.
@param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing)
# disallow running EasyBuild as root
if os.getuid() == 0:
_log.error("You seem to be running EasyBuild with root privileges which is not wise, so let's end this here.")
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs': build_specs,
'command_line': eb_cmd_line,
'pr_path': pr_path,
'robot_path': robot_path,
'silent': testing,
'try_to_generate': try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options, cmdline_options=options)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs, if a query is specified
query = options.search or options.search_short
if query:
search_easyconfigs(query, short=not options.search)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths, options.from_pr, easyconfigs_pkg_paths)
if not paths:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs)
print_msg(txt, log=_log, silent=testing, prefix=False)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
retained_ecs = skip_available(easyconfigs)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
if options.robot:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs)
else:
ordered_ecs = easyconfigs
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
job_info_txt = submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now: %s" % job_info_txt)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
state = full_state['derived']['state']
states = full_state['derived']['states']
if state == ND_free and ND_idle in states:
state = ND_idle # special case for idle
if not state in detailed_res:
detailed_res[state] = []
if go.options.anystate:
states_to_check = states
else:
states_to_check = [state]
# filter the allowed states
if any(x for x in states_to_check if x in report_states):
nagios_res[nagios_state].append(states)
detailed_res[state].append(states)
nodes_found.append(name)
if go.options.regex and not go.options.allregex:
break
if go.options.invert:
nodes_found = [x for x in all_nodes if not x in nodes_found]
if go.options.regex and not go.options.allregex:
# there should only be one node
nagios_state, all_states = nagios_res.items()[0]
states = all_states[0]
if go.options.nagios:
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
"That's not wise, so let's end this here.\n" \
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key('name'):
paths = [obtain_path(software_build_specs, easyconfigs_paths,
try_to_generate=try_to_generate, exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path,
easyconfigs_map[orig_path]))
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def path_matches(path, paths):
"""Check whether given path matches any of the provided paths."""
return any([os.path.samefile(path, p) for p in paths])
else:
ec_file = f
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
# before building starts, take snapshot of environment (watch out -t option!)
orig_environ = copy.deepcopy(os.environ)
os.chdir(os.environ['PWD'])
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs)
if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
easyconfigs = skip_available(easyconfigs, testing=testing)
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs)
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# You should have received a copy of the GNU Library General Public License
# along with vsc-base. If not, see <http://www.gnu.org/licenses/>.
#
"""
Module with various convenience functions and classes to deal with date, time and timezone
@author: Stijn De Weirdt (Ghent University)
"""
import calendar
import re
import time as _time
from datetime import tzinfo, timedelta, datetime, date
try:
any([0, 1])
except:
from vsc.utils.missing import any
class FancyMonth:
"""Convenience class for month math"""
def __init__(self, tmpdate=None, year=None, month=None, day=None):
"""Initialise the month based on first day of month of tmpdate"""
if tmpdate is None:
tmpdate = date.today()
if day is None:
day = tmpdate.day
if month is None:
def configure_step(self):
"""Custom configuration procedure for GROMACS: set configure options for configure or cmake."""
if LooseVersion(self.version) < LooseVersion('4.6'):
self.log.info(
"Using configure script for configuring GROMACS build.")
raise EasyBuildError(
"Configuration procedure for older GROMACS versions not implemented yet."
)
else:
# build a release build
self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release")
# prefer static libraries, if available
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON")
# always specify to use external BLAS/LAPACK
self.cfg.update('configopts',
"-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON")
# disable GUI tools
self.cfg.update('configopts', "-DGMX_X11=OFF")
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "-DGMX_OPENMP=ON")
else:
self.cfg.update('configopts', "-DGMX_OPENMP=OFF")
# enable MPI support if desired
if self.toolchain.options.get('usempi', None):
self.cfg.update('configopts',
"-DGMX_MPI=ON -DGMX_THREAD_MPI=OFF")
else:
self.cfg.update('configopts', "-DGMX_MPI=OFF")
# explicitely disable GPU support if CUDA is not available,
# to avoid that GROMACS find and uses a system-wide CUDA compiler
cuda = get_software_root('CUDA')
if cuda:
self.cfg.update(
'configopts',
"-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
else:
self.cfg.update('configopts', "-DGMX_GPU=OFF")
if get_software_root('imkl'):
# using MKL for FFT, so it will also be used for BLAS/LAPACK
self.cfg.update(
'configopts',
'-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" '
)
mkl_libs = [
os.path.join(os.getenv('LAPACK_LIB_DIR'), lib)
for lib in ['libmkl_lapack.a']
]
self.cfg.update('configopts',
'-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs))
else:
for libname in ['BLAS', 'LAPACK']:
lib_dir = os.getenv('%s_LIB_DIR' % libname)
libs = os.getenv('LIB%s' % libname)
self.cfg.update(
'configopts',
'-DGMX_%s_USER="******"' % (libname, lib_dir, libs))
# set regression test path
prefix = 'regressiontests'
if any([src['name'].startswith(prefix) for src in self.src]):
self.cfg.update(
'configopts',
"-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " %
prefix)
# no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472
if LooseVersion(self.version) < LooseVersion('5.0'):
# enable GSL when it's provided
if get_software_root('GSL'):
self.cfg.update('configopts', "-DGMX_GSL=ON")
else:
self.cfg.update('configopts', "-DGMX_GSL=OFF")
# complete configuration with configure_method of parent
out = super(EB_GROMACS, self).configure_step()
# for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used
if LooseVersion(self.version) >= LooseVersion('4.6.5'):
patterns = [
r"Using external FFT library - \S*",
r"Looking for dgemm_ - found",
r"Looking for cheev_ - found",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
if not regex.search(out):
raise EasyBuildError(
"Pattern '%s' not found in GROMACS configuration output.",
pattern)
def configure_step(self):
"""
Configure for GCC build:
- prepare extra source dirs (GMP, MPFR, MPC, ...)
- create obj dir to build in (GCC doesn't like to be built in source dir)
- add configure and make options, according to .eb spec file
- decide whether or not to do a staged build (which is required to enable PPL/CLooG support)
- set platform_lib based on config.guess output
"""
# self.configopts will be reused in a 3-staged build,
# configopts is only used in first configure
self.configopts = self.cfg['configopts']
# I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them
stage1_info = self.prep_extra_src_dirs("stage1")
configopts = stage1_info['configopts']
# II) update config options
# enable specified language support
if self.cfg['languages']:
self.configopts += " --enable-languages=%s" % ','.join(self.cfg['languages'])
# enable building of libiberty, if desired
if self.cfg['withlibiberty']:
self.configopts += " --enable-install-libiberty"
# enable link-time-optimization (LTO) support, if desired
if self.cfg['withlto']:
self.configopts += " --enable-lto"
else:
self.configopts += " --disable-lto"
# configure for a release build
self.configopts += " --enable-checking=release "
# enable multilib: allow both 32 and 64 bit
if self.cfg['multilib']:
glibc_32bit = [
"glibc.i686", # Fedora, RedHat-based
"glibc.ppc", # "" on Power
"libc6-dev-i386", # Debian-based
"gcc-c++-32bit", # OpenSuSE, SLES
]
if not any([check_os_dependency(dep) for dep in glibc_32bit]):
raise EasyBuildError("Using multilib requires 32-bit glibc (install one of %s, depending on your OS)",
', '.join(glibc_32bit))
self.configopts += " --enable-multilib --with-multilib-list=m32,m64"
else:
self.configopts += " --disable-multilib"
# build both static and dynamic libraries (???)
self.configopts += " --enable-shared=yes --enable-static=yes "
# use POSIX threads
self.configopts += " --enable-threads=posix "
# enable plugin support
self.configopts += " --enable-plugins "
# use GOLD as default linker
if self.cfg['use_gold_linker']:
self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold"
else:
self.configopts += " --enable-gold --enable-ld=default"
# enable bootstrap build for self-containment (unless for staged build)
if not self.stagedbuild:
configopts += " --enable-bootstrap"
else:
configopts += " --disable-bootstrap"
if self.stagedbuild:
#
# STAGE 1: configure GCC build that will be used to build PPL/CLooG
#
self.log.info("Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support...")
self.stage1installdir = os.path.join(self.builddir, 'GCC_stage1_eb')
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.stage1installdir}
else:
# unstaged build, so just run standard configure/make/make install
# set prefixes
self.log.info("Performing regular GCC build...")
configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.installdir}
# prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config
# only do this when multilib is not enabled
if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']:
cfgfile = 'gcc/config/i386/t-linux64'
multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32"
self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'", cfgfile, multilib_osdirnames)
write_file(cfgfile, multilib_osdirnames, append=True)
elif self.cfg['multilib']:
self.log.info("Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled")
# III) create obj dir to build in, and change to it
# GCC doesn't like to be built in the source dir
if self.stagedbuild:
objdir = self.create_dir("stage1_obj")
self.stage1prefix = objdir
else:
objdir = self.create_dir("obj")
# IV) actual configure, but not on default path
cmd = "../configure %s %s" % (self.configopts, configopts)
# instead of relying on uname, we run the same command GCC uses to
# determine the platform
out, ec = run_cmd("../config.guess", simple=False)
if ec == 0:
self.platform_lib = out.rstrip()
self.run_configure_cmd(cmd)
self.disable_lto_mpfr_old_gcc(objdir)
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n"
)
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-")
os.close(fd)
fancylogger.logToFile(logfile)
print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section("config"))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key("name"):
paths = [
obtain_path(
software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing
)
]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(
(
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"
),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing,
)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info(
"Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])
)
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
# You should have received a copy of the GNU Library General Public License
# along with vsc-base. If not, see <http://www.gnu.org/licenses/>.
#
"""
Module with various convenience functions and classes to deal with date, time and timezone
@author: Stijn De Weirdt (Ghent University)
"""
import calendar
import re
import time as _time
from datetime import tzinfo, timedelta, datetime, date
try:
any([0, 1])
except:
from vsc.utils.missing import any
class FancyMonth:
"""Convenience class for month math"""
def __init__(self, tmpdate=None, year=None, month=None, day=None):
"""Initialise the month based on first day of month of tmpdate"""
if tmpdate is None:
tmpdate = date.today()
if day is None:
day = tmpdate.day
if month is None:
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile),
log=None,
silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error(
"No robot paths specified, and unable to determine easybuild-easyconfigs install path."
)
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
if robot_path is None:
robot_path = []
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info(
"Extended list of robot paths with paths for installed easyconfigs: %s"
% robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest':
options.aggregate_regtest,
'allow_modules_tool_mismatch':
options.allow_modules_tool_mismatch,
'check_osdeps':
not options.ignore_osdeps,
'cleanup_builddir':
options.cleanup_builddir,
'command_line':
eb_command_line,
'debug':
options.debug,
'dry_run':
options.dry_run,
'easyblock':
options.easyblock,
'experimental':
options.experimental,
'force':
options.force,
'github_user':
options.github_user,
'group':
options.group,
'ignore_dirs':
options.ignore_dirs,
'modules_footer':
options.modules_footer,
'only_blocks':
options.only_blocks,
'recursive_mod_unload':
options.recursive_module_unload,
'regtest_output_dir':
options.regtest_output_dir,
'retain_all_deps':
retain_all_deps,
'robot_path':
robot_path,
'sequential':
options.sequential,
'silent':
testing,
'set_gid_bit':
options.set_gid_bit,
'skip':
options.skip,
'skip_test_cases':
options.skip_test_cases,
'sticky_bit':
options.sticky_bit,
'stop':
options.stop,
'umask':
options.umask,
'valid_module_classes':
module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate':
not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list()
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path,
query,
short=not options.search,
ignore_dirs=ignore_dirs,
silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
pr_files = fetch_easyconfigs_from_pr(
options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files
if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [
obtain_path(build_specs,
easyconfigs_paths,
try_to_generate=try_to_generate,
exit_on_error=not testing)
]
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(
orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug(
"Looking for missing easyconfig files (%d left) in %s..." %
(len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path,
topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" %
(orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [
d for d in dirnames if not d in options.ignore_dirs
]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" %
(path, err))
def path_matches(path, paths):
"""Check whether given path matches any of the provided paths."""
return any([os.path.samefile(path, p) for p in paths])
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# specified robot paths are preferred over installed easyconfig files
if robot_path:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'group': options.group,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_online': options.regtest_online,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for f in files:
if not generated and try_to_generate and build_specs:
ec_file = tweak(f, None, build_specs)
else:
ec_file = f
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
# tweak obtained easyconfig files, if requested
if try_to_generate and build_specs:
easyconfigs = tweak(easyconfigs, build_specs)
# before building starts, take snapshot of environment (watch out -t option!)
os.chdir(os.environ['PWD'])
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
print_dry_run(easyconfigs,
short=not options.dry_run,
build_specs=build_specs)
if any([
options.dry_run, options.dry_run_short, options.regtest,
options.search, options.search_short
]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
easyconfigs = skip_available(easyconfigs, testing=testing)
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs,
build_specs=build_specs)
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
