def get_software_libdir(name, only_one=True, fs=None): """ Find library subdirectories for the specified software package. Returns the library subdirectory, relative to software root. It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs. @param: name of the software package @param only_one: indicates whether only one lib path is expected to be found @param fs: only retain library subdirs that contain one of the files in this list """ lib_subdirs = ['lib', 'lib64'] root = get_software_root(name) res = [] if root: for lib_subdir in lib_subdirs: if os.path.exists(os.path.join(root, lib_subdir)): if fs is None or any([os.path.exists(os.path.join(root, lib_subdir, f)) for f in fs]): res.append(lib_subdir) # if no library subdir was found, return None if not res: return None if only_one: if len(res) == 1: res = res[0] else: _log.error("Multiple library subdirectories found for %s in %s: %s" % (name, root, ', '.join(res))) return res else: # return None if software package root could not be determined return None
def validate_os_deps(self): """ validate presence of OS dependencies osdependencies should be a single list """ not_found = [] for dep in self['osdependencies']: # make sure we have a tuple if isinstance(dep, basestring): dep = (dep, ) elif not isinstance(dep, tuple): self.log.error( "Non-tuple value type for OS dependency specification: %s (type %s)" % (dep, type(dep))) if not any([check_os_dependency(cand_dep) for cand_dep in dep]): not_found.append(dep) if not_found: self.log.error("One or more OS dependencies were not found: %s" % not_found) else: self.log.info("OS dependencies ok: %s" % self['osdependencies']) return True
def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig # set strictness of run module if self.options.strict: run.strictness = self.options.strict if self.options.deprecated: build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated) if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) if any( [ self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, ] ): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() self._postprocess_config()
def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig # set strictness of run module if self.options.strict: run.strictness = self.options.strict # override current version of EasyBuild with version specified to --deprecated if self.options.deprecated: build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated) # log to specified value of --unittest-file if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) # prepare for --list/--avail if any([ self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, ]): build_easyconfig_constants_dict( ) # runs the easyconfig constants sanity check self._postprocess_list_avail() # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: self.log.error( "Required support for using GitHub API is not available (see warnings)." ) # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: self.log.error( "Python 'keyring' module required for obtaining GitHub token is not available." ) if self.options.github_user is None: self.log.error( "No GitHub user name provided, required for fetching GitHub token." ) token = fetch_github_token(self.options.github_user) if token is None: self.log.error( "Failed to obtain required GitHub token for user '%s'" % self.options.github_user) self._postprocess_config()
def postprocess(self): """Do some postprocessing, in particular print stuff""" if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, ]): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() self._postprocess_config()
def get_version_toolchain(self, version=None, tcname=None, tcversion=None): """Return tuple of version, toolchainname and toolchainversion (possibly using defaults).""" # make sure that requested version/toolchain are supported by this easyconfig versions = [x.get_version_str() for x in self.supported['versions']] if version is None: if 'version' in self.default: version = self.default['version'] self.log.debug("No version specified, using default %s" % version) else: self.log.error("No version specified, no default found.") elif version in versions: self.log.debug("Version '%s' is supported in easyconfig." % version) else: self.log.error("Version '%s' not supported in easyconfig (only %s)" % (version, versions)) tcnames = [tc.tc_name for tc in self.supported['toolchains']] if tcname is None: if 'toolchain' in self.default and 'name' in self.default['toolchain']: tcname = self.default['toolchain']['name'] self.log.debug("No toolchain name specified, using default %s" % tcname) else: self.log.error("No toolchain name specified, no default found.") elif tcname in tcnames: self.log.debug("Toolchain '%s' is supported in easyconfig." % tcname) else: self.log.error("Toolchain '%s' not supported in easyconfig (only %s)" % (tcname, tcnames)) tcs = [tc for tc in self.supported['toolchains'] if tc.tc_name == tcname] if tcversion is None: if 'toolchain' in self.default and 'version' in self.default['toolchain']: tcversion = self.default['toolchain']['version'] self.log.debug("No toolchain version specified, using default %s" % tcversion) else: self.log.error("No toolchain version specified, no default found.") elif any([tc.test(tcname, tcversion) for tc in tcs]): self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion)) else: tup = (tcname, tcversion, tcs) self.log.error("Toolchain '%s' version '%s' not supported in easyconfig (only %s)" % tup) tup = (version, tcname, tcversion) self.log.debug('version %s, tcversion %s, tcname %s' % tup) return tup
def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig # set strictness of run module if self.options.strict: run.strictness = self.options.strict # override current version of EasyBuild with version specified to --deprecated if self.options.deprecated: build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated) # log to specified value of --unittest-file if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) # prepare for --list/--avail if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, ]): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: self.log.error("Required support for using GitHub API is not available (see warnings).") # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.") if self.options.github_user is None: self.log.error("No GitHub user name provided, required for fetching GitHub token.") token = fetch_github_token(self.options.github_user) if token is None: self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user) self._postprocess_config()
def date_parser(txt): """Parse txt @type txt: string @param txt: date to be parsed. Usually in C{YYYY-MM-DD} format, but also C{(BEGIN|END)(THIS|LAST|NEXT)MONTH}, or even C{(BEGIN | END)(JANUARY | FEBRUARY | MARCH | APRIL | MAY | JUNE | JULY | AUGUST | SEPTEMBER | OCTOBER | NOVEMBER | DECEMBER)} """ reserveddate = ('TODAY', ) testsupportedmonths = [ txt.endswith(calendar.month_name[x].upper()) for x in range(1, 13) ] if txt.endswith('MONTH'): m = FancyMonth() res = m.parser(txt) elif any(testsupportedmonths): # set day=1 or this will fail on day's with an index more then the count of days then the month you want to parse # e.g. will fail on 31'st when trying to parse april m = FancyMonth(month=testsupportedmonths.index(True) + 1, day=1) res = m.parser(txt) elif txt in reserveddate: if txt in ('TODAY', ): m = FancyMonth() res = m.date else: msg = 'dateparser: unimplemented reservedword %s' % txt raise (Exception(msg)) else: try: datetuple = [int(x) for x in txt.split("-")] res = date(*datetuple) except: msg = ( "dateparser: failed on '%s' date txt expects a YYYY-MM-DD format or " "reserved words %s") % (txt, ','.join(reserveddate)) raise (Exception(msg)) return res
def validate_os_deps(self): """ validate presence of OS dependencies osdependencies should be a single list """ not_found = [] for dep in self['osdependencies']: # make sure we have a tuple if isinstance(dep, basestring): dep = (dep,) elif not isinstance(dep, tuple): self.log.error("Non-tuple value type for OS dependency specification: %s (type %s)" % (dep, type(dep))) if not any([check_os_dependency(cand_dep) for cand_dep in dep]): not_found.append(dep) if not_found: self.log.error("One or more OS dependencies were not found: %s" % not_found) else: self.log.info("OS dependencies ok: %s" % self['osdependencies']) return True
def date_parser(txt): """Parse txt @type txt: string @param txt: date to be parsed. Usually in C{YYYY-MM-DD} format, but also C{(BEGIN|END)(THIS|LAST|NEXT)MONTH}, or even C{(BEGIN | END)(JANUARY | FEBRUARY | MARCH | APRIL | MAY | JUNE | JULY | AUGUST | SEPTEMBER | OCTOBER | NOVEMBER | DECEMBER)} """ reserveddate = ('TODAY',) testsupportedmonths = [txt.endswith(calendar.month_name[x].upper()) for x in range(1, 13)] if txt.endswith('MONTH'): m = FancyMonth() res = m.parser(txt) elif any(testsupportedmonths): # set day=1 or this will fail on day's with an index more then the count of days then the month you want to parse # e.g. will fail on 31'st when trying to parse april m = FancyMonth(month=testsupportedmonths.index(True) + 1, day=1) res = m.parser(txt) elif txt in reserveddate: if txt in ('TODAY',): m = FancyMonth() res = m.date else: msg = 'dateparser: unimplemented reservedword %s' % txt raise(Exception(msg)) else: try: datetuple = [int(x) for x in txt.split("-")] res = date(*datetuple) except: msg = ("dateparser: failed on '%s' date txt expects a YYYY-MM-DD format or " "reserved words %s") % (txt, ','.join(reserveddate)) raise(Exception(msg)) return res
def get_software_libdir(name, only_one=True, fs=None): """ Find library subdirectories for the specified software package. Returns the library subdirectory, relative to software root. It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs. @param: name of the software package @param only_one: indicates whether only one lib path is expected to be found @param fs: only retain library subdirs that contain one of the files in this list """ lib_subdirs = ['lib', 'lib64'] root = get_software_root(name) res = [] if root: for lib_subdir in lib_subdirs: if os.path.exists(os.path.join(root, lib_subdir)): if fs is None or any([ os.path.exists(os.path.join(root, lib_subdir, f)) for f in fs ]): res.append(lib_subdir) # if no library subdir was found, return None if not res: return None if only_one: if len(res) == 1: res = res[0] else: _log.error( "Multiple library subdirectories found for %s in %s: %s" % (name, root, ', '.join(res))) return res else: # return None if software package root could not be determined return None
def is_dep_in_toolchain_module(self, name): """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" return any(map(lambda m: self.mns.is_short_modname_for(m, name), self.toolchain_dep_mods))
def get_version_toolchain(self, version=None, tcname=None, tcversion=None): """Return tuple of version, toolchainname and toolchainversion (possibly using defaults).""" # make sure that requested version/toolchain are supported by this easyconfig versions = [x.get_version_str() for x in self.supported['versions']] if version is None: if 'version' in self.default: version = self.default['version'] self.log.debug("No version specified, using default %s" % version) else: self.log.error("No version specified, no default found.") elif version in versions: self.log.debug("Version '%s' is supported in easyconfig." % version) else: self.log.error( "Version '%s' not supported in easyconfig (only %s)" % (version, versions)) tcnames = [tc.tc_name for tc in self.supported['toolchains']] if tcname is None: if 'toolchain' in self.default and 'name' in self.default[ 'toolchain']: tcname = self.default['toolchain']['name'] self.log.debug( "No toolchain name specified, using default %s" % tcname) else: self.log.error( "No toolchain name specified, no default found.") elif tcname in tcnames: self.log.debug("Toolchain '%s' is supported in easyconfig." % tcname) else: self.log.error( "Toolchain '%s' not supported in easyconfig (only %s)" % (tcname, tcnames)) tcs = [ tc for tc in self.supported['toolchains'] if tc.tc_name == tcname ] if tcversion is None: if 'toolchain' in self.default and 'version' in self.default[ 'toolchain']: tcversion = self.default['toolchain']['version'] self.log.debug( "No toolchain version specified, using default %s" % tcversion) else: self.log.error( "No toolchain version specified, no default found.") elif any([tc.test(tcname, tcversion) for tc in tcs]): self.log.debug( "Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion)) else: tup = (tcname, tcversion, tcs) self.log.error( "Toolchain '%s' version '%s' not supported in easyconfig (only %s)" % tup) tup = (version, tcname, tcversion) self.log.debug('version %s, tcversion %s, tcname %s' % tup) return tup
def configure_step(self): """Custom configuration procedure for GROMACS: set configure options for configure or cmake.""" if LooseVersion(self.version) < LooseVersion('4.6'): self.log.info( "Using configure script for configuring GROMACS build.") # Use static libraries if possible self.cfg.update('configopts', "--enable-static") # Use external BLAS and LAPACK self.cfg.update('configopts', "--with-external-blas --with-external-lapack") env.setvar('LIBS', "%s %s" % (os.environ['LIBLAPACK'], os.environ['LIBS'])) # Don't use the X window system self.cfg.update('configopts', "--without-x") # OpenMP is not supported for versions older than 4.5. if LooseVersion(self.version) >= LooseVersion('4.5'): # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "--enable-threads") else: self.cfg.update('configopts', "--disable-threads") elif self.toolchain.options.get('openmp', None): raise EasyBuildError( "GROMACS version %s does not support OpenMP" % self.version) # GSL support if get_software_root('GSL'): self.cfg.update('configopts', "--with-gsl") else: self.cfg.update('configopts', "--without-gsl") # actually run configure via ancestor (not direct parent) ConfigureMake.configure_step(self) else: # build a release build self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release") # prefer static libraries, if available if self.toolchain.options.get('dynamic', False): self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=OFF") else: self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON") if self.cfg['double_precision']: self.cfg.update('configopts', "-DGMX_DOUBLE=ON") # always specify to use external BLAS/LAPACK self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON") # disable GUI tools self.cfg.update('configopts', "-DGMX_X11=OFF") # set regression test path prefix = 'regressiontests' if any([src['name'].startswith(prefix) for src in self.src]): major_minor_version = '.'.join(self.version.split('.')[:2]) self.cfg.update( 'configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix) # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "-DGMX_OPENMP=ON") else: self.cfg.update('configopts', "-DGMX_OPENMP=OFF") # disable MPI support for initial, serial/SMP build; actual MPI build is done later self.cfg.update('configopts', "-DGMX_MPI=OFF") # explicitly disable GPU support if CUDA is not available, # to avoid that GROMACS find and uses a system-wide CUDA compiler cuda = get_software_root('CUDA') if cuda: self.cfg.update( 'configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) else: self.cfg.update('configopts', "-DGMX_GPU=OFF") if get_software_root('imkl'): # using MKL for FFT, so it will also be used for BLAS/LAPACK self.cfg.update( 'configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" ' ) mkl_libs = [ os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in ['libmkl_lapack.a'] ] self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs)) else: shlib_ext = get_shared_lib_ext() for libname in ['BLAS', 'LAPACK']: libdir = os.getenv('%s_LIB_DIR' % libname) if self.toolchain.toolchain_family() == toolchain.CRAYPE: libsci_mpi_mp_lib = glob.glob( os.path.join(libdir, 'libsci_*_mpi_mp.a')) if libsci_mpi_mp_lib: self.cfg.update( 'configopts', '-DGMX_%s_USER=%s' % (libname, libsci_mpi_mp_lib[0])) else: raise EasyBuildError( "Failed to find libsci library to link with for %s", libname) else: # -DGMX_BLAS_USER & -DGMX_LAPACK_USER require full path to library libs = os.getenv('%s_STATIC_LIBS' % libname).split(',') libpaths = [ os.path.join(libdir, lib) for lib in libs if lib != 'libgfortran.a' ] self.cfg.update( 'configopts', '-DGMX_%s_USER="******"' % (libname, ';'.join(libpaths))) # if libgfortran.a is listed, make sure it gets linked in too to avoiding linking issues if 'libgfortran.a' in libs: env.setvar( 'LDFLAGS', "%s -lgfortran -lm" % os.environ.get('LDFLAGS', '')) # no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472 if LooseVersion(self.version) < LooseVersion('5.0'): # enable GSL when it's provided if get_software_root('GSL'): self.cfg.update('configopts', "-DGMX_GSL=ON") else: self.cfg.update('configopts', "-DGMX_GSL=OFF") # include flags for linking to zlib/XZ in $LDFLAGS if they're listed as a dep; # this is important for the tests, to correctly link against libxml2 for dep, link_flag in [('XZ', '-llzma'), ('zlib', '-lz')]: root = get_software_root(dep) if root: libdir = get_software_libdir(dep) ldflags = os.environ.get('LDFLAGS', '') env.setvar( 'LDFLAGS', "%s -L%s %s" % (ldflags, os.path.join(root, libdir), link_flag)) # complete configuration with configure_method of parent self.cfg['separate_build_dir'] = True out = super(EB_GROMACS, self).configure_step() # for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used if LooseVersion(self.version) >= LooseVersion('4.6.5'): patterns = [ r"Using external FFT library - \S*", r"Looking for dgemm_ - found", r"Looking for cheev_ - found", ] for pattern in patterns: regex = re.compile(pattern, re.M) if not regex.search(out): raise EasyBuildError( "Pattern '%s' not found in GROMACS configuration output.", pattern)
def configure_step(self): """Custom configuration procedure for GROMACS: set configure options for configure or cmake.""" if LooseVersion(self.version) < LooseVersion('4.6'): self.log.info("Using configure script for configuring GROMACS build.") raise EasyBuildError("Configuration procedure for older GROMACS versions not implemented yet.") else: # build a release build self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release") # prefer static libraries, if available self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON") # always specify to use external BLAS/LAPACK self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON") # disable GUI tools self.cfg.update('configopts', "-DGMX_X11=OFF") # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "-DGMX_OPENMP=ON") else: self.cfg.update('configopts', "-DGMX_OPENMP=OFF") # enable MPI support if desired if self.toolchain.options.get('usempi', None): self.cfg.update('configopts', "-DGMX_MPI=ON -DGMX_THREAD_MPI=OFF") else: self.cfg.update('configopts', "-DGMX_MPI=OFF") # explicitely disable GPU support if CUDA is not available, # to avoid that GROMACS find and uses a system-wide CUDA compiler cuda = get_software_root('CUDA') if cuda: self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) else: self.cfg.update('configopts', "-DGMX_GPU=OFF") if get_software_root('imkl'): # using MKL for FFT, so it will also be used for BLAS/LAPACK self.cfg.update('configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" ') mkl_libs = [os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in ['libmkl_lapack.a']] self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs)) else: for libname in ['BLAS', 'LAPACK']: lib_dir = os.getenv('%s_LIB_DIR' % libname) libs = os.getenv('LIB%s' % libname) self.cfg.update('configopts', '-DGMX_%s_USER="******"' % (libname, lib_dir, libs)) # set regression test path prefix = 'regressiontests' if any([src['name'].startswith(prefix) for src in self.src]): self.cfg.update('configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix) # no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472 if LooseVersion(self.version) < LooseVersion('5.0'): # enable GSL when it's provided if get_software_root('GSL'): self.cfg.update('configopts', "-DGMX_GSL=ON") else: self.cfg.update('configopts', "-DGMX_GSL=OFF") # complete configuration with configure_method of parent out = super(EB_GROMACS, self).configure_step() # for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used if LooseVersion(self.version) >= LooseVersion('4.6.5'): patterns = [ r"Using external FFT library - \S*", r"Looking for dgemm_ - found", r"Looking for cheev_ - found", ] for pattern in patterns: regex = re.compile(pattern, re.M) if not regex.search(out): raise EasyBuildError("Pattern '%s' not found in GROMACS configuration output.", pattern)
def configure_step(self): """Custom configuration procedure for GROMACS: set configure options for configure or cmake.""" if LooseVersion(self.version) < LooseVersion('4.6'): self.log.info("Using configure script for configuring GROMACS build.") # Use static libraries if possible self.cfg.update('configopts', "--enable-static") # Use external BLAS and LAPACK self.cfg.update('configopts', "--with-external-blas --with-external-lapack") env.setvar('LIBS', "%s %s" % (os.environ['LIBLAPACK'], os.environ['LIBS'])) # Don't use the X window system self.cfg.update('configopts', "--without-x") # OpenMP is not supported for versions older than 4.5. if LooseVersion(self.version) >= LooseVersion('4.5'): # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "--enable-threads") else: self.cfg.update('configopts', "--disable-threads") elif self.toolchain.options.get('openmp', None): raise EasyBuildError("GROMACS version %s does not support OpenMP" % self.version) # GSL support if get_software_root('GSL'): self.cfg.update('configopts', "--with-gsl") else: self.cfg.update('configopts', "--without-gsl") # actually run configure via ancestor (not direct parent) ConfigureMake.configure_step(self) else: # build a release build self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release") # prefer static libraries, if available if self.toolchain.options.get('dynamic', False): self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=OFF") else: self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON") if self.cfg['double_precision']: self.cfg.update('configopts', "-DGMX_DOUBLE=ON") # always specify to use external BLAS/LAPACK self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON") # disable GUI tools self.cfg.update('configopts', "-DGMX_X11=OFF") # set regression test path prefix = 'regressiontests' if any([src['name'].startswith(prefix) for src in self.src]): major_minor_version = '.'.join(self.version.split('.')[:2]) self.cfg.update('configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix) # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "-DGMX_OPENMP=ON") else: self.cfg.update('configopts', "-DGMX_OPENMP=OFF") # disable MPI support for initial, serial/SMP build; actual MPI build is done later self.cfg.update('configopts', "-DGMX_MPI=OFF") # explicitly disable GPU support if CUDA is not available, # to avoid that GROMACS find and uses a system-wide CUDA compiler cuda = get_software_root('CUDA') if cuda: self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) else: self.cfg.update('configopts', "-DGMX_GPU=OFF") if get_software_root('imkl'): # using MKL for FFT, so it will also be used for BLAS/LAPACK self.cfg.update('configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" ') mkl_libs = [os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in ['libmkl_lapack.a']] self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs)) else: shlib_ext = get_shared_lib_ext() for libname in ['BLAS', 'LAPACK']: libdir = os.getenv('%s_LIB_DIR' % libname) if self.toolchain.toolchain_family() == toolchain.CRAYPE: libsci_mpi_mp_lib = glob.glob(os.path.join(libdir, 'libsci_*_mpi_mp.a')) if libsci_mpi_mp_lib: self.cfg.update('configopts', '-DGMX_%s_USER=%s' % (libname, libsci_mpi_mp_lib[0])) else: raise EasyBuildError("Failed to find libsci library to link with for %s", libname) else: # -DGMX_BLAS_USER & -DGMX_LAPACK_USER require full path to library libs = os.getenv('%s_STATIC_LIBS' % libname).split(',') libpaths = [os.path.join(libdir, lib) for lib in libs if lib != 'libgfortran.a'] self.cfg.update('configopts', '-DGMX_%s_USER="******"' % (libname, ';'.join(libpaths))) # if libgfortran.a is listed, make sure it gets linked in too to avoiding linking issues if 'libgfortran.a' in libs: env.setvar('LDFLAGS', "%s -lgfortran -lm" % os.environ.get('LDFLAGS', '')) # no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472 if LooseVersion(self.version) < LooseVersion('5.0'): # enable GSL when it's provided if get_software_root('GSL'): self.cfg.update('configopts', "-DGMX_GSL=ON") else: self.cfg.update('configopts', "-DGMX_GSL=OFF") # include flags for linking to zlib/XZ in $LDFLAGS if they're listed as a dep; # this is important for the tests, to correctly link against libxml2 for dep, link_flag in [('XZ', '-llzma'), ('zlib', '-lz')]: root = get_software_root(dep) if root: libdir = get_software_libdir(dep) ldflags = os.environ.get('LDFLAGS', '') env.setvar('LDFLAGS', "%s -L%s %s" % (ldflags, os.path.join(root, libdir), link_flag)) # complete configuration with configure_method of parent self.cfg['separate_build_dir'] = True out = super(EB_GROMACS, self).configure_step() # for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used if LooseVersion(self.version) >= LooseVersion('4.6.5'): patterns = [ r"Using external FFT library - \S*", r"Looking for dgemm_ - found", r"Looking for cheev_ - found", ] for pattern in patterns: regex = re.compile(pattern, re.M) if not regex.search(out): raise EasyBuildError("Pattern '%s' not found in GROMACS configuration output.", pattern)
def configure_step(self): """ Configure for GCC build: - prepare extra source dirs (GMP, MPFR, MPC, ...) - create obj dir to build in (GCC doesn't like to be built in source dir) - add configure and make options, according to .eb spec file - decide whether or not to do a staged build (which is required to enable PPL/CLooG support) - set platform_lib based on config.guess output """ # self.configopts will be reused in a 3-staged build, # configopts is only used in first configure self.configopts = self.cfg['configopts'] # I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them stage1_info = self.prep_extra_src_dirs("stage1") configopts = stage1_info['configopts'] # II) update config options # enable specified language support if self.cfg['languages']: self.configopts += " --enable-languages=%s" % ','.join( self.cfg['languages']) # enable building of libiberty, if desired if self.cfg['withlibiberty']: self.configopts += " --enable-install-libiberty" # enable link-time-optimization (LTO) support, if desired if self.cfg['withlto']: self.configopts += " --enable-lto" else: self.configopts += " --disable-lto" # configure for a release build self.configopts += " --enable-checking=release " # enable multilib: allow both 32 and 64 bit if self.cfg['multilib']: glibc_32bit = [ "glibc.i686", # Fedora, RedHat-based "glibc.ppc", # "" on Power "libc6-dev-i386", # Debian-based "gcc-c++-32bit", # OpenSuSE, SLES ] if not any([check_os_dependency(dep) for dep in glibc_32bit]): raise EasyBuildError( "Using multilib requires 32-bit glibc (install one of %s, depending on your OS)", ', '.join(glibc_32bit)) self.configopts += " --enable-multilib --with-multilib-list=m32,m64" else: self.configopts += " --disable-multilib" # build both static and dynamic libraries (???) self.configopts += " --enable-shared=yes --enable-static=yes " # use POSIX threads self.configopts += " --enable-threads=posix " # enable plugin support self.configopts += " --enable-plugins " # use GOLD as default linker if self.cfg['use_gold_linker']: self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold" else: self.configopts += " --enable-gold --enable-ld=default" # enable bootstrap build for self-containment (unless for staged build) if not self.stagedbuild: configopts += " --enable-bootstrap" else: configopts += " --disable-bootstrap" if self.stagedbuild: # # STAGE 1: configure GCC build that will be used to build PPL/CLooG # self.log.info( "Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support..." ) self.stage1installdir = os.path.join(self.builddir, 'GCC_stage1_eb') configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % { 'p': self.stage1installdir } else: # unstaged build, so just run standard configure/make/make install # set prefixes self.log.info("Performing regular GCC build...") configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % { 'p': self.installdir } # prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config # only do this when multilib is not enabled if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']: cfgfile = 'gcc/config/i386/t-linux64' multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32" self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'", cfgfile, multilib_osdirnames) write_file(cfgfile, multilib_osdirnames, append=True) elif self.cfg['multilib']: self.log.info( "Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled" ) # III) create obj dir to build in, and change to it # GCC doesn't like to be built in the source dir if self.stagedbuild: objdir = self.create_dir("stage1_obj") self.stage1prefix = objdir else: objdir = self.create_dir("obj") # IV) actual configure, but not on default path cmd = "../configure %s %s" % (self.configopts, configopts) # instead of relying on uname, we run the same command GCC uses to # determine the platform out, ec = run_cmd("../config.guess", simple=False) if ec == 0: self.platform_lib = out.rstrip() self.run_configure_cmd(cmd) self.disable_lto_mpfr_old_gcc(objdir)
def is_dep_in_toolchain_module(self, name): """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" return any( map(lambda m: self.mns.is_short_modname_for(m, name), self.toolchain_dep_mods))
def main(testing_data=(None, None, None)): """ Main function: parse command line options, and act accordingly. @param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing) # disallow running EasyBuild as root if os.getuid() == 0: _log.error("You seem to be running EasyBuild with root privileges which is not wise, so let's end this here.") # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs, if a query is specified query = options.search or options.search_short if query: search_easyconfigs(query, short=not options.search) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # determine paths to easyconfigs paths = det_easyconfig_paths(orig_paths, options.from_pr, easyconfigs_pkg_paths) if not paths: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) print_msg(txt, log=_log, silent=testing, prefix=False) # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: retained_ecs = skip_available(easyconfigs) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: if options.robot: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: ordered_ecs = easyconfigs else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) # submit build as job(s), clean up and exit if options.job: job_info_txt = submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now: %s" % job_info_txt) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
state = full_state['derived']['state'] states = full_state['derived']['states'] if state == ND_free and ND_idle in states: state = ND_idle # special case for idle if not state in detailed_res: detailed_res[state] = [] if go.options.anystate: states_to_check = states else: states_to_check = [state] # filter the allowed states if any(x for x in states_to_check if x in report_states): nagios_res[nagios_state].append(states) detailed_res[state].append(states) nodes_found.append(name) if go.options.regex and not go.options.allregex: break if go.options.invert: nodes_found = [x for x in all_nodes if not x in nodes_found] if go.options.regex and not go.options.allregex: # there should only be one node nagios_state, all_states = nagios_res.items()[0] states = all_states[0] if go.options.nagios:
def main(testing_data=(None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \ "That's not wise, so let's end this here.\n" \ "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) # hello world! _log.info(this_is_easybuild()) # set strictness of filetools module if options.strict: filetools.strictness = options.strict if not options.robot is None: if options.robot: _log.info("Using robot path: %s" % options.robot) else: _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.") # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot) easyconfigs_pkg_full_path = None search_path = os.getcwd() if easyconfigs_paths: easyconfigs_pkg_full_path = easyconfigs_paths[0] if not options.robot: search_path = easyconfigs_pkg_full_path else: search_path = options.robot else: _log.info("Failed to determine install path for easybuild-easyconfigs package.") if options.robot: easyconfigs_paths = [options.robot] + easyconfigs_paths # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # search for easyconfigs if options.search: search_file(search_path, options.search, silent=testing) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, software_build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if software_build_specs.has_key('name'): paths = [obtain_path(software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they we found at the given relative paths if easyconfigs_pkg_full_path: # create a mapping from filename to path in easybuild-easyconfigs package install path easyconfigs_map = {} for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path): for filename in filenames: easyconfigs_map.update({filename: os.path.join(subpath, filename)}) # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path for idx, orig_path in enumerate(orig_paths): if not os.path.isabs(orig_path) and not os.path.exists(orig_path): if orig_path in easyconfigs_map: _log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])) orig_paths[idx] = easyconfigs_map[orig_path] # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: regtest_ok = regtest(options, [path[0] for path in paths]) else: # fallback: easybuild-easyconfigs install path regtest_ok = regtest(options, [easyconfigs_pkg_full_path]) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if any([options.search, options.regtest]): cleanup_logfile_and_exit(logfile, testing, True) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) validate_easyconfigs = True retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True validate_easyconfigs = False retain_all_deps = True # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not (os.path.exists(path)): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path) for f in files: if not generated and try_to_generate and software_build_specs: ec_file = easyconfig.tools.tweak(f, None, software_build_specs) else: ec_file = f easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs)) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def path_matches(path, paths): """Check whether given path matches any of the provided paths.""" return any([os.path.samefile(path, p) for p in paths])
else: ec_file = f ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err)) # before building starts, take snapshot of environment (watch out -t option!) orig_environ = copy.deepcopy(os.environ) os.chdir(os.environ['PWD']) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: easyconfigs = skip_available(easyconfigs, testing=testing) # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = []
# You should have received a copy of the GNU Library General Public License # along with vsc-base. If not, see <http://www.gnu.org/licenses/>. # """ Module with various convenience functions and classes to deal with date, time and timezone @author: Stijn De Weirdt (Ghent University) """ import calendar import re import time as _time from datetime import tzinfo, timedelta, datetime, date try: any([0, 1]) except: from vsc.utils.missing import any class FancyMonth: """Convenience class for month math""" def __init__(self, tmpdate=None, year=None, month=None, day=None): """Initialise the month based on first day of month of tmpdate""" if tmpdate is None: tmpdate = date.today() if day is None: day = tmpdate.day if month is None:
def configure_step(self): """Custom configuration procedure for GROMACS: set configure options for configure or cmake.""" if LooseVersion(self.version) < LooseVersion('4.6'): self.log.info( "Using configure script for configuring GROMACS build.") raise EasyBuildError( "Configuration procedure for older GROMACS versions not implemented yet." ) else: # build a release build self.cfg.update('configopts', "-DCMAKE_BUILD_TYPE=Release") # prefer static libraries, if available self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON") # always specify to use external BLAS/LAPACK self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON") # disable GUI tools self.cfg.update('configopts', "-DGMX_X11=OFF") # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "-DGMX_OPENMP=ON") else: self.cfg.update('configopts', "-DGMX_OPENMP=OFF") # enable MPI support if desired if self.toolchain.options.get('usempi', None): self.cfg.update('configopts', "-DGMX_MPI=ON -DGMX_THREAD_MPI=OFF") else: self.cfg.update('configopts', "-DGMX_MPI=OFF") # explicitely disable GPU support if CUDA is not available, # to avoid that GROMACS find and uses a system-wide CUDA compiler cuda = get_software_root('CUDA') if cuda: self.cfg.update( 'configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) else: self.cfg.update('configopts', "-DGMX_GPU=OFF") if get_software_root('imkl'): # using MKL for FFT, so it will also be used for BLAS/LAPACK self.cfg.update( 'configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="$EBROOTMKL/mkl/include" ' ) mkl_libs = [ os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in ['libmkl_lapack.a'] ] self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs)) else: for libname in ['BLAS', 'LAPACK']: lib_dir = os.getenv('%s_LIB_DIR' % libname) libs = os.getenv('LIB%s' % libname) self.cfg.update( 'configopts', '-DGMX_%s_USER="******"' % (libname, lib_dir, libs)) # set regression test path prefix = 'regressiontests' if any([src['name'].startswith(prefix) for src in self.src]): self.cfg.update( 'configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix) # no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472 if LooseVersion(self.version) < LooseVersion('5.0'): # enable GSL when it's provided if get_software_root('GSL'): self.cfg.update('configopts', "-DGMX_GSL=ON") else: self.cfg.update('configopts', "-DGMX_GSL=OFF") # complete configuration with configure_method of parent out = super(EB_GROMACS, self).configure_step() # for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used if LooseVersion(self.version) >= LooseVersion('4.6.5'): patterns = [ r"Using external FFT library - \S*", r"Looking for dgemm_ - found", r"Looking for cheev_ - found", ] for pattern in patterns: regex = re.compile(pattern, re.M) if not regex.search(out): raise EasyBuildError( "Pattern '%s' not found in GROMACS configuration output.", pattern)
def configure_step(self): """ Configure for GCC build: - prepare extra source dirs (GMP, MPFR, MPC, ...) - create obj dir to build in (GCC doesn't like to be built in source dir) - add configure and make options, according to .eb spec file - decide whether or not to do a staged build (which is required to enable PPL/CLooG support) - set platform_lib based on config.guess output """ # self.configopts will be reused in a 3-staged build, # configopts is only used in first configure self.configopts = self.cfg['configopts'] # I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them stage1_info = self.prep_extra_src_dirs("stage1") configopts = stage1_info['configopts'] # II) update config options # enable specified language support if self.cfg['languages']: self.configopts += " --enable-languages=%s" % ','.join(self.cfg['languages']) # enable building of libiberty, if desired if self.cfg['withlibiberty']: self.configopts += " --enable-install-libiberty" # enable link-time-optimization (LTO) support, if desired if self.cfg['withlto']: self.configopts += " --enable-lto" else: self.configopts += " --disable-lto" # configure for a release build self.configopts += " --enable-checking=release " # enable multilib: allow both 32 and 64 bit if self.cfg['multilib']: glibc_32bit = [ "glibc.i686", # Fedora, RedHat-based "glibc.ppc", # "" on Power "libc6-dev-i386", # Debian-based "gcc-c++-32bit", # OpenSuSE, SLES ] if not any([check_os_dependency(dep) for dep in glibc_32bit]): raise EasyBuildError("Using multilib requires 32-bit glibc (install one of %s, depending on your OS)", ', '.join(glibc_32bit)) self.configopts += " --enable-multilib --with-multilib-list=m32,m64" else: self.configopts += " --disable-multilib" # build both static and dynamic libraries (???) self.configopts += " --enable-shared=yes --enable-static=yes " # use POSIX threads self.configopts += " --enable-threads=posix " # enable plugin support self.configopts += " --enable-plugins " # use GOLD as default linker if self.cfg['use_gold_linker']: self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold" else: self.configopts += " --enable-gold --enable-ld=default" # enable bootstrap build for self-containment (unless for staged build) if not self.stagedbuild: configopts += " --enable-bootstrap" else: configopts += " --disable-bootstrap" if self.stagedbuild: # # STAGE 1: configure GCC build that will be used to build PPL/CLooG # self.log.info("Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support...") self.stage1installdir = os.path.join(self.builddir, 'GCC_stage1_eb') configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.stage1installdir} else: # unstaged build, so just run standard configure/make/make install # set prefixes self.log.info("Performing regular GCC build...") configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.installdir} # prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config # only do this when multilib is not enabled if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']: cfgfile = 'gcc/config/i386/t-linux64' multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32" self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'", cfgfile, multilib_osdirnames) write_file(cfgfile, multilib_osdirnames, append=True) elif self.cfg['multilib']: self.log.info("Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled") # III) create obj dir to build in, and change to it # GCC doesn't like to be built in the source dir if self.stagedbuild: objdir = self.create_dir("stage1_obj") self.stage1prefix = objdir else: objdir = self.create_dir("obj") # IV) actual configure, but not on default path cmd = "../configure %s %s" % (self.configopts, configopts) # instead of relying on uname, we run the same command GCC uses to # determine the platform out, ec = run_cmd("../config.guess", simple=False) if ec == 0: self.platform_lib = out.rstrip() self.run_configure_cmd(cmd) self.disable_lto_mpfr_old_gcc(objdir)
def main(testing_data=(None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write( "ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n" ) sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-") os.close(fd) fancylogger.logToFile(logfile) print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) # hello world! _log.info(this_is_easybuild()) # set strictness of filetools module if options.strict: filetools.strictness = options.strict if not options.robot is None: if options.robot: _log.info("Using robot path: %s" % options.robot) else: _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.") # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot) easyconfigs_pkg_full_path = None search_path = os.getcwd() if easyconfigs_paths: easyconfigs_pkg_full_path = easyconfigs_paths[0] if not options.robot: search_path = easyconfigs_pkg_full_path else: search_path = options.robot else: _log.info("Failed to determine install path for easybuild-easyconfigs package.") if options.robot: easyconfigs_paths = [options.robot] + easyconfigs_paths # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section("config")) # search for easyconfigs if options.search: search_file(search_path, options.search, silent=testing) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, software_build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if software_build_specs.has_key("name"): paths = [ obtain_path( software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing ) ] elif not any([options.aggregate_regtest, options.search, options.regtest]): print_error( ( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs" ), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing, ) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they we found at the given relative paths if easyconfigs_pkg_full_path: # create a mapping from filename to path in easybuild-easyconfigs package install path easyconfigs_map = {} for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path): for filename in filenames: easyconfigs_map.update({filename: os.path.join(subpath, filename)}) # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path for idx, orig_path in enumerate(orig_paths): if not os.path.isabs(orig_path) and not os.path.exists(orig_path): if orig_path in easyconfigs_map: _log.info( "Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path]) ) orig_paths[idx] = easyconfigs_map[orig_path] # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: regtest_ok = regtest(options, [path[0] for path in paths]) else: # fallback: easybuild-easyconfigs install path regtest_ok = regtest(options, [easyconfigs_pkg_full_path]) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if any([options.search, options.regtest]): cleanup_logfile_and_exit(logfile, testing, True) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) validate_easyconfigs = True retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True validate_easyconfigs = False retain_all_deps = True # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not (os.path.exists(path)): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path) for f in files: if not generated and try_to_generate and software_build_specs: ec_file = easyconfig.tools.tweak(f, None, software_build_specs) else: ec_file = f easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs)) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write( "ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args eb_config = eb_go.generate_cmd_line(add_default=True) init_session_state.update({'easybuild_configuration': eb_config}) # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error( "No robot paths specified, and unable to determine easybuild-easyconfigs install path." ) # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # specified robot paths are preferred over installed easyconfig files # --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs if robot_path or try_to_generate or options.dep_graph: if robot_path is None: robot_path = [] robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info( "Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'cleanup_builddir': options.cleanup_builddir, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'github_user': options.github_user, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # obtain list of loaded modules, build options must be initialized first modlist = session_module_list() init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) paths = [] if len(orig_paths) == 0: if options.from_pr: pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr) pr_files = fetch_easyconfigs_from_pr( options.from_pr, path=pr_path, github_user=options.github_user) paths = [(path, False) for path in pr_files if path.endswith('.eb')] elif 'name' in build_specs: paths = [ obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing) ] elif not any([ options.aggregate_regtest, options.search, options.search_short, options.regtest ]): print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename( orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug( "Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [ d for d in dirnames if not d in options.ignore_dirs ] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for ec_file in ec_files: # only pass build specs when not generating easyconfig files if try_to_generate: ecs = process_easyconfig(ec_file) else: ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # specified robot paths are preferred over installed easyconfig files if robot_path: robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_online': options.regtest_online, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if 'name' in build_specs: paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for f in files: if not generated and try_to_generate and build_specs: ec_file = tweak(f, None, build_specs) else: ec_file = f ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
# tweak obtained easyconfig files, if requested if try_to_generate and build_specs: easyconfigs = tweak(easyconfigs, build_specs) # before building starts, take snapshot of environment (watch out -t option!) os.chdir(os.environ['PWD']) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) if any([ options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short ]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: easyconfigs = skip_available(easyconfigs, testing=testing) # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing)