def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains]):
self._postprocess_list_avail()
def _set_fftw_variables(self):
if not hasattr(self, 'BLAS_LIB_DIR'):
self.log.raiseException(
"_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)"
)
imklver = get_software_version(self.FFT_MODULE_NAME[0])
picsuff = ''
if self.options.get('pic', None):
picsuff = '_pic'
bitsuff = '_lp64'
if self.options.get('i8', None):
bitsuff = '_ilp64'
compsuff = '_intel'
if get_software_root('icc') is None:
if get_software_root('GCC'):
compsuff = '_gnu'
else:
self.log.error(
"Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries."
)
fftw_libs = ["fftw3xc%s%s" % (compsuff, picsuff)]
if self.options['usempi']:
# add cluster interface for recent imkl versions
if LooseVersion(imklver) >= LooseVersion("11.0.2"):
fftw_libs.append("fftw3x_cdft%s%s" % (bitsuff, picsuff))
elif LooseVersion(imklver) >= LooseVersion("10.3"):
fftw_libs.append("fftw3x_cdft%s" % picsuff)
fftw_libs.append("mkl_cdft_core") # add cluster dft
fftw_libs.extend(self.variables['LIBBLACS'].flatten()
) # add BLACS; use flatten because ListOfList
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
fftw_libs.extend(self.variables['LIBBLAS'].flatten()
) # add BLAS libs (contains dft)
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
self.FFT_LIB_DIR = self.BLAS_LIB_DIR
self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR
# building the FFTW interfaces is optional,
# so make sure libraries are there before FFT_LIB is set
imklroot = get_software_root(self.FFT_MODULE_NAME[0])
fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
# filter out gfortran from list of FFTW libraries to check for, since it's not provided by imkl
check_fftw_libs = [lib for lib in fftw_libs if lib != 'gfortran']
fftw_lib_exists = lambda x: any([
os.path.exists(os.path.join(d, "lib%s.a" % x))
for d in fft_lib_dirs
])
if all([fftw_lib_exists(lib) for lib in check_fftw_libs]):
self.FFT_LIB = fftw_libs
else:
tup = (check_fftw_libs, fft_lib_dirs)
msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % tup
self.log.error(msg)
def _set_fftw_variables(self):
if not hasattr(self, 'BLAS_LIB_DIR'):
self.log.raiseException("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)")
imklver = get_software_version(self.FFT_MODULE_NAME[0])
picsuff = ''
if self.options.get('pic', None):
picsuff = '_pic'
bitsuff = '_lp64'
if self.options.get('i8', None):
bitsuff = '_ilp64'
compsuff = '_intel'
if get_software_root('icc') is None:
if get_software_root('GCC'):
compsuff = '_gnu'
else:
self.log.error("Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries.")
fftw_libs = ["fftw3xc%s%s" % (compsuff, picsuff)]
if self.options['usempi']:
# add cluster interface for recent imkl versions
if LooseVersion(imklver) >= LooseVersion("11.0.2"):
fftw_libs.append("fftw3x_cdft%s%s" % (bitsuff, picsuff))
elif LooseVersion(imklver) >= LooseVersion("10.3"):
fftw_libs.append("fftw3x_cdft%s" % picsuff)
fftw_libs.append("mkl_cdft_core") # add cluster dft
fftw_libs.extend(self.variables['LIBBLACS'].flatten()) # add BLACS; use flatten because ListOfList
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
fftw_libs.extend(self.variables['LIBBLAS'].flatten()) # add BLAS libs (contains dft)
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
self.FFT_LIB_DIR = self.BLAS_LIB_DIR
self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR
# building the FFTW interfaces is optional,
# so make sure libraries are there before FFT_LIB is set
imklroot = get_software_root(self.FFT_MODULE_NAME[0])
fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
# filter out gfortran from list of FFTW libraries to check for, since it's not provided by imkl
check_fftw_libs = [lib for lib in fftw_libs if lib != 'gfortran']
fftw_lib_exists = lambda x: any([os.path.exists(os.path.join(d, "lib%s.a" % x)) for d in fft_lib_dirs])
if all([fftw_lib_exists(lib) for lib in check_fftw_libs]):
self.FFT_LIB = fftw_libs
else:
tup = (check_fftw_libs, fft_lib_dirs)
msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % tup
self.log.error(msg)
def _set_fftw_variables(self):
if not hasattr(self, 'BLAS_LIB_DIR'):
self.log.raiseException(
"_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)"
)
imklver = get_software_version(self.FFT_MODULE_NAME[0])
fftwsuff = ""
if self.options.get('pic', None):
fftwsuff = "_pic"
fftw_libs = ["fftw3xc_intel%s" % fftwsuff]
if self.options['usempi']:
# add cluster interface
if LooseVersion(imklver) < LooseVersion("11.1"):
if LooseVersion(imklver) >= LooseVersion("11.0"):
fftw_libs.append("fftw3x_cdft_lp64%s" % fftwsuff)
elif LooseVersion(imklver) >= LooseVersion("10.3"):
fftw_libs.append("fftw3x_cdft%s" % fftwsuff)
fftw_libs.append("mkl_cdft_core") # add cluster dft
fftw_libs.extend(self.variables['LIBBLACS'].flatten(
)) ## add BLACS; use flatten because ListOfList
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
fftw_libs.extend(self.variables['LIBBLAS'].flatten(
)) ## add core (contains dft) ; use flatten because ListOfList
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
self.FFT_LIB_DIR = self.BLAS_LIB_DIR
self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR
# building the FFTW interfaces is optional,
# so make sure libraries are there before FFT_LIB is set
imklroot = get_software_root(self.FFT_MODULE_NAME[0])
fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
if all([
any([
os.path.exists(os.path.join(d, "lib%s.a" % lib))
for d in fft_lib_dirs
]) for lib in fftw_libs
]):
self.FFT_LIB = fftw_libs
else:
msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % (
fftw_libs, fft_lib_dirs)
self.log.error(msg)
def _set_fftw_variables(self):
if not hasattr(self, 'BLAS_LIB_DIR'):
self.log.raiseException("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)")
imklver = get_software_version(self.FFT_MODULE_NAME[0])
fftwsuff = ""
if self.options.get('pic', None):
fftwsuff = "_pic"
fftw_libs = ["fftw3xc_intel%s" % fftwsuff]
if self.options['usempi']:
# add cluster interface
if LooseVersion(imklver) < LooseVersion("11.1"):
if LooseVersion(imklver) >= LooseVersion("11.0"):
fftw_libs.append("fftw3x_cdft_lp64%s" % fftwsuff)
elif LooseVersion(imklver) >= LooseVersion("10.3"):
fftw_libs.append("fftw3x_cdft%s" % fftwsuff)
fftw_libs.append("mkl_cdft_core") # add cluster dft
fftw_libs.extend(self.variables['LIBBLACS'].flatten()) ## add BLACS; use flatten because ListOfList
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
fftw_libs.extend(self.variables['LIBBLAS'].flatten()) ## add core (contains dft) ; use flatten because ListOfList
self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
self.FFT_LIB_DIR = self.BLAS_LIB_DIR
self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR
# building the FFTW interfaces is optional,
# so make sure libraries are there before FFT_LIB is set
imklroot = get_software_root(self.FFT_MODULE_NAME[0])
fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
if all([any([os.path.exists(os.path.join(d, "lib%s.a" % lib)) for d in fft_lib_dirs]) for lib in fftw_libs]):
self.FFT_LIB = fftw_libs
else:
msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % (fftw_libs, fft_lib_dirs)
self.log.error(msg)
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
"That's not wise, so let's end this here.\n" \
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
(options, orig_paths, opt_parser, cmd_args) = eboptions.parse_options(args=args)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global log
log = fancylogger.getLogger(fname=False)
# hello world!
log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
log.info("Using robot path: %s" % options.robot)
else:
log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
configOptions = {}
if options.pretend:
configOptions['install_path'] = os.path.join(os.environ['HOME'], 'easybuildinstall')
# default location of configfile is set as default in the config option
config.init(options.config, **configOptions)
# search for modules
if options.search:
search_module(search_path, options.search)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key('name'):
paths = [obtain_path(software_build_specs, easyconfigs_paths,
try_to_generate=try_to_generate, exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=log, opt_parser=opt_parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for i in range(len(orig_paths)):
if not os.path.isabs(orig_paths[i]) and not os.path.exists(orig_paths[i]):
if orig_paths[i] in easyconfigs_map:
log.info("Found %s in %s: %s" % (orig_paths[i], easyconfigs_pkg_full_path,
easyconfigs_map[orig_paths[i]]))
orig_paths[i] = easyconfigs_map[orig_paths[i]]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
validate=validate_easyconfigs))
except IOError, err:
log.error("Processing easyconfigs in path %s failed: %s" % (path, err))