Beispiel #1
0
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
                self.options.list_easyblocks, self.options.list_toolchains]):
            self._postprocess_list_avail()
Beispiel #2
0
    def _set_fftw_variables(self):
        if not hasattr(self, 'BLAS_LIB_DIR'):
            self.log.raiseException(
                "_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)"
            )

        imklver = get_software_version(self.FFT_MODULE_NAME[0])

        picsuff = ''
        if self.options.get('pic', None):
            picsuff = '_pic'
        bitsuff = '_lp64'
        if self.options.get('i8', None):
            bitsuff = '_ilp64'
        compsuff = '_intel'
        if get_software_root('icc') is None:
            if get_software_root('GCC'):
                compsuff = '_gnu'
            else:
                self.log.error(
                    "Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries."
                )

        fftw_libs = ["fftw3xc%s%s" % (compsuff, picsuff)]
        if self.options['usempi']:
            # add cluster interface for recent imkl versions
            if LooseVersion(imklver) >= LooseVersion("11.0.2"):
                fftw_libs.append("fftw3x_cdft%s%s" % (bitsuff, picsuff))
            elif LooseVersion(imklver) >= LooseVersion("10.3"):
                fftw_libs.append("fftw3x_cdft%s" % picsuff)
            fftw_libs.append("mkl_cdft_core")  # add cluster dft
            fftw_libs.extend(self.variables['LIBBLACS'].flatten()
                             )  # add BLACS; use flatten because ListOfList

        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
        fftw_libs.extend(self.variables['LIBBLAS'].flatten()
                         )  # add BLAS libs (contains dft)
        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())

        self.FFT_LIB_DIR = self.BLAS_LIB_DIR
        self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR

        # building the FFTW interfaces is optional,
        # so make sure libraries are there before FFT_LIB is set
        imklroot = get_software_root(self.FFT_MODULE_NAME[0])
        fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
        # filter out gfortran from list of FFTW libraries to check for, since it's not provided by imkl
        check_fftw_libs = [lib for lib in fftw_libs if lib != 'gfortran']
        fftw_lib_exists = lambda x: any([
            os.path.exists(os.path.join(d, "lib%s.a" % x))
            for d in fft_lib_dirs
        ])
        if all([fftw_lib_exists(lib) for lib in check_fftw_libs]):
            self.FFT_LIB = fftw_libs
        else:
            tup = (check_fftw_libs, fft_lib_dirs)
            msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % tup
            self.log.error(msg)
    def _set_fftw_variables(self):
        if not hasattr(self, 'BLAS_LIB_DIR'):
            self.log.raiseException("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)")

        imklver = get_software_version(self.FFT_MODULE_NAME[0])

        picsuff = ''
        if self.options.get('pic', None):
            picsuff = '_pic'
        bitsuff = '_lp64'
        if self.options.get('i8', None):
            bitsuff = '_ilp64'
        compsuff = '_intel'
        if get_software_root('icc') is None:
            if get_software_root('GCC'):
                compsuff = '_gnu'
            else:
                self.log.error("Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries.")

        fftw_libs = ["fftw3xc%s%s" % (compsuff, picsuff)]
        if self.options['usempi']:
            # add cluster interface for recent imkl versions
            if LooseVersion(imklver) >= LooseVersion("11.0.2"):
                fftw_libs.append("fftw3x_cdft%s%s" % (bitsuff, picsuff))
            elif LooseVersion(imklver) >= LooseVersion("10.3"):
                fftw_libs.append("fftw3x_cdft%s" % picsuff)
            fftw_libs.append("mkl_cdft_core")  # add cluster dft
            fftw_libs.extend(self.variables['LIBBLACS'].flatten()) # add BLACS; use flatten because ListOfList

        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
        fftw_libs.extend(self.variables['LIBBLAS'].flatten())  # add BLAS libs (contains dft)
        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())

        self.FFT_LIB_DIR = self.BLAS_LIB_DIR
        self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR

        # building the FFTW interfaces is optional,
        # so make sure libraries are there before FFT_LIB is set
        imklroot = get_software_root(self.FFT_MODULE_NAME[0])
        fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
        # filter out gfortran from list of FFTW libraries to check for, since it's not provided by imkl
        check_fftw_libs = [lib for lib in fftw_libs if lib != 'gfortran']
        fftw_lib_exists = lambda x: any([os.path.exists(os.path.join(d, "lib%s.a" % x)) for d in fft_lib_dirs])
        if all([fftw_lib_exists(lib) for lib in check_fftw_libs]):
            self.FFT_LIB = fftw_libs
        else:
            tup = (check_fftw_libs, fft_lib_dirs)
            msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % tup
            self.log.error(msg)
    def _set_fftw_variables(self):
        if not hasattr(self, 'BLAS_LIB_DIR'):
            self.log.raiseException(
                "_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)"
            )

        imklver = get_software_version(self.FFT_MODULE_NAME[0])

        fftwsuff = ""
        if self.options.get('pic', None):
            fftwsuff = "_pic"
        fftw_libs = ["fftw3xc_intel%s" % fftwsuff]
        if self.options['usempi']:
            # add cluster interface
            if LooseVersion(imklver) < LooseVersion("11.1"):
                if LooseVersion(imklver) >= LooseVersion("11.0"):
                    fftw_libs.append("fftw3x_cdft_lp64%s" % fftwsuff)
                elif LooseVersion(imklver) >= LooseVersion("10.3"):
                    fftw_libs.append("fftw3x_cdft%s" % fftwsuff)
                fftw_libs.append("mkl_cdft_core")  # add cluster dft
                fftw_libs.extend(self.variables['LIBBLACS'].flatten(
                ))  ## add BLACS; use flatten because ListOfList

        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
        fftw_libs.extend(self.variables['LIBBLAS'].flatten(
        ))  ## add core (contains dft) ; use flatten because ListOfList
        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())

        self.FFT_LIB_DIR = self.BLAS_LIB_DIR
        self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR

        # building the FFTW interfaces is optional,
        # so make sure libraries are there before FFT_LIB is set
        imklroot = get_software_root(self.FFT_MODULE_NAME[0])
        fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
        if all([
                any([
                    os.path.exists(os.path.join(d, "lib%s.a" % lib))
                    for d in fft_lib_dirs
                ]) for lib in fftw_libs
        ]):
            self.FFT_LIB = fftw_libs
        else:
            msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % (
                fftw_libs, fft_lib_dirs)
            self.log.error(msg)
    def _set_fftw_variables(self):
        if not hasattr(self, 'BLAS_LIB_DIR'):
            self.log.raiseException("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)")

        imklver = get_software_version(self.FFT_MODULE_NAME[0])

        fftwsuff = ""
        if self.options.get('pic', None):
            fftwsuff = "_pic"
        fftw_libs = ["fftw3xc_intel%s" % fftwsuff]
        if self.options['usempi']:
            # add cluster interface
            if LooseVersion(imklver) < LooseVersion("11.1"):
                if LooseVersion(imklver) >= LooseVersion("11.0"):
                    fftw_libs.append("fftw3x_cdft_lp64%s" % fftwsuff)
                elif LooseVersion(imklver) >= LooseVersion("10.3"):
                    fftw_libs.append("fftw3x_cdft%s" % fftwsuff)
                fftw_libs.append("mkl_cdft_core")  # add cluster dft
                fftw_libs.extend(self.variables['LIBBLACS'].flatten()) ## add BLACS; use flatten because ListOfList

        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
        fftw_libs.extend(self.variables['LIBBLAS'].flatten()) ## add core (contains dft) ; use flatten because ListOfList
        self.log.debug('fftw_libs %s' % fftw_libs.__repr__())

        self.FFT_LIB_DIR = self.BLAS_LIB_DIR
        self.FFT_INCLUDE_DIR = self.BLAS_INCLUDE_DIR

        # building the FFTW interfaces is optional,
        # so make sure libraries are there before FFT_LIB is set
        imklroot = get_software_root(self.FFT_MODULE_NAME[0])
        fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
        if all([any([os.path.exists(os.path.join(d, "lib%s.a" % lib)) for d in fft_lib_dirs]) for lib in fftw_libs]):
            self.FFT_LIB = fftw_libs
        else:
            msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % (fftw_libs, fft_lib_dirs)
            self.log.error(msg)
Beispiel #6
0
def main(testing_data=(None, None)):
    """
    Main function:
    @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
    This function will:
    - read easyconfig
    - build software
    """
    # disallow running EasyBuild as root
    if os.getuid() == 0:
        sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
                        "That's not wise, so let's end this here.\n" \
                        "Exiting.\n")
        sys.exit(1)

    # steer behavior when testing main
    testing = testing_data[0] is not None
    args, logfile = testing_data

    # initialise options
    (options, orig_paths, opt_parser, cmd_args) = eboptions.parse_options(args=args)

    # initialise logging for main
    if options.logtostdout:
        fancylogger.logToScreen(enable=True, stdout=True)
    else:
        if logfile is None:
            # mkstemp returns (fd,filename), fd is from os.open, not regular open!
            fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
            os.close(fd)

        fancylogger.logToFile(logfile)
        print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)

    global log
    log = fancylogger.getLogger(fname=False)

    # hello world!
    log.info(this_is_easybuild())

    # set strictness of filetools module
    if options.strict:
        filetools.strictness = options.strict

    if not options.robot is None:
        if options.robot:
            log.info("Using robot path: %s" % options.robot)
        else:
            log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")

    # determine easybuild-easyconfigs package install path
    easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
    easyconfigs_pkg_full_path = None

    if easyconfigs_paths:
        easyconfigs_pkg_full_path = easyconfigs_paths[0]
        if not options.robot:
            search_path = easyconfigs_pkg_full_path
        else:
            search_path = options.robot
    else:
        log.info("Failed to determine install path for easybuild-easyconfigs package.")

    if options.robot:
        easyconfigs_paths = [options.robot] + easyconfigs_paths

    configOptions = {}
    if options.pretend:
        configOptions['install_path'] = os.path.join(os.environ['HOME'], 'easybuildinstall')

    # default location of configfile is set as default in the config option
    config.init(options.config, **configOptions)

    # search for modules
    if options.search:
        search_module(search_path, options.search)

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, software_build_specs) = process_software_build_specs(options)

    paths = []
    if len(orig_paths) == 0:
        if software_build_specs.has_key('name'):
            paths = [obtain_path(software_build_specs, easyconfigs_paths,
                                 try_to_generate=try_to_generate, exit_on_error=not testing)]
        elif not any([options.aggregate_regtest, options.search, options.regtest]):
            print_error(("Please provide one or multiple easyconfig files, or use software build "
                  "options to make EasyBuild search for easyconfigs"),
                  log=log, opt_parser=opt_parser, exit_on_error=not testing)
    else:
        # look for easyconfigs with relative paths in easybuild-easyconfigs package,
        # unless they we found at the given relative paths

        if easyconfigs_pkg_full_path:
            # create a mapping from filename to path in easybuild-easyconfigs package install path
            easyconfigs_map = {}
            for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
                for filename in filenames:
                    easyconfigs_map.update({filename: os.path.join(subpath, filename)})

            # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
            for i in range(len(orig_paths)):
                if not os.path.isabs(orig_paths[i]) and not os.path.exists(orig_paths[i]):
                    if orig_paths[i] in easyconfigs_map:
                        log.info("Found %s in %s: %s" % (orig_paths[i], easyconfigs_pkg_full_path,
                                                         easyconfigs_map[orig_paths[i]]))
                        orig_paths[i] = easyconfigs_map[orig_paths[i]]

        # indicate that specified paths do not contain generated easyconfig files
        paths = [(path, False) for path in orig_paths]

    log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        log.info("Running regression test")
        if paths:
            regtest_ok = regtest(options, [path[0] for path in paths])
        else:  # fallback: easybuild-easyconfigs install path
            regtest_ok = regtest(options, [easyconfigs_pkg_full_path])

        if not regtest_ok:
            log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    if any([options.search, options.regtest]):
        cleanup_logfile_and_exit(logfile, testing, True)

    # building a dependency graph implies force, so that all dependencies are retained
    # and also skips validation of easyconfigs (e.g. checking os dependencies)
    validate_easyconfigs = True
    retain_all_deps = False
    if options.dep_graph:
        log.info("Enabling force to generate dependency graph.")
        options.force = True
        validate_easyconfigs = False
        retain_all_deps = True

    # read easyconfig files
    easyconfigs = []
    for (path, generated) in paths:
        path = os.path.abspath(path)
        if not (os.path.exists(path)):
            print_error("Can't find path %s" % path)

        try:
            files = find_easyconfigs(path)
            for f in files:
                if not generated and try_to_generate and software_build_specs:
                    ec_file = easyconfig.tweak(f, None, software_build_specs)
                else:
                    ec_file = f
                easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
                                                      validate=validate_easyconfigs))
        except IOError, err:
            log.error("Processing easyconfigs in path %s failed: %s" % (path, err))