def configure(self): """Configure FSL build: set FSLDIR env var.""" self.fsldir = self.getcfg('startfrom') env.set('FSLDIR', self.fsldir) # determine FSL machine type cmd = ". %s/etc/fslconf/fsl.sh && echo $FSLMACHTYPE" % self.fsldir (out, _) = run_cmd(cmd, log_all=True, simple=False) fslmachtype = out.strip() self.log.debug("FSL machine type: %s" % fslmachtype) # prepare config # either using matching config, or copy closest match cfgdir = os.path.join(self.fsldir, "config") try: cfgs = os.listdir(cfgdir) best_cfg = difflib.get_close_matches(fslmachtype, cfgs)[0] self.log.debug("Best matching config dir for %s is %s" % (fslmachtype, best_cfg)) if fslmachtype != best_cfg: srcdir = os.path.join(cfgdir, best_cfg) tgtdir = os.path.join(cfgdir, fslmachtype) shutil.copytree(srcdir, tgtdir) self.log.debug("Copied %s to %s" % (srcdir, tgtdir)) except OSError, err: self.log.error("Failed to copy closest matching config dir: %s" % err)
def configure(self): # things might go wrong if a previous install dir is present, so let's get rid of it if not self.getcfg('keeppreviousinstall'): self.log.info("Making sure any old installation is removed before we start the build...") Application.make_dir(self, self.installdir, True, dontcreateinstalldir=True) # additional configuration options add_configopts = '--with-rdma=%s ' % self.getcfg('rdma_type') # use POSIX threads add_configopts += '--with-thread-package=pthreads ' if self.getcfg('debug'): # debug build, with error checking, timing and debug info # note: this will affact performance add_configopts += '--enable-fast=none ' else: # optimized build, no error checking, timing or debug info add_configopts += '--enable-fast ' # enable shared libraries, using GCC and GNU ld options add_configopts += '--enable-shared --enable-sharedlibs=gcc ' # enable Fortran 77/90 and C++ bindings add_configopts += '--enable-f77 --enable-fc --enable-cxx ' # MVAPICH configure script complains when F90 or F90FLAGS are set, # they should be replaced with FC/FCFLAGS instead for (envvar, new_envvar) in [("F90", "FC"), ("F90FLAGS", "FCFLAGS")]: envvar_val = os.getenv(envvar) if envvar_val: if not os.getenv(new_envvar): env.set(new_envvar, envvar_val) env.set(envvar, '') else: self.log.error("Both %(ev)s and %(nev)s set, can I overwrite %(nev)s with %(ev)s (%(evv)s) ?" % { 'ev': envvar, 'nev': new_envvar, 'evv': envvar_val }) # enable specific support options (if desired) if self.getcfg('withmpe'): add_configopts += '--enable-mpe ' if self.getcfg('withlimic2'): add_configopts += '--enable-limic2 ' if self.getcfg('withchkpt'): add_configopts += '--enable-checkpointing --with-hydra-ckpointlib=blcr ' self.updatecfg('configopts', add_configopts) Application.configure(self)
def configure(self): """Configure build: set config options and configure""" if self.toolkit().opts['pic']: self.updatecfg('configopts', "--with-pic") self.updatecfg('configopts', 'FCFLAGS="%s" FC="%s"' % (os.getenv('FFLAGS'), os.getenv('F90'))) # add -DgFortran to CPPFLAGS when building with GCC if self.toolkit().comp_family() == toolkit.GCC: env.set('CPPFLAGS', "%s -DgFortran" % os.getenv('CPPFLAGS')) Application.configure(self)
def set_netcdf_env_vars(log): """Set netCDF environment variables used by other software.""" netcdf = get_software_root('netCDF') if not netcdf: log.error("netCDF module not loaded?") else: env.set('NETCDF', netcdf) log.debug("Set NETCDF to %s" % netcdf) netcdff = get_software_root('netCDF-Fortran') netcdf_ver = get_software_version('netCDF') if not netcdff: if LooseVersion(netcdf_ver) >= LooseVersion("4.2"): log.error("netCDF v4.2 no longer supplies Fortran library, also need netCDF-Fortran") else: env.set('NETCDFF', netcdff) log.debug("Set NETCDFF to %s" % netcdff)
def mpi_cmd_for(self, cmd, nr_ranks): """Construct an MPI command for the given command and number of ranks.""" # parameter values for mpirun command params = {'nr_ranks':nr_ranks, 'cmd':cmd} # different known mpirun commands mpi_cmds = { OPENMPI:"mpirun -n %(nr_ranks)d %(cmd)s", INTEL:"mpirun %(mpdbootfile)s %(nodesfile)s -np %(nr_ranks)d %(cmd)s", } mpi_type = self.mpi_type() # Intel MPI mpirun needs more work if mpi_type == INTEL: # set temporary dir for mdp env.set('I_MPI_MPD_TMPDIR', "/tmp") # set PBS_ENVIRONMENT, so that --file option for mpdboot isn't stripped away env.set('PBS_ENVIRONMENT', "PBS_BATCH_MPI") # create mpdboot file fn = "/tmp/mpdboot" try: if os.path.exists(fn): os.remove(fn) f = open(fn, "w") f.write("localhost ifhn=localhost") f.close() except (OSError, IOError), err: self.log.error("Failed to create file %s: %s" % (fn, err)) params.update({'mpdbootfile':"--file=%s"%fn}) # create nodes file fn = "/tmp/nodes" try: if os.path.exists(fn): os.remove(fn) f = open(fn, "w") f.write("localhost\n" * nr_ranks) f.close() except (OSError, IOError), err: self.log.error("Failed to create file %s: %s" % (fn, err))
def configure(self): """Configure: handle license file and clean home dir.""" # obtain license path self.license = self.getcfg('license') if self.license: self.log.info("Using license %s" % self.license) else: self.log.error("No license defined") # verify license path if not os.path.exists(self.license): self.log.error("Can't find license at %s" % self.license) # set INTEL_LICENSE_FILE env.set("INTEL_LICENSE_FILE", self.license) # clean home directory self.clean_homedir()
def _setVariables(self, dontset=None): """ Sets the environment variables """ self.log.debug("Setting variables: dontset=%s" % dontset) dontsetlist = [] if type(dontset) == str: dontsetlist = dontset.split(',') elif type(dontset) == list: dontsetlist = dontset for key, val in self.vars.items(): if key in dontsetlist: self.log.debug("Not setting environment variable %s (value: %s)." % (key, val)) continue self.log.debug("Setting environment variable %s to %s" % (key, val)) env.set(key, val) # also set unique named variables that can be used in Makefiles # - so you can have 'CFLAGS = $(EBVARCFLAGS)' # -- 'CLFLAGS = $(CFLAGS)' gives '*** Recursive variable `CFLAGS' # references itself (eventually). Stop' error env.set("EBVAR%s" % key, val)
def modifyEnv(old, new): """ Compares 2 os.environ dumps. Adapts final environment. """ oldKeys = old.keys() newKeys = new.keys() for key in newKeys: ## set them all. no smart checking for changed/identical values if key in oldKeys: ## hmm, smart checking with debug logging if not new[key] == old[key]: log.debug("Key in new environment found that is different from old one: %s (%s)" % (key, new[key])) env.set(key, new[key]) else: log.debug("Key in new environment found that is not in old one: %s (%s)" % (key, new[key])) env.set(key, new[key]) for key in oldKeys: if not key in newKeys: log.debug("Key in old environment found that is not in new one: %s (%s)" % (key, old[key])) os.unsetenv(key) del os.environ[key] return 'ok'
def make_install(self): """Actual installation - create silent cfg file - set environment parameters - execute command """ silent = """ ACTIVATION=%s PSET_LICENSE_FILE=%s PSET_INSTALL_DIR=%s ACCEPT_EULA=accept INSTALL_MODE=NONRPM CONTINUE_WITH_OPTIONAL_ERROR=yes """ % (self.getcfg('license_activation'), self.license, self.installdir) # we should be already in the correct directory silentcfg = os.path.join(os.getcwd(), "silent.cfg") try: f = open(silentcfg, 'w') f.write(silent) f.close() except: self.log.exception("Writing silent cfg % failed" % silent) # workaround for mktmp: create tmp dir and use it tmpdir = os.path.join(self.getcfg('startfrom'), 'mytmpdir') try: os.makedirs(tmpdir) except: self.log.exception("Directory %s can't be created" % (tmpdir)) tmppathopt = '' if self.getcfg('usetmppath'): env.set('TMP_PATH', tmpdir) tmppathopt = "-t %s" % tmpdir # set some extra env variables env.set('LOCAL_INSTALL_VERBOSE','1') env.set('VERBOSE_MODE', '1') env.set('INSTALL_PATH', self.installdir) # perform installation cmd = "./install.sh %s -s %s" % (tmppathopt, silentcfg) return run_cmd(cmd, log_all=True, simple=True)
def configure(self): """Configure build: - set some magic environment variables - run configure script - adjust configure.wrf file if needed """ # netCDF dependency set_netcdf_env_vars(self.log) # HDF5 (optional) dependency hdf5 = get_software_root('HDF5') if hdf5: # check if this is parallel HDF5 phdf5_bins = ['h5pcc','ph5diff'] parallel_hdf5 = True for f in phdf5_bins: if not os.path.exists(os.path.join(hdf5, 'bin', f)): parallel_hdf5 = False break if not (hdf5 or parallel_hdf5): self.log.error("Parallel HDF5 module not loaded?") else: env.set('PHDF5', hdf5) else: self.log.info("HDF5 module not loaded, assuming that's OK...") # JasPer dependency check + setting env vars jasper = get_software_root('JasPer') jasperlibdir = os.path.join(jasper, "lib") if jasper: env.set('JASPERINC', os.path.join(jasper, "include")) env.set('JASPERLIB', jasperlibdir) else: if os.getenv('JASPERINC') or os.getenv('JASPERLIB'): self.log.error("JasPer module not loaded, but JASPERINC and/or JASPERLIB still set?") else: self.log.info("JasPer module not loaded, assuming that's OK...") # enable support for large file support in netCDF env.set('WRFIO_NCD_LARGE_FILE_SUPPORT', '1') # patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions patch_perl_script_autoflush(os.path.join("arch", "Config_new.pl")) # determine build type option to look for build_type_option = None self.comp_fam = self.toolkit().comp_family() if self.comp_fam == toolkit.INTEL: build_type_option = "Linux x86_64 i486 i586 i686, ifort compiler with icc" elif self.comp_fam == toolkit.GCC: build_type_option = "x86_64 Linux, gfortran compiler with gcc" else: self.log.error("Don't know how to figure out build type to select.") # fetch selected build type (and make sure it makes sense) known_build_types = ['serial', 'smpar', 'dmpar', 'dm+sm'] self.parallel_build_types = ["dmpar","smpar","dm+sm"] bt = self.getcfg('buildtype') if not bt in known_build_types: self.log.error("Unknown build type: '%s'. Supported build types: %s" % (bt, known_build_types)) # fetch option number based on build type option and selected build type build_type_question = "\s*(?P<nr>[0-9]+).\s*%s\s*\(%s\)" % (build_type_option, bt) # run configure script cmd = "./configure" qa = { # named group in match will be used to construct answer "Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]:": "1", "Compile for nesting? (0=no nesting, 1=basic, 2=preset moves, 3=vortex following) [default 0]:": "0" } no_qa = [] std_qa = { # named group in match will be used to construct answer r"%s.*\n(.*\n)*Enter selection\s*\[[0-9]+-[0-9]+\]\s*:" % build_type_question: "%(nr)s", } run_cmd_qa(cmd, qa, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True) cfgfile= 'configure.wrf' # make sure correct compilers are being used comps = { 'SCC': os.getenv('CC'), 'SFC': os.getenv('F90'), 'CCOMP': os.getenv('CC'), 'DM_FC': os.getenv('MPIF90'), 'DM_CC': "%s -DMPI2_SUPPORT" % os.getenv('MPICC'), } for line in fileinput.input(cfgfile, inplace=1, backup='.orig.comps'): for (k, v) in comps.items(): line = re.sub(r"^(%s\s*=\s*).*$" % k, r"\1 %s" % v, line) sys.stdout.write(line) # rewrite optimization options if desired if self.getcfg('rewriteopts'): # replace default -O3 option in configure.wrf with CFLAGS/FFLAGS from environment self.log.info("Rewriting optimization options in %s" % cfgfile) # set extra flags for Intel compilers # see http://software.intel.com/en-us/forums/showthread.php?t=72109&p=1#146748 if self.comp_fam == toolkit.INTEL: # -O3 -heap-arrays is required to resolve compilation error for envvar in ['CFLAGS', 'FFLAGS']: val = os.getenv(envvar) if '-O3' in val: env.set(envvar, '%s -heap-arrays' % val) self.log.info("Updated %s to '%s'" % (envvar, os.getenv(envvar))) # replace -O3 with desired optimization options for line in fileinput.input(cfgfile, inplace=1, backup='.orig.rewriteopts'): line = re.sub(r"^(FCOPTIM.*)(\s-O3)(\s.*)$", r"\1 %s \3" % os.getenv('FFLAGS'), line) line = re.sub(r"^(CFLAGS_LOCAL.*)(\s-O3)(\s.*)$", r"\1 %s \3" % os.getenv('CFLAGS'), line) sys.stdout.write(line)
def postproc(self): """ The mkl directory structure has thoroughly changed as from version 10.3. Hence post processing is quite different in both situations """ if LooseVersion(self.version()) >= LooseVersion('10.3'): #Add convenient wrapper libs #- form imkl 10.3 if self.getcfg('m32'): self.log.error("32-bit not supported yet for IMKL v%s (>=10.3)" % self.version()) extra = { 'libmkl.so': 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } for fil, txt in extra.items(): dest = os.path.join(self.installdir, 'mkl/lib/intel64', fil) if not os.path.exists(dest): try: f = open(dest, 'w') f.write(txt) f.close() self.log.info("File %s written" % dest) except: self.log.exception("Can't write file %s" % (dest)) # build the mkl interfaces (pic and no-pic) # load the dependencies m = Modules() m.addModule(self.cfg.dependencies()) m.load() if not self.getcfg('interfaces'): return # build the interfaces #- blas95 and lapack95 need more work, ignore for now #lis1=['blas95','fftw2xc','fftw2xf','lapack95'] # blas95 and lapack also need include/.mod to be processed lis1 = ['fftw2xc', 'fftw2xf'] lis2 = ['fftw3xc', 'fftw3xf'] lis3 = ['fftw2x_cdft', 'fftw3x_cdft'] interfacedir = os.path.join(self.installdir, 'mkl/interfaces') try: os.chdir(interfacedir) self.log.info("Changed to interfaces directory %s" % interfacedir) except: self.log.exception("Can't change to interfaces directory %s" % interfacedir) # compiler defaults to icc, but we could be using gcc to create gimkl. makeopts = '' if self.toolkit().comp_family() == toolkit.GCC: makeopts += 'compiler=gnu ' for i in lis1 + lis2 + lis3: if i in lis1: # use INSTALL_DIR and CFLAGS and COPTS cmd = "make -f makefile libintel64" if i in lis2: # use install_to and CFLAGS cmd = "make -f makefile libintel64 install_to=$INSTALL_DIR" if i in lis3: # use INSTALL_DIR and SPEC_OPT extramakeopts = '' if self.toolkit().mpi_type() == toolkit.MPICH2: extramakeopts += 'mpi=mpich2' cmd = "make -f makefile libintel64 %s" % extramakeopts for opt in ['', '-fPIC']: try: tmpbuild = tempfile.mkdtemp() self.log.debug("Created temporary directory %s" % tmpbuild) except: self.log.exception("Creating temporary directory failed") # always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS env.set('INSTALL_DIR', tmpbuild) env.set('SPEC_OPT', opt) env.set('COPTS', opt) env.set('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) os.chdir(intdir) self.log.info("Changed to interface %s directory %s" % (i, intdir)) except: self.log.exception("Can't change to interface %s directory %s" % (i, intdir)) if not run_cmd(cmd, log_all=True, simple=True): self.log.error("Building %s (opt: %s) failed" % (i, opt)) for fil in os.listdir(tmpbuild): if opt == '-fPIC': # add _pic to filename ff = fil.split('.') newfil = '.'.join(ff[:-1]) + '_pic.' + ff[-1] else: newfil = fil dest = os.path.join(self.installdir, 'mkl/lib/intel64', newfil) try: src = os.path.join(tmpbuild, fil) if os.path.isfile(src): shutil.move(src, dest) self.log.info("Moved %s to %s" % (src, dest)) except: self.log.exception("Failed to move %s to %s" % (src, dest)) try: shutil.rmtree(tmpbuild) self.log.debug('Removed temporary directory %s' % tmpbuild) except: self.log.exception("Removing temporary directory %s failed" % tmpbuild) else: #Follow this procedure for mkl version lower than 10.3 #Extra #- build the mkl interfaces (pic and no-pic) #- add wrapper libs # Add convenient libs #- form imkl 10.1 if self.getcfg('m32'): extra = { 'libmkl.so': 'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } else: extra = { 'libmkl.so': 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } for fil, txt in extra.items(): if self.getcfg('m32'): dest = os.path.join(self.installdir, 'lib/32', fil) else: dest = os.path.join(self.installdir, 'lib/em64t', fil) if not os.path.exists(dest): try: f = open(dest, 'w') f.write(txt) f.close() self.log.info("File %s written" % dest) except: self.log.exception("Can't write file %s" % (dest)) # load the dependencies m = Modules() m.addModule(self.cfg.dependencies()) m.load() if not self.getcfg('interfaces'): return # build the interfaces # - blas95 and lapack95 need more work, ignore for now #lis1=['blas95','fftw2xc','fftw2x_cdft','fftw2xf','lapack95'] # blas95 and lapack also need include/.mod to be processed lis1 = ['fftw2xc', 'fftw2x_cdft', 'fftw2xf'] lis2 = ['fftw3xc', 'fftw3xf'] interfacedir = os.path.join(self.installdir, 'interfaces') try: os.chdir(interfacedir) except: self.log.exception("Can't change to interfaces directory %s" % interfacedir) interfacestarget = "libem64t" if self.getcfg('m32'): interfacestarget = "lib32" for i in lis1 + lis2: if i in lis1: # use INSTALL_DIR and SPEC_OPT cmd = "make -f makefile %s" % interfacestarget if i in lis2: # use install_to and CFLAGS cmd = "make -f makefile %s install_to=$INSTALL_DIR" % interfacestarget for opt in ['', '-fPIC']: try: tmpbuild = tempfile.mkdtemp() self.log.debug("Created temporary directory %s" % tmpbuild) except: self.log.exception("Creating temporary directory failed") # always set INSTALL_DIR, SPEC_OPT and CFLAGS env.set('INSTALL_DIR', tmpbuild) env.set('SPEC_OPT', opt) env.set('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) os.chdir(intdir) except: self.log.exception("Can't change to interface %s directory %s" % (i, intdir)) if not run_cmd(cmd, log_all=True, simple=True): self.log.error("Building %s (opt: %s) failed" % (i, opt)) for fil in os.listdir(tmpbuild): if opt == '-fPIC': # add _pic to filename ff = fil.split('.') newfil = '.'.join(ff[:-1]) + '_pic.' + ff[-1] else: newfil = fil if self.getcfg('m32'): dest = os.path.join(self.installdir, 'lib/32', newfil) else: dest = os.path.join(self.installdir, 'lib/em64t', newfil) try: src = os.path.join(tmpbuild, fil) shutil.move(src, dest) self.log.debug("Moved %s to %s" % (src, dest)) except: self.log.exception("Failed to move %s to %s" % (src, dest)) try: shutil.rmtree(tmpbuild) self.log.debug('Removed temporary directory %s' % tmpbuild) except: self.log.exception("Removing temporary directory %s failed" % (tmpbuild))
def configure(self): """Configure OpenFOAM build by setting appropriate environment variables.""" # installation directory env.set("FOAM_INST_DIR", self.installdir) # third party directory self.thrdpartydir = "ThirdParty-%s" % self.version() os.symlink(os.path.join("..", self.thrdpartydir), self.thrdpartydir) env.set("WM_THIRD_PARTY_DIR", os.path.join(self.installdir, self.thrdpartydir)) # compiler comp_fam = self.toolkit().comp_family() if comp_fam == toolkit.GCC: self.wm_compiler="Gcc" elif comp_fam == toolkit.INTEL: self.wm_compiler="Icc" # make sure -no-prec-div is used with Intel compilers self.updatecfg('premakeopts', 'CFLAGS="$CFLAGS -no-prec-div" CXXFLAGS="$CXXFLAGS -no-prec-div"') else: self.log.error("Unknown compiler family, don't know how to set WM_COMPILER") env.set("WM_COMPILER",self.wm_compiler) # type of MPI mpi_type = self.toolkit().mpi_type() if mpi_type == toolkit.INTEL: self.mpipath = os.path.join(get_software_root('IMPI'),'intel64') self.wm_mplib = "IMPI" elif mpi_type == toolkit.QLOGIC: self.mpipath = get_software_root('QLogicMPI') self.wm_mplib = "MPICH" elif mpi_type == toolkit.OPENMPI: self.mpipath = get_software_root('OpenMPI') self.wm_mplib = "MPI-MVAPICH2" else: self.log.error("Unknown MPI, don't know how to set MPI_ARCH_PATH, WM_MPLIB or FOAM_MPI_LIBBIN") env.set("WM_MPLIB", self.wm_mplib) env.set("MPI_ARCH_PATH", self.mpipath) env.set("FOAM_MPI_LIBBIN", self.mpipath) # parallel build spec env.set("WM_NCOMPPROCS", str(self.getcfg('parallel')))