def post_install_step(self): """ The mkl directory structure has thoroughly changed as from version 10.3. Hence post processing is quite different in both situations """ if LooseVersion(self.version) >= LooseVersion('10.3'): # Add convenient wrapper libs # - form imkl 10.3 if self.cfg['m32']: self.log.error( "32-bit not supported yet for IMKL v%s (>=10.3)" % self.version) extra = { 'libmkl.so': 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } for fil, txt in extra.items(): dest = os.path.join(self.installdir, 'mkl/lib/intel64', fil) if not os.path.exists(dest): try: f = open(dest, 'w') f.write(txt) f.close() self.log.info("File %s written" % dest) except: self.log.exception("Can't write file %s" % (dest)) # build the mkl interfaces (pic and no-pic) # load the dependencies m = Modules() m.add_module(self.cfg.dependencies()) m.load() if not self.cfg['interfaces']: return # build the interfaces # - blas95 and lapack95 need more work, ignore for now # lis1=['blas95','fftw2xc','fftw2xf','lapack95'] # blas95 and lapack also need include/.mod to be processed lis1 = ['fftw2xc', 'fftw2xf'] lis2 = ['fftw3xc', 'fftw3xf'] lis3 = ['fftw2x_cdft', 'fftw3x_cdft'] interfacedir = os.path.join(self.installdir, 'mkl/interfaces') try: os.chdir(interfacedir) self.log.info("Changed to interfaces directory %s" % interfacedir) except: self.log.exception("Can't change to interfaces directory %s" % interfacedir) # compiler defaults to icc, but we could be using gcc to create gimkl. makeopts = '' if get_software_root( 'GCC' ): # can't use toolchain.comp_family, because of dummy toolchain makeopts = 'compiler=gnu ' for i in lis1 + lis2 + lis3: if i in lis1: # use INSTALL_DIR and CFLAGS and COPTS cmd = "make -f makefile libintel64" if i in lis2: # use install_to and CFLAGS cmd = "make -f makefile libintel64 install_to=$INSTALL_DIR" if i in lis3: # use INSTALL_DIR and SPEC_OPT extramakeopts = '' if get_software_root( 'MPICH2' ): # can't use toolchain.mpi_family, because of dummy toolchain extramakeopts = 'mpi=mpich2' elif get_software_root('OpenMPI'): extramakeopts = 'mpi=openmpi' cmd = "make -f makefile libintel64 %s" % extramakeopts # add other make options as well cmd = ' '.join([cmd, makeopts]) for opt in ['', '-fPIC']: try: tmpbuild = tempfile.mkdtemp() self.log.debug("Created temporary directory %s" % tmpbuild) except: self.log.exception( "Creating temporary directory failed") # always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS env.setvar('INSTALL_DIR', tmpbuild) env.setvar('SPEC_OPT', opt) env.setvar('COPTS', opt) env.setvar('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) os.chdir(intdir) self.log.info("Changed to interface %s directory %s" % (i, intdir)) except: self.log.exception( "Can't change to interface %s directory %s" % (i, intdir)) if not run_cmd(cmd, log_all=True, simple=True): self.log.error("Building %s (opt: %s) failed" % (i, opt)) for fil in os.listdir(tmpbuild): if opt == '-fPIC': # add _pic to filename ff = fil.split('.') newfil = '.'.join(ff[:-1]) + '_pic.' + ff[-1] else: newfil = fil dest = os.path.join(self.installdir, 'mkl/lib/intel64', newfil) try: src = os.path.join(tmpbuild, fil) if os.path.isfile(src): shutil.move(src, dest) self.log.info("Moved %s to %s" % (src, dest)) except: self.log.exception("Failed to move %s to %s" % (src, dest)) try: rmtree2(tmpbuild) self.log.debug('Removed temporary directory %s' % tmpbuild) except: self.log.exception( "Removing temporary directory %s failed" % tmpbuild) else: # Follow this procedure for mkl version lower than 10.3 # Extra # - build the mkl interfaces (pic and no-pic) # - add wrapper libs # Add convenient libs # - form imkl 10.1 if self.cfg['m32']: extra = { 'libmkl.so': 'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } else: extra = { 'libmkl.so': 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } for fil, txt in extra.items(): if self.cfg['m32']: dest = os.path.join(self.installdir, 'lib/32', fil) else: dest = os.path.join(self.installdir, 'lib/em64t', fil) if not os.path.exists(dest): try: f = open(dest, 'w') f.write(txt) f.close() self.log.info("File %s written" % dest) except: self.log.exception("Can't write file %s" % (dest)) # load the dependencies m = Modules() m.add_module(self.cfg.dependencies()) m.load() if not self.cfg['interfaces']: return # build the interfaces # - blas95 and lapack95 need more work, ignore for now # lis1=['blas95','fftw2xc','fftw2x_cdft','fftw2xf','lapack95'] # blas95 and lapack also need include/.mod to be processed lis1 = ['fftw2xc', 'fftw2x_cdft', 'fftw2xf'] lis2 = ['fftw3xc', 'fftw3xf'] interfacedir = os.path.join(self.installdir, 'interfaces') try: os.chdir(interfacedir) except: self.log.exception("Can't change to interfaces directory %s" % interfacedir) interfacestarget = "libem64t" if self.cfg['m32']: interfacestarget = "lib32" for i in lis1 + lis2: if i in lis1: # use INSTALL_DIR and SPEC_OPT cmd = "make -f makefile %s" % interfacestarget if i in lis2: # use install_to and CFLAGS cmd = "make -f makefile %s install_to=$INSTALL_DIR" % interfacestarget for opt in ['', '-fPIC']: try: tmpbuild = tempfile.mkdtemp() self.log.debug("Created temporary directory %s" % tmpbuild) except: self.log.exception( "Creating temporary directory failed") # always set INSTALL_DIR, SPEC_OPT and CFLAGS env.setvar('INSTALL_DIR', tmpbuild) env.setvar('SPEC_OPT', opt) env.setvar('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) os.chdir(intdir) except: self.log.exception( "Can't change to interface %s directory %s" % (i, intdir)) if not run_cmd(cmd, log_all=True, simple=True): self.log.error("Building %s (opt: %s) failed" % (i, opt)) for fil in os.listdir(tmpbuild): if opt == '-fPIC': # add _pic to filename ff = fil.split('.') newfil = '.'.join(ff[:-1]) + '_pic.' + ff[-1] else: newfil = fil if self.cfg['m32']: dest = os.path.join(self.installdir, 'lib/32', newfil) else: dest = os.path.join(self.installdir, 'lib/em64t', newfil) try: src = os.path.join(tmpbuild, fil) shutil.move(src, dest) self.log.debug("Moved %s to %s" % (src, dest)) except: self.log.exception("Failed to move %s to %s" % (src, dest)) try: rmtree2(tmpbuild) self.log.debug('Removed temporary directory %s' % tmpbuild) except: self.log.exception( "Removing temporary directory %s failed" % (tmpbuild))
def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment onlymod: Boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). """ if not self._toolchain_exists(): self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version)) if self.name == self.DUMMY_NAME: if self.version == self.DUMMY_VERSION: self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies') else: self.log.info('prepare: toolchain dummy mode and loading dependencies') modules = Modules() modules.add_module(self.dependencies) modules.load() return ## Load the toolchain and dependencies modules modules = Modules() modules.add_module([(self.name, self.version)]) modules.add_module(self.dependencies) modules.load() # determine direct toolchain dependencies (legacy, not really used anymore) self.toolchain_dependencies = modules.dependencies_for(self.name, self.version, depth=0) self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dependencies) # verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module toolchain_module_deps = set([mod['name'] for mod in self.toolchain_dependencies]) toolchain_elements_mod_names = set([y for x in dir(self) if x.endswith('_MODULE_NAME') for y in eval("self.%s" % x)]) # filter out toolchain name (e.g. 'GCC') from list of toolchain elements toolchain_elements_mod_names = set([x for x in toolchain_elements_mod_names if not x == self.name]) self.log.debug("List of toolchain dependency modules from loaded toolchain module: %s" % toolchain_module_deps) self.log.debug("List of toolchain elements from toolchain definition: %s" % toolchain_elements_mod_names) if toolchain_module_deps == toolchain_elements_mod_names: self.log.info("List of toolchain dependency modules and toolchain definition match!") else: self.log.error("List of toolchain dependency modules and toolchain definition do not match " \ "(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names)) ## Generate the variables to be set self.set_variables() ## set the variables ## onlymod can be comma-separated string of variables not to be set if onlymod == True: self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod) self.generate_vars() else: self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) ## add LDFLAGS and CPPFLAGS from dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod)
def post_install_step(self): """ The mkl directory structure has thoroughly changed as from version 10.3. Hence post processing is quite different in both situations """ if LooseVersion(self.version) >= LooseVersion('10.3'): #Add convenient wrapper libs #- form imkl 10.3 if self.cfg['m32']: self.log.error("32-bit not supported yet for IMKL v%s (>=10.3)" % self.version) extra = { 'libmkl.so': 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } for fil, txt in extra.items(): dest = os.path.join(self.installdir, 'mkl/lib/intel64', fil) if not os.path.exists(dest): try: f = open(dest, 'w') f.write(txt) f.close() self.log.info("File %s written" % dest) except: self.log.exception("Can't write file %s" % (dest)) # build the mkl interfaces (pic and no-pic) # load the dependencies m = Modules() m.add_module(self.cfg.dependencies()) m.load() if not self.cfg['interfaces']: return # build the interfaces #- blas95 and lapack95 need more work, ignore for now #lis1=['blas95','fftw2xc','fftw2xf','lapack95'] # blas95 and lapack also need include/.mod to be processed lis1 = ['fftw2xc', 'fftw2xf'] lis2 = ['fftw3xc', 'fftw3xf'] lis3 = ['fftw2x_cdft', 'fftw3x_cdft'] interfacedir = os.path.join(self.installdir, 'mkl/interfaces') try: os.chdir(interfacedir) self.log.info("Changed to interfaces directory %s" % interfacedir) except: self.log.exception("Can't change to interfaces directory %s" % interfacedir) # compiler defaults to icc, but we could be using gcc to create gimkl. makeopts = '' if get_software_root('GCC'): # can't use toolchain.comp_family, because of dummy toolchain makeopts = 'compiler=gnu ' for i in lis1 + lis2 + lis3: if i in lis1: # use INSTALL_DIR and CFLAGS and COPTS cmd = "make -f makefile libintel64" if i in lis2: # use install_to and CFLAGS cmd = "make -f makefile libintel64 install_to=$INSTALL_DIR" if i in lis3: # use INSTALL_DIR and SPEC_OPT extramakeopts = '' if get_software_root('MPICH2'): # can't use toolchain.mpi_family, because of dummy toolchain extramakeopts = 'mpi=mpich2' cmd = "make -f makefile libintel64 %s" % extramakeopts # add other make options as well cmd = ' '.join([cmd, makeopts]) for opt in ['', '-fPIC']: try: tmpbuild = tempfile.mkdtemp() self.log.debug("Created temporary directory %s" % tmpbuild) except: self.log.exception("Creating temporary directory failed") # always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS env.setvar('INSTALL_DIR', tmpbuild) env.setvar('SPEC_OPT', opt) env.setvar('COPTS', opt) env.setvar('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) os.chdir(intdir) self.log.info("Changed to interface %s directory %s" % (i, intdir)) except: self.log.exception("Can't change to interface %s directory %s" % (i, intdir)) if not run_cmd(cmd, log_all=True, simple=True): self.log.error("Building %s (opt: %s) failed" % (i, opt)) for fil in os.listdir(tmpbuild): if opt == '-fPIC': # add _pic to filename ff = fil.split('.') newfil = '.'.join(ff[:-1]) + '_pic.' + ff[-1] else: newfil = fil dest = os.path.join(self.installdir, 'mkl/lib/intel64', newfil) try: src = os.path.join(tmpbuild, fil) if os.path.isfile(src): shutil.move(src, dest) self.log.info("Moved %s to %s" % (src, dest)) except: self.log.exception("Failed to move %s to %s" % (src, dest)) try: shutil.rmtree(tmpbuild) self.log.debug('Removed temporary directory %s' % tmpbuild) except: self.log.exception("Removing temporary directory %s failed" % tmpbuild) else: #Follow this procedure for mkl version lower than 10.3 #Extra #- build the mkl interfaces (pic and no-pic) #- add wrapper libs # Add convenient libs #- form imkl 10.1 if self.cfg['m32']: extra = { 'libmkl.so': 'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } else: extra = { 'libmkl.so': 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' } for fil, txt in extra.items(): if self.cfg['m32']: dest = os.path.join(self.installdir, 'lib/32', fil) else: dest = os.path.join(self.installdir, 'lib/em64t', fil) if not os.path.exists(dest): try: f = open(dest, 'w') f.write(txt) f.close() self.log.info("File %s written" % dest) except: self.log.exception("Can't write file %s" % (dest)) # load the dependencies m = Modules() m.add_module(self.cfg.dependencies()) m.load() if not self.cfg['interfaces']: return # build the interfaces # - blas95 and lapack95 need more work, ignore for now #lis1=['blas95','fftw2xc','fftw2x_cdft','fftw2xf','lapack95'] # blas95 and lapack also need include/.mod to be processed lis1 = ['fftw2xc', 'fftw2x_cdft', 'fftw2xf'] lis2 = ['fftw3xc', 'fftw3xf'] interfacedir = os.path.join(self.installdir, 'interfaces') try: os.chdir(interfacedir) except: self.log.exception("Can't change to interfaces directory %s" % interfacedir) interfacestarget = "libem64t" if self.cfg['m32']: interfacestarget = "lib32" for i in lis1 + lis2: if i in lis1: # use INSTALL_DIR and SPEC_OPT cmd = "make -f makefile %s" % interfacestarget if i in lis2: # use install_to and CFLAGS cmd = "make -f makefile %s install_to=$INSTALL_DIR" % interfacestarget for opt in ['', '-fPIC']: try: tmpbuild = tempfile.mkdtemp() self.log.debug("Created temporary directory %s" % tmpbuild) except: self.log.exception("Creating temporary directory failed") # always set INSTALL_DIR, SPEC_OPT and CFLAGS env.setvar('INSTALL_DIR', tmpbuild) env.setvar('SPEC_OPT', opt) env.setvar('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) os.chdir(intdir) except: self.log.exception("Can't change to interface %s directory %s" % (i, intdir)) if not run_cmd(cmd, log_all=True, simple=True): self.log.error("Building %s (opt: %s) failed" % (i, opt)) for fil in os.listdir(tmpbuild): if opt == '-fPIC': # add _pic to filename ff = fil.split('.') newfil = '.'.join(ff[:-1]) + '_pic.' + ff[-1] else: newfil = fil if self.cfg['m32']: dest = os.path.join(self.installdir, 'lib/32', newfil) else: dest = os.path.join(self.installdir, 'lib/em64t', newfil) try: src = os.path.join(tmpbuild, fil) shutil.move(src, dest) self.log.debug("Moved %s to %s" % (src, dest)) except: self.log.exception("Failed to move %s to %s" % (src, dest)) try: shutil.rmtree(tmpbuild) self.log.debug('Removed temporary directory %s' % tmpbuild) except: self.log.exception("Removing temporary directory %s failed" % (tmpbuild))
def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment onlymod: Boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). """ if not self._toolchain_exists(): self.log.raiseException( "No module found for toolchain name '%s' (%s)" % (self.name, self.version)) if self.name == self.DUMMY_NAME: if self.version == self.DUMMY_VERSION: self.log.info( 'prepare: toolchain dummy mode, dummy version; not loading dependencies' ) else: self.log.info( 'prepare: toolchain dummy mode and loading dependencies') modules = Modules() modules.add_module(self.dependencies) modules.load() return # Load the toolchain and dependencies modules modules = Modules() modules.add_module([(self.name, self.version)]) modules.add_module(self.dependencies) modules.load() # determine direct toolchain dependencies (legacy, not really used anymore) self.toolchain_dependencies = modules.dependencies_for(self.name, self.version, depth=0) self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dependencies) # verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module toolchain_module_deps = set( [mod['name'] for mod in self.toolchain_dependencies]) toolchain_elements_mod_names = set([ y for x in dir(self) if x.endswith('_MODULE_NAME') for y in eval("self.%s" % x) ]) # filter out toolchain name (e.g. 'GCC') from list of toolchain elements toolchain_elements_mod_names = set( [x for x in toolchain_elements_mod_names if not x == self.name]) # filter out optional toolchain elements if they're not used in the module for mod_name in toolchain_elements_mod_names.copy(): if not self.is_required(mod_name): if not mod_name in toolchain_module_deps: self.log.debug( "Removing optional module %s from list of toolchain elements." % mod_name) toolchain_elements_mod_names.remove(mod_name) self.log.debug( "List of toolchain dependency modules from loaded toolchain module: %s" % toolchain_module_deps) self.log.debug( "List of toolchain elements from toolchain definition: %s" % toolchain_elements_mod_names) if toolchain_module_deps == toolchain_elements_mod_names: self.log.info( "List of toolchain dependency modules and toolchain definition match!" ) else: self.log.error("List of toolchain dependency modules and toolchain definition do not match " \ "(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names)) # Generate the variables to be set self.set_variables() # set the variables # onlymod can be comma-separated string of variables not to be set if onlymod == True: self.log.debug( "prepare: do not set additional variables onlymod=%s" % onlymod) self.generate_vars() else: self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod)