def install_step(self): """ Execute the all.bash script to build and install the Go compiler, specifying the final installation prefix by setting $GOROOT_FINAL. """ srcdir = os.path.join(self.cfg['start_dir'], 'src') try: os.chdir(srcdir) except OSError as err: raise EasyBuildError("Failed to move to %s: %s", srcdir, err) # $GOROOT_FINAL only specifies the location of the final installation, which gets baked into the binaries # the installation itself is *not* done by the all.bash script, that needs to be done manually # $GOROOT_BOOTSTRAP needs to specify a Go installation directory to build the go toolchain for versions # 1.5 and later. if LooseVersion(self.version) >= LooseVersion('1.5'): go_root = get_software_root('Go') if go_root: cmd = "GOROOT_BOOTSTRAP=%s GOROOT_FINAL=%s ./all.bash" % (go_root, self.installdir) else: raise EasyBuildError("Go is required as a build dependency for installing Go since version 1.5") else: cmd = "GOROOT_FINAL=%s ./all.bash" % self.installdir run_cmd(cmd, log_all=True, simple=False) try: rmtree2(self.installdir) shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks']) except OSError as err: raise EasyBuildError("Failed to copy installation to %s: %s", self.installdir, err)
def extract_step(self): """ Prepare a combined MXNet source tree. Move all submodules to their right place. """ # Extract everything into separate directories. super(EB_MXNet, self).extract_step() mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*')) if len(mxnet_dirs) == 1: self.mxnet_src_dir = mxnet_dirs[0] self.log.debug("MXNet dir is: %s", self.mxnet_src_dir) else: raise EasyBuildError("Failed to find/isolate MXNet source directory: %s", mxnet_dirs) for srcdir in [d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir)]: submodule, _, _ = srcdir.rpartition('-') newdir = os.path.join(self.mxnet_src_dir, submodule) olddir = os.path.join(self.builddir, srcdir) # first remove empty existing directory rmtree2(newdir) try: shutil.move(olddir, newdir) except IOError as err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err) # the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place. newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core") olddir = os.path.join(self.mxnet_src_dir, "dmlc-core") rmtree2(newdir) symlink(olddir, newdir)
def extract_step(self): """ Prepare a combined MXNet source tree. Move all submodules to their right place. """ # Extract everything into separate directories. super(EB_MXNet, self).extract_step() mxnet_dirs = glob.glob(os.path.join(self.builddir, 'mxnet-*')) if len(mxnet_dirs) == 1: self.mxnet_src_dir = mxnet_dirs[0] self.log.debug("MXNet dir is: %s", self.mxnet_src_dir) else: raise EasyBuildError("Failed to find/isolate MXNet source directory: %s", mxnet_dirs) for srcdir in [d for d in os.listdir(self.builddir) if not d.startswith('mxnet-')]: submodule, _, _ = srcdir.rpartition('-') newdir = os.path.join(self.mxnet_src_dir, submodule) olddir = os.path.join(self.builddir, srcdir) # first remove empty existing directory rmtree2(newdir) try: shutil.move(olddir, newdir) except IOError, err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err)
def test_step(self): """Test the built Python package.""" if isinstance(self.cfg['runtest'], basestring): self.testcmd = self.cfg['runtest'] if self.cfg['runtest'] and not self.testcmd is None: extrapath = "" testinstalldir = None if self.testinstall: # install in test directory and export PYTHONPATH try: testinstalldir = tempfile.mkdtemp() mkdir(os.path.join(testinstalldir, self.pylibdir), parents=True) except OSError, err: self.log.error("Failed to create test install dir: %s" % err) tup = (self.cfg['preinstallopts'], testinstalldir, self.cfg['installopts']) cmd = "%s python setup.py install --prefix=%s %s" % tup run_cmd(cmd, log_all=True, simple=True) run_cmd("python -c 'import sys; print(sys.path)'") # print Python search path (debug) extrapath = "export PYTHONPATH=%s:$PYTHONPATH && " % os.path.join(testinstalldir, self.pylibdir) if self.testcmd: cmd = "%s%s" % (extrapath, self.testcmd) run_cmd(cmd, log_all=True, simple=True) if testinstalldir: try: rmtree2(testinstalldir) except OSError, err: self.log.exception("Removing testinstalldir %s failed: %s" % (testinstalldir, err))
def test_cases_step(self): """Run test cases, if specified.""" for test in self.cfg['tests']: # check expected format if not len(test) == 4: raise EasyBuildError("WIEN2k test case not specified in expected format: " "(testcase_name, init_lapw_args, run_lapw_args, [scf_regexp_pattern])") test_name = test[0] init_args = test[1] run_args = test[2] scf_regexp_patterns = test[3] try: cwd = os.getcwd() # WIEN2k enforces that working dir has same name as test case tmpdir = os.path.join(tempfile.mkdtemp(), test_name) scratch = os.path.join(tmpdir, 'scratch') mkdir(scratch, parents=True) env.setvar('SCRATCH', scratch) os.chdir(tmpdir) self.log.info("Running test case %s in %s" % (test_name, tmpdir)) except OSError as err: raise EasyBuildError("Failed to create temporary directory for test %s: %s", test_name, err) # try and find struct file for test test_fp = self.obtain_file("%s.struct" % test_name) try: shutil.copy2(test_fp, tmpdir) except OSError as err: raise EasyBuildError("Failed to copy %s: %s", test_fp, err) # run test cmd = "init_lapw %s" % init_args run_cmd(cmd, log_all=True, simple=True) cmd = "run_lapw %s" % run_args run_cmd(cmd, log_all=True, simple=True) # check output scf_fn = "%s.scf" % test_name self.log.debug("Checking output of test %s in %s" % (str(test), scf_fn)) scftxt = read_file(scf_fn) for regexp_pat in scf_regexp_patterns: regexp = re.compile(regexp_pat, re.M) if not regexp.search(scftxt): raise EasyBuildError("Failed to find pattern %s in %s", regexp.pattern, scf_fn) else: self.log.debug("Found pattern %s in %s" % (regexp.pattern, scf_fn)) # cleanup try: os.chdir(cwd) rmtree2(tmpdir) except OSError as err: raise EasyBuildError("Failed to clean up temporary test dir: %s", err)
def extract_step(self): """ Prepare a combined MXNet source tree. Move all submodules to their right place. """ # Extract everything into separate directories. super(EB_MXNet, self).extract_step() mxnet_dirs = glob.glob(os.path.join(self.builddir, 'mxnet-*')) if len(mxnet_dirs) == 1: self.mxnet_src_dir = mxnet_dirs[0] self.log.debug("MXNet dir is: %s", self.mxnet_src_dir) else: raise EasyBuildError( "Failed to find/isolate MXNet source directory: %s", mxnet_dirs) for srcdir in [ d for d in os.listdir(self.builddir) if not d.startswith('mxnet-') ]: submodule, _, _ = srcdir.rpartition('-') newdir = os.path.join(self.mxnet_src_dir, submodule) olddir = os.path.join(self.builddir, srcdir) # first remove empty existing directory rmtree2(newdir) try: shutil.move(olddir, newdir) except IOError, err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err)
def install_step(self): """Install IMOD using install script.""" # -dir: Choose location of installation directory # -skip: do not attempt to deploy resource files in /etc # -yes: do not prompt for confirmation script = '{0}_{1}{2}.csh'.format(self.name.lower(), self.version, self.cfg['versionsuffix']) cmd = "tcsh {0} -dir {1} -script {1} -skip -yes".format( script, self.installdir) run_cmd(cmd, log_all=True, simple=True) # The assumption by the install script is that installdir will be something # like /usr/local. So it creates, within the specified install location, a # number of additional directories within which to install IMOD. We will, # therefore, move the contents of these directories up and throw away the # directories themselves. Doing so apparently is not a problem so long as # IMOD_DIR is correctly set in the module. link_to_remove = os.path.join(self.installdir, self.name) dir_to_remove = os.path.join( self.installdir, "{0}_{1}".format(self.name.lower(), self.version)) try: for entry in os.listdir(dir_to_remove): shutil.move(os.path.join(dir_to_remove, entry), self.installdir) if os.path.realpath(link_to_remove) != os.path.realpath( dir_to_remove): raise EasyBuildError( "Something went wrong: %s doesn't point to %s", link_to_remove, dir_to_remove) rmtree2(dir_to_remove) os.remove(link_to_remove) except OSError as err: raise EasyBuildError("Failed to clean up install dir: %s", err)
def install_step(self): """Install IMOD using install script.""" # -dir: Choose location of installation directory # -skip: do not attempt to deploy resource files in /etc # -yes: do not prompt for confirmation script = '{0}_{1}{2}.csh'.format(self.name.lower(), self.version, self.cfg['versionsuffix']) cmd = "tcsh {0} -dir {1} -script {1} -skip -yes".format(script, self.installdir) run_cmd(cmd, log_all=True, simple=True) # The assumption by the install script is that installdir will be something # like /usr/local. So it creates, within the specified install location, a # number of additional directories within which to install IMOD. We will, # therefore, move the contents of these directories up and throw away the # directories themselves. Doing so apparently is not a problem so long as # IMOD_DIR is correctly set in the module. link_to_remove = os.path.join(self.installdir, self.name) dir_to_remove = os.path.join(self.installdir, "{0}_{1}".format(self.name.lower(), self.version)) try: for entry in os.listdir(dir_to_remove): shutil.move(os.path.join(dir_to_remove, entry), self.installdir) if os.path.realpath(link_to_remove) != os.path.realpath(dir_to_remove): raise EasyBuildError("Something went wrong: %s doesn't point to %s", link_to_remove, dir_to_remove) rmtree2(dir_to_remove) os.remove(link_to_remove) except OSError as err: raise EasyBuildError("Failed to clean up install dir: %s", err)
def test_step(self): """Test the compilation - default: None """ extrapath = "" testinstalldir = os.path.join(self.builddir, "mytemporarytestinstalldir") if self.testinstall: # Install in test directory and export PYTHONPATH try: os.makedirs(testinstalldir) except OSError: self.log.exception("Creating testinstalldir %s failed" % testinstalldir) ppath = "%s/reallib" % testinstalldir cmd = "python setup.py install --install-scripts=%s --install-purelib=%s %s" % \ (testinstalldir, ppath, self.installopts) run_cmd(cmd, log_all=True, simple=True) if os.environ.has_key('PYTHONPATH'): extrapath = "export PYTHONPATH=%s:%s && " % (ppath, os.environ['PYTHONPATH']) else: extrapath = "export PYTHONPATH=%s && " % ppath if self.runtest: cmd = "%s%s" % (extrapath, self.runtest) run_cmd(cmd, log_all=True, simple=True) if self.testinstall: try: rmtree2(testinstalldir) except OSError, err: self.log.exception("Removing testinstalldir %s failed: %s" % (testinstalldir, err))
def install_step(self): """Install python but only keep the bits we need""" super(EB_Tkinter, self).install_step() tmpdir = tempfile.mkdtemp(dir=self.builddir) pylibdir = os.path.join(self.installdir, os.path.dirname(det_pylibdir())) shlib_ext = get_shared_lib_ext() tkinter_so = os.path.join(pylibdir, 'lib-dynload', '_tkinter*.' + shlib_ext) tkinter_so_hits = glob.glob(tkinter_so) if len(tkinter_so_hits) != 1: raise EasyBuildError("Expected to find exactly one _tkinter*.so: %s", tkinter_so_hits) self.tkinter_so_basename = os.path.basename(tkinter_so_hits[0]) if LooseVersion(self.version) >= LooseVersion('3'): tkparts = ["tkinter", os.path.join("lib-dynload", self.tkinter_so_basename)] else: tkparts = ["lib-tk", os.path.join("lib-dynload", self.tkinter_so_basename)] copy([os.path.join(pylibdir, x) for x in tkparts], tmpdir) rmtree2(self.installdir) mkdir(pylibdir, parents=True) try: shutil.move(os.path.join(tmpdir, tkparts[0]), pylibdir) shutil.move(os.path.join(tmpdir, os.path.basename(tkparts[1])), pylibdir) except (IOError, OSError) as err: raise EasyBuildError("Failed to move Tkinter back to the install directory: %s", err)
def cleanup(self): """ Clean up SVN working copy. """ try: rmtree2(self.wc) except OSError, err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def cleanup(self): """ Clean up SVN working copy. """ try: rmtree2(self.wc) except OSError, err: self.log.exception("Can't remove working copy %s: %s" % (self.wc, err))
def cleanup(self): """ Clean up mercurial working copy. """ try: rmtree2(self.wc) except IOError, err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def cleanup(self): """ Clean up SVN working copy. """ try: rmtree2(self.wc) except OSError as err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def clean_homedir(self): """Remove 'intel' directory from home directory, where stuff is cached.""" intelhome = os.path.join(os.getenv('HOME'), self.home_subdir) if os.path.exists(intelhome): try: rmtree2(intelhome) self.log.info("Cleaning up intel dir %s" % intelhome) except OSError, err: self.log.warning("Cleaning up intel dir %s failed: %s" % (intelhome, err))
def obtain_repo(self, url, force_download=False): """ Downloads the repo from the given url, then updates submodules :param url: url of git repository """ srcpaths = source_paths() if re.match(r"^(https?)://", url): # URL detected, try and download it reponame = url.split('/')[-1].split('.')[0] # Try and find the repo in the sources directory first foundfile = None failedpaths = [] for path in srcpaths: # create list of candidate filepaths namepath = os.path.join(path, self.name) letterpath = os.path.join(path, letter_dir_for(self.name), self.name) candidate_filepaths = [letterpath, namepath, path] # look for repo in those locations for cfp in candidate_filepaths: fullpath = os.path.join(cfp, reponame) if os.path.isdir(fullpath): self.log.info("Found repo %s at %s", reponame, fullpath) foundfile = os.path.abspath(fullpath) if force_download: print_warning( "Found file %s at %s, but re-downloading it anyway..." % (reponame, foundfile)) foundfile = None break else: failedpaths.append(fullpath) if foundfile: if self.dry_run: self.dry_run_msg(" * %s found at %s", filename, foundfile) return foundfile else: # Figure out where to clone the repo to repopath = os.path.join(srcpaths[0], letter_dir_for(self.name), self.name) self.log.info("Creating path %s to clone repo to" % repopath) mkdir(repopath, parents=True) # Remove repo, if it has previously been downloaded (we must be forcing download) fullpath = os.path.join(repopath, reponame) if os.path.exists(fullpath): rmtree2(fullpath) try: if self.download_repo(reponame, url, repopath): return fullpath except IOError, err: raise EasyBuildError( "Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
def cleanup(self): """ Clean up git working copy. """ try: self.wc = os.path.dirname(self.wc) rmtree2(self.wc) except IOError, err: self.log.exception("Can't remove working copy %s: %s" % (self.wc, err))
def cleanup(self): """ Clean up git working copy. """ try: self.wc = os.path.dirname(self.wc) rmtree2(self.wc) except IOError, err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def install_step(self): """Install numpy and remove numpy build dir, so scipy doesn't find it by accident.""" super(EB_numpy, self).install_step() builddir = os.path.join(self.builddir, "numpy") if os.path.isdir(builddir): rmtree2(builddir) else: self.log.debug("build dir %s already clean" % builddir)
def cleanup(self): """ Clean up mercurial working copy. """ try: rmtree2(self.wc) except IOError as err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err)
def install_step(self): """Copy all files in build directory to the install directory""" if self.cfg['install_cmd'] is None: try: # shutil.copytree doesn't allow the target directory to exist already rmtree2(self.installdir) shutil.copytree(self.cfg['start_dir'], self.installdir) except OSError, err: self.log.error("Failed to copy %s to %s: %s" % (self.cfg['start_dir'], self.installdir, err))
def install_step(self): """Copy all files in build directory to the install directory""" if self.cfg['install_cmd'] is None: try: # shutil.copytree doesn't allow the target directory to exist already rmtree2(self.installdir) shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks']) except OSError, err: raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err)
def install_step(self): """Copy all files in build directory to the install directory""" rmtree2(self.installdir) install_script = self.src[0]['name'] adjust_permissions(os.path.join(self.builddir, install_script), stat.S_IRUSR|stat.S_IXUSR) cmd = "%s ./%s -p %s -b -f" % (self.cfg['preinstallopts'], install_script, self.installdir) self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) run_cmd(cmd, log_all=True, simple=True)
def install_step(self): src = self.cfg['start_dir'] # shutil.copytree cannot handle destination dirs that exist already. # On the other hand, Python2.4 cannot create entire paths during copytree. # Therefore, only the final directory is deleted. rmtree2(self.installdir) try: # self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file shutil.copytree(src,self.installdir, symlinks=self.cfg['keepsymlinks']) except: self.log.exception("Copying %s to installation dir %s failed" % (src,self.installdir))
def test_step(self): """Test the built Python package.""" if isinstance(self.cfg['runtest'], basestring): self.testcmd = self.cfg['runtest'] if self.cfg['runtest'] and self.testcmd is not None: extrapath = "" testinstalldir = None if self.testinstall: # install in test directory and export PYTHONPATH try: testinstalldir = tempfile.mkdtemp() for pylibdir in self.all_pylibdirs: mkdir(os.path.join(testinstalldir, pylibdir), parents=True) except OSError, err: raise EasyBuildError( "Failed to create test install dir: %s", err) # print Python search path (just debugging purposes) run_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, verbose=False, trace=False) abs_pylibdirs = [ os.path.join(testinstalldir, pylibdir) for pylibdir in self.all_pylibdirs ] extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join( abs_pylibdirs + ['$PYTHONPATH']) cmd = self.compose_install_command(testinstalldir, extrapath=extrapath) run_cmd(cmd, log_all=True, simple=True, verbose=False) if self.testcmd: cmd = "%s%s" % (extrapath, self.testcmd % { 'python': self.python_cmd }) run_cmd(cmd, log_all=True, simple=True) if testinstalldir: try: rmtree2(testinstalldir) except OSError, err: raise EasyBuildError( "Removing testinstalldir %s failed: %s", testinstalldir, err)
def install_step(self): """Copy all files in build directory to the install directory""" rmtree2(self.installdir) install_script = self.src[0]['name'] adjust_permissions(os.path.join(self.builddir, install_script), stat.S_IRUSR | stat.S_IXUSR) cmd = "%s ./%s -p %s -b -f" % (self.cfg['preinstallopts'], install_script, self.installdir) self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) run_cmd(cmd, log_all=True, simple=True)
def post_install_step(self): """Copy installation to actual installation directory in case of a staged installation.""" if self.cfg['staged_install']: staged_installdir = self.installdir self.installdir = self.actual_installdir try: # copytree expects target directory to not exist yet if os.path.exists(self.installdir): rmtree2(self.installdir) shutil.copytree(staged_installdir, self.installdir) except OSError, err: raise EasyBuildError("Failed to move staged install from %s to %s: %s", staged_installdir, self.installdir, err)
def post_install_step(self): """Copy installation to actual installation directory in case of a staged installation.""" if self.cfg.get('staged_install', False): staged_installdir = self.installdir self.installdir = self.actual_installdir try: # copytree expects target directory to not exist yet if os.path.exists(self.installdir): rmtree2(self.installdir) shutil.copytree(staged_installdir, self.installdir) except OSError, err: raise EasyBuildError("Failed to move staged install from %s to %s: %s", staged_installdir, self.installdir, err)
def install_step(self, src=None): """Install by copying from specified source directory (or 'start_dir' if not specified).""" if src is None: src = self.cfg['start_dir'] # shutil.copytree cannot handle destination dirs that exist already. # On the other hand, Python2.4 cannot create entire paths during copytree. # Therefore, only the final directory is deleted. rmtree2(self.installdir) try: # self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file shutil.copytree(src, self.installdir, symlinks=self.cfg['keepsymlinks']) except OSError, err: self.log.error("Copying %s to installation dir %s failed: %s" % (src, self.installdir, err))
def install_step(self): """Copy all files in build directory to the install directory""" install_cmd = self.cfg.get('install_cmd', None) if install_cmd is None: try: # shutil.copytree doesn't allow the target directory to exist already rmtree2(self.installdir) shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks']) except OSError as err: raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err) else: cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']]) self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) run_cmd(cmd, log_all=True, simple=True)
def install_step(self): """Install numpy and remove numpy build dir, so scipy doesn't find it by accident.""" super(EB_numpy, self).install_step() builddir = os.path.join(self.builddir, "numpy") try: if os.path.isdir(builddir): os.chdir(self.builddir) rmtree2(builddir) else: self.log.debug("build dir %s already clean" % builddir) except OSError as err: raise EasyBuildError("Failed to clean up numpy build dir %s: %s", builddir, err)
def build_image(self, dockerfile): ec = self.easyconfigs[-1]['ec'] module_name = self.mns.det_full_module_name(ec) tempdir = tempfile.mkdtemp(prefix='easybuild-docker') container_name = self.img_name or "%s:latest" % module_name.replace('/', '-') docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.']) print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd) run_cmd(docker_cmd, path=tempdir, stream_output=True) print_msg("Docker image created at %s" % container_name, log=self.log) rmtree2(tempdir)
def install_step(self, src=None): """Install by copying from specified source directory (or 'start_dir' if not specified).""" if src is None: src = self.cfg["start_dir"] # shutil.copytree cannot handle destination dirs that exist already. # On the other hand, Python2.4 cannot create entire paths during copytree. # Therefore, only the final directory is deleted. rmtree2(self.installdir) try: # self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file shutil.copytree(src, self.installdir, symlinks=self.cfg["keepsymlinks"]) except OSError, err: raise EasyBuildError("Copying %s to installation dir %s failed: %s", src, self.installdir, err)
def install_step(self): """Copy all files in build directory to the install directory""" pre_install_step(self.log, self.cfg['pre_install_cmd']) rmtree2(self.installdir) sources = self.cfg['sources'] install_script = sources[0] cmd = "chmod 777 {} && ./{} -p {} -b -f".format(install_script, install_script, self.installdir ) self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) run_cmd(cmd, log_all=True, simple=True) post_install_step(self.log, self.installdir, self.cfg['post_install_cmd'])
def install_step(self): src = self.cfg['start_dir'] # shutil.copytree cannot handle destination dirs that exist already. # On the other hand, Python2.4 cannot create entire paths during copytree. # Therefore, only the final directory is deleted. rmtree2(self.installdir) try: # self.cfg['keepsymlinks'] is False by default except when explicitly put to True in .eb file shutil.copytree(src, self.installdir, symlinks=self.cfg['keepsymlinks']) except: self.log.exception("Copying %s to installation dir %s failed" % (src, self.installdir))
class EB_Go(ConfigureMake): """ Build Go compiler """ def configure_step(self): """No dedicated configure step.""" pass def build_step(self): """No dedicated build step, building & installing is done in one go.""" pass def install_step(self): """ Execute the all.bash script to build and install the Go compiler, specifying the final installation prefix by setting $GOROOT_FINAL. """ srcdir = os.path.join(self.cfg['start_dir'], 'src') try: os.chdir(srcdir) except OSError, err: raise EasyBuildError("Failed to move to %s: %s", srcdir, err) # $GOROOT_FINAL only specifies the location of the final installation, which gets baked into the binaries # the installation itself is *not* done by the all.bash script, that needs to be done manually cmd = "GOROOT_FINAL=%s ./all.bash" % self.installdir run_cmd(cmd, log_all=True, simple=False) try: rmtree2(self.installdir) shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks']) except OSError, err: raise EasyBuildError("Failed to copy installation to %s: %s", self.installdir, err)
def install_step(self): """ Install by unpacking tarball in dist directory, and copying site-packages dir to installdir. """ # locate tarball tarball = None shortver = '.'.join(self.version.split('.')[0:2]) fn_pattern = os.path.join(self.cfg['start_dir'], 'dist', "%s-%s.*.tar.gz" % (self.name, shortver)) matches = glob.glob(fn_pattern) if not matches: raise EasyBuildError("No tarball found at %s", fn_pattern) elif len(matches) > 1: raise EasyBuildError("Multiple matches found for tarball: %s", matches) else: tarball = matches[0] self.log.info("Tarball found at %s" % tarball) # unpack tarball to temporary directory tmpdir = tempfile.mkdtemp() srcdir = extract_file(tarball, tmpdir) if not srcdir: raise EasyBuildError("Unpacking tarball %s failed?", tarball) # locate site-packages dir to copy by diving into unpacked tarball src = srcdir while len(os.listdir(src)) == 1: src = os.path.join(src, os.listdir(src)[0]) if not os.path.basename(src) == 'site-packages': raise EasyBuildError( "Expected to find a site-packages path, but found something else: %s", src) # copy contents of site-packages dir dest = os.path.join(self.installdir, 'site-packages') try: shutil.copytree(src, dest) rmtree2(tmpdir) os.chdir(self.installdir) except OSError, err: raise EasyBuildError("Failed to copy directory %s to %s: %s", src, dest, err)
def install_step(self): """Custom install step for netpbm.""" # Preinstallation to a tmp directory. Can't install directly to installdir because the make command fails if the # directory already exists cmd = "make package pkgdir=%s/pkg" % self.builddir (out, _) = run_cmd(cmd, log_all=True, simple=False) # Move things to installdir copy(["%s/pkg/%s" % (self.builddir, x) for x in os.listdir("%s/pkg/" % self.builddir)], self.installdir) # Need to do manually the last bits of the installation configs = [ ("%s/config_template" % self.installdir, "%s/bin/netpbm-config" % self.installdir), ("%s/pkgconfig_template" % self.installdir, "%s/lib/pkgconfig/netpbm.pc" % self.installdir) ] mkdir("%s/lib/pkgconfig" % self.installdir) for template, config_file in configs: move_file(template, config_file) for line in fileinput.input(config_file, inplace=1, backup='.orig'): if re.match(r"^@", line): continue else: line = re.sub(r'@VERSION@', 'Netpbm %s' % self.version, line) line = re.sub(r'@DATADIR@', '%s/lib' % self.installdir, line) line = re.sub(r'@LINKDIR@', '%s/lib' % self.installdir, line) line = re.sub(r'@INCLUDEDIR@', '%s/include' % self.installdir, line) line = re.sub(r'@BINDIR@', '%s/bin' % self.installdir, line) sys.stdout.write(line) adjust_permissions("%s/bin/netpbm-config" % self.installdir, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) move_file("%s/link/libnetpbm.a" % self.installdir, "%s/lib/libnetpbm.a" % self.installdir) symlink("%s/lib/libnetpbm.so.11" % self.installdir, "%s/lib/libnetpbm.so" % self.installdir) for f in os.listdir("%s/misc/" % self.installdir): move_file("%s/misc/%s" % (self.installdir, f), "%s/lib/%s" % (self.installdir, f)) rmtree2("%s/misc/" % self.installdir) rmtree2("%s/link/" % self.installdir) headers = os.listdir("%s/include/netpbm" % self.installdir) for header in headers: symlink("%s/include/netpbm/%s" % (self.installdir, header), "%s/include/%s" % (self.installdir, header)) return out
def install_step(self): """ Install by unpacking tarball in dist directory, and copying site-packages dir to installdir. """ # locate tarball tarball = None shortver = '.'.join(self.version.split('.')[0:2]) fn_pattern = os.path.join(self.cfg['start_dir'], 'dist', "%s-%s.*.tar.gz" % (self.name, shortver)) matches = glob.glob(fn_pattern) if not matches: raise EasyBuildError("No tarball found at %s", fn_pattern) elif len(matches) > 1: raise EasyBuildError("Multiple matches found for tarball: %s", matches) else: tarball = matches[0] self.log.info("Tarball found at %s" % tarball) # unpack tarball to temporary directory tmpdir = tempfile.mkdtemp() srcdir = extract_file(tarball, tmpdir) if not srcdir: raise EasyBuildError("Unpacking tarball %s failed?", tarball) # locate site-packages dir to copy by diving into unpacked tarball src = srcdir while len(os.listdir(src)) == 1: src = os.path.join(src, os.listdir(src)[0]) if not os.path.basename(src) =='site-packages': raise EasyBuildError("Expected to find a site-packages path, but found something else: %s", src) # copy contents of site-packages dir dest = os.path.join(self.installdir, 'site-packages') try: shutil.copytree(src, dest) rmtree2(tmpdir) os.chdir(self.installdir) except OSError, err: raise EasyBuildError("Failed to copy directory %s to %s: %s", src, dest, err)
def build_image(self, dockerfile): ec = self.easyconfigs[-1]['ec'] module_name = self.mns.det_full_module_name(ec) tempdir = tempfile.mkdtemp(prefix='easybuild-docker') container_name = self.img_name or "%s:latest" % module_name.replace( '/', '-') docker_cmd = ' '.join([ 'sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.' ]) print_msg( "Running '%s', you may need to enter your 'sudo' password..." % docker_cmd) run_cmd(docker_cmd, path=tempdir, stream_output=True) print_msg("Docker image created at %s" % container_name, log=self.log) rmtree2(tempdir)
def install_step(self): """Install python but only keep the bits we need""" super(EB_Tkinter, self).install_step() tmpdir = tempfile.mkdtemp(dir=self.builddir) pylibdir = os.path.join(self.installdir, os.path.dirname(det_pylibdir())) shlib_ext = get_shared_lib_ext() tkinter_so = os.path.join(pylibdir, 'lib-dynload', '_tkinter*.' + shlib_ext) tkinter_so_hits = glob.glob(tkinter_so) if len(tkinter_so_hits) != 1: raise EasyBuildError( "Expected to find exactly one _tkinter*.so: %s", tkinter_so_hits) self.tkinter_so_basename = os.path.basename(tkinter_so_hits[0]) if LooseVersion(self.version) >= LooseVersion('3'): tkparts = [ "tkinter", os.path.join("lib-dynload", self.tkinter_so_basename) ] else: tkparts = [ "lib-tk", os.path.join("lib-dynload", self.tkinter_so_basename) ] copy([os.path.join(pylibdir, x) for x in tkparts], tmpdir) rmtree2(self.installdir) mkdir(pylibdir, parents=True) try: shutil.move(os.path.join(tmpdir, tkparts[0]), pylibdir) shutil.move(os.path.join(tmpdir, os.path.basename(tkparts[1])), pylibdir) except (IOError, OSError) as err: raise EasyBuildError( "Failed to move Tkinter back to the install directory: %s", err)
def install_step(self): """Copy all files in build directory to the install directory""" if self.cfg['install_cmd'] is None: try: cmd = self.cfg['preinstallopts'] if cmd: self.log.info( "Running preinstallopts of %s using command '%s'..." % (self.name, cmd)) run_cmd(cmd, log_all=True, simple=True) self.log.info("Copying %s into '%s'..." % (self.cfg['start_dir'], self.installdir)) # shutil.copytree doesn't allow the target directory to exist already rmtree2(self.installdir) shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks']) except OSError, err: raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err)
class EB_MXNet(MakeCp): """Easyblock to build and install MXNet""" @staticmethod def extra_options(extra_vars=None): """Change default values of options""" extra = MakeCp.extra_options() # files_to_copy is not mandatory here extra['files_to_copy'][2] = CUSTOM return extra def __init__(self, *args, **kwargs): """Initialize custom class variables.""" super(EB_MXNet, self).__init__(*args, **kwargs) self.mxnet_src_dir = None self.py_ext = PythonPackage(self, {'name': self.name, 'version': self.version}) self.py_ext.module_generator = self.module_generator self.r_ext = RPackage(self, {'name': self.name, 'version': self.version}) self.r_ext.module_generator = self.module_generator def extract_step(self): """ Prepare a combined MXNet source tree. Move all submodules to their right place. """ # Extract everything into separate directories. super(EB_MXNet, self).extract_step() mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*')) if len(mxnet_dirs) == 1: self.mxnet_src_dir = mxnet_dirs[0] self.log.debug("MXNet dir is: %s", self.mxnet_src_dir) else: raise EasyBuildError("Failed to find/isolate MXNet source directory: %s", mxnet_dirs) for srcdir in [d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir)]: submodule, _, _ = srcdir.rpartition('-') newdir = os.path.join(self.mxnet_src_dir, submodule) olddir = os.path.join(self.builddir, srcdir) # first remove empty existing directory rmtree2(newdir) try: shutil.move(olddir, newdir) except IOError, err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err) # the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place. newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core") olddir = os.path.join(self.mxnet_src_dir, "dmlc-core") rmtree2(newdir) symlink(olddir, newdir)
def install_step(self): """Copy all files in build directory to the install directory""" install_cmd = self.cfg.get('install_cmd', None) if install_cmd is None: try: # shutil.copytree doesn't allow the target directory to exist already rmtree2(self.installdir) shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks']) except OSError as err: raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err) else: cmd = ' '.join([ self.cfg['preinstallopts'], install_cmd, self.cfg['installopts'] ]) self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) run_cmd(cmd, log_all=True, simple=True)
def test_step(self): """Test the built Python package.""" if isinstance(self.cfg['runtest'], basestring): self.testcmd = self.cfg['runtest'] if self.cfg['runtest'] and self.testcmd is not None: extrapath = "" testinstalldir = None if self.testinstall: # install in test directory and export PYTHONPATH try: testinstalldir = tempfile.mkdtemp() for pylibdir in self.all_pylibdirs: mkdir(os.path.join(testinstalldir, pylibdir), parents=True) except OSError, err: raise EasyBuildError("Failed to create test install dir: %s", err) # print Python search path (just debugging purposes) run_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, verbose=False) abs_pylibdirs = [os.path.join(testinstalldir, pylibdir) for pylibdir in self.all_pylibdirs] extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join(abs_pylibdirs + ['$PYTHONPATH']) cmd = self.compose_install_command(testinstalldir, extrapath=extrapath) run_cmd(cmd, log_all=True, simple=True, verbose=False) if self.testcmd: cmd = "%s%s" % (extrapath, self.testcmd % {'python': self.python_cmd}) run_cmd(cmd, log_all=True, simple=True) if testinstalldir: try: rmtree2(testinstalldir) except OSError, err: raise EasyBuildError("Removing testinstalldir %s failed: %s", testinstalldir, err)
def install_step(self): """Install python but only keep the bits we need""" super(EB_Tkinter, self).install_step() tmpdir = tempfile.mkdtemp(dir=self.builddir) pylibdir = os.path.join(self.installdir, os.path.dirname(det_pylibdir())) tkparts = ["lib-tk", "lib-dynload/_tkinter.so"] copy([os.path.join(pylibdir, x) for x in tkparts], tmpdir) rmtree2(self.installdir) mkdir(pylibdir, parents=True) try: shutil.move(os.path.join(tmpdir, tkparts[0]), pylibdir) shutil.move(os.path.join(tmpdir, os.path.basename(tkparts[1])), pylibdir) except (IOError, OSError) as err: raise EasyBuildError( "Failed to move Tkinter back to the install directory: %s", err)
class EB_numpy(FortranPythonPackage): """Support for installing the numpy Python package as part of a Python installation.""" def __init__(self, *args, **kwargs): """Initialize numpy-specific class variables.""" super(EB_numpy, self).__init__(*args, **kwargs) self.sitecfg = None self.sitecfgfn = 'site.cfg' self.installopts = '' self.testinstall = True self.testcmd = "cd .. && python -c 'import numpy; numpy.test(verbose=2)'" def configure_step(self): """Configure numpy build by composing site.cfg contents.""" # see e.g. https://github.com/numpy/numpy/pull/2809/files self.sitecfg = '\n'.join([ "[DEFAULT]", "library_dirs = %(libs)s", "include_dirs= %(includes)s", "search_static_first=True", ]) if get_software_root("imkl"): extrasiteconfig = '\n'.join([ "[mkl]", "lapack_libs = %(lapack)s", "mkl_libs = %(blas)s", ]) else: # [atlas] the only real alternative, even for non-ATLAS BLAS libs (e.g., OpenBLAS, ACML, ...) # using only the [blas] and [lapack] sections results in sub-optimal builds that don't provide _dotblas.so; # it does require a CBLAS interface to be available for the BLAS library being used # e.g. for ACML, the CBLAS module providing a C interface needs to be used extrasiteconfig = '\n'.join([ "[atlas]", "atlas_libs = %(lapack)s", "[lapack]", "lapack_libs = %(lapack)s", # required by scipy, that uses numpy's site.cfg ]) blas = None lapack = None fft = None if get_software_root("imkl"): # with IMKL, no spaces and use '-Wl:' # redefine 'Wl,' to 'Wl:' so that the patch file can do its job def get_libs_for_mkl(varname): """Get list of libraries as required for MKL patch file.""" libs = self.toolchain.variables['LIB%s' % varname].copy() libs.try_remove(['pthread']) tweaks = { 'prefix': '', 'prefix_begin_end': '-Wl:', 'separator': ',', 'separator_begin_end': ',', } libs.try_function_on_element('change', kwargs=tweaks) libs.SEPARATOR = ',' return str( libs ) # str causes list concatenation and adding prefixes & separators blas = get_libs_for_mkl('BLAS_MT') lapack = get_libs_for_mkl('LAPACK_MT') fft = get_libs_for_mkl('FFT') # make sure the patch file is there # we check for a typical characteristic of a patch file that cooperates with the above # not fool-proof, but better than enforcing a particular patch filename patch_found = False patch_wl_regex = re.compile(r"replace\(':',\s*','\)") for patch in self.patches: # patches are either strings (extension) or dicts (easyblock) if isinstance(patch, dict): patch = patch['path'] if patch_wl_regex.search(open(patch, 'r').read()): patch_found = True break if not patch_found: self.log.error( "Building numpy on top of Intel MKL requires a patch to " "handle -Wl linker flags correctly, which doesn't seem to be there." ) else: # unless Intel MKL is used, $ATLAS should be set to take full control, # and to make sure a fully optimized version is built, including _dotblas.so # which is critical for decent performance of the numpy.dot (matrix dot product) function! env.setvar('ATLAS', '1') lapack = ', '.join([ x for x in self.toolchain.get_variable('LIBLAPACK_MT', typ=list) if x != "pthread" ]) fft = ', '.join(self.toolchain.get_variable('LIBFFT', typ=list)) libs = ':'.join(self.toolchain.get_variable('LDFLAGS', typ=list)) includes = ':'.join(self.toolchain.get_variable('CPPFLAGS', typ=list)) # CBLAS is required for ACML, because it doesn't offer a C interface to BLAS if get_software_root('ACML'): cblasroot = get_software_root('CBLAS') if cblasroot: lapack = ', '.join([lapack, "cblas"]) cblaslib = os.path.join(cblasroot, 'lib') # with numpy as extension, CBLAS might not be included in LDFLAGS because it's not part of a toolchain if not cblaslib in libs: libs = ':'.join([libs, cblaslib]) else: self.log.error( "CBLAS is required next to ACML to provide a C interface to BLAS, but it's not loaded." ) if fft: extrasiteconfig += "\n[fftw]\nlibraries = %s" % fft self.sitecfg = '\n'.join([self.sitecfg, extrasiteconfig]) self.sitecfg = self.sitecfg % { 'blas': blas, 'lapack': lapack, 'libs': libs, 'includes': includes, } super(EB_numpy, self).configure_step() # check configuration (for debugging purposes) cmd = "python setup.py config" run_cmd(cmd, log_all=True, simple=True) def test_step(self): """Run available numpy unit tests, and more.""" super(EB_numpy, self).test_step() # temporarily install numpy, it doesn't alow to be used straight from the source dir tmpdir = tempfile.mkdtemp() cmd = "python setup.py install --prefix=%s %s" % (tmpdir, self.installopts) run_cmd(cmd, log_all=True, simple=True) try: pwd = os.getcwd() os.chdir(tmpdir) except OSError, err: self.log.error("Faild to change to %s: %s" % (tmpdir, err)) # evaluate performance of numpy.dot (3 runs, 3 loops each) size = 1000 max_time_msec = 1000 # 1 second should really do it for a 1000x1000 matrix dot product cmd = ' '.join([ 'export PYTHONPATH=%s:$PYTHONPATH &&' % os.path.join(tmpdir, self.pylibdir), 'python -m timeit -n 3 -r 3', '-s "import numpy; x = numpy.random.random((%(size)d, %(size)d))"' % { 'size': size }, '"numpy.dot(x, x.T)"', ]) (out, ec) = run_cmd(cmd, simple=False) self.log.debug("Test output: %s" % out) # fetch result time_msec = None msec_re = re.compile( "\d+ loops, best of \d+: (?P<time>[0-9.]+) msec per loop") res = msec_re.search(out) if res: time_msec = float(res.group('time')) else: sec_re = re.compile( "\d+ loops, best of \d+: (?P<time>[0-9.]+) sec per loop") res = sec_re.search(out) if res: time_msec = 1000 * float(res.group('time')) else: self.log.error( "Failed to determine time for numpy.dot test run.") # make sure we observe decent performance if time_msec < max_time_msec: self.log.info( "Time for %(size)dx%(size)d matrix dot product: %(time)d msec < %(maxtime)d msec => OK" % { 'size': size, 'time': time_msec, 'maxtime': max_time_msec }) else: self.log.error( "Time for %(size)dx%(size)d matrix dot product: %(time)d msec >= %(maxtime)d msec => ERROR" % { 'size': size, 'time': time_msec, 'maxtime': max_time_msec }) try: os.chdir(pwd) rmtree2(tmpdir) except OSError, err: self.log.error("Failed to change back to %s: %s" % (pwd, err))
def test_step(self): """Run WPS test (requires large dataset to be downloaded). """ wpsdir = None def run_wps_cmd(cmdname, mpi_cmd=True): """Run a WPS command, and check for success.""" cmd = os.path.join(wpsdir, "%s.exe" % cmdname) if mpi_cmd: cmd = self.toolchain.mpi_cmd_for(cmd, 1) (out, _) = run_cmd(cmd, log_all=True, simple=False) re_success = re.compile("Successful completion of %s" % cmdname) if not re_success.search(out): self.log.error("%s.exe failed (pattern '%s' not found)?" % (cmdname, re_success.pattern)) if self.cfg['runtest']: if not self.cfg['testdata']: self.log.error("List of URLs for testdata not provided.") wpsdir = os.path.join(self.builddir, "WPS") try: # create temporary directory tmpdir = tempfile.mkdtemp() os.chdir(tmpdir) # download data testdata_paths = [] for testdata in self.cfg['testdata']: path = self.obtain_file(testdata) if not path: self.log.error("Downloading file from %s failed?" % testdata) testdata_paths.append(path) # unpack data for path in testdata_paths: extract_file(path, tmpdir) # copy namelist.wps file fn = "namelist.wps" shutil.copy2(os.path.join(wpsdir, fn), tmpdir) namelist_file = os.path.join(tmpdir, fn) # GEOGRID # setup directories and files for d in os.listdir(os.path.join(tmpdir, "geog")): os.symlink(os.path.join(tmpdir, "geog", d), os.path.join(tmpdir, d)) # patch namelist.wps file for geogrib for line in fileinput.input(namelist_file, inplace=1, backup='.orig.geogrid'): line = re.sub(r"^(\s*geog_data_path\s*=\s*).*$", r"\1 '%s'" % tmpdir, line) sys.stdout.write(line) # GEOGRID.TBL geogrid_dir = os.path.join(tmpdir, "geogrid") os.mkdir(geogrid_dir) os.symlink(os.path.join(wpsdir, "geogrid", "GEOGRID.TBL.ARW"), os.path.join(geogrid_dir, "GEOGRID.TBL")) # run geogrid.exe run_wps_cmd("geogrid") # UNGRIB # determine start and end time stamps of grib files grib_file_prefix = "fnl_" k = len(grib_file_prefix) fs = [f for f in sorted(os.listdir('.')) if f.startswith(grib_file_prefix)] start = "%s:00:00" % fs[0][k:] end = "%s:00:00" % fs[-1][k:] # patch namelist.wps file for ungrib shutil.copy2(os.path.join(wpsdir, "namelist.wps"), tmpdir) for line in fileinput.input(namelist_file, inplace=1, backup='.orig.ungrib'): line = re.sub(r"^(\s*start_date\s*=\s*).*$", r"\1 '%s','%s'," % (start, start), line) line = re.sub(r"^(\s*end_date\s*=\s*).*$", r"\1 '%s','%s'," % (end, end), line) sys.stdout.write(line) # copy correct Vtable shutil.copy2(os.path.join(wpsdir, "ungrib", "Variable_Tables", "Vtable.ARW"), os.path.join(tmpdir, "Vtable")) # run link_grib.csh script cmd = "%s %s*" % (os.path.join(wpsdir, "link_grib.csh"), grib_file_prefix) run_cmd(cmd, log_all=True, simple=True) # run ungrib.exe run_wps_cmd("ungrib", mpi_cmd=False) # METGRID.TBL metgrid_dir = os.path.join(tmpdir, "metgrid") os.mkdir(metgrid_dir) os.symlink(os.path.join(wpsdir, "metgrid", "METGRID.TBL.ARW"), os.path.join(metgrid_dir, "METGRID.TBL")) # run metgrid.exe run_wps_cmd('metgrid') # clean up rmtree2(tmpdir) os.chdir(self.builddir) except OSError, err: self.log.error("Failed to run WPS test: %s" % err)
def test_step(self): """Run WIEN2k test benchmarks. """ def run_wien2k_test(cmd_arg): """Run a WPS command, and check for success.""" cmd = "x_lapw lapw1 %s" % cmd_arg (out, _) = run_cmd(cmd, log_all=True, simple=False) re_success = re.compile("LAPW1\s+END") if not re_success.search(out): raise EasyBuildError("Test '%s' in %s failed (pattern '%s' not found)?", cmd, os.getcwd(), re_success.pattern) else: self.log.info("Test '%s' seems to have run successfully: %s" % (cmd, out)) if self.cfg['runtest']: if not self.cfg['testdata']: raise EasyBuildError("List of URLs for testdata not provided.") # prepend $PATH with install directory, define $SCRATCH which is used by the tests env.setvar('PATH', "%s:%s" % (self.installdir, os.environ['PATH'])) try: cwd = os.getcwd() # create temporary directory tmpdir = tempfile.mkdtemp() os.chdir(tmpdir) self.log.info("Running tests in %s" % tmpdir) scratch = os.path.join(tmpdir, 'scratch') mkdir(scratch) env.setvar('SCRATCH', scratch) # download data testdata_paths = {} for testdata in self.cfg['testdata']: td_path = self.obtain_file(testdata) if not td_path: raise EasyBuildError("Downloading file from %s failed?", testdata) testdata_paths.update({os.path.basename(testdata): td_path}) self.log.debug('testdata_paths: %s' % testdata_paths) # unpack serial benchmark serial_test_name = "test_case" extract_file(testdata_paths['%s.tar.gz' % serial_test_name], tmpdir) # run serial benchmark os.chdir(os.path.join(tmpdir, serial_test_name)) run_wien2k_test("-c") # unpack parallel benchmark (in serial benchmark dir) parallel_test_name = "mpi-benchmark" extract_file(testdata_paths['%s.tar.gz' % parallel_test_name], tmpdir) # run parallel benchmark os.chdir(os.path.join(tmpdir, serial_test_name)) run_wien2k_test("-p") os.chdir(cwd) rmtree2(tmpdir) except OSError, err: raise EasyBuildError("Failed to run WIEN2k benchmark tests: %s", err) self.log.debug("Current dir: %s" % os.getcwd())
def test_step(self): """Run DOLFIN demos by means of test.""" if self.cfg['runtest']: # set cache/error dirs for Instant tmpdir = tempfile.mkdtemp() instant_cache_dir = os.path.join(tmpdir, '.instant', 'cache') instant_error_dir = os.path.join(tmpdir, '.instant', 'error') try: os.makedirs(instant_cache_dir) os.makedirs(instant_error_dir) except OSError, err: raise EasyBuildError("Failed to create Instant cache/error dirs: %s", err) env_vars = [ ('INSTANT_CACHE_DIR', instant_cache_dir), ('INSTANT_ERROR_DIR', instant_error_dir), ] env_var_cmds = ' && '.join(['export %s="%s"' % (var, val) for (var, val) in env_vars]) # test command templates cmd_template_python = " && ".join([ env_var_cmds, "cd %(dir)s", "python demo_%(name)s.py", "cd -", ]) cpp_cmds = [ env_var_cmds, "cd %(dir)s", ] if LooseVersion(self.version) < LooseVersion('1.1'): cpp_cmds.append("cmake . %s" % self.saved_configopts) cpp_cmds.extend([ "make VERBOSE=1", "./demo_%(name)s", "cd -", ]) cmd_template_cpp = " && ".join(cpp_cmds) # list based on demos available for DOLFIN v1.0.0 pde_demos = ['biharmonic', 'cahn-hilliard', 'hyperelasticity', 'mixed-poisson', 'navier-stokes', 'poisson', 'stokes-iterative'] if LooseVersion(self.version) < LooseVersion('1.1'): demos = [os.path.join('demo', 'la', 'eigenvalue')] + [os.path.join('demo', 'pde', x) for x in pde_demos] else: # verified with v1.6.0 demos = [os.path.join('demo', 'documented', x) for x in pde_demos] # construct commands cmds = [tmpl % {'dir': os.path.join(d, subdir), 'name': os.path.basename(d)} for d in demos for (tmpl, subdir) in [(cmd_template_cpp, 'cpp')]] # exclude Python tests for now, because they 'hang' sometimes (unclear why) # they can be reinstated once run_cmd (or its equivalent) has support for timeouts # see https://github.com/hpcugent/easybuild-framework/issues/581 #for (tmpl, subdir) in [(cmd_template_python, 'python'), (cmd_template_cpp, 'cpp')]] # subdomains-poisson has no C++ get_version, only Python # Python tests excluded, see above #name = 'subdomains-poisson' #path = os.path.join('demo', 'pde', name, 'python') #cmds += [cmd_template_python % {'dir': path, 'name': name}] # supply empty argument to each command for cmd in cmds: run_cmd(cmd, log_all=True) # clean up temporary dir try: rmtree2(tmpdir) except OSError, err: raise EasyBuildError("Failed to remove Instant cache/error dirs: %s", err)