def install_step(self): """Custom install procedure for FLUENT.""" cmd = "./INSTALL -noroot -silent -install_dir %s" % self.installdir run_cmd(cmd, log_all=True, simple=True) adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self): """Install by copying unzipped binaries to 'bin' subdir of installation dir, and fixing permissions.""" bindir = os.path.join(self.installdir, 'bin') try: os.makedirs(bindir) for item in os.listdir(self.cfg['start_dir']): if os.path.isfile(item): shutil.copy2(os.path.join(self.cfg['start_dir'], item), bindir) # make sure binary has executable permissions adjust_permissions(os.path.join(bindir, item), stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) self.log.debug("Copied %s to %s and fixed permissions" % (item, bindir)) else: self.log.warning( "Skipping non-file %s in %s, not copying it." % (item, self.cfg['start_dir'])) except OSError, err: raise EasyBuildError( "Copying binaries in %s to install dir 'bin' failed: %s", self.cfg['start_dir'], err)
def install_step(self): """MATLAB install procedure using 'install' command.""" src = os.path.join(self.cfg['start_dir'], 'install') # make sure install script is executable adjust_permissions(src, stat.S_IXUSR) if LooseVersion(self.version) >= LooseVersion('2016b'): jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre', 'glnxa64', 'jre', 'bin') for perm_dir in [ os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir ]: adjust_permissions(perm_dir, stat.S_IXUSR) # make sure $DISPLAY is not defined, which may lead to (hard to trace) problems # this is a workaround for not being able to specify --nodisplay to the install scripts if 'DISPLAY' in os.environ: os.environ.pop('DISPLAY') if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']: self.cfg['preinstallopts'] = ( 'export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options']) + self.cfg['preinstallopts'] if LooseVersion(self.version) >= LooseVersion('2016b'): change_dir(self.builddir) cmd = "%s %s -v -inputFile %s %s" % (self.cfg['preinstallopts'], src, self.configfile, self.cfg['installopts']) run_cmd(cmd, log_all=True, simple=True)
def install_step(self): """ Install by copying files to install dir """ install_files = [ ('include/bam', self.include_files), ('lib', self.lib_files), ] # v1.3 and more recent supports 'make install', but this only installs (some of) the binaries... if LooseVersion(self.version) >= LooseVersion('1.3'): super(EB_SAMtools, self).install_step() # figure out which bin files are missing, and try copying them missing_bin_files = [] for binfile in self.bin_files: if not os.path.exists(os.path.join(self.installdir, 'bin', os.path.basename(binfile))): missing_bin_files.append(binfile) install_files.append(('bin', missing_bin_files)) else: # copy binaries manually for older versions install_files.append(('bin', self.bin_files)) self.log.debug("Installing files by copying them 'manually': %s", install_files) for (destdir, files) in install_files: for fn in files: dest = os.path.join(self.installdir, destdir, os.path.basename(fn)) copy_file(os.path.join(self.cfg['start_dir'], fn), dest) # enable r-x permissions for group/others perms = stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH adjust_permissions(self.installdir, perms, add=True, recursive=True)
def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" # put fake 'gcc' command in place that just echos its arguments fake_gcc = os.path.join(self.test_prefix, 'fake', 'gcc') write_file(fake_gcc, '#!/bin/bash\necho "$@"') adjust_permissions(fake_gcc, stat.S_IXUSR) os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', '')) # enable --rpath and prepare toolchain init_config(build_options={'rpath': True, 'rpath_filter': ['/ba.*']}) tc = self.get_toolchain('gompi', version='1.3.12') # preparing RPATH wrappers requires --experimental, need to bypass that here tc.log.experimental = lambda x: x tc.prepare() # check whether fake gcc was wrapped and that arguments are what they should be # no -rpath for /bar because of rpath filter out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""') expected = ' '.join([ '-Wl,-rpath=$ORIGIN/../lib', '-Wl,-rpath=$ORIGIN/../lib64', '-Wl,--disable-new-dtags', '-Wl,-rpath=/foo', '%(user)s.c', '-L/foo', '-L/bar', '$FOO', '-DX=""', ]) self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" # command that generates a lot of output before waiting for input # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000! cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; ' cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' (out, ec) = run_cmd_qa(cmd, {'Pick a number: ': '42'}, log_all=True, maxhits=5) self.assertEqual(ec, 0) regex = re.compile("Picked number: 42$") self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out)) # also test with script run as interactive command that quickly exits with non-zero exit code; # see https://github.com/easybuilders/easybuild-framework/issues/3593 script_txt = '\n'.join([ "#/bin/bash", "echo 'Hello, I am about to exit'", "echo 'ERROR: I failed' >&2", "exit 1", ]) script = os.path.join(self.test_prefix, 'test.sh') write_file(script, script_txt) adjust_permissions(script, stat.S_IXUSR) out, ec = run_cmd_qa(script, {}, log_ok=False) self.assertEqual(ec, 1) self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n")
def install_step(self): """Install by copying files to install dir and binary to 'bin' install subdir, and fixing permissions.""" bindir = os.path.join(self.installdir, 'bin') binaries = [ 'foldx_%s.linux' % self.version, # FoldX v2.x 'FoldX.linux64', # FoldX 3.x 'foldx64Linux', # FoldX 3.0-beta6 'foldx3b6', # FoldX 3.0 beta 6.1 >_< ] try: os.makedirs(bindir) for item in os.listdir(self.cfg['start_dir']): if os.path.isfile(item): if item in binaries: shutil.copy2(os.path.join(self.cfg['start_dir'], item), bindir) # make sure binary has executable permissions adjust_permissions(os.path.join(bindir, item), stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH, add=True) self.log.debug("Copied %s to %s and fixed permissions" % (item, bindir)) else: # copy everything else straight into install directory shutil.copy2(os.path.join(self.cfg['start_dir'], item), self.installdir) self.log.debug("Copied %s to install dir %s" % (item, self.installdir)) else: self.log.warning("Skipping non-file %s in %s, not copying it." % (item, self.cfg['start_dir'])) except OSError, err: raise EasyBuildError("Copying binaries in %s to install dir 'bin' failed: %s", self.cfg['start_dir'], err)
def install_step(self): """Custom install procedure for Samcef.""" # e.g.: 17.0-03 => 17, 17.0, 170 maj_ver = self.version.split('.')[0] main_ver = self.version.split('-')[0] flat_ver = ''.join(main_ver.split('.')) qa = { "Default language ? (E=English, F=French, default E): ": 'E', "Default Postscript printer ? (default: none): ": '', "Type RETURN to continue": '', } no_qa = [] std_qa = { r"Installation for:.*[\s\n]*Continue \? \(y/n, default y\):": 'y', r" 1 Install Samcef %s \(%s\)[\s\n]*.*[\s\n]*Install type .*" % (maj_ver, main_ver): '1', r"Install SHORT INTEGERS VERSION \(i4\)[\s\na-zA-Z0-9().-]*Continue \? \(y/n, default y\):": 'n', r"Install LONG INTEGERS VERSION \(i8\)[\s\na-zA-Z0-9().-]*Continue \? \(y/n, default y\):": 'y', r"Pathname of install directory \(.*\):": self.installdir, r"Directory .* exists, use it \? \(y/n, default y\):": 'y', r"Installation directory\s*:.*[\s\n]*.*[\s\n]*Continue \? \(y/n, default:y\):": 'y', # option '2' is SAMCEF + object libraries r"Selection size: 0[\s\n]*Your selection:": '2', # 'i' for install after making a selection r"Selection size: [1-9][0-9]*[\s\n]*Your selection:": 'i', r"Prerequisite OK[\s\n]*Continue \? \(y/n, default y\):": 'y', r"Samcef users can modify these values.*[\s\n]*Continue \? \(y/n, default y\):": 'y', r"What is the hostname of your license server \( without at sign @ \) \? ": 'localhost', r"Confirm .* as the LMS Samtech Licence server \? \(y/n, default n\): ": 'y', r"2 Floating RLM license[\s\n]*Type a value \(1/2, default 1 \)": '2', } # e.g., v170_inst.csh install_script = './v%s_inst.csh' % flat_ver # make sure script is executable (unpack with 7z doesn't preserve permissions) adjust_permissions(install_script, stat.S_IXUSR, add=True) run_cmd_qa(install_script, qa, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True)
def install_step(self): """MCR install procedure using 'install' command.""" src = os.path.join(self.cfg['start_dir'], 'install') # make sure install script is executable adjust_permissions(src, stat.S_IXUSR) # make sure $DISPLAY is not defined, which may lead to (hard to trace) problems # this is a workaround for not being able to specify --nodisplay to the install scripts if 'DISPLAY' in os.environ: os.environ.pop('DISPLAY') if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']: java_options = 'export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options'] self.cfg['preinstallopts'] = java_options + self.cfg['preinstallopts'] configfile = "%s/%s" % (self.builddir, self.configfilename) cmd = "%s ./install -v -inputFile %s %s" % (self.cfg['preinstallopts'], configfile, self.cfg['installopts']) run_cmd(cmd, log_all=True, simple=True) # determine subdirectory (e.g. v84 (2014a, 2014b), v85 (2015a), ...) subdirs = os.listdir(self.installdir) if len(subdirs) == 1: self.subdir = subdirs[0] else: raise EasyBuildError("Found multiple subdirectories, don't know which one to pick: %s", subdirs)
def install_step(self): """Install by copying the contents of the builddir to the installdir (preserving permissions)""" for x in os.listdir(self.cfg['start_dir']): src = os.path.join(self.cfg['start_dir'], x) dst = os.path.join(self.installdir, x) try: if os.path.isdir(src): shutil.copytree(src, dst) # symlink # - dst/Lib to dst/lib # - dst/Include to dst/include for c in ['Lib', 'Include']: nsrc = os.path.join(dst, c) ndst = os.path.join(dst, c.lower()) if os.path.exists(nsrc): os.symlink(nsrc, ndst) # enable r-x permissions for group/others perms = stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH adjust_permissions(dst, perms, add=True, recursive=True, onlydirs=True) else: shutil.copy2(src, dst) except OSError, err: raise EasyBuildError("Copying src %s to dst %s failed: %s", src, dst, err)
def build_step(self): """No build step for SNPhylo.""" # check for required dependencies for dep in ['MUSCLE', 'PHYLIP', 'Python', 'R']: if not get_software_root(dep): raise EasyBuildError("Required dependency '%s' not loaded", dep) # check for required R libraries rver = get_software_version('R') r_libs, _ = run_cmd( "R --vanilla --no-save --slave -e 'print(installed.packages())'", simple=False) for rpkg in ['gdsfmt', 'getopt', 'SNPRelate', 'phangorn']: if not re.search(r'^%s\s.*%s' % (rpkg, rver), r_libs, re.M): raise EasyBuildError("Required R package '%s' not installed", rpkg) # run setup.sh, and send a bunch of newlines as stdin to 'answer' the Q&A; # all questions can be answered with the default answer (if the dependencies are specified correctly); # use run_cmd_qa doesn not work because of buffering issues (questions are not coming through) adjust_permissions('setup.sh', stat.S_IXUSR, add=True) (out, _) = run_cmd('bash ./setup.sh', inp='\n' * 10, simple=False) success_msg = "SNPHYLO is successfully installed!!!" if success_msg not in out: raise EasyBuildError( "Success message '%s' not found in setup.sh output: %s", success_msg, out)
def test_remove_file(self): """Test remove_file""" testfile = os.path.join(self.test_prefix, 'foo') ft.write_file(testfile, 'bar') self.assertTrue(os.path.exists(testfile)) ft.remove_file(testfile) ft.write_file(testfile, 'bar') ft.adjust_permissions(self.test_prefix, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False) self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile) # also test behaviour of remove_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) self.mock_stdout(True) ft.remove_file(testfile) txt = self.get_stdout() self.mock_stdout(False) regex = re.compile("^file [^ ]* removed$") self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def install_step(self): """MCR install procedure using 'install' command.""" src = os.path.join(self.cfg['start_dir'], 'install') # make sure install script is executable adjust_permissions(src, stat.S_IXUSR) # make sure $DISPLAY is not defined, which may lead to (hard to trace) problems # this is a workaround for not being able to specify --nodisplay to the install scripts if 'DISPLAY' in os.environ: os.environ.pop('DISPLAY') if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']: java_options = 'export _JAVA_OPTIONS="%s" && ' % self.cfg[ 'java_options'] self.cfg[ 'preinstallopts'] = java_options + self.cfg['preinstallopts'] configfile = "%s/%s" % (self.builddir, self.configfilename) cmd = "%s ./install -v -inputFile %s %s" % ( self.cfg['preinstallopts'], configfile, self.cfg['installopts']) run_cmd(cmd, log_all=True, simple=True) # determine subdirectory (e.g. v84 (2014a, 2014b), v85 (2015a), ...) subdirs = os.listdir(self.installdir) if len(subdirs) == 1: self.subdir = subdirs[0] else: raise EasyBuildError( "Found multiple subdirectories, don't know which one to pick: %s", subdirs)
def install_step(self): """Install by running install command.""" pgi_env_vars = { 'PGI_ACCEPT_EULA': 'accept', 'PGI_INSTALL_AMD': str(self.cfg['install_amd']).lower(), 'PGI_INSTALL_DIR': self.installdir, 'PGI_INSTALL_JAVA': str(self.cfg['install_java']).lower(), 'PGI_INSTALL_MANAGED': str(self.cfg['install_managed']).lower(), 'PGI_INSTALL_NVIDIA': str(self.cfg['install_nvidia']).lower(), 'PGI_SILENT': 'true', } cmd = "%s ./install" % ' '.join( ['%s=%s' % x for x in sorted(pgi_env_vars.items())]) run_cmd(cmd, log_all=True, simple=True) # make sure localrc uses GCC in PATH, not always the system GCC, and does not use a system g77 but gfortran for subdir in self.pgi_install_subdirs: install_abs_subdir = os.path.join(self.installdir, subdir) filename = os.path.join(install_abs_subdir, "bin", "makelocalrc") for line in fileinput.input(filename, inplace='1', backup='.orig'): line = re.sub(r"^PATH=/", r"#PATH=/", line) sys.stdout.write(line) cmd = "%s -x %s -g77 /" % (filename, install_abs_subdir) run_cmd(cmd, log_all=True, simple=True) # If an OS libnuma is NOT found, makelocalrc creates symbolic links to libpgnuma.so # If we use the EB libnuma, delete those symbolic links to ensure they are not used if get_software_root("numactl"): for filename in ["libnuma.so", "libnuma.so.1"]: path = os.path.join(install_abs_subdir, "lib", filename) if os.path.islink(path): os.remove(path) # install (or update) siterc file to make PGI consider $LIBRARY_PATH and accept -pthread siterc_path = os.path.join(self.installdir, subdir, 'bin', 'siterc') # adding LIBRARY_PATH has the side effect of adding llvm to nollvm siterc files and viceversa # this is done dynamically, so we can't account for it in all cases. Simply do it for the default #isdefault = False #if ((LooseVersion(self.version) > LooseVersion('18') and LooseVersion(self.version) < LooseVersion('19') and # subdir == 'linux86-64-nollvm') or # (LooseVersion(self.version) > LooseVersion('19') and subdir == 'linux86-64-llvm')): # isdefault = True if subdir == os.path.join('linux86-64', self.version): write_file(siterc_path, SITERC_LIBRARY_PATH, append=True) self.log.info( "Appended instructions to pick up $LIBRARY_PATH to siterc file at %s: %s", siterc_path, SITERC_LIBRARY_PATH) write_file(siterc_path, SITERC_PTHREAD_SWITCH, append=True) self.log.info( "Append instructions to replace -pthread with -lpthread to siterc file at %s: %s", siterc_path, SITERC_PTHREAD_SWITCH) # The cuda nvvp tar file has broken permissions adjust_permissions(self.installdir, stat.S_IWUSR, add=True, onlydirs=True)
def test_remove_file(self): """Test remove_file""" testfile = os.path.join(self.test_prefix, 'foo') ft.write_file(testfile, 'bar') self.assertTrue(os.path.exists(testfile)) ft.remove_file(testfile) ft.write_file(testfile, 'bar') ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False) self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile) # also test behaviour of remove_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) self.mock_stdout(True) ft.remove_file(testfile) txt = self.get_stdout() self.mock_stdout(False) regex = re.compile("^file [^ ]* removed$") self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def install_step(self): """ Install binary and a wrapper that loads correct CUDA version. """ src_gctf_bin = os.path.join(self.builddir, 'bin', self.gctf_bin) if not os.path.exists(src_gctf_bin): raise EasyBuildError( "Specified CUDA version has no corresponding Gctf binary") bindir = os.path.join(self.installdir, 'bin') mkdir(bindir) dst_gctf_bin = os.path.join(bindir, self.gctf_bin) copy_file(src_gctf_bin, dst_gctf_bin) exe_perms = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH read_perms = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH perms = read_perms | exe_perms adjust_permissions(dst_gctf_bin, perms, add=True) # Install a wrapper that loads CUDA before starting the binary wrapper = os.path.join(bindir, 'Gctf') txt = '\n'.join([ '#!/bin/bash', '', '# Wrapper for Gctf binary that loads the required', '# version of CUDA', 'module unload %s' % self.cuda_name, 'module add %s' % self.cuda_mod_name, 'exec %s "$@"' % dst_gctf_bin ]) write_file(wrapper, txt) adjust_permissions(wrapper, exe_perms, add=True) symlink('Gctf', os.path.join(bindir, 'gctf'), use_abspath_source=False)
def create_wrapper(wrapper_name, wrapper_comp): """Create for a particular compiler, with a particular name""" wrapper_f = os.path.join(self.installdir, 'bin', wrapper_name) write_file(wrapper_f, WRAPPER_TEMPLATE % wrapper_comp) adjust_permissions( wrapper_f, stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH)
def install_step(self): """MATLAB install procedure using 'install' command.""" src = os.path.join(self.cfg['start_dir'], 'install') # make sure install script is executable adjust_permissions(src, stat.S_IXUSR) if LooseVersion(self.version) >= LooseVersion('2016b'): jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre', 'glnxa64', 'jre', 'bin') for perm_dir in [os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir]: adjust_permissions(perm_dir, stat.S_IXUSR) # make sure $DISPLAY is not defined, which may lead to (hard to trace) problems # this is a workaround for not being able to specify --nodisplay to the install scripts if 'DISPLAY' in os.environ: os.environ.pop('DISPLAY') if not '_JAVA_OPTIONS' in self.cfg['preinstallopts']: self.cfg['preinstallopts'] = ('export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options']) + self.cfg['preinstallopts'] if LooseVersion(self.version) >= LooseVersion('2016b'): change_dir(self.builddir) cmd = "%s %s -v -inputFile %s %s" % (self.cfg['preinstallopts'], src, self.configfile, self.cfg['installopts']) run_cmd(cmd, log_all=True, simple=True)
def test_build_easyconfigs_in_parallel_slurm(self): """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job.""" # install mocked versions of 'sbatch' and 'scontrol' commands sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch') write_file(sbatch, MOCKED_SBATCH) adjust_permissions(sbatch, stat.S_IXUSR, add=True) scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol') write_file(scontrol, MOCKED_SCONTROL) adjust_permissions(scontrol, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join( [os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) topdir = os.path.dirname(os.path.abspath(__file__)) test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, 'job_max_walltime': 5, 'force': True, } init_config(args=['--job-backend=Slurm'], build_options=build_options) easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) self.mock_stdout(True) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.mock_stdout(False) # jobs are submitted for foss & gzip (listed easyconfigs) self.assertEqual(len(jobs), 2) # last job (gzip) has a dependency on second-to-last job (foss) self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a') expected = { 'dependency': 'afterok:%s' % jobs[0].jobid, 'hold': True, 'job-name': 'gzip-1.5-foss-2018a', 'nodes': 1, 'ntasks': 3, 'ntasks-per-node': 3, 'output': '%x-%j.out', 'time': 300, # 60*5 (unit is minutes) 'wrap': "echo '%s'" % test_ec, } self.assertEqual(jobs[1].job_specs, expected)
def configure_step(self): """Configure COMSOL installation: create license file.""" # The tar file comes from the DVD and has 0444 as permission at the top dir. adjust_permissions(self.start_dir, stat.S_IWUSR) default_lic_env_var = 'LMCOMSOL_LICENSE_FILE' lic_specs, self.license_env_var = find_flexlm_license( custom_env_vars=[default_lic_env_var], lic_specs=[self.cfg['license_file']]) if lic_specs: if self.license_env_var is None: self.log.info( "Using COMSOL license specifications from 'license_file': %s", lic_specs) self.license_env_var = default_lic_env_var else: self.log.info( "Using COMSOL license specifications from $%s: %s", self.license_env_var, lic_specs) self.license_file = os.pathsep.join(lic_specs) env.setvar(self.license_env_var, self.license_file) else: msg = "No viable license specifications found; " msg += "specify 'license_file', or define $%s" % default_lic_env_var raise EasyBuildError(msg) copy_file(os.path.join(self.start_dir, 'setupconfig.ini'), self.configfile) config = read_file(self.configfile) config_vars = { 'agree': '1', 'desktopshortcuts': '0', 'fileassoc': '0', 'firewall': '0', 'installdir': self.installdir, 'license': self.license_file, 'licmanager': '0', 'linuxlauncher': '0', 'showgui': '0', 'startmenushortcuts': '0', 'symlinks': '0', } matlab_root = get_software_root("MATLAB") if matlab_root: config_vars.update({'matlabdir': matlab_root}) for key, val in config_vars.items(): regex = re.compile(r"^%s\s*=.*" % key, re.M) config = regex.sub("%s=%s" % (key, val), config) write_file(self.configfile, config) self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def test_end2end_docker_image(self): topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) args = [ toy_ec, '-C', # equivalent with --containerize '--experimental', '--container-type=docker', '--container-config=ubuntu:16.04', '--container-build-image', ] if not which('docker'): error_pattern = "docker not found on your system." self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True) # install mocked versions of 'sudo' and 'docker' commands docker = os.path.join(self.test_prefix, 'bin', 'docker') write_file(docker, MOCKED_DOCKER) adjust_permissions(docker, stat.S_IXUSR, add=True) sudo = os.path.join(self.test_prefix, 'bin', 'sudo') write_file( sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n' ) adjust_permissions(sudo, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join( [os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs = [ "^== docker tool found at %s/bin/docker" % self.test_prefix, "^== Dockerfile definition file created at %s/containers/Dockerfile\.toy-0.0" % self.test_prefix, "^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...", "^== Docker image created at toy-0.0:latest", ] self.check_regexs(regexs, stdout) args.extend(['--force', '--extended-dry-run']) stdout, stderr = self.run_main(args) self.assertFalse(stderr) self.check_regexs(regexs, stdout)
def build_and_install_software(ecs, init_session_state, exit_on_failure=True): """Build and install software for all provided parsed easyconfig files.""" # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) res = [] for ec in ecs: ec_res = {} try: (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) except Exception, err: # purposely catch all exceptions ec_res['success'] = False ec_res['err'] = err ec_res['traceback'] = traceback.format_exc() # keep track of success/total count if ec_res['success']: test_msg = "Successfully built %s" % ec['spec'] else: test_msg = "Build of %s failed" % ec['spec'] if 'err' in ec_res: test_msg += " (err: %s)" % ec_res['err'] # dump test report next to log file test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) if 'log_file' in ec_res and ec_res['log_file']: test_report_fp = "%s_test_report.md" % '.'.join( ec_res['log_file'].split('.')[:-1]) parent_dir = os.path.dirname(test_report_fp) # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used if os.stat(parent_dir).st_mode & 0200: write_file(test_report_fp, test_report_txt) else: adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) write_file(test_report_fp, test_report_txt) adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res['success'] and exit_on_failure: if 'traceback' in ec_res: raise EasyBuildError(ec_res['traceback']) else: raise EasyBuildError(test_msg) res.append((ec, ec_res))
def build_and_install_software(ecs, init_session_state, exit_on_failure=True, hooks=None): """ Build and install software for all provided parsed easyconfig files. :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure :param hooks: list of defined pre- and post-step hooks """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) run_hook(START, hooks) res = [] for ec in ecs: ec_res = {} try: (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, hooks=hooks) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) except Exception, err: # purposely catch all exceptions ec_res['success'] = False ec_res['err'] = err ec_res['traceback'] = traceback.format_exc() # keep track of success/total count if ec_res['success']: test_msg = "Successfully built %s" % ec['spec'] else: test_msg = "Build of %s failed" % ec['spec'] if 'err' in ec_res: test_msg += " (err: %s)" % ec_res['err'] # dump test report next to log file test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) if 'log_file' in ec_res and ec_res['log_file']: test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1]) parent_dir = os.path.dirname(test_report_fp) # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used if os.stat(parent_dir).st_mode & 0200: write_file(test_report_fp, test_report_txt) else: adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) write_file(test_report_fp, test_report_txt) adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res['success'] and exit_on_failure: if 'traceback' in ec_res: raise EasyBuildError(ec_res['traceback']) else: raise EasyBuildError(test_msg) res.append((ec, ec_res))
def post_install_step(self): """Add a shebang to the .py files and make them executable.""" for pyfile in self.pyfiles: pf_path = os.path.join(self.installdir, pyfile) pf_contents = read_file(pf_path) write_file(pf_path, "#!/usr/bin/env python\n" + pf_contents) adjust_permissions(pf_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) super(EB_fastStructure, self).post_install_step()
def install_step(self): """MATLAB install procedure using 'install' command.""" src = os.path.join(self.cfg['start_dir'], 'install') # make sure install script is executable adjust_permissions(src, stat.S_IXUSR) if LooseVersion(self.version) >= LooseVersion('2016b'): jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre', 'glnxa64', 'jre', 'bin') for perm_dir in [os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir]: adjust_permissions(perm_dir, stat.S_IXUSR) # make sure $DISPLAY is not defined, which may lead to (hard to trace) problems # this is a workaround for not being able to specify --nodisplay to the install scripts if 'DISPLAY' in os.environ: os.environ.pop('DISPLAY') if '_JAVA_OPTIONS' not in self.cfg['preinstallopts']: java_opts = 'export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options'] self.cfg['preinstallopts'] = java_opts + self.cfg['preinstallopts'] if LooseVersion(self.version) >= LooseVersion('2016b'): change_dir(self.builddir) # MATLAB installer ignores TMPDIR (always uses /tmp) and might need a large tmpdir tmpdir = "-tmpdir %s" % tempfile.mkdtemp() keys = self.cfg['key'] if keys is None: keys = os.getenv('EB_MATLAB_KEY', '00000-00000-00000-00000-00000-00000-00000-00000-00000-00000') if isinstance(keys, basestring): keys = keys.split(',') # Make one install for each key for key in keys: cmd = ' '.join([ self.cfg['preinstallopts'], src, '-v', tmpdir, '-inputFile', self.configfile, '-fileInstallationKey', key, self.cfg['installopts'], ]) (out, _) = run_cmd(cmd, log_all=True, simple=False) # check installer output for known signs of trouble patterns = [ "Error: You have entered an invalid File Installation Key", ] for pattern in patterns: regex = re.compile(pattern, re.I) if regex.search(out): raise EasyBuildError("Found error pattern '%s' in output of installation command '%s': %s", regex.pattern, cmd, out)
def install_step(self): """Custom install procedure for ANSYS.""" licserv = self.cfg["license_server"] licport = self.cfg["license_server_port"] cmd = "./INSTALL -silent -install_dir %s -licserverinfo %s:%s" % (self.installdir, licport, licserv) run_cmd(cmd, log_all=True, simple=True) adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self): """Custom install procedure for ANSYS.""" licserv = self.cfg['license_server'] licport = self.cfg['license_server_port'] cmd = "./INSTALL -silent -install_dir %s -licserverinfo %s:%s" % (self.installdir, licport, licserv) run_cmd(cmd, log_all=True, simple=True) adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def configure_step(self): """Configure MATLAB installation: create license file.""" licfile = self.cfg['license_file'] if licfile is None: licserv = self.cfg['license_server'] if licserv is None: licserv = os.getenv('EB_MATLAB_LICENSE_SERVER', 'license.example.com') licport = self.cfg['license_server_port'] if licport is None: licport = os.getenv('EB_MATLAB_LICENSE_SERVER_PORT', '00000') # create license file lictxt = '\n'.join([ "SERVER %s 000000000000 %s" % (licserv, licport), "USE_SERVER", ]) licfile = os.path.join(self.builddir, 'matlab.lic') write_file(licfile, lictxt) try: copy_file( os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile) adjust_permissions(self.configfile, stat.S_IWUSR) # read file in binary mode to avoid UTF-8 encoding issues when using Python 3, # due to non-UTF-8 characters... config = read_file(self.configfile, mode='rb') # use raw byte strings (must be 'br', not 'rb'), # required when using Python 3 because file was read in binary mode regdest = re.compile(br"^# destinationFolder=.*", re.M) regagree = re.compile(br"^# agreeToLicense=.*", re.M) regmode = re.compile(br"^# mode=.*", re.M) reglicpath = re.compile(br"^# licensePath=.*", re.M) # must use byte-strings here when using Python 3, see above config = regdest.sub( b"destinationFolder=%s" % self.installdir.encode('utf-8'), config) config = regagree.sub(b"agreeToLicense=Yes", config) config = regmode.sub(b"mode=silent", config) config = reglicpath.sub( b"licensePath=%s" % licfile.encode('utf-8'), config) write_file(self.configfile, config) except IOError as err: raise EasyBuildError( "Failed to create installation config file %s: %s", self.configfile, err) self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def install_step(self): """Install by running install command.""" pgi_env_vars = { 'PGI_ACCEPT_EULA': 'accept', 'PGI_INSTALL_AMD': str(self.cfg['install_amd']).lower(), 'PGI_INSTALL_DIR': self.installdir, 'PGI_INSTALL_JAVA': str(self.cfg['install_java']).lower(), 'PGI_INSTALL_MANAGED': str(self.cfg['install_managed']).lower(), 'PGI_INSTALL_NVIDIA': str(self.cfg['install_nvidia']).lower(), 'PGI_SILENT': 'true', 'PGI_INSTALL_MPI': str(self.cfg['install_mpi']).lower(), 'PGI_MPI_GPU_SUPPORT': str(self.cfg['mpi_gpu_support']).lower(), } cmd = "%s ./install" % ' '.join( ['%s=%s' % x for x in sorted(pgi_env_vars.items())]) run_cmd(cmd, log_all=True, simple=True) # make sure localrc uses GCC in PATH, not always the system GCC, and does not use a system g77 but gfortran install_abs_subdir = os.path.join(self.installdir, self.pgi_install_subdir) filename = os.path.join(install_abs_subdir, "bin", "makelocalrc") for line in fileinput.input(filename, inplace='1', backup='.orig'): line = re.sub(r"^PATH=/", r"#PATH=/", line) sys.stdout.write(line) cmd = "%s -x %s -g77 /" % (filename, install_abs_subdir) run_cmd(cmd, log_all=True, simple=True) # If an OS libnuma is NOT found, makelocalrc creates symbolic links to libpgnuma.so # If we use the EB libnuma, delete those symbolic links to ensure they are not used if get_software_root("numactl"): for subdir in self.pgi_install_subdirs: install_abs_subdir = os.path.join(self.installdir, subdir) for filename in ["libnuma.so", "libnuma.so.1"]: path = os.path.join(install_abs_subdir, "lib", filename) if os.path.islink(path): os.remove(path) # install (or update) siterc file to make PGI consider $LIBRARY_PATH and accept -pthread siterc_path = os.path.join(self.installdir, self.pgi_install_subdir, 'bin', 'siterc') write_file(siterc_path, SITERC_LIBRARY_PATH, append=True) self.log.info( "Appended instructions to pick up $LIBRARY_PATH to siterc file at %s: %s", siterc_path, SITERC_LIBRARY_PATH) write_file(siterc_path, SITERC_PTHREAD_SWITCH, append=True) self.log.info( "Append instructions to replace -pthread with -lpthread to siterc file at %s: %s", siterc_path, SITERC_PTHREAD_SWITCH) # The cuda nvvp tar file has broken permissions adjust_permissions(self.installdir, stat.S_IWUSR, add=True, onlydirs=True)
def install_step(self): super(EB_BWISE, self).install_step() # BWISE expects BCALM to be at exactly this location... bcalmwrapper = """#!/bin/sh $EBROOTBCALM/bin/bcalm "$@" """ write_file(os.path.join(self.installdir, "bin", "bcalm"), bcalmwrapper) adjust_permissions(os.path.join(self.installdir, "bin"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def extract_step(self): """Unpack the source""" if LooseVersion(self.version) < LooseVersion('1.7'): copy_file(self.src[0]['path'], self.builddir) adjust_permissions(os.path.join(self.builddir, self.src[0]['name']), stat.S_IXUSR, add=True) change_dir(self.builddir) run_cmd(os.path.join(self.builddir, self.src[0]['name']), log_all=True, simple=True, inp='') else: PackedBinary.extract_step(self)
def install_step(self): """Custom install procedure for FLUENT.""" extra_args ='' # only include -noroot flag for older versions if LooseVersion(self.version) < LooseVersion('15.0'): extra_args += '-noroot' cmd = "./INSTALL %s -debug -silent -install_dir %s %s" % (extra_args, self.installdir, self.cfg['installopts']) run_cmd(cmd, log_all=True, simple=True) adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def install_step(self): """Copy all files in build directory to the install directory""" rmtree2(self.installdir) install_script = self.src[0]['name'] adjust_permissions(os.path.join(self.builddir, install_script), stat.S_IRUSR|stat.S_IXUSR) cmd = "%s ./%s -p %s -b -f" % (self.cfg['preinstallopts'], install_script, self.installdir) self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) run_cmd(cmd, log_all=True, simple=True)
def install_step(self): """Custom install procedure for NWChem.""" try: # binary bindir = os.path.join(self.installdir, 'bin') mkdir(bindir) shutil.copy( os.path.join(self.cfg['start_dir'], 'bin', self.cfg['target'], 'nwchem'), bindir) # data shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'data'), os.path.join(self.installdir, 'data')) shutil.copytree( os.path.join(self.cfg['start_dir'], 'src', 'basis', 'libraries'), os.path.join(self.installdir, 'data', 'libraries')) shutil.copytree( os.path.join(self.cfg['start_dir'], 'src', 'nwpw', 'libraryps'), os.path.join(self.installdir, 'data', 'libraryps')) except OSError as err: raise EasyBuildError("Failed to install NWChem: %s", err) # create NWChem settings file default_nwchemrc = os.path.join(self.installdir, 'data', 'default.nwchemrc') txt = '\n'.join([ "nwchem_basis_library %(path)s/data/libraries/", "nwchem_nwpw_library %(path)s/data/libraryps/", "ffield amber", "amber_1 %(path)s/data/amber_s/", "amber_2 %(path)s/data/amber_q/", "amber_3 %(path)s/data/amber_x/", "amber_4 %(path)s/data/amber_u/", "spce %(path)s/data/solvents/spce.rst", "charmm_s %(path)s/data/charmm_s/", "charmm_x %(path)s/data/charmm_x/", ]) % { 'path': self.installdir } write_file(default_nwchemrc, txt) # fix permissions in data directory datadir = os.path.join(self.installdir, 'data') adjust_permissions(datadir, stat.S_IROTH, add=True, recursive=True) adjust_permissions(datadir, stat.S_IXOTH, add=True, recursive=True, onlydirs=True)
def obtain_config_guess(download_source_path=None, search_source_paths=None): """ Locate or download an up-to-date config.guess :param download_source_path: Path to download config.guess to :param search_source_paths: Paths to search for config.guess :return: Path to config.guess or None """ log = fancylogger.getLogger('obtain_config_guess') eb_source_paths = source_paths() if download_source_path is None: download_source_path = eb_source_paths[0] else: log.deprecated("Specifying custom source path to download config.guess via 'download_source_path'", '5.0') if search_source_paths is None: search_source_paths = eb_source_paths else: log.deprecated("Specifying custom location to search for updated config.guess via 'search_source_paths'", '5.0') config_guess = 'config.guess' sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') config_guess_path = None # check if config.guess has already been downloaded to source path for path in search_source_paths: cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) if os.path.isfile(cand_config_guess_path) and check_config_guess(cand_config_guess_path): force_download = build_option('force_download') if force_download: print_warning("Found file %s at %s, but re-downloading it anyway..." % (config_guess, cand_config_guess_path)) else: config_guess_path = cand_config_guess_path log.info("Found %s at %s", config_guess, config_guess_path) break if not config_guess_path: cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID if not download_file(config_guess, config_guess_url, cand_config_guess_path): print_warning("Failed to download recent %s to %s", config_guess, cand_config_guess_path, log=log) elif not check_config_guess(cand_config_guess_path): print_warning("Verification failed for file %s, not using it!", cand_config_guess_path, log=log) remove_file(cand_config_guess_path) else: config_guess_path = cand_config_guess_path adjust_permissions(config_guess_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) log.info("Verified %s at %s, using it if required", config_guess, config_guess_path) return config_guess_path
def test_build_easyconfigs_in_parallel_slurm(self): """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job.""" # install mocked versions of 'sbatch' and 'scontrol' commands sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch') write_file(sbatch, MOCKED_SBATCH) adjust_permissions(sbatch, stat.S_IXUSR, add=True) scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol') write_file(scontrol, MOCKED_SCONTROL) adjust_permissions(scontrol, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) topdir = os.path.dirname(os.path.abspath(__file__)) test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, 'job_max_walltime': 5, 'force': True, } init_config(args=['--job-backend=Slurm'], build_options=build_options) easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) self.mock_stdout(True) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.mock_stdout(False) # jobs are submitted for foss & gzip (listed easyconfigs) self.assertEqual(len(jobs), 2) # last job (gzip) has a dependency on second-to-last job (foss) self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a') expected = { 'dependency': 'afterok:%s' % jobs[0].jobid, 'hold': True, 'job-name': 'gzip-1.5-foss-2018a', 'nodes': 1, 'ntasks': 3, 'ntasks-per-node': 3, 'output': '%x-%j.out', 'time': 300, # 60*5 (unit is minutes) 'wrap': "echo '%s'" % test_ec, } self.assertEqual(jobs[1].job_specs, expected)
def extract_step(self): """Unpack the source""" if LooseVersion(self.version) < LooseVersion('1.7'): try: shutil.copy2(self.src[0]['path'], self.builddir) adjust_permissions(os.path.join(self.builddir, self.src[0]['name']), stat.S_IXUSR, add=True) except OSError, err: raise EasyBuildError("Failed copying installer to builddir or adjunting permissions: %s", err) try: os.chdir(self.builddir) except OSError, err: raise EasyBuildError("Failed to move to build dir: %s", err)
def install_step(self): """CPLEX has an interactive installer, so use Q&A""" tmpdir = os.path.join(self.builddir, 'tmp') stagedir = os.path.join(self.builddir, 'staged') change_dir(self.builddir) mkdir(tmpdir) mkdir(stagedir) env.setvar('IATEMPDIR', tmpdir) dst = os.path.join(self.builddir, self.src[0]['name']) cmd = "%s -i console" % dst qanda = { "PRESS <ENTER> TO CONTINUE:": '', 'Press Enter to continue viewing the license agreement, or enter' ' "1" to accept the agreement, "2" to decline it, "3" to print it,' ' or "99" to go back to the previous screen.:': '1', 'ENTER AN ABSOLUTE PATH, OR PRESS <ENTER> TO ACCEPT THE DEFAULT :': self.installdir, 'IS THIS CORRECT? (Y/N):': 'y', 'PRESS <ENTER> TO INSTALL:': '', "PRESS <ENTER> TO EXIT THE INSTALLER:": '', "CHOOSE LOCALE BY NUMBER:": '', "Choose Instance Management Option:": '', "No model content or proprietary data will be sent.\n1- Yes\n2- No\n" "ENTER THE NUMBER OF THE DESIRED CHOICE:": '2', } noqanda = [r'Installing\.\.\..*\n.*------.*\n\n.*============.*\n.*$'] run_cmd_qa(cmd, qanda, no_qa=noqanda, log_all=True, simple=True) # fix permissions on install dir perms = stat.S_IRWXU | stat.S_IXOTH | stat.S_IXGRP | stat.S_IROTH | stat.S_IRGRP adjust_permissions(self.installdir, perms, recursive=False, relative=False) # also install Python bindings if Python is included as a dependency if self.with_python: cwd = change_dir(os.path.join(self.installdir, 'python')) run_cmd("python setup.py install --prefix=%s" % self.installdir) change_dir(cwd)
def configure_step(self): """Configure COMSOL installation: create license file.""" # The tar file comes from the DVD and has 0444 as permission at the top dir. adjust_permissions(self.start_dir, stat.S_IWUSR) default_lic_env_var = 'LMCOMSOL_LICENSE_FILE' lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var], lic_specs=[self.cfg['license_file']]) if lic_specs: if self.license_env_var is None: self.log.info("Using COMSOL license specifications from 'license_file': %s", lic_specs) self.license_env_var = default_lic_env_var else: self.log.info("Using COMSOL license specifications from $%s: %s", self.license_env_var, lic_specs) self.license_file = os.pathsep.join(lic_specs) env.setvar(self.license_env_var, self.license_file) else: msg = "No viable license specifications found; " msg += "specify 'license_file', or define $%s" % default_lic_env_var raise EasyBuildError(msg) copy_file(os.path.join(self.start_dir, 'setupconfig.ini'), self.configfile) config = read_file(self.configfile) config_vars = { 'agree': '1', 'desktopshortcuts': '0', 'fileassoc': '0', 'firewall': '0', 'installdir': self.installdir, 'license': self.license_file, 'licmanager': '0', 'linuxlauncher': '0', 'showgui': '0', 'startmenushortcuts': '0', 'symlinks': '0', } matlab_root = get_software_root("MATLAB") if matlab_root: config_vars.update({'matlabdir': matlab_root}) for key, val in config_vars.items(): regex = re.compile(r"^%s\s*=.*" % key, re.M) config = regex.sub("%s=%s" % (key, val), config) write_file(self.configfile, config) self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def mock_fpm(tmpdir): """Put mocked version of fpm command in place in specified tmpdir.""" # put mocked 'fpm' command in place, just for testing purposes fpm = os.path.join(tmpdir, 'fpm') write_file(fpm, MOCKED_FPM) adjust_permissions(fpm, stat.S_IXUSR, add=True) # also put mocked rpmbuild in place rpmbuild = os.path.join(tmpdir, 'rpmbuild') write_file(rpmbuild, '#!/bin/bash') # only needs to be there, doesn't need to actually do something... adjust_permissions(rpmbuild, stat.S_IXUSR, add=True) os.environ['PATH'] = '%s:%s' % (tmpdir, os.environ['PATH'])
def install_step(self): """Custom install procedure for ANSYS.""" licserv = self.cfg['license_server'] if licserv is None: licserv = os.getenv('EB_ANSYS_LICENSE_SERVER', 'license.example.com') licport = self.cfg['license_server_port'] if licport is None: licport = os.getenv('EB_ANSYS_LICENSE_SERVER_PORT', '2325:1055') cmd = "./INSTALL -silent -install_dir %s -licserverinfo %s:%s" % (self.installdir, licport, licserv) run_cmd(cmd, log_all=True, simple=True) adjust_permissions(self.installdir, stat.S_IWOTH, add=False)
def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs # noqa (ignore unused import) except ImportError: print "GC3Pie not available, skipping test" return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def install_step(self): """COMSOL install procedure using 'install' command.""" setup_script = os.path.join(self.start_dir, 'setup') # make sure setup script is executable adjust_permissions(setup_script, stat.S_IXUSR) # make sure $DISPLAY is not defined, which may lead to (hard to trace) problems # this is a workaround for not being able to specify --nodisplay to the install scripts env.unset_env_vars(['DISPLAY']) cmd = ' '.join([self.cfg['preinstallopts'], setup_script, '-s', self.configfile, self.cfg['installopts']]) run_cmd(cmd, log_all=True, simple=True)
def extract_step(self): """Copy all source files to the build directory""" if self.cfg.get('extract_sources', False): super(Binary, self).extract_step() else: # required for correctly guessing start directory self.src[0]['finalpath'] = self.builddir # copy source to build dir for source in self.src: dst = os.path.join(self.builddir, source['name']) copy_file(source['path'], dst) adjust_permissions(dst, stat.S_IRWXU, add=True)
def build_and_install_software(ecs, init_session_state, exit_on_failure=True): """Build and install software for all provided parsed easyconfig files.""" # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) res = [] for ec in ecs: ec_res = {} try: (ec_res["success"], app_log, err) = build_and_install_one(ec, init_env) ec_res["log_file"] = app_log if not ec_res["success"]: ec_res["err"] = EasyBuildError(err) except Exception, err: # purposely catch all exceptions ec_res["success"] = False ec_res["err"] = err ec_res["traceback"] = traceback.format_exc() # keep track of success/total count if ec_res["success"]: test_msg = "Successfully built %s" % ec["spec"] else: test_msg = "Build of %s failed" % ec["spec"] if "err" in ec_res: test_msg += " (err: %s)" % ec_res["err"] # dump test report next to log file test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) if "log_file" in ec_res and ec_res["log_file"]: test_report_fp = "%s_test_report.md" % ".".join(ec_res["log_file"].split(".")[:-1]) parent_dir = os.path.dirname(test_report_fp) # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used if os.stat(parent_dir).st_mode & 0200: write_file(test_report_fp, test_report_txt) else: adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) write_file(test_report_fp, test_report_txt) adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res["success"] and exit_on_failure: if "traceback" in ec_res: raise EasyBuildError(ec_res["traceback"]) else: raise EasyBuildError(test_msg) res.append((ec, ec_res))
def test_end2end_docker_image(self): topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) args = [ toy_ec, '-C', # equivalent with --containerize '--experimental', '--container-type=docker', '--container-base=ubuntu:16.04', '--container-build-image', ] if not which('docker'): error_pattern = "docker not found on your system." self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True) # install mocked versions of 'sudo' and 'docker' commands docker = os.path.join(self.test_prefix, 'bin', 'docker') write_file(docker, MOCKED_DOCKER) adjust_permissions(docker, stat.S_IXUSR, add=True) sudo = os.path.join(self.test_prefix, 'bin', 'sudo') write_file(sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n') adjust_permissions(sudo, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs = [ "^== docker tool found at %s/bin/docker" % self.test_prefix, "^== Dockerfile definition file created at %s/containers/Dockerfile\.toy-0.0" % self.test_prefix, "^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...", "^== Docker image created at toy-0.0:latest", ] self.check_regexs(regexs, stdout) args.extend(['--force', '--extended-dry-run']) stdout, stderr = self.run_main(args) self.assertFalse(stderr) self.check_regexs(regexs, stdout)
def test_check_pkg_support(self): """Test check_pkg_support().""" # clear $PATH to make sure fpm/rpmbuild can not be found os.environ['PATH'] = '' self.assertErrorRegex(EasyBuildError, "Selected packaging tool 'fpm' not found", check_pkg_support) for binary in ['fpm', 'rpmbuild']: binpath = os.path.join(self.test_prefix, binary) write_file(binpath, '#!/bin/bash') adjust_permissions(binpath, stat.S_IXUSR, add=True) os.environ['PATH'] = self.test_prefix # no errors => support check passes check_pkg_support()
def test_which(self): """Test which function for locating commands.""" python = ft.which('python') self.assertTrue(python and os.path.exists(python) and os.path.isabs(python)) path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available') self.assertTrue(path is None) os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH']) foo, bar = os.path.join(self.test_prefix, 'foo'), os.path.join(self.test_prefix, 'bar') ft.mkdir(foo) ft.adjust_permissions(foo, stat.S_IRUSR|stat.S_IXUSR) ft.write_file(bar, '#!/bin/bash') ft.adjust_permissions(bar, stat.S_IRUSR|stat.S_IXUSR) self.assertEqual(ft.which('foo'), None) self.assertTrue(os.path.samefile(ft.which('bar'), bar))
def install_step(self): """Install by copying unzipped binaries to 'bin' subdir of installation dir, and fixing permissions.""" bindir = os.path.join(self.installdir, 'bin') try: os.makedirs(bindir) for item in os.listdir(self.cfg['start_dir']): if os.path.isfile(item): shutil.copy2(os.path.join(self.cfg['start_dir'], item), bindir) # make sure binary has executable permissions adjust_permissions(os.path.join(bindir, item), stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH, add=True) self.log.debug("Copied %s to %s and fixed permissions" % (item, bindir)) else: self.log.warning("Skipping non-file %s in %s, not copying it." % (item, self.cfg['start_dir'])) except OSError, err: raise EasyBuildError("Copying binaries in %s to install dir 'bin' failed: %s", self.cfg['start_dir'], err)
def test_run_cmd_script(self): """Testing use of run_cmd with shell=False to call external scripts""" py_test_script = os.path.join(self.test_prefix, 'test.py') write_file(py_test_script, '\n'.join([ '#!/usr/bin/python', 'print("hello")', ])) adjust_permissions(py_test_script, stat.S_IXUSR) (out, ec) = run_cmd(py_test_script) self.assertEqual(ec, 0) self.assertEqual(out, "hello\n") (out, ec) = run_cmd([py_test_script], shell=False) self.assertEqual(ec, 0) self.assertEqual(out, "hello\n")
def write_wrapper(self, wrapper_dir, compiler, i_mpi_root): """Helper function to write a compiler wrapper.""" wrapper_txt = INTEL_COMPILER_WRAPPER % { 'compiler_path': which(compiler), 'intel_mpi_root': i_mpi_root, 'cpath': os.getenv('CPATH'), 'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')), 'wrapper_dir': wrapper_dir, } wrapper = os.path.join(wrapper_dir, compiler) write_file(wrapper, wrapper_txt) if self.dry_run: self.dry_run_msg("Wrapper for '%s' was put in place: %s", compiler, wrapper) else: adjust_permissions(wrapper, stat.S_IXUSR) self.log.info("Using wrapper script for '%s': %s", compiler, which(compiler))