def test_copy_file(self): """ Test copy_file """ testdir = os.path.dirname(os.path.abspath(__file__)) tmpdir = self.test_prefix to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(tmpdir, 'toy.eb') ft.copy_file(to_copy, target_path) self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) # also test behaviour of extract_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) # remove target file, it shouldn't get copied under dry run os.remove(target_path) self.mock_stdout(True) ft.copy_file(to_copy, target_path) txt = self.get_stdout() self.mock_stdout(False) self.assertFalse(os.path.exists(target_path)) self.assertTrue(re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt))
def test_patch_step(self): """Test patch step.""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] orig_sources = ec['ec']['sources'][:] toy_patches = [ 'toy-0.0_typo.patch', # test for applying patch ('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy ] self.assertEqual(ec['ec']['patches'], toy_patches) # test applying patches without sources ec['ec']['sources'] = [] eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step) # test actual patching of unpacked sources ec['ec']['sources'] = orig_sources eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() eb.patch_step() # verify that patches were applied toydir = os.path.join(eb.builddir, 'toy-0.0') self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig']) self.assertTrue("and very proud of it" in read_file(os.path.join(toydir, 'toy.source'))) self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n')
def test_run_cmd_log(self): """Test logging of executed commands.""" fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) regex = re.compile('cmd "echo hello" exited with exit code [0-9]* and output:') # command output is not logged by default without debug logging init_logging(logfile, silent=True) self.assertTrue(run_cmd("echo hello")) stop_logging(logfile) self.assertEqual(len(regex.findall(read_file(logfile))), 0) write_file(logfile, '') init_logging(logfile, silent=True) self.assertTrue(run_cmd("echo hello", log_all=True)) stop_logging(logfile) self.assertEqual(len(regex.findall(read_file(logfile))), 1) write_file(logfile, '') # with debugging enabled, exit code and output of command should only get logged once setLogLevelDebug() init_logging(logfile, silent=True) self.assertTrue(run_cmd("echo hello")) stop_logging(logfile) self.assertEqual(len(regex.findall(read_file(logfile))), 1) write_file(logfile, '') init_logging(logfile, silent=True) self.assertTrue(run_cmd("echo hello", log_all=True)) stop_logging(logfile) self.assertEqual(len(regex.findall(read_file(logfile))), 1) write_file(logfile, '')
def test_package(self): """Test package function.""" init_config(build_options={'silent': True}) test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False) mock_fpm(self.test_prefix) # import needs to be done here, since test easyblocks are only included later from easybuild.easyblocks.toy import EB_toy easyblock = EB_toy(ec) # build & install first easyblock.run_all_steps(False) # package using default packaging configuration (FPM to build RPM packages) pkgdir = package(easyblock) pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION) self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile) pkgtxt = read_file(pkgfile) pkgtxt_regex = re.compile("Contents of installdir %s" % easyblock.installdir) self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt)) if DEBUG: print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
def test_read_write_file(self): """Test reading/writing files.""" fp = os.path.join(self.test_prefix, 'test.txt') txt = "test123" ft.write_file(fp, txt) self.assertEqual(ft.read_file(fp), txt) txt2 = '\n'.join(['test', '123']) ft.write_file(fp, txt2, append=True) self.assertEqual(ft.read_file(fp), txt+txt2) # also test behaviour of write_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) foo = os.path.join(self.test_prefix, 'foo.txt') self.mock_stdout(True) ft.write_file(foo, 'bar') txt = self.get_stdout() self.mock_stdout(False) self.assertFalse(os.path.exists(foo)) self.assertTrue(re.match("^file written: .*/foo.txt$", txt)) ft.write_file(foo, 'bar', forced=True) self.assertTrue(os.path.exists(foo)) self.assertEqual(ft.read_file(foo), 'bar')
def test_download_repo(self): """Test download_repo function.""" if self.github_token is None: print "Skipping test_download_repo, no GitHub token available?" return # default: download tarball for master branch of hpcugent/easybuild-easyconfigs repo path = gh.download_repo(path=self.test_prefix) repodir = os.path.join(self.test_prefix, 'hpcugent', 'easybuild-easyconfigs-master') self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue(os.path.exists(repodir)) shafile = os.path.join(repodir, 'latest-sha') self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2015a.eb'))) # existing downloaded repo is not reperformed, except if SHA is different account, repo, branch = 'boegel', 'easybuild-easyblocks', 'develop' repodir = os.path.join(self.test_prefix, account, '%s-%s' % (repo, branch)) latest_sha = gh.fetch_latest_commit_sha(repo, account, branch=branch) # put 'latest-sha' fail in place, check whether repo was (re)downloaded (should not) shafile = os.path.join(repodir, 'latest-sha') write_file(shafile, latest_sha) path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix) self.assertTrue(os.path.samefile(path, repodir)) self.assertEqual(os.listdir(repodir), ['latest-sha']) # remove 'latest-sha' file and verify that download was performed os.remove(shafile) path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix) self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue('easybuild' in os.listdir(repodir)) self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')))
def test_modules_tool_stateless(self): """Check whether ModulesTool instance is stateless between runs.""" test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') # copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load shutil.copytree(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core')) shutil.copytree(os.path.join(test_modules_path, 'Compiler'), os.path.join(self.test_prefix, 'Compiler')) modtxt = read_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2')) modpath_extension = os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2') modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) write_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'), modtxt) modtxt = read_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4')) modpath_extension = os.path.join(self.test_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') mkdir(modpath_extension, parents=True) modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) write_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), modtxt) # force reset of any singletons by reinitiating config init_config() os.environ['MODULEPATH'] = os.path.join(self.test_prefix, 'Core') modtool = modules_tool() if isinstance(modtool, Lmod): load_err_msg = "cannot[\s\n]*be[\s\n]*loaded" else: load_err_msg = "Unable to locate a modulefile" # GCC/4.6.3 is *not* an available Core module self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['GCC/4.6.3']) # GCC/4.7.2 is one of the available Core modules modtool.load(['GCC/4.7.2']) # OpenMPI/1.6.4 becomes available after loading GCC/4.7.2 module modtool.load(['OpenMPI/1.6.4']) modtool.purge() # reset $MODULEPATH, obtain new ModulesTool instance, # which should not remember anything w.r.t. previous $MODULEPATH value os.environ['MODULEPATH'] = test_modules_path modtool = modules_tool() # GCC/4.6.3 is available modtool.load(['GCC/4.6.3']) modtool.purge() # GCC/4.7.2 is available (note: also as non-Core module outside of hierarchy) modtool.load(['GCC/4.7.2']) # OpenMPI/1.6.4 is *not* available with current $MODULEPATH (loaded GCC/4.7.2 was not a hierarchical module) self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['OpenMPI/1.6.4'])
def multidiff(base, files, colored=True): """ Generate a diff for multiple files, all compared to base. @param base: base to compare with @param files: list of files to compare with base @param colored: boolean indicating whether a colored multi-diff should be generated @return: text with multidiff overview """ differ = difflib.Differ() base_lines = read_file(base).split('\n') mdiff = MultiDiff(os.path.basename(base), base_lines, files, colored=colored) # use the MultiDiff class to store the information for filepath in files: lines = read_file(filepath).split('\n') diff = differ.compare(lines, base_lines) filename = os.path.basename(filepath) # contruct map of line number to diff lines and mapping between diff lines # example partial diff: # # - toolchain = {'name': 'goolfc', 'version': '2.6.10'} # ? - ^ ^ # # + toolchain = {'name': 'goolf', 'version': '1.6.20'} # ? ^ ^ # local_diff = {} squigly_dict = {} last_added = None offset = 1 for (i, line) in enumerate(diff): # diff line indicating changed characters on line above, a.k.a. a 'squigly' line if line.startswith(QUESTIONMARK): squigly_dict[last_added] = line offset -= 1 # diff line indicating addition change elif line.startswith(PLUS): local_diff.setdefault(i + offset, []).append((line, filename)) last_added = line # diff line indicated removal change elif line.startswith(MINUS): local_diff.setdefault(i + offset, []).append((line, filename)) last_added = line offset -= 1 # construct the multi-diff based on the constructed dict for line_no in local_diff: for (line, filename) in local_diff[line_no]: mdiff.parse_line(line_no, line.rstrip(), filename, squigly_dict.get(line, '').rstrip()) return str(mdiff)
def test_allow_modules_tool_mismatch(self): """Test allowing mismatch of modules tool with 'module' function.""" # make sure MockModulesTool is available from test.framework.modulestool import MockModulesTool ec_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') # keep track of original module definition so we can restore it orig_module = os.environ.get('module', None) # check whether mismatch between 'module' function and selected modules tool is detected os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" args = [ ec_file, '--modules-tool=MockModulesTool', ] self.eb_main(args, do_build=True) outtxt = read_file(self.logfile) error_regex = re.compile("ERROR .*pattern .* not found in defined 'module' function") self.assertTrue(error_regex.search(outtxt), "Found error w.r.t. module function mismatch: %s" % outtxt[-600:]) # check that --allow-modules-tool-mispatch transforms this error into a warning os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" args = [ ec_file, '--modules-tool=MockModulesTool', '--allow-modules-tool-mismatch', ] self.eb_main(args, do_build=True) outtxt = read_file(self.logfile) warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function") self.assertTrue(warn_regex.search(outtxt), "Found warning w.r.t. module function mismatch: %s" % outtxt[-600:]) # check whether match between 'module' function and selected modules tool is detected os.environ['module'] = "() { eval ` /bin/echo $*`\n}" args = [ ec_file, '--modules-tool=MockModulesTool', '--debug', ] self.eb_main(args, do_build=True) outtxt = read_file(self.logfile) found_regex = re.compile("DEBUG Found pattern .* in defined 'module' function") self.assertTrue(found_regex.search(outtxt), "Found debug message w.r.t. module function: %s" % outtxt[-600:]) # restore 'module' function if orig_module is not None: os.environ['module'] = orig_module else: del os.environ['module']
def test_fetch_parameters_from_easyconfig(self): """Test fetch_parameters_from_easyconfig function.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') toy_ec_file = os.path.join(test_ecs_dir, 'toy-0.0.eb') for ec_file, correct_name, correct_easyblock in [ (toy_ec_file, 'toy', None), (os.path.join(test_ecs_dir, 'goolf-1.4.10.eb'), 'goolf', 'Toolchain'), ]: name, easyblock = fetch_parameters_from_easyconfig(read_file(ec_file), ['name', 'easyblock']) self.assertEqual(name, correct_name) self.assertEqual(easyblock, correct_easyblock) self.assertEqual(fetch_parameters_from_easyconfig(read_file(toy_ec_file), ['description'])[0], "Toy C program.")
def test_is_yeb_format(self): """ Test is_yeb_format function """ testdir = os.path.dirname(os.path.abspath(__file__)) test_yeb = os.path.join(testdir, 'easyconfigs', 'yeb', 'bzip2-1.0.6-GCC-4.9.2.yeb') raw_yeb = read_file(test_yeb) self.assertTrue(is_yeb_format(test_yeb, None)) self.assertTrue(is_yeb_format(None, raw_yeb)) test_eb = os.path.join(testdir, 'easyconfigs', 'gzip-1.4.eb') raw_eb = read_file(test_eb) self.assertFalse(is_yeb_format(test_eb, None)) self.assertFalse(is_yeb_format(None, raw_eb))
def test_read_write_file(self): """Test reading/writing files.""" tmpdir = tempfile.mkdtemp() fp = os.path.join(tmpdir, 'test.txt') txt = "test123" ft.write_file(fp, txt) self.assertEqual(ft.read_file(fp), txt) txt2 = '\n'.join(['test', '123']) ft.write_file(fp, txt2, append=True) self.assertEqual(ft.read_file(fp), txt+txt2) shutil.rmtree(tmpdir)
def get_cpu_family(): """ Determine CPU family. @return: a value from the CPU_FAMILIES list """ family = None vendor = get_cpu_vendor() if vendor in CPU_FAMILIES: family = vendor _log.debug("Using vendor as CPU family: %s" % family) else: # POWER family needs to be determined indirectly via 'cpu' in /proc/cpuinfo if os.path.exists(PROC_CPUINFO_FP): cpuinfo_txt = read_file(PROC_CPUINFO_FP) power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M) if power_regex.search(cpuinfo_txt): family = POWER _log.debug("Determined CPU family using regex '%s' in %s: %s", power_regex.pattern, PROC_CPUINFO_FP, family) if family is None: family = UNKNOWN _log.warning("Failed to determine CPU family, returning %s" % family) return family
def build_and_install_software(module, options, origEnviron, exitOnFailure=True, silent=False): """ Build the software """ spec = module['spec'] print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) # restore original environment _log.info("Resetting environment") filetools.errorsFoundInLog = 0 modify_env(os.environ, origEnviron) cwd = os.getcwd() # load easyblock easyblock = options.easyblock if not easyblock: # try to look in .eb file reg = re.compile(r"^\s*easyblock\s*=(.*)$") txt = read_file(spec) for line in txt.split('\n'): match = reg.search(line) if match: easyblock = eval(match.group(1)) break name = module['module'][0] try: app_class = get_class(easyblock, name=name) app = app_class(spec, debug=options.debug, robot_path=options.robot) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError, err: print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent)
def configure_step(self): """Configure MATLAB installation: create license file.""" # create license file licserv = self.cfg['license_server'] licport = self.cfg['license_server_port'] lictxt = '\n'.join([ "SERVER %s 000000000000 %s" % (licserv, licport), "USE_SERVER", ]) licfile = os.path.join(self.builddir, 'matlab.lic') write_file(licfile, lictxt) try: shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile) config = read_file(self.configfile) regdest = re.compile(r"^# destinationFolder=.*", re.M) regkey = re.compile(r"^# fileInstallationKey=.*", re.M) regagree = re.compile(r"^# agreeToLicense=.*", re.M) regmode = re.compile(r"^# mode=.*", re.M) reglicpath = re.compile(r"^# licensePath=.*", re.M) config = regdest.sub("destinationFolder=%s" % self.installdir, config) key = self.cfg['key'] config = regkey.sub("fileInstallationKey=%s" % key, config) config = regagree.sub("agreeToLicense=Yes", config) config = regmode.sub("mode=silent", config) config = reglicpath.sub("licensePath=%s" % licfile, config) write_file(self.configfile, config) except IOError, err: raise EasyBuildError("Failed to create installation config file %s: %s", self.configfile, err)
def get_cpu_vendor(): """ Try to detect the CPU vendor @return: a value from the VENDORS dict """ vendor = None os_type = get_os_type() if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP): txt = read_file(PROC_CPUINFO_FP) arch = UNKNOWN vendor_regex = re.compile(r"(vendor_id.*?)?\s*:\s*(?P<vendor>(?(1)\S+|(?:IBM|ARM)))") res = vendor_regex.search(txt) if res: arch = res.group('vendor') if arch in VENDORS: vendor = VENDORS[arch] _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s", vendor, vendor_regex.pattern, PROC_CPUINFO_FP) elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" out, ec = run_cmd(cmd, force_in_dry_run=True) out = out.strip() if ec == 0 and out in VENDORS: vendor = VENDORS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) if vendor is None: vendor = UNKNOWN _log.warning("Could not determine CPU vendor on %s, returning %s" % (os_type, vendor)) return vendor
def get_cpu_model(): """ Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz """ model = None os_type = get_os_type() if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP): # we need 'model name' on Linux/x86, but 'model' is there first with different info # 'model name' is not there for Linux/POWER, but 'model' has the right info model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P<model>.*[A-Za-z].+)\s*$", re.M) txt = read_file(PROC_CPUINFO_FP) res = model_regex.search(txt) if res is not None: model = res.group('model').strip() _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s", model_regex.pattern, PROC_CPUINFO_FP, model) elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" out, ec = run_cmd(cmd, force_in_dry_run=True) if ec == 0: model = out.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) if model is None: model = UNKNOWN _log.warning("Failed to determine CPU model, returning %s" % model) return model
def process_easyconfig_file(ec_file): """Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested).""" ectxt = read_file(ec_file) name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock']) derived_easyblock_class = get_easyblock_class(easyblock, name=name, default_fallback=False) fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class) if ectxt != fixed_ectxt: if go.options.backup: try: backup_ec_file = '%s.bk' % ec_file i = 1 while os.path.exists(backup_ec_file): backup_ec_file = '%s.bk%d' % (ec_file, i) i += 1 os.rename(ec_file, backup_ec_file) log.info("Backed up %s to %s" % (ec_file, backup_ec_file)) except OSError, err: raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err) write_file(ec_file, fixed_ectxt) log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt) log.info("%s: fixed" % ec_file)
def get_cpu_vendor(): """Try to detect the cpu identifier will return INTEL, ARM or AMD constant """ regexp = re.compile(r"^vendor_id\s+:\s*(?P<vendorid>\S+)\s*$", re.M) VENDORS = { 'GenuineIntel': INTEL, 'AuthenticAMD': AMD, } os_type = get_os_type() if os_type == LINUX: try: txt = read_file('/proc/cpuinfo', log_error=False) arch = UNKNOWN # vendor_id might not be in the /proc/cpuinfo, so this might fail res = regexp.search(txt) if res: arch = res.groupdict().get('vendorid', UNKNOWN) if arch in VENDORS: return VENDORS[arch] # some embeded linux on arm behaves differently (e.g. raspbian) regexp = re.compile(r"^Processor\s+:\s*(?P<vendorid>ARM\S+)\s*", re.M) res = regexp.search(txt) if res: arch = res.groupdict().get('vendorid', UNKNOWN) if ARM in arch: return ARM except IOError, err: raise SystemToolsException("An error occured while determining CPU vendor since: %s" % err)
def parse_components_list(self): """parse the regex in the components extra_options and select the matching components from the mediaconfig.xml file in the install dir""" mediaconfigpath = os.path.join(self.cfg['start_dir'], 'pset', 'mediaconfig.xml') if not os.path.isfile(mediaconfigpath): raise EasyBuildError("Could not find %s to find list of components." % mediaconfigpath) mediaconfig = read_file(mediaconfigpath) available_components = re.findall("<Abbr>(?P<component>[^<]+)</Abbr>", mediaconfig, re.M) self.log.debug("Intel components found: %s" % available_components) self.log.debug("Using regex list: %s" % self.cfg['components']) if COMP_ALL in self.cfg['components'] or COMP_DEFAULTS in self.cfg['components']: if len(self.cfg['components']) == 1: self.install_components = self.cfg['components'] else: raise EasyBuildError("If you specify %s as components, you cannot specify anything else: %s", ' or '.join([COMP_ALL, COMP_DEFAULTS]), self.cfg['components']) else: self.install_components = [] for comp_regex in self.cfg['components']: comps = [comp for comp in available_components if re.match(comp_regex, comp)] self.install_components.extend(comps) self.log.debug("Components to install: %s" % self.install_components)
def test_robot_archived_easyconfigs(self): """Test whether robot can pick up archived easyconfigs when asked.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb') gzip_ectxt = read_file(gzip_ec) test_ec = os.path.join(self.test_prefix, 'test.eb') tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}" regex = re.compile("^toolchain = .*", re.M) test_ectxt = regex.sub(tc_spec, gzip_ectxt) write_file(test_ec, test_ectxt) ecs, _ = parse_easyconfigs([(test_ec, False)]) self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies, ecs, self.modtool, retain_all_deps=True) # --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs init_config(build_options={ 'consider_archived_easyconfigs': True, 'robot_path': [test_ecs], }) res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True) self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3']) expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb') self.assertTrue(os.path.samefile(res[0]['spec'], expected))
def add_easyconfig(self, cfg, name, version, stats, previous): """ Add the eb-file for software name and version to the repository. stats should be a dict containing statistics. if previous is true -> append the statistics to the file This will return the path to the created file (for use in subclasses) """ # create directory for eb file full_path = os.path.join(self.wc, self.subdir, name) mkdir(full_path, parents=True) # destination dest = os.path.join(full_path, "%s-%s.eb" % (name, version)) txt = "# Built with EasyBuild version %s on %s\n" % (VERBOSE_VERSION, time.strftime("%Y-%m-%d_%H-%M-%S")) # copy file txt += read_file(cfg) # append a line to the eb file so that we don't have git merge conflicts if not previous: statsprefix = "\n# Build statistics\nbuildstats = [" statssuffix = "]\n" else: # statstemplate = "\nbuildstats.append(%s)\n" statsprefix = "\nbuildstats.append(" statssuffix = ")\n" txt += statsprefix + stats_to_str(stats) + statssuffix write_file(dest, txt) return dest
def test_recursive_module_unload(self): """Test generating recursively unloading modules.""" # use temporary paths for build/install paths, make sure sources can be found buildpath = tempfile.mkdtemp() installpath = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp() sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') # use toy-0.0.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb') # check log message with --skip for existing module args = [ eb_file, '--sourcepath=%s' % sourcepath, '--buildpath=%s' % buildpath, '--installpath=%s' % installpath, '--debug', '--force', '--recursive-module-unload', ] self.eb_main(args, do_build=True, verbose=True) toy_module = os.path.join(installpath, 'modules', 'all', 'toy', '0.0-deps') toy_module_txt = read_file(toy_module) is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M) self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) # cleanup shutil.rmtree(buildpath) shutil.rmtree(installpath) shutil.rmtree(tmpdir)
def test_apply_regex_substitutions(self): """Test apply_regex_substitutions function.""" testfile = os.path.join(self.test_prefix, 'test.txt') testtxt = '\n'.join([ "CC = gcc", "CFLAGS = -O3 -g", "FC = gfortran", "FFLAGS = -O3 -g -ffixed-form", ]) ft.write_file(testfile, testtxt) regex_subs = [ (r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"), (r"^(FC\s*=\s*).*$", r"\1${FC}"), (r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"), ] ft.apply_regex_substitutions(testfile, regex_subs) expected_testtxt = '\n'.join([ "CC = ${CC}", "CFLAGS = -O2", "FC = ${FC}", "FFLAGS = -O2 -ffixed-form", ]) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, expected_testtxt)
def get_total_memory(): """ Try to ascertain this node's total memory :return: total memory as an integer, specifically a number of megabytes """ memtotal = None os_type = get_os_type() if os_type == LINUX and is_readable(PROC_MEMINFO_FP): _log.debug("Trying to determine total memory size on Linux via %s", PROC_MEMINFO_FP) meminfo = read_file(PROC_MEMINFO_FP) mem_mo = re.match(r'^MemTotal:\s*(\d+)\s*kB', meminfo, re.M) if mem_mo: memtotal = int(mem_mo.group(1)) / 1024 elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) if ec == 0: memtotal = int(out.strip()) / (1024**2) if memtotal is None: memtotal = UNKNOWN _log.warning("Failed to determine total memory, returning %s", memtotal) return memtotal
def configure_step(self): """Configure MCR installation: create license file.""" configfile = os.path.join(self.builddir, self.configfilename) if LooseVersion(self.version) < LooseVersion('R2015a'): shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), configfile) config = read_file(configfile) # compile regex first since re.sub doesn't accept re.M flag for multiline regex in Python 2.6 regdest = re.compile(r"^# destinationFolder=.*", re.M) regagree = re.compile(r"^# agreeToLicense=.*", re.M) regmode = re.compile(r"^# mode=.*", re.M) config = regdest.sub("destinationFolder=%s" % self.installdir, config) config = regagree.sub("agreeToLicense=Yes", config) config = regmode.sub("mode=silent", config) else: config = '\n'.join([ "destinationFolder=%s" % self.installdir, "agreeToLicense=Yes", "mode=silent", ]) write_file(configfile, config) self.log.debug("configuration file written to %s:\n %s", configfile, config)
def sanity_check_step(self): """Custom sanity check for CUDA.""" if LooseVersion(self.version) > LooseVersion("9"): versionfile = read_file(os.path.join(self.installdir, "version.txt")) if not re.search("Version %s$" % self.version, versionfile): raise EasyBuildError("Unable to find the correct version (%s) in the version.txt file", self.version) shlib_ext = get_shared_lib_ext() chk_libdir = ["lib64"] # Versions higher than 6 do not provide 32 bit libraries if LooseVersion(self.version) < LooseVersion("6"): chk_libdir += ["lib"] culibs = ["cublas", "cudart", "cufft", "curand", "cusparse"] custom_paths = { 'files': [os.path.join("bin", x) for x in ["fatbinary", "nvcc", "nvlink", "ptxas"]] + [os.path.join("%s", "lib%s.%s") % (x, y, shlib_ext) for x in chk_libdir for y in culibs], 'dirs': ["include"], } if LooseVersion(self.version) < LooseVersion('7'): custom_paths['files'].append(os.path.join('open64', 'bin', 'nvopencc')) if LooseVersion(self.version) >= LooseVersion('7'): custom_paths['files'].append(os.path.join("extras", "CUPTI", "lib64", "libcupti.%s") % shlib_ext) custom_paths['dirs'].append(os.path.join("extras", "CUPTI", "include")) super(EB_CUDA, self).sanity_check_step(custom_paths=custom_paths)
def test_job(self): """Test submitting build as a job.""" # set MODULEPATH to included modules orig_modulepath = os.getenv('MODULEPATH', None) os.environ['MODULEPATH'] = os.path.join(os.path.dirname(__file__), 'modules') # use gzip-1.4.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4.eb') # check log message with --job for job_args in [ # options passed are reordered, so order here matters to make tests pass ['--debug'], ['--debug', '--stop=configure', '--try-software-name=foo'], ]: # clear log file outtxt = write_file(self.logfile, '') args = [ eb_file, '--job', ] + job_args try: main((args, self.logfile)) except (SystemExit, Exception), err: pass outtxt = read_file(self.logfile) job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s %s.*\n" % ' .*'.join(job_args) assertmsg = "Info log message with job command template when using --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) self.assertTrue(re.search(job_msg, outtxt), assertmsg)
def dependencies_for(self, mod_name, depth=sys.maxint): """ Obtain a list of dependencies for the given module, determined recursively, up to a specified depth (optionally) """ modfilepath = self.modulefile_path(mod_name) self.log.debug("modulefile path %s: %s" % (mod_name, modfilepath)) modtxt = read_file(modfilepath) loadregex = re.compile(r"^\s+module load\s+(.*)$", re.M) mods = loadregex.findall(modtxt) if depth > 0: # recursively determine dependencies for these dependency modules, until depth is non-positive moddeps = [self.dependencies_for(mod, depth=depth - 1) for mod in mods] else: # ignore any deeper dependencies moddeps = [] # add dependencies of dependency modules only if they're not there yet for moddepdeps in moddeps: for dep in moddepdeps: if not dep in mods: mods.append(dep) return mods
def get_cpu_vendor(): """Try to detect the cpu identifier will return INTEL or AMD constant """ regexp = re.compile(r"^vendor_id\s+:\s*(?P<vendorid>\S+)\s*$", re.M) VENDORS = {"GenuineIntel": INTEL, "AuthenticAMD": AMD} # Linux txt = read_file("/proc/cpuinfo", log_error=False) if txt is not None: arch = regexp.search(txt).groupdict()["vendorid"] if arch in VENDORS: return VENDORS[arch] # Darwin (OS X) out, exitcode = run_cmd("sysctl -n machdep.cpu.vendor") out = out.strip() if not exitcode and out and out in VENDORS: return VENDORS[out] # BSD out, exitcode = run_cmd("sysctl -n hw.model") out = out.strip() if not exitcode and out: return out.split(" ")[0] raise SystemToolsException("Could not detect cpu vendor")
def test_run_cmd_qa_log_all(self): """Test run_cmd_qa with log_output enabled""" (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}, log_all=True) self.assertEqual(ec, 0) self.assertEquals(out, "n: \n1\n2\n3\n4\n5\n") run_cmd_logs = glob.glob( os.path.join(self.test_prefix, '*', 'easybuild-run_cmd_qa*.log')) self.assertEqual(len(run_cmd_logs), 1) run_cmd_log_txt = read_file(run_cmd_logs[0]) extra_pref = "# output for interactive command: echo 'n: '; read n; seq 1 $n\n\n" self.assertEquals(run_cmd_log_txt, extra_pref + "n: \n1\n2\n3\n4\n5\n")
def test_copy_file(self): """ Test copy_file """ testdir = os.path.dirname(os.path.abspath(__file__)) tmpdir = self.test_prefix to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') target_path = os.path.join(tmpdir, 'toy.eb') ft.copy_file(to_copy, target_path) self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) # also test behaviour of copy_file under --dry-run build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) # remove target file, it shouldn't get copied under dry run os.remove(target_path) self.mock_stdout(True) ft.copy_file(to_copy, target_path) txt = self.get_stdout() self.mock_stdout(False) self.assertFalse(os.path.exists(target_path)) self.assertTrue( re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt)) # forced copy, even in dry run mode self.mock_stdout(True) ft.copy_file(to_copy, target_path, force_in_dry_run=True) txt = self.get_stdout() self.mock_stdout(False) self.assertTrue(os.path.exists(target_path)) self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path)) self.assertEqual(txt, '')
def configure_step(self): """Configure MATLAB installation: create license file.""" licserv = self.cfg['license_server'] if licserv is None: licserv = os.getenv('EB_MATLAB_LICENSE_SERVER', 'license.example.com') licport = self.cfg['license_server_port'] if licport is None: licport = os.getenv('EB_MATLAB_LICENSE_SERVER_PORT', '00000') key = self.cfg['key'] if key is None: key = os.getenv( 'EB_MATLAB_KEY', '00000-00000-00000-00000-00000-00000-00000-00000-00000-00000') # create license file lictxt = '\n'.join([ "SERVER %s 000000000000 %s" % (licserv, licport), "USE_SERVER", ]) licfile = os.path.join(self.builddir, 'matlab.lic') write_file(licfile, lictxt) try: shutil.copyfile( os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile) config = read_file(self.configfile) regdest = re.compile(r"^# destinationFolder=.*", re.M) regkey = re.compile(r"^# fileInstallationKey=.*", re.M) regagree = re.compile(r"^# agreeToLicense=.*", re.M) regmode = re.compile(r"^# mode=.*", re.M) reglicpath = re.compile(r"^# licensePath=.*", re.M) config = regdest.sub("destinationFolder=%s" % self.installdir, config) config = regkey.sub("fileInstallationKey=%s" % key, config) config = regagree.sub("agreeToLicense=Yes", config) config = regmode.sub("mode=silent", config) config = reglicpath.sub("licensePath=%s" % licfile, config) write_file(self.configfile, config) except IOError, err: raise EasyBuildError( "Failed to create installation config file %s: %s", self.configfile, err)
def test_apply_regex_substitutions(self): """Test apply_regex_substitutions function.""" testfile = os.path.join(self.test_prefix, 'test.txt') testtxt = '\n'.join([ "CC = gcc", "CFLAGS = -O3 -g", "FC = gfortran", "FFLAGS = -O3 -g -ffixed-form", ]) ft.write_file(testfile, testtxt) regex_subs = [ (r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"), (r"^(FC\s*=\s*).*$", r"\1${FC}"), (r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"), ] ft.apply_regex_substitutions(testfile, regex_subs) expected_testtxt = '\n'.join([ "CC = ${CC}", "CFLAGS = -O2", "FC = ${FC}", "FFLAGS = -O2 -ffixed-form", ]) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, expected_testtxt) # passing empty list of substitions is a no-op ft.write_file(testfile, testtxt) ft.apply_regex_substitutions(testfile, []) new_testtxt = ft.read_file(testfile) self.assertEqual(new_testtxt, testtxt) # clean error on non-existing file error_pat = "Failed to patch .*/nosuchfile.txt: .*No such file or directory" path = os.path.join(self.test_prefix, 'nosuchfile.txt') self.assertErrorRegex(EasyBuildError, error_pat, ft.apply_regex_substitutions, path, regex_subs)
def get_cpu_vendor(): """ Try to detect the CPU vendor :return: a value from the CPU_VENDORS list """ vendor = None os_type = get_os_type() if os_type == LINUX: vendor_regex = None arch = get_cpu_architecture() if arch == X86_64: vendor_regex = re.compile(r"vendor_id\s+:\s*(\S+)") elif arch == POWER: vendor_regex = re.compile(r"model\s+:\s*(\w+)") elif arch in [AARCH32, AARCH64]: vendor_regex = re.compile(r"CPU implementer\s+:\s*(\S+)") if vendor_regex and is_readable(PROC_CPUINFO_FP): vendor_id = None proc_cpuinfo = read_file(PROC_CPUINFO_FP) res = vendor_regex.search(proc_cpuinfo) if res: vendor_id = res.group(1) if vendor_id in VENDOR_IDS: vendor = VENDOR_IDS[vendor_id] _log.debug( "Determined CPU vendor on Linux as being '%s' via regex '%s' in %s", vendor, vendor_regex.pattern, PROC_CPUINFO_FP) elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" out, ec = run_cmd(cmd, force_in_dry_run=True) out = out.strip() if ec == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug( "Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) if vendor is None: vendor = UNKNOWN _log.warning("Could not determine CPU vendor on %s, returning %s" % (os_type, vendor)) return vendor
def test_exclude_path_to_top_of_module_tree(self): """ Make sure that modules under the HierarchicalMNS are correct, w.r.t. not including any load statements for modules that build up the path to the top of the module tree. """ self.orig_module_naming_scheme = config.get_module_naming_scheme() test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [test_ecs_path], 'valid_stops': all_stops, 'validate': False, } os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' init_config(build_options=build_options) self.setup_hierarchical_modules() modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all') mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True) mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True) impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049') imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144') if get_module_syntax() == 'Lua': impi_modfile_path += '.lua' imkl_modfile_path += '.lua' # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included # not for the toolchain or any of the toolchain components, # since both icc/ifort and impi form the path to the top of the module tree tests = [ ('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']), ('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']), ] for ec_file, modfile_path, excluded_deps in tests: ec = EasyConfig(os.path.join(test_ecs_path, ec_file)) eb = EasyBlock(ec) eb.toolchain.prepare() modpath = eb.make_module_step() modfile_path = os.path.join(modpath, modfile_path) modtxt = read_file(modfile_path) for imkl_dep in excluded_deps: tup = (imkl_dep, modfile_path, modtxt) failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup self.assertFalse(re.search("module load %s" % imkl_dep, modtxt), failmsg) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme init_config(build_options=build_options)
def get_system_libs_from_tf(source_dir): """Return the valid values for TF_SYSTEM_LIBS from the TensorFlow source directory""" syslibs_path = os.path.join(source_dir, 'third_party', 'systemlibs', 'syslibs_configure.bzl') result = [] if os.path.exists(syslibs_path): txt = read_file(syslibs_path) valid_libs_match = re.search(r'VALID_LIBS\s*=\s*\[(.*?)\]', txt, re.DOTALL) if not valid_libs_match: raise EasyBuildError('VALID_LIBS definition not found in %s', syslibs_path) result = split_tf_libs_txt(valid_libs_match.group(1)) return result
def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False, reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True): """Helper method to call EasyBuild main function.""" cleanup() myerr = False if logfile is None: logfile = self.logfile # clear log file if logfile: f = open(logfile, 'w') f.write('') f.close() env_before = copy.deepcopy(os.environ) try: main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=self.modtool) except SystemExit as err: if raise_systemexit: raise err except Exception as err: myerr = err if verbose: print("err: %s" % err) if logfile and os.path.exists(logfile): logtxt = read_file(logfile) else: logtxt = None os.chdir(self.cwd) if redo_init_config: # make sure config is reinitialized init_config(with_include=False) # restore environment to what it was before running main, # changes may have been made by eb_main (e.g. $TMPDIR & co) if reset_env: modify_env(os.environ, env_before, verbose=False) tempfile.tempdir = None if myerr and raise_error: raise myerr if return_error: return logtxt, myerr else: return logtxt
def test_get_easyblock_instance(self): """Test get_easyblock_instance function.""" from easybuild.easyblocks.toy import EB_toy testdir = os.path.abspath(os.path.dirname(__file__)) ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) self.assertTrue(isinstance(eb, EB_toy)) # check whether 'This is easyblock' log message is there tup = ('EB_toy', 'easybuild.easyblocks.toy', '.*test/framework/sandbox/easybuild/easyblocks/t/toy.pyc*') eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) logtxt = read_file(eb.logfile) self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt))
def patch_step(self, *args, **kwargs): """ Custom patch step for Python: * patch setup.py when --sysroot EasyBuild configuration setting is used """ super(EB_Python, self).patch_step(*args, **kwargs) # if we're installing Python with an alternate sysroot, # we need to patch setup.py which includes hardcoded paths like /usr/include and /lib64; # this fixes problems like not being able to build the _ssl module ("Could not build the ssl module") sysroot = build_option('sysroot') if sysroot: sysroot_inc_dirs, sysroot_lib_dirs = [], [] for pattern in ['include*', os.path.join('usr', 'include*')]: sysroot_inc_dirs.extend(glob.glob(os.path.join(sysroot, pattern))) if sysroot_inc_dirs: sysroot_inc_dirs = ', '.join(["'%s'" % x for x in sysroot_inc_dirs]) else: raise EasyBuildError("No include directories found in sysroot %s!", sysroot) for pattern in ['lib*', os.path.join('usr', 'lib*')]: sysroot_lib_dirs.extend(glob.glob(os.path.join(sysroot, pattern))) if sysroot_lib_dirs: sysroot_lib_dirs = ', '.join(["'%s'" % x for x in sysroot_lib_dirs]) else: raise EasyBuildError("No lib directories found in sysroot %s!", sysroot) setup_py_fn = 'setup.py' setup_py_txt = read_file(setup_py_fn) # newer Python versions (3.6+) have refactored code, requires different patching approach if "system_include_dirs = " in setup_py_txt: regex_subs = [ (r"(system_include_dirs = \[).*\]", r"\1%s]" % sysroot_inc_dirs), (r"(system_lib_dirs = \[).*\]", r"\1%s]" % sysroot_lib_dirs), ] else: regex_subs = [ (r"^([ ]+)'/usr/include',", r"\1%s," % sysroot_inc_dirs), (r"\['/usr/include'\]", r"[%s]" % sysroot_inc_dirs), (r"^([ ]+)'/lib64', '/usr/lib64',", r"\1%s," % sysroot_lib_dirs), (r"^[ ]+'/lib', '/usr/lib',", ''), ] apply_regex_substitutions(setup_py_fn, regex_subs)
def test_parallel(self): """Test defining of parallellism.""" topdir = os.path.abspath(os.path.dirname(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') toytxt = read_file(toy_ec) handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) write_file(toy_ec1, toytxt + "\nparallel = 123") handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(handle) write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67") # default: parallellism is derived from # available cores + ulimit test_eb = EasyBlock(EasyConfig(toy_ec)) test_eb.check_readiness_step() self.assertTrue( isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0) # only 'parallel' easyconfig parameter specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec1)) test_eb.check_readiness_step() self.assertEqual(test_eb.cfg['parallel'], 123) # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() self.assertEqual(test_eb.cfg['parallel'], 67) # only 'parallel' build option specified init_config(build_options={'parallel': '97', 'validate': False}) test_eb = EasyBlock(EasyConfig(toy_ec)) test_eb.check_readiness_step() self.assertEqual(test_eb.cfg['parallel'], 97) # both 'parallel' build option and easyconfig parameter specified (no 'maxparallel') test_eb = EasyBlock(EasyConfig(toy_ec1)) test_eb.check_readiness_step() self.assertEqual(test_eb.cfg['parallel'], 97) # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option test_eb = EasyBlock(EasyConfig(toy_ec2)) test_eb.check_readiness_step() self.assertEqual(test_eb.cfg['parallel'], 67)
def test_dry_run(self): """Test use of functions under (extended) dry run.""" build_options = { 'extended_dry_run': True, 'silent': False, } init_config(build_options=build_options) self.mock_stdout(True) run_cmd("somecommand foo 123 bar") txt = self.get_stdout() self.mock_stdout(False) expected_regex = re.compile('\n'.join([ r" running command \"somecommand foo 123 bar\"", r" \(in .*\)", ])) self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt)) # check disabling 'verbose' self.mock_stdout(True) run_cmd("somecommand foo 123 bar", verbose=False) txt = self.get_stdout() self.mock_stdout(False) self.assertEqual(txt, '') # check forced run outfile = os.path.join(self.test_prefix, 'cmd.out') self.assertFalse(os.path.exists(outfile)) self.mock_stdout(True) run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True) txt = self.get_stdout() self.mock_stdout(False) # nothing printed to stdout, but command was run self.assertEqual(txt, '') self.assertTrue(os.path.exists(outfile)) self.assertEqual(read_file(outfile), "This is always echoed\n") # Q&A commands self.mock_stdout(True) run_cmd_qa("some_qa_cmd", {'question1': 'answer1'}) txt = self.get_stdout() self.mock_stdout(False) expected_regex = re.compile('\n'.join([ r" running interactive command \"some_qa_cmd\"", r" \(in .*\)", ])) self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt))
def configure_step(self): """Configure COMSOL installation: create license file.""" default_lic_env_var = 'LMCOMSOL_LICENSE_FILE' lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var], lic_specs=[self.cfg['license_file']]) if lic_specs: if self.license_env_var is None: self.log.info("Using COMSOL license specifications from 'license_file': %s", lic_specs) self.license_env_var = default_lic_env_var else: self.log.info("Using COMSOL license specifications from $%s: %s", self.license_env_var, lic_specs) self.license_file = os.pathsep.join(lic_specs) env.setvar(self.license_env_var, self.license_file) else: msg = "No viable license specifications found; " msg += "specify 'license_file', or define $%s" % default_lic_env_var raise EasyBuildError(msg) copy_file(os.path.join(self.start_dir, 'setupconfig.ini'), self.configfile) config = read_file(self.configfile) config_vars = { 'agree': '1', 'desktopshortcuts': '0', 'fileassoc': '0', 'firewall': '0', 'installdir': self.installdir, 'license': self.license_file, 'licmanager': '0', 'linuxlauncher': '0', 'showgui': '0', 'startmenushortcuts': '0', 'symlinks': '0', } matlab_root = get_software_root("MATLAB") if matlab_root: config_vars.update({'matlabdir': matlab_root}) for key, val in config_vars.items(): regex = re.compile(r"^%s\s*=.*" % key, re.M) config = regex.sub("%s=%s" % (key, val), config) write_file(self.configfile, config) self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def add_easyconfig(self, cfg, name, version, stats, previous): """ Add easyconfig to repository :param cfg: location of easyconfig file :param name: software name :param version: software install version, incl. toolchain & versionsuffix :param stats: build stats, to add to archived easyconfig :param previous: list of previous build stats :return: location of archived easyconfig """ # create directory for eb file full_path = os.path.join(self.wc, self.subdir, name) yeb_format = is_yeb_format(cfg, None) if yeb_format: extension = YEB_FORMAT_EXTENSION prefix = "buildstats: [" else: extension = EB_FORMAT_EXTENSION prefix = "buildstats = [" # destination dest = os.path.join(full_path, "%s-%s%s" % (name, version, extension)) txt = "# Built with EasyBuild version %s on %s\n" % ( VERBOSE_VERSION, time.strftime("%Y-%m-%d_%H-%M-%S")) # copy file txt += read_file(cfg) # append a line to the eb file so that we don't have git merge conflicts statscomment = "\n# Build statistics\n" statsprefix = prefix statssuffix = "]\n" if previous: statstxt = statscomment + statsprefix + '\n' for entry in previous + [stats]: statstxt += stats_to_str(entry, isyeb=yeb_format) + ',\n' statstxt += statssuffix else: statstxt = statscomment + statsprefix + stats_to_str( stats, isyeb=yeb_format) + statssuffix txt += statstxt write_file(dest, txt) return dest
def test_toy_tweaked(self): """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') ec_file = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') shutil.copy2(os.path.join(test_ecs_dir, 'toy-0.0.eb'), ec_file) # tweak easyconfig by appending to it ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = 'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'", "modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua "modluafooter = 'io.stderr:write(\"oh hai!\")'" # ignored when module syntax is Tcl ]) write_file(ec_file, ec_extra, append=True) args = [ ec_file, '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', '--force', ] outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.check_toy(self.test_installpath, outtxt, versionsuffix='-tweaked') toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked') if get_module_syntax() == 'Lua': toy_module += '.lua' toy_module_txt = read_file(toy_module) if get_module_syntax() == 'Tcl': self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M)) self.assertTrue(re.search(r'module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0"$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) elif get_module_syntax() == 'Lua': self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^if mode\(\) == "load" then\n\s*io.stderr:write\(".*I AM toy v0.0"\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^io.stderr:write\("oh hai!"\)$', toy_module_txt, re.M)) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
def test_download_repo(self): """Test download_repo function.""" if self.skip_github_tests: print("Skipping test_download_repo, no GitHub token available?") return # default: download tarball for master branch of easybuilders/easybuild-easyconfigs repo path = gh.download_repo(path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) repodir = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-master') self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue(os.path.exists(repodir)) shafile = os.path.join(repodir, 'latest-sha') self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2019b.eb'))) # existing downloaded repo is not reperformed, except if SHA is different account, repo, branch = 'boegel', 'easybuild-easyblocks', 'develop' repodir = os.path.join(self.test_prefix, account, '%s-%s' % (repo, branch)) latest_sha = gh.fetch_latest_commit_sha(repo, account, branch=branch, github_user=GITHUB_TEST_ACCOUNT) # put 'latest-sha' fail in place, check whether repo was (re)downloaded (should not) shafile = os.path.join(repodir, 'latest-sha') write_file(shafile, latest_sha) path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) self.assertTrue(os.path.samefile(path, repodir)) self.assertEqual(os.listdir(repodir), ['latest-sha']) # remove 'latest-sha' file and verify that download was performed os.remove(shafile) path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) self.assertTrue(os.path.samefile(path, repodir)) self.assertTrue('easybuild' in os.listdir(repodir)) self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')))
def test_patch_step(self): """Test patch step.""" test_easyconfigs = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = process_easyconfig( os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] orig_sources = ec['ec']['sources'][:] toy_patches = [ 'toy-0.0_typo.patch', # test for applying patch ('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy ] self.assertEqual(ec['ec']['patches'], toy_patches) # test applying patches without sources ec['ec']['sources'] = [] eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step) # test actual patching of unpacked sources ec['ec']['sources'] = orig_sources eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() eb.patch_step() # verify that patches were applied toydir = os.path.join(eb.builddir, 'toy-0.0') self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig']) self.assertTrue("and very proud of it" in read_file( os.path.join(toydir, 'toy.source'))) self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n')
def test_raw(self): """Test passing of raw contents to EasyConfigParser.""" ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb') ec_txt1 = read_file(ec_file1) ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'gzip-1.5-goolf-1.4.10.eb') ec_txt2 = read_file(ec_file2) ecparser = EasyConfigParser(ec_file1) self.assertEqual(ecparser.rawcontent, ec_txt1) ecparser = EasyConfigParser(rawcontent=ec_txt2) self.assertEqual(ecparser.rawcontent, ec_txt2) # rawcontent supersedes passed filepath ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2) self.assertEqual(ecparser.rawcontent, ec_txt2) ec = ecparser.get_config_dict() self.assertEqual(ec['name'], 'gzip') self.assertEqual(ec['toolchain']['name'], 'goolf') self.assertErrorRegex(EasyBuildError, "Neither filename nor rawcontent provided", EasyConfigParser)
def get_cpu_model(): """ returns cpu model f.ex Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz """ os_type = get_os_type() if os_type == LINUX: regexp = re.compile(r"^model name\s+:\s*(?P<modelname>.+)\s*$", re.M) try: txt = read_file('/proc/cpuinfo', log_error=False) if txt is not None: return regexp.search(txt).groupdict()['modelname'].strip() except IOError, err: raise SystemToolsException( "An error occured when determining CPU model: %s" % err)
def post_install_step(self): """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there.""" setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth') if os.path.exists(setuptools_pth): setuptools_pth_txt = read_file(setuptools_pth) # any line that starts with '/' is a sign of trouble sys_path_regex = re.compile('^/', re.M) if sys_path_regex.search(setuptools_pth_txt): self.log.warning("Found %s, and includes one or more absolute system paths. Removing it.", setuptools_pth) try: os.remove(setuptools_pth) except OSError as err: raise EasyBuildError("Failed to remove %s: %s", setuptools_pth, err)
def test_footer(self): """Test specifying a module footer.""" # use temporary paths for build/install paths, make sure sources can be found buildpath = tempfile.mkdtemp() installpath = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp() sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') # create file containing modules footer module_footer_txt = '\n'.join([ "# test footer", "setenv SITE_SPECIFIC_ENV_VAR foobar", ]) fd, modules_footer = tempfile.mkstemp(prefix='modules-footer-') os.close(fd) f = open(modules_footer, 'w') f.write(module_footer_txt) f.close() # use toy-0.0.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') # check log message with --skip for existing module args = [ eb_file, '--sourcepath=%s' % sourcepath, '--buildpath=%s' % buildpath, '--installpath=%s' % installpath, '--debug', '--force', '--modules-footer=%s' % modules_footer, ] self.eb_main(args, do_build=True) toy_module = os.path.join(installpath, 'modules', 'all', 'toy', '0.0') toy_module_txt = read_file(toy_module) footer_regex = re.compile(r'%s$' % module_footer_txt, re.M) msg = "modules footer '%s' is present in '%s'" % (module_footer_txt, toy_module_txt) self.assertTrue(footer_regex.search(toy_module_txt), msg) # cleanup shutil.rmtree(buildpath) shutil.rmtree(installpath) shutil.rmtree(tmpdir) os.remove(modules_footer)
def get_cpu_model(): """ Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz """ model = None os_type = get_os_type() if os_type == LINUX and is_readable(PROC_CPUINFO_FP): proc_cpuinfo = read_file(PROC_CPUINFO_FP) arch = get_cpu_architecture() if arch in [AARCH32, AARCH64]: # On ARM platforms, no model name is provided in /proc/cpuinfo. However, for vanilla ARM cores # we can reverse-map the part number. vendor = get_cpu_vendor() if vendor == ARM: model_regex = re.compile(r"CPU part\s+:\s*(\S+)", re.M) # There can be big.LITTLE setups with different types of cores! model_ids = model_regex.findall(proc_cpuinfo) if model_ids: id_list = [] for model_id in sorted(set(model_ids)): id_list.append(ARM_CORTEX_IDS.get(model_id, UNKNOWN)) model = vendor + ' ' + ' + '.join(id_list) _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s", model_regex.pattern, PROC_CPUINFO_FP, model) else: # we need 'model name' on Linux/x86, but 'model' is there first with different info # 'model name' is not there for Linux/POWER, but 'model' has the right info model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P<model>.*[A-Za-z].+)\s*$", re.M) res = model_regex.search(proc_cpuinfo) if res is not None: model = res.group('model').strip() _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s", model_regex.pattern, PROC_CPUINFO_FP, model) elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) if ec == 0: model = out.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) if model is None: model = UNKNOWN _log.warning("Failed to determine CPU model, returning %s" % model) return model
def test_run_cmd_log_output(self): """Test run_cmd with log_output enabled""" (out, ec) = run_cmd("seq 1 100", log_output=True) self.assertEqual(ec, 0) self.assertTrue(out.startswith("1\n2\n")) self.assertTrue(out.endswith("99\n100\n")) run_cmd_logs = glob.glob( os.path.join(self.test_prefix, '*', 'easybuild-run_cmd*.log')) self.assertEqual(len(run_cmd_logs), 1) run_cmd_log_txt = read_file(run_cmd_logs[0]) self.assertTrue( run_cmd_log_txt.startswith("# output for command: seq 1 100\n\n")) run_cmd_log_lines = run_cmd_log_txt.split('\n') self.assertEqual(run_cmd_log_lines[2:5], ['1', '2', '3']) self.assertEqual(run_cmd_log_lines[-4:-1], ['98', '99', '100'])
def configure_step(self): """Configure MATLAB installation: create license file.""" licfile = self.cfg['license_file'] if licfile is None: licserv = self.cfg['license_server'] if licserv is None: licserv = os.getenv('EB_MATLAB_LICENSE_SERVER', 'license.example.com') licport = self.cfg['license_server_port'] if licport is None: licport = os.getenv('EB_MATLAB_LICENSE_SERVER_PORT', '00000') # create license file lictxt = '\n'.join([ "SERVER %s 000000000000 %s" % (licserv, licport), "USE_SERVER", ]) licfile = os.path.join(self.builddir, 'matlab.lic') write_file(licfile, lictxt) try: copy_file(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile) adjust_permissions(self.configfile, stat.S_IWUSR) # read file in binary mode to avoid UTF-8 encoding issues when using Python 3, # due to non-UTF-8 characters... config = read_file(self.configfile, mode='rb') # use raw byte strings (must be 'br', not 'rb'), # required when using Python 3 because file was read in binary mode regdest = re.compile(br"^# destinationFolder=.*", re.M) regagree = re.compile(br"^# agreeToLicense=.*", re.M) regmode = re.compile(br"^# mode=.*", re.M) reglicpath = re.compile(br"^# licensePath=.*", re.M) # must use byte-strings here when using Python 3, see above config = regdest.sub(b"destinationFolder=%s" % self.installdir.encode('utf-8'), config) config = regagree.sub(b"agreeToLicense=Yes", config) config = regmode.sub(b"mode=silent", config) config = reglicpath.sub(b"licensePath=%s" % licfile.encode('utf-8'), config) write_file(self.configfile, config) except IOError as err: raise EasyBuildError("Failed to create installation config file %s: %s", self.configfile, err) self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def test_log_levels(self): """Test whether log levels are respected""" fd, tmplog = tempfile.mkstemp() os.close(fd) # set log format, for each regex searching setLogFormat("%(name)s [%(levelname)s] :: %(message)s") # test basic log methods logToFile(tmplog, enable=True) log = getLogger('test_easybuildlog') self.mock_stderr(True) # avoid that some log statement spit out stuff to stderr while tests are running for level in ['ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEVEL']: log.setLevelName(level) log.raiseError = False log.error('kaput') log.deprecated('almost kaput', '10000000000000') log.raiseError = True log.warn('this is a warning') log.info('fyi') log.debug('gdb') log.devel('tmi') self.mock_stderr(False) logToFile(tmplog, enable=False) logtxt = read_file(tmplog) root = getRootLoggerName() prefix = '%s.test_easybuildlog' % root devel_msg = r"%s \[DEVEL\] :: tmi" % prefix debug_msg = r"%s \[DEBUG\] :: gdb" % prefix info_msg = r"%s \[INFO\] :: fyi" % prefix warning_msg = r"%s \[WARNING\] :: this is a warning" % prefix deprecated_msg = r"%s \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % prefix error_msg = r"%s \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % prefix expected_logtxt = '\n'.join([ error_msg, error_msg, deprecated_msg, warning_msg, error_msg, deprecated_msg, warning_msg, info_msg, error_msg, deprecated_msg, warning_msg, info_msg, debug_msg, error_msg, deprecated_msg, warning_msg, info_msg, debug_msg, devel_msg, ]) logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def test_toy_broken(self): """Test deliberately broken toy build.""" tmpdir = tempfile.mkdtemp() broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb") toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') broken_toy_ec_txt = read_file(toy_ec_file) broken_toy_ec_txt += "checksums = ['clearywrongchecksum']" write_file(broken_toy_ec, broken_toy_ec_txt) error_regex = "Checksum verification .* failed" self.assertErrorRegex(EasyBuildError, error_regex, self.test_toy_build, ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True, verbose=False, raise_error=True) # make sure log file is retained, also for failed build log_path_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*.log') self.assertTrue( len(glob.glob(log_path_pattern)) == 1, "Log file found at %s" % log_path_pattern) # make sure individual test report is retained, also for failed build test_report_fp_pattern = os.path.join( tmpdir, 'easybuild-*', 'easybuild-toy-0.0*test_report.md') self.assertTrue( len(glob.glob(test_report_fp_pattern)) == 1, "Test report %s found" % test_report_fp_pattern) # test dumping full test report (doesn't raise an exception) test_report_fp = os.path.join(self.test_buildpath, 'full_test_report.md') self.test_toy_build(ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True, verbose=False, raise_error=True, test_report=test_report_fp) # cleanup shutil.rmtree(tmpdir)
def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_EB_MAIN, path=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. @param repo: repo to download @param branch: branch to download @param account: GitHub account to download repo from @param path: path to extract to """ # make sure path exists, create it if necessary if path is None: path = tempfile.mkdtemp() # add account subdir path = os.path.join(path, account) mkdir(path, parents=True) extracted_dir_name = '%s-%s' % (repo, branch) base_name = '%s.tar.gz' % branch latest_commit_sha = fetch_latest_commit_sha(repo, account, branch) expected_path = os.path.join(path, extracted_dir_name) latest_sha_path = os.path.join(expected_path, 'latest-sha') # check if directory already exists, don't download if 'latest-sha' file indicates that it's up to date if os.path.exists(latest_sha_path): sha = read_file(latest_sha_path).split('\n')[0].rstrip() if latest_commit_sha == sha: _log.debug("Not redownloading %s/%s as it already exists: %s" % (account, repo, expected_path)) return expected_path url = URL_SEPARATOR.join([GITHUB_URL, account, repo, 'archive', base_name]) target_path = os.path.join(path, base_name) _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) download_file(base_name, url, target_path) _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) extracted_path = os.path.join(extract_file(target_path, path), extracted_dir_name) # check if extracted_path exists if not os.path.isdir(extracted_path): raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch) write_file(latest_sha_path, latest_commit_sha) _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) return extracted_path
def test_error_reporting(self): """Make sure error reporting is done correctly (no more log.error, log.exception).""" # easybuild.framework.__file__ provides location to <prefix>/easybuild/framework/__init__.py easybuild_loc = os.path.dirname(os.path.dirname(os.path.abspath(easybuild.framework.__file__))) log_method_regexes = [ re.compile("log\.error\("), re.compile("log\.exception\("), re.compile("log\.raiseException\("), ] for dirpath, _, filenames in os.walk(easybuild_loc): for filename in [f for f in filenames if f.endswith('.py')]: path = os.path.join(dirpath, filename) txt = read_file(path) for regex in log_method_regexes: self.assertFalse(regex.search(txt), "No match for '%s' in %s" % (regex.pattern, path))
def test_list_easyblocks(self): """Test listing easyblock hierarchy.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) # adjust PYTHONPATH such that test easyblocks are found orig_sys_path = sys.path[:] import easybuild sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), 'sandbox'))) easybuild = reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) reload(easybuild.tools.module_naming_scheme ) # required to run options unit tests stand-alone # simple view for list_arg in ['--list-easyblocks', '--list-easyblocks=simple']: # clear log write_file(self.logfile, '') args = [ list_arg, '--unittest-file=%s' % self.logfile, ] try: main((args, dummylogfn, False)) except (SystemExit, Exception), err: pass outtxt = read_file(self.logfile) for pat in [ r"EasyBlock\n", r"|--\s+EB_foo\n|\s+|--\s+EB_foofoo\n", r"|--\s+bar\n", ]: self.assertTrue( re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt))
def test_debug(self): """Test enabling debug logging.""" for debug_arg in ['-d', '--debug']: args = [ '--software-name=somethingrandom', debug_arg, ] try: main((args, self.logfile)) except (SystemExit, Exception), err: myerr = err outtxt = read_file(self.logfile) for log_msg_type in ['DEBUG', 'INFO', 'ERROR']: res = re.search(' %s ' % log_msg_type, outtxt) self.assertTrue(res, "%s log messages are included when using %s: %s" % (log_msg_type, debug_arg, outtxt))