def test_copy_file(self):
""" Test copy_file """
testdir = os.path.dirname(os.path.abspath(__file__))
tmpdir = self.test_prefix
to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
target_path = os.path.join(tmpdir, 'toy.eb')
ft.copy_file(to_copy, target_path)
self.assertTrue(os.path.exists(target_path))
self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
# also test behaviour of extract_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
# remove target file, it shouldn't get copied under dry run
os.remove(target_path)
self.mock_stdout(True)
ft.copy_file(to_copy, target_path)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertFalse(os.path.exists(target_path))
self.assertTrue(re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt))
def test_patch_step(self):
"""Test patch step."""
test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]
orig_sources = ec['ec']['sources'][:]
toy_patches = [
'toy-0.0_typo.patch', # test for applying patch
('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy
]
self.assertEqual(ec['ec']['patches'], toy_patches)
# test applying patches without sources
ec['ec']['sources'] = []
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step)
# test actual patching of unpacked sources
ec['ec']['sources'] = orig_sources
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
eb.patch_step()
# verify that patches were applied
toydir = os.path.join(eb.builddir, 'toy-0.0')
self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig'])
self.assertTrue("and very proud of it" in read_file(os.path.join(toydir, 'toy.source')))
self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n')
def test_run_cmd_log(self):
"""Test logging of executed commands."""
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
regex = re.compile('cmd "echo hello" exited with exit code [0-9]* and output:')
# command output is not logged by default without debug logging
init_logging(logfile, silent=True)
self.assertTrue(run_cmd("echo hello"))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 0)
write_file(logfile, '')
init_logging(logfile, silent=True)
self.assertTrue(run_cmd("echo hello", log_all=True))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 1)
write_file(logfile, '')
# with debugging enabled, exit code and output of command should only get logged once
setLogLevelDebug()
init_logging(logfile, silent=True)
self.assertTrue(run_cmd("echo hello"))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 1)
write_file(logfile, '')
init_logging(logfile, silent=True)
self.assertTrue(run_cmd("echo hello", log_all=True))
stop_logging(logfile)
self.assertEqual(len(regex.findall(read_file(logfile))), 1)
write_file(logfile, '')
def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("Contents of installdir %s" % easyblock.installdir)
self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
if DEBUG:
print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
def test_read_write_file(self):
"""Test reading/writing files."""
fp = os.path.join(self.test_prefix, 'test.txt')
txt = "test123"
ft.write_file(fp, txt)
self.assertEqual(ft.read_file(fp), txt)
txt2 = '\n'.join(['test', '123'])
ft.write_file(fp, txt2, append=True)
self.assertEqual(ft.read_file(fp), txt+txt2)
# also test behaviour of write_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
foo = os.path.join(self.test_prefix, 'foo.txt')
self.mock_stdout(True)
ft.write_file(foo, 'bar')
txt = self.get_stdout()
self.mock_stdout(False)
self.assertFalse(os.path.exists(foo))
self.assertTrue(re.match("^file written: .*/foo.txt$", txt))
ft.write_file(foo, 'bar', forced=True)
self.assertTrue(os.path.exists(foo))
self.assertEqual(ft.read_file(foo), 'bar')
def test_download_repo(self):
"""Test download_repo function."""
if self.github_token is None:
print "Skipping test_download_repo, no GitHub token available?"
return
# default: download tarball for master branch of hpcugent/easybuild-easyconfigs repo
path = gh.download_repo(path=self.test_prefix)
repodir = os.path.join(self.test_prefix, 'hpcugent', 'easybuild-easyconfigs-master')
self.assertTrue(os.path.samefile(path, repodir))
self.assertTrue(os.path.exists(repodir))
shafile = os.path.join(repodir, 'latest-sha')
self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile)))
self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2015a.eb')))
# existing downloaded repo is not reperformed, except if SHA is different
account, repo, branch = 'boegel', 'easybuild-easyblocks', 'develop'
repodir = os.path.join(self.test_prefix, account, '%s-%s' % (repo, branch))
latest_sha = gh.fetch_latest_commit_sha(repo, account, branch=branch)
# put 'latest-sha' fail in place, check whether repo was (re)downloaded (should not)
shafile = os.path.join(repodir, 'latest-sha')
write_file(shafile, latest_sha)
path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix)
self.assertTrue(os.path.samefile(path, repodir))
self.assertEqual(os.listdir(repodir), ['latest-sha'])
# remove 'latest-sha' file and verify that download was performed
os.remove(shafile)
path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix)
self.assertTrue(os.path.samefile(path, repodir))
self.assertTrue('easybuild' in os.listdir(repodir))
self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile)))
self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')))
def test_modules_tool_stateless(self):
"""Check whether ModulesTool instance is stateless between runs."""
test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')
# copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load
shutil.copytree(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core'))
shutil.copytree(os.path.join(test_modules_path, 'Compiler'), os.path.join(self.test_prefix, 'Compiler'))
modtxt = read_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'))
modpath_extension = os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2')
modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M)
write_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'), modtxt)
modtxt = read_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'))
modpath_extension = os.path.join(self.test_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4')
mkdir(modpath_extension, parents=True)
modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M)
write_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), modtxt)
# force reset of any singletons by reinitiating config
init_config()
os.environ['MODULEPATH'] = os.path.join(self.test_prefix, 'Core')
modtool = modules_tool()
if isinstance(modtool, Lmod):
load_err_msg = "cannot[\s\n]*be[\s\n]*loaded"
else:
load_err_msg = "Unable to locate a modulefile"
# GCC/4.6.3 is *not* an available Core module
self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['GCC/4.6.3'])
# GCC/4.7.2 is one of the available Core modules
modtool.load(['GCC/4.7.2'])
# OpenMPI/1.6.4 becomes available after loading GCC/4.7.2 module
modtool.load(['OpenMPI/1.6.4'])
modtool.purge()
# reset $MODULEPATH, obtain new ModulesTool instance,
# which should not remember anything w.r.t. previous $MODULEPATH value
os.environ['MODULEPATH'] = test_modules_path
modtool = modules_tool()
# GCC/4.6.3 is available
modtool.load(['GCC/4.6.3'])
modtool.purge()
# GCC/4.7.2 is available (note: also as non-Core module outside of hierarchy)
modtool.load(['GCC/4.7.2'])
# OpenMPI/1.6.4 is *not* available with current $MODULEPATH (loaded GCC/4.7.2 was not a hierarchical module)
self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['OpenMPI/1.6.4'])
def multidiff(base, files, colored=True):
"""
Generate a diff for multiple files, all compared to base.
@param base: base to compare with
@param files: list of files to compare with base
@param colored: boolean indicating whether a colored multi-diff should be generated
@return: text with multidiff overview
"""
differ = difflib.Differ()
base_lines = read_file(base).split('\n')
mdiff = MultiDiff(os.path.basename(base), base_lines, files, colored=colored)
# use the MultiDiff class to store the information
for filepath in files:
lines = read_file(filepath).split('\n')
diff = differ.compare(lines, base_lines)
filename = os.path.basename(filepath)
# contruct map of line number to diff lines and mapping between diff lines
# example partial diff:
#
# - toolchain = {'name': 'goolfc', 'version': '2.6.10'}
# ? - ^ ^
#
# + toolchain = {'name': 'goolf', 'version': '1.6.20'}
# ? ^ ^
#
local_diff = {}
squigly_dict = {}
last_added = None
offset = 1
for (i, line) in enumerate(diff):
# diff line indicating changed characters on line above, a.k.a. a 'squigly' line
if line.startswith(QUESTIONMARK):
squigly_dict[last_added] = line
offset -= 1
# diff line indicating addition change
elif line.startswith(PLUS):
local_diff.setdefault(i + offset, []).append((line, filename))
last_added = line
# diff line indicated removal change
elif line.startswith(MINUS):
local_diff.setdefault(i + offset, []).append((line, filename))
last_added = line
offset -= 1
# construct the multi-diff based on the constructed dict
for line_no in local_diff:
for (line, filename) in local_diff[line_no]:
mdiff.parse_line(line_no, line.rstrip(), filename, squigly_dict.get(line, '').rstrip())
return str(mdiff)
def test_allow_modules_tool_mismatch(self):
"""Test allowing mismatch of modules tool with 'module' function."""
# make sure MockModulesTool is available
from test.framework.modulestool import MockModulesTool
ec_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb')
# keep track of original module definition so we can restore it
orig_module = os.environ.get('module', None)
# check whether mismatch between 'module' function and selected modules tool is detected
os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
args = [
ec_file,
'--modules-tool=MockModulesTool',
]
self.eb_main(args, do_build=True)
outtxt = read_file(self.logfile)
error_regex = re.compile("ERROR .*pattern .* not found in defined 'module' function")
self.assertTrue(error_regex.search(outtxt), "Found error w.r.t. module function mismatch: %s" % outtxt[-600:])
# check that --allow-modules-tool-mispatch transforms this error into a warning
os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
args = [
ec_file,
'--modules-tool=MockModulesTool',
'--allow-modules-tool-mismatch',
]
self.eb_main(args, do_build=True)
outtxt = read_file(self.logfile)
warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
self.assertTrue(warn_regex.search(outtxt), "Found warning w.r.t. module function mismatch: %s" % outtxt[-600:])
# check whether match between 'module' function and selected modules tool is detected
os.environ['module'] = "() { eval ` /bin/echo $*`\n}"
args = [
ec_file,
'--modules-tool=MockModulesTool',
'--debug',
]
self.eb_main(args, do_build=True)
outtxt = read_file(self.logfile)
found_regex = re.compile("DEBUG Found pattern .* in defined 'module' function")
self.assertTrue(found_regex.search(outtxt), "Found debug message w.r.t. module function: %s" % outtxt[-600:])
# restore 'module' function
if orig_module is not None:
os.environ['module'] = orig_module
else:
del os.environ['module']
def test_fetch_parameters_from_easyconfig(self):
"""Test fetch_parameters_from_easyconfig function."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
toy_ec_file = os.path.join(test_ecs_dir, 'toy-0.0.eb')
for ec_file, correct_name, correct_easyblock in [
(toy_ec_file, 'toy', None),
(os.path.join(test_ecs_dir, 'goolf-1.4.10.eb'), 'goolf', 'Toolchain'),
]:
name, easyblock = fetch_parameters_from_easyconfig(read_file(ec_file), ['name', 'easyblock'])
self.assertEqual(name, correct_name)
self.assertEqual(easyblock, correct_easyblock)
self.assertEqual(fetch_parameters_from_easyconfig(read_file(toy_ec_file), ['description'])[0], "Toy C program.")
def test_is_yeb_format(self):
""" Test is_yeb_format function """
testdir = os.path.dirname(os.path.abspath(__file__))
test_yeb = os.path.join(testdir, 'easyconfigs', 'yeb', 'bzip2-1.0.6-GCC-4.9.2.yeb')
raw_yeb = read_file(test_yeb)
self.assertTrue(is_yeb_format(test_yeb, None))
self.assertTrue(is_yeb_format(None, raw_yeb))
test_eb = os.path.join(testdir, 'easyconfigs', 'gzip-1.4.eb')
raw_eb = read_file(test_eb)
self.assertFalse(is_yeb_format(test_eb, None))
self.assertFalse(is_yeb_format(None, raw_eb))
def test_read_write_file(self):
"""Test reading/writing files."""
tmpdir = tempfile.mkdtemp()
fp = os.path.join(tmpdir, 'test.txt')
txt = "test123"
ft.write_file(fp, txt)
self.assertEqual(ft.read_file(fp), txt)
txt2 = '\n'.join(['test', '123'])
ft.write_file(fp, txt2, append=True)
self.assertEqual(ft.read_file(fp), txt+txt2)
shutil.rmtree(tmpdir)
def get_cpu_family():
"""
Determine CPU family.
@return: a value from the CPU_FAMILIES list
"""
family = None
vendor = get_cpu_vendor()
if vendor in CPU_FAMILIES:
family = vendor
_log.debug("Using vendor as CPU family: %s" % family)
else:
# POWER family needs to be determined indirectly via 'cpu' in /proc/cpuinfo
if os.path.exists(PROC_CPUINFO_FP):
cpuinfo_txt = read_file(PROC_CPUINFO_FP)
power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M)
if power_regex.search(cpuinfo_txt):
family = POWER
_log.debug("Determined CPU family using regex '%s' in %s: %s",
power_regex.pattern, PROC_CPUINFO_FP, family)
if family is None:
family = UNKNOWN
_log.warning("Failed to determine CPU family, returning %s" % family)
return family
def build_and_install_software(module, options, origEnviron, exitOnFailure=True, silent=False):
"""
Build the software
"""
spec = module['spec']
print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
# restore original environment
_log.info("Resetting environment")
filetools.errorsFoundInLog = 0
modify_env(os.environ, origEnviron)
cwd = os.getcwd()
# load easyblock
easyblock = options.easyblock
if not easyblock:
# try to look in .eb file
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
txt = read_file(spec)
for line in txt.split('\n'):
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
name = module['module'][0]
try:
app_class = get_class(easyblock, name=name)
app = app_class(spec, debug=options.debug, robot_path=options.robot)
_log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
except EasyBuildError, err:
print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent)
def configure_step(self):
"""Configure MATLAB installation: create license file."""
# create license file
licserv = self.cfg['license_server']
licport = self.cfg['license_server_port']
lictxt = '\n'.join([
"SERVER %s 000000000000 %s" % (licserv, licport),
"USE_SERVER",
])
licfile = os.path.join(self.builddir, 'matlab.lic')
write_file(licfile, lictxt)
try:
shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile)
config = read_file(self.configfile)
regdest = re.compile(r"^# destinationFolder=.*", re.M)
regkey = re.compile(r"^# fileInstallationKey=.*", re.M)
regagree = re.compile(r"^# agreeToLicense=.*", re.M)
regmode = re.compile(r"^# mode=.*", re.M)
reglicpath = re.compile(r"^# licensePath=.*", re.M)
config = regdest.sub("destinationFolder=%s" % self.installdir, config)
key = self.cfg['key']
config = regkey.sub("fileInstallationKey=%s" % key, config)
config = regagree.sub("agreeToLicense=Yes", config)
config = regmode.sub("mode=silent", config)
config = reglicpath.sub("licensePath=%s" % licfile, config)
write_file(self.configfile, config)
except IOError, err:
raise EasyBuildError("Failed to create installation config file %s: %s", self.configfile, err)
def get_cpu_vendor():
"""
Try to detect the CPU vendor
@return: a value from the VENDORS dict
"""
vendor = None
os_type = get_os_type()
if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP):
txt = read_file(PROC_CPUINFO_FP)
arch = UNKNOWN
vendor_regex = re.compile(r"(vendor_id.*?)?\s*:\s*(?P<vendor>(?(1)\S+|(?:IBM|ARM)))")
res = vendor_regex.search(txt)
if res:
arch = res.group('vendor')
if arch in VENDORS:
vendor = VENDORS[arch]
_log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s",
vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.vendor"
out, ec = run_cmd(cmd, force_in_dry_run=True)
out = out.strip()
if ec == 0 and out in VENDORS:
vendor = VENDORS[out]
_log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
if vendor is None:
vendor = UNKNOWN
_log.warning("Could not determine CPU vendor on %s, returning %s" % (os_type, vendor))
return vendor
def get_cpu_model():
"""
Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
"""
model = None
os_type = get_os_type()
if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP):
# we need 'model name' on Linux/x86, but 'model' is there first with different info
# 'model name' is not there for Linux/POWER, but 'model' has the right info
model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P<model>.*[A-Za-z].+)\s*$", re.M)
txt = read_file(PROC_CPUINFO_FP)
res = model_regex.search(txt)
if res is not None:
model = res.group('model').strip()
_log.debug("Determined CPU model on Linux using regex '%s' in %s: %s",
model_regex.pattern, PROC_CPUINFO_FP, model)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.brand_string"
out, ec = run_cmd(cmd, force_in_dry_run=True)
if ec == 0:
model = out.strip()
_log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
if model is None:
model = UNKNOWN
_log.warning("Failed to determine CPU model, returning %s" % model)
return model
def process_easyconfig_file(ec_file):
"""Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested)."""
ectxt = read_file(ec_file)
name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock'])
derived_easyblock_class = get_easyblock_class(easyblock, name=name, default_fallback=False)
fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class)
if ectxt != fixed_ectxt:
if go.options.backup:
try:
backup_ec_file = '%s.bk' % ec_file
i = 1
while os.path.exists(backup_ec_file):
backup_ec_file = '%s.bk%d' % (ec_file, i)
i += 1
os.rename(ec_file, backup_ec_file)
log.info("Backed up %s to %s" % (ec_file, backup_ec_file))
except OSError, err:
raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err)
write_file(ec_file, fixed_ectxt)
log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt)
log.info("%s: fixed" % ec_file)
def get_cpu_vendor():
"""Try to detect the cpu identifier
will return INTEL, ARM or AMD constant
"""
regexp = re.compile(r"^vendor_id\s+:\s*(?P<vendorid>\S+)\s*$", re.M)
VENDORS = {
'GenuineIntel': INTEL,
'AuthenticAMD': AMD,
}
os_type = get_os_type()
if os_type == LINUX:
try:
txt = read_file('/proc/cpuinfo', log_error=False)
arch = UNKNOWN
# vendor_id might not be in the /proc/cpuinfo, so this might fail
res = regexp.search(txt)
if res:
arch = res.groupdict().get('vendorid', UNKNOWN)
if arch in VENDORS:
return VENDORS[arch]
# some embeded linux on arm behaves differently (e.g. raspbian)
regexp = re.compile(r"^Processor\s+:\s*(?P<vendorid>ARM\S+)\s*", re.M)
res = regexp.search(txt)
if res:
arch = res.groupdict().get('vendorid', UNKNOWN)
if ARM in arch:
return ARM
except IOError, err:
raise SystemToolsException("An error occured while determining CPU vendor since: %s" % err)
def parse_components_list(self):
"""parse the regex in the components extra_options and select the matching components
from the mediaconfig.xml file in the install dir"""
mediaconfigpath = os.path.join(self.cfg['start_dir'], 'pset', 'mediaconfig.xml')
if not os.path.isfile(mediaconfigpath):
raise EasyBuildError("Could not find %s to find list of components." % mediaconfigpath)
mediaconfig = read_file(mediaconfigpath)
available_components = re.findall("<Abbr>(?P<component>[^<]+)</Abbr>", mediaconfig, re.M)
self.log.debug("Intel components found: %s" % available_components)
self.log.debug("Using regex list: %s" % self.cfg['components'])
if COMP_ALL in self.cfg['components'] or COMP_DEFAULTS in self.cfg['components']:
if len(self.cfg['components']) == 1:
self.install_components = self.cfg['components']
else:
raise EasyBuildError("If you specify %s as components, you cannot specify anything else: %s",
' or '.join([COMP_ALL, COMP_DEFAULTS]), self.cfg['components'])
else:
self.install_components = []
for comp_regex in self.cfg['components']:
comps = [comp for comp in available_components if re.match(comp_regex, comp)]
self.install_components.extend(comps)
self.log.debug("Components to install: %s" % self.install_components)
def test_robot_archived_easyconfigs(self):
"""Test whether robot can pick up archived easyconfigs when asked."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb')
gzip_ectxt = read_file(gzip_ec)
test_ec = os.path.join(self.test_prefix, 'test.eb')
tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}"
regex = re.compile("^toolchain = .*", re.M)
test_ectxt = regex.sub(tc_spec, gzip_ectxt)
write_file(test_ec, test_ectxt)
ecs, _ = parse_easyconfigs([(test_ec, False)])
self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies,
ecs, self.modtool, retain_all_deps=True)
# --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs
init_config(build_options={
'consider_archived_easyconfigs': True,
'robot_path': [test_ecs],
})
res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True)
self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3'])
expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb')
self.assertTrue(os.path.samefile(res[0]['spec'], expected))
def add_easyconfig(self, cfg, name, version, stats, previous):
"""
Add the eb-file for software name and version to the repository.
stats should be a dict containing statistics.
if previous is true -> append the statistics to the file
This will return the path to the created file (for use in subclasses)
"""
# create directory for eb file
full_path = os.path.join(self.wc, self.subdir, name)
mkdir(full_path, parents=True)
# destination
dest = os.path.join(full_path, "%s-%s.eb" % (name, version))
txt = "# Built with EasyBuild version %s on %s\n" % (VERBOSE_VERSION, time.strftime("%Y-%m-%d_%H-%M-%S"))
# copy file
txt += read_file(cfg)
# append a line to the eb file so that we don't have git merge conflicts
if not previous:
statsprefix = "\n# Build statistics\nbuildstats = ["
statssuffix = "]\n"
else:
# statstemplate = "\nbuildstats.append(%s)\n"
statsprefix = "\nbuildstats.append("
statssuffix = ")\n"
txt += statsprefix + stats_to_str(stats) + statssuffix
write_file(dest, txt)
return dest
def test_recursive_module_unload(self):
"""Test generating recursively unloading modules."""
# use temporary paths for build/install paths, make sure sources can be found
buildpath = tempfile.mkdtemp()
installpath = tempfile.mkdtemp()
tmpdir = tempfile.mkdtemp()
sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources')
# use toy-0.0.eb easyconfig file that comes with the tests
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb')
# check log message with --skip for existing module
args = [
eb_file,
'--sourcepath=%s' % sourcepath,
'--buildpath=%s' % buildpath,
'--installpath=%s' % installpath,
'--debug',
'--force',
'--recursive-module-unload',
]
self.eb_main(args, do_build=True, verbose=True)
toy_module = os.path.join(installpath, 'modules', 'all', 'toy', '0.0-deps')
toy_module_txt = read_file(toy_module)
is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M)
self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt)
# cleanup
shutil.rmtree(buildpath)
shutil.rmtree(installpath)
shutil.rmtree(tmpdir)
def test_apply_regex_substitutions(self):
"""Test apply_regex_substitutions function."""
testfile = os.path.join(self.test_prefix, 'test.txt')
testtxt = '\n'.join([
"CC = gcc",
"CFLAGS = -O3 -g",
"FC = gfortran",
"FFLAGS = -O3 -g -ffixed-form",
])
ft.write_file(testfile, testtxt)
regex_subs = [
(r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"),
(r"^(FC\s*=\s*).*$", r"\1${FC}"),
(r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"),
]
ft.apply_regex_substitutions(testfile, regex_subs)
expected_testtxt = '\n'.join([
"CC = ${CC}",
"CFLAGS = -O2",
"FC = ${FC}",
"FFLAGS = -O2 -ffixed-form",
])
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, expected_testtxt)
def get_total_memory():
"""
Try to ascertain this node's total memory
:return: total memory as an integer, specifically a number of megabytes
"""
memtotal = None
os_type = get_os_type()
if os_type == LINUX and is_readable(PROC_MEMINFO_FP):
_log.debug("Trying to determine total memory size on Linux via %s", PROC_MEMINFO_FP)
meminfo = read_file(PROC_MEMINFO_FP)
mem_mo = re.match(r'^MemTotal:\s*(\d+)\s*kB', meminfo, re.M)
if mem_mo:
memtotal = int(mem_mo.group(1)) / 1024
elif os_type == DARWIN:
cmd = "sysctl -n hw.memsize"
_log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
if ec == 0:
memtotal = int(out.strip()) / (1024**2)
if memtotal is None:
memtotal = UNKNOWN
_log.warning("Failed to determine total memory, returning %s", memtotal)
return memtotal
def configure_step(self):
"""Configure MCR installation: create license file."""
configfile = os.path.join(self.builddir, self.configfilename)
if LooseVersion(self.version) < LooseVersion('R2015a'):
shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), configfile)
config = read_file(configfile)
# compile regex first since re.sub doesn't accept re.M flag for multiline regex in Python 2.6
regdest = re.compile(r"^# destinationFolder=.*", re.M)
regagree = re.compile(r"^# agreeToLicense=.*", re.M)
regmode = re.compile(r"^# mode=.*", re.M)
config = regdest.sub("destinationFolder=%s" % self.installdir, config)
config = regagree.sub("agreeToLicense=Yes", config)
config = regmode.sub("mode=silent", config)
else:
config = '\n'.join([
"destinationFolder=%s" % self.installdir,
"agreeToLicense=Yes",
"mode=silent",
])
write_file(configfile, config)
self.log.debug("configuration file written to %s:\n %s", configfile, config)
def sanity_check_step(self):
"""Custom sanity check for CUDA."""
if LooseVersion(self.version) > LooseVersion("9"):
versionfile = read_file(os.path.join(self.installdir, "version.txt"))
if not re.search("Version %s$" % self.version, versionfile):
raise EasyBuildError("Unable to find the correct version (%s) in the version.txt file", self.version)
shlib_ext = get_shared_lib_ext()
chk_libdir = ["lib64"]
# Versions higher than 6 do not provide 32 bit libraries
if LooseVersion(self.version) < LooseVersion("6"):
chk_libdir += ["lib"]
culibs = ["cublas", "cudart", "cufft", "curand", "cusparse"]
custom_paths = {
'files': [os.path.join("bin", x) for x in ["fatbinary", "nvcc", "nvlink", "ptxas"]] +
[os.path.join("%s", "lib%s.%s") % (x, y, shlib_ext) for x in chk_libdir for y in culibs],
'dirs': ["include"],
}
if LooseVersion(self.version) < LooseVersion('7'):
custom_paths['files'].append(os.path.join('open64', 'bin', 'nvopencc'))
if LooseVersion(self.version) >= LooseVersion('7'):
custom_paths['files'].append(os.path.join("extras", "CUPTI", "lib64", "libcupti.%s") % shlib_ext)
custom_paths['dirs'].append(os.path.join("extras", "CUPTI", "include"))
super(EB_CUDA, self).sanity_check_step(custom_paths=custom_paths)
def test_job(self):
"""Test submitting build as a job."""
# set MODULEPATH to included modules
orig_modulepath = os.getenv('MODULEPATH', None)
os.environ['MODULEPATH'] = os.path.join(os.path.dirname(__file__), 'modules')
# use gzip-1.4.eb easyconfig file that comes with the tests
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4.eb')
# check log message with --job
for job_args in [ # options passed are reordered, so order here matters to make tests pass
['--debug'],
['--debug', '--stop=configure', '--try-software-name=foo'],
]:
# clear log file
outtxt = write_file(self.logfile, '')
args = [
eb_file,
'--job',
] + job_args
try:
main((args, self.logfile))
except (SystemExit, Exception), err:
pass
outtxt = read_file(self.logfile)
job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s %s.*\n" % ' .*'.join(job_args)
assertmsg = "Info log message with job command template when using --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt)
self.assertTrue(re.search(job_msg, outtxt), assertmsg)
def dependencies_for(self, mod_name, depth=sys.maxint):
"""
Obtain a list of dependencies for the given module, determined recursively, up to a specified depth (optionally)
"""
modfilepath = self.modulefile_path(mod_name)
self.log.debug("modulefile path %s: %s" % (mod_name, modfilepath))
modtxt = read_file(modfilepath)
loadregex = re.compile(r"^\s+module load\s+(.*)$", re.M)
mods = loadregex.findall(modtxt)
if depth > 0:
# recursively determine dependencies for these dependency modules, until depth is non-positive
moddeps = [self.dependencies_for(mod, depth=depth - 1) for mod in mods]
else:
# ignore any deeper dependencies
moddeps = []
# add dependencies of dependency modules only if they're not there yet
for moddepdeps in moddeps:
for dep in moddepdeps:
if not dep in mods:
mods.append(dep)
return mods
def get_cpu_vendor():
"""Try to detect the cpu identifier
will return INTEL or AMD constant
"""
regexp = re.compile(r"^vendor_id\s+:\s*(?P<vendorid>\S+)\s*$", re.M)
VENDORS = {"GenuineIntel": INTEL, "AuthenticAMD": AMD}
# Linux
txt = read_file("/proc/cpuinfo", log_error=False)
if txt is not None:
arch = regexp.search(txt).groupdict()["vendorid"]
if arch in VENDORS:
return VENDORS[arch]
# Darwin (OS X)
out, exitcode = run_cmd("sysctl -n machdep.cpu.vendor")
out = out.strip()
if not exitcode and out and out in VENDORS:
return VENDORS[out]
# BSD
out, exitcode = run_cmd("sysctl -n hw.model")
out = out.strip()
if not exitcode and out:
return out.split(" ")[0]
raise SystemToolsException("Could not detect cpu vendor")
def test_run_cmd_qa_log_all(self):
"""Test run_cmd_qa with log_output enabled"""
(out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'},
log_all=True)
self.assertEqual(ec, 0)
self.assertEquals(out, "n: \n1\n2\n3\n4\n5\n")
run_cmd_logs = glob.glob(
os.path.join(self.test_prefix, '*', 'easybuild-run_cmd_qa*.log'))
self.assertEqual(len(run_cmd_logs), 1)
run_cmd_log_txt = read_file(run_cmd_logs[0])
extra_pref = "# output for interactive command: echo 'n: '; read n; seq 1 $n\n\n"
self.assertEquals(run_cmd_log_txt, extra_pref + "n: \n1\n2\n3\n4\n5\n")
def test_copy_file(self):
""" Test copy_file """
testdir = os.path.dirname(os.path.abspath(__file__))
tmpdir = self.test_prefix
to_copy = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
target_path = os.path.join(tmpdir, 'toy.eb')
ft.copy_file(to_copy, target_path)
self.assertTrue(os.path.exists(target_path))
self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
# also test behaviour of copy_file under --dry-run
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
# remove target file, it shouldn't get copied under dry run
os.remove(target_path)
self.mock_stdout(True)
ft.copy_file(to_copy, target_path)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertFalse(os.path.exists(target_path))
self.assertTrue(
re.search("^copied file .*/toy-0.0.eb to .*/toy.eb", txt))
# forced copy, even in dry run mode
self.mock_stdout(True)
ft.copy_file(to_copy, target_path, force_in_dry_run=True)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertTrue(os.path.exists(target_path))
self.assertTrue(ft.read_file(to_copy) == ft.read_file(target_path))
self.assertEqual(txt, '')
def configure_step(self):
"""Configure MATLAB installation: create license file."""
licserv = self.cfg['license_server']
if licserv is None:
licserv = os.getenv('EB_MATLAB_LICENSE_SERVER',
'license.example.com')
licport = self.cfg['license_server_port']
if licport is None:
licport = os.getenv('EB_MATLAB_LICENSE_SERVER_PORT', '00000')
key = self.cfg['key']
if key is None:
key = os.getenv(
'EB_MATLAB_KEY',
'00000-00000-00000-00000-00000-00000-00000-00000-00000-00000')
# create license file
lictxt = '\n'.join([
"SERVER %s 000000000000 %s" % (licserv, licport),
"USE_SERVER",
])
licfile = os.path.join(self.builddir, 'matlab.lic')
write_file(licfile, lictxt)
try:
shutil.copyfile(
os.path.join(self.cfg['start_dir'], 'installer_input.txt'),
self.configfile)
config = read_file(self.configfile)
regdest = re.compile(r"^# destinationFolder=.*", re.M)
regkey = re.compile(r"^# fileInstallationKey=.*", re.M)
regagree = re.compile(r"^# agreeToLicense=.*", re.M)
regmode = re.compile(r"^# mode=.*", re.M)
reglicpath = re.compile(r"^# licensePath=.*", re.M)
config = regdest.sub("destinationFolder=%s" % self.installdir,
config)
config = regkey.sub("fileInstallationKey=%s" % key, config)
config = regagree.sub("agreeToLicense=Yes", config)
config = regmode.sub("mode=silent", config)
config = reglicpath.sub("licensePath=%s" % licfile, config)
write_file(self.configfile, config)
except IOError, err:
raise EasyBuildError(
"Failed to create installation config file %s: %s",
self.configfile, err)
def test_apply_regex_substitutions(self):
"""Test apply_regex_substitutions function."""
testfile = os.path.join(self.test_prefix, 'test.txt')
testtxt = '\n'.join([
"CC = gcc",
"CFLAGS = -O3 -g",
"FC = gfortran",
"FFLAGS = -O3 -g -ffixed-form",
])
ft.write_file(testfile, testtxt)
regex_subs = [
(r"^(CC)\s*=\s*.*$", r"\1 = ${CC}"),
(r"^(FC\s*=\s*).*$", r"\1${FC}"),
(r"^(.FLAGS)\s*=\s*-O3\s-g(.*)$", r"\1 = -O2\2"),
]
ft.apply_regex_substitutions(testfile, regex_subs)
expected_testtxt = '\n'.join([
"CC = ${CC}",
"CFLAGS = -O2",
"FC = ${FC}",
"FFLAGS = -O2 -ffixed-form",
])
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, expected_testtxt)
# passing empty list of substitions is a no-op
ft.write_file(testfile, testtxt)
ft.apply_regex_substitutions(testfile, [])
new_testtxt = ft.read_file(testfile)
self.assertEqual(new_testtxt, testtxt)
# clean error on non-existing file
error_pat = "Failed to patch .*/nosuchfile.txt: .*No such file or directory"
path = os.path.join(self.test_prefix, 'nosuchfile.txt')
self.assertErrorRegex(EasyBuildError, error_pat,
ft.apply_regex_substitutions, path, regex_subs)
def get_cpu_vendor():
"""
Try to detect the CPU vendor
:return: a value from the CPU_VENDORS list
"""
vendor = None
os_type = get_os_type()
if os_type == LINUX:
vendor_regex = None
arch = get_cpu_architecture()
if arch == X86_64:
vendor_regex = re.compile(r"vendor_id\s+:\s*(\S+)")
elif arch == POWER:
vendor_regex = re.compile(r"model\s+:\s*(\w+)")
elif arch in [AARCH32, AARCH64]:
vendor_regex = re.compile(r"CPU implementer\s+:\s*(\S+)")
if vendor_regex and is_readable(PROC_CPUINFO_FP):
vendor_id = None
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
res = vendor_regex.search(proc_cpuinfo)
if res:
vendor_id = res.group(1)
if vendor_id in VENDOR_IDS:
vendor = VENDOR_IDS[vendor_id]
_log.debug(
"Determined CPU vendor on Linux as being '%s' via regex '%s' in %s",
vendor, vendor_regex.pattern, PROC_CPUINFO_FP)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.vendor"
out, ec = run_cmd(cmd, force_in_dry_run=True)
out = out.strip()
if ec == 0 and out in VENDOR_IDS:
vendor = VENDOR_IDS[out]
_log.debug(
"Determined CPU vendor on DARWIN as being '%s' via cmd '%s" %
(vendor, cmd))
if vendor is None:
vendor = UNKNOWN
_log.warning("Could not determine CPU vendor on %s, returning %s" %
(os_type, vendor))
return vendor
def test_exclude_path_to_top_of_module_tree(self):
"""
Make sure that modules under the HierarchicalMNS are correct,
w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
"""
self.orig_module_naming_scheme = config.get_module_naming_scheme()
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all')
mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True)
mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True)
impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049')
imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144')
if get_module_syntax() == 'Lua':
impi_modfile_path += '.lua'
imkl_modfile_path += '.lua'
# example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
# not for the toolchain or any of the toolchain components,
# since both icc/ifort and impi form the path to the top of the module tree
tests = [
('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']),
('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']),
]
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
eb.toolchain.prepare()
modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
for imkl_dep in excluded_deps:
tup = (imkl_dep, modfile_path, modtxt)
failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
self.assertFalse(re.search("module load %s" % imkl_dep, modtxt), failmsg)
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
def get_system_libs_from_tf(source_dir):
"""Return the valid values for TF_SYSTEM_LIBS from the TensorFlow source directory"""
syslibs_path = os.path.join(source_dir, 'third_party', 'systemlibs',
'syslibs_configure.bzl')
result = []
if os.path.exists(syslibs_path):
txt = read_file(syslibs_path)
valid_libs_match = re.search(r'VALID_LIBS\s*=\s*\[(.*?)\]', txt,
re.DOTALL)
if not valid_libs_match:
raise EasyBuildError('VALID_LIBS definition not found in %s',
syslibs_path)
result = split_tf_libs_txt(valid_libs_match.group(1))
return result
def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False,
reset_env=True, raise_systemexit=False, testing=True, redo_init_config=True):
"""Helper method to call EasyBuild main function."""
cleanup()
myerr = False
if logfile is None:
logfile = self.logfile
# clear log file
if logfile:
f = open(logfile, 'w')
f.write('')
f.close()
env_before = copy.deepcopy(os.environ)
try:
main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=self.modtool)
except SystemExit as err:
if raise_systemexit:
raise err
except Exception as err:
myerr = err
if verbose:
print("err: %s" % err)
if logfile and os.path.exists(logfile):
logtxt = read_file(logfile)
else:
logtxt = None
os.chdir(self.cwd)
if redo_init_config:
# make sure config is reinitialized
init_config(with_include=False)
# restore environment to what it was before running main,
# changes may have been made by eb_main (e.g. $TMPDIR & co)
if reset_env:
modify_env(os.environ, env_before, verbose=False)
tempfile.tempdir = None
if myerr and raise_error:
raise myerr
if return_error:
return logtxt, myerr
else:
return logtxt
def test_get_easyblock_instance(self):
"""Test get_easyblock_instance function."""
from easybuild.easyblocks.toy import EB_toy
testdir = os.path.abspath(os.path.dirname(__file__))
ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0]
eb = get_easyblock_instance(ec)
self.assertTrue(isinstance(eb, EB_toy))
# check whether 'This is easyblock' log message is there
tup = ('EB_toy', 'easybuild.easyblocks.toy', '.*test/framework/sandbox/easybuild/easyblocks/t/toy.pyc*')
eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M)
logtxt = read_file(eb.logfile)
self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt))
def patch_step(self, *args, **kwargs):
"""
Custom patch step for Python:
* patch setup.py when --sysroot EasyBuild configuration setting is used
"""
super(EB_Python, self).patch_step(*args, **kwargs)
# if we're installing Python with an alternate sysroot,
# we need to patch setup.py which includes hardcoded paths like /usr/include and /lib64;
# this fixes problems like not being able to build the _ssl module ("Could not build the ssl module")
sysroot = build_option('sysroot')
if sysroot:
sysroot_inc_dirs, sysroot_lib_dirs = [], []
for pattern in ['include*', os.path.join('usr', 'include*')]:
sysroot_inc_dirs.extend(glob.glob(os.path.join(sysroot, pattern)))
if sysroot_inc_dirs:
sysroot_inc_dirs = ', '.join(["'%s'" % x for x in sysroot_inc_dirs])
else:
raise EasyBuildError("No include directories found in sysroot %s!", sysroot)
for pattern in ['lib*', os.path.join('usr', 'lib*')]:
sysroot_lib_dirs.extend(glob.glob(os.path.join(sysroot, pattern)))
if sysroot_lib_dirs:
sysroot_lib_dirs = ', '.join(["'%s'" % x for x in sysroot_lib_dirs])
else:
raise EasyBuildError("No lib directories found in sysroot %s!", sysroot)
setup_py_fn = 'setup.py'
setup_py_txt = read_file(setup_py_fn)
# newer Python versions (3.6+) have refactored code, requires different patching approach
if "system_include_dirs = " in setup_py_txt:
regex_subs = [
(r"(system_include_dirs = \[).*\]", r"\1%s]" % sysroot_inc_dirs),
(r"(system_lib_dirs = \[).*\]", r"\1%s]" % sysroot_lib_dirs),
]
else:
regex_subs = [
(r"^([ ]+)'/usr/include',", r"\1%s," % sysroot_inc_dirs),
(r"\['/usr/include'\]", r"[%s]" % sysroot_inc_dirs),
(r"^([ ]+)'/lib64', '/usr/lib64',", r"\1%s," % sysroot_lib_dirs),
(r"^[ ]+'/lib', '/usr/lib',", ''),
]
apply_regex_substitutions(setup_py_fn, regex_subs)
def test_parallel(self):
"""Test defining of parallellism."""
topdir = os.path.abspath(os.path.dirname(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
toytxt = read_file(toy_ec)
handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_',
suffix='.eb')
os.close(handle)
write_file(toy_ec1, toytxt + "\nparallel = 123")
handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_',
suffix='.eb')
os.close(handle)
write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67")
# default: parallellism is derived from # available cores + ulimit
test_eb = EasyBlock(EasyConfig(toy_ec))
test_eb.check_readiness_step()
self.assertTrue(
isinstance(test_eb.cfg['parallel'], int)
and test_eb.cfg['parallel'] > 0)
# only 'parallel' easyconfig parameter specified (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 123)
# both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 67)
# only 'parallel' build option specified
init_config(build_options={'parallel': '97', 'validate': False})
test_eb = EasyBlock(EasyConfig(toy_ec))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 97)
# both 'parallel' build option and easyconfig parameter specified (no 'maxparallel')
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 97)
# both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg['parallel'], 67)
def test_dry_run(self):
"""Test use of functions under (extended) dry run."""
build_options = {
'extended_dry_run': True,
'silent': False,
}
init_config(build_options=build_options)
self.mock_stdout(True)
run_cmd("somecommand foo 123 bar")
txt = self.get_stdout()
self.mock_stdout(False)
expected_regex = re.compile('\n'.join([
r" running command \"somecommand foo 123 bar\"",
r" \(in .*\)",
]))
self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt))
# check disabling 'verbose'
self.mock_stdout(True)
run_cmd("somecommand foo 123 bar", verbose=False)
txt = self.get_stdout()
self.mock_stdout(False)
self.assertEqual(txt, '')
# check forced run
outfile = os.path.join(self.test_prefix, 'cmd.out')
self.assertFalse(os.path.exists(outfile))
self.mock_stdout(True)
run_cmd("echo 'This is always echoed' > %s" % outfile, force_in_dry_run=True)
txt = self.get_stdout()
self.mock_stdout(False)
# nothing printed to stdout, but command was run
self.assertEqual(txt, '')
self.assertTrue(os.path.exists(outfile))
self.assertEqual(read_file(outfile), "This is always echoed\n")
# Q&A commands
self.mock_stdout(True)
run_cmd_qa("some_qa_cmd", {'question1': 'answer1'})
txt = self.get_stdout()
self.mock_stdout(False)
expected_regex = re.compile('\n'.join([
r" running interactive command \"some_qa_cmd\"",
r" \(in .*\)",
]))
self.assertTrue(expected_regex.match(txt), "Pattern %s matches with: %s" % (expected_regex.pattern, txt))
def configure_step(self):
"""Configure COMSOL installation: create license file."""
default_lic_env_var = 'LMCOMSOL_LICENSE_FILE'
lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var],
lic_specs=[self.cfg['license_file']])
if lic_specs:
if self.license_env_var is None:
self.log.info("Using COMSOL license specifications from 'license_file': %s", lic_specs)
self.license_env_var = default_lic_env_var
else:
self.log.info("Using COMSOL license specifications from $%s: %s", self.license_env_var, lic_specs)
self.license_file = os.pathsep.join(lic_specs)
env.setvar(self.license_env_var, self.license_file)
else:
msg = "No viable license specifications found; "
msg += "specify 'license_file', or define $%s" % default_lic_env_var
raise EasyBuildError(msg)
copy_file(os.path.join(self.start_dir, 'setupconfig.ini'), self.configfile)
config = read_file(self.configfile)
config_vars = {
'agree': '1',
'desktopshortcuts': '0',
'fileassoc': '0',
'firewall': '0',
'installdir': self.installdir,
'license': self.license_file,
'licmanager': '0',
'linuxlauncher': '0',
'showgui': '0',
'startmenushortcuts': '0',
'symlinks': '0',
}
matlab_root = get_software_root("MATLAB")
if matlab_root:
config_vars.update({'matlabdir': matlab_root})
for key, val in config_vars.items():
regex = re.compile(r"^%s\s*=.*" % key, re.M)
config = regex.sub("%s=%s" % (key, val), config)
write_file(self.configfile, config)
self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def add_easyconfig(self, cfg, name, version, stats, previous):
"""
Add easyconfig to repository
:param cfg: location of easyconfig file
:param name: software name
:param version: software install version, incl. toolchain & versionsuffix
:param stats: build stats, to add to archived easyconfig
:param previous: list of previous build stats
:return: location of archived easyconfig
"""
# create directory for eb file
full_path = os.path.join(self.wc, self.subdir, name)
yeb_format = is_yeb_format(cfg, None)
if yeb_format:
extension = YEB_FORMAT_EXTENSION
prefix = "buildstats: ["
else:
extension = EB_FORMAT_EXTENSION
prefix = "buildstats = ["
# destination
dest = os.path.join(full_path, "%s-%s%s" % (name, version, extension))
txt = "# Built with EasyBuild version %s on %s\n" % (
VERBOSE_VERSION, time.strftime("%Y-%m-%d_%H-%M-%S"))
# copy file
txt += read_file(cfg)
# append a line to the eb file so that we don't have git merge conflicts
statscomment = "\n# Build statistics\n"
statsprefix = prefix
statssuffix = "]\n"
if previous:
statstxt = statscomment + statsprefix + '\n'
for entry in previous + [stats]:
statstxt += stats_to_str(entry, isyeb=yeb_format) + ',\n'
statstxt += statssuffix
else:
statstxt = statscomment + statsprefix + stats_to_str(
stats, isyeb=yeb_format) + statssuffix
txt += statstxt
write_file(dest, txt)
return dest
def test_toy_tweaked(self):
"""Test toy build with tweaked easyconfig, for testing extra easyconfig parameters."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
ec_file = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb')
shutil.copy2(os.path.join(test_ecs_dir, 'toy-0.0.eb'), ec_file)
# tweak easyconfig by appending to it
ec_extra = '\n'.join([
"versionsuffix = '-tweaked'",
"modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}",
"modextravars = {'FOO': 'bar'}",
"modloadmsg = 'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'",
"modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua
"modluafooter = 'io.stderr:write(\"oh hai!\")'" # ignored when module syntax is Tcl
])
write_file(ec_file, ec_extra, append=True)
args = [
ec_file,
'--sourcepath=%s' % self.test_sourcepath,
'--buildpath=%s' % self.test_buildpath,
'--installpath=%s' % self.test_installpath,
'--debug',
'--force',
]
outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
self.check_toy(self.test_installpath, outtxt, versionsuffix='-tweaked')
toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked')
if get_module_syntax() == 'Lua':
toy_module += '.lua'
toy_module_txt = read_file(toy_module)
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M))
self.assertTrue(re.search(r'module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0"$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M))
self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M))
self.assertTrue(re.search(r'^if mode\(\) == "load" then\n\s*io.stderr:write\(".*I AM toy v0.0"\)$',
toy_module_txt, re.M))
self.assertTrue(re.search(r'^io.stderr:write\("oh hai!"\)$', toy_module_txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
def test_download_repo(self):
"""Test download_repo function."""
if self.skip_github_tests:
print("Skipping test_download_repo, no GitHub token available?")
return
# default: download tarball for master branch of easybuilders/easybuild-easyconfigs repo
path = gh.download_repo(path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
repodir = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-master')
self.assertTrue(os.path.samefile(path, repodir))
self.assertTrue(os.path.exists(repodir))
shafile = os.path.join(repodir, 'latest-sha')
self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile)))
self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2019b.eb')))
# existing downloaded repo is not reperformed, except if SHA is different
account, repo, branch = 'boegel', 'easybuild-easyblocks', 'develop'
repodir = os.path.join(self.test_prefix, account, '%s-%s' % (repo, branch))
latest_sha = gh.fetch_latest_commit_sha(repo, account, branch=branch, github_user=GITHUB_TEST_ACCOUNT)
# put 'latest-sha' fail in place, check whether repo was (re)downloaded (should not)
shafile = os.path.join(repodir, 'latest-sha')
write_file(shafile, latest_sha)
path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix,
github_user=GITHUB_TEST_ACCOUNT)
self.assertTrue(os.path.samefile(path, repodir))
self.assertEqual(os.listdir(repodir), ['latest-sha'])
# remove 'latest-sha' file and verify that download was performed
os.remove(shafile)
path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix,
github_user=GITHUB_TEST_ACCOUNT)
self.assertTrue(os.path.samefile(path, repodir))
self.assertTrue('easybuild' in os.listdir(repodir))
self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile)))
self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')))
def test_patch_step(self):
"""Test patch step."""
test_easyconfigs = os.path.join(
os.path.abspath(os.path.dirname(__file__)), 'easyconfigs',
'test_ecs')
ec = process_easyconfig(
os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0]
orig_sources = ec['ec']['sources'][:]
toy_patches = [
'toy-0.0_typo.patch', # test for applying patch
('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy
]
self.assertEqual(ec['ec']['patches'], toy_patches)
# test applying patches without sources
ec['ec']['sources'] = []
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step)
# test actual patching of unpacked sources
ec['ec']['sources'] = orig_sources
eb = EasyBlock(ec['ec'])
eb.fetch_step()
eb.extract_step()
eb.patch_step()
# verify that patches were applied
toydir = os.path.join(eb.builddir, 'toy-0.0')
self.assertEqual(sorted(os.listdir(toydir)),
['toy-extra.txt', 'toy.source', 'toy.source.orig'])
self.assertTrue("and very proud of it" in read_file(
os.path.join(toydir, 'toy.source')))
self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')),
'moar!\n')
def test_raw(self):
"""Test passing of raw contents to EasyConfigParser."""
ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb')
ec_txt1 = read_file(ec_file1)
ec_file2 = os.path.join(TESTDIRBASE, 'v1.0',
'gzip-1.5-goolf-1.4.10.eb')
ec_txt2 = read_file(ec_file2)
ecparser = EasyConfigParser(ec_file1)
self.assertEqual(ecparser.rawcontent, ec_txt1)
ecparser = EasyConfigParser(rawcontent=ec_txt2)
self.assertEqual(ecparser.rawcontent, ec_txt2)
# rawcontent supersedes passed filepath
ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2)
self.assertEqual(ecparser.rawcontent, ec_txt2)
ec = ecparser.get_config_dict()
self.assertEqual(ec['name'], 'gzip')
self.assertEqual(ec['toolchain']['name'], 'goolf')
self.assertErrorRegex(EasyBuildError,
"Neither filename nor rawcontent provided",
EasyConfigParser)
def get_cpu_model():
"""
returns cpu model
f.ex Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
"""
os_type = get_os_type()
if os_type == LINUX:
regexp = re.compile(r"^model name\s+:\s*(?P<modelname>.+)\s*$", re.M)
try:
txt = read_file('/proc/cpuinfo', log_error=False)
if txt is not None:
return regexp.search(txt).groupdict()['modelname'].strip()
except IOError, err:
raise SystemToolsException(
"An error occured when determining CPU model: %s" % err)
def post_install_step(self):
"""Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there."""
setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth')
if os.path.exists(setuptools_pth):
setuptools_pth_txt = read_file(setuptools_pth)
# any line that starts with '/' is a sign of trouble
sys_path_regex = re.compile('^/', re.M)
if sys_path_regex.search(setuptools_pth_txt):
self.log.warning("Found %s, and includes one or more absolute system paths. Removing it.",
setuptools_pth)
try:
os.remove(setuptools_pth)
except OSError as err:
raise EasyBuildError("Failed to remove %s: %s", setuptools_pth, err)
def test_footer(self):
"""Test specifying a module footer."""
# use temporary paths for build/install paths, make sure sources can be found
buildpath = tempfile.mkdtemp()
installpath = tempfile.mkdtemp()
tmpdir = tempfile.mkdtemp()
sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'sandbox', 'sources')
# create file containing modules footer
module_footer_txt = '\n'.join([
"# test footer",
"setenv SITE_SPECIFIC_ENV_VAR foobar",
])
fd, modules_footer = tempfile.mkstemp(prefix='modules-footer-')
os.close(fd)
f = open(modules_footer, 'w')
f.write(module_footer_txt)
f.close()
# use toy-0.0.eb easyconfig file that comes with the tests
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs',
'toy-0.0.eb')
# check log message with --skip for existing module
args = [
eb_file,
'--sourcepath=%s' % sourcepath,
'--buildpath=%s' % buildpath,
'--installpath=%s' % installpath,
'--debug',
'--force',
'--modules-footer=%s' % modules_footer,
]
self.eb_main(args, do_build=True)
toy_module = os.path.join(installpath, 'modules', 'all', 'toy', '0.0')
toy_module_txt = read_file(toy_module)
footer_regex = re.compile(r'%s$' % module_footer_txt, re.M)
msg = "modules footer '%s' is present in '%s'" % (module_footer_txt,
toy_module_txt)
self.assertTrue(footer_regex.search(toy_module_txt), msg)
# cleanup
shutil.rmtree(buildpath)
shutil.rmtree(installpath)
shutil.rmtree(tmpdir)
os.remove(modules_footer)
def get_cpu_model():
"""
Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz
"""
model = None
os_type = get_os_type()
if os_type == LINUX and is_readable(PROC_CPUINFO_FP):
proc_cpuinfo = read_file(PROC_CPUINFO_FP)
arch = get_cpu_architecture()
if arch in [AARCH32, AARCH64]:
# On ARM platforms, no model name is provided in /proc/cpuinfo. However, for vanilla ARM cores
# we can reverse-map the part number.
vendor = get_cpu_vendor()
if vendor == ARM:
model_regex = re.compile(r"CPU part\s+:\s*(\S+)", re.M)
# There can be big.LITTLE setups with different types of cores!
model_ids = model_regex.findall(proc_cpuinfo)
if model_ids:
id_list = []
for model_id in sorted(set(model_ids)):
id_list.append(ARM_CORTEX_IDS.get(model_id, UNKNOWN))
model = vendor + ' ' + ' + '.join(id_list)
_log.debug("Determined CPU model on Linux using regex '%s' in %s: %s",
model_regex.pattern, PROC_CPUINFO_FP, model)
else:
# we need 'model name' on Linux/x86, but 'model' is there first with different info
# 'model name' is not there for Linux/POWER, but 'model' has the right info
model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P<model>.*[A-Za-z].+)\s*$", re.M)
res = model_regex.search(proc_cpuinfo)
if res is not None:
model = res.group('model').strip()
_log.debug("Determined CPU model on Linux using regex '%s' in %s: %s",
model_regex.pattern, PROC_CPUINFO_FP, model)
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.brand_string"
out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
if ec == 0:
model = out.strip()
_log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
if model is None:
model = UNKNOWN
_log.warning("Failed to determine CPU model, returning %s" % model)
return model
def test_run_cmd_log_output(self):
"""Test run_cmd with log_output enabled"""
(out, ec) = run_cmd("seq 1 100", log_output=True)
self.assertEqual(ec, 0)
self.assertTrue(out.startswith("1\n2\n"))
self.assertTrue(out.endswith("99\n100\n"))
run_cmd_logs = glob.glob(
os.path.join(self.test_prefix, '*', 'easybuild-run_cmd*.log'))
self.assertEqual(len(run_cmd_logs), 1)
run_cmd_log_txt = read_file(run_cmd_logs[0])
self.assertTrue(
run_cmd_log_txt.startswith("# output for command: seq 1 100\n\n"))
run_cmd_log_lines = run_cmd_log_txt.split('\n')
self.assertEqual(run_cmd_log_lines[2:5], ['1', '2', '3'])
self.assertEqual(run_cmd_log_lines[-4:-1], ['98', '99', '100'])
def configure_step(self):
"""Configure MATLAB installation: create license file."""
licfile = self.cfg['license_file']
if licfile is None:
licserv = self.cfg['license_server']
if licserv is None:
licserv = os.getenv('EB_MATLAB_LICENSE_SERVER', 'license.example.com')
licport = self.cfg['license_server_port']
if licport is None:
licport = os.getenv('EB_MATLAB_LICENSE_SERVER_PORT', '00000')
# create license file
lictxt = '\n'.join([
"SERVER %s 000000000000 %s" % (licserv, licport),
"USE_SERVER",
])
licfile = os.path.join(self.builddir, 'matlab.lic')
write_file(licfile, lictxt)
try:
copy_file(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile)
adjust_permissions(self.configfile, stat.S_IWUSR)
# read file in binary mode to avoid UTF-8 encoding issues when using Python 3,
# due to non-UTF-8 characters...
config = read_file(self.configfile, mode='rb')
# use raw byte strings (must be 'br', not 'rb'),
# required when using Python 3 because file was read in binary mode
regdest = re.compile(br"^# destinationFolder=.*", re.M)
regagree = re.compile(br"^# agreeToLicense=.*", re.M)
regmode = re.compile(br"^# mode=.*", re.M)
reglicpath = re.compile(br"^# licensePath=.*", re.M)
# must use byte-strings here when using Python 3, see above
config = regdest.sub(b"destinationFolder=%s" % self.installdir.encode('utf-8'), config)
config = regagree.sub(b"agreeToLicense=Yes", config)
config = regmode.sub(b"mode=silent", config)
config = reglicpath.sub(b"licensePath=%s" % licfile.encode('utf-8'), config)
write_file(self.configfile, config)
except IOError as err:
raise EasyBuildError("Failed to create installation config file %s: %s", self.configfile, err)
self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def test_log_levels(self):
"""Test whether log levels are respected"""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
self.mock_stderr(True) # avoid that some log statement spit out stuff to stderr while tests are running
for level in ['ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEVEL']:
log.setLevelName(level)
log.raiseError = False
log.error('kaput')
log.deprecated('almost kaput', '10000000000000')
log.raiseError = True
log.warn('this is a warning')
log.info('fyi')
log.debug('gdb')
log.devel('tmi')
self.mock_stderr(False)
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
prefix = '%s.test_easybuildlog' % root
devel_msg = r"%s \[DEVEL\] :: tmi" % prefix
debug_msg = r"%s \[DEBUG\] :: gdb" % prefix
info_msg = r"%s \[INFO\] :: fyi" % prefix
warning_msg = r"%s \[WARNING\] :: this is a warning" % prefix
deprecated_msg = r"%s \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % prefix
error_msg = r"%s \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % prefix
expected_logtxt = '\n'.join([
error_msg,
error_msg, deprecated_msg, warning_msg,
error_msg, deprecated_msg, warning_msg, info_msg,
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg,
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg, devel_msg,
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def test_toy_broken(self):
"""Test deliberately broken toy build."""
tmpdir = tempfile.mkdtemp()
broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb")
toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs',
'toy-0.0.eb')
broken_toy_ec_txt = read_file(toy_ec_file)
broken_toy_ec_txt += "checksums = ['clearywrongchecksum']"
write_file(broken_toy_ec, broken_toy_ec_txt)
error_regex = "Checksum verification .* failed"
self.assertErrorRegex(EasyBuildError,
error_regex,
self.test_toy_build,
ec_file=broken_toy_ec,
tmpdir=tmpdir,
verify=False,
fails=True,
verbose=False,
raise_error=True)
# make sure log file is retained, also for failed build
log_path_pattern = os.path.join(tmpdir, 'easybuild-*',
'easybuild-toy-0.0*.log')
self.assertTrue(
len(glob.glob(log_path_pattern)) == 1,
"Log file found at %s" % log_path_pattern)
# make sure individual test report is retained, also for failed build
test_report_fp_pattern = os.path.join(
tmpdir, 'easybuild-*', 'easybuild-toy-0.0*test_report.md')
self.assertTrue(
len(glob.glob(test_report_fp_pattern)) == 1,
"Test report %s found" % test_report_fp_pattern)
# test dumping full test report (doesn't raise an exception)
test_report_fp = os.path.join(self.test_buildpath,
'full_test_report.md')
self.test_toy_build(ec_file=broken_toy_ec,
tmpdir=tmpdir,
verify=False,
fails=True,
verbose=False,
raise_error=True,
test_report=test_report_fp)
# cleanup
shutil.rmtree(tmpdir)
def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_EB_MAIN, path=None):
"""
Download entire GitHub repo as a tar.gz archive, and extract it into specified path.
@param repo: repo to download
@param branch: branch to download
@param account: GitHub account to download repo from
@param path: path to extract to
"""
# make sure path exists, create it if necessary
if path is None:
path = tempfile.mkdtemp()
# add account subdir
path = os.path.join(path, account)
mkdir(path, parents=True)
extracted_dir_name = '%s-%s' % (repo, branch)
base_name = '%s.tar.gz' % branch
latest_commit_sha = fetch_latest_commit_sha(repo, account, branch)
expected_path = os.path.join(path, extracted_dir_name)
latest_sha_path = os.path.join(expected_path, 'latest-sha')
# check if directory already exists, don't download if 'latest-sha' file indicates that it's up to date
if os.path.exists(latest_sha_path):
sha = read_file(latest_sha_path).split('\n')[0].rstrip()
if latest_commit_sha == sha:
_log.debug("Not redownloading %s/%s as it already exists: %s" % (account, repo, expected_path))
return expected_path
url = URL_SEPARATOR.join([GITHUB_URL, account, repo, 'archive', base_name])
target_path = os.path.join(path, base_name)
_log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path))
download_file(base_name, url, target_path)
_log.debug("%s downloaded to %s, extracting now" % (base_name, path))
extracted_path = os.path.join(extract_file(target_path, path), extracted_dir_name)
# check if extracted_path exists
if not os.path.isdir(extracted_path):
raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch)
write_file(latest_sha_path, latest_commit_sha)
_log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path))
return extracted_path
def test_error_reporting(self):
"""Make sure error reporting is done correctly (no more log.error, log.exception)."""
# easybuild.framework.__file__ provides location to <prefix>/easybuild/framework/__init__.py
easybuild_loc = os.path.dirname(os.path.dirname(os.path.abspath(easybuild.framework.__file__)))
log_method_regexes = [
re.compile("log\.error\("),
re.compile("log\.exception\("),
re.compile("log\.raiseException\("),
]
for dirpath, _, filenames in os.walk(easybuild_loc):
for filename in [f for f in filenames if f.endswith('.py')]:
path = os.path.join(dirpath, filename)
txt = read_file(path)
for regex in log_method_regexes:
self.assertFalse(regex.search(txt), "No match for '%s' in %s" % (regex.pattern, path))
def test_list_easyblocks(self):
"""Test listing easyblock hierarchy."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy',
suffix='.log')
os.close(fd)
# adjust PYTHONPATH such that test easyblocks are found
orig_sys_path = sys.path[:]
import easybuild
sys.path.append(
os.path.abspath(os.path.join(os.path.dirname(__file__),
'sandbox')))
easybuild = reload(easybuild)
import easybuild.easyblocks
reload(easybuild.easyblocks)
reload(easybuild.tools.module_naming_scheme
) # required to run options unit tests stand-alone
# simple view
for list_arg in ['--list-easyblocks', '--list-easyblocks=simple']:
# clear log
write_file(self.logfile, '')
args = [
list_arg,
'--unittest-file=%s' % self.logfile,
]
try:
main((args, dummylogfn, False))
except (SystemExit, Exception), err:
pass
outtxt = read_file(self.logfile)
for pat in [
r"EasyBlock\n",
r"|--\s+EB_foo\n|\s+|--\s+EB_foofoo\n",
r"|--\s+bar\n",
]:
self.assertTrue(
re.search(pat, outtxt),
"Pattern '%s' is found in output of --list-easyblocks: %s"
% (pat, outtxt))
def test_debug(self):
"""Test enabling debug logging."""
for debug_arg in ['-d', '--debug']:
args = [
'--software-name=somethingrandom',
debug_arg,
]
try:
main((args, self.logfile))
except (SystemExit, Exception), err:
myerr = err
outtxt = read_file(self.logfile)
for log_msg_type in ['DEBUG', 'INFO', 'ERROR']:
res = re.search(' %s ' % log_msg_type, outtxt)
self.assertTrue(res, "%s log messages are included when using %s: %s" % (log_msg_type, debug_arg, outtxt))
