def test_v20(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
# this should be ok: ie the default values
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {
'name': 'system',
'version': 'system'
})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.2')
# changes to this dict should not affect the return value of the next call to get_config_dict
fn = 'test.tar.gz'
ec['sources'].append(fn)
ec_bis = ecp.get_config_dict()
self.assertTrue(fn in ec['sources'])
self.assertFalse(fn in ec_bis['sources'])
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v20(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
# this should be ok: ie the default values
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {
'name': 'dummy',
'version': 'dummy'
})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.2')
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_tweak_one_version(self):
"""Test tweak_one function"""
test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')
# test tweaking of software version (--try-software-version)
tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
# checksums should be reset to empty list, only version should be changed, nothing else
self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]:
val = toy_ec_parsed[key]
self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key)
tweaked_val = tweaked_toy_ec_parsed.get(key)
self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val))
# check behaviour if target file already exists
error_pattern = "File exists, not overwriting it without --force"
self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
# existing file does get overwritten when --force is used
init_config(build_options={'force': True, 'silent': True})
write_file(tweaked_toy_ec, '')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
def parse(self):
"""
Parse the file and set options
mandatory requirements are checked here
"""
if self.build_specs is None:
arg_specs = {}
elif isinstance(self.build_specs, dict):
# build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
arg_specs = self.build_specs
else:
self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
self.log.debug("Obtained specs dict %s" % arg_specs)
self.log.info("Parsing easyconfig file %s with rawcontent: %s" % (self.path, self.rawtxt))
parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt)
parser.set_specifications(arg_specs)
local_vars = parser.get_config_dict()
self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)
# make sure all mandatory parameters are defined
# this includes both generic mandatory parameters and software-specific parameters defined via extra_options
missing_mandatory_keys = [key for key in self.mandatory if key not in local_vars]
if missing_mandatory_keys:
self.log.error("mandatory parameters not provided in %s: %s" % (self.path, missing_mandatory_keys))
# provide suggestions for typos
possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
for key in local_vars if key not in self]
typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
if typos:
self.log.error("You may have some typos in your easyconfig file: %s" %
', '.join(["%s -> %s" % typo for typo in typos]))
# we need toolchain to be set when we call _parse_dependency
for key in ['toolchain'] + local_vars.keys():
# validations are skipped, just set in the config
# do not store variables we don't need
if key in self._config.keys():
if key in ['builddependencies', 'dependencies']:
self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
elif key in ['hiddendependencies']:
self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]]
else:
self[key] = local_vars[key]
tup = (key, self[key], type(self[key]))
self.log.info("setting config option %s: value %s (type: %s)" % tup)
elif key in REPLACED_PARAMETERS:
_log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
else:
self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))
# update templating dictionary
self.generate_template_values()
# indicate that this is a parsed easyconfig
self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
def parse(self):
"""
Parse the file and set options
mandatory requirements are checked here
"""
if self.build_specs is None:
arg_specs = {}
elif isinstance(self.build_specs, dict):
# build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
arg_specs = self.build_specs
else:
self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
self.log.debug("Obtained specs dict %s" % arg_specs)
parser = EasyConfigParser(self.path)
parser.set_specifications(arg_specs)
local_vars = parser.get_config_dict()
self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)
# validate mandatory keys
# TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
missing_keys = [key for key in self.mandatory if key not in local_vars]
if missing_keys:
self.log.error("mandatory variables %s not provided in %s" % (missing_keys, self.path))
# provide suggestions for typos
possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
for key in local_vars if key not in self._config]
typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
if typos:
self.log.error("You may have some typos in your easyconfig file: %s" %
', '.join(["%s -> %s" % typo for typo in typos]))
# we need toolchain to be set when we call _parse_dependency
for key in ['toolchain'] + local_vars.keys():
# validations are skipped, just set in the config
# do not store variables we don't need
if key in self._config.keys() + DEPRECATED_OPTIONS.keys():
if key in ['builddependencies', 'dependencies']:
self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
elif key in ['hiddendependencies']:
self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]]
else:
self[key] = local_vars[key]
tup = (key, self[key], type(self[key]))
self.log.info("setting config option %s: value %s (type: %s)" % tup)
else:
self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))
# update templating dictionary
self.generate_template_values()
# indicate that this is a parsed easyconfig
self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
def test_v10(self):
ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb'))
self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.3')
def test_v10(self):
ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb'))
self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.3')
def test_v10(self):
ecp = EasyConfigParser(os.path.join(TESTDIRBASE, "v1.0", "GCC-4.6.3.eb"))
self.assertEqual(ecp._formatter.VERSION, EasyVersion("1.0"))
ec = ecp.get_config_dict()
self.assertEqual(ec["toolchain"], {"name": "dummy", "version": "dummy"})
self.assertEqual(ec["name"], "GCC")
self.assertEqual(ec["version"], "4.6.3")
def test_check_value_types(self):
"""Test checking of easyconfig parameter value types."""
test_ec = os.path.join(TESTDIRBASE, 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb')
error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*"
ecp = EasyConfigParser(test_ec, auto_convert_value_types=False)
self.assertErrorRegex(EasyBuildError, error_msg_pattern, ecp.get_config_dict)
# test default behaviour: auto-converting of mismatched value types
ecp = EasyConfigParser(test_ec)
ecdict = ecp.get_config_dict()
self.assertEqual(ecdict['version'], '1.4')
def parse(self):
"""
Parse the file and set options
mandatory requirements are checked here
"""
if self.build_specs is None:
arg_specs = {}
elif isinstance(self.build_specs, dict):
# build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
arg_specs = self.build_specs
else:
self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
self.log.debug("Obtained specs dict %s" % arg_specs)
parser = EasyConfigParser(self.path)
parser.set_specifications(arg_specs)
local_vars = parser.get_config_dict()
self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)
# validate mandatory keys
# TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
missing_keys = [key for key in self.mandatory if key not in local_vars]
if missing_keys:
self.log.error("mandatory variables %s not provided in %s" % (missing_keys, self.path))
# provide suggestions for typos
possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
for key in local_vars if key not in self._config]
typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
if typos:
self.log.error("You may have some typos in your easyconfig file: %s" %
', '.join(["%s -> %s" % typo for typo in typos]))
# we need toolchain to be set when we call _parse_dependency
for key in ['toolchain'] + local_vars.keys():
# validations are skipped, just set in the config
# do not store variables we don't need
if key in self._config.keys() + DEPRECATED_OPTIONS.keys():
if key in ['builddependencies', 'dependencies']:
self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
else:
self[key] = local_vars[key]
tup = (key, self[key], type(self[key]))
self.log.info("setting config option %s: value %s (type: %s)" % tup)
else:
self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))
# update templating dictionary
self.generate_template_values()
# indicate that this is a parsed easyconfig
self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
def test_check_value_types(self):
"""Test checking of easyconfig parameter value types."""
test_ec = os.path.join(TESTDIRBASE, 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb')
error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*"
ecp = EasyConfigParser(test_ec, auto_convert_value_types=False)
self.assertErrorRegex(EasyBuildError, error_msg_pattern, ecp.get_config_dict)
# test default behaviour: auto-converting of mismatched value types
ecp = EasyConfigParser(test_ec)
ecdict = ecp.get_config_dict()
self.assertEqual(ecdict['version'], '1.4')
def parse(self, path, format_version=None):
"""
Parse the file and set options
mandatory requirements are checked here
"""
parser = EasyConfigParser(path, format_version=format_version)
local_vars = parser.get_config_dict()
# validate mandatory keys
# TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
missing_keys = [key for key in self.mandatory if key not in local_vars]
if missing_keys:
self.log.error("mandatory variables %s not provided in %s" %
(missing_keys, path))
# provide suggestions for typos
possible_typos = [(key,
difflib.get_close_matches(key.lower(),
self._config.keys(), 1,
0.85))
for key in local_vars if key not in self._config]
typos = [(key, guesses[0]) for (key, guesses) in possible_typos
if len(guesses) == 1]
if typos:
self.log.error(
"You may have some typos in your easyconfig file: %s" %
', '.join(["%s -> %s" % typo for typo in typos]))
self._legacy_license(local_vars)
for key in local_vars:
# validations are skipped, just set in the config
# do not store variables we don't need
if key in self._config:
self[key] = local_vars[key]
self.log.info("setting config option %s: value %s" %
(key, self[key]))
else:
self.log.debug(
"Ignoring unknown config option %s (value: %s)" %
(key, local_vars[key]))
# update templating dictionary
self.generate_template_values()
# indicate that this is a parsed easyconfig
self._config['parsed'] = [
True, "This is a parsed easyconfig", "HIDDEN"
]
def test_tweak_one_version(self):
"""Test tweak_one function"""
test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')
# test tweaking of software version (--try-software-version)
tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
# checksums should be reset to empty list, only version should be changed, nothing else
self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]:
val = toy_ec_parsed[key]
self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key)
tweaked_val = tweaked_toy_ec_parsed.get(key)
self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val))
# check behaviour if target file already exists
error_pattern = "File exists, not overwriting it without --force"
self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
# existing file does get overwritten when --force is used
init_config(build_options={'force': True, 'silent': True})
write_file(tweaked_toy_ec, '')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
def test_v20_deps(self):
"""Test parsing of easyconfig in format v2 that includes dependencies."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'libpng.eb')
ecp = EasyConfigParser(fn)
ec = ecp.get_config_dict()
self.assertEqual(ec['name'], 'libpng')
# first version/toolchain listed is default
self.assertEqual(ec['version'], '1.5.10')
self.assertEqual(ec['toolchain'], {'name': 'goolf', 'version': '1.4.10'})
# dependencies should be parsed correctly
deps = ec['dependencies']
self.assertTrue(isinstance(deps[0], Dependency))
self.assertEqual(deps[0].name(), 'zlib')
self.assertEqual(deps[0].version(), '1.2.5')
fn = os.path.join(TESTDIRBASE, 'v2.0', 'goolf.eb')
ecp = EasyConfigParser(fn)
ec = ecp.get_config_dict()
self.assertEqual(ec['name'], 'goolf')
self.assertEqual(ec['version'], '1.4.10')
self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
# dependencies should be parsed correctly
deps = [
# name, version, versionsuffix, toolchain
('GCC', '4.7.2', None, None),
('OpenMPI', '1.6.4', None, {'name': 'GCC', 'version': '4.7.2'}),
('OpenBLAS', '0.2.6', '-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
('FFTW', '3.3.3', None, {'name': 'gompi', 'version': '1.4.10'}),
('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
]
for i, (name, version, versionsuffix, toolchain) in enumerate(deps):
self.assertEqual(ec['dependencies'][i].name(), name)
self.assertEqual(ec['dependencies'][i].version(), version)
self.assertEqual(ec['dependencies'][i].versionsuffix(), versionsuffix)
self.assertEqual(ec['dependencies'][i].toolchain(), toolchain)
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def check_ec(path, expected_buildstats):
"""Check easyconfig at specified path"""
self.assertTrue(os.path.exists(path))
ectxt = read_file(path)
self.assertTrue(ectxt.startswith("# Built with EasyBuild version"))
self.assertTrue("# Build statistics" in ectxt)
ecdict = EasyConfigParser(path).get_config_dict()
self.assertEqual(ecdict['buildstats'], expected_buildstats)
def test_v20_deps(self):
"""Test parsing of easyconfig in format v2 that includes dependencies."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, "v2.0", "libpng.eb")
ecp = EasyConfigParser(fn)
ec = ecp.get_config_dict()
self.assertEqual(ec["name"], "libpng")
# first version/toolchain listed is default
self.assertEqual(ec["version"], "1.5.10")
self.assertEqual(ec["toolchain"], {"name": "goolf", "version": "1.4.10"})
# dependencies should be parsed correctly
deps = ec["dependencies"]
self.assertTrue(isinstance(deps[0], Dependency))
self.assertEqual(deps[0].name(), "zlib")
self.assertEqual(deps[0].version(), "1.2.5")
fn = os.path.join(TESTDIRBASE, "v2.0", "goolf.eb")
ecp = EasyConfigParser(fn)
ec = ecp.get_config_dict()
self.assertEqual(ec["name"], "goolf")
self.assertEqual(ec["version"], "1.4.10")
self.assertEqual(ec["toolchain"], {"name": "dummy", "version": "dummy"})
# dependencies should be parsed correctly
deps = [
# name, version, versionsuffix, toolchain
("GCC", "4.7.2", None, None),
("OpenMPI", "1.6.4", None, {"name": "GCC", "version": "4.7.2"}),
("OpenBLAS", "0.2.6", "-LAPACK-3.4.2", {"name": "gompi", "version": "1.4.10"}),
("FFTW", "3.3.3", None, {"name": "gompi", "version": "1.4.10"}),
("ScaLAPACK", "2.0.2", "-OpenBLAS-0.2.6-LAPACK-3.4.2", {"name": "gompi", "version": "1.4.10"}),
]
for i, (name, version, versionsuffix, toolchain) in enumerate(deps):
self.assertEqual(ec["dependencies"][i].name(), name)
self.assertEqual(ec["dependencies"][i].version(), version)
self.assertEqual(ec["dependencies"][i].versionsuffix(), versionsuffix)
self.assertEqual(ec["dependencies"][i].toolchain(), toolchain)
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_raw(self):
"""Test passing of raw contents to EasyConfigParser."""
ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC',
'GCC-4.6.3.eb')
ec_txt1 = read_file(ec_file1)
ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'gzip',
'gzip-1.5-goolf-1.4.10.eb')
ec_txt2 = read_file(ec_file2)
ecparser = EasyConfigParser(ec_file1)
self.assertEqual(ecparser.rawcontent, ec_txt1)
ecparser = EasyConfigParser(rawcontent=ec_txt2)
self.assertEqual(ecparser.rawcontent, ec_txt2)
# rawcontent supersedes passed filepath
ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2)
self.assertEqual(ecparser.rawcontent, ec_txt2)
ec = ecparser.get_config_dict()
self.assertEqual(ec['name'], 'gzip')
self.assertEqual(ec['toolchain']['name'], 'goolf')
self.assertErrorRegex(EasyBuildError,
"Neither filename nor rawcontent provided",
EasyConfigParser)
def doit(path, pattern):
"""Do it for all easyconfigs in specified location"""
repo = git.Repo.init(path)
timestamp_regex = re.compile('Date:\s*(?P<timestamp>.*)', re.M)
tc_regex = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)
easyconfigs, toolchains = [], set()
specs = []
for (subpath, _, filenames) in os.walk(path, topdown=True):
if subpath.startswith(os.path.join(path, '.git')):
continue
for filename in filenames:
if filename.endswith(
'.eb'
) and filename != 'TEMPLATE.eb' and pattern in filename:
specs.append(os.path.join(path, subpath, filename))
for idx, spec in enumerate(specs):
print "\rprocessed %d of %d easyconfigs" % (idx, len(specs)),
ec = EasyConfigParser(filename=spec).get_config_dict()
if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME:
toolchain = 'dummy'
else:
toolchain = '%(name)s-%(version)s' % ec['toolchain']
toolchains.add(toolchain)
logtxt = repo.git.log('--reverse', '--date=iso-local', spec)
res = timestamp_regex.search(logtxt)
if res:
timestamp = str(res.group('timestamp'))
else:
raise EasyBuildError("No timestamp found in git log for %s: %s",
spec, logtxt)
easyconfigs.append((timestamp, os.path.basename(spec), toolchain))
print ''
print 'found %d different toolchains' % len(toolchains)
for toolchain in sorted(toolchains):
# datestamp first for correct sorting!
ecs = sorted([(datestamp, ec) for (datestamp, ec, tc) in easyconfigs
if tc == toolchain])
print '%s (%d)' % (toolchain, len(ecs))
print '\toldest: %s (%s)' % (ecs[0][1], ecs[0][0])
print '\tnewest: %s (%s)' % (ecs[-1][1], ecs[-1][0])
def test_v20_extra(self):
fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
self.assertRaises(NotImplementedError, ecp.get_config_dict)
def test_raw(self):
"""Test passing of raw contents to EasyConfigParser."""
ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb')
ec_txt1 = read_file(ec_file1)
ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'gzip', 'gzip-1.5-goolf-1.4.10.eb')
ec_txt2 = read_file(ec_file2)
ecparser = EasyConfigParser(ec_file1)
self.assertEqual(ecparser.rawcontent, ec_txt1)
ecparser = EasyConfigParser(rawcontent=ec_txt2)
self.assertEqual(ecparser.rawcontent, ec_txt2)
# rawcontent supersedes passed filepath
ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2)
self.assertEqual(ecparser.rawcontent, ec_txt2)
ec = ecparser.get_config_dict()
self.assertEqual(ec['name'], 'gzip')
self.assertEqual(ec['toolchain']['name'], 'goolf')
self.assertErrorRegex(EasyBuildError, "Neither filename nor rawcontent provided", EasyConfigParser)
def parse(self, path, format_version=None):
"""
Parse the file and set options
mandatory requirements are checked here
"""
parser = EasyConfigParser(path, format_version=format_version)
local_vars = parser.get_config_dict()
# validate mandatory keys
# TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
missing_keys = [key for key in self.mandatory if key not in local_vars]
if missing_keys:
self.log.error("mandatory variables %s not provided in %s" % (missing_keys, path))
# provide suggestions for typos
possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
for key in local_vars if key not in self._config]
typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
if typos:
self.log.error("You may have some typos in your easyconfig file: %s" %
', '.join(["%s -> %s" % typo for typo in typos]))
self._legacy_license(local_vars)
for key in local_vars:
# validations are skipped, just set in the config
# do not store variables we don't need
if key in self._config:
self[key] = local_vars[key]
self.log.info("setting config option %s: value %s" % (key, self[key]))
else:
self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))
# update templating dictionary
self.generate_template_values()
# indicate that this is a parsed easyconfig
self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
def test_tweak_one_version(self):
"""Test tweak_one function"""
test_easyconfigs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')
# test tweaking of software version (--try-software-version)
tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
tweaked_toy_ec_parsed = EasyConfigParser(
tweaked_toy_ec).get_config_dict()
# checksums should be reset to empty list, only version should be changed, nothing else
self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
for key in [
k for k in toy_ec_parsed.keys()
if k not in ['checksums', 'version']
]:
val = toy_ec_parsed[key]
self.assertTrue(
key in tweaked_toy_ec_parsed,
"Parameter '%s' not defined in tweaked easyconfig file" % key)
tweaked_val = tweaked_toy_ec_parsed.get(key)
self.assertEqual(
val, tweaked_val,
"Different value for %s parameter: %s vs %s" %
(key, val, tweaked_val))
def test_v10(self):
ecp = EasyConfigParser(
os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb'))
self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {
'name': 'system',
'version': 'system'
})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.3')
# changes to this dict should not affect the return value of the next call to get_config_dict
fn = 'test.tar.gz'
ec['sources'].append(fn)
ec_bis = ecp.get_config_dict()
self.assertTrue(fn in ec['sources'])
self.assertFalse(fn in ec_bis['sources'])
def test_v20(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, "v2.0", "GCC.eb")
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion("2.0"))
self.assertTrue("name" in formatter.pyheader_localvars)
self.assertFalse("version" in formatter.pyheader_localvars)
self.assertFalse("toolchain" in formatter.pyheader_localvars)
# this should be ok: ie the default values
ec = ecp.get_config_dict()
self.assertEqual(ec["toolchain"], {"name": "dummy", "version": "dummy"})
self.assertEqual(ec["name"], "GCC")
self.assertEqual(ec["version"], "4.6.2")
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v20(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
# this should be ok: ie the default values
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.2')
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v20_extra(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v20_deps(self):
"""Test parsing of easyconfig in format v2 that includes dependencies."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'libpng.eb')
ecp = EasyConfigParser(fn)
ec = ecp.get_config_dict()
self.assertEqual(ec['name'], 'libpng')
# first version/toolchain listed is default
self.assertEqual(ec['version'], '1.5.10')
self.assertEqual(ec['toolchain'], {'name': 'goolf', 'version': '1.4.10'})
# dependencies should be parsed correctly
deps = ec['dependencies']
self.assertTrue(isinstance(deps[0], Dependency))
self.assertEqual(deps[0].name(), 'zlib')
self.assertEqual(deps[0].version(), '1.2.5')
fn = os.path.join(TESTDIRBASE, 'v2.0', 'goolf.eb')
ecp = EasyConfigParser(fn)
ec = ecp.get_config_dict()
self.assertEqual(ec['name'], 'goolf')
self.assertEqual(ec['version'], '1.4.10')
self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
# dependencies should be parsed correctly
deps = [
# name, version, versionsuffix, toolchain
('GCC', '4.7.2', None, None),
('OpenMPI', '1.6.4', None, {'name': 'GCC', 'version': '4.7.2'}),
('OpenBLAS', '0.2.6', '-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
('FFTW', '3.3.3', None, {'name': 'gompi', 'version': '1.4.10'}),
('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
]
for i, (name, version, versionsuffix, toolchain) in enumerate(deps):
self.assertEqual(ec['dependencies'][i].name(), name)
self.assertEqual(ec['dependencies'][i].version(), version)
self.assertEqual(ec['dependencies'][i].versionsuffix(), versionsuffix)
self.assertEqual(ec['dependencies'][i].toolchain(), toolchain)
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs',
letter_dir_for(name), name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure that $root is not used, since it is not compatible with module files in Lua syntax
res = re.findall('.*\$root.*', ec.rawtxt, re.M)
error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
self.assertFalse(res, error_msg % res)
# make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
old_urls = [
'github.com/hpcugent/easybuild',
'hpcugent.github.com/easybuild',
'hpcugent.github.io/easybuild',
]
for old_url in old_urls:
self.assertFalse(old_url in ec.rawtxt,
"Old URL '%s' not found in %s" % (old_url, spec))
# make sure binutils is included as a build dep if toolchain is GCCcore
if ec['toolchain']['name'] == 'GCCcore':
# with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball
requires_binutils = ec['easyblock'] not in [
'Tarball'
] and ec['name'] not in ['Eigen']
# let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this
# exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore)
if ec['toolchain']['version'] == 'system':
binutils_complete_dependencies = [
'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'
]
requires_binutils &= bool(
ec['name'] not in binutils_complete_dependencies)
# if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled)
requires_binutils &= bool(ec['sources'] or ec['exts_list']
or ec.get('components'))
if requires_binutils:
dep_names = [d['name'] for d in ec.builddependencies()]
self.assertTrue(
'binutils' in dep_names,
"binutils is a build dep in %s: %s" % (spec, dep_names))
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
orig_toolchain = ec_dict['toolchain']
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
# take into account that dumped value for *dependencies may include hard-coded subtoolchains
# if no easyconfig was found for the dependency with the 'parent' toolchain,
# if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
if key in DEPENDENCY_PARAMETERS:
# number of dependencies should remain the same
self.assertEqual(len(orig_val), len(dumped_val))
for orig_dep, dumped_dep in zip(orig_val, dumped_val):
# name/version should always match
self.assertEqual(orig_dep[:2], dumped_dep[:2])
# 3rd value is versionsuffix;
if len(dumped_dep) >= 3:
# if no versionsuffix was specified in original dep spec, then dumped value should be empty string
if len(orig_dep) >= 3:
self.assertEqual(dumped_dep[2], orig_dep[2])
else:
self.assertEqual(dumped_dep[2], '')
# 4th value is toolchain spec
if len(dumped_dep) >= 4:
if len(orig_dep) >= 4:
self.assertEqual(dumped_dep[3], orig_dep[3])
else:
# if a subtoolchain is specifed (only) in the dumped easyconfig,
# it should *not* be the same as the parent toolchain
self.assertNotEqual(dumped_dep[3],
(orig_toolchain['name'],
orig_toolchain['version']))
else:
self.assertEqual(orig_val, dumped_val)
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def list_software(output_format=FORMAT_TXT,
detailed=False,
only_installed=False):
"""
Show list of supported software
:param output_format: output format to use
:param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
:param only_installed: only retain software for which a corresponding module is available
:return: multi-line string presenting requested info
"""
silent = build_option('silent')
ec_paths = find_matching_easyconfigs('*', '*',
build_option('robot_path') or [])
ecs = []
cnt = len(ec_paths)
for idx, ec_path in enumerate(ec_paths):
# full EasyConfig instance is only required when module name is needed
# this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
if only_installed:
ec = process_easyconfig(ec_path, validate=False,
parse_only=True)[0]['ec']
else:
ec = EasyConfigParser(filename=ec_path).get_config_dict()
ecs.append(ec)
print_msg('\r', prefix=False, newline=False, silent=silent)
print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt),
newline=False,
silent=silent)
print_msg('', prefix=False, silent=silent)
software = {}
for ec in ecs:
software.setdefault(ec['name'], [])
if is_system_toolchain(ec['toolchain']['name']):
toolchain = SYSTEM_TOOLCHAIN_NAME
else:
toolchain = '%s/%s' % (ec['toolchain']['name'],
ec['toolchain']['version'])
keys = ['description', 'homepage', 'version', 'versionsuffix']
info = {'toolchain': toolchain}
for key in keys:
info[key] = ec.get(key, '')
# make sure values like homepage & versionsuffix get properly templated
if isinstance(ec, dict):
template_values = template_constant_dict(ec)
for key in keys:
if '%(' in info[key]:
try:
info[key] = info[key] % template_values
except (KeyError, TypeError, ValueError) as err:
_log.debug("Ignoring failure to resolve templates: %s",
err)
software[ec['name']].append(info)
if only_installed:
software[ec['name']][-1].update({'mod_name': ec.full_mod_name})
print_msg("Found %d different software packages" % len(software),
silent=silent)
if only_installed:
avail_mod_names = modules_tool().available()
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
for key in all_software:
for entry in all_software[key]:
if entry['mod_name'] in avail_mod_names:
software.setdefault(key, []).append(entry)
print_msg("Retained %d installed software packages" % len(software),
silent=silent)
return generate_doc('list_software_%s' % output_format,
[software, detailed])
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
"""
Show list of supported software
:param output_format: output format to use
:param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
:param only_installed: only retain software for which a corresponding module is available
:return: multi-line string presenting requested info
"""
silent = build_option('silent')
ec_paths = find_matching_easyconfigs('*', '*', build_option('robot_path') or [])
ecs = []
cnt = len(ec_paths)
for idx, ec_path in enumerate(ec_paths):
# full EasyConfig instance is only required when module name is needed
# this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
if only_installed:
ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]['ec']
else:
ec = EasyConfigParser(filename=ec_path).get_config_dict()
ecs.append(ec)
print_msg('\r', prefix=False, newline=False, silent=silent)
print_msg("Processed %d/%d easyconfigs..." % (idx+1, cnt), newline=False, silent=silent)
print_msg('', prefix=False, silent=silent)
software = {}
for ec in ecs:
software.setdefault(ec['name'], [])
if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME:
toolchain = DUMMY_TOOLCHAIN_NAME
else:
toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version'])
keys = ['description', 'homepage', 'version', 'versionsuffix']
info = {'toolchain': toolchain}
for key in keys:
info[key] = ec.get(key, '')
# make sure values like homepage & versionsuffix get properly templated
if isinstance(ec, dict):
template_values = template_constant_dict(ec, skip_lower=False)
for key in keys:
if '%(' in info[key]:
try:
info[key] = info[key] % template_values
except (KeyError, TypeError, ValueError) as err:
_log.debug("Ignoring failure to resolve templates: %s", err)
software[ec['name']].append(info)
if only_installed:
software[ec['name']][-1].update({'mod_name': ec.full_mod_name})
print_msg("Found %d different software packages" % len(software), silent=silent)
if only_installed:
avail_mod_names = modules_tool().available()
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
for key in all_software:
for entry in all_software[key]:
if entry['mod_name'] in avail_mod_names:
software.setdefault(key, []).append(entry)
print_msg("Retained %d installed software packages" % len(software), silent=silent)
return generate_doc('list_software_%s' % output_format, [software, detailed])
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs',
name.lower()[0], name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
keys = []
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
keys.append(key)
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
self.assertEqual(orig_val, dumped_val)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
"""
Show list of supported software
:param output_format: output format to use
:param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
:param only_installed: only retain software for which a corresponding module is available
:return: multi-line string presenting requested info
"""
silent = build_option("silent")
ec_paths = find_matching_easyconfigs("*", "*", build_option("robot_path") or [])
ecs = []
cnt = len(ec_paths)
for idx, ec_path in enumerate(ec_paths):
# full EasyConfig instance is only required when module name is needed
# this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
if only_installed:
ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]["ec"]
else:
ec = EasyConfigParser(filename=ec_path).get_config_dict()
ecs.append(ec)
print_msg("\r", prefix=False, newline=False, silent=silent)
print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt), newline=False, silent=silent)
print_msg("", prefix=False, silent=silent)
software = {}
for ec in ecs:
software.setdefault(ec["name"], [])
if ec["toolchain"]["name"] == DUMMY_TOOLCHAIN_NAME:
toolchain = DUMMY_TOOLCHAIN_NAME
else:
toolchain = "%s/%s" % (ec["toolchain"]["name"], ec["toolchain"]["version"])
versionsuffix = ec.get("versionsuffix", "")
# make sure versionsuffix gets properly templated
if versionsuffix and isinstance(ec, dict):
template_values = template_constant_dict(ec)
versionsuffix = versionsuffix % template_values
software[ec["name"]].append(
{
"description": ec["description"],
"homepage": ec["homepage"],
"toolchain": toolchain,
"version": ec["version"],
"versionsuffix": versionsuffix,
}
)
if only_installed:
software[ec["name"]][-1].update({"mod_name": ec.full_mod_name})
print_msg("Found %d different software packages" % len(software), silent=silent)
if only_installed:
avail_mod_names = modules_tool().available()
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
for key in all_software:
for entry in all_software[key]:
if entry["mod_name"] in avail_mod_names:
software.setdefault(key, []).append(entry)
print_msg("Retained %d installed software packages" % len(software), silent=silent)
return generate_doc("list_software_%s" % output_format, [software, detailed])
