def test_obtain_ec_for(self):
"""Test obtain_ec_for function."""
init_config(build_options={'silent': True})
test_easyconfigs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
# find existing easyconfigs
specs = {
'name': 'GCC',
'version': '6.4.0',
'versionsuffix': '-2.28',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-6.4.0-2.28.eb')
specs = {
'name': 'ScaLAPACK',
'version': '2.0.2',
'toolchain_name': 'gompi',
'toolchain_version': '2018a',
'versionsuffix': '-OpenBLAS-0.2.20',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file),
'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb')
specs = {
'name': 'ifort',
'versionsuffix': '-GCC-4.9.3-2.25',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file),
'ifort-2016.1.150-GCC-4.9.3-2.25.eb')
# latest version if not specified
specs = {
'name': 'GCC',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-7.3.0-2.30.eb')
# generate non-existing easyconfig
change_dir(self.test_prefix)
specs = {
'name': 'GCC',
'version': '4.9.0',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertTrue(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-4.9.0.eb')
def test_obtain_ec_for(self):
"""Test obtain_ec_for function."""
test_easyconfigs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
# find existing easyconfigs
specs = {
'name': 'GCC',
'version': '4.6.4',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-4.6.4.eb')
specs = {
'name': 'ScaLAPACK',
'version': '2.0.2',
'toolchain_name': 'gompi',
'toolchain_version': '1.4.10',
'versionsuffix': '-OpenBLAS-0.2.6-LAPACK-3.4.2',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(
os.path.basename(ec_file),
'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb')
specs = {
'name': 'ifort',
'versionsuffix': '-GCC-4.9.3-2.25',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file),
'ifort-2016.1.150-GCC-4.9.3-2.25.eb')
# latest version if not specified
specs = {
'name': 'GCC',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-4.9.2.eb')
# generate non-existing easyconfig
os.chdir(self.test_prefix)
specs = {
'name': 'GCC',
'version': '5.4.3',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertTrue(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-5.4.3.eb')
def test_obtain_ec_for(self):
"""Test obtain_ec_for function."""
init_config(build_options={'silent': True})
test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
# find existing easyconfigs
specs = {
'name': 'GCC',
'version': '6.4.0',
'versionsuffix': '-2.28',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-6.4.0-2.28.eb')
specs = {
'name': 'ScaLAPACK',
'version': '2.0.2',
'toolchain_name': 'gompi',
'toolchain_version': '2018a',
'versionsuffix': '-OpenBLAS-0.2.20',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb')
specs = {
'name': 'ifort',
'versionsuffix': '-GCC-4.9.3-2.25',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'ifort-2016.1.150-GCC-4.9.3-2.25.eb')
# latest version if not specified
specs = {
'name': 'GCC',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-7.3.0-2.30.eb')
# generate non-existing easyconfig
os.chdir(self.test_prefix)
specs = {
'name': 'GCC',
'version': '5.4.3',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertTrue(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-5.4.3.eb')
def test_obtain_ec_for(self):
"""Test obtain_ec_for function."""
test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
# find existing easyconfigs
specs = {
'name': 'GCC',
'version': '4.6.4',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-4.6.4.eb')
specs = {
'name': 'ScaLAPACK',
'version': '2.0.2',
'toolchain_name': 'gompi',
'toolchain_version': '1.4.10',
'versionsuffix': '-OpenBLAS-0.2.6-LAPACK-3.4.2',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb')
specs = {
'name': 'ifort',
'versionsuffix': '-GCC-4.9.3-2.25',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'ifort-2016.1.150-GCC-4.9.3-2.25.eb')
# latest version if not specified
specs = {
'name': 'GCC',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertFalse(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-4.9.2.eb')
# generate non-existing easyconfig
os.chdir(self.test_prefix)
specs = {
'name': 'GCC',
'version': '5.4.3',
}
(generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path])
self.assertTrue(generated)
self.assertEqual(os.path.basename(ec_file), 'GCC-5.4.3.eb')
def find_easyconfigs_by_specs(build_specs,
robot_path,
try_to_generate,
testing=False):
"""Find easyconfigs by build specifications."""
generated, ec_file = obtain_ec_for(build_specs, robot_path, None)
if generated:
if try_to_generate:
print_msg(
"Generated an easyconfig file %s, going to use it now..." %
ec_file,
silent=testing)
else:
# (try to) cleanup
try:
os.remove(ec_file)
except OSError as err:
_log.warning(
"Failed to remove generated easyconfig file %s: %s" %
(ec_file, err))
# don't use a generated easyconfig unless generation was requested (using a --try-X option)
raise EasyBuildError(
"Unable to find an easyconfig for the given specifications: %s; "
"to make EasyBuild try to generate a matching easyconfig, "
"use the --try-X options ", build_specs)
return [(ec_file, generated)]
def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=False):
"""Find easyconfigs by build specifications."""
generated, ec_file = obtain_ec_for(build_specs, robot_path, None)
if generated:
if try_to_generate:
print_msg("Generated an easyconfig file %s, going to use it now..." % ec_file, silent=testing)
else:
# (try to) cleanup
try:
os.remove(ec_file)
except OSError, err:
_log.warning("Failed to remove generated easyconfig file %s: %s" % (ec_file, err))
# don't use a generated easyconfig unless generation was requested (using a --try-X option)
raise EasyBuildError("Unable to find an easyconfig for the given specifications: %s; "
"to make EasyBuild try to generate a matching easyconfig, "
"use the --try-X options ", build_specs)
def test_obtain_easyconfig(self):
"""test obtaining an easyconfig file given certain specifications"""
tcname = 'GCC'
tcver = '4.3.2'
patches = ["one.patch"]
# prepare a couple of eb files to test again
fns = ["pi-3.14.eb",
"pi-3.13-GCC-4.3.2.eb",
"pi-3.15-GCC-4.3.2.eb",
"pi-3.15-GCC-4.4.5.eb",
"foo-1.2.3-GCC-4.3.2.eb"]
eb_files = [(fns[0], "\n".join(['name = "pi"',
'version = "3.12"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'patches = %s' % patches
])),
(fns[1], "\n".join(['name = "pi"',
'version = "3.13"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver),
'patches = %s' % patches
])),
(fns[2], "\n".join(['name = "pi"',
'version = "3.15"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver),
'patches = %s' % patches
])),
(fns[3], "\n".join(['name = "pi"',
'version = "3.15"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "4.5.1"}' % tcname,
'patches = %s' % patches
])),
(fns[4], "\n".join(['name = "foo"',
'version = "1.2.3"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver),
'foo_extra1 = "bar"',
]))
]
self.ec_dir = tempfile.mkdtemp()
for (fn, txt) in eb_files:
write_file(os.path.join(self.ec_dir, fn), txt)
# should crash when no suited easyconfig file (or template) is available
specs = {'name': 'nosuchsoftware'}
error_regexp = ".*No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % specs['name']
self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None)
# should find matching easyconfig file
specs = {'name': 'foo', 'version': '1.2.3'}
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], False)
self.assertEqual(res[1], os.path.join(self.ec_dir, fns[-1]))
# should not pick between multiple available toolchain names
name = "pi"
ver = "3.12"
suff = "mysuff"
specs.update({
'name': name,
'version': ver,
'versionsuffix': suff
})
error_regexp = ".*No toolchain name specified, and more than one available: .*"
self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None)
# should be able to generate an easyconfig file that slightly differs
ver = '3.16'
specs.update({
'toolchain_name': tcname,
'toolchain_version': tcver,
'version': ver,
'foo': 'bar123'
})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[1], "%s-%s-%s-%s%s.eb" % (name, ver, tcname, tcver, suff))
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['name'], specs['name'])
self.assertEqual(ec['version'], specs['version'])
self.assertEqual(ec['versionsuffix'], specs['versionsuffix'])
self.assertEqual(ec['toolchain'], {'name': tcname, 'version': tcver})
# can't check for key 'foo', because EasyConfig ignores parameter names it doesn't know about
txt = read_file(res[1])
self.assertTrue(re.search('foo = "%s"' % specs['foo'], txt))
os.remove(res[1])
# should pick correct version, i.e. not newer than what's specified, if a choice needs to be made
ver = '3.14'
specs.update({'version': ver})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['version'], specs['version'])
txt = read_file(res[1])
self.assertTrue(re.search("version = [\"']%s[\"'] .*was: [\"']3.13[\"']" % ver, txt))
os.remove(res[1])
# should pick correct toolchain version as well, i.e. now newer than what's specified, if a choice needs to be made
specs.update({
'version': '3.15',
'toolchain_version': '4.4.5',
})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['version'], specs['version'])
self.assertEqual(ec['toolchain']['version'], specs['toolchain_version'])
txt = read_file(res[1])
pattern = "toolchain = .*version.*[\"']%s[\"'].*was: .*version.*[\"']%s[\"']" % (specs['toolchain_version'], tcver)
self.assertTrue(re.search(pattern, txt))
os.remove(res[1])
# should be able to prepend to list of patches and handle list of dependencies
new_patches = ['two.patch', 'three.patch']
specs.update({
'patches': new_patches[:],
'dependencies': [('foo', '1.2.3'), ('bar', '666', '-bleh', ('gompi', '1.4.10'))],
})
parsed_deps = [
{
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': ec['toolchain'],
'dummy': False,
'mod_name': 'foo/1.2.3-GCC-4.4.5',
},
{
'name': 'bar',
'version': '666',
'versionsuffix': '-bleh',
'toolchain': {'name': 'gompi', 'version': '1.4.10'},
'dummy': False,
'mod_name': 'bar/666-gompi-1.4.10-bleh',
},
]
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['patches'], specs['patches'])
self.assertEqual(ec['dependencies'], parsed_deps)
os.remove(res[1])
# verify append functionality for lists
specs['patches'].insert(0, '')
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['patches'], patches + new_patches)
specs['patches'].remove('')
os.remove(res[1])
# verify prepend functionality for lists
specs['patches'].append('')
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['patches'], new_patches + patches)
os.remove(res[1])
# should use supplied filename
fn = "my.eb"
res = obtain_ec_for(specs, [self.ec_dir], fn)
self.assertEqual(res[0], True)
self.assertEqual(res[1], fn)
os.remove(res[1])
# should use a template if it's there
tpl_path = os.path.join("share", "easybuild", "easyconfigs", "TEMPLATE.eb")
def trim_path(path):
dirs = path.split(os.path.sep)
if len(dirs) > 3 and 'site-packages' in dirs:
if path.endswith('.egg'):
path = os.path.join(*dirs[:-4]) # strip of lib/python2.7/site-packages/*.egg part
else:
path = os.path.join(*dirs[:-3]) # strip of lib/python2.7/site-packages part
return path
tpl_full_path = find_full_path(tpl_path, trim=trim_path)
# only run this test if the TEMPLATE.eb file is available
# TODO: use unittest.skip for this (but only works from Python 2.7)
if tpl_full_path:
shutil.copy2(tpl_full_path, self.ec_dir)
specs.update({'name': 'nosuchsoftware'})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['name'], specs['name'])
os.remove(res[1])
# cleanup
shutil.rmtree(self.ec_dir)