def test_validation(self):
""" test other validations beside mandatory variables """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'stop = "notvalid"',
])
self.prep()
eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops)
self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", eb.validate)
eb['stop'] = 'patch'
# this should now not crash
eb.validate()
eb['osdependencies'] = ['non-existent-dep']
self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", eb.validate)
# dummy toolchain, installversion == version
self.assertEqual(eb.get_installversion(), "3.14")
os.chmod(self.eb_file, 0000)
self.assertErrorRegex(EasyBuildError, "Unexpected IOError", EasyConfig, self.eb_file)
os.chmod(self.eb_file, 0755)
self.contents += "\nsyntax_error'"
self.prep()
self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_parse_yeb(self):
"""Test parsing of .yeb easyconfigs."""
if 'yaml' not in sys.modules:
print "Skipping test_parse_yeb (no PyYAML available)"
return
testdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(testdir, 'easyconfigs')
test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
# test parsing
test_files = [
'bzip2-1.0.6-GCC-4.9.2',
'gzip-1.6-GCC-4.9.2',
'goolf-1.4.10',
'ictce-4.1.13',
'SQLite-3.8.10.2-goolf-1.4.10',
'Python-2.7.10-ictce-4.1.13',
'CrayCCE-5.1.29',
]
for filename in test_files:
ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename))
# compare with parsed result of .eb easyconfig
ec_eb = EasyConfig(os.path.join(test_easyconfigs, '%s.eb' % filename))
no_match = False
for key in sorted(ec_yeb.asdict()):
eb_val = ec_eb[key]
yeb_val = ec_yeb[key]
if key == 'description':
# multi-line string is always terminated with '\n' in YAML, so strip it off
yeb_val = yeb_val.strip()
self.assertEqual(yeb_val, eb_val)
def test_templating(self):
""" test easyconfig templating """
inp = {
'name':'PI',
'version':'3.14',
'namelower':'pi',
'cmd': 'tar xfvz %s',
}
# don't use any escaping insanity here, since it is templated itself
self.contents = '\n'.join([
'name = "%(name)s"',
'version = "%(version)s"',
'homepage = "http://google.com"',
'description = "test easyconfig %%(name)s"',
'toolchain = {"name":"dummy", "version": "dummy2"}',
'source_urls = [(GOOGLECODE_SOURCE)]',
'sources = [SOURCE_TAR_GZ, (SOURCELOWER_TAR_GZ, "%(cmd)s")]',
'sanity_check_paths = {"files": [], "dirs": ["libfoo.%%s" %% SHLIB_EXT]}',
]) % inp
self.prep()
eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops)
eb.validate()
eb.generate_template_values()
self.assertEqual(eb['description'], "test easyconfig PI")
const_dict = dict([(x[0], x[1]) for x in easyconfig.templates.TEMPLATE_CONSTANTS])
self.assertEqual(eb['sources'][0], const_dict['SOURCE_TAR_GZ'] % inp)
self.assertEqual(eb['sources'][1][0], const_dict['SOURCELOWER_TAR_GZ'] % inp)
self.assertEqual(eb['sources'][1][1], 'tar xfvz %s')
self.assertEqual(eb['source_urls'][0], const_dict['GOOGLECODE_SOURCE'] % inp)
self.assertEqual(eb['sanity_check_paths']['dirs'][0], 'libfoo.%s' % get_shared_lib_ext())
# test the escaping insanity here (ie all the crap we allow in easyconfigs)
eb['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s"
self.assertEqual(eb['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s")
def test_validation(self):
""" test other validations beside mandatory parameters """
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'stop = "notvalid"',
])
self.prep()
ec = EasyConfig(self.eb_file, validate=False)
self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", ec.validate)
ec['stop'] = 'patch'
# this should now not crash
ec.validate()
ec['osdependencies'] = ['non-existent-dep']
self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", ec.validate)
# dummy toolchain, installversion == version
self.assertEqual(det_full_ec_version(ec), "3.14")
os.chmod(self.eb_file, 0000)
self.assertErrorRegex(EasyBuildError, "Permission denied", EasyConfig, self.eb_file)
os.chmod(self.eb_file, 0755)
self.contents += "\nsyntax_error'"
self.prep()
self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_filter_deps(self):
"""Test filtered dependencies."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')
ec_file = os.path.join(test_ecs_dir, 'goolf-1.4.10.eb')
ec = EasyConfig(ec_file)
deps = sorted([dep['name'] for dep in ec.dependencies()])
self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'])
# test filtering multiple deps
init_config(build_options={'filter_deps': ['FFTW', 'ScaLAPACK']})
deps = sorted([dep['name'] for dep in ec.dependencies()])
self.assertEqual(deps, ['GCC', 'OpenBLAS', 'OpenMPI'])
# test filtering of non-existing dep
init_config(build_options={'filter_deps': ['zlib']})
deps = sorted([dep['name'] for dep in ec.dependencies()])
self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'])
# test parsing of value passed to --filter-deps
opts = init_config(args=[])
self.assertEqual(opts.filter_deps, None)
opts = init_config(args=['--filter-deps=zlib'])
self.assertEqual(opts.filter_deps, ['zlib'])
opts = init_config(args=['--filter-deps=zlib,ncurses'])
self.assertEqual(opts.filter_deps, ['zlib', 'ncurses'])
def test_validation(self):
""" test other validations beside mandatory variables """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'stop = "notvalid"',
])
self.prep()
ec = EasyConfig(self.eb_file, validate=False)
self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", ec.validate)
ec['stop'] = 'patch'
# this should now not crash
ec.validate()
ec['osdependencies'] = ['non-existent-dep']
self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", ec.validate)
# dummy toolchain, installversion == version
self.assertEqual(det_full_ec_version(ec), "3.14")
os.chmod(self.eb_file, 0000)
self.assertErrorRegex(EasyBuildError, "Permission denied", EasyConfig, self.eb_file)
os.chmod(self.eb_file, 0755)
self.contents += "\nsyntax_error'"
self.prep()
self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_extensions_step(self):
"""Test the extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1"]',
])
self.writeEC()
"""Testcase for extensions"""
# test for proper error message without the exts_defaultclass set
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
self.assertRaises(EasyBuildError, eb.extensions_step, fetch=True)
self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step, fetch=True)
# test if everything works fine if set
self.contents += "\nexts_defaultclass = 'DummyExtension'"
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.extensions_step(fetch=True)
# test for proper error message when skip is set, but no exts_filter is set
self.assertRaises(EasyBuildError, eb.skip_extensions)
self.assertErrorRegex(EasyBuildError, "no exts_filter set", eb.skip_extensions)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_format_equivalence_basic(self):
"""Test whether easyconfigs in different formats are equivalent."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
easyconfigs_path = os.path.join(os.path.dirname(__file__), 'easyconfigs')
# set max diff high enough to make sure the difference is shown in case of problems
self.maxDiff = 10000
for eb_file1, eb_file2, specs in [
('gzip-1.4.eb', 'gzip.eb', {}),
('gzip-1.4.eb', 'gzip.eb', {'version': '1.4'}),
('gzip-1.4.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'dummy', 'version': 'dummy'}}),
('gzip-1.4-GCC-4.6.3.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}}),
('gzip-1.5-goolf-1.4.10.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}}),
('gzip-1.5-ictce-4.1.13.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'ictce', 'version': '4.1.13'}}),
]:
ec1 = EasyConfig(os.path.join(easyconfigs_path, 'v1.0', eb_file1), validate=False)
ec2 = EasyConfig(os.path.join(easyconfigs_path, 'v2.0', eb_file2), validate=False, build_specs=specs)
ec2_dict = ec2.asdict()
# reset mandatory attributes from format2 that are not in format 1
for attr in ['docurls', 'software_license', 'software_license_urls']:
ec2_dict[attr] = None
self.assertEqual(ec1.asdict(), ec2_dict)
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_format_equivalence_basic(self):
"""Test whether easyconfigs in different formats are equivalent."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
easyconfigs_path = os.path.join(os.path.dirname(__file__), 'easyconfigs')
# set max diff high enough to make sure the difference is shown in case of problems
self.maxDiff = 10000
build_options = {
'valid_stops': self.all_stops,
'validate': False,
}
for eb_file1, eb_file2, specs in [
('gzip-1.4.eb', 'gzip.eb', {}),
('gzip-1.4.eb', 'gzip.eb', {'version': '1.4'}),
('gzip-1.4.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'dummy', 'version': 'dummy'}}),
('gzip-1.4-GCC-4.6.3.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}}),
('gzip-1.5-goolf-1.4.10.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}}),
('gzip-1.5-ictce-4.1.13.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'ictce', 'version': '4.1.13'}}),
]:
ec1 = EasyConfig(os.path.join(easyconfigs_path, 'v1.0', eb_file1), build_options=build_options)
ec2 = EasyConfig(os.path.join(easyconfigs_path, 'v2.0', eb_file2), build_options=build_options, build_specs=specs)
self.assertEqual(ec1.asdict(), ec2.asdict())
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_external_module_toolchain(self):
"""Test specifying external (build) dependencies in yaml format."""
if 'yaml' not in sys.modules:
print "Skipping test_external_module_toolchain (no PyYAML available)"
return
ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb',
'CrayCCE-5.1.29.yeb')
metadata = {
'name': ['foo', 'bar'],
'version': ['1.2.3', '3.2.1'],
'prefix': '/foo/bar',
}
build_options = {
'external_modules_metadata': {
'fftw/3.3.4.0': metadata
},
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
easybuild.tools.build_log.EXPERIMENTAL = True
ec = EasyConfig(ecpath)
self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0')
self.assertEqual(ec.dependencies()[1]['external_module_metadata'],
metadata)
def test_check_readiness(self):
"""Test check_readiness method."""
init_config(build_options={'validate': False})
# check that check_readiness step works (adding dependencies, etc.)
ec_file = 'OpenMPI-1.6.4-GCC-4.6.4.eb'
ec_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs', ec_file)
ec = EasyConfig(ec_path)
eb = EasyBlock(ec)
eb.check_readiness_step()
# a proper error should be thrown for dependencies that can't be resolved (module should be there)
tmpdir = tempfile.mkdtemp()
shutil.copy2(ec_path, tmpdir)
ec_path = os.path.join(tmpdir, ec_file)
f = open(ec_path, 'a')
f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n")
f.close()
ec = EasyConfig(ec_path)
eb = EasyBlock(ec)
try:
eb.check_readiness_step()
except EasyBuildError, err:
err_regex = re.compile(
"Missing modules for one or more dependencies: nosuchsoftware/1.2.3-GCC-4.6.4"
)
self.assertTrue(
err_regex.search(str(err)),
"Pattern '%s' found in '%s'" % (err_regex.pattern, err))
def process_easyconfig(path, build_options=None, build_specs=None):
"""
Process easyconfig, returning some information for each block
@param path: path to easyconfig file
@param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
blocks = retrieve_blocks_in_spec(path, build_options.get("only_blocks", None))
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
_log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
ec = EasyConfig(spec, build_options=build_options, build_specs=build_specs)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
_log.exception(msg)
name = ec["name"]
# this app will appear as following module in the list
easyconfig = {
"ec": ec,
"spec": spec,
"module": det_full_module_name(ec),
"dependencies": [],
"builddependencies": [],
}
if len(blocks) > 1:
easyconfig["originalSpec"] = path
# add build dependencies
for dep in ec.builddependencies():
_log.debug("Adding build dependency %s for app %s." % (dep, name))
easyconfig["builddependencies"].append(dep)
# add dependencies (including build dependencies)
for dep in ec.dependencies():
_log.debug("Adding dependency %s for app %s." % (dep, name))
easyconfig["dependencies"].append(dep)
# add toolchain as dependency too
if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
dep = ec.toolchain.as_dict()
_log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig["dependencies"].append(dep)
del ec
# this is used by the parallel builder
easyconfig["unresolved_deps"] = copy.deepcopy(easyconfig["dependencies"])
easyconfigs.append(easyconfig)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True):
"""
Process easyconfig, returning some information for each block
"""
blocks = retrieve_blocks_in_spec(path, onlyBlocks)
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
# - use mod? __init__ and importCfg are ignored.
_log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
all_stops = [x[0] for x in EasyBlock.get_steps()]
ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
_log.exception(msg)
name = ec['name']
# this app will appear as following module in the list
easyconfig = {
'spec': spec,
'module': (ec.name, ec.get_installversion()),
'dependencies': [],
'builddependencies': [],
}
if len(blocks) > 1:
easyconfig['originalSpec'] = path
# add build dependencies
for dep in ec.builddependencies():
deptup = (dep['name'], dep['tc'])
_log.debug("Adding build dependency %s for app %s." % (deptup, name))
easyconfig['builddependencies'].append(deptup)
# add dependencies (including build dependencies)
for dep in ec.dependencies():
deptup = (dep['name'], dep['tc'])
_log.debug("Adding dependency %s for app %s." % (deptup, name))
easyconfig['dependencies'].append(deptup)
# add toolchain as dependency too
if ec.toolchain.name != 'dummy':
dep = (ec.toolchain.name, ec.toolchain.version)
_log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
del ec
# this is used by the parallel builder
easyconfig['unresolvedDependencies'] = copy.copy(easyconfig['dependencies'])
easyconfigs.append(easyconfig)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True):
"""
Process easyconfig, returning some information for each block
"""
blocks = retrieve_blocks_in_spec(path, onlyBlocks)
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
# - use mod? __init__ and importCfg are ignored.
_log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
all_stops = [x[0] for x in EasyBlock.get_steps()]
ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
_log.exception(msg)
name = ec["name"]
# this app will appear as following module in the list
easyconfig = {
"spec": spec,
"module": (ec.name, ec.get_installversion()),
"dependencies": [],
"builddependencies": [],
}
if len(blocks) > 1:
easyconfig["originalSpec"] = path
# add build dependencies
for dep in ec.builddependencies():
deptup = (dep["name"], dep["tc"])
_log.debug("Adding build dependency %s for app %s." % (deptup, name))
easyconfig["builddependencies"].append(deptup)
# add dependencies (including build dependencies)
for dep in ec.dependencies():
deptup = (dep["name"], dep["tc"])
_log.debug("Adding dependency %s for app %s." % (deptup, name))
easyconfig["dependencies"].append(deptup)
# add toolchain as dependency too
if ec.toolchain.name != "dummy":
dep = (ec.toolchain.name, ec.toolchain.version)
_log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig["dependencies"].append(dep)
del ec
# this is used by the parallel builder
easyconfig["unresolvedDependencies"] = copy.copy(easyconfig["dependencies"])
easyconfigs.append(easyconfig)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True):
"""
Process easyconfig, returning some information for each block
"""
blocks = retrieve_blocks_in_spec(path, onlyBlocks)
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
# - use mod? __init__ and importCfg are ignored.
_log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
all_stops = [x[0] for x in EasyBlock.get_steps()]
ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
_log.exception(msg)
name = ec['name']
# this app will appear as following module in the list
easyconfig = {
'ec': ec,
'spec': spec,
'module': det_full_module_name(ec),
'dependencies': [],
'builddependencies': [],
}
if len(blocks) > 1:
easyconfig['originalSpec'] = path
# add build dependencies
for dep in ec.builddependencies():
_log.debug("Adding build dependency %s for app %s." % (dep, name))
easyconfig['builddependencies'].append(dep)
# add dependencies (including build dependencies)
for dep in ec.dependencies():
_log.debug("Adding dependency %s for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
# add toolchain as dependency too
if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
dep = ec.toolchain.as_dict()
_log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
del ec
# this is used by the parallel builder
easyconfig['unresolved_deps'] = copy.deepcopy(easyconfig['dependencies'])
easyconfigs.append(easyconfig)
def test_extra_options(self):
""" extra_options should allow other variables to be stored """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'toolchainopts = { "static": True}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
self.assertRaises(KeyError, lambda: eb['custom_key'])
extra_vars = [('custom_key',
['default', "This is a default key",
easyconfig.CUSTOM])]
eb = EasyConfig(self.eb_file, extra_vars, valid_stops=self.all_stops)
self.assertEqual(eb['custom_key'], 'default')
eb['custom_key'] = "not so default"
self.assertEqual(eb['custom_key'], 'not so default')
self.contents += "\ncustom_key = 'test'"
self.prep()
eb = EasyConfig(self.eb_file, extra_vars, valid_stops=self.all_stops)
self.assertEqual(eb['custom_key'], 'test')
eb['custom_key'] = "not so default"
self.assertEqual(eb['custom_key'], 'not so default')
# test if extra toolchain options are being passed
self.assertEqual(eb.toolchain.options['static'], True)
extra_vars.extend([
('mandatory_key',
['default', 'another mandatory key', easyconfig.MANDATORY])
])
# test extra mandatory vars
self.assertErrorRegex(EasyBuildError,
r"mandatory variables \S* not provided",
EasyConfig, self.eb_file, extra_vars)
self.contents += '\nmandatory_key = "value"'
self.prep()
eb = EasyConfig(self.eb_file, extra_vars, valid_stops=self.all_stops)
self.assertEqual(eb['mandatory_key'], 'value')
def test_make_module_extend_modpath(self):
"""Test for make_module_extend_modpath"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'moduleclass = "compiler"',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
self.assertEqual(eb.make_module_extend_modpath(), '')
usermodsdir = 'my/own/modules'
modclasses = ['compiler', 'tools']
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
build_options = {
'subdir_user_modules': usermodsdir,
'valid_module_classes': modclasses,
}
init_config(build_options=build_options)
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
txt = eb.make_module_extend_modpath()
if get_module_syntax() == 'Tcl':
regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses]
home = r'\$env\(HOME\)'
regexs.extend([
# extension for user modules is guarded
r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir),
])
elif get_module_syntax() == 'Lua':
regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses]
home = r'os.getenv\("HOME"\)'
regexs.extend([
# extension for user modules is guarded
r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir),
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def test_extensions_sanity_check(self):
"""Test sanity check aspect of extensions."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb'))
# purposely put sanity check command in place that breaks the build,
# to check whether sanity check is only run once;
# sanity check commands are checked after checking sanity check paths, so this should work
toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')])
# this import only works here, since EB_toy is a test easyblock
from easybuild.easyblocks.toy import EB_toy
eb = EB_toy(toy_ec)
eb.silent = True
eb.run_all_steps(True)
def test_extensions_sanity_check(self):
"""Test sanity check aspect of extensions."""
test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb'))
# purposely put sanity check command in place that breaks the build,
# to check whether sanity check is only run once;
# sanity check commands are checked after checking sanity check paths, so this should work
toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')])
# this import only works here, since EB_toy is a test easyblock
from easybuild.easyblocks.toy import EB_toy
eb = EB_toy(toy_ec)
eb.silent = True
eb.run_all_steps(True)
def test_extra_options(self):
""" extra_options should allow other variables to be stored """
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'toolchainopts = { "static": True}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file)
self.assertErrorRegex(EasyBuildError, "unknown easyconfig parameter", lambda: eb['custom_key'])
extra_vars = {'custom_key': ['default', "This is a default key", easyconfig.CUSTOM]}
eb = EasyConfig(self.eb_file, extra_options=extra_vars)
self.assertEqual(eb['custom_key'], 'default')
eb['custom_key'] = "not so default"
self.assertEqual(eb['custom_key'], 'not so default')
self.contents += "\ncustom_key = 'test'"
self.prep()
eb = EasyConfig(self.eb_file, extra_options=extra_vars)
self.assertEqual(eb['custom_key'], 'test')
eb['custom_key'] = "not so default"
self.assertEqual(eb['custom_key'], 'not so default')
# test if extra toolchain options are being passed
self.assertEqual(eb.toolchain.options['static'], True)
# test extra mandatory parameters
extra_vars.update({'mandatory_key': ['default', 'another mandatory key', easyconfig.MANDATORY]})
self.assertErrorRegex(EasyBuildError, r"mandatory parameters not provided",
EasyConfig, self.eb_file, extra_options=extra_vars)
self.contents += '\nmandatory_key = "value"'
self.prep()
eb = EasyConfig(self.eb_file, extra_options=extra_vars)
self.assertEqual(eb['mandatory_key'], 'value')
def test_parse_yeb(self):
"""Test parsing of .yeb easyconfigs."""
if 'yaml' not in sys.modules:
print "Skipping test_parse_yeb (no PyYAML available)"
return
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
easybuild.tools.build_log.EXPERIMENTAL = True
testdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(testdir, 'easyconfigs')
test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
# test parsing
test_files = [
'bzip2-1.0.6-GCC-4.9.2',
'gzip-1.6-GCC-4.9.2',
'goolf-1.4.10',
'ictce-4.1.13',
'SQLite-3.8.10.2-goolf-1.4.10',
'Python-2.7.10-ictce-4.1.13',
'CrayCCE-5.1.29',
'toy-0.0',
]
for filename in test_files:
ec_yeb = EasyConfig(
os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename))
# compare with parsed result of .eb easyconfig
ec_file = glob.glob(
os.path.join(test_easyconfigs, 'test_ecs', '*', '*',
'%s.eb' % filename))[0]
ec_eb = EasyConfig(ec_file)
no_match = False
for key in sorted(ec_yeb.asdict()):
eb_val = ec_eb[key]
yeb_val = ec_yeb[key]
if key == 'description':
# multi-line string is always terminated with '\n' in YAML, so strip it off
yeb_val = yeb_val.strip()
self.assertEqual(yeb_val, eb_val)
def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
test_easyconfigs = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
ec = EasyConfig(os.path.join(test_easyconfigs,
'toy-0.0-gompi-1.3.12-test.eb'),
validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(
pkgdir,
'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("Contents of installdir %s" %
easyblock.installdir)
self.assertTrue(
pkgtxt_regex.search(pkgtxt),
"Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
def test_mandatory(self):
""" make sure all checking of mandatory variables works """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
])
self.prep()
self.assertErrorRegex(EasyBuildError,
"mandatory variables .* not provided",
EasyConfig, self.eb_file)
self.contents += '\n' + '\n'.join([
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
])
self.prep()
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
self.assertEqual(eb['name'], "pi")
self.assertEqual(eb['version'], "3.14")
self.assertEqual(eb['homepage'], "http://google.com")
self.assertEqual(eb['toolchain'], {
"name": "dummy",
"version": "dummy"
})
self.assertEqual(eb['description'], "test easyconfig")
def test_exts_list(self):
"""Test handling of list of extensions."""
os.environ['EASYBUILD_SOURCEPATH'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
init_config()
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = [',
' ("ext1", "ext_ver1", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": ["http://example.com/"]',
' }),',
' ("ext2", "ext_ver2", {',
' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail
' "source_urls": [("http://example.com", "suffix")],'
' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail
' "checksums": [',
' "504c7036558938f997c1c269a01d7458",', # checksum for source (gzip-1.4.eb)
' "ddd5161154f5db67701525123129ff09",', # checksum for patch (toy-0.0.eb)
' ],',
' }),',
']',
])
self.prep()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
exts_sources = eb.fetch_extension_sources()
def runTest(self):
""" test easyconfig templating """
eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops)
eb.validate()
eb.generate_template_values()
self.assertEqual(eb['description'], "test easyconfig PI")
const_dict = dict([(x[0], x[1]) for x in easyconfig.templates.TEMPLATE_CONSTANTS])
self.assertEqual(eb['sources'][0], const_dict['SOURCE_TAR_GZ'] % self.inp)
self.assertEqual(eb['sources'][1][0], const_dict['SOURCELOWER_TAR_GZ'] % self.inp)
self.assertEqual(eb['sources'][1][1], 'tar xfvz %s')
self.assertEqual(eb['source_urls'][0], const_dict['GOOGLECODE_SOURCE'] % self.inp)
# test the escaping insanity here (ie all the crap we allow in easyconfigs)
eb['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s"
self.assertEqual(eb['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s")
def test_toolchain(self):
"""Test whether toolchain is initialized correctly."""
ec_file = find_full_path(
os.path.join('test', 'framework', 'easyconfigs', 'gzip-1.4.eb'))
ec = EasyConfig(ec_file, validate=False)
tc = ec.toolchain
self.assertTrue('debug' in tc.options)
def test_skip_extensions_step(self):
"""Test the skip_extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1", "ext2"]',
'exts_filter = ("if [ %(ext_name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")',
'exts_defaultclass = "DummyExtension"',
])
# check if skip skips correct extensions
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.skip = True
eb.extensions_step(fetch=True)
# 'ext1' should be in eb.exts
self.assertTrue('ext1' in [y for x in eb.exts for y in x.values()])
# 'ext2' should not
self.assertFalse('ext2' in [y for x in eb.exts for y in x.values()])
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search("^prepend-path\s+CPATH\s+.*$", guess, re.M))
# cleanup
eb.close_log()
os.remove(eb.logfile)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True):
"""
Process easyconfig, returning some information for each block
"""
blocks = retrieve_blocks_in_spec(path, onlyBlocks)
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
# - use mod? __init__ and importCfg are ignored.
log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
all_stops = [x[0] for x in EasyBlock.get_steps()]
ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
log.exception(msg)
name = ec['name']
# this app will appear as following module in the list
easyconfig = {
'spec': spec,
'module': (ec.name, ec.get_installversion()),
'dependencies': []
}
if len(blocks) > 1:
easyconfig['originalSpec'] = path
for d in ec.dependencies():
dep = (d['name'], d['tc'])
log.debug("Adding dependency %s for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
if ec.toolchain.name != 'dummy':
dep = (ec.toolchain.name, ec.toolchain.version)
log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
del ec
# this is used by the parallel builder
easyconfig['unresolvedDependencies'] = copy.copy(easyconfig['dependencies'])
easyconfigs.append(easyconfig)
def test_hierarchical_mns(self):
"""Test hierarchical module naming scheme."""
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
ec = EasyConfig(os.path.join(ecs_dir, 'GCC-4.7.2.eb'))
self.assertEqual(ActiveMNS().det_full_module_name(ec),
'Core/GCC/4.7.2')
self.assertEqual(ActiveMNS().det_short_module_name(ec), 'GCC/4.7.2')
self.assertEqual(ActiveMNS().det_module_subdir(ec), 'Core')
self.assertEqual(ActiveMNS().det_modpath_extensions(ec),
['Compiler/GCC/4.7.2'])
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core'])
ec = EasyConfig(os.path.join(ecs_dir, 'OpenMPI-1.6.4-GCC-4.7.2.eb'))
self.assertEqual(ActiveMNS().det_full_module_name(ec),
'Compiler/GCC/4.7.2/OpenMPI/1.6.4')
self.assertEqual(ActiveMNS().det_short_module_name(ec),
'OpenMPI/1.6.4')
self.assertEqual(ActiveMNS().det_module_subdir(ec),
'Compiler/GCC/4.7.2')
self.assertEqual(ActiveMNS().det_modpath_extensions(ec),
['MPI/GCC/4.7.2/OpenMPI/1.6.4'])
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core'])
ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(ActiveMNS().det_full_module_name(ec),
'MPI/GCC/4.7.2/OpenMPI/1.6.4/gzip/1.5')
self.assertEqual(ActiveMNS().det_short_module_name(ec), 'gzip/1.5')
self.assertEqual(ActiveMNS().det_module_subdir(ec),
'MPI/GCC/4.7.2/OpenMPI/1.6.4')
self.assertEqual(ActiveMNS().det_modpath_extensions(ec), [])
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core'])
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
def test_parse_yeb(self):
"""Test parsing of .yeb easyconfigs."""
if 'yaml' not in sys.modules:
print "Skipping test_parse_yeb (no PyYAML available)"
return
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
easybuild.tools.build_log.EXPERIMENTAL = True
testdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(testdir, 'easyconfigs')
test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
# test parsing
test_files = [
'bzip2-1.0.6-GCC-4.9.2',
'gzip-1.6-GCC-4.9.2',
'goolf-1.4.10',
'ictce-4.1.13',
'SQLite-3.8.10.2-goolf-1.4.10',
'Python-2.7.10-ictce-4.1.13',
'CrayCCE-5.1.29',
'toy-0.0',
]
for filename in test_files:
ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename))
# compare with parsed result of .eb easyconfig
ec_file = glob.glob(os.path.join(test_easyconfigs, 'test_ecs', '*', '*', '%s.eb' % filename))[0]
ec_eb = EasyConfig(ec_file)
no_match = False
for key in sorted(ec_yeb.asdict()):
eb_val = ec_eb[key]
yeb_val = ec_yeb[key]
if key == 'description':
# multi-line string is always terminated with '\n' in YAML, so strip it off
yeb_val = yeb_val.strip()
self.assertEqual(yeb_val, eb_val)
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [('GCC', '4.6.4')]
hiddendeps = [('toy', '0.0-deps')]
alldeps = deps + hiddendeps # hidden deps must be included in list of deps
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = %s" % str(alldeps),
"hiddendependencies = %s" % str(hiddendeps),
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M))
self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
for (key, val) in modextravars.items():
regex = re.compile('^setenv\s+%s\s+"%s"$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
regex = re.compile('^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in deps:
regex = re.compile('^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in hiddendeps:
regex = re.compile('^\s*module load %s/.%s\s*$' % (name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths):
"""Test whether active module naming scheme returns expected values."""
ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*','*', ecfile))[0])
self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname))
self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname)
self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir)
self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts)
self.assertEqual(ActiveMNS().det_user_modpath_extensions(ec), user_modpath_exts)
self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths)
def template_module_only_test(self,
easyblock,
name='foo',
version='1.3.2',
extra_txt=''):
"""Test whether all easyblocks are compatible with --module-only."""
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# read easyblock Python module
f = open(easyblock, "r")
txt = f.read()
f.close()
# obtain easyblock class name using regex
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_options = app_class.extra_options()
for (key, val) in extra_options.items():
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# run all steps, most should be skipped
orig_workdir = os.getcwd()
try:
app.run_all_steps(run_test_cases=False)
finally:
os.chdir(orig_workdir)
modfile = os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.3.2')
luamodfile = '%s.lua' % modfile
self.assertTrue(
os.path.exists(modfile) or os.path.exists(luamodfile),
"Module file %s or %s was generated" % (modfile, luamodfile))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_init_test(self, easyblock):
"""Test whether all easyconfigs can be initialized."""
def check_extra_options_format(extra_options):
"""Make sure extra_options value is of correct format."""
# EasyBuild v1.x
self.assertTrue(isinstance(extra_options, list))
for extra_option in extra_options:
self.assertTrue(isinstance(extra_option, tuple))
self.assertEqual(len(extra_option), 2)
self.assertTrue(isinstance(extra_option[0], basestring))
self.assertTrue(isinstance(extra_option[1], list))
self.assertEqual(len(extra_option[1]), 3)
# EasyBuild v2.0 (breaks backward compatibility compared to v1.x)
#self.assertTrue(isinstance(extra_options, dict))
#for key in extra_options:
# self.assertTrue(isinstance(extra_options[key], list))
# self.assertTrue(len(extra_options[key]), 3)
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# obtain easyblock class name using regex
f = open(easyblock, "r")
txt = f.read()
f.close()
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_easyblock_class(ebname)
extra_options = app_class.extra_options()
check_extra_options_format(extra_options)
# extend easyconfig to make sure mandatory custom easyconfig paramters are defined
extra_txt = ''
for (key, val) in extra_options:
if val[2] == MANDATORY:
extra_txt += '%s = "foo"\n' % key
# write easyconfig file
self.writeEC(ebname, extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(EasyConfig(self.eb_file))
# cleanup
app.close_log()
os.remove(app.logfile)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def test_external_module_toolchain(self):
"""Test specifying external (build) dependencies."""
ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb', 'CrayCCE-5.1.29.yeb')
metadata = {
'name': ['foo', 'bar'],
'version': ['1.2.3', '3.2.1'],
'prefix': '/foo/bar',
}
build_options = {
'external_modules_metadata': {'fftw/3.3.4.0': metadata},
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
easybuild.tools.build_log.EXPERIMENTAL = True
ec = EasyConfig(ecpath)
self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0')
self.assertEqual(ec.dependencies()[1]['external_module_metadata'], metadata)
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'], det_full_ec_version(ec))
return fn
def test_exclude_path_to_top_of_module_tree(self):
"""
Make sure that modules under the HierarchicalMNS are correct,
w.r.t. not including any load statements for modules that build up the path to the top of the module tree.
"""
self.orig_module_naming_scheme = config.get_module_naming_scheme()
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all')
mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True)
mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True)
impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049')
imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144')
if get_module_syntax() == 'Lua':
impi_modfile_path += '.lua'
imkl_modfile_path += '.lua'
# example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included
# not for the toolchain or any of the toolchain components,
# since both icc/ifort and impi form the path to the top of the module tree
tests = [
('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']),
('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']),
]
for ec_file, modfile_path, excluded_deps in tests:
ec = EasyConfig(os.path.join(test_ecs_path, ec_file))
eb = EasyBlock(ec)
eb.toolchain.prepare()
modpath = eb.make_module_step()
modfile_path = os.path.join(modpath, modfile_path)
modtxt = read_file(modfile_path)
for dep in excluded_deps:
tup = (dep, modfile_path, modtxt)
failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
if get_module_syntax() == 'Tcl':
self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg)
elif get_module_syntax() == 'Lua':
self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
eb['dependencies'] = ["wrong type"]
self.assertErrorRegex(EasyBuildError,
"wrong type from unsupported type",
eb.dependencies)
eb['dependencies'] = [()]
self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies)
eb['dependencies'] = [{'name': "test"}]
self.assertErrorRegex(EasyBuildError, "without version",
eb.dependencies)
def test_toolchain_inspection(self):
"""Test whether available toolchain inspection functionality is working."""
build_options = {
'robot_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'),
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb'))
self.assertEqual(['/'.join([x['name'], x['version']]) for x in det_toolchain_compilers(ec)], ['GCC/4.7.2'])
self.assertEqual(det_toolchain_mpi(ec)['name'], 'OpenMPI')
ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'hwloc-1.6.2-GCC-4.6.4.eb'))
tc_comps = det_toolchain_compilers(ec)
self.assertEqual(['/'.join([x['name'], x['version']]) for x in tc_comps], ['GCC/4.6.4'])
self.assertEqual(det_toolchain_mpi(ec), None)
ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'))
self.assertEqual(det_toolchain_compilers(ec), None)
self.assertEqual(det_toolchain_mpi(ec), None)
def test_make_module_extra(self):
"""Test for make_module_extra."""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
if get_module_syntax() == 'Tcl':
expected_default = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+\"\$root"',
r'setenv\s+EBVERSIONPI\s+"3.14"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"',
r'setenv\s+EBVERSIONPI\s+"6.28"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
elif get_module_syntax() == 'Lua':
expected_default = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", root\)',
r'setenv\("EBVERSIONPI", "3.14"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)',
r'setenv\("EBVERSIONPI", "6.28"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
defaulttxt = eb.make_module_extra().strip()
self.assertTrue(
expected_default.match(defaulttxt),
"Pattern %s found in %s" % (expected_default.pattern, defaulttxt))
alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28',
altversion='6.28').strip()
self.assertTrue(
expected_alt.match(alttxt),
"Pattern %s found in %s" % (expected_alt.pattern, alttxt))
def test_active_pns(self):
"""Test use of ActivePNS."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
ec = EasyConfig(os.path.join(test_easyconfigs, 'OpenMPI-1.6.4-GCC-4.6.4.eb'), validate=False)
pns = ActivePNS()
# default: EasyBuild package naming scheme, pkg release 1
self.assertEqual(pns.name(ec), 'OpenMPI-1.6.4-GCC-4.6.4')
self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION)
self.assertEqual(pns.release(ec), '1')
def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
topdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy',
'toy-0.0-gompi-1.3.12-test.eb'),
validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# write a dummy log and report file to make sure they don't get packaged
logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log")
write_file(logfile, "I'm a logfile")
reportfile = os.path.join(easyblock.installdir, log_path(),
"report.md")
write_file(reportfile, "I'm a reportfile")
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(
pkgdir,
'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" %
easyblock.installdir)
self.assertTrue(
pkgtxt_regex.search(pkgtxt),
"Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
no_logfiles_regex = re.compile(
r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS',
re.DOTALL | re.MULTILINE)
self.assertFalse(
no_logfiles_regex.search(pkgtxt), "Pattern not '%s' found in: %s" %
(no_logfiles_regex.pattern, pkgtxt))
if DEBUG:
print "The FPM script debug output"
print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
print "The Package File"
print read_file(pkgfile)
def runTest(self):
""" test other validations beside mandatory variables """
eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops)
self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", eb.validate)
eb['stop'] = 'patch'
# this should now not crash
eb.validate()
eb['osdependencies'] = ['non-existent-dep']
self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", eb.validate)
# dummy toolchain, installversion == version
self.assertEqual(eb.get_installversion(), "3.14")
os.chmod(self.eb_file, 0000)
self.assertErrorRegex(EasyBuildError, "Unexpected IOError", EasyConfig, self.eb_file)
os.chmod(self.eb_file, 0755)
self.contents += "\nsyntax_error'"
self.setUp()
self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo")
self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ())
self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'})
err_msg = "Incorrect external dependency specification"
self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,))
self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER))
def runTest(self):
ver = "1.2.3"
verpref = "myprefix"
versuff = "mysuffix"
tcname = "mytc"
tcver = "4.1.2"
extra_patches = ['t5.patch', 't6.patch']
homepage = "http://www.justatest.com"
tweaks = {
'version': ver,
'versionprefix': verpref,
'versionsuffix': versuff,
'toolchain_version': tcver,
'patches': extra_patches
}
tweak(self.eb_file, self.tweaked_fn, tweaks)
eb = EasyConfig(self.tweaked_fn, valid_stops=self.all_stops)
self.assertEqual(eb['version'], ver)
self.assertEqual(eb['versionprefix'], verpref)
self.assertEqual(eb['versionsuffix'], versuff)
self.assertEqual(eb['toolchain']['version'], tcver)
self.assertEqual(eb['patches'], extra_patches + self.patches)
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# eb['toolchain']['version'] = tcver does not work as expected with templating enabled
eb.enable_templating = False
eb['version'] = ver
eb['toolchain']['version'] = tcver
eb.enable_templating = True
eb.dump(self.eb_file)
tweaks = {
'toolchain_name': tcname,
'patches': extra_patches[0:1],
'homepage': homepage,
'foo': "bar"
}
tweak(self.eb_file, self.tweaked_fn, tweaks)
eb = EasyConfig(self.tweaked_fn, valid_stops=self.all_stops)
self.assertEqual(eb['toolchain']['name'], tcname)
self.assertEqual(eb['toolchain']['version'], tcver)
self.assertEqual(eb['patches'], extra_patches[0:1] + self.patches)
self.assertEqual(eb['version'], ver)
self.assertEqual(eb['homepage'], homepage)
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
eb['dependencies'] = ["wrong type"]
self.assertErrorRegex(EasyBuildError, "wrong type from unsupported type", eb.dependencies)
eb['dependencies'] = [()]
self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies)
eb['dependencies'] = [{'name': "test"}]
self.assertErrorRegex(EasyBuildError, "without version", eb.dependencies)
def test_external_dependencies(self):
"""Test specifying external (build) dependencies."""
ectxt = read_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb'))
toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb')
# just specify some of the test modules we ship, doesn't matter where they come from
ectxt += "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]"
ectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]"
write_file(toy_ec, ectxt)
build_options = {
'valid_module_classes': module_classes(),
'external_modules_metadata': ConfigObj(),
}
init_config(build_options=build_options)
ec = EasyConfig(toy_ec)
builddeps = ec.builddependencies()
self.assertEqual(len(builddeps), 1)
self.assertEqual(builddeps[0]['short_mod_name'], 'somebuilddep/0.1')
self.assertEqual(builddeps[0]['full_mod_name'], 'somebuilddep/0.1')
self.assertEqual(builddeps[0]['external_module'], True)
deps = ec.dependencies()
self.assertEqual(len(deps), 4)
correct_deps = ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'somebuilddep/0.1']
self.assertEqual([d['short_mod_name'] for d in deps], correct_deps)
self.assertEqual([d['full_mod_name'] for d in deps], correct_deps)
self.assertEqual([d['external_module'] for d in deps], [False, True, True, True])
metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg')
metadatatxt = '\n'.join(['[foobar/1.2.3]', 'name = foo,bar', 'version = 1.2.3,3.2.1', 'prefix = /foo/bar'])
write_file(metadata, metadatatxt)
cfg = init_config(args=['--external-modules-metadata=%s' % metadata])
build_options = {
'external_modules_metadata': cfg.external_modules_metadata,
'valid_module_classes': module_classes(),
}
init_config(build_options=build_options)
ec = EasyConfig(toy_ec)
self.assertEqual(ec.dependencies()[2]['short_mod_name'], 'foobar/1.2.3')
self.assertEqual(ec.dependencies()[2]['external_module'], True)
metadata = {
'name': ['foo', 'bar'],
'version': ['1.2.3', '3.2.1'],
'prefix': '/foo/bar',
}
self.assertEqual(ec.dependencies()[2]['external_module_metadata'], metadata)
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file, build_options={'valid_stops': self.all_stops})
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo")
self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ())
self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'})
def runTest(self):
""" test all possible ways of specifying dependencies """
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(first['tc'], '1.1-GCC-4.6.3')
self.assertEqual(second['tc'], '2.2-GCC-4.6.3')
eb['dependencies'] = ["wrong type"]
self.assertErrorRegex(EasyBuildError, "wrong type from unsupported type", eb.dependencies)
eb['dependencies'] = [()]
self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies)
eb['dependencies'] = [{'name': "test"}]
self.assertErrorRegex(EasyBuildError, "without version", eb.dependencies)
def test_update(self):
"""Test use of update() method for EasyConfig instances."""
toy_ebfile = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb')
ec = EasyConfig(toy_ebfile)
# for string values: append
ec.update('unpack_options', '--strip-components=1')
self.assertEqual(ec['unpack_options'].strip(), '--strip-components=1')
ec.update('description', "- just a test")
self.assertEqual(ec['description'].strip(), "Toy C program. - just a test")
# spaces in between multiple updates for stirng values
ec.update('configopts', 'CC="$CC"')
ec.update('configopts', 'CXX="$CXX"')
self.assertTrue(ec['configopts'].strip().endswith('CC="$CC" CXX="$CXX"'))
# for list values: extend
ec.update('patches', ['foo.patch', 'bar.patch'])
self.assertEqual(ec['patches'], ['toy-0.0_typo.patch', 'foo.patch', 'bar.patch'])
def test_tweaking(self):
"""test tweaking ability of easyconfigs"""
fd, tweaked_fn = tempfile.mkstemp(prefix='easybuild-tweaked-', suffix='.eb')
os.close(fd)
patches = ["t1.patch", ("t2.patch", 1), ("t3.patch", "test"), ("t4.h", "include")]
self.contents = '\n'.join([
'name = "pi"',
'homepage = "http://www.google.com"',
'description = "dummy description"',
'version = "3.14"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'patches = %s',
]) % str(patches)
self.prep()
ver = "1.2.3"
verpref = "myprefix"
versuff = "mysuffix"
tcname = "mytc"
tcver = "4.1.2"
new_patches = ['t5.patch', 't6.patch']
homepage = "http://www.justatest.com"
tweaks = {
'version': ver,
'versionprefix': verpref,
'versionsuffix': versuff,
'toolchain_version': tcver,
'patches': new_patches
}
tweak(self.eb_file, tweaked_fn, tweaks)
eb = EasyConfig(tweaked_fn, valid_stops=self.all_stops)
self.assertEqual(eb['version'], ver)
self.assertEqual(eb['versionprefix'], verpref)
self.assertEqual(eb['versionsuffix'], versuff)
self.assertEqual(eb['toolchain']['version'], tcver)
self.assertEqual(eb['patches'], new_patches)
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# eb['toolchain']['version'] = tcver does not work as expected with templating enabled
eb.enable_templating = False
eb['version'] = ver
eb['toolchain']['version'] = tcver
eb.enable_templating = True
eb.dump(self.eb_file)
tweaks = {
'toolchain_name': tcname,
'patches': new_patches[:1],
'homepage': homepage,
'foo': "bar"
}
tweak(self.eb_file, tweaked_fn, tweaks)
eb = EasyConfig(tweaked_fn, valid_stops=self.all_stops)
self.assertEqual(eb['toolchain']['name'], tcname)
self.assertEqual(eb['toolchain']['version'], tcver)
self.assertEqual(eb['patches'], new_patches[:1])
self.assertEqual(eb['version'], ver)
self.assertEqual(eb['homepage'], homepage)
# specify patches as string, eb should promote it to a list because original value was a list
tweaks['patches'] = new_patches[0]
eb = EasyConfig(tweaked_fn, valid_stops=self.all_stops)
self.assertEqual(eb['patches'], [new_patches[0]])
# cleanup
os.remove(tweaked_fn)
def test_obtain_easyconfig(self):
"""test obtaining an easyconfig file given certain specifications"""
tcname = 'GCC'
tcver = '4.3.2'
patches = ["one.patch"]
# prepare a couple of eb files to test again
fns = ["pi-3.14.eb",
"pi-3.13-GCC-4.3.2.eb",
"pi-3.15-GCC-4.3.2.eb",
"pi-3.15-GCC-4.4.5.eb",
"foo-1.2.3-GCC-4.3.2.eb"]
eb_files = [(fns[0], "\n".join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.12"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'patches = %s' % patches
])),
(fns[1], "\n".join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.13"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver),
'patches = %s' % patches
])),
(fns[2], "\n".join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.15"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver),
'patches = %s' % patches
])),
(fns[3], "\n".join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.15"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "4.5.1"}' % tcname,
'patches = %s' % patches
])),
(fns[4], "\n".join([
'easyblock = "ConfigureMake"',
'name = "foo"',
'version = "1.2.3"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver),
'foo_extra1 = "bar"',
]))
]
self.ec_dir = tempfile.mkdtemp()
for (fn, txt) in eb_files:
write_file(os.path.join(self.ec_dir, fn), txt)
# should crash when no suited easyconfig file (or template) is available
specs = {'name': 'nosuchsoftware'}
error_regexp = ".*No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % specs['name']
self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None)
# should find matching easyconfig file
specs = {
'name': 'foo',
'version': '1.2.3'
}
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], False)
self.assertEqual(res[1], os.path.join(self.ec_dir, fns[-1]))
# should not pick between multiple available toolchain names
name = "pi"
ver = "3.12"
suff = "mysuff"
specs.update({
'name': name,
'version': ver,
'versionsuffix': suff
})
error_regexp = ".*No toolchain name specified, and more than one available: .*"
self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None)
# should be able to generate an easyconfig file that slightly differs
ver = '3.16'
specs.update({
'toolchain_name': tcname,
'toolchain_version': tcver,
'version': ver,
'start_dir': 'bar123'
})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[1], "%s-%s-%s-%s%s.eb" % (name, ver, tcname, tcver, suff))
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['name'], specs['name'])
self.assertEqual(ec['version'], specs['version'])
self.assertEqual(ec['versionsuffix'], specs['versionsuffix'])
self.assertEqual(ec['toolchain'], {'name': tcname, 'version': tcver})
self.assertEqual(ec['start_dir'], specs['start_dir'])
os.remove(res[1])
specs.update({
'foo': 'bar123'
})
self.assertErrorRegex(EasyBuildError, "Unkown easyconfig parameter: foo",
obtain_ec_for, specs, [self.ec_dir], None)
del specs['foo']
# should pick correct version, i.e. not newer than what's specified, if a choice needs to be made
ver = '3.14'
specs.update({'version': ver})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['version'], specs['version'])
txt = read_file(res[1])
self.assertTrue(re.search("^version = [\"']%s[\"']$" % ver, txt, re.M))
os.remove(res[1])
# should pick correct toolchain version as well, i.e. now newer than what's specified, if a choice needs to be made
specs.update({
'version': '3.15',
'toolchain_version': '4.4.5',
})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['version'], specs['version'])
self.assertEqual(ec['toolchain']['version'], specs['toolchain_version'])
txt = read_file(res[1])
pattern = "^toolchain = .*version.*[\"']%s[\"'].*}$" % specs['toolchain_version']
self.assertTrue(re.search(pattern, txt, re.M))
os.remove(res[1])
# should be able to prepend to list of patches and handle list of dependencies
new_patches = ['two.patch', 'three.patch']
specs.update({
'patches': new_patches[:],
'dependencies': [('foo', '1.2.3'), ('bar', '666', '-bleh', ('gompi', '1.4.10'))],
'hiddendependencies': [('test', '3.2.1')],
})
parsed_deps = [
{
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '',
'toolchain': ec['toolchain'],
'dummy': False,
'short_mod_name': 'foo/1.2.3-GCC-4.4.5',
'full_mod_name': 'foo/1.2.3-GCC-4.4.5',
'hidden': False,
'external_module': False,
'external_module_metadata': {},
},
{
'name': 'bar',
'version': '666',
'versionsuffix': '-bleh',
'toolchain': {'name': 'gompi', 'version': '1.4.10'},
'dummy': False,
'short_mod_name': 'bar/666-gompi-1.4.10-bleh',
'full_mod_name': 'bar/666-gompi-1.4.10-bleh',
'hidden': False,
'external_module': False,
'external_module_metadata': {},
},
{
'name': 'test',
'version': '3.2.1',
'versionsuffix': '',
'toolchain': ec['toolchain'],
'dummy': False,
'short_mod_name': 'test/.3.2.1-GCC-4.4.5',
'full_mod_name': 'test/.3.2.1-GCC-4.4.5',
'hidden': True,
'external_module': False,
'external_module_metadata': {},
},
]
# hidden dependencies must be included in list of dependencies
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
error_pattern = "Hidden dependencies with visible module names .* not in list of dependencies: .*"
self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, res[1])
specs['dependencies'].append(('test', '3.2.1'))
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['patches'], specs['patches'])
self.assertEqual(ec.dependencies(), parsed_deps)
# hidden dependencies are filtered from list of dependencies
self.assertFalse('test/3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['dependencies']])
self.assertTrue('test/.3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['hiddendependencies']])
os.remove(res[1])
# hidden dependencies are also filtered from list of dependencies when validation is skipped
res = obtain_ec_for(specs, [self.ec_dir], None)
ec = EasyConfig(res[1], validate=False)
self.assertFalse('test/3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['dependencies']])
self.assertTrue('test/.3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['hiddendependencies']])
os.remove(res[1])
# verify append functionality for lists
specs['patches'].insert(0, '')
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['patches'], patches + new_patches)
specs['patches'].remove('')
os.remove(res[1])
# verify prepend functionality for lists
specs['patches'].append('')
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['patches'], new_patches + patches)
os.remove(res[1])
# should use supplied filename
fn = "my.eb"
res = obtain_ec_for(specs, [self.ec_dir], fn)
self.assertEqual(res[0], True)
self.assertEqual(res[1], fn)
os.remove(res[1])
# should use a template if it's there
tpl_path = os.path.join("share", "easybuild", "easyconfigs", "TEMPLATE.eb")
def trim_path(path):
dirs = path.split(os.path.sep)
if len(dirs) > 3 and 'site-packages' in dirs:
if path.endswith('.egg'):
path = os.path.join(*dirs[:-4]) # strip of lib/python2.7/site-packages/*.egg part
else:
path = os.path.join(*dirs[:-3]) # strip of lib/python2.7/site-packages part
return path
tpl_full_path = find_full_path(tpl_path, trim=trim_path)
# only run this test if the TEMPLATE.eb file is available
# TODO: use unittest.skip for this (but only works from Python 2.7)
if tpl_full_path:
shutil.copy2(tpl_full_path, self.ec_dir)
specs.update({'name': 'nosuchsoftware'})
res = obtain_ec_for(specs, [self.ec_dir], None)
self.assertEqual(res[0], True)
ec = EasyConfig(res[1])
self.assertEqual(ec['name'], specs['name'])
os.remove(res[1])
# cleanup
shutil.rmtree(self.ec_dir)
def test_robot_find_minimal_toolchain_of_dependency(self):
"""Test robot_find_minimal_toolchain_of_dependency."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
#
# First test that it can do basic resolution
#
gzip15 = {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
get_toolchain_hierarchy.clear()
new_gzip15_toolchain = robot_find_minimal_toolchain_of_dependency(gzip15, self.modtool)
self.assertEqual(new_gzip15_toolchain, gzip15['toolchain'])
# no easyconfig for gzip 1.4 with matching non-dummy (sub)toolchain
gzip14 = {
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
get_toolchain_hierarchy.clear()
self.assertEqual(robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool), None)
gzip14['toolchain'] = {'name': 'gompi', 'version': '1.4.10'}
#
# Second test also including dummy toolchain
#
init_config(build_options={
'add_dummy_to_minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
# specify alternative parent toolchain
gompi_1410 = {'name': 'gompi', 'version': '1.4.10'}
get_toolchain_hierarchy.clear()
new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool, parent_tc=gompi_1410)
self.assertTrue(new_gzip14_toolchain != gzip14['toolchain'])
self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''})
# default: use toolchain from dependency
gzip14['toolchain'] = gompi_1410
get_toolchain_hierarchy.clear()
new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool)
self.assertTrue(new_gzip14_toolchain != gzip14['toolchain'])
self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''})
# check reversed order (parent tc first) and skipping of parent tc itself
dep = {
'name': 'SQLite',
'version': '3.8.10.2',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool)
self.assertEqual(res, {'name': 'GCC', 'version': '4.7.2'})
res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool, parent_first=True)
self.assertEqual(res, {'name': 'goolf', 'version': '1.4.10'})
#
# Finally test if it can recognise existing modules and use those
#
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_txt = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),", # available with goolf/1.4.10, gompi/1.4.10 and GCC/4.7.2
"]",
])
write_file(barec, barec_txt)
# check without --minimal-toolchains
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
bar = EasyConfig(barec)
expected_dep_versions = [
'1.6.4-GCC-4.7.2',
'0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'3.8.10.2-goolf-1.4.10',
]
for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions):
self.assertEqual(det_full_ec_version(dep), expected_dep_version)
# check with --minimal-toolchains enabled
init_config(build_options={
'minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
bar = EasyConfig(barec)
# check that all bar dependencies have been processed as expected
expected_dep_versions[-1] = '3.8.10.2-GCC-4.7.2'
for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions):
self.assertEqual(det_full_ec_version(dep), expected_dep_version)
# Add the gompi/1.4.10 version of SQLite as an available module
module_parent = os.path.join(self.test_prefix, 'minimal_toolchain_modules')
module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-gompi-1.4.10')
module_txt = '\n'.join([
"#%Module",
"set root /tmp/SQLite/3.8.10.2",
"setenv EBROOTSQLITE $root",
"setenv EBVERSIONSQLITE 3.8.10.2",
"setenv EBDEVELSQLITE $root/easybuild/SQLite-3.8.10.2-easybuild-devel",
])
write_file(module_file, module_txt)
os.environ['MODULEPATH'] = module_parent # Add the parent directory to the MODULEPATH
invalidate_module_caches_for(module_parent)
# Reinitialize the environment for the updated MODULEPATH and use_existing_modules
init_config(build_options={
'minimal_toolchains': True,
'use_existing_modules': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
# Check gompi is now being picked up
bar = EasyConfig(barec) # Re-parse the parent easyconfig
sqlite = bar.dependencies()[3]
self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-gompi-1.4.10')
# Add the goolf version as an available version and check that gets precedence over the gompi version
module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-goolf-1.4.10')
write_file(module_file, module_txt)
invalidate_module_caches_for(module_parent)
bar = EasyConfig(barec) # Re-parse the parent easyconfig
sqlite = bar.dependencies()[3]
self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-goolf-1.4.10')
# fs.walk yields the same results as os.walk, so should be interchangable
# same for fs.join and os.path.join
for root, subfolders, files in walk(options.path):
if '.git' in subfolders:
log.info("found .git subfolder, ignoring it")
subfolders.remove('.git')
for ec_file in files:
if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]:
log.warning("SKIPPING %s/%s" % (root, ec_file))
continue
ec_file = join(root, ec_file)
ec_file = read(ec_file)
try:
ec = EasyConfig(ec_file)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec)
# check if an easyblock exists
module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
else:
ec.easyblock = None
configs.append(ec)
names.append(ec.name)
except Exception, err:
log.error("faulty easyconfig %s: %s" % (ec_file, err))
log.info("Found easyconfigs: %s" % [x.name for x in configs])
