def test_extensions_step(self):
"""Test the extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1"]',
])
self.writeEC()
"""Testcase for extensions"""
# test for proper error message without the exts_defaultclass set
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
self.assertRaises(EasyBuildError, eb.extensions_step, fetch=True)
self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step, fetch=True)
# test if everything works fine if set
self.contents += "\nexts_defaultclass = 'DummyExtension'"
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.extensions_step(fetch=True)
# test for proper error message when skip is set, but no exts_filter is set
self.assertRaises(EasyBuildError, eb.skip_extensions)
self.assertErrorRegex(EasyBuildError, "no exts_filter set", eb.skip_extensions)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_install_path(self):
"""Test install_path function."""
# defaults
self.assertEqual(install_path(), os.path.join(self.test_installpath, 'software'))
self.assertEqual(install_path('software'), os.path.join(self.test_installpath, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(self.test_installpath, 'modules'))
self.assertEqual(install_path('modules'), os.path.join(self.test_installpath, 'modules'))
self.assertErrorRegex(EasyBuildError, "Unknown type specified", install_path, typ='foo')
args = [
'--subdir-software', 'SOFT',
'--installpath', '/foo',
]
os.environ['EASYBUILD_SUBDIR_MODULES'] = 'MOD'
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/foo', 'SOFT'))
self.assertEqual(install_path(typ='mod'), os.path.join('/foo', 'MOD'))
del os.environ['EASYBUILD_SUBDIR_MODULES']
args = [
'--installpath', '/prefix',
'--installpath-modules', '/foo',
]
os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] = '/bar/baz'
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/bar', 'baz'))
self.assertEqual(install_path(typ='mod'), '/foo')
del os.environ['EASYBUILD_INSTALLPATH_SOFTWARE']
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/prefix', 'software'))
self.assertEqual(install_path(typ='mod'), '/foo')
def test_generaloption_overrides_legacy(self):
"""Test whether generaloption overrides legacy configuration."""
self.purge_environment()
# if both legacy and generaloption style configuration is mixed, generaloption wins
legacy_prefix = os.path.join(self.tmpdir, 'legacy')
go_prefix = os.path.join(self.tmpdir, 'generaloption')
# legacy env vars
os.environ['EASYBUILDPREFIX'] = legacy_prefix
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build')
# generaloption env vars
os.environ['EASYBUILD_INSTALLPATH'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(legacy_prefix,
'ebfiles_repo'))
del os.environ['EASYBUILDPREFIX']
# legacy env vars
os.environ['EASYBUILDBUILDPATH'] = os.path.join(
legacy_prefix, 'buildagain')
# generaloption env vars
os.environ['EASYBUILD_PREFIX'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(go_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDBUILDPATH']
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--config', config_file, # force empty oldstyle config file
'--prefix', prefix,
'--installpath', install,
'--repositorypath', repopath,
]
options = self.configure_options(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertEqual(options.config, config_file)
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
self.purge_environment()
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = init_config(args=[])
self.assertEqual(build_path(), buildpath_env_var)
self.assertEqual(install_path(), os.path.join(prefix, 'software'))
self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')])
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--config', config_file, # force empty oldstyle config file
'--prefix', prefix,
'--installpath', install,
'--repositorypath', repopath,
]
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertEqual(options.config, config_file)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--config', config_file, # force empty oldstyle config file
'--installpath', install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, subdir_software))
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--config',
config_file, # force empty oldstyle config file
'--prefix',
prefix,
'--installpath',
install,
'--repositorypath',
repopath,
]
options = self.configure_options(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'software'))
self.assertEqual(install_path(typ='mod'),
os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertEqual(options.config, config_file)
def test_generaloption_overrides_legacy(self):
"""Test whether generaloption overrides legacy configuration."""
self.purge_environment()
# if both legacy and generaloption style configuration is mixed, generaloption wins
legacy_prefix = os.path.join(self.tmpdir, 'legacy')
go_prefix = os.path.join(self.tmpdir, 'generaloption')
# legacy env vars
os.environ['EASYBUILDPREFIX'] = legacy_prefix
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build')
# generaloption env vars
os.environ['EASYBUILD_INSTALLPATH'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(legacy_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDPREFIX']
# legacy env vars
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'buildagain')
# generaloption env vars
os.environ['EASYBUILD_PREFIX'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(go_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDBUILDPATH']
def test_install_path(self):
"""Test install_path function."""
# defaults
self.assertEqual(install_path(), os.path.join(self.test_installpath, 'software'))
self.assertEqual(install_path('software'), os.path.join(self.test_installpath, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(self.test_installpath, 'modules'))
self.assertEqual(install_path('modules'), os.path.join(self.test_installpath, 'modules'))
self.assertErrorRegex(EasyBuildError, "Unknown type specified", install_path, typ='foo')
args = [
'--subdir-software', 'SOFT',
'--installpath', '/foo',
]
os.environ['EASYBUILD_SUBDIR_MODULES'] = 'MOD'
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/foo', 'SOFT'))
self.assertEqual(install_path(typ='mod'), os.path.join('/foo', 'MOD'))
del os.environ['EASYBUILD_SUBDIR_MODULES']
args = [
'--installpath', '/prefix',
'--installpath-modules', '/foo',
]
os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] = '/bar/baz'
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/bar', 'baz'))
self.assertEqual(install_path(typ='mod'), '/foo')
del os.environ['EASYBUILD_INSTALLPATH_SOFTWARE']
init_config(args=args)
self.assertEqual(install_path(), os.path.join('/prefix', 'software'))
self.assertEqual(install_path(typ='mod'), '/foo')
def test_extensions_step(self):
"""Test the extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1"]',
])
self.writeEC()
"""Testcase for extensions"""
# test for proper error message without the exts_defaultclass set
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
self.assertRaises(EasyBuildError, eb.extensions_step, fetch=True)
self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step, fetch=True)
# test if everything works fine if set
self.contents += "\nexts_defaultclass = 'DummyExtension'"
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.extensions_step(fetch=True)
# test for proper error message when skip is set, but no exts_filter is set
self.assertRaises(EasyBuildError, eb.skip_extensions)
self.assertErrorRegex(EasyBuildError, "no exts_filter set", eb.skip_extensions)
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = self.configure_options(args=[])
self.assertEqual(build_path(), buildpath_env_var)
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--config', config_file, # force empty oldstyle config file
'--prefix', prefix,
'--installpath', install,
'--repositorypath', repopath,
]
options = self.configure_options(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'software'))
self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertEqual(options.config, config_file)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--config', config_file, # force empty oldstyle config file
'--installpath', install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
options = self.configure_options(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, subdir_software))
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def test_make_module_extend_modpath(self):
"""Test for make_module_extend_modpath"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'moduleclass = "compiler"',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
self.assertEqual(eb.make_module_extend_modpath(), '')
usermodsdir = 'my/own/modules'
modclasses = ['compiler', 'tools']
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
build_options = {
'subdir_user_modules': usermodsdir,
'valid_module_classes': modclasses,
}
init_config(build_options=build_options)
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
txt = eb.make_module_extend_modpath()
if get_module_syntax() == 'Tcl':
regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses]
home = r'\$env\(HOME\)'
regexs.extend([
# extension for user modules is guarded
r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir),
])
elif get_module_syntax() == 'Lua':
regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses]
home = r'os.getenv\("HOME"\)'
regexs.extend([
# extension for user modules is guarded
r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir),
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def test_make_module_extend_modpath(self):
"""Test for make_module_extend_modpath"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'moduleclass = "compiler"',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
self.assertEqual(eb.make_module_extend_modpath(), '')
usermodsdir = 'my/own/modules'
modclasses = ['compiler', 'tools']
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
build_options = {
'subdir_user_modules': usermodsdir,
'valid_module_classes': modclasses,
}
init_config(build_options=build_options)
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
txt = eb.make_module_extend_modpath()
if get_module_syntax() == 'Tcl':
regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses]
home = r'\$env\(HOME\)'
regexs.extend([
# extension for user modules is guarded
r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir),
])
elif get_module_syntax() == 'Lua':
regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses]
home = r'os.getenv\("HOME"\)'
regexs.extend([
# extension for user modules is guarded
r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir),
# no per-moduleclass extension for user modules
r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir),
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def create_files(self):
"""
Creates the absolute filename for the module.
"""
module_path = install_path('mod')
# Fake mode: set installpath to temporary dir
if self.fake:
self.tmpdir = tempfile.mkdtemp()
_log.debug("Fake mode: using %s (instead of %s)" %
(self.tmpdir, module_path))
module_path = self.tmpdir
# Real file goes in 'all' category
self.filename = os.path.join(module_path, GENERAL_CLASS, self.app.name,
self.app.get_installversion())
# Make symlink in moduleclass category
classPath = os.path.join(module_path, self.app.cfg['moduleclass'],
self.app.name)
classPathFile = os.path.join(classPath, self.app.get_installversion())
# Create directories and links
for directory in [
os.path.dirname(x) for x in [self.filename, classPathFile]
]:
if not os.path.isdir(directory):
try:
os.makedirs(directory)
except OSError, err:
_log.exception("Couldn't make directory %s: %s" %
(directory, err))
def check_module_path(self):
"""
Check if MODULEPATH is set and change it if necessary.
"""
# if self.mod_paths is not specified, use $MODULEPATH and make sure the EasyBuild module path is in there (first)
if self.mod_paths is None:
# take module path from environment
self.mod_paths = [
x for x in nub(os.environ.get('MODULEPATH', '').split(':'))
if len(x) > 0
]
self.log.debug("self.mod_paths set based on $MODULEPATH: %s" %
self.mod_paths)
# determine module path for EasyBuild install path to be included in $MODULEPATH
eb_modpath = os.path.join(install_path(typ='modules'),
GENERAL_CLASS)
# make sure EasyBuild module path is in 1st place
self.mod_paths = [x for x in self.mod_paths if not x == eb_modpath]
self.mod_paths.insert(0, eb_modpath)
self.log.info(
"Prepended list of module paths with path used by EasyBuild: %s"
% eb_modpath)
# set the module path environment accordingly
os.environ['MODULEPATH'] = ':'.join(self.mod_paths)
self.log.info("$MODULEPATH set based on list of module paths: %s" %
os.environ['MODULEPATH'])
def load(self, modules, mod_paths=None, purge=False, init_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param init_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired by restoring initial environment
# actually running 'module purge' is futile (and wrong/broken on some systems, e.g. Cray)
if purge:
# restore initial environment if provided
if init_env is None:
raise EasyBuildError(
"Initial environment required when purging before loading, but not available"
)
else:
restore_env(init_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path('mod'),
build_option('suffix_modules_path'),
mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module('load', mod)
def create_files(self):
"""
Creates the absolute filename for the module.
"""
module_path = install_path('mod')
# Fake mode: set installpath to temporary dir
if self.fake:
self.tmpdir = tempfile.mkdtemp()
log.debug("Fake mode: using %s (instead of %s)" % (self.tmpdir, module_path))
module_path = self.tmpdir
# Real file goes in 'all' category
self.filename = os.path.join(module_path, GENERAL_CLASS, self.app.name, self.app.get_installversion())
# Make symlink in moduleclass category
classPath = os.path.join(module_path, self.app.cfg['moduleclass'], self.app.name)
classPathFile = os.path.join(classPath, self.app.get_installversion())
# Create directories and links
for directory in [os.path.dirname(x) for x in [self.filename, classPathFile]]:
if not os.path.isdir(directory):
try:
os.makedirs(directory)
except OSError, err:
log.exception("Couldn't make directory %s: %s" % (directory, err))
def load(self, modules, mod_paths=None, purge=False, orig_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param orig_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired
if purge:
self.purge()
# restore original environment if provided
if orig_env is not None:
modify_env(os.environ, orig_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module('load', mod)
def load(self, modules, mod_paths=None, purge=False, orig_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param orig_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired
if purge:
self.purge()
# restore original environment if provided
if orig_env is not None:
restore_env(orig_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path('mod'),
build_option('suffix_modules_path'),
mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module('load', mod)
def test_skip_extensions_step(self):
"""Test the skip_extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1", "ext2"]',
'exts_filter = ("if [ %(ext_name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")',
'exts_defaultclass = "DummyExtension"',
])
# check if skip skips correct extensions
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.skip = True
eb.extensions_step(fetch=True)
# 'ext1' should be in eb.exts
self.assertTrue('ext1' in [y for x in eb.exts for y in x.values()])
# 'ext2' should not
self.assertFalse('ext2' in [y for x in eb.exts for y in x.values()])
# cleanup
eb.close_log()
os.remove(eb.logfile)
def check_module_path(self):
"""
Check if MODULEPATH is set and change it if necessary.
"""
# if self.mod_paths is not specified, define it and make sure the EasyBuild module path is in there (first)
if self.mod_paths is None:
# take (unique) module paths from environment
self.set_mod_paths()
self.log.debug("self.mod_paths set based on $MODULEPATH: %s" %
self.mod_paths)
# determine module path for EasyBuild install path to be included in $MODULEPATH
eb_modpath = os.path.join(install_path(typ='modules'),
build_option('suffix_modules_path'))
# make sure EasyBuild module path is in 1st place
self.prepend_module_path(eb_modpath)
self.log.info(
"Prepended list of module paths with path used by EasyBuild: %s"
% eb_modpath)
# set the module path environment accordingly
for mod_path in self.mod_paths[::-1]:
self.use(mod_path)
self.log.info(
"$MODULEPATH set based on list of module paths (via 'module use'): %s"
% os.environ['MODULEPATH'])
def create_files(self):
"""
Creates the absolute filename for the module.
"""
module_path = config.install_path('mod')
# Fake mode: set installpath to temporary dir
if self.fake:
self.tmpdir = tempfile.mkdtemp()
_log.debug("Fake mode: using %s (instead of %s)" % (self.tmpdir, module_path))
module_path = self.tmpdir
# Real file goes in 'all' category
self.filename = os.path.join(module_path, GENERAL_CLASS, det_full_module_name(self.app.cfg))
# Make symlink in moduleclass category
classPathFile = os.path.join(module_path, self.app.cfg['moduleclass'], det_full_module_name(self.app.cfg))
# Create directories and links
for path in [os.path.dirname(x) for x in [self.filename, classPathFile]]:
mkdir(path, parents=True)
# Make a symlink from classpathFile to self.filename
try:
# remove symlink if its there (even if it's broken)
if os.path.lexists(classPathFile):
os.remove(classPathFile)
# remove module file if it's there (it'll be recreated), see Application.make_module
if os.path.exists(self.filename):
os.remove(self.filename)
os.symlink(self.filename, classPathFile)
except OSError, err:
_log.exception("Failed to create symlink from %s to %s: %s" % (classPathFile, self.filename, err))
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search("^prepend-path\s+CPATH\s+.*$", guess, re.M))
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_skip_extensions_step(self):
"""Test the skip_extensions_step"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'exts_list = ["ext1", "ext2"]',
'exts_filter = ("if [ %(ext_name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")',
'exts_defaultclass = "DummyExtension"',
])
# check if skip skips correct extensions
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.builddir = config.build_path()
eb.installdir = config.install_path()
eb.skip = True
eb.extensions_step(fetch=True)
# 'ext1' should be in eb.exts
self.assertTrue('ext1' in [y for x in eb.exts for y in x.values()])
# 'ext2' should not
self.assertFalse('ext2' in [y for x in eb.exts for y in x.values()])
# cleanup
eb.close_log()
os.remove(eb.logfile)
def load(self, modules, mod_paths=None, purge=False, init_env=None):
"""
Load all requested modules.
@param modules: list of modules to load
@param mod_paths: list of module paths to activate before loading
@param purge: whether or not a 'module purge' should be run before loading
@param init_env: original environment to restore after running 'module purge'
"""
if mod_paths is None:
mod_paths = []
# purge all loaded modules if desired by restoring initial environment
# actually running 'module purge' is futile (and wrong/broken on some systems, e.g. Cray)
if purge:
# restore initial environment if provided
if init_env is None:
raise EasyBuildError("Initial environment required when purging before loading, but not available")
else:
restore_env(init_env)
# make sure $MODULEPATH is set correctly after purging
self.check_module_path()
# extend $MODULEPATH if needed
for mod_path in mod_paths:
full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path)
self.prepend_module_path(full_mod_path)
for mod in modules:
self.run_module('load', mod)
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search("^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search("^prepend-path\s+CPATH\s+.*$", guess, re.M))
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [("GCC", "4.6.4")]
hiddendeps = [("toy", "0.0-deps")]
alldeps = deps + hiddendeps # hidden deps must be included in list of deps
modextravars = {"PI": "3.1415", "FOO": "bar"}
modextrapaths = {"PATH": "pibin", "CPATH": "pi/include"}
self.contents = "\n".join(
[
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = %s" % str(alldeps),
"hiddendependencies = %s" % str(hiddendeps),
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
]
)
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ["MODULEPATH"] = os.path.join(test_dir, "modules")
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
# eb.builddir = self.test_buildpath
eb.installdir = os.path.join(config.install_path(), "pi", "3.14")
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, "r")
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split("\n")[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M))
self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
for (key, val) in modextravars.items():
regex = re.compile('^setenv\s+%s\s+"%s"$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
regex = re.compile("^prepend-path\s+%s\s+\$root/%s$" % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in deps:
regex = re.compile("^\s*module load %s\s*$" % os.path.join(name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in hiddendeps:
regex = re.compile("^\s*module load %s/.%s\s*$" % (name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [('GCC', '4.6.4')]
hiddendeps = [('toy', '0.0-deps')]
alldeps = deps + hiddendeps # hidden deps must be included in list of deps
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = %s" % str(alldeps),
"hiddendependencies = %s" % str(hiddendeps),
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M))
self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
for (key, val) in modextravars.items():
regex = re.compile('^setenv\s+%s\s+"%s"$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
regex = re.compile('^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in deps:
regex = re.compile('^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in hiddendeps:
regex = re.compile('^\s*module load %s/.%s\s*$' % (name, ver), re.M)
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
def get_modules_path(self, fake=False, mod_path_suffix=None):
"""Return path to directory where module files should be generated in."""
mod_path = install_path('mod')
if fake:
self.log.debug("Fake mode: using %s (instead of %s)" % (self.fake_mod_path, mod_path))
mod_path = self.fake_mod_path
if mod_path_suffix is None:
mod_path_suffix = build_option('suffix_modules_path')
return os.path.join(mod_path, mod_path_suffix)
def set_fake(self, fake):
"""Determine whether this ModuleGenerator instance should generate fake modules."""
_log.debug("Updating fake for this ModuleGenerator instance to %s (was %s)" % (fake, self.fake))
self.fake = fake
# fake mode: set installpath to temporary dir
if self.fake:
self.tmpdir = tempfile.mkdtemp()
_log.debug("Fake mode: using %s (instead of %s)" % (self.tmpdir, self.module_path))
self.module_path = self.tmpdir
else:
self.module_path = config.install_path('mod')
def set_fake(self, fake):
"""Determine whether this ModuleGenerator instance should generate fake modules."""
_log.debug("Updating fake for this ModuleGenerator instance to %s (was %s)" % (fake, self.fake))
self.fake = fake
# fake mode: set installpath to temporary dir
if self.fake:
self.tmpdir = tempfile.mkdtemp()
_log.debug("Fake mode: using %s (instead of %s)" % (self.tmpdir, self.module_path))
self.module_path = self.tmpdir
else:
self.module_path = config.install_path('mod')
def get_modules_path(self, fake=False, mod_path_suffix=None):
"""Return path to directory where module files should be generated in."""
mod_path = install_path('mod')
if fake:
self.log.debug("Fake mode: using %s (instead of %s)" % (self.fake_mod_path, mod_path))
mod_path = self.fake_mod_path
if mod_path_suffix is None:
mod_path_suffix = build_option('suffix_modules_path')
return os.path.join(mod_path, mod_path_suffix)
def test_make_module_extra(self):
"""Test for make_module_extra."""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
if get_module_syntax() == 'Tcl':
expected_default = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+\"\$root"',
r'setenv\s+EBVERSIONPI\s+"3.14"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"',
r'setenv\s+EBVERSIONPI\s+"6.28"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
elif get_module_syntax() == 'Lua':
expected_default = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", root\)',
r'setenv\("EBVERSIONPI", "3.14"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)',
r'setenv\("EBVERSIONPI", "6.28"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
defaulttxt = eb.make_module_extra().strip()
self.assertTrue(
expected_default.match(defaulttxt),
"Pattern %s found in %s" % (expected_default.pattern, defaulttxt))
alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28',
altversion='6.28').strip()
self.assertTrue(
expected_alt.match(alttxt),
"Pattern %s found in %s" % (expected_alt.pattern, alttxt))
def install_step(self):
"""
Actual installation
- create silent cfg file
- execute command
"""
if LooseVersion(self.version) >= LooseVersion('8.1'):
silent_cfg_names_map = {}
if LooseVersion(self.version) >= LooseVersion('9.0'):
# To generate the correct itacvars.sh (called by traceanalyzer) we need to change the default install dir
silent_cfg_names_map = {
'install_dir': install_path(),
}
super(EB_itac,
self).install_step(silent_cfg_names_map=silent_cfg_names_map)
# itac v9.0.1 installer create itac/<version> subdir, so stuff needs to be moved afterwards
#if LooseVersion(self.version) >= LooseVersion('9.0'):
# super(EB_itac, self).move_after_install()
else:
silent = \
"""
[itac]
INSTALLDIR=%(ins)s
LICENSEPATH=%(lic)s
INSTALLMODE=NONRPM
INSTALLUSER=NONROOT
INSTALL_ITA=YES
INSTALL_ITC=YES
DEFAULT_MPI=%(mpi)s
EULA=accept
""" % {'lic': self.license_file, 'ins': self.installdir, 'mpi': self.cfg['preferredmpi']}
# already in correct directory
silentcfg = os.path.join(os.getcwd(), "silent.cfg")
f = open(silentcfg, 'w')
f.write(silent)
f.close()
self.log.debug("Contents of %s: %s" % (silentcfg, silent))
tmpdir = os.path.join(os.getcwd(), self.version, 'mytmpdir')
try:
os.makedirs(tmpdir)
except OSError, err:
raise EasyBuildError("Directory %s can't be created: %s",
tmpdir, err)
cmd = "./install.sh --tmp-dir=%s --silent=%s" % (tmpdir, silentcfg)
run_cmd(cmd, log_all=True, simple=True)
def test_make_module_extra(self):
"""Test for make_module_extra."""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
if get_module_syntax() == 'Tcl':
expected_default = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+\"\$root"',
r'setenv\s+EBVERSIONPI\s+"3.14"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"',
r'setenv\s+EBVERSIONPI\s+"6.28"',
r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"',
]))
elif get_module_syntax() == 'Lua':
expected_default = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", root\)',
r'setenv\("EBVERSIONPI", "3.14"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
expected_alt = re.compile(r'\n'.join([
r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)',
r'setenv\("EBVERSIONPI", "6.28"\)',
r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)',
]))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
defaulttxt = eb.make_module_extra().strip()
self.assertTrue(expected_default.match(defaulttxt),
"Pattern %s found in %s" % (expected_default.pattern, defaulttxt))
alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28', altversion='6.28').strip()
self.assertTrue(expected_alt.match(alttxt),
"Pattern %s found in %s" % (expected_alt.pattern, alttxt))
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname,
silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(
tcdep['name'])
self._simulated_load_dependency_module(
tcdep['name'], tcdep['version'],
{'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod,
silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version,
{'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(
os.path.join(install_path('mod'), mod_path_suffix,
modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = [('foo', '1.2.3')]",
"builddependencies = [('bar', '9.8.7')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
# test if module is generated correctly
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(
re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(
re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt,
re.M))
self.assertTrue(
re.search(
'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt,
re.M))
for (key, val) in modextravars.items():
self.assertTrue(
re.search('^setenv\s+%s\s+"%s"$' % (key, val), txt, re.M))
for (key, val) in modextrapaths.items():
self.assertTrue(
re.search('^prepend-path\s+%s\s+\$root/%s$' % (key, val), txt,
re.M))
def skip_available(easyconfigs, testing=False):
"""Skip building easyconfigs for which a module is already available."""
m = modules_tool()
easyconfigs, check_easyconfigs = [], easyconfigs
for ec in check_easyconfigs:
module = ec["module"]
mod = "%s (version %s)" % (module[0], module[1])
modspath = mk_module_path(curr_module_paths() + [os.path.join(config.install_path("mod"), "all")])
if m.exists(module[0], module[1], modspath):
msg = "%s is already installed (module found in %s), skipping " % (mod, modspath)
print_msg(msg, log=_log, silent=testing)
_log.info(msg)
else:
_log.debug("%s is not installed yet, so retaining it" % mod)
easyconfigs.append(ec)
return easyconfigs
def skip_available(easyconfigs, testing=False):
"""Skip building easyconfigs for which a module is already available."""
m = modules_tool()
easyconfigs, check_easyconfigs = [], easyconfigs
for ec in check_easyconfigs:
module = ec['module']
mod = "%s (version %s)" % (module[0], module[1])
modspath = mk_module_path(curr_module_paths() + [os.path.join(config.install_path("mod"), 'all')])
if m.exists(module[0], module[1], modspath):
msg = "%s is already installed (module found in %s), skipping " % (mod, modspath)
print_msg(msg, log=_log, silent=testing)
_log.info(msg)
else:
_log.debug("%s is not installed yet, so retaining it" % mod)
easyconfigs.append(ec)
return easyconfigs
def test_make_module_dep_hmns(self):
"""Test for make_module_dep under HMNS"""
test_ecs_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
'test_ecs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.4.10'}",
'dependencies = [',
" ('FFTW', '3.3.3'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
eb.make_builddir()
eb.prepare_step()
# GCC, OpenMPI and hwloc modules should *not* be included in loads for dependencies
mod_dep_txt = eb.make_module_dep()
for mod in ['GCC/4.7.2', 'OpenMPI/1.6.4']:
regex = re.compile('load.*%s' % mod)
self.assertFalse(
regex.search(mod_dep_txt),
"Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
regex = re.compile('load.*FFTW/3.3.3')
self.assertTrue(
regex.search(mod_dep_txt),
"Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M))
self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M))
self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M))
self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# cleanup
eb.close_log()
os.remove(eb.logfile)
def test_make_module_dep_hmns(self):
"""Test for make_module_dep under HMNS"""
test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
all_stops = [x[0] for x in EasyBlock.get_steps()]
build_options = {
'check_osdeps': False,
'robot_path': [test_ecs_path],
'valid_stops': all_stops,
'validate': False,
}
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
init_config(build_options=build_options)
self.setup_hierarchical_modules()
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.4.10'}",
'dependencies = [',
" ('FFTW', '3.3.3'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
eb.make_builddir()
eb.prepare_step()
# GCC, OpenMPI and hwloc modules should *not* be included in loads for dependencies
mod_dep_txt = eb.make_module_dep()
for mod in ['GCC/4.7.2', 'OpenMPI/1.6.4']:
regex = re.compile('load.*%s' % mod)
self.assertFalse(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
regex = re.compile('load.*FFTW/3.3.3')
self.assertTrue(regex.search(mod_dep_txt), "Pattern '%s' found in: %s" % (regex.pattern, mod_dep_txt))
def check_module_path(self):
"""
Check if MODULEPATH is set and change it if necessary.
"""
# if self.mod_paths is not specified, use $MODULEPATH and make sure the EasyBuild module path is in there (first)
if self.mod_paths is None:
# take module path from environment
self.mod_paths = [x for x in nub(os.environ.get('MODULEPATH', '').split(':')) if len(x) > 0]
self.log.debug("self.mod_paths set based on $MODULEPATH: %s" % self.mod_paths)
# determine module path for EasyBuild install path to be included in $MODULEPATH
eb_modpath = os.path.join(install_path(typ='modules'), GENERAL_CLASS)
# make sure EasyBuild module path is in 1st place
self.mod_paths = [x for x in self.mod_paths if not x == eb_modpath]
self.mod_paths.insert(0, eb_modpath)
self.log.info("Prepended list of module paths with path used by EasyBuild: %s" % eb_modpath)
# set the module path environment accordingly
os.environ['MODULEPATH'] = ':'.join(self.mod_paths)
self.log.info("$MODULEPATH set based on list of module paths: %s" % os.environ['MODULEPATH'])
def check_module_path(self):
"""
Check if MODULEPATH is set and change it if necessary.
"""
# if self.mod_paths is not specified, define it and make sure the EasyBuild module path is in there (first)
if self.mod_paths is None:
# take (unique) module paths from environment
self.set_mod_paths()
self.log.debug("self.mod_paths set based on $MODULEPATH: %s" % self.mod_paths)
# determine module path for EasyBuild install path to be included in $MODULEPATH
eb_modpath = os.path.join(install_path(typ='modules'), build_option('suffix_modules_path'))
# make sure EasyBuild module path is in 1st place
self.prepend_module_path(eb_modpath)
self.log.info("Prepended list of module paths with path used by EasyBuild: %s" % eb_modpath)
# set the module path environment accordingly
for mod_path in self.mod_paths[::-1]:
self.use(mod_path)
self.log.info("$MODULEPATH set based on list of module paths (via 'module use'): %s" % os.environ['MODULEPATH'])
def _load_toolchain_module(self, silent=False):
"""Load toolchain module."""
tc_mod = self.det_short_module_name()
if self.dry_run:
dry_run_msg("Loading toolchain module...\n", silent=silent)
# load toolchain module, or simulate load of toolchain components if it is not available
if self.modules_tool.exist([tc_mod], skip_avail=True)[0]:
self.modules_tool.load([tc_mod])
dry_run_msg("module load %s" % tc_mod, silent=silent)
else:
# first simulate loads for toolchain dependencies, if required information is available
if self.tcdeps is not None:
for tcdep in self.tcdeps:
modname = tcdep['short_mod_name']
dry_run_msg("module load %s [SIMULATED]" % modname, silent=silent)
# 'use '$EBROOTNAME' as value for dep install prefix (looks nice in dry run output)
deproot = '$%s' % get_software_root_env_var_name(tcdep['name'])
self._simulated_load_dependency_module(tcdep['name'], tcdep['version'], {'prefix': deproot})
dry_run_msg("module load %s [SIMULATED]" % tc_mod, silent=silent)
# use name of $EBROOT* env var as value for $EBROOT* env var (results in sensible dry run output)
tcroot = '$%s' % get_software_root_env_var_name(self.name)
self._simulated_load_dependency_module(self.name, self.version, {'prefix': tcroot})
else:
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath))
# load modules for all dependencies
self.log.debug("Loading module for toolchain: %s", tc_mod)
trace_msg("loading toolchain module: " + tc_mod)
self.modules_tool.load([tc_mod])
# append toolchain module to list of modules
self.modules.append(tc_mod)
def create_files(self):
"""
Creates the absolute filename for the module.
"""
module_path = config.install_path('mod')
# Fake mode: set installpath to temporary dir
if self.fake:
self.tmpdir = tempfile.mkdtemp()
_log.debug("Fake mode: using %s (instead of %s)" %
(self.tmpdir, module_path))
module_path = self.tmpdir
# Real file goes in 'all' category
self.filename = os.path.join(module_path, GENERAL_CLASS,
det_full_module_name(self.app.cfg))
# Make symlink in moduleclass category
classPathFile = os.path.join(module_path, self.app.cfg['moduleclass'],
det_full_module_name(self.app.cfg))
# Create directories and links
for path in [
os.path.dirname(x) for x in [self.filename, classPathFile]
]:
mkdir(path, parents=True)
# Make a symlink from classpathFile to self.filename
try:
# remove symlink if its there (even if it's broken)
if os.path.lexists(classPathFile):
os.remove(classPathFile)
# remove module file if it's there (it'll be recreated), see Application.make_module
if os.path.exists(self.filename):
os.remove(self.filename)
os.symlink(self.filename, classPathFile)
except OSError, err:
_log.exception("Failed to create symlink from %s to %s: %s" %
(classPathFile, self.filename, err))
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = [('foo', '1.2.3')]",
"builddependencies = [('bar', '9.8.7')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
# test if module is generated correctly
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
modpath = os.path.join(eb.make_module_step(), name, version)
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
f = open(modpath, 'r')
txt = f.read()
f.close()
self.assertTrue(re.search("^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M))
self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M))
self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
for (key, val) in modextravars.items():
self.assertTrue(re.search('^setenv\s+%s\s+"%s"$' % (key, val), txt, re.M))
for (key, val) in modextrapaths.items():
self.assertTrue(re.search('^prepend-path\s+%s\s+\$root/%s$' % (key, val), txt, re.M))
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
self.purge_environment()
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build', 'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = init_config(args=[])
self.assertEqual(build_path(), buildpath_env_var)
self.assertEqual(install_path(), os.path.join(prefix, 'software'))
self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')])
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--configfiles', config_file, # force empty config file
'--prefix', prefix,
'--installpath', install,
'--repositorypath', repopath,
'--subdir-software', 'APPS',
]
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'APPS'))
self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertTrue(config_file in options.configfiles)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--configfiles', config_file, # force empty config file
'--installpath', install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, subdir_software))
self.assertEqual(install_path('mod'), installpath_modules)
# subdir options *must* be relative (to --installpath)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software
error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path"
self.assertErrorRegex(EasyBuildError, error_regex, init_config)
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
args = [
'--configfiles',
config_file,
'--debug',
'--buildpath',
testpath1,
'--installpath-software',
installpath_software,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [
os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')
]) # default
self.assertEqual(install_path(),
installpath_software) # via cmdline arg
self.assertEqual(install_path('mod'),
os.path.join(testpath2, 'modules')) # via config file
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(
prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs')
shutil.copytree(test_ecs_dir,
os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
orig_sys_path = sys.path[:]
sys.path.insert(
0, tmpdir
) # prepend to give it preference over possible other installed easyconfigs pkgs
# test with config file passed via environment variable
# also test for existence of HOME and USER by adding paths to robot-paths
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
'sourcepath = %(DEFAULT_REPOSITORYPATH)s',
'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir',
'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s',
'installpath-modules=%s' % installpath_modules,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath',
testpath2,
]
options = init_config(args=args)
topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild')
self.assertEqual(install_path(), os.path.join(topdir,
'software')) # default
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
self.assertEqual(get_repositorypath(),
[os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'
]) # via config file
# hardcoded first entry
self.assertEqual(options.robot_paths[0], '/tmp/foo')
# resolved value for %(sourcepath)s template
self.assertEqual(
options.robot_paths[1],
os.path.join(os.getenv('HOME'), '.local', 'easybuild',
'ebfiles_repo'))
# resolved value for HOME constant
self.assertEqual(options.robot_paths[2], os.getenv('HOME'))
# resolved value that uses USER constant
self.assertEqual(options.robot_paths[3],
os.path.join('/tmp', os.getenv('USER')))
# first path in DEFAULT_ROBOT_PATHS
self.assertEqual(options.robot_paths[4],
os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath',
testpath3,
]
options = init_config(args=args)
self.assertEqual(
source_paths(),
[testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(),
os.path.join(testpath3,
'software')) # via command line
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
sys.path[:] = orig_sys_path
def test_generaloption_config(self):
"""Test new-style configuration (based on generaloption)."""
self.purge_environment()
# check whether configuration via environment variables works as expected
prefix = os.path.join(self.tmpdir, 'testprefix')
buildpath_env_var = os.path.join(self.tmpdir, 'envvar', 'build',
'path')
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_BUILDPATH'] = buildpath_env_var
options = init_config(args=[])
self.assertEqual(build_path(), buildpath_env_var)
self.assertEqual(install_path(), os.path.join(prefix, 'software'))
self.assertEqual(get_repositorypath(),
[os.path.join(prefix, 'ebfiles_repo')])
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_BUILDPATH']
# check whether configuration via command line arguments works
prefix = os.path.join(self.tmpdir, 'test1')
install = os.path.join(self.tmpdir, 'test2', 'install')
repopath = os.path.join(self.tmpdir, 'test2', 'repo')
config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
write_file(config_file, '')
args = [
'--configfiles',
config_file, # force empty config file
'--prefix',
prefix,
'--installpath',
install,
'--repositorypath',
repopath,
'--subdir-software',
'APPS',
]
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install, 'APPS'))
self.assertEqual(install_path(typ='mod'),
os.path.join(install, 'modules'))
self.assertEqual(options.installpath, install)
self.assertTrue(config_file in options.configfiles)
# check mixed command line/env var configuration
prefix = os.path.join(self.tmpdir, 'test3')
install = os.path.join(self.tmpdir, 'test4', 'install')
subdir_software = 'eb-soft'
args = [
'--configfiles',
config_file, # force empty config file
'--installpath',
install,
]
os.environ['EASYBUILD_PREFIX'] = prefix
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules
options = init_config(args=args)
self.assertEqual(build_path(), os.path.join(prefix, 'build'))
self.assertEqual(install_path(), os.path.join(install,
subdir_software))
self.assertEqual(install_path('mod'), installpath_modules)
# subdir options *must* be relative (to --installpath)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software
error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path"
self.assertErrorRegex(EasyBuildError, error_regex, init_config)
del os.environ['EASYBUILD_PREFIX']
del os.environ['EASYBUILD_SUBDIR_SOFTWARE']
def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()
# create fake directories and files that should be guessed
os.makedirs(eb.installdir)
open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar')
open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar')
os.mkdir(os.path.join(eb.installdir, 'bin'))
os.mkdir(os.path.join(eb.installdir, 'share'))
os.mkdir(os.path.join(eb.installdir, 'share', 'man'))
# this is not a path that should be picked up
os.mkdir(os.path.join(eb.installdir, 'CPATH'))
guess = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(
re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$",
guess, re.M))
self.assertTrue(
re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$",
guess, re.M))
self.assertTrue(
re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$",
guess, re.M))
self.assertTrue(
re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M))
self.assertFalse(
re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(
re.search(
r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$',
guess, re.M))
self.assertTrue(
re.search(
r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$',
guess, re.M))
self.assertTrue(
re.search(
r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$',
guess, re.M))
self.assertTrue(
re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$',
guess, re.M))
self.assertFalse(
re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
# check for behavior when a string value is used as dict value by make_module_req_guesses
eb.make_module_req_guess = lambda: {'PATH': 'bin'}
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(
re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(
re.match(
r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$',
txt, re.M))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
# check for correct behaviour if empty string is specified as one of the values
# prepend-path statements should be included for both the 'bin' subdir and the install root
eb.make_module_req_guess = lambda: {'PATH': ['bin', '']}
txt = eb.make_module_req()
if get_module_syntax() == 'Tcl':
self.assertTrue(
re.search(r"\nprepend-path\s+PATH\s+\$root/bin\n", txt, re.M))
self.assertTrue(
re.search(r"\nprepend-path\s+PATH\s+\$root\n", txt, re.M))
elif get_module_syntax() == 'Lua':
self.assertTrue(
re.search(
r'\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n',
txt, re.M))
self.assertTrue(
re.search(r'\nprepend_path\("PATH", root\)\n', txt, re.M))
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
# cleanup
eb.close_log()
os.remove(eb.logfile)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 0 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = "'%s<0.11.4'" % VSC_INSTALL
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# determine PyPI URLs for individual packages
pkg_urls = []
for pkg in EASYBUILD_PACKAGES:
# format of pkg entries in templates: "'<pkg_filename>',"
pkg_filename = templates[pkg][1:-2]
# the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
# which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)
# determine download URL via PyPI's 'simple' API
pkg_simple = None
try:
pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
except (urllib2.URLError, urllib2.HTTPError) as err:
# failing to figure out the package download URl may be OK when source tarballs are provided
if sourcepath:
info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
else:
raise err
if pkg_simple:
pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
res = pkg_url_part_regex.search(pkg_simple)
if res:
pkg_url_part = res.group(1)
else:
error_msg = "Failed to determine PyPI package URL for %s using pattern '%s': %s\n"
error(error_msg % (pkg, pkg_url_part_regex.pattern, pkg_simple))
pkg_url = 'https://pypi.python.org/' + pkg_url_part
pkg_urls.append(pkg_url)
templates.update({
'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
handle.write(ebfile_txt)
handle.close()
debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure EasyBuild can find EasyBuild-*.eb easyconfig file when it needs to;
# (for example when HierarchicalMNS is used as module naming scheme,
# see https://github.com/easybuilders/easybuild-framework/issues/2393)
eb_args.append('--robot-paths=%s:' % tmpdir)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
if print_debug:
os.environ['EASYBUILD_DEBUG'] = '1'
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def test_make_module_dep(self):
"""Test for make_module_dep"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
if get_module_syntax() == 'Tcl':
tc_load = '\n'.join([
"if { ![ is-loaded gompi/1.1.0-no-OFED ] } {",
" module load gompi/1.1.0-no-OFED",
"}",
])
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
lapack_load = '\n'.join([
"if { ![ is-loaded LAPACK/3.4.0-gompi-1.1.0-no-OFED ] } {",
" module load LAPACK/3.4.0-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
tc_load = '\n'.join([
'if not isloaded("gompi/1.1.0-no-OFED") then',
' load("gompi/1.1.0-no-OFED")',
'end',
])
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
lapack_load = '\n'.join([
'if not isloaded("LAPACK/3.4.0-gompi-1.1.0-no-OFED") then',
' load("LAPACK/3.4.0-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(eb.make_module_dep().strip(), expected)
# provide swap info for FFTW to trigger an extra 'unload FFTW'
unload_info = {
'FFTW/3.3.1-gompi-1.1.0-no-OFED': 'FFTW',
}
if get_module_syntax() == 'Tcl':
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module unload FFTW",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' unload("FFTW")',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False,
"Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(
eb.make_module_dep(unload_info=unload_info).strip(), expected)
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
write_file(oldstyle_config_file, '')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
args = [
'--configfiles', config_file,
'--debug',
'--buildpath', testpath1,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default
self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file
# test with config file passed via environment variable
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath', testpath2,
]
options = init_config(args=args)
self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath', testpath3,
]
options = init_config(args=args)
self.assertEqual(source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
def test_legacy_config_file(self):
"""Test finding/using legacy configuration files."""
self.purge_environment()
cfg_fn = self.configure(args=[])
self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))
configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""
buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'test', 'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
logdir = 'somedir'
logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware'
modsuffix = 'modulesgohere'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create user config file on default location
myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
if not os.path.exists(os.path.dirname(myconfigfile)):
os.makedirs(os.path.dirname(myconfigfile))
write_file(myconfigfile, configtxt % configdict)
# redefine home so we can test user config file on default location
home = os.environ.get('HOME', None)
os.environ['HOME'] = self.tmpdir
init_config()
cfg_fn = self.configure(args=[])
if home is not None:
os.environ['HOME'] = home
# check finding and use of config file
self.assertEqual(cfg_fn, myconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
# redefine config file entries for proper testing below
buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo', 'path')
logdir = 'somedir_custom'
logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware_custom'
modsuffix = 'modulesgohere_custom'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix }
# create custom config file, and point to it
mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
if not os.path.exists(os.path.dirname(mycustomconfigfile)):
os.makedirs(os.path.dirname(mycustomconfigfile))
write_file(mycustomconfigfile, configtxt % configdict)
os.environ['EASYBUILDCONFIG'] = mycustomconfigfile
# reconfigure
init_config()
cfg_fn = self.configure(args=[])
# verify configuration
self.assertEqual(cfg_fn, mycustomconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
def test_legacy_env_vars(self):
"""Test legacy environment variables."""
self.purge_environment()
# build path
test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
del os.environ['EASYBUILDBUILDPATH']
# source path(s)
test_sourcepaths = [
os.path.join(self.tmpdir, 'source', 'path'),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
]),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
os.path.join(self.tmpdir, 'source', 'path3'),
]),
]
for test_sourcepath in test_sourcepaths:
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths(), test_sourcepath.split(':'))
del os.environ['EASYBUILDSOURCEPATH']
test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
# install path
init_config()
test_installpath = os.path.join(self.tmpdir, 'install', 'path')
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(source_paths()[0], os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
del os.environ['EASYBUILDINSTALLPATH']
# prefix: should change build/install/source/repo paths
init_config()
test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
os.environ['EASYBUILDPREFIX'] = test_prefixpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
# build/source/install path overrides prefix
init_config()
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDBUILDPATH']
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], test_sourcepath)
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDSOURCEPATH']
init_config()
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
def test_make_module_dep(self):
"""Test for make_module_dep"""
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}",
'dependencies = [',
" ('FFTW', '3.3.1'),",
" ('LAPACK', '3.4.0'),",
']',
])
self.writeEC()
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
if get_module_syntax() == 'Tcl':
tc_load = '\n'.join([
"if { ![ is-loaded gompi/1.1.0-no-OFED ] } {",
" module load gompi/1.1.0-no-OFED",
"}",
])
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
lapack_load = '\n'.join([
"if { ![ is-loaded LAPACK/3.4.0-gompi-1.1.0-no-OFED ] } {",
" module load LAPACK/3.4.0-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
tc_load = '\n'.join([
'if not isloaded("gompi/1.1.0-no-OFED") then',
' load("gompi/1.1.0-no-OFED")',
'end',
])
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
lapack_load = '\n'.join([
'if not isloaded("LAPACK/3.4.0-gompi-1.1.0-no-OFED") then',
' load("LAPACK/3.4.0-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(eb.make_module_dep().strip(), expected)
# provide swap info for FFTW to trigger an extra 'unload FFTW'
unload_info = {
'FFTW/3.3.1-gompi-1.1.0-no-OFED': 'FFTW',
}
if get_module_syntax() == 'Tcl':
fftw_load = '\n'.join([
"if { ![ is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED ] } {",
" module unload FFTW",
" module load FFTW/3.3.1-gompi-1.1.0-no-OFED",
"}",
])
elif get_module_syntax() == 'Lua':
fftw_load = '\n'.join([
'if not isloaded("FFTW/3.3.1-gompi-1.1.0-no-OFED") then',
' unload("FFTW")',
' load("FFTW/3.3.1-gompi-1.1.0-no-OFED")',
'end',
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
expected = tc_load + '\n\n' + fftw_load + '\n\n' + lapack_load
self.assertEqual(eb.make_module_dep(unload_info=unload_info).strip(), expected)
def test_make_module_step(self):
"""Test the make_module_step"""
name = "pi"
version = "3.14"
deps = [('GCC', '4.6.4')]
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
'version = "%s"' % version,
'homepage = "http://example.com"',
'description = "test easyconfig"',
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
"dependencies = [('GCC', '4.6.4'), ('toy', '0.0-deps')]",
"builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]",
# hidden deps must be included in list of (build)deps
"hiddendependencies = [('toy', '0.0-deps'), ('OpenMPI', '1.6.4-GCC-4.6.4')]",
"modextravars = %s" % str(modextravars),
"modextrapaths = %s" % str(modextrapaths),
])
test_dir = os.path.dirname(os.path.abspath(__file__))
os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')
# test if module is generated correctly
self.writeEC()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
eb.check_readiness_step()
modpath = os.path.join(eb.make_module_step(), name, version)
if get_module_syntax() == 'Lua':
modpath += '.lua'
self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)
# verify contents of module
txt = read_file(modpath)
if get_module_syntax() == 'Tcl':
self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0]))
self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M))
self.assertTrue(re.search(r"^set\s+root\s+%s$" % eb.installdir, txt, re.M))
ebroot_regex = re.compile(r'^setenv\s+EBROOT%s\s+"\$root"\s*$' % name.upper(), re.M)
self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt))
self.assertTrue(re.search(r'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M))
elif get_module_syntax() == 'Lua':
ebroot_regex = re.compile(r'^setenv\("EBROOT%s", root\)$' % name.upper(), re.M)
self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt))
self.assertTrue(re.search(r'^setenv\("EBVERSION%s", "%s"\)$' % (name.upper(), version), txt, re.M))
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for (key, val) in modextravars.items():
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (key, val) in modextrapaths.items():
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('GCC', '4.6.4')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('toy', '0.0-deps')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
for (name, ver) in [('OpenMPI', '1.6.4-GCC-4.6.4')]:
if get_module_syntax() == 'Tcl':
regex = re.compile(r'^\s*module load %s/.?%s\s*$' % (name, ver), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^\s*load\("%s/.?%s"\)$' % (name, ver), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertFalse(regex.search(txt), "Pattern '%s' *not* found in %s" % (regex.pattern, txt))
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
args = [
'--configfiles', config_file,
'--debug',
'--buildpath', testpath1,
'--installpath-software', installpath_software,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default
self.assertEqual(install_path(), installpath_software) # via cmdline arg
self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
orig_sys_path = sys.path[:]
sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs
# test with config file passed via environment variable
# also test for existence of HOME and USER by adding paths to robot-paths
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
'sourcepath = %(DEFAULT_REPOSITORYPATH)s',
'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir',
'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s',
'installpath-modules=%s' % installpath_modules,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath', testpath2,
]
options = init_config(args=args)
topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild')
self.assertEqual(install_path(), os.path.join(topdir, 'software')) # default
self.assertEqual(install_path('mod'), installpath_modules), # via config file
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file
# hardcoded first entry
self.assertEqual(options.robot_paths[0], '/tmp/foo')
# resolved value for %(sourcepath)s template
self.assertEqual(options.robot_paths[1], os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'ebfiles_repo'))
# resolved value for HOME constant
self.assertEqual(options.robot_paths[2], os.getenv('HOME'))
# resolved value that uses USER constant
self.assertEqual(options.robot_paths[3], os.path.join('/tmp', os.getenv('USER')))
# first path in DEFAULT_ROBOT_PATHS
self.assertEqual(options.robot_paths[4], os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath', testpath3,
]
options = init_config(args=args)
self.assertEqual(source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line
self.assertEqual(install_path('mod'), installpath_modules), # via config file
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
sys.path[:] = orig_sys_path
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
write_file(oldstyle_config_file, '')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
args = [
'--configfiles',
config_file,
'--debug',
'--buildpath',
testpath1,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [
os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')
]) # default
self.assertEqual(install_path(),
os.path.join(testpath2,
'software')) # via config file
# test with config file passed via environment variable
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath',
testpath2,
]
options = init_config(args=args)
self.assertEqual(install_path(),
os.path.join(os.getenv('HOME'), '.local', 'easybuild',
'software')) # default
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath',
testpath3,
]
options = init_config(args=args)
self.assertEqual(
source_paths(),
[testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(),
os.path.join(testpath3,
'software')) # via command line
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
