def avail_job_backends(check_usable=True):
"""
Return all known job execution backends.
"""
import_available_modules('easybuild.tools.job')
class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)])
return class_dict
def avail_job_backends(check_usable=True):
"""
Return all known job execution backends.
"""
import_available_modules('easybuild.tools.job')
class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)])
return class_dict
def get_convert_class(class_name):
"""Return the Convert class with specified class name class_name"""
res = [x for x in nub(get_subclasses(Convert)) if x.__name__ == class_name]
if len(res) == 1:
return res[0]
else:
_log.error('More then one Convert subclass found for name %s: %s' % (class_name, res))
def what_sched(requested):
"""Return the scheduler class """
def sched_to_key(klass):
"""Return key for specified scheduler class which can be used for sorting."""
# use lowercase class name for sorting
key = klass.__name__.lower()
# prefix key for SLURM scheduler class with '_'
# this is done to consider SLURM before PBS, since $PBS* environment variables may be defined in SLURM job env
if key == 'slurm':
key = '_' + key
return key
# exclude Coupler class which also is a subclass of Sched, since it's not an actual scheduler
found_sched = sorted([c for c in get_subclasses(Sched) if c.__name__ != 'Coupler'], key=sched_to_key)
# Get local scheduler
local_sched = get_local_sched(found_sched)
# first, try to use the scheduler that was requested
if requested:
for sched in found_sched:
if sched._is_sched_for(requested):
return sched, found_sched
LOGGER.warn("%s scheduler was requested, but mympirun failed to find an implementation", requested)
# next, try to use the scheduler defined by environment variables
for sched in found_sched:
if sched.SCHED_ENVIRON_NODE_INFO in os.environ and sched.SCHED_ENVIRON_ID in os.environ:
return sched, found_sched
# If that fails, try to force the local scheduler
LOGGER.debug("No scheduler found in environment, trying local")
return local_sched, found_sched
def get_convert_class(class_name):
"""Return the Convert class with specified class name class_name"""
res = [x for x in nub(get_subclasses(Convert)) if x.__name__ == class_name]
if len(res) == 1:
return res[0]
else:
_log.error('More then one Convert subclass found for name %s: %s' % (class_name, res))
def get_convert_class(class_name):
"""Return the Convert class with specified class name class_name"""
res = [x for x in nub(get_subclasses(Convert)) if x.__name__ == class_name]
if len(res) == 1:
return res[0]
else:
raise EasyBuildError("More than one Convert subclass found for name %s: %s", class_name, res)
def whatMPI(name):
"""
Return the scriptname and the MPI class
"""
fullscriptname = os.path.abspath(name)
scriptname = os.path.basename(fullscriptname)
found_mpi = get_subclasses(MPI)
# check on scriptname
for mpi in found_mpi:
if mpi._is_mpiscriptname_for(scriptname):
stripfake() # mandatory before return at this point
return scriptname, mpi, found_mpi
# not called through alias
# stripfake is in which
mpirunname = which(['mpirun'])
if mpirunname is None:
return None, None, found_mpi
for mpi in found_mpi:
if mpi._is_mpirun_for(mpirunname):
return scriptname, mpi, found_mpi
# return found mpirunname
return mpirunname, None, found_mpi
def pingpongfactory(pptype, comm, p, log):
"""a factory for creating PingPong objects"""
for cls in get_subclasses(PingPongSR, include_base_class=True):
if "PingPong%s" % pptype == cls.__name__:
return cls(comm, p, log)
raise KeyError
def what_sched(requested):
"""Return the scheduler class """
# import all modules in this dir: http://stackoverflow.com/a/16853487
for loader, modulename, _ in pkgutil.walk_packages(
[os.path.dirname(__file__)]):
loader.find_module(modulename).load_module(modulename)
found_sched = get_subclasses(Sched)
# first, try to use the scheduler that was requested
if requested:
for sched in found_sched:
if sched._is_sched_for(requested):
return sched, found_sched
LOGGER.warn(
"%s scheduler was requested, but mympirun failed to find an implementation",
requested)
# next, try to use the scheduler defined by environment variables
for sched in found_sched:
if sched.SCHED_ENVIRON_ID in os.environ:
return sched, found_sched
# If that fails, try to force the local scheduler
for sched in found_sched:
LOGGER.debug("No scheduler found in environment, trying local")
if sched._is_sched_for("local"):
return sched, found_sched
# if there is no local scheduler, return None
return None, found_sched
def what_sched(requested):
"""Return the scheduler class """
# import all modules in this dir: http://stackoverflow.com/a/16853487
for loader, modulename, _ in pkgutil.walk_packages([os.path.dirname(__file__)]):
loader.find_module(modulename).load_module(modulename)
found_sched = get_subclasses(Sched)
# first, try to use the scheduler that was requested
if requested:
for sched in found_sched:
if sched._is_sched_for(requested):
return sched, found_sched
LOGGER.warn("%s scheduler was requested, but mympirun failed to find an implementation", requested)
# next, try to use the scheduler defined by environment variables
for sched in found_sched:
if sched.SCHED_ENVIRON_ID in os.environ:
return sched, found_sched
# If that fails, try to force the local scheduler
for sched in found_sched:
LOGGER.debug("No scheduler found in environment, trying local")
if sched._is_sched_for("local"):
return sched, found_sched
# if there is no local scheduler, return None
return None, found_sched
def whatMPI(name):
"""
Return the scriptname and the MPI class
"""
fullscriptname = os.path.abspath(name)
scriptname = os.path.basename(fullscriptname)
found_mpi = get_subclasses(MPI)
# check on scriptname
for mpi in found_mpi:
if mpi._is_mpiscriptname_for(scriptname):
stripfake() # mandatory before return at this point
return scriptname, mpi, found_mpi
# not called through alias
# stripfake is in which
mpirunname = which(['mpirun'])
if mpirunname is None:
return None, None, found_mpi
for mpi in found_mpi:
if mpi._is_mpirun_for(mpirunname):
return scriptname, mpi, found_mpi
# return found mpirunname
return mpirunname, None, found_mpi
def get_format_version_classes(version=None):
"""Return the (usable) subclasses from EasyConfigFormat that have a matching version."""
all_classes = get_subclasses(EasyConfigFormat)
if version is None:
return all_classes
else:
return [x for x in all_classes if x.VERSION == version and x.USABLE]
def avail_package_naming_schemes():
"""
Returns the list of valed naming schemes that are in the easybuild.package.package_naming_scheme namespace
"""
import_available_modules('easybuild.tools.package.package_naming_scheme')
class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)])
return class_dict
def search_toolchain(name):
"""
Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains.
:param name: toolchain name
:return: Toolchain instance (or None), found_toolchains
"""
package = easybuild.tools.toolchain
check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX
if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name):
# import all available toolchains, so we know about them
tc_modules = import_available_modules('easybuild.toolchains')
# make sure all defined toolchain constants are available in toolchain module
tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX)
for tc_mod in tc_modules:
# determine classes imported in this module
mod_classes = []
for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
if hasattr(elem, '__module__'):
# exclude the toolchain class defined in that module
if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
_log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__)
mod_classes.append(elem)
# look for constants in modules of imported classes, and make them available
for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
for elem in dir(mod_class_mod):
res = tc_const_re.match(elem)
if res:
tc_const_name = res.group(1)
tc_const_value = getattr(mod_class_mod, elem)
_log.debug("Found constant %s ('%s') in module %s, adding it to %s",
tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__)
if hasattr(package, tc_const_name):
cur_value = getattr(package, tc_const_name)
if not tc_const_value == cur_value:
raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.",
package.__name__, tc_const_name, cur_value, tc_const_value)
else:
setattr(package, tc_const_name, tc_const_value)
# indicate that processing of toolchain constants is done, so it's not done again
setattr(package, check_attr_name, True)
else:
_log.debug("Skipping importing of toolchain modules, processing of toolchain constants is already done.")
# obtain all subclasses of toolchain
found_tcs = nub(get_subclasses(Toolchain))
# filter found toolchain subclasses based on whether they can be used a toolchains
found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]
for tc in found_tcs:
if tc._is_toolchain_for(name):
return tc, found_tcs
return None, found_tcs
def pairfactory(pairmode, seed=None, rng=None, pairid=None, logger=None):
"""A factory for creating Pair objects"""
logger.debug("in pairfactory with pairmode %s", pairmode)
for cls in get_subclasses(Pair, include_base_class=True):
if pairmode == cls.__name__.lower():
return cls(seed, rng, pairid, logger)
raise KeyError
def get_convert_class(class_name):
"""Return the Convert class with specified class name class_name"""
res = [x for x in nub(get_subclasses(Convert)) if x.__name__ == class_name]
if len(res) == 1:
return res[0]
else:
raise EasyBuildError(
"More than one Convert subclass found for name %s: %s", class_name,
res)
def avail_modules_tools():
"""
Return all known modules tools.
"""
class_dict = dict([(x.__name__, x) for x in get_subclasses(ModulesTool)])
# filter out legacy Modules class
if 'Modules' in class_dict:
del class_dict['Modules']
return class_dict
def avail_modules_tools():
"""
Return all known modules tools.
"""
class_dict = dict([(x.__name__, x) for x in get_subclasses(ModulesTool)])
# filter out legacy Modules class
if 'Modules' in class_dict:
del class_dict['Modules']
return class_dict
def what_licenses():
"""Return a dict of License subclasses names and license instances"""
res = {}
for lic in get_subclasses(License):
if lic.HIDDEN:
continue
res[lic.__name__] = lic
return res
def get_job(classname, options):
"""
This is a job factory.
Returns an instance of classname initialized with options
"""
for cls in get_subclasses(Job):
if cls._is_job_for(classname):
return cls(options)
getLogger().error("No job class found for %s", classname)
def get_job(classname, options):
"""
This is a job factory.
Returns an instance of classname initialized with options
"""
for cls in get_subclasses(Job):
if cls._is_job_for(classname):
return cls(options)
getLogger().error("No job class found for %s", classname)
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
# all ModuleNamingScheme subclasses available in easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)])
return avail_mnss
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
# all ModuleNamingScheme subclasses available in easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)])
return avail_mnss
def avail_repositories(check_useable=True):
"""
Return all available repositories.
check_useable: boolean, if True, only return usable repositories
"""
class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable])
if not 'FileRepository' in class_dict:
_log.error('avail_repositories: FileRepository missing from list of repositories')
return class_dict
def avail_repositories(check_useable=True):
"""
Return all available repositories.
check_useable: boolean, if True, only return usable repositories
"""
import_available_modules('easybuild.tools.repository')
class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable])
if not 'FileRepository' in class_dict:
raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories")
return class_dict
def test_mympirun_aliases_setup(self):
"""Make sure that list of mympirun aliases included in setup.py is synced"""
from setup import MYMPIRUN_ALIASES
# make sure all modules in vsc.mympirun.mpi are imported
for loader, modname, _ in pkgutil.walk_packages([os.path.dirname(mpim.__file__)]):
loader.find_module(modname).load_module(modname)
# determine actual list of mympirun aliases
mympirun_aliases = ['myscoop']
for mpiclass in get_subclasses(mpim.MPI):
mympirun_aliases.extend(mpiclass._mpiscriptname_for)
self.assertEqual(MYMPIRUN_ALIASES, nub(sorted(mympirun_aliases)))
def what_mpi(name):
"""
Return the path of the selected mpirun and its class.
@param name: The name of the executable used to run mympirun
@return: A triplet containing the following variables:
- The path to the executable used to run mympirun (should be the path to an mpirun implementation)
- The corresponding python class of the MPI variant
- The python classes of the supported MPI flavors (from the various .py files in mympirun/mpi)
"""
# import all modules in this dir: http://stackoverflow.com/a/16853487
for loader, modulename, _ in pkgutil.walk_packages([os.path.dirname(__file__)]):
loader.find_module(modulename).load_module(modulename)
supp_mpi_impl = get_subclasses(MPI) # supported MPI implementations
# remove fake mpirun from $PATH
stripfake()
# get the path of the mpirun executable
mpirun_path = which('mpirun')
if mpirun_path is None:
# no MPI implementation installed
LOGGER.warn("no mpirun command found")
return None, None, supp_mpi_impl
scriptname = os.path.basename(os.path.abspath(name))
# check if mympirun was called by a known mpirun alias (like
# ompirun for OpenMPI or mhmpirun for mpich)
for mpi in supp_mpi_impl:
if mpi._is_mpiscriptname_for(scriptname):
LOGGER.debug("%s was used to call mympirun", scriptname)
return scriptname, mpi, supp_mpi_impl
# mympirun was not called through a known alias, so find out which MPI
# implementation the user has installed
for mpi in supp_mpi_impl:
if mpi._is_mpirun_for(mpirun_path):
return scriptname, mpi, supp_mpi_impl
# no specific flavor found, default to mpirun_path
LOGGER.warn("The executable that called mympirun (%s) isn't supported"
", defaulting to %s", name, mpirun_path)
return mpirun_path, None, supp_mpi_impl
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
mns_attr = 'AVAIL_MODULE_NAMING_SCHEMES'
if not hasattr(module_naming_scheme, mns_attr):
# all subclasses of ModuleNamingScheme available in the easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)])
# cache dict of available module naming scheme in module constant
setattr(module_naming_scheme, mns_attr, avail_mnss)
return avail_mnss
else:
return getattr(module_naming_scheme, mns_attr)
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
mns_attr = 'AVAIL_MODULE_NAMING_SCHEMES'
if not hasattr(module_naming_scheme, mns_attr):
# all subclasses of ModuleNamingScheme available in the easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x)
for x in get_subclasses(ModuleNamingScheme)])
# cache dict of available module naming scheme in module constant
setattr(module_naming_scheme, mns_attr, avail_mnss)
return avail_mnss
else:
return getattr(module_naming_scheme, mns_attr)
def what_mpi(name):
"""
Return the path of the selected mpirun and its class.
@param name: The name of the executable used to run mympirun
@return: A triplet containing the following variables:
- The path to the executable used to run mympirun (should be the path to an mpirun implementation)
- The corresponding python class of the MPI variant
- The python classes of the supported MPI flavors (from the various .py files in mympirun/mpi)
"""
# The coupler is also a subclass of MPI, but it isn't and MPI implementation
supp_mpi_impl = [x for x in get_subclasses(MPI) if x.__name__ != 'Coupler'] # supported MPI implementations
# remove fake mpirun from $PATH
stripfake()
# get the path of the mpirun executable
mpirun_path = which('mpirun')
if mpirun_path is None:
# no MPI implementation installed
LOGGER.warn("no mpirun command found")
return None, None, supp_mpi_impl
scriptname = os.path.basename(os.path.abspath(name))
# check if mympirun was called by a known mpirun alias (like
# ompirun for OpenMPI or mhmpirun for mpich)
for mpi in supp_mpi_impl:
if mpi._is_mpiscriptname_for(scriptname) and mpi._is_mpirun_for(mpirun_path):
LOGGER.debug("%s was used to call mympirun", scriptname)
return scriptname, mpi, supp_mpi_impl
# mympirun was not called through a known alias, so find out which MPI
# implementation the user has installed
for mpi in supp_mpi_impl:
if mpi._is_mpirun_for(mpirun_path):
return scriptname, mpi, supp_mpi_impl
# no specific flavor found, default to mpirun_path
LOGGER.warn("The executable that called mympirun (%s) isn't supported, defaulting to %s", name, mpirun_path)
return mpirun_path, None, supp_mpi_impl
def avail_module_generators():
"""
Return all known module syntaxes.
"""
return dict([(k.SYNTAX, k) for k in get_subclasses(ModuleGenerator)])
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files]
ec_files = [fil for fil in ec_files if not "v2.0" in fil] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn], det_full_module_name_mg(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar', det_full_module_name_ec(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
for test_mns_mod in ['test_module_naming_scheme', 'test_module_naming_scheme_all']:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
mns_mod = __import__(mns_path, globals(), locals(), [''])
test_mnss = dict([(x.__name__, x) for x in get_subclasses(mns_mod.ModuleNamingScheme)])
easybuild.tools.module_naming_scheme.AVAIL_MODULE_NAMING_SCHEMES.update(test_mnss)
init_config(build_options=build_options)
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
# test module naming scheme using all available easyconfig parameters
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeAll'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/698cacc77167c6824f597f0b6371cad5e6749922',
'gzip-1.4.eb': 'gzip/d240a51c643ec42e709d405d958c7b26f5a25d5a',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/cea02d332af7044ae5faf762cea2ef6ffed014d2',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/f1dbb38c4518a15fc8bb1fbf797ceda02f0cacd0',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/3ef9ac73b468c989f5a47b30098d340e92c3d0da',
'toy-0.0.eb': 'toy/778417f0e140ebbaebd60d0f98c8b2411f980edf',
'toy-0.0-multiple.eb': 'toy/2d45f3cde87dedf30662f4a005023d56d2532bf0',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
]:
self.assertEqual(det_full_module_name_ec(dep_spec), ec2mod_map[dep_ec])
# restore default module naming scheme, and retest
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [
os.path.join(subdir, fil)
for (subdir, _, files) in os.walk(ecs_dir) for fil in files
]
ec_files = [fil for fil in ec_files if not "v2.0" in fil
] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn],
det_full_module_name_mg(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar',
det_full_module_name_ec(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
for test_mns_mod in [
'test_module_naming_scheme', 'test_module_naming_scheme_all'
]:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
mns_mod = __import__(mns_path, globals(), locals(), [''])
test_mnss = dict([
(x.__name__, x)
for x in get_subclasses(mns_mod.ModuleNamingScheme)
])
easybuild.tools.module_naming_scheme.AVAIL_MODULE_NAMING_SCHEMES.update(
test_mnss)
init_config(build_options=build_options)
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb':
'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
# test module naming scheme using all available easyconfig parameters
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeAll'
init_config(build_options=build_options)
# note: these checksums will change if another easyconfig parameter is added
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/698cacc77167c6824f597f0b6371cad5e6749922',
'gzip-1.4.eb': 'gzip/d240a51c643ec42e709d405d958c7b26f5a25d5a',
'gzip-1.4-GCC-4.6.3.eb':
'gzip/cea02d332af7044ae5faf762cea2ef6ffed014d2',
'gzip-1.5-goolf-1.4.10.eb':
'gzip/f1dbb38c4518a15fc8bb1fbf797ceda02f0cacd0',
'gzip-1.5-ictce-4.1.13.eb':
'gzip/3ef9ac73b468c989f5a47b30098d340e92c3d0da',
'toy-0.0.eb': 'toy/778417f0e140ebbaebd60d0f98c8b2411f980edf',
'toy-0.0-multiple.eb':
'toy/2d45f3cde87dedf30662f4a005023d56d2532bf0',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {
'name': 'goolf',
'version': '1.4.10'
},
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
}),
]:
self.assertEqual(det_full_module_name_ec(dep_spec),
ec2mod_map[dep_ec])
# restore default module naming scheme, and retest
os.environ[
'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def search_toolchain(name):
"""
Find a toolchain with matching name
returns toolchain (or None), found_toolchains
"""
log = fancylogger.getLogger("search_toolchain")
# import all available toolchains, so we know about them
tc_modules = []
for path in sys.path:
for module in glob.glob(os.path.join(path, 'easybuild', 'toolchains', '*.py')):
if not module.endswith('__init__.py'):
modpath = "easybuild.toolchains.%s" % module.split(os.path.sep)[-1].split('.')[0]
log.debug("importing toolchain module %s" % modpath)
tc_modules.append(__import__(modpath, globals(), locals(), ['']))
# make sure all defined toolchain constants are available in toolchain module
package = easybuild.tools.toolchain
tc_const_prefix = 'TC_CONSTANT_'
tc_const_re = re.compile('^%s(.*)$' % tc_const_prefix)
for tc_mod in tc_modules:
# determine classes imported in this module
mod_classes = []
for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
if hasattr(elem, '__module__'):
# exclude the toolchain class defined in that module
if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__)
mod_classes.append(elem)
# look for constants in modules of imported classes, and make them available
for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
for elem in dir(mod_class_mod):
res = tc_const_re.match(elem)
if res:
tc_const_name = res.group(1)
tc_const_value = getattr(mod_class_mod, elem)
log.debug("Found constant %s ('%s') in module %s, adding it to %s" % (tc_const_name,
tc_const_value,
mod_class_mod.__name__,
package.__name__))
if hasattr(package, tc_const_name):
cur_value = getattr(package, tc_const_name)
if not tc_const_value == cur_value:
log.error("Constant %s.%s defined as '%s', can't set it to '%s'." % (package.__name__,
tc_const_name,
cur_value,
tc_const_value
))
else:
setattr(package, tc_const_name, tc_const_value)
# obtain all subclasses of toolchain
found_tcs = nub(get_subclasses(Toolchain))
# filter found toolchain subclasses based on whether they can be used a toolchains
found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]
for tc in found_tcs:
if tc._is_toolchain_for(name):
return tc, found_tcs
return None, found_tcs
def test_module_naming_scheme(self):
"""Test using default module naming scheme."""
all_stops = [x[0] for x in EasyBlock.get_steps()]
init_config(build_options={'valid_stops': all_stops})
ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs')
ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files]
ec_files = [fil for fil in ec_files if not "v2.0" in fil] # TODO FIXME: drop this once 2.0 support works
build_options = {
'check_osdeps': False,
'robot_path': [ecs_dir],
'valid_stops': all_stops,
'validate': False,
}
init_config(build_options=build_options)
def test_mns():
"""Test default module naming scheme."""
# test default naming scheme
for ec_file in ec_files:
ec_path = os.path.abspath(ec_file)
ecs = process_easyconfig(ec_path, validate=False)
# derive module name directly from easyconfig file name
ec_fn = os.path.basename(ec_file)
if ec_fn in ec2mod_map:
# only check first, ignore any others (occurs when blocks are used (format v1.0 only))
self.assertEqual(ec2mod_map[ec_fn], det_full_module_name_mg(ecs[0]['ec']))
# test default module naming scheme
default_ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/1.4-GCC-4.6.3',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1.5-goolf-1.4.10',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/1.5-ictce-4.1.13',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0-somesuffix', # first block sets versionsuffix to '-somesuffix'
}
ec2mod_map = default_ec2mod_map
test_mns()
# generating module name from non-parsed easyconfig works fine
non_parsed = {
'name': 'foo',
'version': '1.2.3',
'versionsuffix': '-bar',
'toolchain': {
'name': 't00ls',
'version': '6.6.6',
},
}
self.assertEqual('foo/1.2.3-t00ls-6.6.6-bar', det_full_module_name_ec(non_parsed))
# install custom module naming scheme dynamically
test_mns_parent_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
sys.path.append(test_mns_parent_dir)
reload(easybuild)
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# make sure test module naming schemes are available
for test_mns_mod in ['test_module_naming_scheme', 'test_module_naming_scheme_all']:
mns_path = "easybuild.tools.module_naming_scheme.%s" % test_mns_mod
mns_mod = __import__(mns_path, globals(), locals(), [''])
test_mnss = dict([(x.__name__, x) for x in get_subclasses(mns_mod.ModuleNamingScheme)])
easybuild.tools.module_naming_scheme.AVAIL_MODULE_NAMING_SCHEMES.update(test_mnss)
init_config(build_options=build_options)
# test simple custom module naming scheme
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/4.6.3',
'gzip-1.4.eb': 'gzip/1.4',
'gzip-1.4-GCC-4.6.3.eb': 'gnu/gzip/1.4',
'gzip-1.5-goolf-1.4.10.eb': 'gnu/openmpi/gzip/1.5',
'gzip-1.5-ictce-4.1.13.eb': 'intel/intelmpi/gzip/1.5',
'toy-0.0.eb': 'toy/0.0',
'toy-0.0-multiple.eb': 'toy/0.0', # test module naming scheme ignores version suffixes
}
test_mns()
# test module naming scheme using all available easyconfig parameters
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingSchemeAll'
init_config(build_options=build_options)
ec2mod_map = {
'GCC-4.6.3.eb': 'GCC/afd4d25a1a2cdb1364c55274fb6929fab622f652',
'gzip-1.4.eb': 'gzip/b6306986fb95a06ad8bd2a09689d8997ff3e80dd',
'gzip-1.4-GCC-4.6.3.eb': 'gzip/3c2d54583487828c21e17ed185eac372cabc5bb0',
'gzip-1.5-goolf-1.4.10.eb': 'gzip/1fb1e3787d6063e05a04b2c054faf00dbe1dfe97',
'gzip-1.5-ictce-4.1.13.eb': 'gzip/78c9afa1ff09994fe38d796b7569ce4b175e3551',
'toy-0.0.eb': 'toy/494518267cc5ed64c4250c5fbd1730a6e48fde17',
'toy-0.0-multiple.eb': 'toy/02822d81743944e1c072fc3c717c666da70f1be6',
}
test_mns()
# test determining module name for dependencies (i.e. non-parsed easyconfigs)
# using a module naming scheme that requires all easyconfig parameters
for dep_ec, dep_spec in [
('GCC-4.6.3.eb', {
'name': 'GCC',
'version': '4.6.3',
'versionsuffix': '',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
('gzip-1.5-goolf-1.4.10.eb', {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}),
('toy-0.0-multiple.eb', {
'name': 'toy',
'version': '0.0',
'versionsuffix': '-multiple',
'toolchain': {'name': 'dummy', 'version': 'dummy'},
}),
]:
self.assertEqual(det_full_module_name_ec(dep_spec), ec2mod_map[dep_ec])
# restore default module naming scheme, and retest
os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
init_config(build_options=build_options)
ec2mod_map = default_ec2mod_map
test_mns()
def make_init(self):
""" add all the options to generaloption, so it can correctly parse the command line arguments """
opts = {
# long option: (description, type, action, default, short option)
"basepath": ("Directory (preferably shared) to use for temporary mympirun files (default: HOME).",
"str", "store", None),
'branchcount': ("Set the hydra branchcount", "int", "store", None),
"debuglvl": ("Specify debug level", "int", "store", 0),
"debugmpi": ("Enable MPI level debugging", None, "store_true", False),
"dry-run": ("Dry run mode, just print command that will be executed", None, 'store_true', False, 'D'),
"double": ("Run double the amount of processes (equivalent to --multi 2)", None, "store_true", False),
"hybrid": ("Run in hybrid mode, specify number of processes per node.", "int", "store", None, 'h'),
"launcher": ("The launcher to be used by Hydra (used in recent Intel MPI versions (> 4.1))"
"for example: ssh, pbsdsh, ..", "str", "store", None),
"logtofile": ("redirect the logging of mympirun to a file (instead of stdout/stderr)",
"str", "store", None),
"mpdbootverbose": ("Run verbose mpdboot", None, "store_true", False),
"mpirunoptions": ("String with options to pass to mpirun (will be appended to generate command)",
"str", "store", None),
"multi": ("Run the amount of processes multiplied by the given integer", "int", "store", None),
"noenvmodules": ("Don't pass the environment modules variables",
None, "store_true", False),
"order": ("Reorder the generated nodelist (default: normal. supports: sort, random[_<seed>])",
"str", "store", None),
"output": ("redirect the output of mpirun to a file (instead of stdout/stderr)",
"str", "store", None),
"output-check-timeout": ("Warn when no stdout/stderr was seen after start (in seconds; negative number "
"disables this test", "int", "store", DEFAULT_TIMEOUT),
"output-check-fatal": ("Exit with code %s instead of warn in case of output check timeout" % TIMEOUT_CODE,
None, "store_true", False),
"overridepin": (("Let mympriun set the affinity (default: disabled, left over to MPI implementation). "
"Supported types: 'compact','spread','cycle' (add 'pin' postfix for single core pinning, "
"e.g. 'cyclepin')."), "str", "store", None),
# don't set it by default. It will be set if needed (eg ipath)
"pinmpi": ("Disable MPI pinning", None, "store_true", True),
"rdma": ("Force rdma device", None, "store_true", None),
"schedtype": ("Specify scheduler (eg local, pbs...; will try to guess by default).",
"str", "store", None, "S"),
"setmpi": ("Specify MPI flavor (eg mpich2, openmpi...; will try to guess by default).",
"str", "store", None, "M"),
"showmpi": ("Print the known MPI classes and exit", None, "store_true", False, 'm'),
"showsched": ("Print the known Sched classes and exit", None, "store_true", False, 's'),
"sockets-per-node": ("Number of sockets per node (default: 0, i.e. try to detect #sockets "
"from /proc/cpuinfo)", "int", "store", 0),
"ssh": ("Force ssh for mpd startup (will try to use optimised method by default)",
None, "store_false", True),
"stats": ("Set MPI statistics level", "int", "store", 0),
"universe": (("Start only this number of processes instead of all (e.g. for MPI_Spawn) Total size of the "
"universe is all requested processes.)"), "int", "store", None),
'use_psm': ("Use Performance Scaled Messaging", None, "store_true", None),
"variablesprefix": (("Comma-separated list of exact names or prefixes to match environment variables "
"(<prefix>_ should match) to pass through."), "string", "extend", []),
}
descr = ["mympirun options", "General advanced mympirun options"]
prefix = ''
self.log.debug("Add advanced option parser: options %s, description %s, prefix %s", opts, descr, prefix)
self.add_group_parser(opts, descr, prefix=prefix)
# for all MPI classes, get the additional options
for mpi in get_subclasses(MPI):
if mpi.RUNTIMEOPTION is not None:
# don't try to add the same set of options twice (based on prefix)
prefix = mpi.RUNTIMEOPTION['prefix']
if prefix not in self.dict_by_prefix():
opts = mpi.RUNTIMEOPTION['options']
descr = mpi.RUNTIMEOPTION['description']
self.log.debug("Add MPI subclass %s option parser prefix %s descr %s opts %s",
mpi.__name__, prefix, descr, opts)
self.add_group_parser(opts, descr, prefix=prefix)
def avail_module_generators():
"""
Return all known module syntaxes.
"""
return dict([(k.SYNTAX, k) for k in get_subclasses(ModuleGenerator)])
