def test_import_available_modules(self):
"""Test for import_available_modules function."""
res = import_available_modules('easybuild.tools.repository')
self.assertEqual(len(res), 5)
# don't check all, since some required specific Python packages to be installed...
self.assertTrue(easybuild.tools.repository.filerepo in res)
# replicate situation where import_available_modules failed when running in directory where modules are located
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2659
# and https://github.com/easybuilders/easybuild-framework/issues/2742
test123 = os.path.join(self.test_prefix, 'test123')
mkdir(test123)
write_file(os.path.join(test123, '__init__.py'), '')
write_file(os.path.join(test123, 'one.py'), '')
write_file(os.path.join(test123, 'two.py'), '')
write_file(os.path.join(test123, 'three.py'), '')
change_dir(self.test_prefix)
res = import_available_modules('test123')
import test123.one
import test123.two
import test123.three
self.assertEqual([test123.one, test123.three, test123.two], res)
def test_import_available_modules(self):
"""Test for import_available_modules function."""
res = import_available_modules('easybuild.tools.repository')
self.assertEqual(len(res), 5)
# don't check all, since some required specific Python packages to be installed...
self.assertTrue(easybuild.tools.repository.filerepo in res)
# replicate situation where import_available_modules failed when running in directory where modules are located
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2659
# and https://github.com/easybuilders/easybuild-framework/issues/2742
test123 = os.path.join(self.test_prefix, 'test123')
mkdir(test123)
write_file(os.path.join(test123, '__init__.py'), '')
write_file(os.path.join(test123, 'one.py'), '')
write_file(os.path.join(test123, 'two.py'), '')
write_file(os.path.join(test123, 'three.py'), '')
# this test relies on having an empty entry in sys.path (which represents the current working directory)
# may not be there (e.g. when testing with Python 3.7)
if '' not in sys.path:
sys.path.insert(0, '')
change_dir(self.test_prefix)
res = import_available_modules('test123')
import test123.one
import test123.two
import test123.three
self.assertEqual([test123.one, test123.three, test123.two], res)
def avail_job_backends(check_usable=True):
"""
Return all known job execution backends.
"""
import_available_modules('easybuild.tools.job')
class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)])
return class_dict
def avail_job_backends(check_usable=True):
"""
Return all known job execution backends.
"""
import_available_modules('easybuild.tools.job')
class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)])
return class_dict
def avail_package_naming_schemes():
"""
Returns the list of valed naming schemes that are in the easybuild.package.package_naming_scheme namespace
"""
import_available_modules('easybuild.tools.package.package_naming_scheme')
class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)])
return class_dict
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
# all ModuleNamingScheme subclasses available in easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)])
return avail_mnss
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
# all ModuleNamingScheme subclasses available in easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)])
return avail_mnss
def avail_repositories(check_useable=True):
"""
Return all available repositories.
check_useable: boolean, if True, only return usable repositories
"""
import_available_modules('easybuild.tools.repository')
class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable])
if 'FileRepository' not in class_dict:
raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories")
return class_dict
def avail_repositories(check_useable=True):
"""
Return all available repositories.
check_useable: boolean, if True, only return usable repositories
"""
import_available_modules('easybuild.tools.repository')
class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable])
if not 'FileRepository' in class_dict:
_log.error('avail_repositories: FileRepository missing from list of repositories')
return class_dict
def search_toolchain(name):
"""
Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains.
:param name: toolchain name
:return: Toolchain instance (or None), found_toolchains
"""
package = easybuild.tools.toolchain
check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX
if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name):
# import all available toolchains, so we know about them
tc_modules = import_available_modules('easybuild.toolchains')
# make sure all defined toolchain constants are available in toolchain module
tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX)
for tc_mod in tc_modules:
# determine classes imported in this module
mod_classes = []
for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
if hasattr(elem, '__module__'):
# exclude the toolchain class defined in that module
if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
elem_name = elem.__name__ if hasattr(elem, '__name__') else elem
_log.debug("Adding %s to list of imported classes used for looking for constants", elem_name)
mod_classes.append(elem)
# look for constants in modules of imported classes, and make them available
for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
for elem in dir(mod_class_mod):
res = tc_const_re.match(elem)
if res:
tc_const_name = res.group(1)
tc_const_value = getattr(mod_class_mod, elem)
_log.debug("Found constant %s ('%s') in module %s, adding it to %s",
tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__)
if hasattr(package, tc_const_name):
cur_value = getattr(package, tc_const_name)
if not tc_const_value == cur_value:
raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.",
package.__name__, tc_const_name, cur_value, tc_const_value)
else:
setattr(package, tc_const_name, tc_const_value)
# indicate that processing of toolchain constants is done, so it's not done again
setattr(package, check_attr_name, True)
else:
_log.debug("Skipping importing of toolchain modules, processing of toolchain constants is already done.")
# obtain all subclasses of toolchain
found_tcs = nub(get_subclasses(Toolchain))
# filter found toolchain subclasses based on whether they can be used a toolchains
found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]
for tc in found_tcs:
if tc._is_toolchain_for(name):
return tc, found_tcs
return None, found_tcs
def search_toolchain(name):
"""
Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains.
:param name: toolchain name
:return: Toolchain instance (or None), found_toolchains
"""
package = easybuild.tools.toolchain
check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX
if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name):
# import all available toolchains, so we know about them
tc_modules = import_available_modules('easybuild.toolchains')
# make sure all defined toolchain constants are available in toolchain module
tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX)
for tc_mod in tc_modules:
# determine classes imported in this module
mod_classes = []
for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
if hasattr(elem, '__module__'):
# exclude the toolchain class defined in that module
if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
_log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__)
mod_classes.append(elem)
# look for constants in modules of imported classes, and make them available
for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
for elem in dir(mod_class_mod):
res = tc_const_re.match(elem)
if res:
tc_const_name = res.group(1)
tc_const_value = getattr(mod_class_mod, elem)
_log.debug("Found constant %s ('%s') in module %s, adding it to %s",
tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__)
if hasattr(package, tc_const_name):
cur_value = getattr(package, tc_const_name)
if not tc_const_value == cur_value:
raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.",
package.__name__, tc_const_name, cur_value, tc_const_value)
else:
setattr(package, tc_const_name, tc_const_value)
# indicate that processing of toolchain constants is done, so it's not done again
setattr(package, check_attr_name, True)
else:
_log.debug("Skipping importing of toolchain modules, processing of toolchain constants is already done.")
# obtain all subclasses of toolchain
found_tcs = nub(get_subclasses(Toolchain))
# filter found toolchain subclasses based on whether they can be used a toolchains
found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]
for tc in found_tcs:
if tc._is_toolchain_for(name):
return tc, found_tcs
return None, found_tcs
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
mns_attr = 'AVAIL_MODULE_NAMING_SCHEMES'
if not hasattr(module_naming_scheme, mns_attr):
# all subclasses of ModuleNamingScheme available in the easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)])
# cache dict of available module naming scheme in module constant
setattr(module_naming_scheme, mns_attr, avail_mnss)
return avail_mnss
else:
return getattr(module_naming_scheme, mns_attr)
def avail_module_naming_schemes():
"""
Returns a list of available module naming schemes.
"""
mns_attr = 'AVAIL_MODULE_NAMING_SCHEMES'
if not hasattr(module_naming_scheme, mns_attr):
# all subclasses of ModuleNamingScheme available in the easybuild.tools.module_naming_scheme namespace are eligible
import_available_modules('easybuild.tools.module_naming_scheme')
# construct name-to-class dict of available module naming scheme
avail_mnss = dict([(x.__name__, x)
for x in get_subclasses(ModuleNamingScheme)])
# cache dict of available module naming scheme in module constant
setattr(module_naming_scheme, mns_attr, avail_mnss)
return avail_mnss
else:
return getattr(module_naming_scheme, mns_attr)
def test_gen_easyblocks(self):
""" Test gen_easyblocks_overview_rst function """
gen_easyblocks_pkg = 'easybuild.easyblocks.generic'
modules = import_available_modules(gen_easyblocks_pkg)
common_params = {
'ConfigureMake': ['configopts', 'buildopts', 'installopts'],
}
doc_functions = ['build_step', 'configure_step', 'test_step']
eb_overview = gen_easyblocks_overview_rst(gen_easyblocks_pkg,
'easyconfigs', common_params,
doc_functions)
ebdoc = '\n'.join(eb_overview)
# extensive check for ConfigureMake easyblock
check_configuremake = '\n'.join([
".. _ConfigureMake:",
'',
"``ConfigureMake``",
"=================",
'',
"(derives from EasyBlock)",
'',
"Dummy support for building and installing applications with configure/make/make install.",
'',
"Commonly used easyconfig parameters with ``ConfigureMake`` easyblock",
"--------------------------------------------------------------------",
'',
"==================== ================================================================",
"easyconfig parameter description ",
"==================== ================================================================",
"configopts Extra options passed to configure (default already has --prefix)",
"buildopts Extra options passed to make step (default already has -j X) ",
"installopts Extra options for installation ",
"==================== ================================================================",
])
self.assertTrue(check_configuremake in ebdoc)
names = []
for mod in modules:
for name, obj in inspect.getmembers(mod, inspect.isclass):
eb_class = getattr(mod, name)
# skip imported classes that are not easyblocks
if eb_class.__module__.startswith(gen_easyblocks_pkg):
self.assertTrue(name in ebdoc)
names.append(name)
toc = [":ref:`" + n + "`" for n in sorted(set(names))]
pattern = " - ".join(toc)
regex = re.compile(pattern)
self.assertTrue(re.search(regex, ebdoc),
"Pattern %s found in %s" % (regex.pattern, ebdoc))
def test_gen_easyblocks(self):
""" Test gen_easyblocks_overview_rst function """
module = 'easybuild.easyblocks.generic'
modules = import_available_modules(module)
common_params = {
'ConfigureMake' : ['configopts', 'buildopts', 'installopts'],
}
doc_functions = ['build_step', 'configure_step', 'test_step']
eb_overview = gen_easyblocks_overview_rst(module, 'easyconfigs', common_params, doc_functions)
ebdoc = '\n'.join(eb_overview)
# extensive check for ConfigureMake easyblock
check_configuremake = '\n'.join([
".. _ConfigureMake:",
'',
"``ConfigureMake``",
"=================",
'',
"(derives from EasyBlock)",
'',
"Dummy support for building and installing applications with configure/make/make install.",
'',
"Commonly used easyconfig parameters with ``ConfigureMake`` easyblock",
"--------------------------------------------------------------------",
'',
"==================== ================================================================",
"easyconfig parameter description ",
"==================== ================================================================",
"configopts Extra options passed to configure (default already has --prefix)",
"buildopts Extra options passed to make step (default already has -j X) ",
"installopts Extra options for installation ",
"==================== ================================================================",
])
self.assertTrue(check_configuremake in ebdoc)
names = []
for mod in modules:
for name, obj in inspect.getmembers(mod, inspect.isclass):
eb_class = getattr(mod, name)
# skip imported classes that are not easyblocks
if eb_class.__module__.startswith(module):
self.assertTrue(name in ebdoc)
names.append(name)
toc = [":ref:`" + n + "`" for n in sorted(names)]
pattern = " - ".join(toc)
regex = re.compile(pattern)
self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))
def gen_easyblocks_overview_rst(package_name,
path_to_examples,
common_params={},
doc_functions=[]):
"""
Compose overview of all easyblocks in the given package in rst format
"""
modules = import_available_modules(package_name)
doc = []
all_blocks = []
# get all blocks
for mod in modules:
for name, obj in inspect.getmembers(mod, inspect.isclass):
eb_class = getattr(mod, name)
# skip imported classes that are not easyblocks
if eb_class.__module__.startswith(
package_name) and eb_class not in all_blocks:
all_blocks.append(eb_class)
for eb_class in sorted(all_blocks, key=lambda c: c.__name__):
doc.extend(
gen_easyblock_doc_section_rst(eb_class, path_to_examples,
common_params, doc_functions,
all_blocks))
title = 'Overview of generic easyblocks'
heading = [
'*(this page was generated automatically using* ``easybuild.tools.docs.gen_easyblocks_overview_rst()`` *)*',
'',
'=' * len(title),
title,
'=' * len(title),
'',
]
contents = [
":ref:`" + b.__name__ + "`"
for b in sorted(all_blocks, key=lambda b: b.__name__)
]
toc = ' - '.join(contents)
heading.append(toc)
heading.append('')
return heading + doc
def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]):
"""
Compose overview of all easyblocks in the given package in rst format
"""
modules = import_available_modules(package_name)
docs = []
all_blocks = []
# get all blocks
for mod in modules:
for name,obj in inspect.getmembers(mod, inspect.isclass):
eb_class = getattr(mod, name)
# skip imported classes that are not easyblocks
if eb_class.__module__.startswith(package_name) and eb_class not in all_blocks:
all_blocks.append(eb_class)
for eb_class in sorted(all_blocks, key=lambda c: c.__name__):
docs.append(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks))
title = 'Overview of generic easyblocks'
heading = [
'*(this page was generated automatically using* ``easybuild.tools.docs.gen_easyblocks_overview_rst()`` *)*',
'',
'=' * len(title),
title,
'=' * len(title),
'',
]
contents = [":ref:`" + b.__name__ + "`" for b in sorted(all_blocks, key=lambda b: b.__name__)]
toc = ' - '.join(contents)
heading.append(toc)
heading.append('')
return heading + docs
