def test_import_available_modules(self): """Test for import_available_modules function.""" res = import_available_modules('easybuild.tools.repository') self.assertEqual(len(res), 5) # don't check all, since some required specific Python packages to be installed... self.assertTrue(easybuild.tools.repository.filerepo in res) # replicate situation where import_available_modules failed when running in directory where modules are located # cfr. https://github.com/easybuilders/easybuild-framework/issues/2659 # and https://github.com/easybuilders/easybuild-framework/issues/2742 test123 = os.path.join(self.test_prefix, 'test123') mkdir(test123) write_file(os.path.join(test123, '__init__.py'), '') write_file(os.path.join(test123, 'one.py'), '') write_file(os.path.join(test123, 'two.py'), '') write_file(os.path.join(test123, 'three.py'), '') change_dir(self.test_prefix) res = import_available_modules('test123') import test123.one import test123.two import test123.three self.assertEqual([test123.one, test123.three, test123.two], res)
def test_import_available_modules(self): """Test for import_available_modules function.""" res = import_available_modules('easybuild.tools.repository') self.assertEqual(len(res), 5) # don't check all, since some required specific Python packages to be installed... self.assertTrue(easybuild.tools.repository.filerepo in res) # replicate situation where import_available_modules failed when running in directory where modules are located # cfr. https://github.com/easybuilders/easybuild-framework/issues/2659 # and https://github.com/easybuilders/easybuild-framework/issues/2742 test123 = os.path.join(self.test_prefix, 'test123') mkdir(test123) write_file(os.path.join(test123, '__init__.py'), '') write_file(os.path.join(test123, 'one.py'), '') write_file(os.path.join(test123, 'two.py'), '') write_file(os.path.join(test123, 'three.py'), '') # this test relies on having an empty entry in sys.path (which represents the current working directory) # may not be there (e.g. when testing with Python 3.7) if '' not in sys.path: sys.path.insert(0, '') change_dir(self.test_prefix) res = import_available_modules('test123') import test123.one import test123.two import test123.three self.assertEqual([test123.one, test123.three, test123.two], res)
def avail_job_backends(check_usable=True): """ Return all known job execution backends. """ import_available_modules('easybuild.tools.job') class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)]) return class_dict
def avail_package_naming_schemes(): """ Returns the list of valed naming schemes that are in the easybuild.package.package_naming_scheme namespace """ import_available_modules('easybuild.tools.package.package_naming_scheme') class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)]) return class_dict
def avail_module_naming_schemes(): """ Returns a list of available module naming schemes. """ # all ModuleNamingScheme subclasses available in easybuild.tools.module_naming_scheme namespace are eligible import_available_modules('easybuild.tools.module_naming_scheme') # construct name-to-class dict of available module naming scheme avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)]) return avail_mnss
def avail_repositories(check_useable=True): """ Return all available repositories. check_useable: boolean, if True, only return usable repositories """ import_available_modules('easybuild.tools.repository') class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable]) if 'FileRepository' not in class_dict: raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories") return class_dict
def avail_repositories(check_useable=True): """ Return all available repositories. check_useable: boolean, if True, only return usable repositories """ import_available_modules('easybuild.tools.repository') class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable]) if not 'FileRepository' in class_dict: _log.error('avail_repositories: FileRepository missing from list of repositories') return class_dict
def search_toolchain(name): """ Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains. :param name: toolchain name :return: Toolchain instance (or None), found_toolchains """ package = easybuild.tools.toolchain check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name): # import all available toolchains, so we know about them tc_modules = import_available_modules('easybuild.toolchains') # make sure all defined toolchain constants are available in toolchain module tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX) for tc_mod in tc_modules: # determine classes imported in this module mod_classes = [] for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]: if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: elem_name = elem.__name__ if hasattr(elem, '__name__') else elem _log.debug("Adding %s to list of imported classes used for looking for constants", elem_name) mod_classes.append(elem) # look for constants in modules of imported classes, and make them available for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]: for elem in dir(mod_class_mod): res = tc_const_re.match(elem) if res: tc_const_name = res.group(1) tc_const_value = getattr(mod_class_mod, elem) _log.debug("Found constant %s ('%s') in module %s, adding it to %s", tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) if hasattr(package, tc_const_name): cur_value = getattr(package, tc_const_name) if not tc_const_value == cur_value: raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", package.__name__, tc_const_name, cur_value, tc_const_value) else: setattr(package, tc_const_name, tc_const_value) # indicate that processing of toolchain constants is done, so it's not done again setattr(package, check_attr_name, True) else: _log.debug("Skipping importing of toolchain modules, processing of toolchain constants is already done.") # obtain all subclasses of toolchain found_tcs = nub(get_subclasses(Toolchain)) # filter found toolchain subclasses based on whether they can be used a toolchains found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)] for tc in found_tcs: if tc._is_toolchain_for(name): return tc, found_tcs return None, found_tcs
def search_toolchain(name): """ Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains. :param name: toolchain name :return: Toolchain instance (or None), found_toolchains """ package = easybuild.tools.toolchain check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name): # import all available toolchains, so we know about them tc_modules = import_available_modules('easybuild.toolchains') # make sure all defined toolchain constants are available in toolchain module tc_const_re = re.compile('^%s(.*)$' % TC_CONST_PREFIX) for tc_mod in tc_modules: # determine classes imported in this module mod_classes = [] for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]: if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: _log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__) mod_classes.append(elem) # look for constants in modules of imported classes, and make them available for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]: for elem in dir(mod_class_mod): res = tc_const_re.match(elem) if res: tc_const_name = res.group(1) tc_const_value = getattr(mod_class_mod, elem) _log.debug("Found constant %s ('%s') in module %s, adding it to %s", tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) if hasattr(package, tc_const_name): cur_value = getattr(package, tc_const_name) if not tc_const_value == cur_value: raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", package.__name__, tc_const_name, cur_value, tc_const_value) else: setattr(package, tc_const_name, tc_const_value) # indicate that processing of toolchain constants is done, so it's not done again setattr(package, check_attr_name, True) else: _log.debug("Skipping importing of toolchain modules, processing of toolchain constants is already done.") # obtain all subclasses of toolchain found_tcs = nub(get_subclasses(Toolchain)) # filter found toolchain subclasses based on whether they can be used a toolchains found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)] for tc in found_tcs: if tc._is_toolchain_for(name): return tc, found_tcs return None, found_tcs
def avail_module_naming_schemes(): """ Returns a list of available module naming schemes. """ mns_attr = 'AVAIL_MODULE_NAMING_SCHEMES' if not hasattr(module_naming_scheme, mns_attr): # all subclasses of ModuleNamingScheme available in the easybuild.tools.module_naming_scheme namespace are eligible import_available_modules('easybuild.tools.module_naming_scheme') # construct name-to-class dict of available module naming scheme avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)]) # cache dict of available module naming scheme in module constant setattr(module_naming_scheme, mns_attr, avail_mnss) return avail_mnss else: return getattr(module_naming_scheme, mns_attr)
def test_gen_easyblocks(self): """ Test gen_easyblocks_overview_rst function """ gen_easyblocks_pkg = 'easybuild.easyblocks.generic' modules = import_available_modules(gen_easyblocks_pkg) common_params = { 'ConfigureMake': ['configopts', 'buildopts', 'installopts'], } doc_functions = ['build_step', 'configure_step', 'test_step'] eb_overview = gen_easyblocks_overview_rst(gen_easyblocks_pkg, 'easyconfigs', common_params, doc_functions) ebdoc = '\n'.join(eb_overview) # extensive check for ConfigureMake easyblock check_configuremake = '\n'.join([ ".. _ConfigureMake:", '', "``ConfigureMake``", "=================", '', "(derives from EasyBlock)", '', "Dummy support for building and installing applications with configure/make/make install.", '', "Commonly used easyconfig parameters with ``ConfigureMake`` easyblock", "--------------------------------------------------------------------", '', "==================== ================================================================", "easyconfig parameter description ", "==================== ================================================================", "configopts Extra options passed to configure (default already has --prefix)", "buildopts Extra options passed to make step (default already has -j X) ", "installopts Extra options for installation ", "==================== ================================================================", ]) self.assertTrue(check_configuremake in ebdoc) names = [] for mod in modules: for name, obj in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(gen_easyblocks_pkg): self.assertTrue(name in ebdoc) names.append(name) toc = [":ref:`" + n + "`" for n in sorted(set(names))] pattern = " - ".join(toc) regex = re.compile(pattern) self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))
def test_gen_easyblocks(self): """ Test gen_easyblocks_overview_rst function """ module = 'easybuild.easyblocks.generic' modules = import_available_modules(module) common_params = { 'ConfigureMake' : ['configopts', 'buildopts', 'installopts'], } doc_functions = ['build_step', 'configure_step', 'test_step'] eb_overview = gen_easyblocks_overview_rst(module, 'easyconfigs', common_params, doc_functions) ebdoc = '\n'.join(eb_overview) # extensive check for ConfigureMake easyblock check_configuremake = '\n'.join([ ".. _ConfigureMake:", '', "``ConfigureMake``", "=================", '', "(derives from EasyBlock)", '', "Dummy support for building and installing applications with configure/make/make install.", '', "Commonly used easyconfig parameters with ``ConfigureMake`` easyblock", "--------------------------------------------------------------------", '', "==================== ================================================================", "easyconfig parameter description ", "==================== ================================================================", "configopts Extra options passed to configure (default already has --prefix)", "buildopts Extra options passed to make step (default already has -j X) ", "installopts Extra options for installation ", "==================== ================================================================", ]) self.assertTrue(check_configuremake in ebdoc) names = [] for mod in modules: for name, obj in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(module): self.assertTrue(name in ebdoc) names.append(name) toc = [":ref:`" + n + "`" for n in sorted(names)] pattern = " - ".join(toc) regex = re.compile(pattern) self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc))
def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]): """ Compose overview of all easyblocks in the given package in rst format """ modules = import_available_modules(package_name) doc = [] all_blocks = [] # get all blocks for mod in modules: for name, obj in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith( package_name) and eb_class not in all_blocks: all_blocks.append(eb_class) for eb_class in sorted(all_blocks, key=lambda c: c.__name__): doc.extend( gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks)) title = 'Overview of generic easyblocks' heading = [ '*(this page was generated automatically using* ``easybuild.tools.docs.gen_easyblocks_overview_rst()`` *)*', '', '=' * len(title), title, '=' * len(title), '', ] contents = [ ":ref:`" + b.__name__ + "`" for b in sorted(all_blocks, key=lambda b: b.__name__) ] toc = ' - '.join(contents) heading.append(toc) heading.append('') return heading + doc
def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]): """ Compose overview of all easyblocks in the given package in rst format """ modules = import_available_modules(package_name) docs = [] all_blocks = [] # get all blocks for mod in modules: for name,obj in inspect.getmembers(mod, inspect.isclass): eb_class = getattr(mod, name) # skip imported classes that are not easyblocks if eb_class.__module__.startswith(package_name) and eb_class not in all_blocks: all_blocks.append(eb_class) for eb_class in sorted(all_blocks, key=lambda c: c.__name__): docs.append(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks)) title = 'Overview of generic easyblocks' heading = [ '*(this page was generated automatically using* ``easybuild.tools.docs.gen_easyblocks_overview_rst()`` *)*', '', '=' * len(title), title, '=' * len(title), '', ] contents = [":ref:`" + b.__name__ + "`" for b in sorted(all_blocks, key=lambda b: b.__name__)] toc = ' - '.join(contents) heading.append(toc) heading.append('') return heading + docs