def _create_from_python(cls, filename): try: mod = util.import_module_from_file(filename) except ImportError as e: # import_module_from_file() may raise an ImportError if the # configuration file is under ReFrame's top-level directory raise ConfigError( f"could not load Python configuration file: '{filename}'" ) from e if hasattr(mod, 'settings'): # Looks like an old style config raise ConfigError( f"the syntax of the configuration file {filename!r} " f"is no longer supported; please convert it using the " f"'--upgrade-config-file' option" ) mod = util.import_module_from_file(filename) if not hasattr(mod, 'site_configuration'): raise ConfigError( f"not a valid Python configuration file: '{filename}'" ) return _SiteConfig(mod.site_configuration, filename)
def test_load_unknown_path(self): try: util.import_module_from_file('/foo') self.fail() except ImportError as e: self.assertEqual('foo', e.name) self.assertEqual('/foo', e.path)
def test_load_unknown_path(self): try: util.import_module_from_file('/foo') pytest.fail() except ImportError as e: assert 'foo' == e.name assert '/foo' == e.path
def test_load_relative(self): with os_ext.change_dir('reframe'): # Load a module from a directory up module = util.import_module_from_file('../reframe/__init__.py') self.assertEqual(reframe.VERSION, module.VERSION) self.assertEqual('reframe', module.__name__) self.assertIs(module, sys.modules.get('reframe')) # Load a module from the current directory module = util.import_module_from_file('utility/os_ext.py') self.assertEqual('reframe.utility.os_ext', module.__name__) self.assertIs(module, sys.modules.get('reframe.utility.os_ext'))
def test_import_from_file_load_relative(): with osext.change_dir('reframe'): # Load a module from a directory up module = util.import_module_from_file('../reframe/__init__.py') assert reframe.VERSION == module.VERSION assert 'reframe' == module.__name__ assert module is sys.modules.get('reframe') # Load a module from the current directory module = util.import_module_from_file('utility/osext.py') assert 'reframe.utility.osext' == module.__name__ assert module is sys.modules.get('reframe.utility.osext')
def load_from_file(self, filename, force=False): if not self._validate_source(filename): return [] return self.load_from_module( util.import_module_from_file(filename, force) )
def load_settings_from_file(filename): global _settings try: _settings = util.import_module_from_file(filename).settings return _settings except Exception as e: raise ConfigError("could not load configuration file `%s'" % filename) from e
def test_import_from_file_existing_module_name(tmp_path): test_file = tmp_path / 'os.py' with open(test_file, 'w') as fp: print('var = 1', file=fp) module = util.import_module_from_file(test_file) assert module.var == 1 assert not hasattr(module, 'path') assert hasattr(os, 'path')
def test_load_outside_pkg(self): module = util.import_module_from_file(os.path.__file__) # os imports the OS-specific path libraries under the name `path`. Our # importer will import the actual file, thus the module name should be # the real one. assert (module is sys.modules.get('posixpath') or module is sys.modules.get('ntpath') or module is sys.modules.get('macpath'))
def _create_from_python(cls, filename): try: mod = util.import_module_from_file(filename) except ImportError as e: # import_module_from_file() may raise an ImportError if the # configuration file is under ReFrame's top-level directory raise ConfigError( f"could not load Python configuration file: '{filename}'" ) from e if hasattr(mod, 'settings'): # Looks like an old style config raise ReframeDeprecationWarning( f"the syntax of the configuration file '{filename}' " f"is deprecated") mod = util.import_module_from_file(filename) if not hasattr(mod, 'site_configuration'): raise ConfigError( f"not a valid Python configuration file: '{filename}'") return _SiteConfig(mod.site_configuration, filename)
def _object_hook(json): filename = json.pop('__rfm_file__', None) typename = json.pop('__rfm_class__', None) if filename is None or typename is None: return json mod = util.import_module_from_file(filename) cls = getattr(mod, typename) obj = cls.__new__(cls) obj.__dict__.update(json) if hasattr(obj, '__rfm_json_decode__'): obj.__rfm_json_decode__(json) return obj
def load_from_file(self, filename, force=False): if not self._validate_source(filename): return [] try: return self.load_from_module( util.import_module_from_file(filename, force)) except Exception: exc_info = sys.exc_info() if not is_severe(*exc_info): # Simply skip the file in this case getlogger().warning( f"skipping test file {filename!r}: {what(*exc_info)} " f"(rerun with '-v' for more information)") getlogger().verbose(traceback.format_exc()) return [] else: raise
def test_load_directory(self): module = util.import_module_from_file('reframe') self.assertEqual(reframe.VERSION, module.VERSION) self.assertEqual('reframe', module.__name__) self.assertIs(module, sys.modules.get('reframe'))
def test_import_from_file_load_twice(): filename = os.path.abspath('reframe') module1 = util.import_module_from_file(filename) module2 = util.import_module_from_file(filename) assert module1 is module2
def test_load_twice(self): filename = os.path.abspath('reframe') module1 = util.import_module_from_file(filename) module2 = util.import_module_from_file(filename) self.assertIs(module1, module2)
def test_import_from_file_load_directory_relative(): with osext.change_dir('reframe'): module = util.import_module_from_file('../reframe') assert reframe.VERSION == module.VERSION assert 'reframe' == module.__name__ assert module is sys.modules.get('reframe')
def test_import_from_file_load_directory(): module = util.import_module_from_file('reframe') assert reframe.VERSION == module.VERSION assert 'reframe' == module.__name__ assert module is sys.modules.get('reframe')
def test_import_from_file_load_abspath(): filename = os.path.abspath('reframe/__init__.py') module = util.import_module_from_file(filename) assert reframe.VERSION == module.VERSION assert 'reframe' == module.__name__ assert module is sys.modules.get('reframe')
def test_load_directory_relative(self): with os_ext.change_dir('reframe'): module = util.import_module_from_file('../reframe') self.assertEqual(reframe.VERSION, module.VERSION) self.assertEqual('reframe', module.__name__) self.assertIs(module, sys.modules.get('reframe'))
def test_load_namespace_package(self): module = util.import_module_from_file('unittests/resources') self.assertIn('unittests', sys.modules) self.assertIn('unittests.resources', sys.modules)
def test_import_from_file_load_namespace_package(): util.import_module_from_file('unittests/resources') assert 'unittests' in sys.modules assert 'unittests.resources' in sys.modules
import re import semver import os import yaml import reframe as rfm import reframe.core.exceptions as error import reframe.utility as util import reframe.utility.osext as osext import reframe.utility.sanity as sn import reframe.utility.udeps as udeps from reframe.core.exceptions import SanityError spacklib = util.import_module_from_file( os.path.join(os.path.dirname(__file__), 'src', 'spack_util', 'spacklib.py')) spackconfig = util.import_module_from_file( os.path.join(os.path.dirname(__file__), 'src', 'spack_util', 'spack_config.py')) SPACK_VERSIONS = ['develop', '0.15.4', '0.16.1'] # TODO find a mechanism to include intel in the list base_cuda_compilers = ['gcc', 'cce'] base_generic_compilers = ['gcc', 'cce', 'pgi'] cuda_compatible_compilers = spacklib.generate_compiler_list_for_testing( [r'cdt-cuda/'], base_cuda_compilers) generic_compilers = spacklib.generate_compiler_list_for_testing( [r'cdt/'], base_generic_compilers)
def test_load_relpath(self): module = util.import_module_from_file('reframe/__init__.py') self.assertEqual(reframe.VERSION, module.VERSION) self.assertEqual('reframe', module.__name__) self.assertIs(module, sys.modules.get('reframe'))
def load_from_file(self, filename, **check_args): if not self._validate_source(filename): return [] return self.load_from_module(util.import_module_from_file(filename))
def test_import_from_file_load_relpath(): module = util.import_module_from_file('reframe/__init__.py') assert reframe.VERSION == module.VERSION assert 'reframe' == module.__name__ assert module is sys.modules.get('reframe')
def convert_old_config(filename, newfilename=None): old_config = util.import_module_from_file(filename).settings converted = { 'systems': [], 'environments': [], 'logging': [], } perflogdir = {} old_systems = old_config.site_configuration['systems'].items() for sys_name, sys_spec in old_systems: sys_dict = {'name': sys_name} system_perflogdir = sys_spec.pop('perflogdir', None) perflogdir.setdefault(system_perflogdir, []) perflogdir[system_perflogdir].append(sys_name) sys_dict.update(sys_spec) # hostnames is now a required property if 'hostnames' not in sys_spec: sys_dict['hostnames'] = [] # Make variables dictionary into a list of lists if 'variables' in sys_spec: sys_dict['variables'] = [ [vname, v] for vname, v in sys_dict['variables'].items() ] # Make partitions dictionary into a list if 'partitions' in sys_spec: sys_dict['partitions'] = [] for pname, p in sys_spec['partitions'].items(): new_p = {'name': pname} new_p.update(p) if p['scheduler'] == 'nativeslurm': new_p['scheduler'] = 'slurm' new_p['launcher'] = 'srun' elif p['scheduler'] == 'local': new_p['scheduler'] = 'local' new_p['launcher'] = 'local' else: sched, launcher, *_ = p['scheduler'].split('+') new_p['scheduler'] = sched new_p['launcher'] = launcher # Make resources dictionary into a list if 'resources' in p: new_p['resources'] = [{ 'name': rname, 'options': r } for rname, r in p['resources'].items()] # Make variables dictionary into a list of lists if 'variables' in p: new_p['variables'] = [ [vname, v] for vname, v in p['variables'].items() ] if 'container_platforms' in p: new_p['container_platforms'] = [] for cname, c in p['container_platforms'].items(): new_c = {'type': cname} new_c.update(c) if 'variables' in c: new_c['variables'] = [ [vn, v] for vn, v in c['variables'].items() ] new_p['container_platforms'].append(new_c) sys_dict['partitions'].append(new_p) converted['systems'].append(sys_dict) old_environs = old_config.site_configuration['environments'].items() for env_target, env_entries in old_environs: for ename, e in env_entries.items(): new_env = {'name': ename} if env_target != '*': new_env['target_systems'] = [env_target] new_env.update(e) # Convert variables dictionary to a list of lists if 'variables' in e: new_env['variables'] = [[vname, v] for vname, v in e['variables'].items()] # Type attribute is not used anymore if 'type' in new_env: del new_env['type'] converted['environments'].append(new_env) if 'modes' in old_config.site_configuration: converted['modes'] = [] old_modes = old_config.site_configuration['modes'].items() for target_mode, mode_entries in old_modes: for mname, m in mode_entries.items(): new_mode = {'name': mname, 'options': m} if target_mode != '*': new_mode['target_systems'] = [target_mode] converted['modes'].append(new_mode) def handler_list(handler_config, basedir=None): ret = [] for h in handler_config: new_h = h.copy() new_h['level'] = h['level'].lower() if h['type'] == 'graylog': # `host` and `port` attribute are converted to `address` new_h['address'] = h['host'] if 'port' in h: new_h['address'] += ':' + h['port'] elif h['type'] == 'filelog' and basedir is not None: new_h['basedir'] = basedir ret.append(new_h) return ret for basedir, target_systems in perflogdir.items(): converted['logging'].append({ 'level': old_config.logging_config['level'].lower(), 'handlers': handler_list(old_config.logging_config['handlers']), 'handlers_perflog': handler_list(old_config.perf_logging_config['handlers'], basedir=basedir), 'target_systems': target_systems }) if basedir is None: del converted['logging'][-1]['target_systems'] converted['general'] = [{}] if hasattr(old_config, 'checks_path'): converted['general'][0]['check_search_path'] = old_config.checks_path if hasattr(old_config, 'checks_path_recurse'): converted['general'][0][ 'check_search_recursive'] = old_config.checks_path_recurse if converted['general'] == [{}]: del converted['general'] contents = (f"#\n# This file was automatically generated " f"by ReFrame based on '{filename}'.\n#\n\n" f"site_configuration = {util.ppretty(converted)}\n") contents = '\n'.join(l if len(l) < 80 else f'{l} # noqa: E501' for l in contents.split('\n')) if newfilename: with open(newfilename, 'w') as fp: if newfilename.endswith('.json'): json.dump(converted, fp, indent=4) else: fp.write(contents) else: with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as fp: fp.write(contents) return fp.name
def test_load_abspath(self): filename = os.path.abspath('reframe/__init__.py') module = util.import_module_from_file(filename) self.assertEqual(reframe.VERSION, module.VERSION) self.assertEqual('reframe', module.__name__) self.assertIs(module, sys.modules.get('reframe'))