def _create_from_python(cls, filename):
try:
mod = util.import_module_from_file(filename)
except ImportError as e:
# import_module_from_file() may raise an ImportError if the
# configuration file is under ReFrame's top-level directory
raise ConfigError(
f"could not load Python configuration file: '{filename}'"
) from e
if hasattr(mod, 'settings'):
# Looks like an old style config
raise ConfigError(
f"the syntax of the configuration file {filename!r} "
f"is no longer supported; please convert it using the "
f"'--upgrade-config-file' option"
)
mod = util.import_module_from_file(filename)
if not hasattr(mod, 'site_configuration'):
raise ConfigError(
f"not a valid Python configuration file: '{filename}'"
)
return _SiteConfig(mod.site_configuration, filename)
def test_load_unknown_path(self):
try:
util.import_module_from_file('/foo')
self.fail()
except ImportError as e:
self.assertEqual('foo', e.name)
self.assertEqual('/foo', e.path)
def test_load_unknown_path(self):
try:
util.import_module_from_file('/foo')
pytest.fail()
except ImportError as e:
assert 'foo' == e.name
assert '/foo' == e.path
def test_load_relative(self):
with os_ext.change_dir('reframe'):
# Load a module from a directory up
module = util.import_module_from_file('../reframe/__init__.py')
self.assertEqual(reframe.VERSION, module.VERSION)
self.assertEqual('reframe', module.__name__)
self.assertIs(module, sys.modules.get('reframe'))
# Load a module from the current directory
module = util.import_module_from_file('utility/os_ext.py')
self.assertEqual('reframe.utility.os_ext', module.__name__)
self.assertIs(module, sys.modules.get('reframe.utility.os_ext'))
def test_import_from_file_load_relative():
with osext.change_dir('reframe'):
# Load a module from a directory up
module = util.import_module_from_file('../reframe/__init__.py')
assert reframe.VERSION == module.VERSION
assert 'reframe' == module.__name__
assert module is sys.modules.get('reframe')
# Load a module from the current directory
module = util.import_module_from_file('utility/osext.py')
assert 'reframe.utility.osext' == module.__name__
assert module is sys.modules.get('reframe.utility.osext')
def load_from_file(self, filename, force=False):
if not self._validate_source(filename):
return []
return self.load_from_module(
util.import_module_from_file(filename, force)
)
def load_settings_from_file(filename):
global _settings
try:
_settings = util.import_module_from_file(filename).settings
return _settings
except Exception as e:
raise ConfigError("could not load configuration file `%s'" %
filename) from e
def test_import_from_file_existing_module_name(tmp_path):
test_file = tmp_path / 'os.py'
with open(test_file, 'w') as fp:
print('var = 1', file=fp)
module = util.import_module_from_file(test_file)
assert module.var == 1
assert not hasattr(module, 'path')
assert hasattr(os, 'path')
def test_load_outside_pkg(self):
module = util.import_module_from_file(os.path.__file__)
# os imports the OS-specific path libraries under the name `path`. Our
# importer will import the actual file, thus the module name should be
# the real one.
assert (module is sys.modules.get('posixpath')
or module is sys.modules.get('ntpath')
or module is sys.modules.get('macpath'))
def _create_from_python(cls, filename):
try:
mod = util.import_module_from_file(filename)
except ImportError as e:
# import_module_from_file() may raise an ImportError if the
# configuration file is under ReFrame's top-level directory
raise ConfigError(
f"could not load Python configuration file: '{filename}'"
) from e
if hasattr(mod, 'settings'):
# Looks like an old style config
raise ReframeDeprecationWarning(
f"the syntax of the configuration file '{filename}' "
f"is deprecated")
mod = util.import_module_from_file(filename)
if not hasattr(mod, 'site_configuration'):
raise ConfigError(
f"not a valid Python configuration file: '{filename}'")
return _SiteConfig(mod.site_configuration, filename)
def _object_hook(json):
filename = json.pop('__rfm_file__', None)
typename = json.pop('__rfm_class__', None)
if filename is None or typename is None:
return json
mod = util.import_module_from_file(filename)
cls = getattr(mod, typename)
obj = cls.__new__(cls)
obj.__dict__.update(json)
if hasattr(obj, '__rfm_json_decode__'):
obj.__rfm_json_decode__(json)
return obj
def load_from_file(self, filename, force=False):
if not self._validate_source(filename):
return []
try:
return self.load_from_module(
util.import_module_from_file(filename, force))
except Exception:
exc_info = sys.exc_info()
if not is_severe(*exc_info):
# Simply skip the file in this case
getlogger().warning(
f"skipping test file {filename!r}: {what(*exc_info)} "
f"(rerun with '-v' for more information)")
getlogger().verbose(traceback.format_exc())
return []
else:
raise
def test_load_directory(self):
module = util.import_module_from_file('reframe')
self.assertEqual(reframe.VERSION, module.VERSION)
self.assertEqual('reframe', module.__name__)
self.assertIs(module, sys.modules.get('reframe'))
def test_import_from_file_load_twice():
filename = os.path.abspath('reframe')
module1 = util.import_module_from_file(filename)
module2 = util.import_module_from_file(filename)
assert module1 is module2
def test_load_twice(self):
filename = os.path.abspath('reframe')
module1 = util.import_module_from_file(filename)
module2 = util.import_module_from_file(filename)
self.assertIs(module1, module2)
def test_import_from_file_load_directory_relative():
with osext.change_dir('reframe'):
module = util.import_module_from_file('../reframe')
assert reframe.VERSION == module.VERSION
assert 'reframe' == module.__name__
assert module is sys.modules.get('reframe')
def test_import_from_file_load_directory():
module = util.import_module_from_file('reframe')
assert reframe.VERSION == module.VERSION
assert 'reframe' == module.__name__
assert module is sys.modules.get('reframe')
def test_import_from_file_load_abspath():
filename = os.path.abspath('reframe/__init__.py')
module = util.import_module_from_file(filename)
assert reframe.VERSION == module.VERSION
assert 'reframe' == module.__name__
assert module is sys.modules.get('reframe')
def test_load_directory_relative(self):
with os_ext.change_dir('reframe'):
module = util.import_module_from_file('../reframe')
self.assertEqual(reframe.VERSION, module.VERSION)
self.assertEqual('reframe', module.__name__)
self.assertIs(module, sys.modules.get('reframe'))
def test_load_namespace_package(self):
module = util.import_module_from_file('unittests/resources')
self.assertIn('unittests', sys.modules)
self.assertIn('unittests.resources', sys.modules)
def test_import_from_file_load_namespace_package():
util.import_module_from_file('unittests/resources')
assert 'unittests' in sys.modules
assert 'unittests.resources' in sys.modules
import re
import semver
import os
import yaml
import reframe as rfm
import reframe.core.exceptions as error
import reframe.utility as util
import reframe.utility.osext as osext
import reframe.utility.sanity as sn
import reframe.utility.udeps as udeps
from reframe.core.exceptions import SanityError
spacklib = util.import_module_from_file(
os.path.join(os.path.dirname(__file__), 'src', 'spack_util',
'spacklib.py'))
spackconfig = util.import_module_from_file(
os.path.join(os.path.dirname(__file__), 'src', 'spack_util',
'spack_config.py'))
SPACK_VERSIONS = ['develop', '0.15.4', '0.16.1']
# TODO find a mechanism to include intel in the list
base_cuda_compilers = ['gcc', 'cce']
base_generic_compilers = ['gcc', 'cce', 'pgi']
cuda_compatible_compilers = spacklib.generate_compiler_list_for_testing(
[r'cdt-cuda/'], base_cuda_compilers)
generic_compilers = spacklib.generate_compiler_list_for_testing(
[r'cdt/'], base_generic_compilers)
def test_load_relpath(self):
module = util.import_module_from_file('reframe/__init__.py')
self.assertEqual(reframe.VERSION, module.VERSION)
self.assertEqual('reframe', module.__name__)
self.assertIs(module, sys.modules.get('reframe'))
def load_from_file(self, filename, **check_args):
if not self._validate_source(filename):
return []
return self.load_from_module(util.import_module_from_file(filename))
def test_import_from_file_load_relpath():
module = util.import_module_from_file('reframe/__init__.py')
assert reframe.VERSION == module.VERSION
assert 'reframe' == module.__name__
assert module is sys.modules.get('reframe')
def convert_old_config(filename, newfilename=None):
old_config = util.import_module_from_file(filename).settings
converted = {
'systems': [],
'environments': [],
'logging': [],
}
perflogdir = {}
old_systems = old_config.site_configuration['systems'].items()
for sys_name, sys_spec in old_systems:
sys_dict = {'name': sys_name}
system_perflogdir = sys_spec.pop('perflogdir', None)
perflogdir.setdefault(system_perflogdir, [])
perflogdir[system_perflogdir].append(sys_name)
sys_dict.update(sys_spec)
# hostnames is now a required property
if 'hostnames' not in sys_spec:
sys_dict['hostnames'] = []
# Make variables dictionary into a list of lists
if 'variables' in sys_spec:
sys_dict['variables'] = [
[vname, v] for vname, v in sys_dict['variables'].items()
]
# Make partitions dictionary into a list
if 'partitions' in sys_spec:
sys_dict['partitions'] = []
for pname, p in sys_spec['partitions'].items():
new_p = {'name': pname}
new_p.update(p)
if p['scheduler'] == 'nativeslurm':
new_p['scheduler'] = 'slurm'
new_p['launcher'] = 'srun'
elif p['scheduler'] == 'local':
new_p['scheduler'] = 'local'
new_p['launcher'] = 'local'
else:
sched, launcher, *_ = p['scheduler'].split('+')
new_p['scheduler'] = sched
new_p['launcher'] = launcher
# Make resources dictionary into a list
if 'resources' in p:
new_p['resources'] = [{
'name': rname,
'options': r
} for rname, r in p['resources'].items()]
# Make variables dictionary into a list of lists
if 'variables' in p:
new_p['variables'] = [
[vname, v] for vname, v in p['variables'].items()
]
if 'container_platforms' in p:
new_p['container_platforms'] = []
for cname, c in p['container_platforms'].items():
new_c = {'type': cname}
new_c.update(c)
if 'variables' in c:
new_c['variables'] = [
[vn, v] for vn, v in c['variables'].items()
]
new_p['container_platforms'].append(new_c)
sys_dict['partitions'].append(new_p)
converted['systems'].append(sys_dict)
old_environs = old_config.site_configuration['environments'].items()
for env_target, env_entries in old_environs:
for ename, e in env_entries.items():
new_env = {'name': ename}
if env_target != '*':
new_env['target_systems'] = [env_target]
new_env.update(e)
# Convert variables dictionary to a list of lists
if 'variables' in e:
new_env['variables'] = [[vname, v]
for vname, v in e['variables'].items()]
# Type attribute is not used anymore
if 'type' in new_env:
del new_env['type']
converted['environments'].append(new_env)
if 'modes' in old_config.site_configuration:
converted['modes'] = []
old_modes = old_config.site_configuration['modes'].items()
for target_mode, mode_entries in old_modes:
for mname, m in mode_entries.items():
new_mode = {'name': mname, 'options': m}
if target_mode != '*':
new_mode['target_systems'] = [target_mode]
converted['modes'].append(new_mode)
def handler_list(handler_config, basedir=None):
ret = []
for h in handler_config:
new_h = h.copy()
new_h['level'] = h['level'].lower()
if h['type'] == 'graylog':
# `host` and `port` attribute are converted to `address`
new_h['address'] = h['host']
if 'port' in h:
new_h['address'] += ':' + h['port']
elif h['type'] == 'filelog' and basedir is not None:
new_h['basedir'] = basedir
ret.append(new_h)
return ret
for basedir, target_systems in perflogdir.items():
converted['logging'].append({
'level':
old_config.logging_config['level'].lower(),
'handlers':
handler_list(old_config.logging_config['handlers']),
'handlers_perflog':
handler_list(old_config.perf_logging_config['handlers'],
basedir=basedir),
'target_systems':
target_systems
})
if basedir is None:
del converted['logging'][-1]['target_systems']
converted['general'] = [{}]
if hasattr(old_config, 'checks_path'):
converted['general'][0]['check_search_path'] = old_config.checks_path
if hasattr(old_config, 'checks_path_recurse'):
converted['general'][0][
'check_search_recursive'] = old_config.checks_path_recurse
if converted['general'] == [{}]:
del converted['general']
contents = (f"#\n# This file was automatically generated "
f"by ReFrame based on '{filename}'.\n#\n\n"
f"site_configuration = {util.ppretty(converted)}\n")
contents = '\n'.join(l if len(l) < 80 else f'{l} # noqa: E501'
for l in contents.split('\n'))
if newfilename:
with open(newfilename, 'w') as fp:
if newfilename.endswith('.json'):
json.dump(converted, fp, indent=4)
else:
fp.write(contents)
else:
with tempfile.NamedTemporaryFile(mode='w', suffix='.py',
delete=False) as fp:
fp.write(contents)
return fp.name
def test_load_abspath(self):
filename = os.path.abspath('reframe/__init__.py')
module = util.import_module_from_file(filename)
self.assertEqual(reframe.VERSION, module.VERSION)
self.assertEqual('reframe', module.__name__)
self.assertIs(module, sys.modules.get('reframe'))
