def test_load_overlapping(self):
e0 = renv.Environment(name='e0', variables=[('a', '1'), ('b', '2')])
e1 = renv.Environment(name='e1', variables=[('b', '3'), ('c', '4')])
e0.load()
e1.load()
self.assertFalse(e0.is_loaded)
self.assertTrue(e1.is_loaded)
def test_env_conflict(base_environ, user_runtime, modules_system):
env0 = env.Environment('env0', ['testmod_foo', 'testmod_boo'])
env1 = env.Environment('env1', ['testmod_bar'])
rt.loadenv(env0, env1)
for m in env1.modules:
assert modules_system.is_module_loaded(m)
for m in env0.modules:
assert not modules_system.is_module_loaded(m)
def test_env_not_equal(base_environ):
env1 = env.Environment('env1', modules=['foo', 'bar'])
env2 = env.Environment('env2', modules=['foo', 'bar'])
assert env1 != env2
# Variables are ordered, because they might depend on each other
env1 = env.Environment('env1', variables=[('a', 1), ('b', 2)])
env2 = env.Environment('env1', variables=[('b', 2), ('a', 1)])
assert env1 != env2
def test_not_equal(self):
env1 = renv.Environment('env1', modules=['foo', 'bar'])
env2 = renv.Environment('env2', modules=['foo', 'bar'])
self.assertNotEqual(env1, env2)
# Variables are ordered, because they might depend on each other
env1 = renv.Environment('env1', variables=[('a', 1), ('b', 2)])
env1 = renv.Environment('env1', variables=[('b', 2), ('a', 1)])
self.assertNotEqual(env1, env2)
def test_conflicting_environments(self):
self.setup_modules_system()
envfoo = env.Environment(name='envfoo',
modules=['testmod_foo', 'testmod_boo'])
envbar = env.Environment(name='envbar', modules=['testmod_bar'])
env.load(envfoo, envbar)
for m in envbar.modules:
assert self.modules_system.is_module_loaded(m)
for m in envfoo.modules:
assert not self.modules_system.is_module_loaded(m)
def test_conflicting_environments(self):
envfoo = renv.Environment(name='envfoo',
modules=['testmod_foo', 'testmod_boo'])
envbar = renv.Environment(name='envbar', modules=['testmod_bar'])
envfoo.load()
envbar.load()
for m in envbar.modules:
self.assertTrue(get_modules_system().is_module_loaded(m))
for m in envfoo.modules:
self.assertFalse(get_modules_system().is_module_loaded(m))
def setUp(self):
self.modules_system = None
os.environ['_var0'] = 'val0'
os.environ['_var1'] = 'val1'
self.environ_save = env.snapshot()
self.environ = env.Environment(name='TestEnv1',
modules=['testmod_foo'],
variables=[('_var0', 'val1'),
('_var2', '$_var0'),
('_var3', '${_var1}')])
self.environ_other = env.Environment(name='TestEnv2',
modules=['testmod_boo'],
variables={'_var4': 'val4'})
def test_conflict_environ_after_module_force_load(self):
self.setup_modules_system()
self.modules_system.load_module('testmod_foo')
envbar = env.Environment(name='envbar', modules=['testmod_bar'])
snapshot, _ = env.load(envbar)
snapshot.restore()
assert self.modules_system.is_module_loaded('testmod_foo')
def test_env_conflict_after_module_load_force(base_environ, user_runtime,
modules_system):
modules_system.load_module('testmod_foo')
env0 = env.Environment(name='env0', modules=['testmod_bar'])
snapshot, _ = rt.loadenv(env0)
snapshot.restore()
assert modules_system.is_module_loaded('testmod_foo')
def test_conflict_environ_after_module_force_load(self):
self.setup_modules_system()
self.modules_system.load_module('testmod_foo')
envbar = renv.Environment(name='envbar', modules=['testmod_bar'])
envbar.load()
envbar.unload()
self.assertTrue(self.modules_system.is_module_loaded('testmod_foo'))
def env0():
return env.Environment('TestEnv1', ['testmod_foo'],
[('_var0', 'val1'), ('_var2', '$_var0'),
('_var3', '${_var1}')], {
'foo': 1,
'bar': 2
})
def test_emit_loadenv_failure(user_runtime):
snap = rt.snapshot()
environ = env.Environment('test', modules=['testmod_foo', 'testmod_xxx'])
# Suppress the module load error and verify that the original environment
# is preserved
with contextlib.suppress(EnvironError):
rt.emit_loadenv_commands(environ)
assert rt.snapshot() == snap
def setUp(self):
get_modules_system().searchpath_add(fixtures.TEST_MODULES)
# Always add a base module; this is a workaround for the modules
# environment's inconsistent behaviour, that starts with an empty
# LOADEDMODULES variable and ends up removing it completely if all
# present modules are removed.
get_modules_system().load_module('testmod_base')
os.environ['_fookey1'] = 'origfoo'
os.environ['_fookey1b'] = 'foovalue1'
os.environ['_fookey2b'] = 'foovalue2'
self.environ_save = renv.EnvironmentSnapshot()
self.environ = renv.Environment(name='TestEnv1',
modules=['testmod_foo'])
self.environ.set_variable(name='_fookey1', value='value1')
self.environ.set_variable(name='_fookey2', value='value2')
self.environ.set_variable(name='_fookey1', value='value3')
self.environ.set_variable(name='_fookey3b', value='$_fookey1b')
self.environ.set_variable(name='_fookey4b', value='${_fookey2b}')
self.environ_other = renv.Environment(name='TestEnv2',
modules=['testmod_boo'])
self.environ_other.set_variable(name='_fookey11', value='value11')
def test_emit_loadenv_commands_mapping_with_conflict(base_environ,
user_runtime,
modules_system):
if modules_system.name == 'tmod4':
pytest.skip('test scenario not valid for tmod4')
e0 = env.Environment(name='e0', modules=['testmod_ext'])
ms = rt.runtime().modules_system
ms.load_mapping('testmod_ext: testmod_ext testmod_foo')
expected_commands = [
ms.emit_load_commands('testmod_ext')[0],
ms.emit_unload_commands('testmod_bar')[0],
ms.emit_load_commands('testmod_foo')[0],
]
assert expected_commands == rt.emit_loadenv_commands(e0)
def test_conflict_environ_after_module_load(self):
get_modules_system().load_module('testmod_foo')
envfoo = renv.Environment(name='envfoo', modules=['testmod_foo'])
envfoo.load()
envfoo.unload()
self.assertTrue(get_modules_system().is_module_loaded('testmod_foo'))
def test_load_non_overlapping(self):
e0 = env.Environment(name='e0', variables=[('a', '1'), ('b', '2')])
e1 = env.Environment(name='e1', variables=[('c', '3'), ('d', '4')])
env.load(e0, e1)
assert e0.is_loaded
assert e1.is_loaded
def test_not_equal(self):
env1 = renv.Environment('env1', modules=['foo', 'bar'])
env2 = renv.Environment('env2', modules=['foo', 'bar'])
self.assertNotEqual(env1, env2)
def load_from_dict(self, site_config):
if not isinstance(site_config, collections.abc.Mapping):
raise TypeError('site configuration is not a dict')
# We do all the necessary imports here and not on the top, because we
# want to remove import time dependencies
import reframe.core.environments as m_env
from reframe.core.systems import System, SystemPartition
sysconfig = site_config.get('systems', None)
envconfig = site_config.get('environments', None)
modes = site_config.get('modes', {})
if not sysconfig:
raise ValueError('no entry for systems was found')
if not envconfig:
raise ValueError('no entry for environments was found')
# Convert envconfig to a ScopedDict
try:
envconfig = fields.ScopedDict(envconfig)
except TypeError:
raise TypeError('environments configuration '
'is not a scoped dictionary') from None
# Convert modes to a `ScopedDict`; note that `modes` will implicitly
# converted to a scoped dict here, since `self._modes` is a
# `ScopedDictField`.
try:
self._modes = modes
except TypeError:
raise TypeError('modes configuration '
'is not a scoped dictionary') from None
def create_env(system, partition, name):
# Create an environment instance
try:
config = envconfig['%s:%s:%s' % (system, partition, name)]
except KeyError:
raise ConfigError("could not find a definition for `%s'" %
name) from None
if not isinstance(config, collections.abc.Mapping):
raise TypeError("config for `%s' is not a dictionary" % name)
try:
envtype = m_env.__dict__[config['type']]
return envtype(name, **config)
except KeyError:
raise ConfigError("no type specified for environment `%s'" %
name) from None
# Populate the systems directory
for sys_name, config in sysconfig.items():
if not isinstance(config, dict):
raise TypeError('system configuration is not a dictionary')
if not isinstance(config['partitions'], collections.abc.Mapping):
raise TypeError('partitions must be a dictionary')
sys_descr = config.get('descr', sys_name)
sys_hostnames = config.get('hostnames', [])
# The System's constructor provides also reasonable defaults, but
# since we are going to set them anyway from the values provided by
# the configuration, we should set default values here. The stage,
# output and log directories default to None, since they are going
# to be set dynamically by the runtime.
sys_prefix = config.get('prefix', '.')
sys_stagedir = config.get('stagedir', None)
sys_outputdir = config.get('outputdir', None)
sys_perflogdir = config.get('perflogdir', None)
sys_resourcesdir = config.get('resourcesdir', '.')
sys_modules_system = config.get('modules_system', None)
# Expand variables
if sys_prefix:
sys_prefix = os_ext.expandvars(sys_prefix)
if sys_stagedir:
sys_stagedir = os_ext.expandvars(sys_stagedir)
if sys_outputdir:
sys_outputdir = os_ext.expandvars(sys_outputdir)
if sys_perflogdir:
sys_perflogdir = os_ext.expandvars(sys_perflogdir)
if sys_resourcesdir:
sys_resourcesdir = os_ext.expandvars(sys_resourcesdir)
# Create the preload environment for the system
sys_preload_env = m_env.Environment(
name='__rfm_env_%s' % sys_name,
modules=config.get('modules', []),
variables=config.get('variables', {}))
system = System(name=sys_name,
descr=sys_descr,
hostnames=sys_hostnames,
preload_env=sys_preload_env,
prefix=sys_prefix,
stagedir=sys_stagedir,
outputdir=sys_outputdir,
perflogdir=sys_perflogdir,
resourcesdir=sys_resourcesdir,
modules_system=sys_modules_system)
for part_name, partconfig in config.get('partitions', {}).items():
if not isinstance(partconfig, collections.abc.Mapping):
raise TypeError("partition `%s' not configured "
"as a dictionary" % part_name)
part_descr = partconfig.get('descr', part_name)
part_scheduler, part_launcher = self.get_schedsystem_config(
partconfig.get('scheduler', 'local+local'))
part_local_env = m_env.Environment(
name='__rfm_env_%s' % part_name,
modules=partconfig.get('modules', []),
variables=partconfig.get('variables', {}).items())
part_environs = [
create_env(sys_name, part_name, e)
for e in partconfig.get('environs', [])
]
part_access = partconfig.get('access', [])
part_resources = partconfig.get('resources', {})
part_max_jobs = partconfig.get('max_jobs', 1)
system.add_partition(
SystemPartition(name=part_name,
descr=part_descr,
scheduler=part_scheduler,
launcher=part_launcher,
access=part_access,
environs=part_environs,
resources=part_resources,
local_env=part_local_env,
max_jobs=part_max_jobs))
self._systems[sys_name] = system
def env1():
return env.Environment('TestEnv2', ['testmod_boo'], {'_var4': 'val4'})
def test_load_overlapping(base_environ):
e0 = env.Environment(name='e0', variables=[('a', '1'), ('b', '2')])
e1 = env.Environment(name='e1', variables=[('b', '3'), ('c', '4')])
rt.loadenv(e0, e1)
assert not rt.is_env_loaded(e0)
assert rt.is_env_loaded(e1)
def test_env_equal(base_environ):
env1 = env.Environment('env1', modules=['foo', 'bar'])
env2 = env.Environment('env1', modules=['bar', 'foo'])
assert env1 == env2
assert env2 == env1
