def test_delitem(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
# delete key
del scoped_dict['a:k1']
self.assertEqual(7, scoped_dict['a:k1'])
# delete a whole scope
del scoped_dict['*']
self.assertRaises(KeyError, exec, "scoped_dict[':k4']", globals(),
locals())
self.assertRaises(KeyError, exec, "scoped_dict['a:k3']", globals(),
locals())
# try to delete a non-existent key
self.assertRaises(KeyError, exec, "del scoped_dict['a:k4']", globals(),
locals())
def test_iter_items(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
expected_items = [('a:k1', 1), ('a:k2', 2), ('a:b:k1', 3),
('a:b:k3', 4), ('a:b:c:k2', 5), ('a:b:c:k3', 6),
('*:k1', 7), ('*:k3', 9), ('*:k4', 10)]
self.assertEqual(sorted(expected_items),
sorted(item for item in scoped_dict.items()))
def test_setitem(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
scoped_dict['a:k2'] = 20
scoped_dict['c:k2'] = 30
scoped_dict[':k4'] = 40
scoped_dict['*:k5'] = 50
scoped_dict['k6'] = 60
self.assertEqual(20, scoped_dict['a:k2'])
self.assertEqual(30, scoped_dict['c:k2'])
self.assertEqual(40, scoped_dict[':k4'])
self.assertEqual(50, scoped_dict['k5'])
self.assertEqual(60, scoped_dict['k6'])
def test_iter_keys(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
expected_keys = [
'a:k1', 'a:k2', 'a:b:k1', 'a:b:k3', 'a:b:c:k2', 'a:b:c:k3', '*:k1',
'*:k3', '*:k4'
]
self.assertEqual(sorted(expected_keys),
sorted(k for k in scoped_dict.keys()))
def test_contains(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
# Test simple lookup
self.assertIn('a:k1', scoped_dict)
self.assertIn('a:k2', scoped_dict)
self.assertIn('a:k3', scoped_dict)
self.assertIn('a:k4', scoped_dict)
self.assertIn('a:b:k1', scoped_dict)
self.assertIn('a:b:k2', scoped_dict)
self.assertIn('a:b:k3', scoped_dict)
self.assertIn('a:b:k4', scoped_dict)
self.assertIn('a:b:c:k1', scoped_dict)
self.assertIn('a:b:c:k1', scoped_dict)
self.assertIn('a:b:c:k1', scoped_dict)
self.assertIn('a:b:c:k1', scoped_dict)
# Test global scope
self.assertIn('k1', scoped_dict)
self.assertNotIn('k2', scoped_dict)
self.assertIn('k3', scoped_dict)
self.assertIn('k4', scoped_dict)
self.assertIn(':k1', scoped_dict)
self.assertNotIn(':k2', scoped_dict)
self.assertIn(':k3', scoped_dict)
self.assertIn(':k4', scoped_dict)
self.assertIn('*:k1', scoped_dict)
self.assertNotIn('*:k2', scoped_dict)
self.assertIn('*:k3', scoped_dict)
self.assertIn('*:k4', scoped_dict)
# Try to get full scopes as keys
self.assertNotIn('a', scoped_dict)
self.assertNotIn('a:b', scoped_dict)
self.assertNotIn('a:b:c', scoped_dict)
self.assertNotIn('a:b:c:d', scoped_dict)
self.assertNotIn('*', scoped_dict)
self.assertNotIn('', scoped_dict)
def test_update(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
scoped_dict_alt = fields.ScopedDict({'a': {'k1': 3, 'k2': 5}})
scoped_dict_alt.update({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
self.assertEqual(scoped_dict, scoped_dict_alt)
def test_scoped_dict_field(self):
class FieldTester:
field = fields.ScopedDictField('field', int)
field_maybe_none = fields.ScopedDictField('field_maybe_none',
int,
allow_none=True)
tester = FieldTester()
# Test valid assignments
tester.field = {
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
}
tester.field_maybe_none = None
# Check that we have indeed a ScopedDict here
self.assertIsInstance(FieldTester.field, fields.ScopedDictField)
self.assertIsInstance(tester.field, fields.ScopedDict)
self.assertEqual(10, tester.field['a:k4'])
# Test invalid assignments
self.assertRaises(TypeError, exec, 'tester.field = {1: "a", 2: "b" }',
globals(), locals())
self.assertRaises(TypeError, exec,
"tester.field = [('a', 1), ('b', 2)]", globals(),
locals())
self.assertRaises(
TypeError, exec, """tester.field = {'a': {1: 'k1'},
'b': {2: 'k2'}}""", globals(),
locals())
# Test assigning a ScopedDict already
tester.field = fields.ScopedDict({})
def test_construction(self):
d = {'a': {'k1': 3, 'k2': 4}, 'b': {'k3': 5}}
namespace_dict = fields.ScopedDict()
namespace_dict = fields.ScopedDict(d)
# Change local dict and verify that the stored values are not affected
d['a']['k1'] = 10
d['b']['k3'] = 10
self.assertEqual(3, namespace_dict['a:k1'])
self.assertEqual(5, namespace_dict['b:k3'])
del d['b']
self.assertIn('b:k3', namespace_dict)
self.assertRaises(TypeError, fields.ScopedDict, 1)
self.assertRaises(TypeError, fields.ScopedDict, {'a': 1, 'b': 2})
self.assertRaises(TypeError, fields.ScopedDict, [('a', 1), ('b', 2)])
self.assertRaises(TypeError, fields.ScopedDict, {
'a': {
1: 'k1'
},
'b': {
2: 'k2'
}
})
def test_iter_values(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
expected_values = [1, 2, 3, 4, 5, 6, 7, 9, 10]
self.assertEqual(expected_values,
sorted(v for v in scoped_dict.values()))
def load_from_dict(self, site_config):
if not isinstance(site_config, collections.abc.Mapping):
raise TypeError('site configuration is not a dict')
# We do all the necessary imports here and not on the top, because we
# want to remove import time dependencies
import reframe.core.environments as m_env
from reframe.core.systems import System, SystemPartition
sysconfig = site_config.get('systems', None)
envconfig = site_config.get('environments', None)
modes = site_config.get('modes', {})
if not sysconfig:
raise ValueError('no entry for systems was found')
if not envconfig:
raise ValueError('no entry for environments was found')
# Convert envconfig to a ScopedDict
try:
envconfig = fields.ScopedDict(envconfig)
except TypeError:
raise TypeError('environments configuration '
'is not a scoped dictionary') from None
# Convert modes to a `ScopedDict`; note that `modes` will implicitly
# converted to a scoped dict here, since `self._modes` is a
# `ScopedDictField`.
try:
self._modes = modes
except TypeError:
raise TypeError('modes configuration '
'is not a scoped dictionary') from None
def create_env(system, partition, name):
# Create an environment instance
try:
config = envconfig['%s:%s:%s' % (system, partition, name)]
except KeyError:
raise ConfigError("could not find a definition for `%s'" %
name) from None
if not isinstance(config, collections.abc.Mapping):
raise TypeError("config for `%s' is not a dictionary" % name)
try:
envtype = m_env.__dict__[config['type']]
return envtype(name, **config)
except KeyError:
raise ConfigError("no type specified for environment `%s'" %
name) from None
# Populate the systems directory
for sys_name, config in sysconfig.items():
if not isinstance(config, dict):
raise TypeError('system configuration is not a dictionary')
if not isinstance(config['partitions'], collections.abc.Mapping):
raise TypeError('partitions must be a dictionary')
sys_descr = config.get('descr', sys_name)
sys_hostnames = config.get('hostnames', [])
# The System's constructor provides also reasonable defaults, but
# since we are going to set them anyway from the values provided by
# the configuration, we should set default values here. The stage,
# output and log directories default to None, since they are going
# to be set dynamically by the runtime.
sys_prefix = config.get('prefix', '.')
sys_stagedir = config.get('stagedir', None)
sys_outputdir = config.get('outputdir', None)
sys_perflogdir = config.get('perflogdir', None)
sys_resourcesdir = config.get('resourcesdir', '.')
sys_modules_system = config.get('modules_system', None)
# Expand variables
if sys_prefix:
sys_prefix = os_ext.expandvars(sys_prefix)
if sys_stagedir:
sys_stagedir = os_ext.expandvars(sys_stagedir)
if sys_outputdir:
sys_outputdir = os_ext.expandvars(sys_outputdir)
if sys_perflogdir:
sys_perflogdir = os_ext.expandvars(sys_perflogdir)
if sys_resourcesdir:
sys_resourcesdir = os_ext.expandvars(sys_resourcesdir)
# Create the preload environment for the system
sys_preload_env = m_env.Environment(
name='__rfm_env_%s' % sys_name,
modules=config.get('modules', []),
variables=config.get('variables', {}))
system = System(name=sys_name,
descr=sys_descr,
hostnames=sys_hostnames,
preload_env=sys_preload_env,
prefix=sys_prefix,
stagedir=sys_stagedir,
outputdir=sys_outputdir,
perflogdir=sys_perflogdir,
resourcesdir=sys_resourcesdir,
modules_system=sys_modules_system)
for part_name, partconfig in config.get('partitions', {}).items():
if not isinstance(partconfig, collections.abc.Mapping):
raise TypeError("partition `%s' not configured "
"as a dictionary" % part_name)
part_descr = partconfig.get('descr', part_name)
part_scheduler, part_launcher = self.get_schedsystem_config(
partconfig.get('scheduler', 'local+local'))
part_local_env = m_env.Environment(
name='__rfm_env_%s' % part_name,
modules=partconfig.get('modules', []),
variables=partconfig.get('variables', {}).items())
part_environs = [
create_env(sys_name, part_name, e)
for e in partconfig.get('environs', [])
]
part_access = partconfig.get('access', [])
part_resources = partconfig.get('resources', {})
part_max_jobs = partconfig.get('max_jobs', 1)
system.add_partition(
SystemPartition(name=part_name,
descr=part_descr,
scheduler=part_scheduler,
launcher=part_launcher,
access=part_access,
environs=part_environs,
resources=part_resources,
local_env=part_local_env,
max_jobs=part_max_jobs))
self._systems[sys_name] = system
def test_key_resolution(self):
scoped_dict = fields.ScopedDict({
'a': {
'k1': 1,
'k2': 2
},
'a:b': {
'k1': 3,
'k3': 4
},
'a:b:c': {
'k2': 5,
'k3': 6
},
'*': {
'k1': 7,
'k3': 9,
'k4': 10
}
})
self.assertEqual(1, scoped_dict['a:k1'])
self.assertEqual(2, scoped_dict['a:k2'])
self.assertEqual(9, scoped_dict['a:k3'])
self.assertEqual(10, scoped_dict['a:k4'])
self.assertEqual(3, scoped_dict['a:b:k1'])
self.assertEqual(2, scoped_dict['a:b:k2'])
self.assertEqual(4, scoped_dict['a:b:k3'])
self.assertEqual(10, scoped_dict['a:b:k4'])
self.assertEqual(3, scoped_dict['a:b:c:k1'])
self.assertEqual(5, scoped_dict['a:b:c:k2'])
self.assertEqual(6, scoped_dict['a:b:c:k3'])
self.assertEqual(10, scoped_dict['a:b:c:k4'])
# Test global scope
self.assertEqual(7, scoped_dict['k1'])
self.assertRaises(KeyError, exec, "scoped_dict['k2']", globals(),
locals())
self.assertEqual(9, scoped_dict['k3'])
self.assertEqual(10, scoped_dict['k4'])
self.assertEqual(7, scoped_dict[':k1'])
self.assertRaises(KeyError, exec, "scoped_dict[':k2']", globals(),
locals())
self.assertEqual(9, scoped_dict[':k3'])
self.assertEqual(10, scoped_dict[':k4'])
self.assertEqual(7, scoped_dict['*:k1'])
self.assertRaises(KeyError, exec, "scoped_dict['*:k2']", globals(),
locals())
self.assertEqual(9, scoped_dict['*:k3'])
self.assertEqual(10, scoped_dict['*:k4'])
# Try to fool it, by requesting keys with scope names
self.assertRaises(KeyError, exec, "scoped_dict['a']", globals(),
locals())
self.assertRaises(KeyError, exec, "scoped_dict['a:b']", globals(),
locals())
self.assertRaises(KeyError, exec, "scoped_dict['a:b:c']", globals(),
locals())
self.assertRaises(KeyError, exec, "scoped_dict['a:b:c:d']", globals(),
locals())
self.assertRaises(KeyError, exec, "scoped_dict['*']", globals(),
locals())
self.assertRaises(KeyError, exec, "scoped_dict['']", globals(),
locals())
