def load_from_dict(self, site_config):
if not isinstance(site_config, collections.abc.Mapping):
raise TypeError('site configuration is not a dict')
sysconfig = site_config.get('systems', None)
envconfig = site_config.get('environments', None)
modes = site_config.get('modes', {})
if not sysconfig:
raise ValueError('no entry for systems was found')
if not envconfig:
raise ValueError('no entry for environments was found')
# Convert envconfig to a ScopedDict
try:
envconfig = ScopedDict(envconfig)
except TypeError:
raise TypeError('environments configuration '
'is not a scoped dictionary') from None
# Convert modes to a `ScopedDict`; note that `modes` will implicitly
# converted to a scoped dict here, since `self._models` is a
# `ScopedDictField`.
try:
self._modes = modes
except TypeError:
raise TypeError('modes configuration '
'is not a scoped dictionary') from None
def create_env(system, partition, name):
# Create an environment instance
try:
config = envconfig['%s:%s:%s' % (system, partition, name)]
except KeyError as e:
raise ConfigError("could not find a definition for `%s'" %
name) from None
if not isinstance(config, collections.abc.Mapping):
raise TypeError("config for `%s' is not a dictionary" % name)
try:
import reframe.core.environments as m_env
envtype = m_env.__dict__[config['type']]
return envtype(name, **config)
except KeyError:
raise ConfigError("no type specified for environment `%s'" %
name) from None
# Populate the systems directory
for sys_name, config in sysconfig.items():
if not isinstance(config, dict):
raise TypeError('system configuration is not a dictionary')
if not isinstance(config['partitions'], collections.abc.Mapping):
raise TypeError('partitions must be a dictionary')
sys_descr = config.get('descr', sys_name)
sys_hostnames = config.get('hostnames', [])
# The System's constructor provides also reasonable defaults, but
# since we are going to set them anyway from the values provided by
# the configuration, we should set default values here. The stage,
# output and log directories default to None, since they are
# going to be set dynamically by the ResourcesManager
sys_prefix = config.get('prefix', '.')
sys_stagedir = config.get('stagedir', None)
sys_outputdir = config.get('outputdir', None)
sys_logdir = config.get('logdir', None)
sys_resourcesdir = config.get('resourcesdir', '.')
sys_modules_system = config.get('modules_system', None)
# Expand variables
if sys_prefix:
sys_prefix = os.path.expandvars(sys_prefix)
if sys_stagedir:
sys_stagedir = os.path.expandvars(sys_stagedir)
if sys_outputdir:
sys_outputdir = os.path.expandvars(sys_outputdir)
if sys_logdir:
sys_logdir = os.path.expandvars(sys_logdir)
if sys_resourcesdir:
sys_resourcesdir = os.path.expandvars(sys_resourcesdir)
system = System(name=sys_name,
descr=sys_descr,
hostnames=sys_hostnames,
prefix=sys_prefix,
stagedir=sys_stagedir,
outputdir=sys_outputdir,
logdir=sys_logdir,
resourcesdir=sys_resourcesdir,
modules_system=sys_modules_system)
for part_name, partconfig in config.get('partitions', {}).items():
if not isinstance(partconfig, collections.abc.Mapping):
raise TypeError("partition `%s' not configured "
"as a dictionary" % part_name)
part_descr = partconfig.get('descr', part_name)
part_scheduler, part_launcher = self.get_schedsystem_config(
partconfig.get('scheduler', 'local+local'))
part_local_env = Environment(
name='__rfm_env_%s' % part_name,
modules=partconfig.get('modules', []),
variables=partconfig.get('variables', {}))
part_environs = [
create_env(sys_name, part_name, e)
for e in partconfig.get('environs', [])
]
part_access = partconfig.get('access', [])
part_resources = partconfig.get('resources', {})
part_max_jobs = partconfig.get('max_jobs', 1)
system.add_partition(
SystemPartition(name=part_name,
descr=part_descr,
scheduler=part_scheduler,
launcher=part_launcher,
access=part_access,
environs=part_environs,
resources=part_resources,
local_env=part_local_env,
max_jobs=part_max_jobs))
self._systems[sys_name] = system
def load_from_dict(self, site_config):
if not isinstance(site_config, collections.abc.Mapping):
raise TypeError('site configuration is not a dict')
# We do all the necessary imports here and not on the top, because we
# want to remove import time dependencies
import reframe.core.environments as m_env
from reframe.core.systems import System, SystemPartition
sysconfig = site_config.get('systems', None)
envconfig = site_config.get('environments', None)
modes = site_config.get('modes', {})
if not sysconfig:
raise ValueError('no entry for systems was found')
if not envconfig:
raise ValueError('no entry for environments was found')
# Convert envconfig to a ScopedDict
try:
envconfig = fields.ScopedDict(envconfig)
except TypeError:
raise TypeError('environments configuration '
'is not a scoped dictionary') from None
# Convert modes to a `ScopedDict`; note that `modes` will implicitly
# converted to a scoped dict here, since `self._modes` is a
# `ScopedDictField`.
try:
self._modes = modes
except TypeError:
raise TypeError('modes configuration '
'is not a scoped dictionary') from None
def create_env(system, partition, name):
# Create an environment instance
try:
config = envconfig['%s:%s:%s' % (system, partition, name)]
except KeyError:
raise ConfigError(
"could not find a definition for `%s'" % name
) from None
if not isinstance(config, collections.abc.Mapping):
raise TypeError("config for `%s' is not a dictionary" % name)
return m_env.ProgEnvironment(name, **config)
# Populate the systems directory
for sys_name, config in sysconfig.items():
if not isinstance(config, dict):
raise TypeError('system configuration is not a dictionary')
if not isinstance(config['partitions'], collections.abc.Mapping):
raise TypeError('partitions must be a dictionary')
sys_descr = config.get('descr', sys_name)
sys_hostnames = config.get('hostnames', [])
# The System's constructor provides also reasonable defaults, but
# since we are going to set them anyway from the values provided by
# the configuration, we should set default values here. The stage,
# output and log directories default to None, since they are going
# to be set dynamically by the runtime.
sys_prefix = config.get('prefix', '.')
sys_stagedir = config.get('stagedir', None)
sys_outputdir = config.get('outputdir', None)
sys_perflogdir = config.get('perflogdir', None)
sys_resourcesdir = config.get('resourcesdir', '.')
sys_modules_system = config.get('modules_system', None)
# Expand variables
if sys_prefix:
sys_prefix = os_ext.expandvars(sys_prefix)
if sys_stagedir:
sys_stagedir = os_ext.expandvars(sys_stagedir)
if sys_outputdir:
sys_outputdir = os_ext.expandvars(sys_outputdir)
if sys_perflogdir:
sys_perflogdir = os_ext.expandvars(sys_perflogdir)
if sys_resourcesdir:
sys_resourcesdir = os_ext.expandvars(sys_resourcesdir)
# Create the preload environment for the system
sys_preload_env = m_env.Environment(
name='__rfm_env_%s' % sys_name,
modules=config.get('modules', []),
variables=config.get('variables', {})
)
system = System(name=sys_name,
descr=sys_descr,
hostnames=sys_hostnames,
preload_env=sys_preload_env,
prefix=sys_prefix,
stagedir=sys_stagedir,
outputdir=sys_outputdir,
perflogdir=sys_perflogdir,
resourcesdir=sys_resourcesdir,
modules_system=sys_modules_system)
for part_name, partconfig in config.get('partitions', {}).items():
if not isinstance(partconfig, collections.abc.Mapping):
raise TypeError("partition `%s' not configured "
"as a dictionary" % part_name)
part_descr = partconfig.get('descr', part_name)
part_scheduler, part_launcher = self.get_schedsystem_config(
partconfig.get('scheduler', 'local+local')
)
part_local_env = m_env.Environment(
name='__rfm_env_%s' % part_name,
modules=partconfig.get('modules', []),
variables=partconfig.get('variables', {}).items()
)
part_environs = [
create_env(sys_name, part_name, e)
for e in partconfig.get('environs', [])
]
part_access = partconfig.get('access', [])
part_resources = partconfig.get('resources', {})
part_max_jobs = partconfig.get('max_jobs', 1)
part = SystemPartition(name=part_name,
descr=part_descr,
scheduler=part_scheduler,
launcher=part_launcher,
access=part_access,
environs=part_environs,
resources=part_resources,
local_env=part_local_env,
max_jobs=part_max_jobs)
container_platforms = partconfig.get('container_platforms', {})
for cp, env_spec in container_platforms.items():
cp_env = m_env.Environment(
name='__rfm_env_%s' % cp,
modules=env_spec.get('modules', []),
variables=env_spec.get('variables', {})
)
part.add_container_env(cp, cp_env)
system.add_partition(part)
self._systems[sys_name] = system
def __init__(self, site_config):
self._site_config = site_config
self._system = System.create(site_config)
self._current_run = 0
self._timestamp = datetime.now()
def test_system_create():
site_config = config.load_config('unittests/resources/settings.py')
site_config.select_subconfig('testsys:gpu')
system = System.create(site_config)
assert system.name == 'testsys'
assert system.descr == 'Fake system for unit tests'
assert system.hostnames == ['testsys']
assert system.modules_system.name == 'nomod'
assert system.preload_environ.modules == ['foo/1.0']
assert system.preload_environ.variables == {'FOO_CMD': 'foobar'}
assert system.prefix == '.rfm_testing'
assert system.stagedir == ''
assert system.outputdir == ''
assert system.resourcesdir == '.rfm_testing/resources'
assert len(system.partitions) == 1
partition = system.partitions[0]
assert partition.name == 'gpu'
assert partition.fullname == 'testsys:gpu'
assert partition.descr == 'GPU partition'
assert partition.scheduler.registered_name == 'slurm'
assert partition.launcher_type.registered_name == 'srun'
assert partition.access == []
assert partition.container_environs == {}
assert partition.local_env.modules == ['foogpu']
assert partition.local_env.modules_detailed == [{
'name': 'foogpu',
'collection': False,
'path': '/foo'
}]
assert partition.local_env.variables == {'FOO_GPU': 'yes'}
assert partition.max_jobs == 10
assert partition.time_limit is None
assert len(partition.environs) == 2
assert partition.environment('PrgEnv-gnu').cc == 'cc'
assert partition.environment('PrgEnv-gnu').cflags == []
assert partition.environment('PrgEnv-gnu').extras == {'foo': 2, 'bar': 'y'}
# Check resource instantiation
resource_spec = partition.get_resource('gpu', num_gpus_per_node=16)
assert resource_spec == ['--gres=gpu:16']
resources_spec = partition.get_resource('datawarp',
capacity='100GB',
stagein_src='/foo')
assert resources_spec == [
'#DW jobdw capacity=100GB', '#DW stage_in source=/foo'
]
# Check processor info
assert partition.processor.info is not None
assert partition.processor.topology is not None
assert partition.processor.arch == 'skylake'
assert partition.processor.num_cpus == 8
assert partition.processor.num_cpus_per_core == 2
assert partition.processor.num_cpus_per_socket == 8
assert partition.processor.num_sockets == 1
assert partition.processor.num_cores == 4
assert partition.processor.num_cores_per_socket == 4
assert partition.processor.num_numa_nodes == 1
assert partition.processor.num_cores_per_numa_node == 4