def test_pick_dep_version(self):
"""Test pick_dep_version function."""
self.assertEqual(pick_dep_version(None), None)
self.assertEqual(pick_dep_version('1.2.3'), '1.2.3')
dep_ver_dict = {
'arch=x86_64': '1.2.3-amd64',
'arch=POWER': '1.2.3-ppc64le',
}
st.get_cpu_architecture = lambda: X86_64
self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-amd64')
st.get_cpu_architecture = lambda: POWER
self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le')
error_pattern = "Unknown value type for version"
self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, ('1.2.3', '4.5.6'))
def test_pick_dep_version(self):
"""Test pick_dep_version function."""
self.assertEqual(pick_dep_version(None), None)
self.assertEqual(pick_dep_version('1.2.3'), '1.2.3')
dep_ver_dict = {
'arch=x86_64': '1.2.3-amd64',
'arch=POWER': '1.2.3-ppc64le',
}
st.get_cpu_architecture = lambda: X86_64
self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-amd64')
st.get_cpu_architecture = lambda: POWER
self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le')
error_pattern = "Unknown value type for version"
self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version,
('1.2.3', '4.5.6'))
# check support for using 'arch=*' as fallback key
dep_ver_dict = {
'arch=*': '1.2.3',
'arch=foo': '1.2.3-foo',
'arch=POWER': '1.2.3-ppc64le',
}
self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le')
del dep_ver_dict['arch=POWER']
self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3')
# check how faulty input is handled
self.assertErrorRegex(EasyBuildError, "Found empty dict as version!",
pick_dep_version, {})
error_pattern = r"Unexpected keys in version: bar,foo \(only 'arch=' keys are supported\)"
self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version,
{
'foo': '1.2',
'bar': '2.3'
})
error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value"
self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version,
1.23)
def template_constant_dict(config,
ignore=None,
skip_lower=None,
toolchain=None):
"""Create a dict for templating the values in the easyconfigs.
- config is a dict with the structure of EasyConfig._config
"""
if skip_lower is not None:
_log.deprecated(
"Use of 'skip_lower' named argument for template_constant_dict has no effect anymore",
'4.0')
# TODO find better name
# ignore
if ignore is None:
ignore = []
# make dict
template_values = {}
_log.debug("config: %s", config)
# set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
template_values['arch'] = platform.uname()[4]
# step 1: add TEMPLATE_NAMES_EASYCONFIG
for name in TEMPLATE_NAMES_EASYCONFIG:
if name in ignore:
continue
# check if this template name is already handled
if template_values.get(name[0]) is not None:
continue
if name[0].startswith('toolchain_'):
tc = config.get('toolchain')
if tc is not None:
template_values['toolchain_name'] = tc.get('name', None)
template_values['toolchain_version'] = tc.get('version', None)
# only go through this once
ignore.extend(['toolchain_name', 'toolchain_version'])
elif name[0].startswith('version_'):
# parse major and minor version numbers
version = config['version']
if version is not None:
_log.debug("version found in easyconfig is %s", version)
version = version.split('.')
try:
major = version[0]
template_values['version_major'] = major
minor = version[1]
template_values['version_minor'] = minor
template_values['version_major_minor'] = '.'.join(
[major, minor])
except IndexError:
# if there is no minor version, skip it
pass
# only go through this once
ignore.extend(
['version_major', 'version_minor', 'version_major_minor'])
elif name[0].endswith('letter'):
# parse first letters
if name[0].startswith('name'):
softname = config['name']
if softname is not None:
template_values['nameletter'] = softname[0]
else:
raise EasyBuildError(
"Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
# step 2: define *ver and *shortver templates
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
# copy to avoid changing original list below
deps = copy.copy(config.get('dependencies', []))
# also consider build dependencies for *ver and *shortver templates;
# we need to be a bit careful here, because for iterative installations
# (when multi_deps is used for example) the builddependencies value may be a list of lists
# first, determine if we have an EasyConfig instance
# (indirectly by checking for 'iterating' and 'iterate_options' attributes,
# because we can't import the EasyConfig class here without introducing
# a cyclic import...);
# we need to know to determine whether we're iterating over a list of build dependencies
is_easyconfig = hasattr(config, 'iterating') and hasattr(
config, 'iterate_options')
if is_easyconfig:
# if we're iterating over different lists of build dependencies,
# only consider build dependencies when we're actually in iterative mode!
if 'builddependencies' in config.iterate_options:
if config.iterating:
deps += config.get('builddependencies', [])
else:
deps += config.get('builddependencies', [])
for dep in deps:
if isinstance(dep, dict):
dep_name, dep_version = dep['name'], dep['version']
# take into account dependencies marked as external modules,
# where name/version may have to be harvested from metadata available for that external module
if dep.get('external_module', False):
metadata = dep.get('external_module_metadata', {})
if dep_name is None:
# name is a list in metadata, just take first value (if any)
dep_name = metadata.get('name', [None])[0]
if dep_version is None:
# version is a list in metadata, just take first value (if any)
dep_version = metadata.get('version', [None])[0]
elif isinstance(dep, (list, tuple)):
dep_name, dep_version = dep[0], dep[1]
else:
raise EasyBuildError("Unexpected type for dependency: %s", dep)
if isinstance(dep_name, string_type) and dep_name.lower(
) == name.lower() and dep_version:
dep_version = pick_dep_version(dep_version)
template_values['%sver' % pref] = dep_version
dep_version_parts = dep_version.split('.')
template_values['%smajver' % pref] = dep_version_parts[0]
if len(dep_version_parts) > 1:
template_values['%sminver' % pref] = dep_version_parts[1]
template_values['%sshortver' % pref] = '.'.join(
dep_version_parts[:2])
break
# step 3: add remaining from config
for name in TEMPLATE_NAMES_CONFIG:
if name in ignore:
continue
if name in config:
template_values[name] = config[name]
_log.debug('name: %s, config: %s', name, config[name])
# step 4. make lower variants
for name in TEMPLATE_NAMES_LOWER:
if name in ignore:
continue
value = config.get(name) or template_values.get(name)
if value is None:
continue
try:
template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % {
'name': name
}] = value.lower()
except Exception:
_log.warning(
"Failed to get .lower() for name %s value %s (type %s)", name,
value, type(value))
# step 5. add additional conditional templates
if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'):
try:
# get prefix for commands to be run with mpi runtime using default number of ranks
mpi_cmd_prefix = toolchain.mpi_cmd_prefix()
if mpi_cmd_prefix is not None:
template_values['mpi_cmd_prefix'] = mpi_cmd_prefix
except EasyBuildError as err:
# don't fail just because we couldn't resolve this template
_log.warning(
"Failed to create mpi_cmd_prefix template, error was:\n%s",
err)
return template_values
def template_constant_dict(config, ignore=None, skip_lower=None):
"""Create a dict for templating the values in the easyconfigs.
- config is a dict with the structure of EasyConfig._config
"""
if skip_lower is not None:
_log.deprecated(
"Use of 'skip_lower' named argument for template_constant_dict has no effect anymore",
'4.0')
# TODO find better name
# ignore
if ignore is None:
ignore = []
# make dict
template_values = {}
_log.debug("config: %s", config)
# set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
template_values['arch'] = platform.uname()[4]
# step 1: add TEMPLATE_NAMES_EASYCONFIG
for name in TEMPLATE_NAMES_EASYCONFIG:
if name in ignore:
continue
# check if this template name is already handled
if template_values.get(name[0]) is not None:
continue
if name[0].startswith('toolchain_'):
tc = config.get('toolchain')
if tc is not None:
template_values['toolchain_name'] = tc.get('name', None)
template_values['toolchain_version'] = tc.get('version', None)
# only go through this once
ignore.extend(['toolchain_name', 'toolchain_version'])
elif name[0].startswith('version_'):
# parse major and minor version numbers
version = config['version']
if version is not None:
_log.debug("version found in easyconfig is %s", version)
version = version.split('.')
try:
major = version[0]
template_values['version_major'] = major
minor = version[1]
template_values['version_minor'] = minor
template_values['version_major_minor'] = '.'.join(
[major, minor])
except IndexError:
# if there is no minor version, skip it
pass
# only go through this once
ignore.extend(
['version_major', 'version_minor', 'version_major_minor'])
elif name[0].endswith('letter'):
# parse first letters
if name[0].startswith('name'):
softname = config['name']
if softname is not None:
template_values['nameletter'] = softname[0]
else:
raise EasyBuildError(
"Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
# step 2: define *ver and *shortver templates
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
# copy to avoid changing original list below
deps = copy.copy(config.get('dependencies', []))
# only consider build dependencies for defining *ver and *shortver templates if we're in iterative mode
if hasattr(config, 'iterating') and config.iterating:
deps += config.get('builddependencies', [])
for dep in deps:
if isinstance(dep, dict):
dep_name, dep_version = dep['name'], dep['version']
elif isinstance(dep, (list, tuple)):
dep_name, dep_version = dep[0], dep[1]
else:
raise EasyBuildError("Unexpected type for dependency: %s", dep)
if isinstance(dep_name,
string_type) and dep_name.lower() == name.lower():
dep_version = pick_dep_version(dep_version)
template_values['%sver' % pref] = dep_version
dep_version_parts = dep_version.split('.')
template_values['%smajver' % pref] = dep_version_parts[0]
template_values['%sshortver' % pref] = '.'.join(
dep_version_parts[:2])
break
# step 3: add remaining from config
for name in TEMPLATE_NAMES_CONFIG:
if name in ignore:
continue
if name in config:
template_values[name] = config[name]
_log.debug('name: %s, config: %s', name, config[name])
# step 4. make lower variants
for name in TEMPLATE_NAMES_LOWER:
if name in ignore:
continue
value = config.get(name) or template_values.get(name)
if value is None:
continue
try:
template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % {
'name': name
}] = value.lower()
except Exception:
_log.warning(
"Failed to get .lower() for name %s value %s (type %s)", name,
value, type(value))
return template_values
def template_constant_dict(config,
ignore=None,
skip_lower=None,
toolchain=None):
"""Create a dict for templating the values in the easyconfigs.
- config is a dict with the structure of EasyConfig._config
"""
if skip_lower is not None:
_log.deprecated(
"Use of 'skip_lower' named argument for template_constant_dict has no effect anymore",
'4.0')
# TODO find better name
# ignore
if ignore is None:
ignore = []
# make dict
template_values = {}
_log.debug("config: %s", config)
# set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
template_values['arch'] = platform.uname()[4]
# step 1: add TEMPLATE_NAMES_EASYCONFIG
for name in TEMPLATE_NAMES_EASYCONFIG:
if name in ignore:
continue
# check if this template name is already handled
if template_values.get(name[0]) is not None:
continue
if name[0].startswith('toolchain_'):
tc = config.get('toolchain')
if tc is not None:
template_values['toolchain_name'] = tc.get('name', None)
template_values['toolchain_version'] = tc.get('version', None)
# only go through this once
ignore.extend(['toolchain_name', 'toolchain_version'])
elif name[0].startswith('version_'):
# parse major and minor version numbers
version = config['version']
if version is not None:
_log.debug("version found in easyconfig is %s", version)
version = version.split('.')
try:
major = version[0]
template_values['version_major'] = major
minor = version[1]
template_values['version_minor'] = minor
template_values['version_major_minor'] = '.'.join(
[major, minor])
except IndexError:
# if there is no minor version, skip it
pass
# only go through this once
ignore.extend(
['version_major', 'version_minor', 'version_major_minor'])
elif name[0].endswith('letter'):
# parse first letters
if name[0].startswith('name'):
softname = config['name']
if softname is not None:
template_values['nameletter'] = softname[0]
elif name[0] == 'module_name':
template_values['module_name'] = getattr(config, 'short_mod_name',
None)
else:
raise EasyBuildError(
"Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
# step 2: define *ver and *shortver templates
if TEMPLATE_SOFTWARE_VERSIONS:
name_to_prefix = dict(
(name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS)
deps = config.get('dependencies', [])
# also consider build dependencies for *ver and *shortver templates;
# we need to be a bit careful here, because for iterative installations
# (when multi_deps is used for example) the builddependencies value may be a list of lists
# first, determine if we have an EasyConfig instance
# (indirectly by checking for 'iterating' and 'iterate_options' attributes,
# because we can't import the EasyConfig class here without introducing
# a cyclic import...);
# we need to know to determine whether we're iterating over a list of build dependencies
is_easyconfig = hasattr(config, 'iterating') and hasattr(
config, 'iterate_options')
if is_easyconfig:
# if we're iterating over different lists of build dependencies,
# only consider build dependencies when we're actually in iterative mode!
if 'builddependencies' in config.iterate_options:
if config.iterating:
build_deps = config.get('builddependencies')
else:
build_deps = None
else:
build_deps = config.get('builddependencies')
if build_deps:
# Don't use += to avoid changing original list
deps = deps + build_deps
# include all toolchain deps (e.g. CUDAcore component in fosscuda);
# access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain!
if config._toolchain is not None and config._toolchain.tcdeps:
# If we didn't create a new list above do it here
if build_deps:
deps.extend(config._toolchain.tcdeps)
else:
deps = deps + config._toolchain.tcdeps
for dep in deps:
if isinstance(dep, dict):
dep_name, dep_version = dep['name'], dep['version']
# take into account dependencies marked as external modules,
# where name/version may have to be harvested from metadata available for that external module
if dep.get('external_module', False):
metadata = dep.get('external_module_metadata', {})
if dep_name is None:
# name is a list in metadata, just take first value (if any)
dep_name = metadata.get('name', [None])[0]
if dep_version is None:
# version is a list in metadata, just take first value (if any)
dep_version = metadata.get('version', [None])[0]
elif isinstance(dep, (list, tuple)):
dep_name, dep_version = dep[0], dep[1]
else:
raise EasyBuildError("Unexpected type for dependency: %s", dep)
if isinstance(dep_name, string_type) and dep_version:
pref = name_to_prefix.get(dep_name.lower())
if pref:
dep_version = pick_dep_version(dep_version)
template_values['%sver' % pref] = dep_version
dep_version_parts = dep_version.split('.')
template_values['%smajver' % pref] = dep_version_parts[0]
if len(dep_version_parts) > 1:
template_values['%sminver' %
pref] = dep_version_parts[1]
template_values['%sshortver' % pref] = '.'.join(
dep_version_parts[:2])
# step 3: add remaining from config
for name in TEMPLATE_NAMES_CONFIG:
if name in ignore:
continue
if name in config:
template_values[name] = config[name]
_log.debug('name: %s, config: %s', name, config[name])
# step 4. make lower variants
for name in TEMPLATE_NAMES_LOWER:
if name in ignore:
continue
value = config.get(name) or template_values.get(name)
if value is None:
continue
try:
template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % {
'name': name
}] = value.lower()
except Exception:
_log.warning(
"Failed to get .lower() for name %s value %s (type %s)", name,
value, type(value))
# keep track of names of defined templates until now,
# so we can check whether names of additional dynamic template values are all known
common_template_names = set(template_values.keys())
# step 5. add additional conditional templates
if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'):
try:
# get prefix for commands to be run with mpi runtime using default number of ranks
mpi_cmd_prefix = toolchain.mpi_cmd_prefix()
if mpi_cmd_prefix is not None:
template_values['mpi_cmd_prefix'] = mpi_cmd_prefix
except EasyBuildError as err:
# don't fail just because we couldn't resolve this template
_log.warning(
"Failed to create mpi_cmd_prefix template, error was:\n%s",
err)
# step 6. CUDA compute capabilities
# Use the commandline / easybuild config option if given, else use the value from the EC (as a default)
cuda_compute_capabilities = build_option(
'cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
if cuda_compute_capabilities:
template_values['cuda_compute_capabilities'] = ','.join(
cuda_compute_capabilities)
template_values['cuda_cc_space_sep'] = ' '.join(
cuda_compute_capabilities)
template_values['cuda_cc_semicolon_sep'] = ';'.join(
cuda_compute_capabilities)
sm_values = [
'sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities
]
template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
unknown_names = []
for key in template_values:
dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC)
if not (key in common_template_names or key in dynamic_template_names):
unknown_names.append(key)
if unknown_names:
raise EasyBuildError(
"One or more template values found with unknown name: %s",
','.join(unknown_names))
return template_values