def test_pick_dep_version(self): """Test pick_dep_version function.""" self.assertEqual(pick_dep_version(None), None) self.assertEqual(pick_dep_version('1.2.3'), '1.2.3') dep_ver_dict = { 'arch=x86_64': '1.2.3-amd64', 'arch=POWER': '1.2.3-ppc64le', } st.get_cpu_architecture = lambda: X86_64 self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-amd64') st.get_cpu_architecture = lambda: POWER self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le') error_pattern = "Unknown value type for version" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, ('1.2.3', '4.5.6'))
def test_pick_dep_version(self): """Test pick_dep_version function.""" self.assertEqual(pick_dep_version(None), None) self.assertEqual(pick_dep_version('1.2.3'), '1.2.3') dep_ver_dict = { 'arch=x86_64': '1.2.3-amd64', 'arch=POWER': '1.2.3-ppc64le', } st.get_cpu_architecture = lambda: X86_64 self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-amd64') st.get_cpu_architecture = lambda: POWER self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le') error_pattern = "Unknown value type for version" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, ('1.2.3', '4.5.6')) # check support for using 'arch=*' as fallback key dep_ver_dict = { 'arch=*': '1.2.3', 'arch=foo': '1.2.3-foo', 'arch=POWER': '1.2.3-ppc64le', } self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le') del dep_ver_dict['arch=POWER'] self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3') # check how faulty input is handled self.assertErrorRegex(EasyBuildError, "Found empty dict as version!", pick_dep_version, {}) error_pattern = r"Unexpected keys in version: bar,foo \(only 'arch=' keys are supported\)" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, { 'foo': '1.2', 'bar': '2.3' }) error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value" self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, 1.23)
def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None): """Create a dict for templating the values in the easyconfigs. - config is a dict with the structure of EasyConfig._config """ if skip_lower is not None: _log.deprecated( "Use of 'skip_lower' named argument for template_constant_dict has no effect anymore", '4.0') # TODO find better name # ignore if ignore is None: ignore = [] # make dict template_values = {} _log.debug("config: %s", config) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: continue # check if this template name is already handled if template_values.get(name[0]) is not None: continue if name[0].startswith('toolchain_'): tc = config.get('toolchain') if tc is not None: template_values['toolchain_name'] = tc.get('name', None) template_values['toolchain_version'] = tc.get('version', None) # only go through this once ignore.extend(['toolchain_name', 'toolchain_version']) elif name[0].startswith('version_'): # parse major and minor version numbers version = config['version'] if version is not None: _log.debug("version found in easyconfig is %s", version) version = version.split('.') try: major = version[0] template_values['version_major'] = major minor = version[1] template_values['version_minor'] = minor template_values['version_major_minor'] = '.'.join( [major, minor]) except IndexError: # if there is no minor version, skip it pass # only go through this once ignore.extend( ['version_major', 'version_minor', 'version_major_minor']) elif name[0].endswith('letter'): # parse first letters if name[0].startswith('name'): softname = config['name'] if softname is not None: template_values['nameletter'] = softname[0] else: raise EasyBuildError( "Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: define *ver and *shortver templates for name, pref in TEMPLATE_SOFTWARE_VERSIONS: # copy to avoid changing original list below deps = copy.copy(config.get('dependencies', [])) # also consider build dependencies for *ver and *shortver templates; # we need to be a bit careful here, because for iterative installations # (when multi_deps is used for example) the builddependencies value may be a list of lists # first, determine if we have an EasyConfig instance # (indirectly by checking for 'iterating' and 'iterate_options' attributes, # because we can't import the EasyConfig class here without introducing # a cyclic import...); # we need to know to determine whether we're iterating over a list of build dependencies is_easyconfig = hasattr(config, 'iterating') and hasattr( config, 'iterate_options') if is_easyconfig: # if we're iterating over different lists of build dependencies, # only consider build dependencies when we're actually in iterative mode! if 'builddependencies' in config.iterate_options: if config.iterating: deps += config.get('builddependencies', []) else: deps += config.get('builddependencies', []) for dep in deps: if isinstance(dep, dict): dep_name, dep_version = dep['name'], dep['version'] # take into account dependencies marked as external modules, # where name/version may have to be harvested from metadata available for that external module if dep.get('external_module', False): metadata = dep.get('external_module_metadata', {}) if dep_name is None: # name is a list in metadata, just take first value (if any) dep_name = metadata.get('name', [None])[0] if dep_version is None: # version is a list in metadata, just take first value (if any) dep_version = metadata.get('version', [None])[0] elif isinstance(dep, (list, tuple)): dep_name, dep_version = dep[0], dep[1] else: raise EasyBuildError("Unexpected type for dependency: %s", dep) if isinstance(dep_name, string_type) and dep_name.lower( ) == name.lower() and dep_version: dep_version = pick_dep_version(dep_version) template_values['%sver' % pref] = dep_version dep_version_parts = dep_version.split('.') template_values['%smajver' % pref] = dep_version_parts[0] if len(dep_version_parts) > 1: template_values['%sminver' % pref] = dep_version_parts[1] template_values['%sshortver' % pref] = '.'.join( dep_version_parts[:2]) break # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: if name in ignore: continue if name in config: template_values[name] = config[name] _log.debug('name: %s, config: %s', name, config[name]) # step 4. make lower variants for name in TEMPLATE_NAMES_LOWER: if name in ignore: continue value = config.get(name) or template_values.get(name) if value is None: continue try: template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % { 'name': name }] = value.lower() except Exception: _log.warning( "Failed to get .lower() for name %s value %s (type %s)", name, value, type(value)) # step 5. add additional conditional templates if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'): try: # get prefix for commands to be run with mpi runtime using default number of ranks mpi_cmd_prefix = toolchain.mpi_cmd_prefix() if mpi_cmd_prefix is not None: template_values['mpi_cmd_prefix'] = mpi_cmd_prefix except EasyBuildError as err: # don't fail just because we couldn't resolve this template _log.warning( "Failed to create mpi_cmd_prefix template, error was:\n%s", err) return template_values
def template_constant_dict(config, ignore=None, skip_lower=None): """Create a dict for templating the values in the easyconfigs. - config is a dict with the structure of EasyConfig._config """ if skip_lower is not None: _log.deprecated( "Use of 'skip_lower' named argument for template_constant_dict has no effect anymore", '4.0') # TODO find better name # ignore if ignore is None: ignore = [] # make dict template_values = {} _log.debug("config: %s", config) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: continue # check if this template name is already handled if template_values.get(name[0]) is not None: continue if name[0].startswith('toolchain_'): tc = config.get('toolchain') if tc is not None: template_values['toolchain_name'] = tc.get('name', None) template_values['toolchain_version'] = tc.get('version', None) # only go through this once ignore.extend(['toolchain_name', 'toolchain_version']) elif name[0].startswith('version_'): # parse major and minor version numbers version = config['version'] if version is not None: _log.debug("version found in easyconfig is %s", version) version = version.split('.') try: major = version[0] template_values['version_major'] = major minor = version[1] template_values['version_minor'] = minor template_values['version_major_minor'] = '.'.join( [major, minor]) except IndexError: # if there is no minor version, skip it pass # only go through this once ignore.extend( ['version_major', 'version_minor', 'version_major_minor']) elif name[0].endswith('letter'): # parse first letters if name[0].startswith('name'): softname = config['name'] if softname is not None: template_values['nameletter'] = softname[0] else: raise EasyBuildError( "Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: define *ver and *shortver templates for name, pref in TEMPLATE_SOFTWARE_VERSIONS: # copy to avoid changing original list below deps = copy.copy(config.get('dependencies', [])) # only consider build dependencies for defining *ver and *shortver templates if we're in iterative mode if hasattr(config, 'iterating') and config.iterating: deps += config.get('builddependencies', []) for dep in deps: if isinstance(dep, dict): dep_name, dep_version = dep['name'], dep['version'] elif isinstance(dep, (list, tuple)): dep_name, dep_version = dep[0], dep[1] else: raise EasyBuildError("Unexpected type for dependency: %s", dep) if isinstance(dep_name, string_type) and dep_name.lower() == name.lower(): dep_version = pick_dep_version(dep_version) template_values['%sver' % pref] = dep_version dep_version_parts = dep_version.split('.') template_values['%smajver' % pref] = dep_version_parts[0] template_values['%sshortver' % pref] = '.'.join( dep_version_parts[:2]) break # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: if name in ignore: continue if name in config: template_values[name] = config[name] _log.debug('name: %s, config: %s', name, config[name]) # step 4. make lower variants for name in TEMPLATE_NAMES_LOWER: if name in ignore: continue value = config.get(name) or template_values.get(name) if value is None: continue try: template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % { 'name': name }] = value.lower() except Exception: _log.warning( "Failed to get .lower() for name %s value %s (type %s)", name, value, type(value)) return template_values
def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None): """Create a dict for templating the values in the easyconfigs. - config is a dict with the structure of EasyConfig._config """ if skip_lower is not None: _log.deprecated( "Use of 'skip_lower' named argument for template_constant_dict has no effect anymore", '4.0') # TODO find better name # ignore if ignore is None: ignore = [] # make dict template_values = {} _log.debug("config: %s", config) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: continue # check if this template name is already handled if template_values.get(name[0]) is not None: continue if name[0].startswith('toolchain_'): tc = config.get('toolchain') if tc is not None: template_values['toolchain_name'] = tc.get('name', None) template_values['toolchain_version'] = tc.get('version', None) # only go through this once ignore.extend(['toolchain_name', 'toolchain_version']) elif name[0].startswith('version_'): # parse major and minor version numbers version = config['version'] if version is not None: _log.debug("version found in easyconfig is %s", version) version = version.split('.') try: major = version[0] template_values['version_major'] = major minor = version[1] template_values['version_minor'] = minor template_values['version_major_minor'] = '.'.join( [major, minor]) except IndexError: # if there is no minor version, skip it pass # only go through this once ignore.extend( ['version_major', 'version_minor', 'version_major_minor']) elif name[0].endswith('letter'): # parse first letters if name[0].startswith('name'): softname = config['name'] if softname is not None: template_values['nameletter'] = softname[0] elif name[0] == 'module_name': template_values['module_name'] = getattr(config, 'short_mod_name', None) else: raise EasyBuildError( "Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: define *ver and *shortver templates if TEMPLATE_SOFTWARE_VERSIONS: name_to_prefix = dict( (name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS) deps = config.get('dependencies', []) # also consider build dependencies for *ver and *shortver templates; # we need to be a bit careful here, because for iterative installations # (when multi_deps is used for example) the builddependencies value may be a list of lists # first, determine if we have an EasyConfig instance # (indirectly by checking for 'iterating' and 'iterate_options' attributes, # because we can't import the EasyConfig class here without introducing # a cyclic import...); # we need to know to determine whether we're iterating over a list of build dependencies is_easyconfig = hasattr(config, 'iterating') and hasattr( config, 'iterate_options') if is_easyconfig: # if we're iterating over different lists of build dependencies, # only consider build dependencies when we're actually in iterative mode! if 'builddependencies' in config.iterate_options: if config.iterating: build_deps = config.get('builddependencies') else: build_deps = None else: build_deps = config.get('builddependencies') if build_deps: # Don't use += to avoid changing original list deps = deps + build_deps # include all toolchain deps (e.g. CUDAcore component in fosscuda); # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! if config._toolchain is not None and config._toolchain.tcdeps: # If we didn't create a new list above do it here if build_deps: deps.extend(config._toolchain.tcdeps) else: deps = deps + config._toolchain.tcdeps for dep in deps: if isinstance(dep, dict): dep_name, dep_version = dep['name'], dep['version'] # take into account dependencies marked as external modules, # where name/version may have to be harvested from metadata available for that external module if dep.get('external_module', False): metadata = dep.get('external_module_metadata', {}) if dep_name is None: # name is a list in metadata, just take first value (if any) dep_name = metadata.get('name', [None])[0] if dep_version is None: # version is a list in metadata, just take first value (if any) dep_version = metadata.get('version', [None])[0] elif isinstance(dep, (list, tuple)): dep_name, dep_version = dep[0], dep[1] else: raise EasyBuildError("Unexpected type for dependency: %s", dep) if isinstance(dep_name, string_type) and dep_version: pref = name_to_prefix.get(dep_name.lower()) if pref: dep_version = pick_dep_version(dep_version) template_values['%sver' % pref] = dep_version dep_version_parts = dep_version.split('.') template_values['%smajver' % pref] = dep_version_parts[0] if len(dep_version_parts) > 1: template_values['%sminver' % pref] = dep_version_parts[1] template_values['%sshortver' % pref] = '.'.join( dep_version_parts[:2]) # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: if name in ignore: continue if name in config: template_values[name] = config[name] _log.debug('name: %s, config: %s', name, config[name]) # step 4. make lower variants for name in TEMPLATE_NAMES_LOWER: if name in ignore: continue value = config.get(name) or template_values.get(name) if value is None: continue try: template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % { 'name': name }] = value.lower() except Exception: _log.warning( "Failed to get .lower() for name %s value %s (type %s)", name, value, type(value)) # keep track of names of defined templates until now, # so we can check whether names of additional dynamic template values are all known common_template_names = set(template_values.keys()) # step 5. add additional conditional templates if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'): try: # get prefix for commands to be run with mpi runtime using default number of ranks mpi_cmd_prefix = toolchain.mpi_cmd_prefix() if mpi_cmd_prefix is not None: template_values['mpi_cmd_prefix'] = mpi_cmd_prefix except EasyBuildError as err: # don't fail just because we couldn't resolve this template _log.warning( "Failed to create mpi_cmd_prefix template, error was:\n%s", err) # step 6. CUDA compute capabilities # Use the commandline / easybuild config option if given, else use the value from the EC (as a default) cuda_compute_capabilities = build_option( 'cuda_compute_capabilities') or config.get('cuda_compute_capabilities') if cuda_compute_capabilities: template_values['cuda_compute_capabilities'] = ','.join( cuda_compute_capabilities) template_values['cuda_cc_space_sep'] = ' '.join( cuda_compute_capabilities) template_values['cuda_cc_semicolon_sep'] = ';'.join( cuda_compute_capabilities) sm_values = [ 'sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities ] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) unknown_names = [] for key in template_values: dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC) if not (key in common_template_names or key in dynamic_template_names): unknown_names.append(key) if unknown_names: raise EasyBuildError( "One or more template values found with unknown name: %s", ','.join(unknown_names)) return template_values