def __init__(self, mself, ext, extra_params=None): """ Constructor for Extension class :param mself: parent Easyblock instance :param ext: dictionary with extension metadata (name, version, src, patches, options, ...) :param extra_params: extra custom easyconfig parameters to take into account for this extension """ self.master = mself self.log = self.master.log self.cfg = self.master.cfg.copy(validate=False) self.ext = copy.deepcopy(ext) self.dry_run = self.master.dry_run if 'name' not in self.ext: raise EasyBuildError( "'name' is missing in supplied class instance 'ext'.") name, version = self.ext['name'], self.ext.get('version', None) # parent sanity check paths/commands are not relevant for extension self.cfg['sanity_check_commands'] = [] self.cfg['sanity_check_paths'] = [] # construct dict with template values that can be used self.cfg.template_values.update( template_constant_dict({ 'name': name, 'version': version })) # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template( copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) if extra_params: self.cfg.extend_params(extra_params, overwrite=False) # custom easyconfig parameters for extension are included in self.options # make sure they are merged into self.cfg so they can be queried; # unknown easyconfig parameters are ignored since self.options may include keys only there for extensions; # this allows to specify custom easyconfig parameters on a per-extension basis for key in self.options: if key in self.cfg: self.cfg[key] = resolve_template(self.options[key], self.cfg.template_values) self.log.debug( "Customising known easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, self.cfg[key]) else: self.log.debug( "Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, self.options[key]) self.sanity_check_fail_msgs = []
def __init__(self, mself, ext, extra_params=None): """ Constructor for Extension class :param mself: parent Easyblock instance :param ext: dictionary with extension metadata (name, version, src, patches, options, ...) :param extra_params: extra custom easyconfig parameters to take into account for this extension """ self.master = mself self.log = self.master.log self.cfg = self.master.cfg.copy(validate=False) self.ext = copy.deepcopy(ext) self.dry_run = self.master.dry_run if 'name' not in self.ext: raise EasyBuildError("'name' is missing in supplied class instance 'ext'.") name, version = self.ext['name'], self.ext.get('version', None) # parent sanity check paths/commands and postinstallcmds are not relevant for extension self.cfg['sanity_check_commands'] = [] self.cfg['sanity_check_paths'] = [] self.cfg['postinstallcmds'] = [] # construct dict with template values that can be used self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version})) # Add install/builddir templates with values from master. for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None)) # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) if extra_params: self.cfg.extend_params(extra_params, overwrite=False) # custom easyconfig parameters for extension are included in self.options # make sure they are merged into self.cfg so they can be queried; # unknown easyconfig parameters are ignored since self.options may include keys only there for extensions; # this allows to specify custom easyconfig parameters on a per-extension basis for key, value in self.options.items(): if key in self.cfg: self.cfg[key] = value self.log.debug("Customising known easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, value) else: self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, value) self.sanity_check_fail_msgs = [] self.async_cmd_info = None self.async_cmd_output = None self.async_cmd_check_cnt = None # initial read size should be relatively small, # to avoid hanging for a long time until desired output is available in async_cmd_check self.async_cmd_read_size = 1024
def _generate_template_values(self, ignore=None, skip_lower=True): """Actual code to generate the template values""" if self.template_values is None: self.template_values = {} # step 0. self.template_values can/should be updated from outside easyconfig # (eg the run_setp code in EasyBlock) # step 1-3 work with easyconfig.templates constants # use a copy to make sure the original is not touched/modified template_values = template_constant_dict(copy.deepcopy(self._config), ignore=ignore, skip_lower=skip_lower) # update the template_values dict self.template_values.update(template_values) # cleanup None values for k, v in self.template_values.items(): if v is None: del self.template_values[k]
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False): """ Show list of supported software :param output_format: output format to use :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) :param only_installed: only retain software for which a corresponding module is available :return: multi-line string presenting requested info """ silent = build_option('silent') ec_paths = find_matching_easyconfigs('*', '*', build_option('robot_path') or []) ecs = [] cnt = len(ec_paths) for idx, ec_path in enumerate(ec_paths): # full EasyConfig instance is only required when module name is needed # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser if only_installed: ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]['ec'] else: ec = EasyConfigParser(filename=ec_path).get_config_dict() ecs.append(ec) print_msg('\r', prefix=False, newline=False, silent=silent) print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt), newline=False, silent=silent) print_msg('', prefix=False, silent=silent) software = {} for ec in ecs: software.setdefault(ec['name'], []) if is_system_toolchain(ec['toolchain']['name']): toolchain = SYSTEM_TOOLCHAIN_NAME else: toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version']) keys = ['description', 'homepage', 'version', 'versionsuffix'] info = {'toolchain': toolchain} for key in keys: info[key] = ec.get(key, '') # make sure values like homepage & versionsuffix get properly templated if isinstance(ec, dict): template_values = template_constant_dict(ec) for key in keys: if '%(' in info[key]: try: info[key] = info[key] % template_values except (KeyError, TypeError, ValueError) as err: _log.debug("Ignoring failure to resolve templates: %s", err) software[ec['name']].append(info) if only_installed: software[ec['name']][-1].update({'mod_name': ec.full_mod_name}) print_msg("Found %d different software packages" % len(software), silent=silent) if only_installed: avail_mod_names = modules_tool().available() # rebuild software, only retain entries with a corresponding available module software, all_software = {}, software for key in all_software: for entry in all_software[key]: if entry['mod_name'] in avail_mod_names: software.setdefault(key, []).append(entry) print_msg("Retained %d installed software packages" % len(software), silent=silent) return generate_doc('list_software_%s' % output_format, [software, detailed])
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False): """ Show list of supported software :param output_format: output format to use :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) :param only_installed: only retain software for which a corresponding module is available :return: multi-line string presenting requested info """ silent = build_option('silent') ec_paths = find_matching_easyconfigs('*', '*', build_option('robot_path') or []) ecs = [] cnt = len(ec_paths) for idx, ec_path in enumerate(ec_paths): # full EasyConfig instance is only required when module name is needed # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser if only_installed: ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]['ec'] else: ec = EasyConfigParser(filename=ec_path).get_config_dict() ecs.append(ec) print_msg('\r', prefix=False, newline=False, silent=silent) print_msg("Processed %d/%d easyconfigs..." % (idx+1, cnt), newline=False, silent=silent) print_msg('', prefix=False, silent=silent) software = {} for ec in ecs: software.setdefault(ec['name'], []) if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME: toolchain = DUMMY_TOOLCHAIN_NAME else: toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version']) keys = ['description', 'homepage', 'version', 'versionsuffix'] info = {'toolchain': toolchain} for key in keys: info[key] = ec.get(key, '') # make sure values like homepage & versionsuffix get properly templated if isinstance(ec, dict): template_values = template_constant_dict(ec, skip_lower=False) for key in keys: if '%(' in info[key]: try: info[key] = info[key] % template_values except (KeyError, TypeError, ValueError) as err: _log.debug("Ignoring failure to resolve templates: %s", err) software[ec['name']].append(info) if only_installed: software[ec['name']][-1].update({'mod_name': ec.full_mod_name}) print_msg("Found %d different software packages" % len(software), silent=silent) if only_installed: avail_mod_names = modules_tool().available() # rebuild software, only retain entries with a corresponding available module software, all_software = {}, software for key in all_software: for entry in all_software[key]: if entry['mod_name'] in avail_mod_names: software.setdefault(key, []).append(entry) print_msg("Retained %d installed software packages" % len(software), silent=silent) return generate_doc('list_software_%s' % output_format, [software, detailed])
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False): """ Show list of supported software :param output_format: output format to use :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) :param only_installed: only retain software for which a corresponding module is available :return: multi-line string presenting requested info """ silent = build_option("silent") ec_paths = find_matching_easyconfigs("*", "*", build_option("robot_path") or []) ecs = [] cnt = len(ec_paths) for idx, ec_path in enumerate(ec_paths): # full EasyConfig instance is only required when module name is needed # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser if only_installed: ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]["ec"] else: ec = EasyConfigParser(filename=ec_path).get_config_dict() ecs.append(ec) print_msg("\r", prefix=False, newline=False, silent=silent) print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt), newline=False, silent=silent) print_msg("", prefix=False, silent=silent) software = {} for ec in ecs: software.setdefault(ec["name"], []) if ec["toolchain"]["name"] == DUMMY_TOOLCHAIN_NAME: toolchain = DUMMY_TOOLCHAIN_NAME else: toolchain = "%s/%s" % (ec["toolchain"]["name"], ec["toolchain"]["version"]) versionsuffix = ec.get("versionsuffix", "") # make sure versionsuffix gets properly templated if versionsuffix and isinstance(ec, dict): template_values = template_constant_dict(ec) versionsuffix = versionsuffix % template_values software[ec["name"]].append( { "description": ec["description"], "homepage": ec["homepage"], "toolchain": toolchain, "version": ec["version"], "versionsuffix": versionsuffix, } ) if only_installed: software[ec["name"]][-1].update({"mod_name": ec.full_mod_name}) print_msg("Found %d different software packages" % len(software), silent=silent) if only_installed: avail_mod_names = modules_tool().available() # rebuild software, only retain entries with a corresponding available module software, all_software = {}, software for key in all_software: for entry in all_software[key]: if entry["mod_name"] in avail_mod_names: software.setdefault(key, []).append(entry) print_msg("Retained %d installed software packages" % len(software), silent=silent) return generate_doc("list_software_%s" % output_format, [software, detailed])