def __init__(self, mself, ext, extra_params=None):
"""
Constructor for Extension class
:param mself: parent Easyblock instance
:param ext: dictionary with extension metadata (name, version, src, patches, options, ...)
:param extra_params: extra custom easyconfig parameters to take into account for this extension
"""
self.master = mself
self.log = self.master.log
self.cfg = self.master.cfg.copy(validate=False)
self.ext = copy.deepcopy(ext)
self.dry_run = self.master.dry_run
if 'name' not in self.ext:
raise EasyBuildError(
"'name' is missing in supplied class instance 'ext'.")
name, version = self.ext['name'], self.ext.get('version', None)
# parent sanity check paths/commands are not relevant for extension
self.cfg['sanity_check_commands'] = []
self.cfg['sanity_check_paths'] = []
# construct dict with template values that can be used
self.cfg.template_values.update(
template_constant_dict({
'name': name,
'version': version
}))
# list of source/patch files: we use an empty list as default value like in EasyBlock
self.src = resolve_template(self.ext.get('src', []),
self.cfg.template_values)
self.patches = resolve_template(self.ext.get('patches', []),
self.cfg.template_values)
self.options = resolve_template(
copy.deepcopy(self.ext.get('options', {})),
self.cfg.template_values)
if extra_params:
self.cfg.extend_params(extra_params, overwrite=False)
# custom easyconfig parameters for extension are included in self.options
# make sure they are merged into self.cfg so they can be queried;
# unknown easyconfig parameters are ignored since self.options may include keys only there for extensions;
# this allows to specify custom easyconfig parameters on a per-extension basis
for key in self.options:
if key in self.cfg:
self.cfg[key] = resolve_template(self.options[key],
self.cfg.template_values)
self.log.debug(
"Customising known easyconfig parameter '%s' for extension %s/%s: %s",
key, name, version, self.cfg[key])
else:
self.log.debug(
"Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s",
key, name, version, self.options[key])
self.sanity_check_fail_msgs = []
def __init__(self, mself, ext, extra_params=None):
"""
Constructor for Extension class
:param mself: parent Easyblock instance
:param ext: dictionary with extension metadata (name, version, src, patches, options, ...)
:param extra_params: extra custom easyconfig parameters to take into account for this extension
"""
self.master = mself
self.log = self.master.log
self.cfg = self.master.cfg.copy(validate=False)
self.ext = copy.deepcopy(ext)
self.dry_run = self.master.dry_run
if 'name' not in self.ext:
raise EasyBuildError("'name' is missing in supplied class instance 'ext'.")
name, version = self.ext['name'], self.ext.get('version', None)
# parent sanity check paths/commands and postinstallcmds are not relevant for extension
self.cfg['sanity_check_commands'] = []
self.cfg['sanity_check_paths'] = []
self.cfg['postinstallcmds'] = []
# construct dict with template values that can be used
self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version}))
# Add install/builddir templates with values from master.
for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None))
# list of source/patch files: we use an empty list as default value like in EasyBlock
self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values)
self.src_extract_cmd = self.ext.get('extract_cmd', None)
self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values)
self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values)
if extra_params:
self.cfg.extend_params(extra_params, overwrite=False)
# custom easyconfig parameters for extension are included in self.options
# make sure they are merged into self.cfg so they can be queried;
# unknown easyconfig parameters are ignored since self.options may include keys only there for extensions;
# this allows to specify custom easyconfig parameters on a per-extension basis
for key, value in self.options.items():
if key in self.cfg:
self.cfg[key] = value
self.log.debug("Customising known easyconfig parameter '%s' for extension %s/%s: %s",
key, name, version, value)
else:
self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s",
key, name, version, value)
self.sanity_check_fail_msgs = []
self.async_cmd_info = None
self.async_cmd_output = None
self.async_cmd_check_cnt = None
# initial read size should be relatively small,
# to avoid hanging for a long time until desired output is available in async_cmd_check
self.async_cmd_read_size = 1024
def _generate_template_values(self, ignore=None, skip_lower=True):
"""Actual code to generate the template values"""
if self.template_values is None:
self.template_values = {}
# step 0. self.template_values can/should be updated from outside easyconfig
# (eg the run_setp code in EasyBlock)
# step 1-3 work with easyconfig.templates constants
# use a copy to make sure the original is not touched/modified
template_values = template_constant_dict(copy.deepcopy(self._config),
ignore=ignore, skip_lower=skip_lower)
# update the template_values dict
self.template_values.update(template_values)
# cleanup None values
for k, v in self.template_values.items():
if v is None:
del self.template_values[k]
def list_software(output_format=FORMAT_TXT,
detailed=False,
only_installed=False):
"""
Show list of supported software
:param output_format: output format to use
:param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
:param only_installed: only retain software for which a corresponding module is available
:return: multi-line string presenting requested info
"""
silent = build_option('silent')
ec_paths = find_matching_easyconfigs('*', '*',
build_option('robot_path') or [])
ecs = []
cnt = len(ec_paths)
for idx, ec_path in enumerate(ec_paths):
# full EasyConfig instance is only required when module name is needed
# this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
if only_installed:
ec = process_easyconfig(ec_path, validate=False,
parse_only=True)[0]['ec']
else:
ec = EasyConfigParser(filename=ec_path).get_config_dict()
ecs.append(ec)
print_msg('\r', prefix=False, newline=False, silent=silent)
print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt),
newline=False,
silent=silent)
print_msg('', prefix=False, silent=silent)
software = {}
for ec in ecs:
software.setdefault(ec['name'], [])
if is_system_toolchain(ec['toolchain']['name']):
toolchain = SYSTEM_TOOLCHAIN_NAME
else:
toolchain = '%s/%s' % (ec['toolchain']['name'],
ec['toolchain']['version'])
keys = ['description', 'homepage', 'version', 'versionsuffix']
info = {'toolchain': toolchain}
for key in keys:
info[key] = ec.get(key, '')
# make sure values like homepage & versionsuffix get properly templated
if isinstance(ec, dict):
template_values = template_constant_dict(ec)
for key in keys:
if '%(' in info[key]:
try:
info[key] = info[key] % template_values
except (KeyError, TypeError, ValueError) as err:
_log.debug("Ignoring failure to resolve templates: %s",
err)
software[ec['name']].append(info)
if only_installed:
software[ec['name']][-1].update({'mod_name': ec.full_mod_name})
print_msg("Found %d different software packages" % len(software),
silent=silent)
if only_installed:
avail_mod_names = modules_tool().available()
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
for key in all_software:
for entry in all_software[key]:
if entry['mod_name'] in avail_mod_names:
software.setdefault(key, []).append(entry)
print_msg("Retained %d installed software packages" % len(software),
silent=silent)
return generate_doc('list_software_%s' % output_format,
[software, detailed])
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
"""
Show list of supported software
:param output_format: output format to use
:param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
:param only_installed: only retain software for which a corresponding module is available
:return: multi-line string presenting requested info
"""
silent = build_option('silent')
ec_paths = find_matching_easyconfigs('*', '*', build_option('robot_path') or [])
ecs = []
cnt = len(ec_paths)
for idx, ec_path in enumerate(ec_paths):
# full EasyConfig instance is only required when module name is needed
# this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
if only_installed:
ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]['ec']
else:
ec = EasyConfigParser(filename=ec_path).get_config_dict()
ecs.append(ec)
print_msg('\r', prefix=False, newline=False, silent=silent)
print_msg("Processed %d/%d easyconfigs..." % (idx+1, cnt), newline=False, silent=silent)
print_msg('', prefix=False, silent=silent)
software = {}
for ec in ecs:
software.setdefault(ec['name'], [])
if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME:
toolchain = DUMMY_TOOLCHAIN_NAME
else:
toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version'])
keys = ['description', 'homepage', 'version', 'versionsuffix']
info = {'toolchain': toolchain}
for key in keys:
info[key] = ec.get(key, '')
# make sure values like homepage & versionsuffix get properly templated
if isinstance(ec, dict):
template_values = template_constant_dict(ec, skip_lower=False)
for key in keys:
if '%(' in info[key]:
try:
info[key] = info[key] % template_values
except (KeyError, TypeError, ValueError) as err:
_log.debug("Ignoring failure to resolve templates: %s", err)
software[ec['name']].append(info)
if only_installed:
software[ec['name']][-1].update({'mod_name': ec.full_mod_name})
print_msg("Found %d different software packages" % len(software), silent=silent)
if only_installed:
avail_mod_names = modules_tool().available()
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
for key in all_software:
for entry in all_software[key]:
if entry['mod_name'] in avail_mod_names:
software.setdefault(key, []).append(entry)
print_msg("Retained %d installed software packages" % len(software), silent=silent)
return generate_doc('list_software_%s' % output_format, [software, detailed])
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
"""
Show list of supported software
:param output_format: output format to use
:param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
:param only_installed: only retain software for which a corresponding module is available
:return: multi-line string presenting requested info
"""
silent = build_option("silent")
ec_paths = find_matching_easyconfigs("*", "*", build_option("robot_path") or [])
ecs = []
cnt = len(ec_paths)
for idx, ec_path in enumerate(ec_paths):
# full EasyConfig instance is only required when module name is needed
# this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
if only_installed:
ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]["ec"]
else:
ec = EasyConfigParser(filename=ec_path).get_config_dict()
ecs.append(ec)
print_msg("\r", prefix=False, newline=False, silent=silent)
print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt), newline=False, silent=silent)
print_msg("", prefix=False, silent=silent)
software = {}
for ec in ecs:
software.setdefault(ec["name"], [])
if ec["toolchain"]["name"] == DUMMY_TOOLCHAIN_NAME:
toolchain = DUMMY_TOOLCHAIN_NAME
else:
toolchain = "%s/%s" % (ec["toolchain"]["name"], ec["toolchain"]["version"])
versionsuffix = ec.get("versionsuffix", "")
# make sure versionsuffix gets properly templated
if versionsuffix and isinstance(ec, dict):
template_values = template_constant_dict(ec)
versionsuffix = versionsuffix % template_values
software[ec["name"]].append(
{
"description": ec["description"],
"homepage": ec["homepage"],
"toolchain": toolchain,
"version": ec["version"],
"versionsuffix": versionsuffix,
}
)
if only_installed:
software[ec["name"]][-1].update({"mod_name": ec.full_mod_name})
print_msg("Found %d different software packages" % len(software), silent=silent)
if only_installed:
avail_mod_names = modules_tool().available()
# rebuild software, only retain entries with a corresponding available module
software, all_software = {}, software
for key in all_software:
for entry in all_software[key]:
if entry["mod_name"] in avail_mod_names:
software.setdefault(key, []).append(entry)
print_msg("Retained %d installed software packages" % len(software), silent=silent)
return generate_doc("list_software_%s" % output_format, [software, detailed])