def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
"""
Construct formatted string representation of iterable parameter (list/tuple/dict), including comments.
@param param_name: parameter name
@param param_val: parameter value
@param outer: reformat for top-level parameter, or not
@param addlen: # characters to add to line length
"""
param_strval = str(param_val)
res = param_strval
# determine whether line would be too long
# note: this does not take into account the parameter name + '=', only the value
line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH
forced = param_name in REFORMAT_FORCED_PARAMS
if param_name in REFORMAT_SKIPPED_PARAMS:
self.log.info("Skipping reformatting value for parameter '%s'", param_name)
elif outer:
# only reformat outer (iterable) values for (too) long lines (or for select parameters)
if isinstance(param_val, (list, tuple, dict)) and ((len(param_val) > 1 and line_too_long) or forced):
item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n'
# start with opening character: [, (, {
res = '%s\n' % param_strval[0]
# add items one-by-one, special care for dict values (order of keys, different format for elements)
if isinstance(param_val, dict):
ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(param_name, sorted(param_val.keys()))
for item_key in ordered_item_keys:
item_val = param_val[item_key]
comment = self._get_item_comments(param_name, item_val).get(str(item_val), '')
key_pref = quote_py_str(item_key) + ': '
addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(comment)
formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen)
res += item_tmpl % {
'comment': comment,
'item': key_pref + formatted_item_val,
}
else: # list, tuple
for item in param_val:
comment = self._get_item_comments(param_name, item).get(str(item), '')
addlen = addlen + len(INDENT_4SPACES) + len(comment)
res += item_tmpl % {
'comment': comment,
'item': self._reformat_line(param_name, item, addlen=addlen)
}
# end with closing character: ], ), }
res += param_strval[-1]
else:
# dependencies are already dumped as strings, so they do not need to be quoted again
if isinstance(param_val, basestring) and param_name not in DEPENDENCY_PARAMETERS:
res = quote_py_str(param_val)
return res
def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
"""
Construct formatted string representation of iterable parameter (list/tuple/dict), including comments.
:param param_name: parameter name
:param param_val: parameter value
:param outer: reformat for top-level parameter, or not
:param addlen: # characters to add to line length
"""
param_strval = str(param_val)
res = param_strval
# determine whether line would be too long
# note: this does not take into account the parameter name + '=', only the value
line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH
forced = param_name in REFORMAT_FORCED_PARAMS
if param_name in REFORMAT_SKIPPED_PARAMS:
self.log.info("Skipping reformatting value for parameter '%s'", param_name)
elif outer:
# only reformat outer (iterable) values for (too) long lines (or for select parameters)
if isinstance(param_val, (list, tuple, dict)) and ((len(param_val) > 1 and line_too_long) or forced):
item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n'
# start with opening character: [, (, {
res = '%s\n' % param_strval[0]
# add items one-by-one, special care for dict values (order of keys, different format for elements)
if isinstance(param_val, dict):
ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(param_name, sorted(param_val.keys()))
for item_key in ordered_item_keys:
item_val = param_val[item_key]
comment = self._get_item_comments(param_name, item_val).get(str(item_val), '')
key_pref = quote_py_str(item_key) + ': '
addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(comment)
formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen)
res += item_tmpl % {
'comment': comment,
'item': key_pref + formatted_item_val,
}
else: # list, tuple
for item in param_val:
comment = self._get_item_comments(param_name, item).get(str(item), '')
addlen = addlen + len(INDENT_4SPACES) + len(comment)
res += item_tmpl % {
'comment': comment,
'item': self._reformat_line(param_name, item, addlen=addlen)
}
# end with closing character: ], ), }
res += param_strval[-1]
else:
# dependencies are already dumped as strings, so they do not need to be quoted again
if isinstance(param_val, basestring) and param_name not in DEPENDENCY_PARAMETERS:
res = quote_py_str(param_val)
return res
def dump(self, ecfg, default_values, templ_const, templ_val):
"""
Dump easyconfig in format v1.
@param ecfg: EasyConfig instance
@param default_values: default values for easyconfig parameters
@param templ_const: known template constants
@param templ_val: known template values
"""
# include header comments first
dump = self.comments['header'][:]
# print easyconfig parameters ordered and in groups specified above
params, printed_keys = self._find_defined_params(
ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val)
dump.extend(params)
# print other easyconfig parameters at the end
keys_to_ignore = printed_keys + LAST_PARAMS
for key in default_values:
if key not in keys_to_ignore and ecfg[key] != default_values[key]:
dump.extend(
self._find_param_with_comments(key,
quote_py_str(ecfg[key]),
templ_const, templ_val))
dump.append('')
# print last parameters
params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS],
default_values, templ_const,
templ_val)
dump.extend(params)
return '\n'.join(dump)
def _find_defined_params(self, ecfg, keyset, default_values, templ_const,
templ_val):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
# the value for 'dependencies' may have been modified after parsing via filter_hidden_deps
if key == 'dependencies':
val = ecfg[key] + ecfg['hiddendependencies']
else:
val = ecfg[key]
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
if key in DEPENDENCY_PARAMETERS:
valstr = [
dump_dependency(d, ecfg['toolchain']) for d in val
]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(
self._find_param_with_comments(key, valstr,
templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def dump(self, ecfg, default_values, templ_const, templ_val):
"""
Dump easyconfig in format v1.
@param ecfg: EasyConfig instance
@param default_values: default values for easyconfig parameters
@param templ_const: known template constants
@param templ_val: known template values
"""
# include header comments first
dump = self.comments['header'][:]
# print easyconfig parameters ordered and in groups specified above
params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val)
dump.extend(params)
# print other easyconfig parameters at the end
keys_to_ignore = printed_keys + LAST_PARAMS
for key in default_values:
if key not in keys_to_ignore and ecfg[key] != default_values[key]:
dump.extend(self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val))
dump.append('')
# print last parameters
params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val)
dump.extend(params)
dump.extend(self.comments['tail'])
return '\n'.join(dump)
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
# the value for 'dependencies' may have been modified after parsing via filter_hidden_deps
if key == 'dependencies':
val = ecfg[key] + ecfg['hiddendependencies']
else:
val = ecfg[key]
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
if key in DEPENDENCY_PARAMETERS:
valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
val = copy.deepcopy(ecfg[key])
# include hidden deps back in list of (build)dependencies, they were filtered out via filter_hidden_deps
if key == 'dependencies':
val.extend([d for d in ecfg['hiddendependencies'] if not d['build_only']])
elif key == 'builddependencies':
val.extend([d for d in ecfg['hiddendependencies'] if d['build_only']])
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
if key in DEPENDENCY_PARAMETERS:
valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
elif key == 'toolchain':
valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
val = copy.deepcopy(ecfg[key])
# include hidden deps back in list of (build)dependencies, they were filtered out via filter_hidden_deps
if key == 'dependencies':
val.extend([d for d in ecfg['hiddendependencies'] if not d['build_only']])
elif key == 'builddependencies':
val.extend([d for d in ecfg['hiddendependencies'] if d['build_only']])
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
if key in DEPENDENCY_PARAMETERS:
valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
elif key == 'toolchain':
valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def dump(self,
ecfg,
default_values,
templ_const,
templ_val,
toolchain_hierarchy=None):
"""
Dump easyconfig in format v1.
:param ecfg: EasyConfig instance
:param default_values: default values for easyconfig parameters
:param templ_const: known template constants
:param templ_val: known template values
:param toolchain_hierarchy: hierarchy of toolchains for easyconfig
"""
# figoure out whether we should be strict about the format of sanity_check_paths;
# if enhance_sanity_check is set, then both files/dirs keys are not strictly required...
self.strict_sanity_check_paths_keys = not ecfg['enhance_sanity_check']
# include header comments first
dump = self.comments['header'][:]
# print easyconfig parameters ordered and in groups specified above
params, printed_keys = self._find_defined_params(
ecfg,
GROUPED_PARAMS,
default_values,
templ_const,
templ_val,
toolchain_hierarchy=toolchain_hierarchy)
dump.extend(params)
# print other easyconfig parameters at the end
keys_to_ignore = printed_keys + LAST_PARAMS
for key in default_values:
mandatory = ecfg.is_mandatory_param(key)
non_default_value = ecfg[key] != default_values[key]
if key not in keys_to_ignore and (mandatory or non_default_value):
dump.extend(
self._find_param_with_comments(key,
quote_py_str(ecfg[key]),
templ_const, templ_val))
dump.append('')
# print last parameters
params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS],
default_values, templ_const,
templ_val)
dump.extend(params)
dump.extend(self.comments['tail'])
return '\n'.join(dump)
def _find_defined_params(self, ecfg, keyset, default_values, templ_const,
templ_val):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
val = ecfg[key]
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them;
# take into account that these parameters may be iterative (i.e. a list of lists of parsed deps)
if key in DEPENDENCY_PARAMETERS:
if key in ecfg.iterate_options:
if 'multi_deps' in ecfg:
# the way that builddependencies are constructed with multi_deps
# we just need to dump the first entry without the dependencies
# that are listed in multi_deps
valstr = [
dump_dependency(d, ecfg['toolchain'])
for d in val[0]
if d['name'] not in ecfg['multi_deps']
]
else:
valstr = [[
dump_dependency(d, ecfg['toolchain'])
for d in dep
] for dep in val]
else:
valstr = [
dump_dependency(d, ecfg['toolchain'])
for d in val
]
elif key == 'toolchain':
valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[
key]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(
self._find_param_with_comments(key, valstr,
templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def dump_dependency(dep, toolchain):
"""Dump parsed dependency in tuple format"""
if dep['external_module']:
res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name'])
else:
# mininal spec: (name, version)
tup = (dep['name'], dep['version'])
if dep['toolchain'] != toolchain:
if dep['dummy']:
tup += (dep['versionsuffix'], True)
else:
tup += (dep['versionsuffix'], (dep['toolchain']['name'], dep['toolchain']['version']))
elif dep['versionsuffix']:
tup += (dep['versionsuffix'],)
res = str(tup)
return res
def dump_dependency(dep, toolchain):
"""Dump parsed dependency in tuple format"""
if dep["external_module"]:
res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep["full_mod_name"])
else:
# mininal spec: (name, version)
tup = (dep["name"], dep["version"])
if dep["toolchain"] != toolchain:
if dep["dummy"]:
tup += (dep["versionsuffix"], True)
else:
tup += (dep["versionsuffix"], (dep["toolchain"]["name"], dep["toolchain"]["version"]))
elif dep["versionsuffix"]:
tup += (dep["versionsuffix"],)
res = str(tup)
return res
def dump_dependency(dep, toolchain):
"""Dump parsed dependency in tuple format"""
if dep['external_module']:
res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name'])
else:
# mininal spec: (name, version)
tup = (dep['name'], dep['version'])
if dep['toolchain'] != toolchain:
if dep['dummy']:
tup += (dep['versionsuffix'], True)
else:
tup += (dep['versionsuffix'], (dep['toolchain']['name'], dep['toolchain']['version']))
elif dep['versionsuffix']:
tup += (dep['versionsuffix'],)
res = str(tup)
return res
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
val = ecfg[key]
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them;
# take into account that these parameters may be iterative (i.e. a list of lists of parsed deps)
if key in DEPENDENCY_PARAMETERS:
if key in ecfg.iterate_options:
if 'multi_deps' in ecfg:
# the way that builddependencies are constructed with multi_deps
# we just need to dump the first entry without the dependencies
# that are listed in multi_deps
valstr = [dump_dependency(d, ecfg['toolchain']) for d in val[0]
if d['name'] not in ecfg['multi_deps']]
else:
valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val]
else:
valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
elif key == 'toolchain':
valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def dump_dependency(dep, toolchain, toolchain_hierarchy=None):
"""Dump parsed dependency in tuple format"""
if not toolchain_hierarchy:
toolchain_hierarchy = [toolchain]
if dep['external_module']:
res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name'])
else:
# minimal spec: (name, version)
tup = (dep['name'], dep['version'])
if all(dep['toolchain'] != subtoolchain
for subtoolchain in toolchain_hierarchy):
if dep[SYSTEM_TOOLCHAIN_NAME]:
tup += (dep['versionsuffix'], True)
else:
tup += (dep['versionsuffix'], (dep['toolchain']['name'],
dep['toolchain']['version']))
elif dep['versionsuffix']:
tup += (dep['versionsuffix'], )
res = str(tup)
return res
def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
"""
Construct formatted string representation of iterable parameter (list/tuple/dict), including comments.
:param param_name: parameter name
:param param_val: parameter value
:param outer: reformat for top-level parameter, or not
:param addlen: # characters to add to line length
"""
param_strval = str(param_val)
res = param_strval
# determine whether line would be too long
# note: this does not take into account the parameter name + '=', only the value
line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH
forced = param_name in REFORMAT_FORCED_PARAMS
list_of_lists_of_tuples_param = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES
if param_name in REFORMAT_SKIPPED_PARAMS:
self.log.info("Skipping reformatting value for parameter '%s'",
param_name)
elif outer:
# only reformat outer (iterable) values for (too) long lines (or for select parameters)
if isinstance(param_val, (list, tuple, dict)) and (
(len(param_val) > 1 or line_too_long) or forced):
item_tmpl = INDENT_4SPACES + '%(item)s,%(inline_comment)s\n'
start_char, end_char = param_strval[0], param_strval[-1]
# start with opening character: [, (, {
res = '%s\n' % start_char
# add items one-by-one, special care for dict values (order of keys, different format for elements)
if isinstance(param_val, dict):
ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(
param_name, sorted(param_val.keys()))
for item_key in ordered_item_keys:
if item_key in param_val:
item_val = param_val[item_key]
item_comments = self._get_item_comments(
param_name, item_val)
elif param_name == 'sanity_check_paths' and not self.strict_sanity_check_paths_keys:
item_val = []
item_comments = {}
self.log.info(
"Using default value for '%s' in sanity_check_paths: %s",
item_key, item_val)
else:
raise EasyBuildError(
"Missing mandatory key '%s' in %s.", item_key,
param_name)
inline_comment = item_comments.get('inline', '')
item_tmpl_dict = {'inline_comment': inline_comment}
for comment in item_comments.get('above', []):
res += INDENT_4SPACES + comment + '\n'
key_pref = quote_py_str(item_key) + ': '
addlen = addlen + len(INDENT_4SPACES) + len(
key_pref) + len(inline_comment)
formatted_item_val = self._reformat_line(param_name,
item_val,
addlen=addlen)
item_tmpl_dict['item'] = key_pref + formatted_item_val
res += item_tmpl % item_tmpl_dict
else: # list, tuple
for item in param_val:
item_comments = self._get_item_comments(
param_name, item)
inline_comment = item_comments.get('inline', '')
item_tmpl_dict = {'inline_comment': inline_comment}
for comment in item_comments.get('above', []):
res += INDENT_4SPACES + comment + '\n'
addlen = addlen + len(INDENT_4SPACES) + len(
inline_comment)
# the tuples are really strings here that are constructed from the dependency dicts
# so for a plain list of builddependencies param_val is a list of strings here;
# and for iterated builddependencies it is a list of lists of strings
is_list_of_lists_of_tuples = isinstance(
item, list) and all(
isinstance(x, str) for x in item)
if list_of_lists_of_tuples_param and is_list_of_lists_of_tuples:
itemstr = '[' + (',\n ' + INDENT_4SPACES).join([
self._reformat_line(param_name,
subitem,
outer=True,
addlen=addlen)
for subitem in item
]) + ']'
else:
itemstr = self._reformat_line(param_name,
item,
addlen=addlen)
item_tmpl_dict['item'] = itemstr
res += item_tmpl % item_tmpl_dict
# take into account possible closing comments
# see https://github.com/easybuilders/easybuild-framework/issues/3082
end_comments = self._get_item_comments(param_name, end_char)
for comment in end_comments.get('above', []):
res += INDENT_4SPACES + comment + '\n'
# end with closing character (']', ')', '}'), incl. possible inline comment
res += end_char
if 'inline' in end_comments:
res += end_comments['inline']
else:
# dependencies are already dumped as strings, so they do not need to be quoted again
if isinstance(
param_val,
string_type) and param_name not in DEPENDENCY_PARAMETERS:
res = quote_py_str(param_val)
return res