def _reformat_line(self, param_name, param_val, outer=False, addlen=0): """ Construct formatted string representation of iterable parameter (list/tuple/dict), including comments. @param param_name: parameter name @param param_val: parameter value @param outer: reformat for top-level parameter, or not @param addlen: # characters to add to line length """ param_strval = str(param_val) res = param_strval # determine whether line would be too long # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) elif outer: # only reformat outer (iterable) values for (too) long lines (or for select parameters) if isinstance(param_val, (list, tuple, dict)) and ((len(param_val) > 1 and line_too_long) or forced): item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n' # start with opening character: [, (, { res = '%s\n' % param_strval[0] # add items one-by-one, special care for dict values (order of keys, different format for elements) if isinstance(param_val, dict): ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(param_name, sorted(param_val.keys())) for item_key in ordered_item_keys: item_val = param_val[item_key] comment = self._get_item_comments(param_name, item_val).get(str(item_val), '') key_pref = quote_py_str(item_key) + ': ' addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(comment) formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen) res += item_tmpl % { 'comment': comment, 'item': key_pref + formatted_item_val, } else: # list, tuple for item in param_val: comment = self._get_item_comments(param_name, item).get(str(item), '') addlen = addlen + len(INDENT_4SPACES) + len(comment) res += item_tmpl % { 'comment': comment, 'item': self._reformat_line(param_name, item, addlen=addlen) } # end with closing character: ], ), } res += param_strval[-1] else: # dependencies are already dumped as strings, so they do not need to be quoted again if isinstance(param_val, basestring) and param_name not in DEPENDENCY_PARAMETERS: res = quote_py_str(param_val) return res
def _reformat_line(self, param_name, param_val, outer=False, addlen=0): """ Construct formatted string representation of iterable parameter (list/tuple/dict), including comments. :param param_name: parameter name :param param_val: parameter value :param outer: reformat for top-level parameter, or not :param addlen: # characters to add to line length """ param_strval = str(param_val) res = param_strval # determine whether line would be too long # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) elif outer: # only reformat outer (iterable) values for (too) long lines (or for select parameters) if isinstance(param_val, (list, tuple, dict)) and ((len(param_val) > 1 and line_too_long) or forced): item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n' # start with opening character: [, (, { res = '%s\n' % param_strval[0] # add items one-by-one, special care for dict values (order of keys, different format for elements) if isinstance(param_val, dict): ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(param_name, sorted(param_val.keys())) for item_key in ordered_item_keys: item_val = param_val[item_key] comment = self._get_item_comments(param_name, item_val).get(str(item_val), '') key_pref = quote_py_str(item_key) + ': ' addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(comment) formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen) res += item_tmpl % { 'comment': comment, 'item': key_pref + formatted_item_val, } else: # list, tuple for item in param_val: comment = self._get_item_comments(param_name, item).get(str(item), '') addlen = addlen + len(INDENT_4SPACES) + len(comment) res += item_tmpl % { 'comment': comment, 'item': self._reformat_line(param_name, item, addlen=addlen) } # end with closing character: ], ), } res += param_strval[-1] else: # dependencies are already dumped as strings, so they do not need to be quoted again if isinstance(param_val, basestring) and param_name not in DEPENDENCY_PARAMETERS: res = quote_py_str(param_val) return res
def dump(self, ecfg, default_values, templ_const, templ_val): """ Dump easyconfig in format v1. @param ecfg: EasyConfig instance @param default_values: default values for easyconfig parameters @param templ_const: known template constants @param templ_val: known template values """ # include header comments first dump = self.comments['header'][:] # print easyconfig parameters ordered and in groups specified above params, printed_keys = self._find_defined_params( ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val) dump.extend(params) # print other easyconfig parameters at the end keys_to_ignore = printed_keys + LAST_PARAMS for key in default_values: if key not in keys_to_ignore and ecfg[key] != default_values[key]: dump.extend( self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val)) dump.append('') # print last parameters params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val) dump.extend(params) return '\n'.join(dump)
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ eclines = [] printed_keys = [] for group in keyset: printed = False for key in group: # the value for 'dependencies' may have been modified after parsing via filter_hidden_deps if key == 'dependencies': val = ecfg[key] + ecfg['hiddendependencies'] else: val = ecfg[key] if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them if key in DEPENDENCY_PARAMETERS: valstr = [ dump_dependency(d, ecfg['toolchain']) for d in val ] else: valstr = quote_py_str(ecfg[key]) eclines.extend( self._find_param_with_comments(key, valstr, templ_const, templ_val)) printed_keys.append(key) printed = True if printed: eclines.append('') return eclines, printed_keys
def dump(self, ecfg, default_values, templ_const, templ_val): """ Dump easyconfig in format v1. @param ecfg: EasyConfig instance @param default_values: default values for easyconfig parameters @param templ_const: known template constants @param templ_val: known template values """ # include header comments first dump = self.comments['header'][:] # print easyconfig parameters ordered and in groups specified above params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val) dump.extend(params) # print other easyconfig parameters at the end keys_to_ignore = printed_keys + LAST_PARAMS for key in default_values: if key not in keys_to_ignore and ecfg[key] != default_values[key]: dump.extend(self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val)) dump.append('') # print last parameters params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val) dump.extend(params) dump.extend(self.comments['tail']) return '\n'.join(dump)
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ eclines = [] printed_keys = [] for group in keyset: printed = False for key in group: # the value for 'dependencies' may have been modified after parsing via filter_hidden_deps if key == 'dependencies': val = ecfg[key] + ecfg['hiddendependencies'] else: val = ecfg[key] if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them if key in DEPENDENCY_PARAMETERS: valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] else: valstr = quote_py_str(ecfg[key]) eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val)) printed_keys.append(key) printed = True if printed: eclines.append('') return eclines, printed_keys
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ eclines = [] printed_keys = [] for group in keyset: printed = False for key in group: val = copy.deepcopy(ecfg[key]) # include hidden deps back in list of (build)dependencies, they were filtered out via filter_hidden_deps if key == 'dependencies': val.extend([d for d in ecfg['hiddendependencies'] if not d['build_only']]) elif key == 'builddependencies': val.extend([d for d in ecfg['hiddendependencies'] if d['build_only']]) if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them if key in DEPENDENCY_PARAMETERS: valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key] else: valstr = quote_py_str(ecfg[key]) eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val)) printed_keys.append(key) printed = True if printed: eclines.append('') return eclines, printed_keys
def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): """ Dump easyconfig in format v1. :param ecfg: EasyConfig instance :param default_values: default values for easyconfig parameters :param templ_const: known template constants :param templ_val: known template values :param toolchain_hierarchy: hierarchy of toolchains for easyconfig """ # figoure out whether we should be strict about the format of sanity_check_paths; # if enhance_sanity_check is set, then both files/dirs keys are not strictly required... self.strict_sanity_check_paths_keys = not ecfg['enhance_sanity_check'] # include header comments first dump = self.comments['header'][:] # print easyconfig parameters ordered and in groups specified above params, printed_keys = self._find_defined_params( ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val, toolchain_hierarchy=toolchain_hierarchy) dump.extend(params) # print other easyconfig parameters at the end keys_to_ignore = printed_keys + LAST_PARAMS for key in default_values: mandatory = ecfg.is_mandatory_param(key) non_default_value = ecfg[key] != default_values[key] if key not in keys_to_ignore and (mandatory or non_default_value): dump.extend( self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val)) dump.append('') # print last parameters params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val) dump.extend(params) dump.extend(self.comments['tail']) return '\n'.join(dump)
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ eclines = [] printed_keys = [] for group in keyset: printed = False for key in group: val = ecfg[key] if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them; # take into account that these parameters may be iterative (i.e. a list of lists of parsed deps) if key in DEPENDENCY_PARAMETERS: if key in ecfg.iterate_options: if 'multi_deps' in ecfg: # the way that builddependencies are constructed with multi_deps # we just need to dump the first entry without the dependencies # that are listed in multi_deps valstr = [ dump_dependency(d, ecfg['toolchain']) for d in val[0] if d['name'] not in ecfg['multi_deps'] ] else: valstr = [[ dump_dependency(d, ecfg['toolchain']) for d in dep ] for dep in val] else: valstr = [ dump_dependency(d, ecfg['toolchain']) for d in val ] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[ key] else: valstr = quote_py_str(ecfg[key]) eclines.extend( self._find_param_with_comments(key, valstr, templ_const, templ_val)) printed_keys.append(key) printed = True if printed: eclines.append('') return eclines, printed_keys
def dump_dependency(dep, toolchain): """Dump parsed dependency in tuple format""" if dep['external_module']: res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name']) else: # mininal spec: (name, version) tup = (dep['name'], dep['version']) if dep['toolchain'] != toolchain: if dep['dummy']: tup += (dep['versionsuffix'], True) else: tup += (dep['versionsuffix'], (dep['toolchain']['name'], dep['toolchain']['version'])) elif dep['versionsuffix']: tup += (dep['versionsuffix'],) res = str(tup) return res
def dump_dependency(dep, toolchain): """Dump parsed dependency in tuple format""" if dep["external_module"]: res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep["full_mod_name"]) else: # mininal spec: (name, version) tup = (dep["name"], dep["version"]) if dep["toolchain"] != toolchain: if dep["dummy"]: tup += (dep["versionsuffix"], True) else: tup += (dep["versionsuffix"], (dep["toolchain"]["name"], dep["toolchain"]["version"])) elif dep["versionsuffix"]: tup += (dep["versionsuffix"],) res = str(tup) return res
def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ eclines = [] printed_keys = [] for group in keyset: printed = False for key in group: val = ecfg[key] if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them; # take into account that these parameters may be iterative (i.e. a list of lists of parsed deps) if key in DEPENDENCY_PARAMETERS: if key in ecfg.iterate_options: if 'multi_deps' in ecfg: # the way that builddependencies are constructed with multi_deps # we just need to dump the first entry without the dependencies # that are listed in multi_deps valstr = [dump_dependency(d, ecfg['toolchain']) for d in val[0] if d['name'] not in ecfg['multi_deps']] else: valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] else: valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key] else: valstr = quote_py_str(ecfg[key]) eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val)) printed_keys.append(key) printed = True if printed: eclines.append('') return eclines, printed_keys
def dump_dependency(dep, toolchain, toolchain_hierarchy=None): """Dump parsed dependency in tuple format""" if not toolchain_hierarchy: toolchain_hierarchy = [toolchain] if dep['external_module']: res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name']) else: # minimal spec: (name, version) tup = (dep['name'], dep['version']) if all(dep['toolchain'] != subtoolchain for subtoolchain in toolchain_hierarchy): if dep[SYSTEM_TOOLCHAIN_NAME]: tup += (dep['versionsuffix'], True) else: tup += (dep['versionsuffix'], (dep['toolchain']['name'], dep['toolchain']['version'])) elif dep['versionsuffix']: tup += (dep['versionsuffix'], ) res = str(tup) return res
def _reformat_line(self, param_name, param_val, outer=False, addlen=0): """ Construct formatted string representation of iterable parameter (list/tuple/dict), including comments. :param param_name: parameter name :param param_val: parameter value :param outer: reformat for top-level parameter, or not :param addlen: # characters to add to line length """ param_strval = str(param_val) res = param_strval # determine whether line would be too long # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS list_of_lists_of_tuples_param = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) elif outer: # only reformat outer (iterable) values for (too) long lines (or for select parameters) if isinstance(param_val, (list, tuple, dict)) and ( (len(param_val) > 1 or line_too_long) or forced): item_tmpl = INDENT_4SPACES + '%(item)s,%(inline_comment)s\n' start_char, end_char = param_strval[0], param_strval[-1] # start with opening character: [, (, { res = '%s\n' % start_char # add items one-by-one, special care for dict values (order of keys, different format for elements) if isinstance(param_val, dict): ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get( param_name, sorted(param_val.keys())) for item_key in ordered_item_keys: if item_key in param_val: item_val = param_val[item_key] item_comments = self._get_item_comments( param_name, item_val) elif param_name == 'sanity_check_paths' and not self.strict_sanity_check_paths_keys: item_val = [] item_comments = {} self.log.info( "Using default value for '%s' in sanity_check_paths: %s", item_key, item_val) else: raise EasyBuildError( "Missing mandatory key '%s' in %s.", item_key, param_name) inline_comment = item_comments.get('inline', '') item_tmpl_dict = {'inline_comment': inline_comment} for comment in item_comments.get('above', []): res += INDENT_4SPACES + comment + '\n' key_pref = quote_py_str(item_key) + ': ' addlen = addlen + len(INDENT_4SPACES) + len( key_pref) + len(inline_comment) formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen) item_tmpl_dict['item'] = key_pref + formatted_item_val res += item_tmpl % item_tmpl_dict else: # list, tuple for item in param_val: item_comments = self._get_item_comments( param_name, item) inline_comment = item_comments.get('inline', '') item_tmpl_dict = {'inline_comment': inline_comment} for comment in item_comments.get('above', []): res += INDENT_4SPACES + comment + '\n' addlen = addlen + len(INDENT_4SPACES) + len( inline_comment) # the tuples are really strings here that are constructed from the dependency dicts # so for a plain list of builddependencies param_val is a list of strings here; # and for iterated builddependencies it is a list of lists of strings is_list_of_lists_of_tuples = isinstance( item, list) and all( isinstance(x, str) for x in item) if list_of_lists_of_tuples_param and is_list_of_lists_of_tuples: itemstr = '[' + (',\n ' + INDENT_4SPACES).join([ self._reformat_line(param_name, subitem, outer=True, addlen=addlen) for subitem in item ]) + ']' else: itemstr = self._reformat_line(param_name, item, addlen=addlen) item_tmpl_dict['item'] = itemstr res += item_tmpl % item_tmpl_dict # take into account possible closing comments # see https://github.com/easybuilders/easybuild-framework/issues/3082 end_comments = self._get_item_comments(param_name, end_char) for comment in end_comments.get('above', []): res += INDENT_4SPACES + comment + '\n' # end with closing character (']', ')', '}'), incl. possible inline comment res += end_char if 'inline' in end_comments: res += end_comments['inline'] else: # dependencies are already dumped as strings, so they do not need to be quoted again if isinstance( param_val, string_type) and param_name not in DEPENDENCY_PARAMETERS: res = quote_py_str(param_val) return res