def test_v20(self): """Test parsing of easyconfig in format v2.""" # hard enable experimental orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb') ecp = EasyConfigParser(fn) formatter = ecp._formatter self.assertEqual(formatter.VERSION, EasyVersion('2.0')) self.assertTrue('name' in formatter.pyheader_localvars) self.assertFalse('version' in formatter.pyheader_localvars) self.assertFalse('toolchain' in formatter.pyheader_localvars) # this should be ok: ie the default values ec = ecp.get_config_dict() self.assertEqual(ec['toolchain'], { 'name': 'dummy', 'version': 'dummy' }) self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.2') # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v20(self): """Test parsing of easyconfig in format v2.""" # hard enable experimental orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb') ecp = EasyConfigParser(fn) formatter = ecp._formatter self.assertEqual(formatter.VERSION, EasyVersion('2.0')) self.assertTrue('name' in formatter.pyheader_localvars) self.assertFalse('version' in formatter.pyheader_localvars) self.assertFalse('toolchain' in formatter.pyheader_localvars) # this should be ok: ie the default values ec = ecp.get_config_dict() self.assertEqual(ec['toolchain'], { 'name': 'system', 'version': 'system' }) self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.2') # changes to this dict should not affect the return value of the next call to get_config_dict fn = 'test.tar.gz' ec['sources'].append(fn) ec_bis = ecp.get_config_dict() self.assertTrue(fn in ec['sources']) self.assertFalse(fn in ec_bis['sources']) # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def get_format_version(txt): """Get the easyconfig format version as EasyVersion instance.""" res = FORMAT_VERSION_REGEXP.search(txt) format_version = None if res is not None: try: maj_min = res.groupdict() format_version = EasyVersion(FORMAT_VERSION_TEMPLATE % maj_min) except (KeyError, TypeError), err: raise EasyBuildError("Failed to get version from match %s: %s", res.groups(), err)
def test_v10(self): ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb')) self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0')) ec = ecp.get_config_dict() self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'}) self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.3')
class EasyConfigFormat(object): """EasyConfigFormat class""" VERSION = EasyVersion( '0.0') # dummy EasyVersion instance (shouldn't be None) USABLE = False # disable this class as usable format def __init__(self): """Initialise the EasyConfigFormat class""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) if not len(self.VERSION) == len(FORMAT_VERSION_TEMPLATE.split('.')): raise EasyBuildError( 'Invalid version number %s (incorrect length)', self.VERSION) self.rawtext = None # text version of the easyconfig self._comments = {} # comments in easyconfig file self.header = None # easyconfig header (e.g., format version, license, ...) self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...) self.specs = {} @property def comments(self): """Return comments in easyconfig file""" return self._comments def set_specifications(self, specs): """Set specifications.""" self.log.debug('Set copy of specs %s' % specs) self.specs = copy.deepcopy(specs) def get_config_dict(self): """Returns a single easyconfig dictionary.""" raise NotImplementedError def validate(self): """Verify the easyconfig format""" raise NotImplementedError def parse(self, txt, **kwargs): """Parse the txt according to this format. This is highly version specific""" raise NotImplementedError def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): """Dump easyconfig according to this format. This is highly version specific""" raise NotImplementedError def extract_comments(self, rawtxt): """Extract comments from raw content.""" raise NotImplementedError
def test_v20_extra(self): fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb') ecp = EasyConfigParser(fn) formatter = ecp._formatter self.assertEqual(formatter.VERSION, EasyVersion('2.0')) self.assertTrue('name' in formatter.pyheader_localvars) self.assertFalse('version' in formatter.pyheader_localvars) self.assertFalse('toolchain' in formatter.pyheader_localvars) self.assertRaises(NotImplementedError, ecp.get_config_dict)
class FormatOneZero(EasyConfigFormatConfigObj): """Support for easyconfig format 1.x""" VERSION = EasyVersion('1.0') USABLE = True # TODO: disable it at some point, too insecure PYHEADER_ALLOWED_BUILTINS = None # allow all PYHEADER_MANDATORY = [ 'version', 'name', 'toolchain', 'homepage', 'description' ] PYHEADER_BLACKLIST = [] def validate(self): """Format validation""" # minimal checks self._validate_pyheader() def get_config_dict(self): """ Return parsed easyconfig as a dictionary, based on specified arguments. This is easyconfig format 1.x, so there is only one easyconfig instance available. """ spec_version = self.specs.get('version', None) spec_tc = self.specs.get('toolchain', {}) spec_tc_name = spec_tc.get('name', None) spec_tc_version = spec_tc.get('version', None) cfg = self.pyheader_localvars if spec_version is not None and not spec_version == cfg['version']: self.log.error('Requested version %s not available, only %s' % (spec_version, cfg['version'])) tc_name = cfg['toolchain']['name'] tc_version = cfg['toolchain']['version'] if spec_tc_name is not None and not spec_tc_name == tc_name: self.log.error( 'Requested toolchain name %s not available, only %s' % (spec_tc_name, tc_name)) if spec_tc_version is not None and not spec_tc_version == tc_version: self.log.error( 'Requested toolchain version %s not available, only %s' % (spec_tc_version, tc_version)) return cfg def parse(self, txt): """ Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced. """ super(FormatOneZero, self).parse(txt, strict_section_markers=True)
def test_v20_extra(self): """Test parsing of easyconfig in format v2.""" # hard enable experimental orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb') ecp = EasyConfigParser(fn) formatter = ecp._formatter self.assertEqual(formatter.VERSION, EasyVersion('2.0')) self.assertTrue('name' in formatter.pyheader_localvars) self.assertFalse('version' in formatter.pyheader_localvars) self.assertFalse('toolchain' in formatter.pyheader_localvars) # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v10(self): ecp = EasyConfigParser( os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb')) self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0')) ec = ecp.get_config_dict() self.assertEqual(ec['toolchain'], { 'name': 'system', 'version': 'system' }) self.assertEqual(ec['name'], 'GCC') self.assertEqual(ec['version'], '4.6.3') # changes to this dict should not affect the return value of the next call to get_config_dict fn = 'test.tar.gz' ec['sources'].append(fn) ec_bis = ecp.get_config_dict() self.assertTrue(fn in ec['sources']) self.assertFalse(fn in ec_bis['sources'])
class EasyConfigFormat(object): """EasyConfigFormat class""" VERSION = EasyVersion( '0.0') # dummy EasyVersion instance (shouldn't be None) USABLE = False # disable this class as usable format def __init__(self): """Initialise the EasyConfigFormat class""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) if not len(self.VERSION) == len(FORMAT_VERSION_TEMPLATE.split('.')): self.log.error('Invalid version number %s (incorrect length)' % self.VERSION) self.rawtext = None # text version of the easyconfig self.header = None # easyconfig header (e.g., format version, license, ...) self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...) def get_config_dict(self, version=None, toolchain_name=None, toolchain_version=None): """Returns a single easyconfig dictionary.""" raise NotImplementedError def validate(self): """Verify the easyconfig format""" raise NotImplementedError def parse(self, txt, **kwargs): """Parse the txt according to this format. This is highly version specific""" raise NotImplementedError def dump(self): """Dump easyconfig according to this format. This is higly version specific""" raise NotImplementedError
class FormatOneZero(EasyConfigFormatConfigObj): """Support for easyconfig format 1.x""" VERSION = EasyVersion('1.0') USABLE = True # TODO: disable it at some point, too insecure PYHEADER_ALLOWED_BUILTINS = None # allow all PYHEADER_MANDATORY = [ 'version', 'name', 'toolchain', 'homepage', 'description' ] PYHEADER_BLACKLIST = [] def validate(self): """Format validation""" # minimal checks self._validate_pyheader() def get_config_dict(self): """ Return parsed easyconfig as a dictionary, based on specified arguments. This is easyconfig format 1.x, so there is only one easyconfig instance available. """ spec_version = self.specs.get('version', None) spec_tc = self.specs.get('toolchain', {}) spec_tc_name = spec_tc.get('name', None) spec_tc_version = spec_tc.get('version', None) cfg = self.pyheader_localvars if spec_version is not None and not spec_version == cfg['version']: raise EasyBuildError('Requested version %s not available, only %s', spec_version, cfg['version']) tc_name = cfg.get('toolchain', {}).get('name', None) tc_version = cfg.get('toolchain', {}).get('version', None) if spec_tc_name is not None and not spec_tc_name == tc_name: raise EasyBuildError( 'Requested toolchain name %s not available, only %s', spec_tc_name, tc_name) if spec_tc_version is not None and not spec_tc_version == tc_version: raise EasyBuildError( 'Requested toolchain version %s not available, only %s', spec_tc_version, tc_version) return cfg def parse(self, txt): """ Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced. """ super(FormatOneZero, self).parse(txt, strict_section_markers=True) def _reformat_line(self, param_name, param_val, outer=False, addlen=0): """ Construct formatted string representation of iterable parameter (list/tuple/dict), including comments. @param param_name: parameter name @param param_val: parameter value @param outer: reformat for top-level parameter, or not @param addlen: # characters to add to line length """ param_strval = str(param_val) res = param_strval # determine whether line would be too long # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) elif outer: # only reformat outer (iterable) values for (too) long lines (or for select parameters) if isinstance(param_val, (list, tuple, dict)) and ( (len(param_val) > 1 and line_too_long) or forced): item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n' # start with opening character: [, (, { res = '%s\n' % param_strval[0] # add items one-by-one, special care for dict values (order of keys, different format for elements) if isinstance(param_val, dict): ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get( param_name, sorted(param_val.keys())) for item_key in ordered_item_keys: item_val = param_val[item_key] comment = self._get_item_comments( param_name, item_val).get(str(item_val), '') key_pref = quote_py_str(item_key) + ': ' addlen = addlen + len(INDENT_4SPACES) + len( key_pref) + len(comment) formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen) res += item_tmpl % { 'comment': comment, 'item': key_pref + formatted_item_val, } else: # list, tuple for item in param_val: comment = self._get_item_comments(param_name, item).get( str(item), '') addlen = addlen + len(INDENT_4SPACES) + len(comment) res += item_tmpl % { 'comment': comment, 'item': self._reformat_line( param_name, item, addlen=addlen) } # end with closing character: ], ), } res += param_strval[-1] else: # dependencies are already dumped as strings, so they do not need to be quoted again if isinstance( param_val, basestring) and param_name not in DEPENDENCY_PARAMETERS: res = quote_py_str(param_val) return res def _get_item_comments(self, key, val): """Get per-item comments for specified parameter name/value.""" item_comments = {} for comment_key, comment_val in self.comments['iter'].get(key, {}).items(): if str(val) in comment_key: item_comments[str(val)] = comment_val return item_comments def _find_param_with_comments(self, key, val, templ_const, templ_val): """Find parameter definition and accompanying comments, to include in dumped easyconfig file.""" res = [] val = self._reformat_line(key, val, outer=True) # templates if key not in EXCLUDED_KEYS_REPLACE_TEMPLATES: new_val = to_template_str(val, templ_const, templ_val) # avoid self-referencing templated parameter definitions if not r'%(' + key in new_val: val = new_val if key in self.comments['inline']: res.append("%s = %s%s" % (key, val, self.comments['inline'][key])) else: if key in self.comments['above']: res.extend(self.comments['above'][key]) res.append("%s = %s" % (key, val)) return res def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ eclines = [] printed_keys = [] for group in keyset: printed = False for key in group: val = copy.deepcopy(ecfg[key]) # include hidden deps back in list of (build)dependencies, they were filtered out via filter_hidden_deps if key == 'dependencies': val.extend([ d for d in ecfg['hiddendependencies'] if not d['build_only'] ]) elif key == 'builddependencies': val.extend([ d for d in ecfg['hiddendependencies'] if d['build_only'] ]) if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them if key in DEPENDENCY_PARAMETERS: valstr = [ dump_dependency(d, ecfg['toolchain']) for d in val ] else: valstr = quote_py_str(ecfg[key]) eclines.extend( self._find_param_with_comments(key, valstr, templ_const, templ_val)) printed_keys.append(key) printed = True if printed: eclines.append('') return eclines, printed_keys def dump(self, ecfg, default_values, templ_const, templ_val): """ Dump easyconfig in format v1. @param ecfg: EasyConfig instance @param default_values: default values for easyconfig parameters @param templ_const: known template constants @param templ_val: known template values """ # include header comments first dump = self.comments['header'][:] # print easyconfig parameters ordered and in groups specified above params, printed_keys = self._find_defined_params( ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val) dump.extend(params) # print other easyconfig parameters at the end keys_to_ignore = printed_keys + LAST_PARAMS for key in default_values: if key not in keys_to_ignore and ecfg[key] != default_values[key]: dump.extend( self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val)) dump.append('') # print last parameters params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val) dump.extend(params) dump.extend(self.comments['tail']) return '\n'.join(dump) def extract_comments(self, rawtxt): """ Extract comments from raw content. Discriminates between comment header, comments above a line (parameter definition), and inline comments. Inline comments on items of iterable values are also extracted. """ self.comments = { 'above': {}, # comments for a particular parameter definition 'header': [], # header comment lines 'inline': {}, # inline comments 'iter': {}, # (inline) comments on elements of iterable values 'tail': [], } rawlines = rawtxt.split('\n') # extract header first while rawlines[0].startswith('#'): self.comments['header'].append(rawlines.pop(0)) parsed_ec = self.get_config_dict() while rawlines: rawline = rawlines.pop(0) if rawline.startswith('#'): comment = [] # comment could be multi-line while rawline is not None and (rawline.startswith('#') or not rawline): # drop empty lines (that don't even include a #) if rawline: comment.append(rawline) # grab next line (if more lines are left) if rawlines: rawline = rawlines.pop(0) else: rawline = None if rawline is None: self.comments['tail'] = comment else: key = rawline.split('=', 1)[0].strip() self.comments['above'][key] = comment elif '#' in rawline: # inline comment comment_key, comment_val = None, None comment = rawline.rsplit('#', 1)[1].strip() # check whether this line is parameter definition; # if not, assume it's a continuation of a multi-line value if re.match(r'^[a-z_]+\s*=', rawline): comment_key = rawline.split('=', 1)[0].strip() else: # determine parameter value where the item value on this line is a part of for key, val in parsed_ec.items(): item_val = re.sub(r',$', r'', rawline.rsplit('#', 1)[0].strip()) if not isinstance(val, basestring) and item_val in str(val): comment_key, comment_val = key, item_val break # check if hash actually indicated a comment; or is part of the value if comment_key in parsed_ec: if comment.replace("'", '').replace('"', '') not in str( parsed_ec[comment_key]): if comment_val: self.comments['iter'].setdefault( comment_key, {})[comment_val] = ' # ' + comment else: self.comments['inline'][ comment_key] = ' # ' + comment
# split into blocks using regex pieces = reg_block.split(txt) # the first block contains common statements common = pieces.pop(0) # determine version of easyconfig format ec_format_version = get_format_version(txt) if ec_format_version is None: ec_format_version = FORMAT_DEFAULT_VERSION _log.debug( "retrieve_blocks_in_spec: derived easyconfig format version: %s" % ec_format_version) # blocks in easyconfigs are only supported in easyconfig format 1.0 if pieces and ec_format_version == EasyVersion('1.0'): # make a map of blocks blocks = [] while pieces: block_name = pieces.pop(0) block_contents = pieces.pop(0) if block_name in [b['name'] for b in blocks]: raise EasyBuildError("Found block %s twice in %s.", block_name, spec) block = {'name': block_name, 'contents': block_contents} # dependency block dep_block = reg_dep_block.search(block_contents) if dep_block:
# split into blocks using regex pieces = reg_block.split(txt) # the first block contains common statements common = pieces.pop(0) # determine version of easyconfig format ec_format_version = get_format_version(txt) if ec_format_version is None: ec_format_version = FORMAT_DEFAULT_VERSION _log.debug( "retrieve_blocks_in_spec: derived easyconfig format version: %s" % ec_format_version) # blocks in easyconfigs are only supported in format versions prior to 2.0 if pieces and ec_format_version < EasyVersion('2.0'): # make a map of blocks blocks = [] while pieces: block_name = pieces.pop(0) block_contents = pieces.pop(0) if block_name in [b['name'] for b in blocks]: raise EasyBuildError("Found block %s twice in %s.", block_name, spec) block = {'name': block_name, 'contents': block_contents} # dependency block dep_block = reg_dep_block.search(block_contents) if dep_block:
from easybuild.framework.easyconfig.format.version import ToolchainVersionOperator, VersionOperator from easybuild.framework.easyconfig.format.convert import Dependency from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import Section INDENT_4SPACES = ' ' * 4 # format is mandatory major.minor FORMAT_VERSION_KEYWORD = "EASYCONFIGFORMAT" FORMAT_VERSION_TEMPLATE = "%(major)s.%(minor)s" FORMAT_VERSION_HEADER_TEMPLATE = "# %s %s\n" % ( FORMAT_VERSION_KEYWORD, FORMAT_VERSION_TEMPLATE) # must end in newline FORMAT_VERSION_REGEXP = re.compile( r'^#\s+%s\s*(?P<major>\d+)\.(?P<minor>\d+)\s*$' % FORMAT_VERSION_KEYWORD, re.M) FORMAT_DEFAULT_VERSION = EasyVersion('1.0') DEPENDENCY_PARAMETERS = [ 'builddependencies', 'dependencies', 'hiddendependencies' ] # values for these keys will not be templated in dump() EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'homepage', 'name', 'toolchain', 'version'] \ + DEPENDENCY_PARAMETERS # ordered groups of keys to obtain a nice looking easyconfig file GROUPED_PARAMS = [ ['easyblock'], ['name', 'version', 'versionprefix', 'versionsuffix'], ['homepage', 'description'], ['toolchain', 'toolchainopts'],
def retrieve_blocks_in_spec(spec, only_blocks, silent=False): """ Easyconfigs can contain blocks (headed by a [Title]-line) which contain commands specific to that block. Commands in the beginning of the file above any block headers are common and shared between each block. """ reg_block = re.compile(r"^\s*\[([\w.-]+)\]\s*$", re.M) reg_dep_block = re.compile(r"^\s*block\s*=(\s*.*?)\s*$", re.M) spec_fn = os.path.basename(spec) txt = read_file(spec) # split into blocks using regex pieces = reg_block.split(txt) # the first block contains common statements common = pieces.pop(0) # determine version of easyconfig format ec_format_version = get_format_version(txt) if ec_format_version is None: ec_format_version = FORMAT_DEFAULT_VERSION _log.debug( "retrieve_blocks_in_spec: derived easyconfig format version: %s" % ec_format_version) # blocks in easyconfigs are only supported in easyconfig format 1.0 if pieces and ec_format_version == EasyVersion('1.0'): # make a map of blocks blocks = [] while pieces: block_name = pieces.pop(0) block_contents = pieces.pop(0) if block_name in [b['name'] for b in blocks]: raise EasyBuildError("Found block %s twice in %s.", block_name, spec) block = {'name': block_name, 'contents': block_contents} # dependency block dep_block = reg_dep_block.search(block_contents) if dep_block: dependencies = eval(dep_block.group(1)) if type(dependencies) == list: block['dependencies'] = dependencies else: block['dependencies'] = [dependencies] blocks.append(block) # make a new easyconfig for each block # they will be processed in the same order as they are all described in the original file specs = [] for block in blocks: name = block['name'] if only_blocks and not (name in only_blocks): print_msg("Skipping block %s-%s" % (spec_fn, name), silent=silent) continue (fd, block_path) = tempfile.mkstemp(prefix='easybuild-', suffix='%s-%s' % (spec_fn, name)) os.close(fd) txt = common if 'dependencies' in block: for dep in block['dependencies']: if dep not in [b['name'] for b in blocks]: raise EasyBuildError( "Block %s depends on %s, but block was not found.", name, dep) dep = [b for b in blocks if b['name'] == dep][0] txt += "\n# Dependency block %s" % (dep['name']) txt += dep['contents'] txt += "\n# Main block %s" % name txt += block['contents'] write_file(block_path, txt) specs.append(block_path) _log.debug("Found %s block(s) in %s" % (len(specs), spec)) return specs else: # no blocks, one file return [spec]
class FormatTwoZero(EasyConfigFormatConfigObj): """ Support for easyconfig format 2.0 Simple extension of FormatOneZero with configparser blocks Doesn't set version and toolchain/toolchain version like in FormatOneZero; referencing 'version' directly in pyheader doesn't work => use templating '%(version)s' NOT in 2.0 - order preservation: need more recent ConfigObj (more recent Python as minimal version) - nested sections (need other ConfigParser/ConfigObj, eg INITools) - type validation - command line generation (--try-X command line options) """ VERSION = EasyVersion('2.0') USABLE = True PYHEADER_ALLOWED_BUILTINS = ['len', 'False', 'True'] PYHEADER_MANDATORY = ['name', 'homepage', 'description', 'software_license', 'software_license_urls', 'docurls'] PYHEADER_BLACKLIST = ['version', 'toolchain'] NAME_DOCSTRING_REGEX_TEMPLATE = r'^\s*@%s\s*:\s*(?P<name>\S.*?)\s*$' # non-greedy match in named pattern AUTHOR_DOCSTRING_REGEX = re.compile(NAME_DOCSTRING_REGEX_TEMPLATE % 'author', re.M) MAINTAINER_DOCSTRING_REGEX = re.compile(NAME_DOCSTRING_REGEX_TEMPLATE % 'maintainer', re.M) AUTHOR_REQUIRED = True MAINTAINER_REQUIRED = False def validate(self): """Format validation""" self._check_docstring() self._validate_pyheader() def _check_docstring(self): """ Verify docstring. field :author: people who contributed to the easyconfig field @maintainer: people who can be contacted in case of problems """ authors = [] maintainers = [] for auth_reg in self.AUTHOR_DOCSTRING_REGEX.finditer(self.docstring): res = auth_reg.groupdict() authors.append(res['name']) for maint_reg in self.MAINTAINER_DOCSTRING_REGEX.finditer(self.docstring): res = maint_reg.groupdict() maintainers.append(res['name']) if self.AUTHOR_REQUIRED and not authors: raise EasyBuildError("No author in docstring (regex: '%s')", self.AUTHOR_DOCSTRING_REGEX.pattern) if self.MAINTAINER_REQUIRED and not maintainers: raise EasyBuildError("No maintainer in docstring (regex: '%s')", self.MAINTAINER_DOCSTRING_REGEX.pattern) def get_config_dict(self): """Return the best matching easyconfig dict""" self.log.experimental(self.__class__.__name__) # the toolchain name/version should not be specified in the pyheader, # but other toolchain options are allowed cfg = copy.deepcopy(self.pyheader_localvars) self.log.debug("Config dict based on Python header: %s" % cfg) co = EBConfigObj(self.configobj) version = self.specs.get('version', None) tc_spec = self.specs.get('toolchain', {}) toolchain_name = tc_spec.get('name', None) toolchain_version = tc_spec.get('version', None) # parse and interpret, dealing with defaults etc version, tcname, tcversion = co.get_version_toolchain(version, toolchain_name, toolchain_version) # format 2.0 will squash self.log.debug('Squashing with version %s and toolchain %s' % (version, (tcname, tcversion))) res = co.squash(version, tcname, tcversion) cfg.update(res) self.log.debug("Config dict after processing applicable easyconfig sections: %s" % cfg) # FIXME what about updating dict values/appending to list values? # FIXME how do we allow both redefining and updating? = and +=? # update config with correct version/toolchain (to avoid using values specified in default section) cfg.update({ 'version': version, 'toolchain': {'name': tcname, 'version': tcversion}, }) self.log.debug("Final config dict (including correct version/toolchain): %s" % cfg) return cfg def extract_comments(self, rawtxt): """Extract comments from raw content.""" # this is fine-ish, it only implies that comments will be lost for format v2 easyconfig files that are dumped self.log.warning("Extraction of comments not supported yet for easyconfig format v2")
class FormatOneZero(EasyConfigFormatConfigObj): """Support for easyconfig format 1.x""" VERSION = EasyVersion('1.0') USABLE = True # TODO: disable it at some point, too insecure PYHEADER_ALLOWED_BUILTINS = None # allow all PYHEADER_MANDATORY = [ 'version', 'name', 'toolchain', 'homepage', 'description' ] PYHEADER_BLACKLIST = [] def __init__(self, *args, **kwargs): """FormatOneZero constructor.""" super(FormatOneZero, self).__init__(*args, **kwargs) self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.strict_sanity_check_paths_keys = True def validate(self): """Format validation""" # minimal checks self._validate_pyheader() def get_config_dict(self): """ Return parsed easyconfig as a dictionary, based on specified arguments. This is easyconfig format 1.x, so there is only one easyconfig instance available. """ spec_version = self.specs.get('version', None) spec_tc = self.specs.get('toolchain', {}) spec_tc_name = spec_tc.get('name', None) spec_tc_version = spec_tc.get('version', None) cfg = self.pyheader_localvars if spec_version is not None and not spec_version == cfg['version']: raise EasyBuildError('Requested version %s not available, only %s', spec_version, cfg['version']) tc_name = cfg.get('toolchain', {}).get('name', None) tc_version = cfg.get('toolchain', {}).get('version', None) if spec_tc_name is not None and not spec_tc_name == tc_name: raise EasyBuildError( 'Requested toolchain name %s not available, only %s', spec_tc_name, tc_name) if spec_tc_version is not None and not spec_tc_version == tc_version: raise EasyBuildError( 'Requested toolchain version %s not available, only %s', spec_tc_version, tc_version) # avoid passing anything by reference, so next time get_config_dict is called # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file; # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails... cfg_copy = {} for key in cfg: if key != '__builtins__': cfg_copy[key] = copy.deepcopy(cfg[key]) return cfg_copy def parse(self, txt): """ Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced. """ self.rawcontent = txt super(FormatOneZero, self).parse(self.rawcontent, strict_section_markers=True) def _reformat_line(self, param_name, param_val, outer=False, addlen=0): """ Construct formatted string representation of iterable parameter (list/tuple/dict), including comments. :param param_name: parameter name :param param_val: parameter value :param outer: reformat for top-level parameter, or not :param addlen: # characters to add to line length """ param_strval = str(param_val) res = param_strval # determine whether line would be too long # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS list_of_lists_of_tuples_param = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) elif outer: # only reformat outer (iterable) values for (too) long lines (or for select parameters) if isinstance(param_val, (list, tuple, dict)) and ( (len(param_val) > 1 or line_too_long) or forced): item_tmpl = INDENT_4SPACES + '%(item)s,%(inline_comment)s\n' start_char, end_char = param_strval[0], param_strval[-1] # start with opening character: [, (, { res = '%s\n' % start_char # add items one-by-one, special care for dict values (order of keys, different format for elements) if isinstance(param_val, dict): ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get( param_name, sorted(param_val.keys())) for item_key in ordered_item_keys: if item_key in param_val: item_val = param_val[item_key] item_comments = self._get_item_comments( param_name, item_val) elif param_name == 'sanity_check_paths' and not self.strict_sanity_check_paths_keys: item_val = [] item_comments = {} self.log.info( "Using default value for '%s' in sanity_check_paths: %s", item_key, item_val) else: raise EasyBuildError( "Missing mandatory key '%s' in %s.", item_key, param_name) inline_comment = item_comments.get('inline', '') item_tmpl_dict = {'inline_comment': inline_comment} for comment in item_comments.get('above', []): res += INDENT_4SPACES + comment + '\n' key_pref = quote_py_str(item_key) + ': ' addlen = addlen + len(INDENT_4SPACES) + len( key_pref) + len(inline_comment) formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen) item_tmpl_dict['item'] = key_pref + formatted_item_val res += item_tmpl % item_tmpl_dict else: # list, tuple for item in param_val: item_comments = self._get_item_comments( param_name, item) inline_comment = item_comments.get('inline', '') item_tmpl_dict = {'inline_comment': inline_comment} for comment in item_comments.get('above', []): res += INDENT_4SPACES + comment + '\n' addlen = addlen + len(INDENT_4SPACES) + len( inline_comment) # the tuples are really strings here that are constructed from the dependency dicts # so for a plain list of builddependencies param_val is a list of strings here; # and for iterated builddependencies it is a list of lists of strings is_list_of_lists_of_tuples = isinstance( item, list) and all( isinstance(x, str) for x in item) if list_of_lists_of_tuples_param and is_list_of_lists_of_tuples: itemstr = '[' + (',\n ' + INDENT_4SPACES).join([ self._reformat_line(param_name, subitem, outer=True, addlen=addlen) for subitem in item ]) + ']' else: itemstr = self._reformat_line(param_name, item, addlen=addlen) item_tmpl_dict['item'] = itemstr res += item_tmpl % item_tmpl_dict # take into account possible closing comments # see https://github.com/easybuilders/easybuild-framework/issues/3082 end_comments = self._get_item_comments(param_name, end_char) for comment in end_comments.get('above', []): res += INDENT_4SPACES + comment + '\n' # end with closing character (']', ')', '}'), incl. possible inline comment res += end_char if 'inline' in end_comments: res += end_comments['inline'] else: # dependencies are already dumped as strings, so they do not need to be quoted again if isinstance( param_val, string_type) and param_name not in DEPENDENCY_PARAMETERS: res = quote_py_str(param_val) return res def _get_item_comments(self, key, val): """Get per-item comments for specified parameter name/value.""" item_comments = {} for comment_key, comment_val in self.comments['iterabove'].get( key, {}).items(): if str(val) in comment_key: item_comments['above'] = comment_val for comment_key, comment_val in self.comments['iterinline'].get( key, {}).items(): if str(val) in comment_key: item_comments['inline'] = comment_val return item_comments def _find_param_with_comments(self, key, val, templ_const, templ_val): """Find parameter definition and accompanying comments, to include in dumped easyconfig file.""" res = [] val = self._reformat_line(key, val, outer=True) # templates if key not in EXCLUDED_KEYS_REPLACE_TEMPLATES: val = to_template_str(key, val, templ_const, templ_val) if key in self.comments['inline']: res.append("%s = %s%s" % (key, val, self.comments['inline'][key])) else: if key in self.comments['above']: res.extend(self.comments['above'][key]) res.append("%s = %s" % (key, val)) return res def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val, toolchain_hierarchy=None): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ eclines = [] printed_keys = [] for group in keyset: printed = False for key in group: val = ecfg[key] if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them; # take into account that these parameters may be iterative (i.e. a list of lists of parsed deps) if key in DEPENDENCY_PARAMETERS: if key in ecfg.iterate_options: if 'multi_deps' in ecfg: # the way that builddependencies are constructed with multi_deps # we just need to dump the first entry without the dependencies # that are listed in multi_deps valstr = [ dump_dependency( d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy ) for d in val[0] if d['name'] not in ecfg['multi_deps'] ] else: valstr = [[ dump_dependency( d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy ) for d in dep ] for dep in val] else: valstr = [ dump_dependency( d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy) for d in val ] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[ key] else: valstr = quote_py_str(ecfg[key]) eclines.extend( self._find_param_with_comments(key, valstr, templ_const, templ_val)) printed_keys.append(key) printed = True if printed: eclines.append('') return eclines, printed_keys def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): """ Dump easyconfig in format v1. :param ecfg: EasyConfig instance :param default_values: default values for easyconfig parameters :param templ_const: known template constants :param templ_val: known template values :param toolchain_hierarchy: hierarchy of toolchains for easyconfig """ # figoure out whether we should be strict about the format of sanity_check_paths; # if enhance_sanity_check is set, then both files/dirs keys are not strictly required... self.strict_sanity_check_paths_keys = not ecfg['enhance_sanity_check'] # include header comments first dump = self.comments['header'][:] # print easyconfig parameters ordered and in groups specified above params, printed_keys = self._find_defined_params( ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val, toolchain_hierarchy=toolchain_hierarchy) dump.extend(params) # print other easyconfig parameters at the end keys_to_ignore = printed_keys + LAST_PARAMS for key in default_values: mandatory = ecfg.is_mandatory_param(key) non_default_value = ecfg[key] != default_values[key] if key not in keys_to_ignore and (mandatory or non_default_value): dump.extend( self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val)) dump.append('') # print last parameters params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val) dump.extend(params) dump.extend(self.comments['tail']) return '\n'.join(dump) @property def comments(self): """ Return comments (and extract them first if needed). """ if not self._comments: self.extract_comments(self.rawcontent) return self._comments def extract_comments(self, rawtxt): """ Extract comments from raw content. Discriminates between comment header, comments above a line (parameter definition), and inline comments. Inline comments on items of iterable values are also extracted. """ self._comments = { 'above': {}, # comments above a parameter definition 'header': [], # header comment lines 'inline': {}, # inline comments 'iterabove': {}, # comment above elements of iterable values 'iterinline': {}, # inline comments on elements of iterable values 'tail': [], # comment at the end of the easyconfig file } parsed_ec = self.get_config_dict() comment_regex = re.compile(r'^\s*#') param_def_regex = re.compile(r'^([a-z_0-9]+)\s*=') whitespace_regex = re.compile(r'^\s*$') def clean_part(part): """Helper function to strip off trailing whitespace + trailing quotes.""" return part.rstrip().rstrip("'").rstrip('"') def split_on_comment_hash(line, param_key): """Helper function to split line on first (actual) comment character '#'.""" # string representation of easyconfig parameter value, # used to check if supposed comment isn't actual part of the parameter value # (and thus not actually a comment at all) param_strval = str(parsed_ec.get(param_key)) parts = line.split('#') # first part (before first #) is definitely not part of comment before_comment = parts.pop(0) # strip out parts that look like a comment but are actually part of a parameter value while parts and ('#' + clean_part(parts[0])) in param_strval: before_comment += '#' + parts.pop(0) comment = '#'.join(parts) return before_comment, comment.strip() def grab_more_comment_lines(lines, param_key): """Grab more comment lines.""" comment_lines = [] while lines and (comment_regex.match(lines[0]) or whitespace_regex.match(lines[0])): line = lines.pop(0) _, actual_comment = split_on_comment_hash(line, param_key) # prefix comment with '#' unless line was empty if line.strip(): actual_comment = '# ' + actual_comment comment_lines.append(actual_comment.strip()) return comment_lines rawlines = rawtxt.split('\n') # extract header first (include empty lines too) self.comments['header'] = grab_more_comment_lines(rawlines, None) last_param_key = None while rawlines: rawline = rawlines.pop(0) # keep track of last parameter definition we have seen, # current line may be (the start of) a parameter definition res = param_def_regex.match(rawline) if res: key = res.group(1) if key in parsed_ec: last_param_key = key if last_param_key: before_comment, inline_comment = split_on_comment_hash( rawline, last_param_key) # short-circuit to next line in case there are no actual comments on this (non-empty) line if before_comment and not inline_comment: continue # lines that start with a hash indicate (start of a block of) comment line(s) if rawline.startswith('#'): comment = [rawline] + grab_more_comment_lines( rawlines, last_param_key) if rawlines: # try to pin comment to parameter definition below it # don't consume the line yet though, it may also include inline comments... res = param_def_regex.match(rawlines[0]) if res: last_param_key = res.group(1) self.comments['above'][last_param_key] = comment else: # if the comment is not above a parameter definition, # then it must be a comment for an item of an iterable parameter value before_comment, _ = split_on_comment_hash( rawlines[0], last_param_key) comment_key = before_comment.rstrip() self.comments['iterabove'].setdefault( last_param_key, {})[comment_key] = comment else: # if there are no more lines, the comment (block) is at the tail self.comments['tail'] = comment elif '#' in rawline: # if there's a hash character elsewhere in the line (not at the start), # there are a couple of possibilities: # - inline comment for a parameter definition (at the end of a non-empty line) # - indented comment for an item value of an iterable easyconfig parameter (list, dict, ...) # - inline comment for an item value of an iterable easyconfig parameter before_comment, comment = split_on_comment_hash( rawline, last_param_key) comment = ('# ' + comment).rstrip() # first check whether current line is an easyconfig parameter definition # if so, the comment is an inline comment if param_def_regex.match(before_comment): self.comments['inline'][last_param_key] = ' ' + comment # if there's only whitespace before the comment, # then we have an indented comment, and we need to figure out for what exactly elif whitespace_regex.match(before_comment): # first consume possible additional comment lines with same indentation comment = [comment] + grab_more_comment_lines( rawlines, last_param_key) before_comment, inline_comment = split_on_comment_hash( rawlines.pop(0), last_param_key) comment_key = before_comment.rstrip() self.comments['iterabove'].setdefault( last_param_key, {})[comment_key] = comment if inline_comment: inline_comment = (' # ' + inline_comment).rstrip() self.comments['iterinline'].setdefault( last_param_key, {})[comment_key] = inline_comment else: # inline comment for item of iterable value comment_key = before_comment.rstrip() self.comments['iterinline'].setdefault( last_param_key, {})[comment_key] = ' ' + comment self.log.debug("Extracted comments:\n%s", pprint.pformat(self.comments, width=120))