def test_v20(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
# this should be ok: ie the default values
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {
'name': 'dummy',
'version': 'dummy'
})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.2')
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v20(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
# this should be ok: ie the default values
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {
'name': 'system',
'version': 'system'
})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.2')
# changes to this dict should not affect the return value of the next call to get_config_dict
fn = 'test.tar.gz'
ec['sources'].append(fn)
ec_bis = ecp.get_config_dict()
self.assertTrue(fn in ec['sources'])
self.assertFalse(fn in ec_bis['sources'])
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def get_format_version(txt):
"""Get the easyconfig format version as EasyVersion instance."""
res = FORMAT_VERSION_REGEXP.search(txt)
format_version = None
if res is not None:
try:
maj_min = res.groupdict()
format_version = EasyVersion(FORMAT_VERSION_TEMPLATE % maj_min)
except (KeyError, TypeError), err:
raise EasyBuildError("Failed to get version from match %s: %s", res.groups(), err)
def test_v10(self):
ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb'))
self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.3')
class EasyConfigFormat(object):
"""EasyConfigFormat class"""
VERSION = EasyVersion(
'0.0') # dummy EasyVersion instance (shouldn't be None)
USABLE = False # disable this class as usable format
def __init__(self):
"""Initialise the EasyConfigFormat class"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
if not len(self.VERSION) == len(FORMAT_VERSION_TEMPLATE.split('.')):
raise EasyBuildError(
'Invalid version number %s (incorrect length)', self.VERSION)
self.rawtext = None # text version of the easyconfig
self._comments = {} # comments in easyconfig file
self.header = None # easyconfig header (e.g., format version, license, ...)
self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...)
self.specs = {}
@property
def comments(self):
"""Return comments in easyconfig file"""
return self._comments
def set_specifications(self, specs):
"""Set specifications."""
self.log.debug('Set copy of specs %s' % specs)
self.specs = copy.deepcopy(specs)
def get_config_dict(self):
"""Returns a single easyconfig dictionary."""
raise NotImplementedError
def validate(self):
"""Verify the easyconfig format"""
raise NotImplementedError
def parse(self, txt, **kwargs):
"""Parse the txt according to this format. This is highly version specific"""
raise NotImplementedError
def dump(self,
ecfg,
default_values,
templ_const,
templ_val,
toolchain_hierarchy=None):
"""Dump easyconfig according to this format. This is highly version specific"""
raise NotImplementedError
def extract_comments(self, rawtxt):
"""Extract comments from raw content."""
raise NotImplementedError
def test_v20_extra(self):
fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
self.assertRaises(NotImplementedError, ecp.get_config_dict)
class FormatOneZero(EasyConfigFormatConfigObj):
"""Support for easyconfig format 1.x"""
VERSION = EasyVersion('1.0')
USABLE = True # TODO: disable it at some point, too insecure
PYHEADER_ALLOWED_BUILTINS = None # allow all
PYHEADER_MANDATORY = [
'version', 'name', 'toolchain', 'homepage', 'description'
]
PYHEADER_BLACKLIST = []
def validate(self):
"""Format validation"""
# minimal checks
self._validate_pyheader()
def get_config_dict(self):
"""
Return parsed easyconfig as a dictionary, based on specified arguments.
This is easyconfig format 1.x, so there is only one easyconfig instance available.
"""
spec_version = self.specs.get('version', None)
spec_tc = self.specs.get('toolchain', {})
spec_tc_name = spec_tc.get('name', None)
spec_tc_version = spec_tc.get('version', None)
cfg = self.pyheader_localvars
if spec_version is not None and not spec_version == cfg['version']:
self.log.error('Requested version %s not available, only %s' %
(spec_version, cfg['version']))
tc_name = cfg['toolchain']['name']
tc_version = cfg['toolchain']['version']
if spec_tc_name is not None and not spec_tc_name == tc_name:
self.log.error(
'Requested toolchain name %s not available, only %s' %
(spec_tc_name, tc_name))
if spec_tc_version is not None and not spec_tc_version == tc_version:
self.log.error(
'Requested toolchain version %s not available, only %s' %
(spec_tc_version, tc_version))
return cfg
def parse(self, txt):
"""
Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced.
"""
super(FormatOneZero, self).parse(txt, strict_section_markers=True)
def test_v20_extra(self):
"""Test parsing of easyconfig in format v2."""
# hard enable experimental
orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
easybuild.tools.build_log.EXPERIMENTAL = True
fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb')
ecp = EasyConfigParser(fn)
formatter = ecp._formatter
self.assertEqual(formatter.VERSION, EasyVersion('2.0'))
self.assertTrue('name' in formatter.pyheader_localvars)
self.assertFalse('version' in formatter.pyheader_localvars)
self.assertFalse('toolchain' in formatter.pyheader_localvars)
# restore
easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_v10(self):
ecp = EasyConfigParser(
os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb'))
self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))
ec = ecp.get_config_dict()
self.assertEqual(ec['toolchain'], {
'name': 'system',
'version': 'system'
})
self.assertEqual(ec['name'], 'GCC')
self.assertEqual(ec['version'], '4.6.3')
# changes to this dict should not affect the return value of the next call to get_config_dict
fn = 'test.tar.gz'
ec['sources'].append(fn)
ec_bis = ecp.get_config_dict()
self.assertTrue(fn in ec['sources'])
self.assertFalse(fn in ec_bis['sources'])
class EasyConfigFormat(object):
"""EasyConfigFormat class"""
VERSION = EasyVersion(
'0.0') # dummy EasyVersion instance (shouldn't be None)
USABLE = False # disable this class as usable format
def __init__(self):
"""Initialise the EasyConfigFormat class"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
if not len(self.VERSION) == len(FORMAT_VERSION_TEMPLATE.split('.')):
self.log.error('Invalid version number %s (incorrect length)' %
self.VERSION)
self.rawtext = None # text version of the easyconfig
self.header = None # easyconfig header (e.g., format version, license, ...)
self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...)
def get_config_dict(self,
version=None,
toolchain_name=None,
toolchain_version=None):
"""Returns a single easyconfig dictionary."""
raise NotImplementedError
def validate(self):
"""Verify the easyconfig format"""
raise NotImplementedError
def parse(self, txt, **kwargs):
"""Parse the txt according to this format. This is highly version specific"""
raise NotImplementedError
def dump(self):
"""Dump easyconfig according to this format. This is higly version specific"""
raise NotImplementedError
class FormatOneZero(EasyConfigFormatConfigObj):
"""Support for easyconfig format 1.x"""
VERSION = EasyVersion('1.0')
USABLE = True # TODO: disable it at some point, too insecure
PYHEADER_ALLOWED_BUILTINS = None # allow all
PYHEADER_MANDATORY = [
'version', 'name', 'toolchain', 'homepage', 'description'
]
PYHEADER_BLACKLIST = []
def validate(self):
"""Format validation"""
# minimal checks
self._validate_pyheader()
def get_config_dict(self):
"""
Return parsed easyconfig as a dictionary, based on specified arguments.
This is easyconfig format 1.x, so there is only one easyconfig instance available.
"""
spec_version = self.specs.get('version', None)
spec_tc = self.specs.get('toolchain', {})
spec_tc_name = spec_tc.get('name', None)
spec_tc_version = spec_tc.get('version', None)
cfg = self.pyheader_localvars
if spec_version is not None and not spec_version == cfg['version']:
raise EasyBuildError('Requested version %s not available, only %s',
spec_version, cfg['version'])
tc_name = cfg.get('toolchain', {}).get('name', None)
tc_version = cfg.get('toolchain', {}).get('version', None)
if spec_tc_name is not None and not spec_tc_name == tc_name:
raise EasyBuildError(
'Requested toolchain name %s not available, only %s',
spec_tc_name, tc_name)
if spec_tc_version is not None and not spec_tc_version == tc_version:
raise EasyBuildError(
'Requested toolchain version %s not available, only %s',
spec_tc_version, tc_version)
return cfg
def parse(self, txt):
"""
Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced.
"""
super(FormatOneZero, self).parse(txt, strict_section_markers=True)
def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
"""
Construct formatted string representation of iterable parameter (list/tuple/dict), including comments.
@param param_name: parameter name
@param param_val: parameter value
@param outer: reformat for top-level parameter, or not
@param addlen: # characters to add to line length
"""
param_strval = str(param_val)
res = param_strval
# determine whether line would be too long
# note: this does not take into account the parameter name + '=', only the value
line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH
forced = param_name in REFORMAT_FORCED_PARAMS
if param_name in REFORMAT_SKIPPED_PARAMS:
self.log.info("Skipping reformatting value for parameter '%s'",
param_name)
elif outer:
# only reformat outer (iterable) values for (too) long lines (or for select parameters)
if isinstance(param_val, (list, tuple, dict)) and (
(len(param_val) > 1 and line_too_long) or forced):
item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n'
# start with opening character: [, (, {
res = '%s\n' % param_strval[0]
# add items one-by-one, special care for dict values (order of keys, different format for elements)
if isinstance(param_val, dict):
ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(
param_name, sorted(param_val.keys()))
for item_key in ordered_item_keys:
item_val = param_val[item_key]
comment = self._get_item_comments(
param_name, item_val).get(str(item_val), '')
key_pref = quote_py_str(item_key) + ': '
addlen = addlen + len(INDENT_4SPACES) + len(
key_pref) + len(comment)
formatted_item_val = self._reformat_line(param_name,
item_val,
addlen=addlen)
res += item_tmpl % {
'comment': comment,
'item': key_pref + formatted_item_val,
}
else: # list, tuple
for item in param_val:
comment = self._get_item_comments(param_name,
item).get(
str(item), '')
addlen = addlen + len(INDENT_4SPACES) + len(comment)
res += item_tmpl % {
'comment':
comment,
'item':
self._reformat_line(
param_name, item, addlen=addlen)
}
# end with closing character: ], ), }
res += param_strval[-1]
else:
# dependencies are already dumped as strings, so they do not need to be quoted again
if isinstance(
param_val,
basestring) and param_name not in DEPENDENCY_PARAMETERS:
res = quote_py_str(param_val)
return res
def _get_item_comments(self, key, val):
"""Get per-item comments for specified parameter name/value."""
item_comments = {}
for comment_key, comment_val in self.comments['iter'].get(key,
{}).items():
if str(val) in comment_key:
item_comments[str(val)] = comment_val
return item_comments
def _find_param_with_comments(self, key, val, templ_const, templ_val):
"""Find parameter definition and accompanying comments, to include in dumped easyconfig file."""
res = []
val = self._reformat_line(key, val, outer=True)
# templates
if key not in EXCLUDED_KEYS_REPLACE_TEMPLATES:
new_val = to_template_str(val, templ_const, templ_val)
# avoid self-referencing templated parameter definitions
if not r'%(' + key in new_val:
val = new_val
if key in self.comments['inline']:
res.append("%s = %s%s" % (key, val, self.comments['inline'][key]))
else:
if key in self.comments['above']:
res.extend(self.comments['above'][key])
res.append("%s = %s" % (key, val))
return res
def _find_defined_params(self, ecfg, keyset, default_values, templ_const,
templ_val):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
val = copy.deepcopy(ecfg[key])
# include hidden deps back in list of (build)dependencies, they were filtered out via filter_hidden_deps
if key == 'dependencies':
val.extend([
d for d in ecfg['hiddendependencies']
if not d['build_only']
])
elif key == 'builddependencies':
val.extend([
d for d in ecfg['hiddendependencies']
if d['build_only']
])
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
if key in DEPENDENCY_PARAMETERS:
valstr = [
dump_dependency(d, ecfg['toolchain']) for d in val
]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(
self._find_param_with_comments(key, valstr,
templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def dump(self, ecfg, default_values, templ_const, templ_val):
"""
Dump easyconfig in format v1.
@param ecfg: EasyConfig instance
@param default_values: default values for easyconfig parameters
@param templ_const: known template constants
@param templ_val: known template values
"""
# include header comments first
dump = self.comments['header'][:]
# print easyconfig parameters ordered and in groups specified above
params, printed_keys = self._find_defined_params(
ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val)
dump.extend(params)
# print other easyconfig parameters at the end
keys_to_ignore = printed_keys + LAST_PARAMS
for key in default_values:
if key not in keys_to_ignore and ecfg[key] != default_values[key]:
dump.extend(
self._find_param_with_comments(key,
quote_py_str(ecfg[key]),
templ_const, templ_val))
dump.append('')
# print last parameters
params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS],
default_values, templ_const,
templ_val)
dump.extend(params)
dump.extend(self.comments['tail'])
return '\n'.join(dump)
def extract_comments(self, rawtxt):
"""
Extract comments from raw content.
Discriminates between comment header, comments above a line (parameter definition), and inline comments.
Inline comments on items of iterable values are also extracted.
"""
self.comments = {
'above': {}, # comments for a particular parameter definition
'header': [], # header comment lines
'inline': {}, # inline comments
'iter': {}, # (inline) comments on elements of iterable values
'tail': [],
}
rawlines = rawtxt.split('\n')
# extract header first
while rawlines[0].startswith('#'):
self.comments['header'].append(rawlines.pop(0))
parsed_ec = self.get_config_dict()
while rawlines:
rawline = rawlines.pop(0)
if rawline.startswith('#'):
comment = []
# comment could be multi-line
while rawline is not None and (rawline.startswith('#')
or not rawline):
# drop empty lines (that don't even include a #)
if rawline:
comment.append(rawline)
# grab next line (if more lines are left)
if rawlines:
rawline = rawlines.pop(0)
else:
rawline = None
if rawline is None:
self.comments['tail'] = comment
else:
key = rawline.split('=', 1)[0].strip()
self.comments['above'][key] = comment
elif '#' in rawline: # inline comment
comment_key, comment_val = None, None
comment = rawline.rsplit('#', 1)[1].strip()
# check whether this line is parameter definition;
# if not, assume it's a continuation of a multi-line value
if re.match(r'^[a-z_]+\s*=', rawline):
comment_key = rawline.split('=', 1)[0].strip()
else:
# determine parameter value where the item value on this line is a part of
for key, val in parsed_ec.items():
item_val = re.sub(r',$', r'',
rawline.rsplit('#', 1)[0].strip())
if not isinstance(val,
basestring) and item_val in str(val):
comment_key, comment_val = key, item_val
break
# check if hash actually indicated a comment; or is part of the value
if comment_key in parsed_ec:
if comment.replace("'", '').replace('"', '') not in str(
parsed_ec[comment_key]):
if comment_val:
self.comments['iter'].setdefault(
comment_key,
{})[comment_val] = ' # ' + comment
else:
self.comments['inline'][
comment_key] = ' # ' + comment
# split into blocks using regex
pieces = reg_block.split(txt)
# the first block contains common statements
common = pieces.pop(0)
# determine version of easyconfig format
ec_format_version = get_format_version(txt)
if ec_format_version is None:
ec_format_version = FORMAT_DEFAULT_VERSION
_log.debug(
"retrieve_blocks_in_spec: derived easyconfig format version: %s" %
ec_format_version)
# blocks in easyconfigs are only supported in easyconfig format 1.0
if pieces and ec_format_version == EasyVersion('1.0'):
# make a map of blocks
blocks = []
while pieces:
block_name = pieces.pop(0)
block_contents = pieces.pop(0)
if block_name in [b['name'] for b in blocks]:
raise EasyBuildError("Found block %s twice in %s.", block_name,
spec)
block = {'name': block_name, 'contents': block_contents}
# dependency block
dep_block = reg_dep_block.search(block_contents)
if dep_block:
# split into blocks using regex
pieces = reg_block.split(txt)
# the first block contains common statements
common = pieces.pop(0)
# determine version of easyconfig format
ec_format_version = get_format_version(txt)
if ec_format_version is None:
ec_format_version = FORMAT_DEFAULT_VERSION
_log.debug(
"retrieve_blocks_in_spec: derived easyconfig format version: %s" %
ec_format_version)
# blocks in easyconfigs are only supported in format versions prior to 2.0
if pieces and ec_format_version < EasyVersion('2.0'):
# make a map of blocks
blocks = []
while pieces:
block_name = pieces.pop(0)
block_contents = pieces.pop(0)
if block_name in [b['name'] for b in blocks]:
raise EasyBuildError("Found block %s twice in %s.", block_name,
spec)
block = {'name': block_name, 'contents': block_contents}
# dependency block
dep_block = reg_dep_block.search(block_contents)
if dep_block:
from easybuild.framework.easyconfig.format.version import ToolchainVersionOperator, VersionOperator
from easybuild.framework.easyconfig.format.convert import Dependency
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.configobj import Section
INDENT_4SPACES = ' ' * 4
# format is mandatory major.minor
FORMAT_VERSION_KEYWORD = "EASYCONFIGFORMAT"
FORMAT_VERSION_TEMPLATE = "%(major)s.%(minor)s"
FORMAT_VERSION_HEADER_TEMPLATE = "# %s %s\n" % (
FORMAT_VERSION_KEYWORD, FORMAT_VERSION_TEMPLATE) # must end in newline
FORMAT_VERSION_REGEXP = re.compile(
r'^#\s+%s\s*(?P<major>\d+)\.(?P<minor>\d+)\s*$' % FORMAT_VERSION_KEYWORD,
re.M)
FORMAT_DEFAULT_VERSION = EasyVersion('1.0')
DEPENDENCY_PARAMETERS = [
'builddependencies', 'dependencies', 'hiddendependencies'
]
# values for these keys will not be templated in dump()
EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'homepage', 'name', 'toolchain', 'version'] \
+ DEPENDENCY_PARAMETERS
# ordered groups of keys to obtain a nice looking easyconfig file
GROUPED_PARAMS = [
['easyblock'],
['name', 'version', 'versionprefix', 'versionsuffix'],
['homepage', 'description'],
['toolchain', 'toolchainopts'],
def retrieve_blocks_in_spec(spec, only_blocks, silent=False):
"""
Easyconfigs can contain blocks (headed by a [Title]-line)
which contain commands specific to that block. Commands in the beginning of the file
above any block headers are common and shared between each block.
"""
reg_block = re.compile(r"^\s*\[([\w.-]+)\]\s*$", re.M)
reg_dep_block = re.compile(r"^\s*block\s*=(\s*.*?)\s*$", re.M)
spec_fn = os.path.basename(spec)
txt = read_file(spec)
# split into blocks using regex
pieces = reg_block.split(txt)
# the first block contains common statements
common = pieces.pop(0)
# determine version of easyconfig format
ec_format_version = get_format_version(txt)
if ec_format_version is None:
ec_format_version = FORMAT_DEFAULT_VERSION
_log.debug(
"retrieve_blocks_in_spec: derived easyconfig format version: %s" %
ec_format_version)
# blocks in easyconfigs are only supported in easyconfig format 1.0
if pieces and ec_format_version == EasyVersion('1.0'):
# make a map of blocks
blocks = []
while pieces:
block_name = pieces.pop(0)
block_contents = pieces.pop(0)
if block_name in [b['name'] for b in blocks]:
raise EasyBuildError("Found block %s twice in %s.", block_name,
spec)
block = {'name': block_name, 'contents': block_contents}
# dependency block
dep_block = reg_dep_block.search(block_contents)
if dep_block:
dependencies = eval(dep_block.group(1))
if type(dependencies) == list:
block['dependencies'] = dependencies
else:
block['dependencies'] = [dependencies]
blocks.append(block)
# make a new easyconfig for each block
# they will be processed in the same order as they are all described in the original file
specs = []
for block in blocks:
name = block['name']
if only_blocks and not (name in only_blocks):
print_msg("Skipping block %s-%s" % (spec_fn, name),
silent=silent)
continue
(fd,
block_path) = tempfile.mkstemp(prefix='easybuild-',
suffix='%s-%s' % (spec_fn, name))
os.close(fd)
txt = common
if 'dependencies' in block:
for dep in block['dependencies']:
if dep not in [b['name'] for b in blocks]:
raise EasyBuildError(
"Block %s depends on %s, but block was not found.",
name, dep)
dep = [b for b in blocks if b['name'] == dep][0]
txt += "\n# Dependency block %s" % (dep['name'])
txt += dep['contents']
txt += "\n# Main block %s" % name
txt += block['contents']
write_file(block_path, txt)
specs.append(block_path)
_log.debug("Found %s block(s) in %s" % (len(specs), spec))
return specs
else:
# no blocks, one file
return [spec]
class FormatTwoZero(EasyConfigFormatConfigObj):
"""
Support for easyconfig format 2.0
Simple extension of FormatOneZero with configparser blocks
Doesn't set version and toolchain/toolchain version like in FormatOneZero;
referencing 'version' directly in pyheader doesn't work => use templating '%(version)s'
NOT in 2.0
- order preservation: need more recent ConfigObj (more recent Python as minimal version)
- nested sections (need other ConfigParser/ConfigObj, eg INITools)
- type validation
- command line generation (--try-X command line options)
"""
VERSION = EasyVersion('2.0')
USABLE = True
PYHEADER_ALLOWED_BUILTINS = ['len', 'False', 'True']
PYHEADER_MANDATORY = ['name', 'homepage', 'description', 'software_license', 'software_license_urls', 'docurls']
PYHEADER_BLACKLIST = ['version', 'toolchain']
NAME_DOCSTRING_REGEX_TEMPLATE = r'^\s*@%s\s*:\s*(?P<name>\S.*?)\s*$' # non-greedy match in named pattern
AUTHOR_DOCSTRING_REGEX = re.compile(NAME_DOCSTRING_REGEX_TEMPLATE % 'author', re.M)
MAINTAINER_DOCSTRING_REGEX = re.compile(NAME_DOCSTRING_REGEX_TEMPLATE % 'maintainer', re.M)
AUTHOR_REQUIRED = True
MAINTAINER_REQUIRED = False
def validate(self):
"""Format validation"""
self._check_docstring()
self._validate_pyheader()
def _check_docstring(self):
"""
Verify docstring.
field :author: people who contributed to the easyconfig
field @maintainer: people who can be contacted in case of problems
"""
authors = []
maintainers = []
for auth_reg in self.AUTHOR_DOCSTRING_REGEX.finditer(self.docstring):
res = auth_reg.groupdict()
authors.append(res['name'])
for maint_reg in self.MAINTAINER_DOCSTRING_REGEX.finditer(self.docstring):
res = maint_reg.groupdict()
maintainers.append(res['name'])
if self.AUTHOR_REQUIRED and not authors:
raise EasyBuildError("No author in docstring (regex: '%s')", self.AUTHOR_DOCSTRING_REGEX.pattern)
if self.MAINTAINER_REQUIRED and not maintainers:
raise EasyBuildError("No maintainer in docstring (regex: '%s')", self.MAINTAINER_DOCSTRING_REGEX.pattern)
def get_config_dict(self):
"""Return the best matching easyconfig dict"""
self.log.experimental(self.__class__.__name__)
# the toolchain name/version should not be specified in the pyheader,
# but other toolchain options are allowed
cfg = copy.deepcopy(self.pyheader_localvars)
self.log.debug("Config dict based on Python header: %s" % cfg)
co = EBConfigObj(self.configobj)
version = self.specs.get('version', None)
tc_spec = self.specs.get('toolchain', {})
toolchain_name = tc_spec.get('name', None)
toolchain_version = tc_spec.get('version', None)
# parse and interpret, dealing with defaults etc
version, tcname, tcversion = co.get_version_toolchain(version, toolchain_name, toolchain_version)
# format 2.0 will squash
self.log.debug('Squashing with version %s and toolchain %s' % (version, (tcname, tcversion)))
res = co.squash(version, tcname, tcversion)
cfg.update(res)
self.log.debug("Config dict after processing applicable easyconfig sections: %s" % cfg)
# FIXME what about updating dict values/appending to list values?
# FIXME how do we allow both redefining and updating? = and +=?
# update config with correct version/toolchain (to avoid using values specified in default section)
cfg.update({
'version': version,
'toolchain': {'name': tcname, 'version': tcversion},
})
self.log.debug("Final config dict (including correct version/toolchain): %s" % cfg)
return cfg
def extract_comments(self, rawtxt):
"""Extract comments from raw content."""
# this is fine-ish, it only implies that comments will be lost for format v2 easyconfig files that are dumped
self.log.warning("Extraction of comments not supported yet for easyconfig format v2")
class FormatOneZero(EasyConfigFormatConfigObj):
"""Support for easyconfig format 1.x"""
VERSION = EasyVersion('1.0')
USABLE = True # TODO: disable it at some point, too insecure
PYHEADER_ALLOWED_BUILTINS = None # allow all
PYHEADER_MANDATORY = [
'version', 'name', 'toolchain', 'homepage', 'description'
]
PYHEADER_BLACKLIST = []
def __init__(self, *args, **kwargs):
"""FormatOneZero constructor."""
super(FormatOneZero, self).__init__(*args, **kwargs)
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.strict_sanity_check_paths_keys = True
def validate(self):
"""Format validation"""
# minimal checks
self._validate_pyheader()
def get_config_dict(self):
"""
Return parsed easyconfig as a dictionary, based on specified arguments.
This is easyconfig format 1.x, so there is only one easyconfig instance available.
"""
spec_version = self.specs.get('version', None)
spec_tc = self.specs.get('toolchain', {})
spec_tc_name = spec_tc.get('name', None)
spec_tc_version = spec_tc.get('version', None)
cfg = self.pyheader_localvars
if spec_version is not None and not spec_version == cfg['version']:
raise EasyBuildError('Requested version %s not available, only %s',
spec_version, cfg['version'])
tc_name = cfg.get('toolchain', {}).get('name', None)
tc_version = cfg.get('toolchain', {}).get('version', None)
if spec_tc_name is not None and not spec_tc_name == tc_name:
raise EasyBuildError(
'Requested toolchain name %s not available, only %s',
spec_tc_name, tc_name)
if spec_tc_version is not None and not spec_tc_version == tc_version:
raise EasyBuildError(
'Requested toolchain version %s not available, only %s',
spec_tc_version, tc_version)
# avoid passing anything by reference, so next time get_config_dict is called
# we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file;
# we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails...
cfg_copy = {}
for key in cfg:
if key != '__builtins__':
cfg_copy[key] = copy.deepcopy(cfg[key])
return cfg_copy
def parse(self, txt):
"""
Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced.
"""
self.rawcontent = txt
super(FormatOneZero, self).parse(self.rawcontent,
strict_section_markers=True)
def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
"""
Construct formatted string representation of iterable parameter (list/tuple/dict), including comments.
:param param_name: parameter name
:param param_val: parameter value
:param outer: reformat for top-level parameter, or not
:param addlen: # characters to add to line length
"""
param_strval = str(param_val)
res = param_strval
# determine whether line would be too long
# note: this does not take into account the parameter name + '=', only the value
line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH
forced = param_name in REFORMAT_FORCED_PARAMS
list_of_lists_of_tuples_param = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES
if param_name in REFORMAT_SKIPPED_PARAMS:
self.log.info("Skipping reformatting value for parameter '%s'",
param_name)
elif outer:
# only reformat outer (iterable) values for (too) long lines (or for select parameters)
if isinstance(param_val, (list, tuple, dict)) and (
(len(param_val) > 1 or line_too_long) or forced):
item_tmpl = INDENT_4SPACES + '%(item)s,%(inline_comment)s\n'
start_char, end_char = param_strval[0], param_strval[-1]
# start with opening character: [, (, {
res = '%s\n' % start_char
# add items one-by-one, special care for dict values (order of keys, different format for elements)
if isinstance(param_val, dict):
ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(
param_name, sorted(param_val.keys()))
for item_key in ordered_item_keys:
if item_key in param_val:
item_val = param_val[item_key]
item_comments = self._get_item_comments(
param_name, item_val)
elif param_name == 'sanity_check_paths' and not self.strict_sanity_check_paths_keys:
item_val = []
item_comments = {}
self.log.info(
"Using default value for '%s' in sanity_check_paths: %s",
item_key, item_val)
else:
raise EasyBuildError(
"Missing mandatory key '%s' in %s.", item_key,
param_name)
inline_comment = item_comments.get('inline', '')
item_tmpl_dict = {'inline_comment': inline_comment}
for comment in item_comments.get('above', []):
res += INDENT_4SPACES + comment + '\n'
key_pref = quote_py_str(item_key) + ': '
addlen = addlen + len(INDENT_4SPACES) + len(
key_pref) + len(inline_comment)
formatted_item_val = self._reformat_line(param_name,
item_val,
addlen=addlen)
item_tmpl_dict['item'] = key_pref + formatted_item_val
res += item_tmpl % item_tmpl_dict
else: # list, tuple
for item in param_val:
item_comments = self._get_item_comments(
param_name, item)
inline_comment = item_comments.get('inline', '')
item_tmpl_dict = {'inline_comment': inline_comment}
for comment in item_comments.get('above', []):
res += INDENT_4SPACES + comment + '\n'
addlen = addlen + len(INDENT_4SPACES) + len(
inline_comment)
# the tuples are really strings here that are constructed from the dependency dicts
# so for a plain list of builddependencies param_val is a list of strings here;
# and for iterated builddependencies it is a list of lists of strings
is_list_of_lists_of_tuples = isinstance(
item, list) and all(
isinstance(x, str) for x in item)
if list_of_lists_of_tuples_param and is_list_of_lists_of_tuples:
itemstr = '[' + (',\n ' + INDENT_4SPACES).join([
self._reformat_line(param_name,
subitem,
outer=True,
addlen=addlen)
for subitem in item
]) + ']'
else:
itemstr = self._reformat_line(param_name,
item,
addlen=addlen)
item_tmpl_dict['item'] = itemstr
res += item_tmpl % item_tmpl_dict
# take into account possible closing comments
# see https://github.com/easybuilders/easybuild-framework/issues/3082
end_comments = self._get_item_comments(param_name, end_char)
for comment in end_comments.get('above', []):
res += INDENT_4SPACES + comment + '\n'
# end with closing character (']', ')', '}'), incl. possible inline comment
res += end_char
if 'inline' in end_comments:
res += end_comments['inline']
else:
# dependencies are already dumped as strings, so they do not need to be quoted again
if isinstance(
param_val,
string_type) and param_name not in DEPENDENCY_PARAMETERS:
res = quote_py_str(param_val)
return res
def _get_item_comments(self, key, val):
"""Get per-item comments for specified parameter name/value."""
item_comments = {}
for comment_key, comment_val in self.comments['iterabove'].get(
key, {}).items():
if str(val) in comment_key:
item_comments['above'] = comment_val
for comment_key, comment_val in self.comments['iterinline'].get(
key, {}).items():
if str(val) in comment_key:
item_comments['inline'] = comment_val
return item_comments
def _find_param_with_comments(self, key, val, templ_const, templ_val):
"""Find parameter definition and accompanying comments, to include in dumped easyconfig file."""
res = []
val = self._reformat_line(key, val, outer=True)
# templates
if key not in EXCLUDED_KEYS_REPLACE_TEMPLATES:
val = to_template_str(key, val, templ_const, templ_val)
if key in self.comments['inline']:
res.append("%s = %s%s" % (key, val, self.comments['inline'][key]))
else:
if key in self.comments['above']:
res.extend(self.comments['above'][key])
res.append("%s = %s" % (key, val))
return res
def _find_defined_params(self,
ecfg,
keyset,
default_values,
templ_const,
templ_val,
toolchain_hierarchy=None):
"""
Determine parameters in the dumped easyconfig file which have a non-default value.
"""
eclines = []
printed_keys = []
for group in keyset:
printed = False
for key in group:
val = ecfg[key]
if val != default_values[key]:
# dependency easyconfig parameters were parsed, so these need special care to 'unparse' them;
# take into account that these parameters may be iterative (i.e. a list of lists of parsed deps)
if key in DEPENDENCY_PARAMETERS:
if key in ecfg.iterate_options:
if 'multi_deps' in ecfg:
# the way that builddependencies are constructed with multi_deps
# we just need to dump the first entry without the dependencies
# that are listed in multi_deps
valstr = [
dump_dependency(
d,
ecfg['toolchain'],
toolchain_hierarchy=toolchain_hierarchy
) for d in val[0]
if d['name'] not in ecfg['multi_deps']
]
else:
valstr = [[
dump_dependency(
d,
ecfg['toolchain'],
toolchain_hierarchy=toolchain_hierarchy
) for d in dep
] for dep in val]
else:
valstr = [
dump_dependency(
d,
ecfg['toolchain'],
toolchain_hierarchy=toolchain_hierarchy)
for d in val
]
elif key == 'toolchain':
valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[
key]
else:
valstr = quote_py_str(ecfg[key])
eclines.extend(
self._find_param_with_comments(key, valstr,
templ_const, templ_val))
printed_keys.append(key)
printed = True
if printed:
eclines.append('')
return eclines, printed_keys
def dump(self,
ecfg,
default_values,
templ_const,
templ_val,
toolchain_hierarchy=None):
"""
Dump easyconfig in format v1.
:param ecfg: EasyConfig instance
:param default_values: default values for easyconfig parameters
:param templ_const: known template constants
:param templ_val: known template values
:param toolchain_hierarchy: hierarchy of toolchains for easyconfig
"""
# figoure out whether we should be strict about the format of sanity_check_paths;
# if enhance_sanity_check is set, then both files/dirs keys are not strictly required...
self.strict_sanity_check_paths_keys = not ecfg['enhance_sanity_check']
# include header comments first
dump = self.comments['header'][:]
# print easyconfig parameters ordered and in groups specified above
params, printed_keys = self._find_defined_params(
ecfg,
GROUPED_PARAMS,
default_values,
templ_const,
templ_val,
toolchain_hierarchy=toolchain_hierarchy)
dump.extend(params)
# print other easyconfig parameters at the end
keys_to_ignore = printed_keys + LAST_PARAMS
for key in default_values:
mandatory = ecfg.is_mandatory_param(key)
non_default_value = ecfg[key] != default_values[key]
if key not in keys_to_ignore and (mandatory or non_default_value):
dump.extend(
self._find_param_with_comments(key,
quote_py_str(ecfg[key]),
templ_const, templ_val))
dump.append('')
# print last parameters
params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS],
default_values, templ_const,
templ_val)
dump.extend(params)
dump.extend(self.comments['tail'])
return '\n'.join(dump)
@property
def comments(self):
"""
Return comments (and extract them first if needed).
"""
if not self._comments:
self.extract_comments(self.rawcontent)
return self._comments
def extract_comments(self, rawtxt):
"""
Extract comments from raw content.
Discriminates between comment header, comments above a line (parameter definition), and inline comments.
Inline comments on items of iterable values are also extracted.
"""
self._comments = {
'above': {}, # comments above a parameter definition
'header': [], # header comment lines
'inline': {}, # inline comments
'iterabove': {}, # comment above elements of iterable values
'iterinline': {}, # inline comments on elements of iterable values
'tail': [], # comment at the end of the easyconfig file
}
parsed_ec = self.get_config_dict()
comment_regex = re.compile(r'^\s*#')
param_def_regex = re.compile(r'^([a-z_0-9]+)\s*=')
whitespace_regex = re.compile(r'^\s*$')
def clean_part(part):
"""Helper function to strip off trailing whitespace + trailing quotes."""
return part.rstrip().rstrip("'").rstrip('"')
def split_on_comment_hash(line, param_key):
"""Helper function to split line on first (actual) comment character '#'."""
# string representation of easyconfig parameter value,
# used to check if supposed comment isn't actual part of the parameter value
# (and thus not actually a comment at all)
param_strval = str(parsed_ec.get(param_key))
parts = line.split('#')
# first part (before first #) is definitely not part of comment
before_comment = parts.pop(0)
# strip out parts that look like a comment but are actually part of a parameter value
while parts and ('#' + clean_part(parts[0])) in param_strval:
before_comment += '#' + parts.pop(0)
comment = '#'.join(parts)
return before_comment, comment.strip()
def grab_more_comment_lines(lines, param_key):
"""Grab more comment lines."""
comment_lines = []
while lines and (comment_regex.match(lines[0])
or whitespace_regex.match(lines[0])):
line = lines.pop(0)
_, actual_comment = split_on_comment_hash(line, param_key)
# prefix comment with '#' unless line was empty
if line.strip():
actual_comment = '# ' + actual_comment
comment_lines.append(actual_comment.strip())
return comment_lines
rawlines = rawtxt.split('\n')
# extract header first (include empty lines too)
self.comments['header'] = grab_more_comment_lines(rawlines, None)
last_param_key = None
while rawlines:
rawline = rawlines.pop(0)
# keep track of last parameter definition we have seen,
# current line may be (the start of) a parameter definition
res = param_def_regex.match(rawline)
if res:
key = res.group(1)
if key in parsed_ec:
last_param_key = key
if last_param_key:
before_comment, inline_comment = split_on_comment_hash(
rawline, last_param_key)
# short-circuit to next line in case there are no actual comments on this (non-empty) line
if before_comment and not inline_comment:
continue
# lines that start with a hash indicate (start of a block of) comment line(s)
if rawline.startswith('#'):
comment = [rawline] + grab_more_comment_lines(
rawlines, last_param_key)
if rawlines:
# try to pin comment to parameter definition below it
# don't consume the line yet though, it may also include inline comments...
res = param_def_regex.match(rawlines[0])
if res:
last_param_key = res.group(1)
self.comments['above'][last_param_key] = comment
else:
# if the comment is not above a parameter definition,
# then it must be a comment for an item of an iterable parameter value
before_comment, _ = split_on_comment_hash(
rawlines[0], last_param_key)
comment_key = before_comment.rstrip()
self.comments['iterabove'].setdefault(
last_param_key, {})[comment_key] = comment
else:
# if there are no more lines, the comment (block) is at the tail
self.comments['tail'] = comment
elif '#' in rawline:
# if there's a hash character elsewhere in the line (not at the start),
# there are a couple of possibilities:
# - inline comment for a parameter definition (at the end of a non-empty line)
# - indented comment for an item value of an iterable easyconfig parameter (list, dict, ...)
# - inline comment for an item value of an iterable easyconfig parameter
before_comment, comment = split_on_comment_hash(
rawline, last_param_key)
comment = ('# ' + comment).rstrip()
# first check whether current line is an easyconfig parameter definition
# if so, the comment is an inline comment
if param_def_regex.match(before_comment):
self.comments['inline'][last_param_key] = ' ' + comment
# if there's only whitespace before the comment,
# then we have an indented comment, and we need to figure out for what exactly
elif whitespace_regex.match(before_comment):
# first consume possible additional comment lines with same indentation
comment = [comment] + grab_more_comment_lines(
rawlines, last_param_key)
before_comment, inline_comment = split_on_comment_hash(
rawlines.pop(0), last_param_key)
comment_key = before_comment.rstrip()
self.comments['iterabove'].setdefault(
last_param_key, {})[comment_key] = comment
if inline_comment:
inline_comment = (' # ' + inline_comment).rstrip()
self.comments['iterinline'].setdefault(
last_param_key, {})[comment_key] = inline_comment
else:
# inline comment for item of iterable value
comment_key = before_comment.rstrip()
self.comments['iterinline'].setdefault(
last_param_key, {})[comment_key] = ' ' + comment
self.log.debug("Extracted comments:\n%s",
pprint.pformat(self.comments, width=120))
