def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
"""
Generate prepend-path statements for the given list of paths
@param key: environment variable to prepend paths to
@param paths: list of paths to prepend
@param allow_abs: allow providing of absolute paths
@param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
"""
if isinstance(paths, basestring):
self.log.debug("Wrapping %s into a list before using it to prepend path %s", paths, key)
paths = [paths]
abspaths = []
for path in paths:
if os.path.isabs(path):
if allow_abs:
abspaths.append(quote_str(path))
else:
raise EasyBuildError(
"Absolute path %s passed to prepend_paths which only expects relative paths.", path
)
else:
# use pathJoin for (non-empty) relative paths
if path:
if expand_relpaths:
abspaths.append(self.PATH_JOIN_TEMPLATE % path)
else:
abspaths.append(quote_str(path))
else:
abspaths.append("root")
statements = [self.PREPEND_PATH_TEMPLATE % (key, p) for p in abspaths]
return "".join(statements)
def test_use(self):
"""Test generating module use statements."""
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
# Test regular 'module use' statements
expected = ''.join([
'module use "/some/path"\n',
'module use "/foo/bar/baz"\n',
])
self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected)
# Test guarded 'module use' statements using prefix
expected = ''.join([
'if { [ file isdirectory [ file join "/foo" "/some/path" ] ] } {\n',
' module use [ file join "/foo" "/some/path" ]\n',
'}\n',
])
self.assertEqual(self.modgen.use(["/some/path"], prefix=quote_str("/foo"), guarded=True), expected)
else:
# Test regular 'module use' statements
expected = ''.join([
'prepend_path("MODULEPATH", "/some/path")\n',
'prepend_path("MODULEPATH", "/foo/bar/baz")\n',
])
self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected)
# Test guarded 'module use' statements using prefix
expected = ''.join([
'if isDir(pathJoin("/foo", "/some/path")) then\n',
' prepend_path("MODULEPATH", pathJoin("/foo", "/some/path"))\n',
'end\n',
])
self.assertEqual(self.modgen.use(["/some/path"], prefix=quote_str("/foo"), guarded=True), expected)
def stats_to_str(stats):
"""
Pretty print build statistics to string.
"""
if not isinstance(stats, (OrderedDict, dict)):
raise EasyBuildError("Can only pretty print build stats in dictionary form, not of type %s", type(stats))
txt = "{\n"
pref = " "
for (k, v) in stats.items():
txt += "%s%s: %s,\n" % (pref, quote_str(k), quote_str(v))
txt += "}"
return txt
def set_environment(self, key, value, relpath=False):
"""
Generate setenv statement for the given key/value pair.
"""
# quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
if relpath:
if value:
val = quote_str(os.path.join("$root", value))
else:
val = '"$root"'
else:
val = quote_str(value)
return "setenv\t%s\t\t%s\n" % (key, val)
def avail_easyconfig_params_rst(title, grouped_params):
"""
Compose overview of available easyconfig parameters, in RST format.
"""
# main title
doc = [
title,
'=' * len(title),
'',
]
for grpname in grouped_params:
# group section title
title = "%s parameters" % grpname
table_titles = ["**Parameter name**", "**Description**", "**Default value**"]
table_values = [
['``%s``' % name for name in grouped_params[grpname].keys()], # parameter name
[x[0] for x in grouped_params[grpname].values()], # description
[str(quote_str(x[1])) for x in grouped_params[grpname].values()] # default value
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
doc.append('')
return '\n'.join(doc)
def avail_easyconfig_params_rst(title, grouped_params):
"""
Compose overview of available easyconfig parameters, in RST format.
"""
# main title
lines = [
title,
'=' * len(title),
'',
]
for grpname in grouped_params:
# group section title
lines.append("%s parameters" % grpname)
lines.extend(['-' * len(lines[-1]), ''])
titles = ["**Parameter name**", "**Description**", "**Default value**"]
values = [
['``' + name + '``' for name in grouped_params[grpname].keys()], # parameter name
[x[0] for x in grouped_params[grpname].values()], # description
[str(quote_str(x[1])) for x in grouped_params[grpname].values()] # default value
]
lines.extend(mk_rst_table(titles, values))
lines.append('')
return '\n'.join(lines)
def avail_easyconfig_params_rst(title, grouped_params):
"""
Compose overview of available easyconfig parameters, in RST format.
"""
def det_col_width(entries, title):
"""Determine column width based on column title and list of entries."""
return max(map(len, entries + [title]))
# main title
lines = [
title,
'=' * len(title),
'',
]
for grpname in grouped_params:
# group section title
lines.append("%s parameters" % grpname)
lines.extend(['-' * len(lines[-1]), ''])
name_title = "**Parameter name**"
descr_title = "**Description**"
dflt_title = "**Default value**"
# figure out column widths
nw = det_col_width(grouped_params[grpname].keys(), name_title) + 4 # +4 for raw format ("``foo``")
dw = det_col_width([x[0] for x in grouped_params[grpname].values()], descr_title)
dfw = det_col_width([str(quote_str(x[1])) for x in grouped_params[grpname].values()], dflt_title)
# 3 columns (name, description, default value), left-aligned, {c} is fill char
line_tmpl = "{0:{c}<%s} {1:{c}<%s} {2:{c}<%s}" % (nw, dw, dfw)
table_line = line_tmpl.format('', '', '', c='=', nw=nw, dw=dw, dfw=dfw)
# table header
lines.append(table_line)
lines.append(line_tmpl.format(name_title, descr_title, dflt_title, c=' '))
lines.append(line_tmpl.format('', '', '', c='-'))
# table rows by parameter
for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
rawname = '``%s``' % name
lines.append(line_tmpl.format(rawname, descr, str(quote_str(dflt)), c=' '))
lines.append(table_line)
lines.append('')
return '\n'.join(lines)
def msg_on_load(self, msg):
"""
Add a message that should be printed when loading the module.
"""
# escape any (non-escaped) characters with special meaning by prefixing them with a backslash
msg = re.sub(r'((?<!\\)[%s])'% ''.join(self.CHARS_TO_ESCAPE), r'\\\1', msg)
print_cmd = "puts stderr %s" % quote_str(msg)
return '\n'.join(['', self.conditional_statement("module-info mode load", print_cmd)])
def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
"""
Generate prepend_path or append_path statements for the given list of paths
:param key: environment variable to prepend/append paths to
:param paths: list of paths to prepend/append
:param prepend: whether to prepend (True) or append (False) paths
:param allow_abs: allow providing of absolute paths
:param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
"""
if prepend:
update_type = 'prepend'
else:
update_type = 'append'
if not self.define_env_var(key):
self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key)
return ''
if isinstance(paths, basestring):
self.log.debug("Wrapping %s into a list before using it to %s path %s", update_type, paths, key)
paths = [paths]
abspaths = []
for path in paths:
if os.path.isabs(path):
if allow_abs:
abspaths.append(quote_str(path))
else:
raise EasyBuildError("Absolute path %s passed to update_paths which only expects relative paths.",
path)
else:
# use pathJoin for (non-empty) relative paths
if path:
if expand_relpaths:
abspaths.append(self.PATH_JOIN_TEMPLATE % path)
else:
abspaths.append(quote_str(path))
else:
abspaths.append('root')
statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
statements.append('')
return '\n'.join(statements)
def stats_to_str(stats, isyeb=False):
"""
Pretty print build statistics to string.
"""
if not isinstance(stats, (OrderedDict, dict)):
raise EasyBuildError("Can only pretty print build stats in dictionary form, not of type %s", type(stats))
txt = "{\n"
pref = " "
for key in sorted(stats):
if isyeb:
val = stats[key]
if isinstance(val, tuple):
val = list(val)
key, val = quote_yaml_special_chars(key), quote_yaml_special_chars(val)
else:
key, val = quote_str(key), quote_str(stats[key])
txt += "%s%s: %s,\n" % (pref, key, val)
txt += "}"
return txt
def set_environment(self, key, value, relpath=False):
"""
Generate a quoted setenv statement for the given key/value pair.
"""
if relpath:
if value:
val = self.PATH_JOIN_TEMPLATE % value
else:
val = 'root'
else:
val = quote_str(value)
return 'setenv("%s", %s)\n' % (key, val)
def msg_on_load(self, msg):
"""
Add a message that should be printed when loading the module.
"""
# escape any (non-escaped) characters with special meaning by prefixing them with a backslash
msg = re.sub(r'((?<!\\)[%s])' % ''.join(self.CHARS_TO_ESCAPE), r'\\\1',
msg)
print_cmd = "puts stderr %s" % quote_str(msg)
return '\n'.join([
'',
self.conditional_statement("module-info mode load", print_cmd)
])
def set_environment(self, key, value, relpath=False):
"""
Generate a quoted setenv statement for the given key/value pair.
:param key: name of environment variable to define
:param value: value to define environment variable with
:param relpath: value is path relative to installation prefix
"""
if not self.define_env_var(key):
self.log.info("Not including statement to define environment variable $%s, as specified", key)
return ''
# quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
if relpath:
if value:
val = quote_str(os.path.join('$root', value))
else:
val = '"$root"'
else:
val = quote_str(value)
return 'setenv\t%s\t\t%s\n' % (key, val)
def set_environment(self, key, value, relpath=False):
"""
Generate a quoted setenv statement for the given key/value pair.
"""
if relpath:
if value:
val = self.PATH_JOIN_TEMPLATE % value
else:
val = "root"
else:
val = quote_str(value)
return 'setenv("%s", %s)\n' % (key, val)
def set_environment(self, key, value, relpath=False):
"""
Generate a quoted setenv statement for the given key/value pair.
:param key: name of environment variable to define
:param value: value to define environment variable with
:param relpath: value is path relative to installation prefix
"""
if not self.define_env_var(key):
self.log.info("Not including statement to define environment variable $%s, as specified", key)
return ''
# quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
if relpath:
if value:
val = quote_str(os.path.join('$root', value))
else:
val = '"$root"'
else:
val = quote_str(value)
return 'setenv\t%s\t\t%s\n' % (key, val)
def test_use(self):
"""Test generating module use statements."""
if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl:
# Test regular 'module use' statements
expected = ''.join([
'module use "/some/path"\n',
'module use "/foo/bar/baz"\n',
])
self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]),
expected)
# Test guarded 'module use' statements using prefix
expected = ''.join([
'if { [ file isdirectory [ file join "/foo" "/some/path" ] ] } {\n',
' module use [ file join "/foo" "/some/path" ]\n',
'}\n',
])
self.assertEqual(
self.modgen.use(["/some/path"],
prefix=quote_str("/foo"),
guarded=True), expected)
else:
# Test regular 'module use' statements
expected = ''.join([
'prepend_path("MODULEPATH", "/some/path")\n',
'prepend_path("MODULEPATH", "/foo/bar/baz")\n',
])
self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]),
expected)
# Test guarded 'module use' statements using prefix
expected = ''.join([
'if isDir(pathJoin("/foo", "/some/path")) then\n',
' prepend_path("MODULEPATH", pathJoin("/foo", "/some/path"))\n',
'end\n',
])
self.assertEqual(
self.modgen.use(["/some/path"],
prefix=quote_str("/foo"),
guarded=True), expected)
def stats_to_str(stats, isyeb=False):
"""
Pretty print build statistics to string.
"""
if not isinstance(stats, (OrderedDict, dict)):
raise EasyBuildError(
"Can only pretty print build stats in dictionary form, not of type %s",
type(stats))
txt = "{\n"
pref = " "
for key in sorted(stats):
if isyeb:
val = stats[key]
if isinstance(val, tuple):
val = list(val)
key, val = quote_yaml_special_chars(key), quote_yaml_special_chars(
val)
else:
key, val = quote_str(key), quote_str(stats[key])
txt += "%s%s: %s,\n" % (pref, key, val)
txt += "}"
return txt
def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
"""
Generate prepend-path statements for the given list of paths
@param key: environment variable to prepend paths to
@param paths: list of paths to prepend
@param allow_abs: allow providing of absolute paths
@param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
"""
if isinstance(paths, basestring):
self.log.debug(
"Wrapping %s into a list before using it to prepend path %s",
paths, key)
paths = [paths]
abspaths = []
for path in paths:
if os.path.isabs(path):
if allow_abs:
abspaths.append(quote_str(path))
else:
raise EasyBuildError(
"Absolute path %s passed to prepend_paths which only expects relative paths.",
path)
else:
# use pathJoin for (non-empty) relative paths
if path:
if expand_relpaths:
abspaths.append(self.PATH_JOIN_TEMPLATE % path)
else:
abspaths.append(quote_str(path))
else:
abspaths.append('root')
statements = [self.PREPEND_PATH_TEMPLATE % (key, p) for p in abspaths]
statements.append('')
return '\n'.join(statements)
def set_environment(self, key, value, relpath=False):
"""
Generate a quoted setenv statement for the given key/value pair.
:param key: name of environment variable to define
:param value: value to define environment variable with
:param relpath: value is path relative to installation prefix
"""
if not self.define_env_var(key):
self.log.info("Not including statement to define environment variable $%s, as specified", key)
return ''
if relpath:
if value:
val = self.PATH_JOIN_TEMPLATE % value
else:
val = 'root'
else:
val = quote_str(value)
return 'setenv("%s", %s)\n' % (key, val)
def test_deprecated_options(self):
"""Test whether deprecated options are handled correctly."""
deprecated_options = [
('makeopts', 'buildopts', 'CC=foo'),
('premakeopts', 'prebuildopts', ['PATH=%(builddir)s/foo:$PATH', 'PATH=%(builddir)s/bar:$PATH']),
]
clean_contents = [
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'buildininstalldir = True',
]
# alternative option is ready to use
for depr_opt, new_opt, val in deprecated_options:
self.contents = '\n'.join(clean_contents + ['%s = %s' % (depr_opt, quote_str(val))])
self.prep()
ec = EasyConfig(self.eb_file)
self.assertEqual(ec[depr_opt], ec[new_opt])
def set_environment(self, key, value, relpath=False):
"""
Generate a quoted setenv statement for the given key/value pair.
:param key: name of environment variable to define
:param value: value to define environment variable with
:param relpath: value is path relative to installation prefix
"""
if not self.define_env_var(key):
self.log.info("Not including statement to define environment variable $%s, as specified", key)
return ''
if relpath:
if value:
val = self.PATH_JOIN_TEMPLATE % value
else:
val = 'root'
else:
val = quote_str(value)
return 'setenv("%s", %s)\n' % (key, val)
def use(self, paths, prefix=None, guarded=False):
"""
Generate module use statements for given list of module paths.
:param paths: list of module path extensions to generate use statements for; paths will be quoted
:param prefix: optional path prefix; not quoted, i.e., can be a statement
:param guarded: use statements will be guarded to only apply if path exists
"""
use_statements = []
for path in paths:
quoted_path = quote_str(path)
if prefix:
full_path = 'pathJoin(%s, %s)' % (prefix, quoted_path)
else:
full_path = quoted_path
prepend_modulepath = self.UPDATE_PATH_TEMPLATE % ('prepend', 'MODULEPATH', full_path)
if guarded:
cond_statement = self.conditional_statement('isDir(%s)' % full_path, prepend_modulepath)
use_statements.append(cond_statement)
else:
use_statements.append(prepend_modulepath + '\n')
return ''.join(use_statements)
def avail_easyconfig_params_txt(title, grouped_params):
"""
Compose overview of available easyconfig parameters, in plain text format.
"""
# main title
doc = ["%s:" % title, ""]
for grpname in grouped_params:
# group section title
doc.append(grpname.upper())
doc.append("-" * len(doc[-1]))
# determine width of 'name' column, to left-align descriptions
nw = max(map(len, grouped_params[grpname].keys()))
# line by parameter
for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
doc.append("{0:<{nw}} {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw))
doc.append("")
return "\n".join(doc)
def use(self, paths, prefix=None, guarded=False):
"""
Generate module use statements for given list of module paths.
:param paths: list of module path extensions to generate use statements for; paths will be quoted
:param prefix: optional path prefix; not quoted, i.e., can be a statement
:param guarded: use statements will be guarded to only apply if path exists
"""
use_statements = []
for path in paths:
quoted_path = quote_str(path)
if prefix:
full_path = 'pathJoin(%s, %s)' % (prefix, quoted_path)
else:
full_path = quoted_path
if guarded:
cond_statement = self.conditional_statement('isDir(%s)' % full_path,
self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', full_path))
use_statements.append(cond_statement)
else:
use_statements.append(self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', full_path) + '\n')
return ''.join(use_statements)
def use(self, paths, prefix=None, guarded=False):
"""
Generate module use statements for given list of module paths.
:param paths: list of module path extensions to generate use statements for; paths will be quoted
:param prefix: optional path prefix; not quoted, i.e., can be a statement
:param guarded: use statements will be guarded to only apply if path exists
"""
use_statements = []
for path in paths:
quoted_path = quote_str(path)
if prefix:
full_path = '[ file join %s %s ]' % (prefix, quoted_path)
else:
full_path = quoted_path
if guarded:
cond_statement = self.conditional_statement('file isdirectory %s' % full_path,
'module use %s' % full_path)
use_statements.append(cond_statement)
else:
use_statements.append("module use %s\n" % full_path)
return ''.join(use_statements)
def use(self, paths, prefix=None, guarded=False):
"""
Generate module use statements for given list of module paths.
:param paths: list of module path extensions to generate use statements for; paths will be quoted
:param prefix: optional path prefix; not quoted, i.e., can be a statement
:param guarded: use statements will be guarded to only apply if path exists
"""
use_statements = []
for path in paths:
quoted_path = quote_str(path)
if prefix:
full_path = '[ file join %s %s ]' % (prefix, quoted_path)
else:
full_path = quoted_path
if guarded:
cond_statement = self.conditional_statement('file isdirectory %s' % full_path,
'module use %s' % full_path)
use_statements.append(cond_statement)
else:
use_statements.append("module use %s\n" % full_path)
return ''.join(use_statements)
def _inject_constants_dict(self, txt):
"""Inject constants so they are resolved when actually parsing the YAML text."""
constants_dict = build_easyconfig_constants_dict()
lines = txt.splitlines()
# extract possible YAML header, for example
# %YAML 1.2
# ---
yaml_header = []
for i, line in enumerate(lines):
if line.startswith(YAML_DIR):
if lines[i+1].startswith(YAML_SEP):
yaml_header.extend([lines.pop(i), lines.pop(i)])
injected_constants = ['__CONSTANTS__: ']
for key, value in constants_dict.items():
injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value)))
full_txt = '\n'.join(yaml_header + injected_constants + lines)
return full_txt
def _inject_constants_dict(self, txt):
"""Inject constants so they are resolved when actually parsing the YAML text."""
constants_dict = build_easyconfig_constants_dict()
lines = txt.splitlines()
# extract possible YAML header, for example
# %YAML 1.2
# ---
yaml_header = []
for i, line in enumerate(lines):
if line.startswith(YAML_DIR):
if lines[i+1].startswith(YAML_SEP):
yaml_header.extend([lines.pop(i), lines.pop(i)])
injected_constants = ['__CONSTANTS__: ']
for key, value in constants_dict.items():
injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value)))
full_txt = '\n'.join(yaml_header + injected_constants + lines)
return full_txt
def avail_easyconfig_params_txt(title, grouped_params):
"""
Compose overview of available easyconfig parameters, in plain text format.
"""
# main title
lines = [
'%s:' % title,
'',
]
for grpname in grouped_params:
# group section title
lines.append(grpname.upper())
lines.append('-' * len(lines[-1]))
# determine width of 'name' column, to left-align descriptions
nw = max(map(len, grouped_params[grpname].keys()))
# line by parameter
for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
lines.append("{0:<{nw}} {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw))
lines.append('')
return '\n'.join(lines)
def prepend_paths(self, key, paths, allow_abs=False):
"""
Generate prepend-path statements for the given list of paths
"""
if isinstance(paths, basestring):
self.log.debug("Wrapping %s into a list before using it to prepend path %s", paths, key)
paths = [paths]
for i, path in enumerate(paths):
if os.path.isabs(path):
if allow_abs:
paths[i] = quote_str(path)
else:
raise EasyBuildError("Absolute path %s passed to prepend_paths which only expects relative paths.",
path)
else:
# use pathJoin for (non-empty) relative paths
if path:
paths[i] = self.PATH_JOIN_TEMPLATE % path
else:
paths[i] = 'root'
statements = [self.PREPEND_PATH_TEMPLATE % (key, p) for p in paths]
return ''.join(statements)
def tweak(src_fn, target_fn, tweaks):
"""
Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
once and then use the 'version' variable to construct the list of sources, and possibly other
parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)
The tweaks should be specified in a dictionary, with parameters and keys that map to the values
to be set.
Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.
If no target filename is provided, a target filepath is generated based on the contents of
the tweaked easyconfig file.
"""
# read easyconfig file
ectxt = read_file(src_fn)
_log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" %
ectxt)
# determine new toolchain if it's being changed
keys = tweaks.keys()
if 'toolchain_name' in keys or 'toolchain_version' in keys:
tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)
res = tc_regexp.search(ectxt)
if not res:
_log.error("No toolchain found in easyconfig file %s?" % src_fn)
toolchain = eval(res.group(1))
for key in ['name', 'version']:
tc_key = "toolchain_%s" % key
if tc_key in keys:
toolchain.update({key: tweaks[tc_key]})
tweaks.pop(tc_key)
class TcDict(dict):
"""A special dict class that represents trivial toolchains properly."""
def __repr__(self):
return "{'name': '%(name)s', 'version': '%(version)s'}" % self
tweaks.update({
'toolchain':
TcDict({
'name': toolchain['name'],
'version': toolchain['version']
})
})
_log.debug("New toolchain constructed: %s" % tweaks['toolchain'])
additions = []
# we need to treat list values seperately, i.e. we prepend to the current value (if any)
for (key, val) in tweaks.items():
if isinstance(val, list):
regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)
res = regexp.search(ectxt)
if res:
newval = "%s + %s" % (val, res.group(1))
ectxt = regexp.sub(
"%s = %s # tweaked by EasyBuild (was: %s)" %
(key, newval, res.group(1)), ectxt)
_log.info("Tweaked %s list to '%s'" % (key, newval))
else:
additions.append("%s = %s # added by EasyBuild" % (key, val))
tweaks.pop(key)
# add parameters or replace existing ones
for (key, val) in tweaks.items():
regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)
_log.debug("Regexp pattern for replacing '%s': %s" %
(key, regexp.pattern))
res = regexp.search(ectxt)
if res:
# only tweak if the value is different
diff = True
try:
_log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1))))
diff = not eval(res.group(1)) == val
except (NameError, SyntaxError):
# if eval fails, just fall back to string comparison
_log.debug(
"eval failed for \"%s\", falling back to string comparison against \"%s\"..."
% (res.group(1), val))
diff = not res.group(1) == val
if diff:
ectxt = regexp.sub(
"%s = %s # tweaked by EasyBuild (was: %s)" %
(key, quote_str(val), res.group(1)), ectxt)
_log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
else:
additions.append("%s = %s" % (key, quote_str(val)))
if additions:
_log.info(
"Adding additional parameters to tweaked easyconfig file: %s")
ectxt += "\n\n# added by EasyBuild as dictated by command line options\n"
ectxt += '\n'.join(additions) + '\n'
_log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)
# come up with suiting file name for tweaked easyconfig file if none was specified
if not target_fn:
fn = None
try:
# obtain temporary filename
fd, tmpfn = tempfile.mkstemp()
os.close(fd)
# write easyconfig to temporary file
write_file(tmpfn, ectxt)
# determine suiting filename
fn = ec_filename_for(tmpfn)
# get rid of temporary file
os.remove(tmpfn)
except OSError, err:
_log.error(
"Failed to determine suiting filename for tweaked easyconfig file: %s"
% err)
target_fn = os.path.join(tempfile.gettempdir(), fn)
_log.debug("Generated file name for tweaked easyconfig file: %s" %
target_fn)
def use(self, paths):
"""
Generate module use statements for given list of module paths.
@param paths: list of module path extensions to generate use statements for
"""
return ''.join([self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', quote_str(p)) for p in paths])
def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params,
doc_functions, all_blocks):
"""
Compose overview of one easyblock given class object of the easyblock in rst format
"""
classname = eb_class.__name__
doc = [
'.. _' + classname + ':',
'',
'``' + classname + '``',
'=' * (len(classname) + 4),
'',
]
bases = []
for b in eb_class.__bases__:
base = ':ref:`' + b.__name__ + '`' if b in all_blocks else b.__name__
bases.append(base)
derived = '(derives from ' + ', '.join(bases) + ')'
doc.extend([derived, ''])
# Description (docstring)
if eb_class.__doc__ is not None:
doc.extend([eb_class.__doc__.strip(), ''])
# Add extra options, if any
if eb_class.extra_options():
title = 'Extra easyconfig parameters specific to ``%s`` easyblock' % classname
ex_opt = eb_class.extra_options()
keys = sorted(ex_opt.keys())
values = [ex_opt[k] for k in keys]
table_titles = ['easyconfig parameter', 'description', 'default value']
table_values = [
['``' + key + '``' for key in keys], # parameter name
[val[1] for val in values], # description
['``' + str(quote_str(val[0])) + '``'
for val in values] # default value
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
# Add commonly used parameters
if classname in common_params:
title = 'Commonly used easyconfig parameters with ``%s`` easyblock' % classname
table_titles = ['easyconfig parameter', 'description']
table_values = [
[opt for opt in common_params[classname]],
[DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
doc.append('')
# Add docstring for custom steps
custom = []
inh = ''
f = None
for func in doc_functions:
if func in eb_class.__dict__:
f = eb_class.__dict__[func]
if f.__doc__:
custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh)
if custom:
title = 'Customised steps in ``' + classname + '`` easyblock'
doc.extend([title, '-' * len(title)] + custom)
doc.append('')
# Add example if available
if os.path.exists(os.path.join(path_to_examples, '%s.eb' % classname)):
title = 'Example easyconfig for ``' + classname + '`` easyblock'
doc.extend([title, '-' * len(title), '', '.. code::', ''])
for line in read_file(os.path.join(path_to_examples,
classname + '.eb')).split('\n'):
doc.append(INDENT_4SPACES + line)
doc.append('') # empty line after literal block
return doc
def use(self, paths):
"""
Generate module use statements for given list of module paths.
@param paths: list of module path extensions to generate use statements for
"""
return "".join([self.PREPEND_PATH_TEMPLATE % ("MODULEPATH", quote_str(p)) for p in paths])
def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks):
"""
Compose overview of one easyblock given class object of the easyblock in rst format
"""
classname = eb_class.__name__
doc = [".. _" + classname + ":", "", "``" + classname + "``", "=" * (len(classname) + 4), ""]
bases = []
for b in eb_class.__bases__:
base = ":ref:`" + b.__name__ + "`" if b in all_blocks else b.__name__
bases.append(base)
derived = "(derives from " + ", ".join(bases) + ")"
doc.extend([derived, ""])
# Description (docstring)
doc.extend([eb_class.__doc__.strip(), ""])
# Add extra options, if any
if eb_class.extra_options():
title = "Extra easyconfig parameters specific to ``%s`` easyblock" % classname
ex_opt = eb_class.extra_options()
table_titles = ["easyconfig parameter", "description", "default value"]
table_values = [
["``" + key + "``" for key in ex_opt], # parameter name
[val[1] for val in ex_opt.values()], # description
["``" + str(quote_str(val[0])) + "``" for val in ex_opt.values()], # default value
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
# Add commonly used parameters
if classname in common_params:
title = "Commonly used easyconfig parameters with ``%s`` easyblock" % classname
table_titles = ["easyconfig parameter", "description"]
table_values = [
[opt for opt in common_params[classname]],
[DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
]
doc.extend(rst_title_and_table(title, table_titles, table_values))
doc.append("")
# Add docstring for custom steps
custom = []
inh = ""
f = None
for func in doc_functions:
if func in eb_class.__dict__:
f = eb_class.__dict__[func]
if f.__doc__:
custom.append("* ``" + func + "`` - " + f.__doc__.strip() + inh)
if custom:
title = "Customised steps in ``" + classname + "`` easyblock"
doc.extend([title, "-" * len(title)] + custom)
doc.append("")
# Add example if available
if os.path.exists(os.path.join(path_to_examples, "%s.eb" % classname)):
title = "Example easyconfig for ``" + classname + "`` easyblock"
doc.extend([title, "-" * len(title), "", ".. code::", ""])
for line in read_file(os.path.join(path_to_examples, classname + ".eb")).split("\n"):
doc.append(INDENT_4SPACES + line)
doc.append("") # empty line after literal block
return doc
def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks):
"""
Compose overview of one easyblock given class object of the easyblock in rst format
"""
classname = eb_class.__name__
lines = [
'.. _' + classname + ':',
'',
'``' + classname + '``',
'=' * (len(classname)+4),
'',
]
bases = []
for b in eb_class.__bases__:
base = ':ref:`' + b.__name__ +'`' if b in all_blocks else b.__name__
bases.append(base)
derived = '(derives from ' + ', '.join(bases) + ')'
lines.extend([derived, ''])
# Description (docstring)
lines.extend([eb_class.__doc__.strip(), ''])
# Add extra options, if any
if eb_class.extra_options():
extra_parameters = 'Extra easyconfig parameters specific to ``' + classname + '`` easyblock'
lines.extend([extra_parameters, '-' * len(extra_parameters), ''])
ex_opt = eb_class.extra_options()
titles = ['easyconfig parameter', 'description', 'default value']
values = [
['``' + key + '``' for key in ex_opt], # parameter name
[val[1] for val in ex_opt.values()], # description
['``' + str(quote_str(val[0])) + '``' for val in ex_opt.values()] # default value
]
lines.extend(mk_rst_table(titles, values))
# Add commonly used parameters
if classname in common_params:
commonly_used = 'Commonly used easyconfig parameters with ``' + classname + '`` easyblock'
lines.extend([commonly_used, '-' * len(commonly_used)])
titles = ['easyconfig parameter', 'description']
values = [
[opt for opt in common_params[classname]],
[DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
]
lines.extend(mk_rst_table(titles, values))
lines.append('')
# Add docstring for custom steps
custom = []
inh = ''
f = None
for func in doc_functions:
if func in eb_class.__dict__:
f = eb_class.__dict__[func]
if f.__doc__:
custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh)
if custom:
title = 'Customised steps in ``' + classname + '`` easyblock'
lines.extend([title, '-' * len(title)] + custom)
lines.append('')
# Add example if available
if os.path.exists(os.path.join(path_to_examples, '%s.eb' % classname)):
title = 'Example for ``' + classname + '`` easyblock'
lines.extend(['', title, '-' * len(title), '', '::', ''])
for line in read_file(os.path.join(path_to_examples, classname+'.eb')).split('\n'):
lines.append(' ' + line.strip())
lines.append('') # empty line after literal block
return '\n'.join(lines)
# add versionsuffix if Python is specified as a dependency
if any(dep[0] == 'Python' for dep in cfg.get('dependencies', [])):
if not_found_yet('versionsuffix'):
cfg['versionsuffix'] = '-Python-%(pyver)s'
# add empty sanity_check_paths
if 'sanity_check_paths' not in cfg:
cfg['sanity_check_paths'] = {'files': [], 'dirs': []}
# enable use_pip & sanity_pip_check
if cfg.get('easyblock') in ['PythonBundle', 'PythonPackage']:
cfg.update({
'use_pip': True,
'sanity_pip_check': True,
})
# enable download_dep_fail
if cfg.get('easyblock') == 'PythonPackage':
cfg['download_dep_fail'] = True
pprint.pprint(cfg)
ec_raw = '\n'.join("%s = %s" % (key, quote_str(cfg[key])) for key in cfg)
ec = EasyConfig(None, rawtxt=ec_raw)
full_ec_ver = det_full_ec_version(ec)
fn = os.path.join('%s-%s.eb' % (cfg['name'], full_ec_ver))
ec.dump(fn)
info("Easyconfig file created: %s" % fn)
depstring = ''
for dep in deps:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
depstring += " --depends '%s'" % dep_pkgname
cmdlist = [
PKG_TOOL_FPM,
'--workdir', workdir,
'--name', pkgname,
'--provides', pkgname,
'-t', pkgtype, # target
'-s', 'dir', # source
'--version', pkgver,
'--iteration', pkgrel,
'--description', quote_str(easyblock.cfg["description"]),
'--url', easyblock.cfg["homepage"],
depstring,
easyblock.installdir,
easyblock.module_generator.get_module_filepath(),
]
cmd = ' '.join(cmdlist)
_log.debug("The flattened cmdlist looks like: %s", cmd)
run_cmd(cmd, log_all=True, simple=True)
_log.info("Created %s package(s) in %s", pkgtype, workdir)
try:
os.chdir(origdir)
except OSError, err:
raise EasyBuildError("Failed to chdir back to %s: %s", origdir, err)
def set_environment(self, key, value):
"""
Generate setenv statement for the given key/value pair.
"""
# quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
return 'setenv\t%s\t\t%s\n' % (key, quote_str(value))
def set_alias(self, key, value):
"""
Generate set-alias statement in modulefile for the given key/value pair.
"""
# quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
return 'set_alias("%s", %s)\n' % (key, quote_str(value))
_log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern))
res = regexp.search(ectxt)
if res:
# only tweak if the value is different
diff = True
try:
_log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1))))
diff = not eval(res.group(1)) == val
except (NameError, SyntaxError):
# if eval fails, just fall back to string comparison
_log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % (res.group(1), val))
diff = not res.group(1) == val
if diff:
ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, quote_str(val), res.group(1)), ectxt)
_log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
else:
additions.append("%s = %s" % (key, quote_str(val)))
if additions:
_log.info("Adding additional parameters to tweaked easyconfig file: %s")
ectxt += "\n\n# added by EasyBuild as dictated by command line options\n"
ectxt += '\n'.join(additions) + '\n'
_log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)
# come up with suiting file name for tweaked easyconfig file if none was specified
if not target_fn:
fn = None
def set_alias(self, key, value):
"""
Generate set-alias statement in modulefile for the given key/value pair.
"""
# quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
return 'set-alias\t%s\t\t%s\n' % (key, quote_str(value))
if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME:
toolchain_dict = easyblock.toolchain.as_dict()
deps.extend([toolchain_dict])
deps.extend(easyblock.cfg.dependencies())
_log.debug("The dependencies to be added to the package are: %s",
pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies()))
depstring = ''
for dep in deps:
if dep.get('external_module', False):
_log.debug("Skipping dep marked as external module: %s", dep['name'])
else:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
depstring += " --depends %s" % quote_str(dep_pkgname)
# Excluding the EasyBuild logs and test reports that might be in the installdir
exclude_files_glob = [
os.path.join(log_path(), "*.log"),
os.path.join(log_path(), "*.md"),
]
# stripping off leading / to match expected glob in fpm
exclude_files_glob = [
'--exclude %s' % quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
for x in exclude_files_glob
]
_log.debug("The list of excluded files passed to fpm: %s", exclude_files_glob)
cmdlist = [
PKG_TOOL_FPM,
'--workdir', workdir,
if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME:
toolchain_dict = easyblock.toolchain.as_dict()
deps.extend([toolchain_dict])
deps.extend(easyblock.cfg.dependencies())
_log.debug("The dependencies to be added to the package are: %s",
pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies()))
depstring = ''
for dep in deps:
if dep.get('external_module', False):
_log.debug("Skipping dep marked as external module: %s", dep['name'])
else:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
depstring += " --depends %s" % quote_str(dep_pkgname)
# Excluding the EasyBuild logs and test reports that might be in the installdir
exclude_files_glob = [
os.path.join(log_path(), "*.log"),
os.path.join(log_path(), "*.md"),
]
# stripping off leading / to match expected glob in fpm
exclude_files_glob = [
'--exclude %s' % quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
for x in exclude_files_glob
]
_log.debug("The list of excluded files passed to fpm: %s", exclude_files_glob)
cmdlist = [
PKG_TOOL_FPM,
'--workdir', workdir,
def tweak(src_fn, target_fn, tweaks):
"""
Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
once and then use the 'version' variable to construct the list of sources, and possibly other
parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)
The tweaks should be specified in a dictionary, with parameters and keys that map to the values
to be set.
Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.
If no target filename is provided, a target filepath is generated based on the contents of
the tweaked easyconfig file.
"""
# read easyconfig file
ectxt = read_file(src_fn)
_log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" % ectxt)
# determine new toolchain if it's being changed
keys = tweaks.keys()
if 'toolchain_name' in keys or 'toolchain_version' in keys:
tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)
res = tc_regexp.search(ectxt)
if not res:
_log.error("No toolchain found in easyconfig file %s?" % src_fn)
toolchain = eval(res.group(1))
for key in ['name', 'version']:
tc_key = "toolchain_%s" % key
if tc_key in keys:
toolchain.update({key: tweaks[tc_key]})
tweaks.pop(tc_key)
class TcDict(dict):
"""A special dict class that represents trivial toolchains properly."""
def __repr__(self):
return "{'name': '%(name)s', 'version': '%(version)s'}" % self
tweaks.update({'toolchain': TcDict({'name': toolchain['name'], 'version': toolchain['version']})})
_log.debug("New toolchain constructed: %s" % tweaks['toolchain'])
additions = []
# we need to treat list values seperately, i.e. we prepend to the current value (if any)
for (key, val) in tweaks.items():
if isinstance(val, list):
regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)
res = regexp.search(ectxt)
if res:
newval = "%s + %s" % (val, res.group(1))
ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, newval, res.group(1)), ectxt)
_log.info("Tweaked %s list to '%s'" % (key, newval))
else:
additions.append("%s = %s # added by EasyBuild" % (key, val))
tweaks.pop(key)
# add parameters or replace existing ones
for (key, val) in tweaks.items():
regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M)
_log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern))
res = regexp.search(ectxt)
if res:
# only tweak if the value is different
diff = True
try:
_log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1))))
diff = not eval(res.group(1)) == val
except (NameError, SyntaxError):
# if eval fails, just fall back to string comparison
_log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % (res.group(1), val))
diff = not res.group(1) == val
if diff:
ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, quote_str(val), res.group(1)), ectxt)
_log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
else:
additions.append("%s = %s" % (key, quote_str(val)))
if additions:
_log.info("Adding additional parameters to tweaked easyconfig file: %s")
ectxt += "\n\n# added by EasyBuild as dictated by command line options\n"
ectxt += '\n'.join(additions) + '\n'
_log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)
# come up with suiting file name for tweaked easyconfig file if none was specified
if not target_fn:
fn = None
try:
# obtain temporary filename
fd, tmpfn = tempfile.mkstemp()
os.close(fd)
# write easyconfig to temporary file
write_file(tmpfn, ectxt)
# determine suiting filename
fn = ec_filename_for(tmpfn)
# get rid of temporary file
os.remove(tmpfn)
except OSError, err:
_log.error("Failed to determine suiting filename for tweaked easyconfig file: %s" % err)
target_fn = os.path.join(tempfile.gettempdir(), fn)
_log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn)
def tweak_one(src_fn, target_fn, tweaks, targetdir=None):
"""
Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
once and then use the 'version' variable to construct the list of sources, and possibly other
parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)
The tweaks should be specified in a dictionary, with parameters and keys that map to the values
to be set.
Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.
If no target filename is provided, a target filepath is generated based on the contents of
the tweaked easyconfig file.
"""
# read easyconfig file
ectxt = read_file(src_fn)
_log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" % ectxt)
# determine new toolchain if it's being changed
keys = tweaks.keys()
if 'toolchain_name' in keys or 'toolchain_version' in keys:
# note: this assumes that the toolchain spec is single-line
tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)
res = tc_regexp.search(ectxt)
if not res:
raise EasyBuildError("No toolchain found in easyconfig file %s: %s", src_fn, ectxt)
toolchain = eval(res.group(1))
for key in ['name', 'version']:
tc_key = "toolchain_%s" % key
if tc_key in keys:
toolchain.update({key: tweaks[tc_key]})
tweaks.pop(tc_key)
class TcDict(dict):
"""A special dict class that represents trivial toolchains properly."""
def __repr__(self):
return "{'name': '%(name)s', 'version': '%(version)s'}" % self
tweaks.update({'toolchain': TcDict({'name': toolchain['name'], 'version': toolchain['version']})})
_log.debug("New toolchain constructed: %s" % tweaks['toolchain'])
additions = []
# automagically clear out list of checksums if software version is being tweaked
if 'version' in tweaks and 'checksums' not in tweaks:
tweaks['checksums'] = []
_log.warning("Tweaking version: checksums cleared, verification disabled.")
# we need to treat list values seperately, i.e. we prepend to the current value (if any)
for (key, val) in tweaks.items():
if isinstance(val, list):
regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>\[(.|\n)*\])\s*$" % key, re.M)
res = regexp.search(ectxt)
if res:
fval = [x for x in val if x != ''] # filter out empty strings
# determine to prepend/append or overwrite by checking first/last list item
# - input ending with comma (empty tail list element) => prepend
# - input starting with comma (empty head list element) => append
# - no empty head/tail list element => overwrite
if not val:
newval = '[]'
_log.debug("Clearing %s to empty list (was: %s)" % (key, res.group('val')))
elif val[0] == '':
newval = "%s + %s" % (res.group('val'), fval)
_log.debug("Appending %s to %s" % (fval, key))
elif val[-1] == '':
newval = "%s + %s" % (fval, res.group('val'))
_log.debug("Prepending %s to %s" % (fval, key))
else:
newval = "%s" % fval
_log.debug("Overwriting %s with %s" % (key, fval))
ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt)
_log.info("Tweaked %s list to '%s'" % (key, newval))
elif get_easyconfig_parameter_default(key) != val:
additions.append("%s = %s" % (key, val))
tweaks.pop(key)
# add parameters or replace existing ones
for (key, val) in tweaks.items():
regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>.*)$" % key, re.M)
_log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern))
res = regexp.search(ectxt)
if res:
# only tweak if the value is different
diff = True
try:
_log.debug("eval(%s): %s" % (res.group('val'), eval(res.group('val'))))
diff = eval(res.group('val')) != val
except (NameError, SyntaxError):
# if eval fails, just fall back to string comparison
_log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"...",
res.group('val'), val)
diff = res.group('val') != val
if diff:
ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt)
_log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
elif get_easyconfig_parameter_default(key) != val:
additions.append("%s = %s" % (key, quote_str(val)))
if additions:
_log.info("Adding additional parameters to tweaked easyconfig file: %s" % additions)
ectxt = '\n'.join([ectxt] + additions)
_log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)
# come up with suiting file name for tweaked easyconfig file if none was specified
if target_fn is None:
fn = None
try:
# obtain temporary filename
fd, tmpfn = tempfile.mkstemp()
os.close(fd)
# write easyconfig to temporary file
write_file(tmpfn, ectxt)
# determine suiting filename
fn = ec_filename_for(tmpfn)
# get rid of temporary file
os.remove(tmpfn)
except OSError, err:
raise EasyBuildError("Failed to determine suiting filename for tweaked easyconfig file: %s", err)
if targetdir is None:
targetdir = tempfile.gettempdir()
target_fn = os.path.join(targetdir, fn)
_log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn)
deps.extend(easyblock.cfg.dependencies())
_log.debug(
"The dependencies to be added to the package are: %s",
pprint.pformat([easyblock.toolchain.as_dict()] +
easyblock.cfg.dependencies()))
depstring = ''
for dep in deps:
if dep.get('external_module', False):
_log.debug("Skipping dep marked as external module: %s",
dep['name'])
else:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
depstring += " --depends %s" % quote_str(dep_pkgname)
# Excluding the EasyBuild logs and test reports that might be in the installdir
exclude_files_glob = [
os.path.join(log_path(), "*.log"),
os.path.join(log_path(), "*.md"),
]
# stripping off leading / to match expected glob in fpm
exclude_files_glob = [
'--exclude %s' %
quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
for x in exclude_files_glob
]
_log.debug("The list of excluded files passed to fpm: %s",
exclude_files_glob)
cmdlist = [
def tweak_one(src_fn, target_fn, tweaks, targetdir=None):
"""
Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions.
Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version
once and then use the 'version' variable to construct the list of sources, and possibly other
parameters that depend on the version (e.g. list of patch files, dependencies, version suffix, ...)
The tweaks should be specified in a dictionary, with parameters and keys that map to the values
to be set.
Reads easyconfig file at path <src_fn>, and writes the tweaked easyconfig file to <target_fn>.
If no target filename is provided, a target filepath is generated based on the contents of
the tweaked easyconfig file.
"""
# read easyconfig file
ectxt = read_file(src_fn)
_log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" %
ectxt)
# determine new toolchain if it's being changed
keys = tweaks.keys()
if 'toolchain_name' in keys or 'toolchain_version' in keys:
tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)
res = tc_regexp.search(ectxt)
if not res:
raise EasyBuildError("No toolchain found in easyconfig file %s?",
src_fn)
toolchain = eval(res.group(1))
for key in ['name', 'version']:
tc_key = "toolchain_%s" % key
if tc_key in keys:
toolchain.update({key: tweaks[tc_key]})
tweaks.pop(tc_key)
class TcDict(dict):
"""A special dict class that represents trivial toolchains properly."""
def __repr__(self):
return "{'name': '%(name)s', 'version': '%(version)s'}" % self
tweaks.update({
'toolchain':
TcDict({
'name': toolchain['name'],
'version': toolchain['version']
})
})
_log.debug("New toolchain constructed: %s" % tweaks['toolchain'])
additions = []
# automagically clear out list of checksums if software version is being tweaked
if 'version' in tweaks and 'checksums' not in tweaks:
tweaks['checksums'] = []
_log.warning(
"Tweaking version: checksums cleared, verification disabled.")
# we need to treat list values seperately, i.e. we prepend to the current value (if any)
for (key, val) in tweaks.items():
if isinstance(val, list):
regexp = re.compile(
r"^(?P<key>\s*%s)\s*=\s*(?P<val>\[(.|\n)*\])\s*$" % key, re.M)
res = regexp.search(ectxt)
if res:
fval = [x for x in val if x != ''] # filter out empty strings
# determine to prepend/append or overwrite by checking first/last list item
# - input ending with comma (empty tail list element) => prepend
# - input starting with comma (empty head list element) => append
# - no empty head/tail list element => overwrite
if not val:
newval = '[]'
_log.debug("Clearing %s to empty list (was: %s)" %
(key, res.group('val')))
elif val[0] == '':
newval = "%s + %s" % (res.group('val'), fval)
_log.debug("Appending %s to %s" % (fval, key))
elif val[-1] == '':
newval = "%s + %s" % (fval, res.group('val'))
_log.debug("Prepending %s to %s" % (fval, key))
else:
newval = "%s" % fval
_log.debug("Overwriting %s with %s" % (key, fval))
ectxt = regexp.sub("%s = %s" % (res.group('key'), newval),
ectxt)
_log.info("Tweaked %s list to '%s'" % (key, newval))
elif get_easyconfig_parameter_default(key) != val:
additions.append("%s = %s" % (key, val))
tweaks.pop(key)
# add parameters or replace existing ones
for (key, val) in tweaks.items():
regexp = re.compile(r"^(?P<key>\s*%s)\s*=\s*(?P<val>.*)$" % key, re.M)
_log.debug("Regexp pattern for replacing '%s': %s" %
(key, regexp.pattern))
res = regexp.search(ectxt)
if res:
# only tweak if the value is different
diff = True
try:
_log.debug("eval(%s): %s" %
(res.group('val'), eval(res.group('val'))))
diff = eval(res.group('val')) != val
except (NameError, SyntaxError):
# if eval fails, just fall back to string comparison
_log.debug(
"eval failed for \"%s\", falling back to string comparison against \"%s\"...",
res.group('val'), val)
diff = res.group('val') != val
if diff:
ectxt = regexp.sub(
"%s = %s" % (res.group('key'), quote_str(val)), ectxt)
_log.info("Tweaked '%s' to '%s'" % (key, quote_str(val)))
elif get_easyconfig_parameter_default(key) != val:
additions.append("%s = %s" % (key, quote_str(val)))
if additions:
_log.info(
"Adding additional parameters to tweaked easyconfig file: %s" %
additions)
ectxt = '\n'.join([ectxt] + additions)
_log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt)
# come up with suiting file name for tweaked easyconfig file if none was specified
if target_fn is None:
fn = None
try:
# obtain temporary filename
fd, tmpfn = tempfile.mkstemp()
os.close(fd)
# write easyconfig to temporary file
write_file(tmpfn, ectxt)
# determine suiting filename
fn = ec_filename_for(tmpfn)
# get rid of temporary file
os.remove(tmpfn)
except OSError, err:
raise EasyBuildError(
"Failed to determine suiting filename for tweaked easyconfig file: %s",
err)
if targetdir is None:
targetdir = tempfile.gettempdir()
target_fn = os.path.join(targetdir, fn)
_log.debug("Generated file name for tweaked easyconfig file: %s" %
target_fn)